Add pseudocode for policy- and value-iteration

source-code/Pseudocode/Policy-Iteration/Makefile

@@ -0,0 +1,36 @@
+SOURCE = Policy-Iteration
+DELAY = 80
+DENSITY = 300
+WIDTH = 512
+
+make:
+	pdflatex $(SOURCE).tex -output-format=pdf
+	pdflatex $(SOURCE).tex -output-format=pdf
+	make clean
+
+clean:
+	rm -rf  $(TARGET) *.class *.html *.log *.aux *.data *.gnuplot
+
+gif:
+	pdfcrop $(SOURCE).pdf
+	convert -verbose -delay $(DELAY) -loop 0 -density $(DENSITY) $(SOURCE)-crop.pdf $(SOURCE).gif
+	make clean
+
+png:
+	make
+	make svg
+	inkscape $(SOURCE).svg -w $(WIDTH) --export-png=$(SOURCE).png
+
+transparentGif:
+	convert $(SOURCE).pdf -transparent white result.gif
+	make clean
+
+svg:
+	make
+	#inkscape $(SOURCE).pdf --export-plain-svg=$(SOURCE).svg
+	pdf2svg $(SOURCE).pdf $(SOURCE).svg
+	# Necessary, as pdf2svg does not always create valid svgs:
+	inkscape $(SOURCE).svg --export-plain-svg=$(SOURCE).svg
+	rsvg-convert -a -w $(WIDTH) -f svg $(SOURCE).svg -o $(SOURCE)2.svg
+	inkscape $(SOURCE)2.svg --export-plain-svg=$(SOURCE).svg
+	rm $(SOURCE)2.svg

source-code/Pseudocode/Policy-Iteration/Policy-Iteration.tex

@@ -0,0 +1,39 @@
+\documentclass{article}
+\usepackage[pdftex,active,tightpage]{preview}
+\setlength\PreviewBorder{2mm}
+
+\usepackage[utf8]{inputenc} % this is needed for umlauts
+\usepackage[ngerman]{babel} % this is needed for umlauts
+\usepackage[T1]{fontenc}    % this is needed for correct output of umlauts in pdf
+\usepackage{amssymb,amsmath,amsfonts} % nice math rendering
+\usepackage{braket} % needed for \Set
+\usepackage[noend]{algorithm,algpseudocode}
+
+\begin{document}
+\begin{preview}
+    \begin{algorithm}[H]
+        \begin{algorithmic}
+        \Require
+        \Statex Sates $\mathcal{X} = \{1, \dots, n_x\}$
+        \Statex Actions $\mathcal{A} = \{1, \dots, n_a\},\qquad A: \mathcal{X} \Rightarrow \mathcal{A}$
+        \Procedure{PolicyIteration}{$\mathcal{X}$, $A$}
+            \State Initialize $\pi$ arbitrarily
+            \While{$\pi$ is not converged}
+                \State $J \gets$ solve system of linear equations $I - \alpha \cdot F(\pi) \cdot J = g(\pi)$
+
+                \For{$x \in \mathcal{X}$}
+                    \For{$a \in A(x)$}
+                        \State $Q(x, a) \gets g(x, a) + \alpha \sum_{j=1}^{n_x} f_{xj}(a) \cdot J(j)$
+                    \EndFor
+                \EndFor
+                \For{$x \in \mathcal{X}$}
+                    \State $\pi(x) \gets \arg \min_a \{Q(x, a)\}$
+                \EndFor
+            \EndWhile
+        \EndProcedure
+        \end{algorithmic}
+    \caption{Policy Iteration}
+    \label{alg:policy-iteration}
+    \end{algorithm}
+\end{preview}
+\end{document}

source-code/Pseudocode/Policy-Iteration/README.md

@@ -0,0 +1,3 @@
+Compiled example
+----------------
+![Example](Policy-Iteration.png)

source-code/Pseudocode/Value-Iteration/Makefile

@@ -0,0 +1,36 @@
+SOURCE = Value-Iteration
+DELAY = 80
+DENSITY = 300
+WIDTH = 512
+
+make:
+	pdflatex $(SOURCE).tex -output-format=pdf
+	pdflatex $(SOURCE).tex -output-format=pdf
+	make clean
+
+clean:
+	rm -rf  $(TARGET) *.class *.html *.log *.aux *.data *.gnuplot
+
+gif:
+	pdfcrop $(SOURCE).pdf
+	convert -verbose -delay $(DELAY) -loop 0 -density $(DENSITY) $(SOURCE)-crop.pdf $(SOURCE).gif
+	make clean
+
+png:
+	make
+	make svg
+	inkscape $(SOURCE).svg -w $(WIDTH) --export-png=$(SOURCE).png
+
+transparentGif:
+	convert $(SOURCE).pdf -transparent white result.gif
+	make clean
+
+svg:
+	make
+	#inkscape $(SOURCE).pdf --export-plain-svg=$(SOURCE).svg
+	pdf2svg $(SOURCE).pdf $(SOURCE).svg
+	# Necessary, as pdf2svg does not always create valid svgs:
+	inkscape $(SOURCE).svg --export-plain-svg=$(SOURCE).svg
+	rsvg-convert -a -w $(WIDTH) -f svg $(SOURCE).svg -o $(SOURCE)2.svg
+	inkscape $(SOURCE)2.svg --export-plain-svg=$(SOURCE).svg
+	rm $(SOURCE)2.svg

source-code/Pseudocode/Value-Iteration/README.md

@@ -0,0 +1,3 @@
+Compiled example
+----------------
+![Example](Value-Iteration.png)

source-code/Pseudocode/Value-Iteration/Value-Iteration.tex

@@ -0,0 +1,41 @@
+\documentclass{article}
+\usepackage[pdftex,active,tightpage]{preview}
+\setlength\PreviewBorder{2mm}
+
+\usepackage[utf8]{inputenc} % this is needed for umlauts
+\usepackage[ngerman]{babel} % this is needed for umlauts
+\usepackage[T1]{fontenc}    % this is needed for correct output of umlauts in pdf
+\usepackage{amssymb,amsmath,amsfonts} % nice math rendering
+\usepackage{braket} % needed for \Set
+\usepackage[noend]{algorithm,algpseudocode}
+
+\begin{document}
+\begin{preview}
+    \begin{algorithm}[H]
+        \begin{algorithmic}
+        \Require
+        \Statex Sates $\mathcal{X} = \{1, \dots, n_x\}$
+        \Statex Actions $\mathcal{A} = \{1, \dots, n_a\},\qquad A: \mathcal{X} \Rightarrow \mathcal{A}$
+        % TODO: g? \alpha?
+        % Q?
+        % f_ij(a)
+        \Procedure{ValueIteration}{$\mathcal{X}$, $A$}
+            \State Initialize $J, J': \mathcal{X} \rightarrow \mathbb{R}_0^+$ arbitrarily
+            \While{$J$ is not converged}
+                \State $J' \gets J$
+                \For{$x \in \mathcal{X}$}
+                    \For{$a \in A(x)$}
+                        \State $Q(x, a) \gets g(x, a) + \alpha \sum_{j=1}^{n_x} f_{xj}(a) \cdot J'(j)$
+                    \EndFor
+                \EndFor
+                \For{$x \in \mathcal{X}$}
+                    \State $J(x) \gets \min_a \{Q(x, a)\}$
+                \EndFor
+            \EndWhile
+        \EndProcedure
+        \end{algorithmic}
+    \caption{Calculate value function}
+    \label{alg:calculateLegendreSymbol}
+    \end{algorithm}
+\end{preview}
+\end{document}