q-learning.tex 2.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546
  1. \documentclass{article}
  2. \usepackage[pdftex,active,tightpage]{preview}
  3. \setlength\PreviewBorder{2mm}
  4. \usepackage[utf8]{inputenc} % this is needed for umlauts
  5. \usepackage[ngerman]{babel} % this is needed for umlauts
  6. \usepackage[T1]{fontenc} % this is needed for correct output of umlauts in pdf
  7. \usepackage{amssymb,amsmath,amsfonts} % nice math rendering
  8. \usepackage{braket} % needed for \Set
  9. \usepackage{caption}
  10. \usepackage{algorithm}
  11. \usepackage[noend]{algpseudocode}
  12. \DeclareCaptionFormat{myformat}{#3}
  13. \captionsetup[algorithm]{format=myformat}
  14. \begin{document}
  15. \begin{preview}
  16. \begin{algorithm}[H]
  17. \begin{algorithmic}
  18. \Require
  19. \Statex Sates $\mathcal{X} = \{1, \dots, n_x\}$
  20. \Statex Actions $\mathcal{A} = \{1, \dots, n_a\},\qquad A: \mathcal{X} \Rightarrow \mathcal{A}$
  21. \Statex Reward function $R: \mathcal{X} \times \mathcal{A} \rightarrow \mathbb{R}$
  22. \Statex Black-box (probabilistic) transition function $T: \mathcal{X} \times \mathcal{A} \rightarrow \mathcal{X}$
  23. \Statex Learning rate $\alpha \in [0, 1]$, typically $\alpha = 0.1$
  24. \Statex Discounting factor $\gamma \in [0, 1]$
  25. \Procedure{QLearning}{$\mathcal{X}$, $A$, $R$, $T$, $\alpha$, $\gamma$}
  26. \State Initialize $Q: \mathcal{X} \times \mathcal{A} \rightarrow \mathbb{R}$ arbitrarily
  27. \State Start in state $s \in \mathcal{X}$
  28. \While{$Q$ is not converged}
  29. \State Select $a \in \mathcal{A}$ by $Q$ and an exploration policy (e.g. $\varepsilon$ greedy)
  30. \State $r \gets R(s, a)$
  31. \State $s' \gets T(s, a)$ \Comment{Receive the new state}
  32. \State $Q(s', a) \gets (1 - \alpha) \cdot Q(s, a) + \alpha \cdot (r + \gamma \cdot \max_{a'} Q(s', a'))$
  33. \State $s \gets s'$
  34. \EndWhile
  35. \Return $Q$
  36. \EndProcedure
  37. \end{algorithmic}
  38. \caption{$Q$-learning: Learn function $Q: \mathcal{X} \times \mathcal{A} \rightarrow \mathbb{R}$}
  39. \label{alg:q-learning}
  40. \end{algorithm}
  41. \end{preview}
  42. \end{document}