diff --git a/BLcourse2.3/03_one_dim_SVI.ipynb b/BLcourse2.3/03_one_dim_SVI.ipynb index 0520fa0366033d88482c76a09f7bf646e7ba4eed..f60508eba28f67c8992c710c4d837f9ba1187282 100644 --- a/BLcourse2.3/03_one_dim_SVI.ipynb +++ b/BLcourse2.3/03_one_dim_SVI.ipynb @@ -17,6 +17,8 @@ "$\\newcommand{\\predve}[1]{\\mathbf{#1}}$\n", "$\\newcommand{\\test}[1]{#1_*}$\n", "$\\newcommand{\\testtest}[1]{#1_{**}}$\n", + "$\\newcommand{\\dd}{{\\rm{d}}}$\n", + "$\\newcommand{\\lt}[1]{_{\\text{#1}}}$\n", "$\\DeclareMathOperator{\\diag}{diag}$\n", "$\\DeclareMathOperator{\\cov}{cov}$" ] @@ -141,6 +143,9 @@ "2015](https://proceedings.mlr.press/v38/hensman15.html). The model is\n", "\"sparse\" since it works with a set of *inducing* points $(\\ma Z, \\ve u),\n", "\\ve u=f(\\ma Z)$ which is much smaller than the train data $(\\ma X, \\ve y)$.\n", + "See also [the GPJax\n", + "docs](https://docs.jaxgaussianprocesses.com/_examples/uncollapsed_vi) for a\n", + "nice introduction.\n", "\n", "We have the same hyper parameters as before\n", "\n", @@ -260,16 +265,31 @@ "source": [ "# Fit GP to data: optimize hyper params\n", "\n", - "Now we optimize the GP hyper parameters by doing a GP-specific variational inference (VI),\n", - "where we optimize not the log marginal likelihood (ExactGP case),\n", - "but an ELBO (evidence lower bound) objective\n", + "Now we optimize the GP hyper parameters by doing a GP-specific variational\n", + "inference (VI), where we don't maximize the log marginal likelihood (ExactGP\n", + "case), but an ELBO (\"evidence lower bound\") objective -- a lower bound on the\n", + "marginal likelihood (the \"evidence\"). In variational inference, an ELBO objective\n", + "shows up when minimizing the KL divergence between\n", + "an approximate and the true posterior\n", "\n", "$$\n", - "\\max_\\ve\\zeta\\left(\\mathbb E_{q_{\\ve\\psi}(\\ve u)}\\left[\\ln p(\\ve y|\\ve u) \\right] -\n", - "D_{\\text{KL}}(q_{\\ve\\psi}(\\ve u)\\Vert p(\\ve u))\\right)\n", + " p(w|y) = \\frac{p(y|w)\\,p(w)}{\\int p(y|w)\\,p(w)\\,\\dd w}\n", + " = \\frac{p(y|w)\\,p(w)}{p(y)}\n", "$$\n", "\n", - "with respect to\n", + "$$\n", + " \\ve\\zeta^* = \\text{arg}\\min_{\\ve\\zeta} D\\lt{KL}(q_{\\ve\\zeta}(w)\\,\\Vert\\, p(w|y))\n", + "$$\n", + "\n", + "to obtain the optimal variational parameters $\\ve\\zeta^*$ to approximate\n", + "$p(w|y)$ with $q_{\\ve\\zeta^*}(w)$.\n", + "\n", + "In our case the two distributions are the approximate\n", + "\n", + "$$q_{\\ve\\zeta}(\\mathbf f)=\\int p(\\mathbf f|\\ve u)\\,q_{\\ve\\psi}(\\ve u)\\,\\dd\\ve u\\quad(\\text{\"variational strategy\"})$$\n", + "\n", + "and the true $p(\\mathbf f|\\mathcal D)$ posterior over function values. We\n", + "optimize with respect to\n", "\n", "$$\\ve\\zeta = [\\ell, \\sigma_n^2, s, c, \\ve\\psi] $$\n", "\n", diff --git a/BLcourse2.3/03_one_dim_SVI.py b/BLcourse2.3/03_one_dim_SVI.py index 644e61a37b818301dd4604bb25d74b64cb5be511..8f34d40034dbda1a5a20c03fe4b1e55608d26e81 100644 --- a/BLcourse2.3/03_one_dim_SVI.py +++ b/BLcourse2.3/03_one_dim_SVI.py @@ -25,6 +25,8 @@ # $\newcommand{\predve}[1]{\mathbf{#1}}$ # $\newcommand{\test}[1]{#1_*}$ # $\newcommand{\testtest}[1]{#1_{**}}$ +# $\newcommand{\dd}{{\rm{d}}}$ +# $\newcommand{\lt}[1]{_{\text{#1}}}$ # $\DeclareMathOperator{\diag}{diag}$ # $\DeclareMathOperator{\cov}{cov}$ @@ -108,6 +110,9 @@ ax.legend() # 2015](https://proceedings.mlr.press/v38/hensman15.html). The model is # "sparse" since it works with a set of *inducing* points $(\ma Z, \ve u), # \ve u=f(\ma Z)$ which is much smaller than the train data $(\ma X, \ve y)$. +# See also [the GPJax +# docs](https://docs.jaxgaussianprocesses.com/_examples/uncollapsed_vi) for a +# nice introduction. # # We have the same hyper parameters as before # @@ -183,16 +188,31 @@ likelihood.noise_covar.noise = 0.3 # # Fit GP to data: optimize hyper params # -# Now we optimize the GP hyper parameters by doing a GP-specific variational inference (VI), -# where we optimize not the log marginal likelihood (ExactGP case), -# but an ELBO (evidence lower bound) objective +# Now we optimize the GP hyper parameters by doing a GP-specific variational +# inference (VI), where we don't maximize the log marginal likelihood (ExactGP +# case), but an ELBO ("evidence lower bound") objective -- a lower bound on the +# marginal likelihood (the "evidence"). In variational inference, an ELBO objective +# shows up when minimizing the KL divergence between +# an approximate and the true posterior # # $$ -# \max_\ve\zeta\left(\mathbb E_{q_{\ve\psi}(\ve u)}\left[\ln p(\ve y|\ve u) \right] - -# D_{\text{KL}}(q_{\ve\psi}(\ve u)\Vert p(\ve u))\right) +# p(w|y) = \frac{p(y|w)\,p(w)}{\int p(y|w)\,p(w)\,\dd w} +# = \frac{p(y|w)\,p(w)}{p(y)} # $$ # -# with respect to +# $$ +# \ve\zeta^* = \text{arg}\min_{\ve\zeta} D\lt{KL}(q_{\ve\zeta}(w)\,\Vert\, p(w|y)) +# $$ +# +# to obtain the optimal variational parameters $\ve\zeta^*$ to approximate +# $p(w|y)$ with $q_{\ve\zeta^*}(w)$. +# +# In our case the two distributions are the approximate +# +# $$q_{\ve\zeta}(\mathbf f)=\int p(\mathbf f|\ve u)\,q_{\ve\psi}(\ve u)\,\dd\ve u\quad(\text{"variational strategy"})$$ +# +# and the true $p(\mathbf f|\mathcal D)$ posterior over function values. We +# optimize with respect to # # $$\ve\zeta = [\ell, \sigma_n^2, s, c, \ve\psi] $$ #