From f7c77937ce52e8b6b7f3a3ab1879a178f14abc58 Mon Sep 17 00:00:00 2001 From: Thomas Viehmann Date: Tue, 21 Nov 2017 22:16:40 +0100 Subject: [PATCH] minibatch mode --- candlegp/models/gpmc.py | 3 ++- candlegp/models/gpr.py | 3 ++- candlegp/models/model.py | 10 +++++----- candlegp/models/sgpmc.py | 15 ++++++++++----- candlegp/models/sgpr.py | 7 +++++-- candlegp/models/svgp.py | 13 +++++++++---- candlegp/models/vgp.py | 3 ++- 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/candlegp/models/gpmc.py b/candlegp/models/gpmc.py index d191a49..a24681a 100644 --- a/candlegp/models/gpmc.py +++ b/candlegp/models/gpmc.py @@ -54,7 +54,7 @@ def __init__(self, X, Y, kern, likelihood, self.V = parameter.Param(self.X.data.new(self.num_data, self.num_latent).zero_()) self.V.prior = priors.Gaussian(self.X.data.new(1).fill_(0.), self.X.data.new(1).fill_(1.)) - def compute_log_likelihood(self): + def compute_log_likelihood(self, X=None, Y=None): """ Construct a tf function to compute the likelihood of a general GP model. @@ -62,6 +62,7 @@ def compute_log_likelihood(self): \log p(Y, V | theta). """ + assert X is None and Y is None, "{} does not support minibatch mode".format(str(type(self))) K = self.kern.K(self.X) L = torch.potrf( K + Variable(torch.eye(self.X.size(0), out=K.data.new()) * self.jitter_level), upper=False) diff --git a/candlegp/models/gpr.py b/candlegp/models/gpr.py index ab22fc5..8bd3c0b 100644 --- a/candlegp/models/gpr.py +++ b/candlegp/models/gpr.py @@ -44,13 +44,14 @@ def __init__(self, X, Y, kern, mean_function=None, **kwargs): super(GPR,self).__init__(X, Y, kern, likelihood, mean_function, **kwargs) self.num_latent = Y.size(1) - def compute_log_likelihood(self): + def compute_log_likelihood(self, X = None, Y = None): """ Construct a tensorflow function to compute the likelihood. \log p(Y | theta). """ + assert X is None and Y is None, "{} does not support minibatch mode".format(str(type(self))) K = self.kern.K(self.X) + Variable(torch.eye(self.X.size(0),out=self.X.data.new())) * self.likelihood.variance.get() L = torch.potrf(K, upper=False) m = self.mean_function(self.X) diff --git a/candlegp/models/model.py b/candlegp/models/model.py index 3c8c7fc..07bd8d8 100644 --- a/candlegp/models/model.py +++ b/candlegp/models/model.py @@ -75,19 +75,19 @@ def compute_log_prior(self): pass @abc.abstractmethod - def compute_log_likelihood(self): + def compute_log_likelihood(self, X=None, Y=None): """Compute the log likelihood of the model.""" pass - def objective(self): - pos_objective = self.compute_log_likelihood() + def objective(self, X=None, Y=None): + pos_objective = self.compute_log_likelihood(X, Y) for param in self.parameters(): if isinstance(param, parameter.ParamWithPrior): pos_objective = pos_objective + param.get_prior() return -pos_objective - def forward(self): - return self.objective() + def forward(self, X=None, Y=None): + return self.objective(X, Y) @abc.abstractmethod def predict_f(self, Xnew, full_cov=False): diff --git a/candlegp/models/sgpmc.py b/candlegp/models/sgpmc.py index bd4e847..29f1742 100644 --- a/candlegp/models/sgpmc.py +++ b/candlegp/models/sgpmc.py @@ -85,10 +85,10 @@ def __init__(self, X, Y, kern, likelihood, Z, self.num_latent = num_latent or Y.size(1) self.num_inducing = Z.size(0) self.Z = parameter.Param(Z) - self.V = parameter.Param(self.X.data.new(self.num_inducing, self.num_latent).zero_()) - self.V.prior = priors.Gaussian(self.X.data.new(1).fill_(0.), self.X.data.new(1).fill_(1.)) + self.V = parameter.Param(self.Z.data.new(self.num_inducing, self.num_latent).zero_()) + self.V.prior = priors.Gaussian(self.Z.data.new(1).fill_(0.), self.Z.data.new(1).fill_(1.)) - def compute_log_likelihood(self): + def compute_log_likelihood(self, X=None, Y=None): """ Construct a tf function to compute the likelihood of a general GP model. @@ -96,9 +96,14 @@ def compute_log_likelihood(self): \log p(Y, V | theta). """ + if X is None: + X = self.X + if Y is None: + Y = self.Y + # get the (marginals of) q(f): exactly predicting! - fmean, fvar = self.predict_f(self.X, full_cov=False) - return self.likelihood.variational_expectations(fmean, fvar, self.Y).sum() + fmean, fvar = self.predict_f(X, full_cov=False) + return self.likelihood.variational_expectations(fmean, fvar, Y).sum() def predict_f(self, Xnew, full_cov=False): """ diff --git a/candlegp/models/sgpr.py b/candlegp/models/sgpr.py index 4dedd8d..7391c35 100644 --- a/candlegp/models/sgpr.py +++ b/candlegp/models/sgpr.py @@ -117,12 +117,14 @@ def __init__(self, X, Y, kern, Z, mean_function=None, **kwargs): self.num_data = X.size(0) self.num_latent = Y.size(1) - def compute_log_likelihood(self): + def compute_log_likelihood(self, X=None, Y=None): """ For a derivation of the terms in here, see the associated SGPR notebook. """ + assert X is None and Y is None, "{} does not support minibatch mode".format(str(type(self))) + num_inducing = self.Z.size(0) num_data = self.Y.size(0) output_dim = self.Y.size(1) @@ -240,11 +242,12 @@ def _common_terms(self): return err, nu, Luu, L, alpha, beta, gamma - def compute_log_likelihood(self): + def compute_log_likelihood(self, X=None, Y=None): """ Construct a tensorflow function to compute the bound on the marginal likelihood. """ + assert X is None and Y is None, "{} does not support minibatch mode".format(str(type(self))) # FITC approximation to the log marginal likelihood is # log ( normal( y | mean, K_fitc ) ) diff --git a/candlegp/models/svgp.py b/candlegp/models/svgp.py index 043e5b1..e6aa603 100644 --- a/candlegp/models/svgp.py +++ b/candlegp/models/svgp.py @@ -92,22 +92,27 @@ def prior_KL(self): KL = kullback_leiblers.gauss_kl(self.q_mu.get(), self.q_sqrt.get(), K) return KL - def compute_log_likelihood(self): + def compute_log_likelihood(self, X=None, Y=None): """ This gives a variational bound on the model likelihood. """ + if X is None: + X = self.X + if Y is None: + Y = self.Y + # Get prior KL. KL = self.prior_KL() # Get conditionals - fmean, fvar = self.predict_f(self.X, full_cov=False) + fmean, fvar = self.predict_f(X, full_cov=False) # Get variational expectations. - var_exp = self.likelihood.variational_expectations(fmean, fvar, self.Y) + var_exp = self.likelihood.variational_expectations(fmean, fvar, Y) # re-scale for minibatch size - scale = float(self.num_data) / self.X.size(0) + scale = float(self.num_data) / X.size(0) return var_exp.sum() * scale - KL diff --git a/candlegp/models/vgp.py b/candlegp/models/vgp.py index 1ac0e10..428a2bc 100644 --- a/candlegp/models/vgp.py +++ b/candlegp/models/vgp.py @@ -63,7 +63,7 @@ def __init__(self, X, Y, kern, likelihood, q_sqrt = torch.eye(self.num_data, out=self.X.data.new()).unsqueeze(2).expand(-1,-1,self.num_latent) self.q_sqrt = parameter.LowerTriangularParam(q_sqrt) # should the diagonal be all positive? - def compute_log_likelihood(self): + def compute_log_likelihood(self, X=None, Y=None): """ This method computes the variational lower bound on the likelihood, which is: @@ -76,6 +76,7 @@ def compute_log_likelihood(self): """ + assert X is None and Y is None, "{} does not support minibatch mode".format(str(type(self))) # Get prior KL. KL = kullback_leiblers.gauss_kl_white(self.q_mu.get(), self.q_sqrt.get())