Source Code for Module paramz.examples.ridge_regression

  1  ''' 
  2  Created on 16 Oct 2015 
  3   
  4  @author: Max Zwiessele 
  5  ''' 
  6  import numpy as np 
  7  from ..model import Model 
  8  from ..core.observable_array import ObsAr 
  9  from ..param import Param 
 10  from ..caching import Cacher 
 11  from ..parameterized import Parameterized 
 12   
 13 -class RidgeRegression(Model): 
 14      ''' 
 15      Ridge regression with regularization. 
 16   
 17      For any regularization to work we to gradient based optimization. 
 18      ''' 
 19 -    def __init__(self, X, Y, regularizer=None, basis=None, name='ridge_regression'): 
 20          ''' 
 21          :param array-like X: the inputs X of the regression problem 
 22          :param array-like Y: the outputs Y 
 23          :param :py:class:`paramz.examples.ridge_regression.Regularizer` regularizer: the regularizer to use 
 24          :param str name: the name of this regression object 
 25          ''' 
 26          super(RidgeRegression, self).__init__(name=name) 
 27          assert X.ndim == 2, 'inputs need to be at least a column vector' 
 28          assert Y.ndim == 2, 'inputs need to be at least a column vector' 
 29   
 30          self.X = ObsAr(X) 
 31          self.Y = ObsAr(Y) 
 32   
 33          if basis is None: 
 34              basis = Polynomial(1) 
 35          self.basis = basis 
 36   
 37          if regularizer is None: 
 38              regularizer = Ridge(1) 
 39          self.regularizer = regularizer 
 40   
 41          self.regularizer.init(basis, X.shape[1]) 
 42   
 43          self.link_parameters(self.regularizer, self.basis) 
 44   
 45      @property 
 46 -    def weights(self): 
 47          return self.regularizer.weights 
 48   
 49      @property 
 50 -    def degree(self): 
 51          return self.basis.degree 
 52   
 53      @property 
 54 -    def _phi(self): 
 55          return self.basis._basis 
 56   
 57 -    def phi(self, Xpred, degrees=None): 
 58          """ 
 59          Compute the design matrix for this model 
 60          using the degrees given by the index array 
 61          in degrees 
 62   
 63          :param array-like Xpred: inputs to compute the design matrix for 
 64          :param array-like degrees: array of degrees to use [default=range(self.degree+1)] 
 65          :returns array-like phi: The design matrix [degree x #samples x #dimensions] 
 66          """ 
 67          assert Xpred.shape[1] == self.X.shape[1], "Need to predict with same shape as training data." 
 68          if degrees is None: 
 69              degrees = range(self.basis.degree+1) 
 70          tmp_phi = np.empty((len(degrees), Xpred.shape[0], Xpred.shape[1])) 
 71          for i, w in enumerate(degrees): 
 72              # Objective function 
 73              tmpX = self._phi(Xpred, w) 
 74              tmp_phi[i] = tmpX * self.weights[[w], :] 
 75          return tmp_phi 
 76   
 77 -    def parameters_changed(self): 
 78          tmp_outer = 0. 
 79          for i in range(self.degree+1): 
 80              # Objective function 
 81              tmp_X = self._phi(self.X, i) 
 82              target_f = tmp_X.dot(self.weights[[i], :].T) 
 83              tmp_outer += target_f 
 84   
 85          tmp_outer = (self.Y-tmp_outer) 
 86          for i in range(self.degree+1): 
 87              tmp_X = self._phi(self.X, i) 
 88              # gradient: 
 89              # Note, that we updated the weights gradients 
 90              # in the basis first. So here we update 
 91              # and add in the gradient for the bound. 
 92              self.weights.gradient[i] -= 2*(tmp_outer*tmp_X).sum(0) 
 93          self._obj = (((tmp_outer)**2).sum() + self.regularizer.error.sum()) 
 94   
 95 -    def objective_function(self): 
 96          return self._obj 
 97   
 98 -    def predict(self, Xnew): 
 99          tmp_outer = 0. 
100          for i in range(self.degree+1): 
101              # Objective function 
102              tmp_X = self._phi(Xnew, i) 
103              tmp_outer += tmp_X.dot(self.weights[[i], :].T) 
104          return tmp_outer 
105   
106   
107   
108 -class Basis(Parameterized): 
109 -    def __init__(self, degree, name='basis'): 
110          """ 
111          Basis class for computing the design matrix phi(X). The weights are held 
112          in the regularizer, so that this only represents the design matrix. 
113          """ 
114          super(Basis, self).__init__(name=name) 
115          self.degree = degree 
116          # One way of setting up the caching is by the following, the 
117          # other is by the decorator. 
118          self._basis = Cacher(self.basis, self.degree+1, [], []) 
119          # If we do not set up the caching ourself, using the decorator, we 
120          # cannot set the limit of the cacher easily to the degree+1. 
121          # It could be done on runtime, by inspecting self.cache and setting 
122          # the limit of the right cacher, but that is more complex then necessary 
123   
124 -    def basis(self, X, i): 
125          """ 
126          Return the ith basis dimension. 
127          In the polynomial case, this is X**index. 
128          You can write your own basis function here, inheriting from this class 
129          and the gradients will still check. 
130   
131          Note: i will be zero for the first degree. This means we 
132          have also a bias in the model, which makes the problem of having an explicit 
133          bias obsolete. 
134          """ 
135          raise NotImplementedError('Implement the basis you want to optimize over.') 
136   
137 -class Polynomial(Basis): 
138 -    def __init__(self, degree, name='polynomial'): 
139          super(Polynomial, self).__init__(degree, name) 
140   
141 -    def basis(self, X, i): 
142          return X**i 
143   
144   
145   
146 -class Regularizer(Parameterized): 
147 -    def __init__(self, lambda_, name='regularizer'): 
148          super(Regularizer, self).__init__(name=name) 
149          self.lambda_ = lambda_ 
150          self._initialized = False 
151   
152 -    def init(self, basis, input_dim): 
153          if self._initialized: 
154              raise RuntimeError("already initialized, please use new object.") 
155          weights = np.ones((basis.degree + 1, input_dim))*np.arange(basis.degree+1)[:,None] 
156          for i in range(basis.degree+1): 
157              weights[[i]] /= basis.basis(weights[[i]], i) 
158   
159          if not isinstance(weights, Param): 
160              if weights.ndim == 1: 
161                  weights = weights[:,None] 
162              weights = Param('weights', weights) 
163          else: 
164              assert weights.ndim == 2, 'weights needs to be at least a column vector' 
165          self.weights = weights 
166          self.link_parameter(weights) 
167          self._initialized = True 
168          self.update_error() 
169   
170 -    def parameters_changed(self): 
171          if self._initialized: 
172              self.update_error() 
173   
174 -    def update_error(self): 
175          raise NotImplementedError('Set the error `error` and gradient of weights in here') 
176   
177 -class Lasso(Regularizer): 
178 -    def __init__(self, lambda_, name='Lasso'): 
179          super(Lasso, self).__init__(lambda_, name) 
180   
181 -    def update_error(self): 
182          self.error = self.lambda_*np.sum(np.abs(self.weights), 1) 
183          self.weights.gradient[:] = self.lambda_*np.sign(self.weights) 
184   
185 -class Ridge(Regularizer): 
186 -    def __init__(self, lambda_, name='Ridge'): 
187          super(Ridge, self).__init__(lambda_, name) 
188   
189 -    def update_error(self): 
190          self.error = self.lambda_*np.sum(self.weights**2, 1) 
191          self.weights.gradient[:] = self.lambda_*2*self.weights 
192