Source Code for Module crino.criterion

  1  # -*- coding: utf-8 -*- 
  2   
  3  #    Copyright (c) 2014-2015 Soufiane Belharbi, Clément Chatelain, 
  4  #    Romain Hérault, Julien Lerouge, Romain Modzelewski (LITIS - EA 4108). 
  5  #    All rights reserved. 
  6  # 
  7  #    This file is part of Crino. 
  8  # 
  9  #    Crino is free software: you can redistribute it and/or modify 
 10  #    it under the terms of the GNU Lesser General Public License as published 
 11  #    by the Free Software Foundation, either version 3 of the License, or 
 12  #    (at your option) any later version. 
 13  # 
 14  #    Crino is distributed in the hope that it will be useful, 
 15  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
 16  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 17  #    GNU Lesser General Public License for more details. 
 18  # 
 19  #    You should have received a copy of the GNU Lesser General Public License 
 20  #    along with Crino. If not, see <http://www.gnu.org/licenses/>. 
 21   
 22  """ 
 23  The `criterion` module provides some differentiable loss functions in order to 
 24  perform a gradient descent on a hand-crafted neural network. 
 25   
 26  The currently implemented criterions are : 
 27      - `CrossEntropy` 
 28      - `MeanSquareError` 
 29      - `MeanAbsoluteError` 
 30      - `NegativeLogLikelihood` 
 31   
 32  See their respective documentations for their mathematical expressions and their 
 33  use cases. 
 34    
 35  :see: `module` 
 36  """ 
 37   
 38  import theano.tensor as T 
 39   
 40 -class Criterion: 
 41      """ 
 42      The Criterion class handles the loss computation between **ŷ** (the `outputs` vector of a `Module`) 
 43      and **y** (the `targets` vector). This loss has to be differentiable, in order to perform a gradient descent. 
 44   
 45      :attention: This is an abstract class, it must be derived to be used. 
 46      """ 
 47 -    def __init__(self, outputs, targets): 
 48          """ 
 49          Constructs a new `Criterion` object. 
 50   
 51          :Parameters: 
 52              outputs : :theano:`TensorVariable` 
 53                  The symbolic outputs vector of a network 
 54              targets : :theano:`TensorVariable` 
 55                  The symbolic targets vector 
 56          """ 
 57   
 58          self.outputs = outputs 
 59          """ 
 60          :ivar: The symbolic outputs vector of the network, denoted :math:`\mathbf{\hat{y}}`. 
 61          :type: :theano:`TensorVariable` 
 62          """ 
 63   
 64          self.targets = targets 
 65          """ 
 66          :ivar: The symbolic targets vector, denoted :math:`\mathbf{y}`, that will be estimated by the `outputs`. 
 67          :type: :theano:`TensorVariable` 
 68          """ 
 69   
 70          self.expression = None 
 71          """ 
 72          :ivar: The symbolic expression that expresses the loss between the `outputs` and the `targets` vectors. 
 73          :type: :theano:`TensorVariable` 
 74          """ 
 75   
 76          self.prepare() 
 77   
 78 -    def prepare(self): 
 79          """ 
 80          Computes the symbolic expression of the loss. 
 81   
 82          :attention: It must be implemented in derived classes. 
 83          """ 
 84          raise NotImplementedError("This class must be derived.") 
 85   
 86   
 87 -class CrossEntropy(Criterion): 
 88      """ 
 89      The cross-entropy criterion is well suited for targets vector 
 90      that are normalized between 0 and 1, used along with a final 
 91      `Sigmoid` activation module. 
 92   
 93      It has been experimentally demonstrated that an `AutoEncoder` 
 94      trains faster with a cross-entropy criterion. 
 95   
 96      The cross-entropy loss can be written as follows : 
 97   
 98      :math:`L_{CE} = -1/N \ \sum_{k=1}^{N} (y\cdot log(\hat{y}) + (1-y)\cdot log(1-\hat{y}))` 
 99      """ 
100   
101 -    def __init__(self, outputs, targets): 
102          """ 
103          Constructs a new `CrossEntropy` criterion. 
104   
105          :Parameters: 
106              outputs : :theano:`TensorVariable` 
107                  The symbolic `outputs` vector of the network 
108              targets : :theano:`TensorVariable` 
109                  The symbolic `targets` vector 
110          """ 
111          Criterion.__init__(self, outputs, targets) 
112   
113 -    def prepare(self): 
114          """ Computes the cross-entropy symbolic expression. """ 
115          self.expression = -T.mean(self.targets*T.log(self.outputs) + (1-self.targets)*T.log(1-self.outputs)) 
116   
117   
118 -class MeanSquareError(Criterion): 
119      """ 
120      The mean square error criterion is used in the least squares method, 
121      it is well suited for data fitting. 
122   
123      The mean square loss can be written as follows : 
124   
125      :math:`L_{MSE} = -1/N \ \sum_{k=1}^{N} (y-\hat{y})^2` 
126      """ 
127   
128 -    def __init__(self, outputs, targets): 
129          """ 
130          Constructs a new `MeanSquareError` criterion. 
131   
132          :Parameters: 
133              outputs : :theano:`TensorVariable` 
134                  The symbolic `outputs` vector of the network 
135              targets : :theano:`TensorVariable` 
136                  The symbolic `targets` vector 
137          """ 
138          Criterion.__init__(self, outputs, targets) 
139   
140 -    def prepare(self): 
141          """ Computes the mean square error symbolic expression. """ 
142          self.expression = T.mean(T.sqr(self.outputs - self.targets)) 
143   
144   
145 -class MeanAbsoluteError(Criterion): 
146      """ 
147      The mean absolute error criterion is used in the least absolute deviations method. 
148   
149      The mean absolute loss can be written as follows : 
150   
151      :math:`L_{MSE} = -1/N \ \sum_{k=1}^{N} |y-\hat{y}|` 
152      """ 
153   
154 -    def __init__(self, outputs, targets): 
155          """ 
156          Constructs a new `MeanAbsoluteError` criterion. 
157   
158          :Parameters: 
159              outputs : :theano:`TensorVariable` 
160                  The symbolic `outputs` vector of a network 
161              targets : :theano:`TensorVariable` 
162                  The symbolic `targets` vector 
163          """ 
164          Criterion.__init__(self, outputs, targets) 
165   
166 -    def prepare(self): 
167          """ Computes the mean absolute error symbolic expression. """ 
168          self.expression = T.mean(T.abs_(self.outputs - self.targets)) 
169   
170 -class NegativeLogLikelihood(Criterion): 
171      """ 
172      The Negative Log Likelihood criterion is used in classification task. 
173   
174      It is meant to be connected to a Softmax Module as the last layer of the MLP. 
175   
176      The output size should be the same as the number of possible classes. 
177   
178      The target size should be the same as the number of possible classes, with value in between [0..1]. 
179      If not weighted, all 0 except 1 on the correct class. 
180   
181      The Negative Log Likelihood can be written as follows : 
182   
183      :math:`L_{NLL} = -log(<\hat{y},y>)` 
184      """ 
185   
186 -    def __init__(self, outputs, targets): 
187          """ 
188          Constructs a new `NegativeLogLikelihood` criterion. 
189   
190          :Parameters: 
191              outputs : :theano:`TensorVariable` 
192                  The symbolic `outputs` vector of a network 
193              targets : :theano:`TensorVariable` 
194                  The symbolic `targets` vector 
195          """ 
196          Criterion.__init__(self, outputs, targets) 
197   
198 -    def prepare(self): 
199          """ Computes the Negative Log Likelihood symbolic expression. """ 
200          self.expression = -T.log(T.sum(self.outputs*self.targets)) 
201