commit f3629a2ce1f606baa6447e1d7fb4ee25a4cc58ff
Author: Étienne Simon <esimon@esimon.eu>
Date: Fri, 11 Apr 2014 19:20:19 +0200
Add class for Embeddings
Diffstat:
| A | embeddings.py | | | 57 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
1 file changed, 57 insertions(+), 0 deletions(-)
diff --git a/embeddings.py b/embeddings.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python2
+
+import numpy
+import theano
+import theano.tensor as T
+import theano.sparse as S
+
+class Embeddings(object):
+ """ Embeddings matrix class.
+
+ This class has one parameter:
+ E -- the "Embeddings"
+ """
+ def __init__(self, rng, number, dimension, tag):
+ """ Initialise the parameter.
+
+ Keyword arguments:
+ rng -- module for random number generation
+ number -- number of embeddings
+ dimension -- dimension of the embeddings
+ tag -- name of the embeddings for parameter declaration
+ """
+
+ self.number = number
+ self.dimension = dimension
+
+ E_bound = numpy.sqrt(6. / dimension)
+ E_values = rng.uniform(low=-E_bound, high=E_bound, size=(number, dimension))
+ E_values = E_values / numpy.sqrt(numpy.sum(E_values **2, axis=1))
+ self.E = theano.shared(name=tag, value=numpy.asarray(E_values, dtype=theano.config.floatX))
+
+ self.params = [E]
+
+ def embed(self, entities):
+ """ Embed given entities.
+
+ Keyword arguments:
+ entities -- a sparse matrix of size ('x', self.number)
+ """
+ return S.dot(entities, self.E)
+
+ def L1_norm(self):
+ """ Compute the L1-norm of the embeddings parameter. """
+ return T.sum(T.abs(self.E))
+
+ def sqrL2_norm(self):
+ """ Compute the squared L2-norm of the embeddings parameter. """
+ return T.sum(T.sqr(self.E))
+
+ def sgd_updates(self, cost, learning_rate):
+ """ Compute the updates to perform a SGD step w.r.t. a given cost.
+
+ Keyword arguments:
+ cost -- The cost to optimise.
+ learning_rate -- The learning rate used for gradient descent.
+ """
+ return [(self.E, self.E - learning_rate * T.grad(cost=cost, wrt=self.E))]
