mila-udem / blocks

    """Scales the step size by a running average of the recent step norms.

    Parameters

    ----------

    decay_rate : float, optional

        How fast the running average decays, value in [0, 1]

        (lower is faster).  Defaults to 0.9.

    max_scaling : float, optional

        Maximum scaling of the step size, in case the running average is

        really small. Needs to be greater than 0. Defaults to 1e5.

    Notes

    -----

    This step rule is intended to be used in conjunction with another

    step rule, _e.g._ :class:`Scale`. For an all-batteries-included

    experience, look at :class:`RMSProp`.

    In general, this step rule should be used _before_ other step rules,

    because it has normalization properties that may undo their work.

    For instance, it should be applied first when used in conjunction

    with :class:`Scale`.

    For more information, see [Hint2014]_.

    """

    def __init__(self, decay_rate=0.9, max_scaling=1e5):

        if not 0.0 <= decay_rate <= 1.0:

            raise ValueError("decay rate needs to be in [0, 1]")

        if max_scaling <= 0:

            raise ValueError("max. scaling needs to be greater than 0")

        self.decay_rate = shared_floatx(decay_rate, "decay_rate")

        add_role(self.decay_rate, ALGORITHM_HYPERPARAMETER)

        self.epsilon = 1. / max_scaling

    def compute_step(self, parameter, previous_step):

        mean_square_step_tm1 = _create_algorithm_buffer_for(

            parameter, "mean_square_step_tm1")

        mean_square_step_t = (

            self.decay_rate * mean_square_step_tm1 +

            (1 - self.decay_rate) * tensor.sqr(previous_step))

        rms_step_t = tensor.maximum(

            tensor.sqrt(mean_square_step_t), self.epsilon)

        step = previous_step / rms_step_t

        updates = [(mean_square_step_tm1, mean_square_step_t)]

        return step, updates

class RMSProp(CompositeRule):

    """Scales the step size by a running average of the recent step norms.

    Combines :class:`BasicRMSProp` and :class:`Scale` to form the step rule

    described in [Hint2014]_.

    """Implements the AdaGrad learning rule.

    Parameters

    ----------

    learning_rate : float, optional

        Step size.

        Default value is set to 0.0002.

    epsilon : float, optional

        Stabilizing constant for one over root of sum of squares.

        Defaults to 1e-6.

    Notes

    -----

    For more information, see [ADAGRAD]_.

    .. [ADADGRAD] Duchi J, Hazan E, Singer Y.,

       *Adaptive subgradient methods for online learning and

        stochastic optimization*,

       http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf

    """

    def __init__(self, learning_rate=0.002, epsilon=1e-6):

        self.learning_rate = shared_floatx(learning_rate, "learning_rate")

        self.epsilon = shared_floatx(epsilon, "epsilon")

        add_role(self.learning_rate, ALGORITHM_HYPERPARAMETER)

        add_role(self.epsilon, ALGORITHM_HYPERPARAMETER)

    def compute_step(self, parameter, previous_step):

        name = 'adagrad_sqs'

        if parameter.name:

            name += '_' + parameter.name

        ssq = _create_algorithm_buffer_for(parameter, name=name)

        ssq_t = (tensor.sqr(previous_step) + ssq)

        step = (self.learning_rate * previous_step /

                (tensor.sqrt(ssq_t) + self.epsilon))

        updates = [(ssq, ssq_t)]

        return step, updates

class Adam(StepRule):

    """Adam optimizer as described in [King2014]_.

    .. [King2014] Diederik Kingma, Jimmy Ba,

       *Adam: A Method for Stochastic Optimization*,