package jsat.math.optimization.stochastic;
import jsat.linear.DenseVector;
import jsat.linear.IndexValue;
import jsat.linear.ScaledVector;
import jsat.linear.Vec;
/**
* AdaDelta is inspired by {@link AdaGrad} and was developed for use primarily
* in neural networks. It still maintains a per feature learning rate, however
* unlike AdaGrad the learning rates may increase over time and are highly
* robust to any individual learning rate.
*
* See: Zeiler, M. D. (2012). ADADELTA: An Adaptive Learning Rate Method.
* CoRR, abs/1212.5.
*
* @author Edward Raff
*/
public class AdaDelta implements GradientUpdater
{
private static final long serialVersionUID = 5855631993426837618L;
private double rho;
private Vec gSqrd;
private Vec deltaXSqrt;
private double biasGSqrd;
private double deltaBiasSqrt;
private double eps = 0.0001;
/**
* Creates a new AdaDelta updater using a decay rate of 0.95
*/
public AdaDelta()
{
this(0.95);
}
/**
* Creates a new AdaDelta updater
* @param rho the decay rate to use
*/
public AdaDelta(double rho)
{
setRho(rho);
}
/**
* Copy constructor
* @param toCopy the object to copy
*/
public AdaDelta(AdaDelta toCopy)
{
this.rho = toCopy.rho;
if(toCopy.gSqrd != null)
{
this.gSqrd = toCopy.gSqrd.clone();
this.deltaXSqrt = toCopy.deltaXSqrt.clone();
}
this.biasGSqrd = toCopy.biasGSqrd;
this.deltaBiasSqrt = toCopy.deltaBiasSqrt;
}
/**
* Sets the decay rate used by AdaDelta. Lower values focus more on the
* current gradient, where higher values incorporate a longer history.
*
* @param rho the decay rate in (0, 1) to use
*/
public void setRho(double rho)
{
if(rho <= 0 || rho >= 1 || Double.isNaN(rho))
throw new IllegalArgumentException("Rho must be in (0, 1)");
this.rho = rho;
}
/**
*
* @return the decay rate that will be used
*/
public double getRho()
{
return rho;
}
@Override
public void update(Vec x, Vec grad, double eta)
{
update(x, grad, eta, 0, 0);
}
@Override
public double update(Vec x, Vec grad, double eta, double bias, double biasGrad)
{
gSqrd.mutableMultiply(rho);
biasGSqrd *= rho;
for(IndexValue iv : grad)
{
final int indx = iv.getIndex();
final double grad_i = iv.getValue();
gSqrd.increment(indx, grad_i*grad_i*(1-rho));//step 4
final double gSqrd_i = gSqrd.get(indx);
final double deltaX_i = deltaXSqrt.get(indx);
final double newDeltaX_i = -Math.sqrt((deltaX_i+eps)/(gSqrd_i+eps))*grad_i;//step 5
x.increment(indx, eta*newDeltaX_i);//step 7
deltaXSqrt.increment(indx, (1-rho)/rho*newDeltaX_i*newDeltaX_i);//step 6, using (1-rho)/rho so we can multiply by rho at the end to get the correct result
}
//step 6 correction, apply rho to the left hand side
deltaXSqrt.mutableMultiply(rho);
//bias term
biasGSqrd += biasGrad*biasGrad*(1-rho);
double newDeltaBias = Math.sqrt((deltaBiasSqrt+eps)/(biasGSqrd+eps))*biasGrad;
double biasUpdate = eta*newDeltaBias;
deltaBiasSqrt += (1-rho)/rho*newDeltaBias*newDeltaBias;
deltaBiasSqrt *= rho;
return biasUpdate;
}
@Override
public AdaDelta clone()
{
return new AdaDelta(this);
}
@Override
public void setup(int d)
{
gSqrd = new ScaledVector(new DenseVector(d));
deltaXSqrt = new ScaledVector(new DenseVector(d));
deltaBiasSqrt = biasGSqrd = 0;
}
}
Ads help maintain this website.