package jsat.math.optimization.stochastic;
import jsat.linear.DenseVector;
import jsat.linear.IndexValue;
import jsat.linear.ScaledVector;
import jsat.linear.Vec;
/**
* rmsprop is an adpative learning weight scheme proposed by Geoffrey Hinton.
* Provides an adaptive learning rate for each individual feature
*
* @author Edward Raff
*/
public class RMSProp implements GradientUpdater
{
private static final long serialVersionUID = 3512851084092042727L;
private double rho;
private Vec daigG;
private double biasG;
/**
* Creates a new RMSProp updater that uses a decay rate of 0.9
*/
public RMSProp()
{
this(0.9);
}
/**
* Creates a new RMSProp updater
* @param rho the decay rate to use
*/
public RMSProp(double rho)
{
setRho(rho);
}
/**
* Sets the decay rate used by rmsprop. Lower values focus more on the
* current gradient, where higher values incorporate a longer history.
*
* @param rho the decay rate in (0, 1) to use
*/
public void setRho(double rho)
{
if(rho <= 0 || rho >= 1 || Double.isNaN(rho))
throw new IllegalArgumentException("Rho should be a value in (0, 1) not " + rho);
this.rho = rho;
}
/**
*
* @return the decay rate parameter to use
*/
public double getRho()
{
return rho;
}
/**
* Copy constructor
* @param toCopy the object to copy
*/
public RMSProp(RMSProp toCopy)
{
if(toCopy.daigG != null)
this.daigG = toCopy.daigG.clone();
this.rho = toCopy.rho;
this.biasG = toCopy.biasG;
}
@Override
public void update(Vec x, Vec grad, double eta)
{
update(x, grad, eta, 0, 0);
}
@Override
public double update(Vec x, Vec grad, double eta, double bias, double biasGrad)
{
daigG.mutableMultiply(rho);
for(IndexValue iv : grad)
{
final int indx = iv.getIndex();
final double grad_i = iv.getValue();
daigG.increment(indx, (1-rho)*grad_i*grad_i);
double g_iiRoot = Math.max(Math.sqrt(daigG.get(indx)), Math.abs(grad_i));//tiny grad sqrd could result in zero
x.increment(indx, -eta*grad_i/g_iiRoot);
}
biasG *= rho;
biasG += (1-rho)*biasGrad*biasGrad;
double g_iiRoot = Math.max(Math.sqrt(biasG), Math.abs(biasGrad));//tiny grad sqrd could result in zero
return eta*biasGrad/g_iiRoot;
}
@Override
public RMSProp clone()
{
return new RMSProp(this);
}
@Override
public void setup(int d)
{
daigG = new ScaledVector(new DenseVector(d));
biasG = 0;
}
}
Ads help maintain this website.