package jsat.math.optimization.stochastic;
import jsat.linear.DenseVector;
import jsat.linear.IndexValue;
import jsat.linear.ScaledVector;
import jsat.linear.Vec;
import static java.lang.Math.*;
/**
* Adam is inspired by {@link RMSProp} and {@link AdaGrad}, where the former can
* be seen as a special case of Adam. Adam has been shown to work well in
* training neural networks, and still converges well with sparse gradients.
* NOTE: that while it will converge, Adam dose not support sparse updates. So
* runtime when in highly sparse environments will be hampered.
*
* See: Kingma, D. P.,&Ba, J. L. (2015). Adam: A Method for Stochastic
* Optimization. In ICLR.
*
* @author Edward Raff
*/
public class Adam implements GradientUpdater
{
private static final long serialVersionUID = 5352504067435579553L;
//internal state
/**
* 1st moment vector
*/
private Vec m;
/**
* 2nd moment vector
*/
private Vec v;
/**
* time step
*/
private long t;
//parameters of the algo
private double alpha;
private double beta_1;
private double beta_2;
private double eps;
private double lambda;
private double vBias;
private double mBias;
public static final double DEFAULT_ALPHA = 0.0002;
public static final double DEFAULT_BETA_1 = 0.1;
public static final double DEFAULT_BETA_2 = 0.001;
public static final double DEFAULT_EPS = 1e-8;
public static final double DEFAULT_LAMBDA = 1e-8;
public Adam()
{
this(DEFAULT_ALPHA, DEFAULT_BETA_1, DEFAULT_BETA_2, DEFAULT_EPS, DEFAULT_LAMBDA);
}
public Adam(double alpha, double beta_1, double beta_2, double eps, double lambda)
{
if(alpha <= 0 || Double.isInfinite(alpha) || Double.isNaN(alpha))
throw new IllegalArgumentException("alpha must be a positive value, not " + alpha);
if(beta_1 <= 0 || beta_1 > 1 || Double.isInfinite(beta_1) || Double.isNaN(beta_1))
throw new IllegalArgumentException("beta_1 must be in (0, 1], not " + beta_1);
if(beta_2 <= 0 || beta_2 > 1 || Double.isInfinite(beta_2) || Double.isNaN(beta_2))
throw new IllegalArgumentException("beta_2 must be in (0, 1], not " + beta_2);
if(pow(1-beta_1, 2) / sqrt(1-beta_2) >= 1)
throw new IllegalArgumentException("the required property (1-beta_1)^2 / sqrt(1-beta_2) < 1, is not held by beta_1=" + beta_1 + " and beta_2=" + beta_2 );
if(lambda <= 0 || lambda >= 1 || Double.isInfinite(lambda) || Double.isNaN(lambda))
throw new IllegalArgumentException("lambda must be in (0, 1), not " + lambda);
this.alpha = alpha;
this.beta_1 = beta_1;
this.beta_2 = beta_2;
this.eps = eps;
this.lambda = lambda;
}
/**
* Copy constructor
* @param toCopy the object to copy
*/
public Adam(Adam toCopy)
{
this.alpha = toCopy.alpha;
this.beta_1 = toCopy.beta_1;
this.beta_2 = toCopy.beta_2;
this.eps = toCopy.eps;
this.lambda = toCopy.lambda;
this.t = toCopy.t;
this.mBias = toCopy.mBias;
this.vBias = toCopy.vBias;
if(toCopy.m != null)
{
this.m = toCopy.m.clone();
this.v = toCopy.v.clone();
}
}
@Override
public void update(Vec x, Vec grad, double eta)
{
update(x, grad, eta, 0, 0);
}
@Override
public double update(Vec x, Vec grad, double eta, double bias, double biasGrad)
{
t++;
//(Decay the first moment running average coefficient
double beta_1t = 1 - (1-beta_1)*pow(lambda, t-1);
//(Get gradients w.r.t. stochastic objective at timestep t)
//grad is already that value
//(Update biased first moment estimate)
m.mutableMultiply(1-beta_1t);
m.mutableAdd(beta_1t, grad);
mBias = (1-beta_1t)+beta_1t*biasGrad;
//(Update biased second raw moment estimate)
v.mutableMultiply(1-beta_2);
vBias = (1-beta_2)*vBias + beta_2 *biasGrad *biasGrad;
for(final IndexValue iv : grad)
{
final double g_i = iv.getValue();
v.increment(iv.getIndex(), beta_2*(g_i*g_i));
}
/*
* "Note that the efficiency of algorithm 1 can, at the expense of
* clarity, be improved upon by changing the order of computation, e.g.
* by replacing the last three lines in the loop with the following
* line:"
* θ_t = θ_{t−1} −[α ·√(1−(1−β_2)^t) · (1−(1−β_1)^t)−1] ·m_t/√v_t
*/
double cnst = eta*alpha*sqrt(1-pow((1-beta_2), t))/(1-pow((1-beta_1), t));
//while the algorithm may converge well with sparse data, m and v are likely to all be non-zero after observing lots of data.
for(int i = 0; i < m.length(); i++)
x.increment(i, -cnst * m.get(i)/(sqrt(v.get(i))+eps));
return cnst * mBias/(sqrt(vBias)+eps);
}
@Override
public Adam clone()
{
return new Adam(this);
}
@Override
public void setup(int d)
{
t = 0;
m = new ScaledVector(new DenseVector(d));
v = new ScaledVector(new DenseVector(d));
vBias = mBias = 0;
}
}
Ads help maintain this website.