package jsat.math.optimization.stochastic;

import jsat.linear.ConstantVector;
import jsat.linear.DenseVector;
import jsat.linear.IndexValue;
import jsat.linear.Vec;

/**
 * AdaGrad provides an adaptive learning rate for each individual feature
*
* See: Duchi, J., Hazan, E.,&Singer, Y. (2011). Adaptive Subgradient * Methods for Online Learning and Stochastic Optimization. Journal of * Machine Learning Research, 12, 2121–2159. * * @author Edward Raff */ public class AdaGrad implements GradientUpdater { private static final long serialVersionUID = 5138474612999751777L; private Vec daigG; private double biasG; /** * Creates a new AdaGrad updater */ public AdaGrad() { } /** * Copy constructor * @param toCopy the object to copy */ public AdaGrad(AdaGrad toCopy) { if(toCopy.daigG != null) this.daigG = toCopy.daigG.clone(); this.biasG = toCopy.biasG; } @Override public void update(Vec x, Vec grad, double eta) { update(x, grad, eta, 0, 0); } @Override public double update(Vec x, Vec grad, double eta, double bias, double biasGrad) { for(IndexValue iv : grad) { final int indx = iv.getIndex(); final double grad_i = iv.getValue(); final double g_ii = daigG.get(indx); x.increment(indx, -eta*grad_i/Math.sqrt(g_ii)); daigG.increment(indx, grad_i*grad_i); } double biasUpdate = eta*biasGrad/Math.sqrt(biasG); biasG += biasGrad*biasGrad; return biasUpdate; } @Override public AdaGrad clone() { return new AdaGrad(this); } @Override public void setup(int d) { daigG = new DenseVector(new ConstantVector(1.0, d)); biasG = 1; } }

Ads help maintain this website.