package jsat.classifiers.neuralnetwork;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
import jsat.classifiers.neuralnetwork.activations.ActivationLayer;
import jsat.classifiers.neuralnetwork.initializers.BiastInitializer;
import jsat.classifiers.neuralnetwork.initializers.WeightInitializer;
import jsat.classifiers.neuralnetwork.regularizers.Max2NormRegularizer;
import jsat.classifiers.neuralnetwork.regularizers.WeightRegularizer;
import jsat.linear.DenseMatrix;
import jsat.linear.DenseVector;
import jsat.linear.Matrix;
import jsat.linear.Vec;
import jsat.math.decayrates.DecayRate;
import jsat.math.decayrates.NoDecay;
import jsat.math.optimization.stochastic.GradientUpdater;
import jsat.math.optimization.stochastic.SimpleSGD;
import jsat.utils.SystemInfo;
import jsat.utils.random.RandomUtil;
import jsat.utils.random.XORWOW;
/**
* This class provides a highly configurable and generalized method of training
* a neural network using Stochastic Gradient Decent.
*
* Note, the API of this class may change in the future.
*
* @author Edward Raff
*/
public class SGDNetworkTrainer implements Serializable
{
private static final long serialVersionUID = 5753653181230693131L;
/**
* An array where the length indicates the number of layers and the value of
* each index indicates the number of neurons in that layer. This includes
* both the input and output layers
*/
private int[] layerSizes;
/**
* The base learning rate to use
*/
private double eta;
/**
* The dropout probability for the input layers
*/
private double p_i;
/**
* The integer threshold to used when sampling a value from
* {@link Random#nextInt() } to get the correct dropout probability
*/
private int p_i_intThresh;
/**
* The dropout probability for the hidden layers.
*/
private double p_o;
/**
* The integer threshold to used when sampling a value from
* {@link Random#nextInt() } to get the correct dropout probability
*/
private int p_o_intThresh;
/**
* The gradient updater to use for updating weights and biases
*/
private GradientUpdater updater = new SimpleSGD();
/**
* The weight regularization method
*/
private WeightRegularizer regularizer = new Max2NormRegularizer(15);
/**
* The method to initialize all neuron connection weights from
*/
private WeightInitializer weightInit;
/**
* The method to initialize all neuron bias values from
*/
private BiastInitializer biasInit;
/**
* This list contains the neuron weight connection matrix for each layer
* after the input layer
*/
private List W;
/**
* This list contains the gradients to update the weight matrices by
*/
private List W_deltas;
/**
* This list contains the gradient updaters used for each layer, where there
* is a list of each matrix and each matrix has a list for each row.
*/
private List> W_updaters;
/**
* This list contains the neuron bias connections for each layer after the
* input layer
*/
private List B;
/**
* This list contains the gradients to update the weight biases by
*/
private List B_deltas;
/**
* This list contains the gradient updaters used for each set of bias
* connections
*/
private List B_updaters;
/**
* This list contains the activation method for each layer after the input
* layer
*/
private List layersActivation;
/**
* The decay rate to apply to the base learning rate
*/
private DecayRate etaDecay = new NoDecay();
/**
* The time step, incremented after every mini batch
*/
private int time;
/**
* Matrices for storing the activations of each layer
*/
private Matrix[] activations;
private Matrix[] unactivated;
private Matrix[] deltas;
/**
* Creates a new SGD network training that uses dropout
*/
public SGDNetworkTrainer()
{
setDropoutInput(0.2);
setDropoutHidden(0.5);
}
/**
* Copy constructor
* @param toCopy the object to copy
*/
public SGDNetworkTrainer(SGDNetworkTrainer toCopy)
{
this.layerSizes = Arrays.copyOf(toCopy.layerSizes, toCopy.layerSizes.length);
this.eta = toCopy.eta;
this.weightInit = toCopy.weightInit.clone();
this.biasInit = toCopy.biasInit.clone();
this.regularizer = toCopy.regularizer.clone();
this.updater = toCopy.updater.clone();
this.setDropoutInput(toCopy.getDropoutInput());
this.setDropoutHidden(toCopy.getDropoutHidden());
if(toCopy.W != null)
{
this.W = new ArrayList();
for(Matrix w : toCopy.W)
this.W.add(w.clone());
this.B = new ArrayList();
for(Vec b : toCopy.B)
this.B.add(b.clone());
}
if(toCopy.W_deltas != null)
{
this.W_deltas = new ArrayList();
for(Matrix w : toCopy.W_deltas)
this.W_deltas.add(w.clone());
this.B_deltas = new ArrayList();
for(Vec b : toCopy.B_deltas)
this.B_deltas.add(b.clone());
}
if(toCopy.W_updaters != null)
{
this.W_updaters = new ArrayList>();
for(List updaters : toCopy.W_updaters)
{
List copyUpdaters = new ArrayList(updaters.size());
this.W_updaters.add(copyUpdaters);
for(GradientUpdater item : updaters)
copyUpdaters.add(item.clone());
}
this.B_updaters = new ArrayList(toCopy.B_updaters);
for(GradientUpdater item : toCopy.B_updaters)
this.B_updaters.add(item.clone());
}
this.layersActivation = new ArrayList(toCopy.layersActivation.size());
for(ActivationLayer activation : toCopy.layersActivation)
this.layersActivation.add(activation.clone());
}
/**
* Sets the probability of dropping a value from the input layer
* @param p the probability in [0, 1) of dropping a value in the input layer
*/
public void setDropoutInput(double p)
{
if(p < 0 || p >= 1 || Double.isNaN(p))
throw new IllegalArgumentException("Dropout probability must be in [0,1) not " + p);
p_i = p;
p_i_intThresh = (int) (0xffffffffL*p_i+Integer.MIN_VALUE);
}
/**
*
* @return the dropout probability for the input layer
*/
public double getDropoutInput()
{
return p_i;
}
/**
* Sets the probability of dropping a value from the hidden layer
* @param p the probability in [0, 1) of dropping a value in the hidden
* layer
*/
public void setDropoutHidden(double p)
{
if(p < 0 || p >= 1 || Double.isNaN(p))
throw new IllegalArgumentException("Dropout probability must be in [0,1) not " + p);
p_o = p;
p_o_intThresh = (int) (0xffffffffL*p_o+Integer.MIN_VALUE);
}
/**
*
* @return the dropout probability for the hidden layers
*/
public double getDropoutHidden()
{
return p_o;
}
/**
* Sets the decay rate on the global learning rate over time
* @param etaDecay the decay rate to use
*/
public void setEtaDecay(DecayRate etaDecay)
{
this.etaDecay = etaDecay;
}
/**
*
* @return the decay rate in use
*/
public DecayRate getEtaDecay()
{
return etaDecay;
}
/**
* Sets the base global learning rate.
* @param eta the learning rate to use
*/
public void setEta(double eta)
{
if(eta <= 0 || Double.isNaN(eta) || Double.isInfinite(eta))
throw new IllegalArgumentException("eta must be a positive constant, not " + eta);
this.eta = eta;
}
/**
*
* @return the global learning rate used
*/
public double getEta()
{
return eta;
}
/**
* Sets the method of regularizing the connections weights
* @param regularizer the method of regularizing the network
*/
public void setRegularizer(WeightRegularizer regularizer)
{
this.regularizer = regularizer;
}
/**
*
* @return the regularizer for the network
*/
public WeightRegularizer getRegularizer()
{
return regularizer;
}
/**
* Sets the array indicating the total number of layers in the network and
* the sizes of each layer. The length of the array is the number of layers
* and the value at each index is the size of that layer.
* @param layerSizes the array of layer sizes
*/
public void setLayerSizes(int... layerSizes)
{
this.layerSizes = layerSizes;
}
/**
*
* @return the array of layer sizes in the network
*/
public int[] getLayerSizes()
{
return layerSizes;
}
/**
* Sets the list of layer activations for all layers other than the input
* layer.
* @param layersActivation the list of hidden and output layer activations
*/
public void setLayersActivation(List layersActivation)
{
this.layersActivation = layersActivation;
}
/**
* Sets the gradient update that will be used when updating the weight
* matrices and bias terms.
* @param updater the updater to use
*/
public void setGradientUpdater(GradientUpdater updater)
{
this.updater = updater;
}
/**
*
* @return the gradient updater used
*/
public GradientUpdater getGradientUpdater()
{
return updater;
}
/**
* Sets the method used to initialize matrix connection weights
* @param weightInit the weight initialization method
*/
public void setWeightInit(WeightInitializer weightInit)
{
this.weightInit = weightInit;
}
/**
*
* @return the weight initialization method
*/
public WeightInitializer getWeightInit()
{
return weightInit;
}
/**
* Sets the method to use when initializing neuron bias values
* @param biasInit the bias initialization method
*/
public void setBiasInit(BiastInitializer biasInit)
{
this.biasInit = biasInit;
}
/**
*
* @return the bias initialization method
*/
public BiastInitializer getBiasInit()
{
return biasInit;
}
/**
* Prepares the network by creating all needed structure, initializing
* weights, and preparing it for updates
*/
public void setup()
{
assert (layersActivation.size() == layerSizes.length-1);
W = new ArrayList(layersActivation.size());
B = new ArrayList(layersActivation.size());
Random rand = RandomUtil.getRandom();
for(int l = 1; l < layerSizes.length; l++)
{
W.add(new DenseMatrix(layerSizes[l], layerSizes[l-1]));
weightInit.init(W.get(W.size()-1), rand);
B.add(new DenseVector(layerSizes[l]));
biasInit.init(B.get(B.size()-1), layerSizes[l-1], rand);
}
time = 0;
prepareForUpdating();
}
/**
* This method assumes that the neural network structure is already in
* place, and prepares only the structure needed to perform updates.
* Any gradient related information that was being used before (such as
* momentum when performing updates) will be lost
*/
private void prepareForUpdating()
{
W_deltas = new ArrayList(layersActivation.size());
W_updaters = new ArrayList>(layersActivation.size());
B_deltas = new ArrayList(layersActivation.size());
B_updaters = new ArrayList(layersActivation.size());
for(int l = 1; l < layerSizes.length; l++)
{
W_deltas.add(new DenseMatrix(layerSizes[l], layerSizes[l-1]));
B_deltas.add(new DenseVector(layerSizes[l]));
//updaters
List W_updaters_l = new ArrayList(layerSizes[l]);
for(int i = 0; i < layerSizes[l]; i++)
{
GradientUpdater W_updater = updater.clone();
W_updater.setup(layerSizes[l-1]);
W_updaters_l.add(W_updater);
}
W_updaters.add(W_updaters_l);
B_updaters.add(updater.clone());
B_updaters.get(B_updaters.size()-1).setup(layerSizes[l]);
}
activations = new Matrix[layersActivation.size()];
unactivated = new Matrix[layersActivation.size()];
deltas = new Matrix[layersActivation.size()];
}
/**
* Calling this method indicates that the user has no intentions of updating
* the network again and is ready to use it for prediction. This will remove
* objects not needed for prediction and do cleanup.
*/
public void finishUpdating()
{
W_deltas = null;
W_updaters = null;
B_deltas = null;
B_updaters = null;
activations = unactivated = deltas = null;
W.get(0).mutableMultiply(1.0-p_i);
B.get(0).mutableMultiply(1.0-p_i);
for(int i = 1; i < W.size(); i++)
{
W.get(i).mutableMultiply(1.0-p_o);
B.get(i).mutableMultiply(1.0-p_o);
}
}
/**
* Performs a mini-batch update of the network using the given input and
* output pairs
* @param x the list of input values
* @param y the list of output values
* @return the error incurred on the given mini batch
*/
public double updateMiniBatch(List x, List y)
{
return updateMiniBatch(x, y, null);
}
/**
* Performs a mini-batch update of the network using the given input and
* output pairs
* @param x the list of input values
* @param y the list of output values
* @param ex the source of threads for parallel computation, may be
* {@code null}
* @return the error incurred on the given mini batch
*/
public double updateMiniBatch(List x, List y, ExecutorService ex)
{
Random rand = RandomUtil.getRandom();
for(Matrix w : W_deltas)
w.zeroOut();
for(Vec b : B_deltas)
b.zeroOut();
for(int i = 0; i < layersActivation.size(); i++)
{
//TODO isntead of making a whole new matrix every time, use a submatrix when bigger and enlarge when too small
if(activations[i] == null || activations[i].cols() != x.size())
activations[i] = new DenseMatrix(layerSizes[i+1], x.size());
if(unactivated[i] == null || unactivated[i].cols() != x.size())
unactivated[i] = new DenseMatrix(layerSizes[i+1], x.size());
if(deltas[i] == null || deltas[i].cols() != x.size())
deltas[i] = new DenseMatrix(layerSizes[i+1], x.size());
}
Matrix X = new DenseMatrix(layerSizes[0], x.size());
for (int j = 0; j < x.size(); j++)
x.get(j).copyTo(X.getColumnView(j));
if(p_i > 0)
applyDropout(X, p_i_intThresh, rand, ex);
double errorMade = 0;
feedforward(X, activations, unactivated, ex, rand);
errorMade = backpropagateError(deltas, activations, x, y, errorMade, ex, unactivated);
accumulateUpdates(X, activations, deltas, ex, x);
double eta_cur = etaDecay.rate(time++, eta);
if(ex == null)
applyGradient(eta_cur);
else
applyGradient(eta_cur, ex);
return errorMade;
}
private void feedforward(Matrix X, Matrix[] activationsM, Matrix[] unactivatedM, ExecutorService ex, Random rand)
{
//feed forward
for (int l = 0; l < layersActivation.size(); l++)
{
final Matrix a_lprev = (l == 0 ? X : activationsM[l - 1]);
final Matrix a_l = activationsM[l];
final Matrix z_l = unactivatedM[(l)];
z_l.zeroOut();
if(ex == null)
W.get(l).multiply(a_lprev, z_l);
else
W.get(l).multiply(a_lprev, z_l, ex);
//add the bias term back in
final Vec B_l = B.get(l);
if (ex == null)
{
for (int i = 0; i < z_l.rows(); i++)
{
final double B_li = B_l.get(i);
for (int j = 0; j < z_l.cols(); j++)
z_l.increment(i, j, B_li);
}
}
else
{
final CountDownLatch latch = new CountDownLatch(SystemInfo.LogicalCores);
for (int id = 0; id < SystemInfo.LogicalCores; id++)
{
final int ID = id;
ex.submit(new Runnable()
{
@Override
public void run()
{
for (int i = ID; i < z_l.rows(); i += SystemInfo.LogicalCores)
{
final double B_li = B_l.get(i);
for (int j = 0; j < z_l.cols(); j++)
z_l.increment(i, j, B_li);
}
latch.countDown();
}
});
}
try
{
latch.await();
}
catch (InterruptedException ex1)
{
Logger.getLogger(SGDNetworkTrainer.class.getName()).log(Level.SEVERE, null, ex1);
}
}
if (p_o > 0 && l != layersActivation.size() - 1)
applyDropout(z_l, p_o_intThresh, rand, ex);
layersActivation.get(l).activate(z_l, a_l, false);
}
}
/**
* Feeds the given singular pattern through the network and computes its
* activations
* @param x the input vector to feed forward through the network
* @return the final activation for this network
*/
public Vec feedfoward(Vec x)
{
Vec a_lprev = x;
for (int l = 0; l < layersActivation.size(); l++)
{
Vec z_l = new DenseVector(layerSizes[l+1]);
z_l.zeroOut();
W.get(l).multiply(a_lprev, 1.0, z_l);
//add the bias term back in
final Vec B_l = B.get(l);
z_l.mutableAdd(B_l);
layersActivation.get(l).activate(z_l, z_l);
a_lprev = z_l;
}
return a_lprev;
}
private double backpropagateError(Matrix[] deltasM, Matrix[] activationsM, List x, List y, double errorMade, ExecutorService ex, Matrix[] unactivatedM)
{
//backpropagate the error
for (int l = layersActivation.size() - 1; l >= 0; l--)
{
Matrix delta_l = deltasM[l];
if (l == layersActivation.size() - 1)//output layer
{
activationsM[(l)].copyTo(delta_l);
for(int r = 0; r < x.size(); r++)
{
delta_l.getColumnView(r).mutableSubtract(y.get(r));
errorMade += delta_l.getColumnView(r).pNorm(2);
}
}
else//any other layer
{
delta_l.zeroOut();
if(ex == null)
W.get(l+1).transposeMultiply(deltasM[l+1], delta_l);
else
W.get(l+1).transposeMultiply(deltasM[l+1], delta_l, ex);
layersActivation.get(l).backprop(unactivatedM[l], activationsM[l], delta_l, delta_l, false);
}
}
return errorMade;
}
private void accumulateUpdates(Matrix X, Matrix[] activationsM, Matrix[] deltasM, ExecutorService ex, final List x)
{
final double invXsize = 1.0/x.size();
//accumulate updates
for (int l = 0; l < layersActivation.size(); l++)
{
final Matrix a_lprev = (l == 0 ? X : activationsM[(l - 1)]);
final Matrix delta_l = deltasM[l];
if(ex == null)
delta_l.multiplyTranspose(a_lprev, W_deltas.get(l));
else
delta_l.multiplyTranspose(a_lprev, W_deltas.get(l), ex);
W_deltas.get(l).mutableMultiply(invXsize);
final Vec B_delta_l = B_deltas.get(l);
if(ex == null)
for(int i = 0; i < delta_l.rows(); i++)
{
double change = 0;
for(int j = 0; j < delta_l.cols(); j++)
change += delta_l.get(i, j);
B_delta_l.increment(i, change*invXsize);
}
else
{
final CountDownLatch latch = new CountDownLatch(Math.min(SystemInfo.LogicalCores, delta_l.rows()));
for(int id = 0; id < SystemInfo.LogicalCores; id++)
{
final int ID = id;
ex.submit(new Runnable()
{
@Override
public void run()
{
for(int i = ID; i < delta_l.rows(); i+=SystemInfo.LogicalCores)
{
double change = 0;
for(int j = 0; j < delta_l.cols(); j++)
change += delta_l.get(i, j);
B_delta_l.increment(i, change*invXsize);
}
latch.countDown();
}
});
}
try
{
latch.await();
}
catch (InterruptedException ex1)
{
Logger.getLogger(SGDNetworkTrainer.class.getName()).log(Level.SEVERE, null, ex1);
}
}
}
}
private void applyGradient(double eta_cur)
{
//apply gradient
for(int l = 0; l < layersActivation.size(); l++)
{
B_updaters.get(l).update(B.get(l), B_deltas.get(l), eta_cur);
final Matrix W_l = W.get(l);
final Matrix W_dl = W_deltas.get(l);
for(int i = 0; i < W_l.rows(); i++)
{
Vec W_li = W_l.getRowView(i);
W_updaters.get(l).get(i).update(W_li, W_dl.getRowView(i), eta_cur);
}
regularizer.applyRegularization(W_l, B.get(l));
}
}
private void applyGradient(final double eta_cur, ExecutorService ex)
{
List> futures = new ArrayList>();
//apply gradient
for(int l = 0; l < layersActivation.size(); l++)
{
B_updaters.get(l).update(B.get(l), B_deltas.get(l), eta_cur);
final Matrix W_l = W.get(l);
final Matrix W_dl = W_deltas.get(l);
final int L = l;
for(int indx = 0; indx < W_l.rows(); indx++)
{
final int i = indx;
futures.add(ex.submit(new Runnable()
{
@Override
public void run()
{
Vec W_li = W_l.getRowView(i);
W_updaters.get(L).get(i).update(W_li, W_dl.getRowView(i), eta_cur);
B.get(L).set(i, regularizer.applyRegularizationToRow(W_li, B.get(L).get(i)));
}
}));
}
}
try
{
for(Future> future : futures)
future.get();
}
catch (InterruptedException e)
{
}
catch (ExecutionException e)
{
}
}
/**
* Applies dropout to the given matrix
* @param X the matrix to dropout values from
* @param randThresh the threshold that a random integer must be less than to get dropped out
* @param rand the source of randomness
* @param ex the source of threads for parlallel computation, or {@code null}
*/
private static void applyDropout(final Matrix X, final int randThresh, final Random rand, ExecutorService ex)
{
if (ex == null)
{
for (int i = 0; i < X.rows(); i++)
for (int j = 0; j < X.cols(); j++)
if (rand.nextInt() < randThresh)
X.set(i, j, 0.0);
}
else
{
final CountDownLatch latch = new CountDownLatch(SystemInfo.LogicalCores);
for(int id = 0; id < SystemInfo.LogicalCores; id++)
{
final int ID = id;
ex.submit(new Runnable()
{
@Override
public void run()
{
for (int i = ID; i < X.rows(); i+=SystemInfo.LogicalCores)
for (int j = 0; j < X.cols(); j++)
if (rand.nextInt() < randThresh)
X.set(i, j, 0.0);
latch.countDown();
}
});
}
try
{
latch.await();
}
catch (InterruptedException ex1)
{
Logger.getLogger(SGDNetworkTrainer.class.getName()).log(Level.SEVERE, null, ex1);
}
}
}
@Override
protected SGDNetworkTrainer clone()
{
return new SGDNetworkTrainer(this);
}
}
Ads help maintain this website.