package jsat.classifiers.neuralnetwork;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import jsat.classifiers.CategoricalResults;
import jsat.classifiers.ClassificationDataSet;
import jsat.classifiers.Classifier;
import jsat.classifiers.DataPoint;
import jsat.classifiers.neuralnetwork.activations.ActivationLayer;
import jsat.classifiers.neuralnetwork.activations.ReLU;
import jsat.classifiers.neuralnetwork.activations.SoftmaxLayer;
import jsat.classifiers.neuralnetwork.initializers.ConstantInit;
import jsat.classifiers.neuralnetwork.initializers.GaussianNormalInit;
import jsat.classifiers.neuralnetwork.regularizers.Max2NormRegularizer;
import jsat.linear.SparseVector;
import jsat.linear.Vec;
import jsat.math.optimization.stochastic.AdaDelta;
import jsat.parameters.Parameter;
import jsat.parameters.Parameterized;
import jsat.utils.IntList;
import jsat.utils.ListUtils;
import jsat.utils.concurrent.ParallelUtils;

/**
 * This class provides a neural network based on Geoffrey Hinton's 
 * Deep Rectified Dropout Nets. It is parameterized 
 * to be "simpler" in that the default batch size and gradient updating method
 * should require no tuning to get decent results
*
* NOTE: Training neural networks is computationally expensive, you may want to * consider a GPU implementation from another source. * * @author Edward Raff */ public class DReDNetSimple implements Classifier, Parameterized { private static final long serialVersionUID = -342281027279571332L; private SGDNetworkTrainer network; private int[] hiddenSizes; private int batchSize = 256; private int epochs = 100; /** * Creates a new DRedNet that uses two hidden layers with 1024 neurons each. * A batch size of 256 and 100 epochs will be used. */ public DReDNetSimple() { this(1024, 1024); } /** * Create a new DReDNet that uses the specified number of hidden layers. A * batch size of 256 and 100 epochs will be used. * @param hiddenLayerSizes the length indicates the number of hidden layers, * and the value in each index is the number of neurons in that layer */ public DReDNetSimple(int... hiddenLayerSizes) { setHiddenSizes(hiddenLayerSizes); } /** * Sets the hidden layer sizes for this network. The size of the array is * the number of hidden layers and the value in each index denotes the size * of that layer. * @param hiddenSizes */ public void setHiddenSizes(int[] hiddenSizes) { for(int i = 0; i < hiddenSizes.length; i++) if(hiddenSizes[i] <= 0) throw new IllegalArgumentException("Hidden layer " + i + " must contain a positive number of neurons, not " + hiddenSizes[i]); this.hiddenSizes = Arrays.copyOf(hiddenSizes, hiddenSizes.length); } /** * * @return the array of hidden layer sizes */ public int[] getHiddenSizes() { return hiddenSizes; } /** * Sets the batch size for updates * @param batchSize the number of items to compute the gradient from */ public void setBatchSize(int batchSize) { this.batchSize = batchSize; } /** * * @return the number of data points to use for one gradient computation */ public int getBatchSize() { return batchSize; } /** * Sets the number of epochs to perform * @param epochs the number of training iterations through the whole data * set */ public void setEpochs(int epochs) { if(epochs <= 0) throw new IllegalArgumentException("Number of epochs must be positive"); this.epochs = epochs; } /** * * @return the number of training iterations through the data set */ public int getEpochs() { return epochs; } @Override public CategoricalResults classify(DataPoint data) { Vec x = data.getNumericalValues(); Vec y = network.feedfoward(x); return new CategoricalResults(y.arrayCopy()); } @Override public void train(ClassificationDataSet dataSet, boolean parallel) { setup(dataSet); List X = dataSet.getDataVectors(); List Y = new ArrayList(dataSet.getSampleSize()); for(int i = 0; i < dataSet.getSampleSize(); i++) { SparseVector sv = new SparseVector(dataSet.getClassSize(), 1); sv.set(dataSet.getDataPointCategory(i), 1.0); Y.add(sv); } IntList randOrder = new IntList(X.size()); ListUtils.addRange(randOrder, 0, X.size(), 1); List Xmini = new ArrayList<>(batchSize); List Ymini = new ArrayList<>(batchSize); ExecutorService threadPool = ParallelUtils.getNewExecutor(parallel); for(int epoch = 0; epoch < epochs; epoch++) { long start = System.currentTimeMillis(); double epochError = 0; Collections.shuffle(randOrder); for(int i = 0; i < X.size(); i+=batchSize) { int to = Math.min(i+batchSize, X.size()); Xmini.clear(); Ymini.clear(); for(int j = i; j < to; j++) { Xmini.add(X.get(j)); Ymini.add(Y.get(j)); } double localErr; if(parallel) localErr = network.updateMiniBatch(Xmini, Ymini, threadPool); else localErr = network.updateMiniBatch(Xmini, Ymini); epochError += localErr; } long end = System.currentTimeMillis(); // System.out.println("Epoch " + epoch + " had error " + epochError + " took " + (end-start)/1000.0 + " seconds"); } network.finishUpdating(); } private void setup(ClassificationDataSet dataSet) { network = new SGDNetworkTrainer(); int[] sizes = new int[hiddenSizes.length+2]; sizes[0] = dataSet.getNumNumericalVars(); for(int i = 0; i < hiddenSizes.length; i++) sizes[i+1] = hiddenSizes[i]; sizes[sizes.length-1] = dataSet.getClassSize(); network.setLayerSizes(sizes); List activations = new ArrayList<>(hiddenSizes.length+2); for(int size : hiddenSizes) activations.add(new ReLU()); activations.add(new SoftmaxLayer()); network.setLayersActivation(activations); network.setRegularizer(new Max2NormRegularizer(25)); network.setWeightInit(new GaussianNormalInit(1e-2)); network.setBiasInit(new ConstantInit(0.1)); network.setEta(1.0); network.setGradientUpdater(new AdaDelta()); network.setup(); } @Override public boolean supportsWeightedData() { return false; } @Override public DReDNetSimple clone() { DReDNetSimple clone = new DReDNetSimple(hiddenSizes); if(this.network != null) clone.network = this.network.clone(); clone.batchSize = this.batchSize; clone.epochs = this.epochs; return clone; } }

Ads help maintain this website.