/*
* Encog(tm) Core v3.4 - Java Version
* http://www.heatonresearch.com/encog/
* https://github.com/encog/encog-java-core
* Copyright 2008-2016 Heaton Research, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* For more information on Heaton Research copyrights, licenses
* and trademarks visit:
* http://www.heatonresearch.com/copyright
*/
package org.encog.neural.networks.training.propagation;
import java.util.Random;
import org.encog.Encog;
import org.encog.engine.network.activation.ActivationFunction;
import org.encog.mathutil.error.ErrorCalculation;
import org.encog.ml.data.MLDataPair;
import org.encog.ml.data.MLDataSet;
import org.encog.ml.data.basic.BasicMLDataPair;
import org.encog.neural.error.ErrorFunction;
import org.encog.neural.flat.FlatNetwork;
import org.encog.util.EngineArray;
import org.encog.util.concurrency.EngineTask;
/**
* Worker class for the mulithreaded training of flat networks.
*/
public class GradientWorker implements EngineTask {
/**
* Used to generate randomness for dropout
*/
protected Random dropoutRandomSource = new Random();
/**
* The network to train.
*/
private final FlatNetwork network;
/**
* The error calculation method.
*/
private final ErrorCalculation errorCalculation = new ErrorCalculation();
/**
* The actual values from the neural network.
*/
private final double[] actual;
/**
* The deltas for each layer.
*/
private final double[] layerDelta;
/**
* The neuron counts, per layer.
*/
private final int[] layerCounts;
/**
* The feed counts, per layer.
*/
private final int[] layerFeedCounts;
/**
* The layer indexes.
*/
private final int[] layerIndex;
/**
* The index to each layer's weights and thresholds.
*/
private final int[] weightIndex;
/**
* The output from each layer.
*/
private final double[] layerOutput;
/**
* The sums.
*/
private final double[] layerSums;
/**
* The gradients.
*/
private final double[] gradients;
/**
* The weights and thresholds.
*/
private final double[] weights;
/**
* The pair to use for training.
*/
private final MLDataPair pair;
/**
* The training data.
*/
private final MLDataSet training;
/**
* The high end of the training data.
*/
private final int low;
/**
* The low end of the training.
*/
private final int high;
/**
* The owner.
*/
private final GradientWorkerOwner owner;
/**
* Derivative add constant. Used to combat flat spot.
*/
private double[] flatSpot;
/**
* The error function to use.
*/
private final ErrorFunction errorFunction;
private double[] layerDropoutRates;
/**
* Construct a gradient worker.
*
* @param theNetwork
* The network to train.
* @param theOwner
* The owner that is doing the training.
* @param theTraining
* The training data.
* @param theLow
* The low index to use in the training data.
* @param theHigh
* The high index to use in the training data.
* @param flatSpot The flatspot additions for each layer
* @param ef Error function
*/
public GradientWorker(final FlatNetwork theNetwork,
final GradientWorkerOwner theOwner,
final MLDataSet theTraining, final int theLow,
final int theHigh, final double[] flatSpot,
ErrorFunction ef) {
this.network = theNetwork;
this.training = theTraining;
this.low = theLow;
this.high = theHigh;
this.owner = theOwner;
this.flatSpot = flatSpot;
this.errorFunction = ef;
this.layerDelta = new double[network.getLayerOutput().length];
this.gradients = new double[network.getWeights().length];
this.actual = new double[network.getOutputCount()];
this.weights = network.getWeights();
this.layerIndex = network.getLayerIndex();
this.layerCounts = network.getLayerCounts();
this.layerDropoutRates = network.getLayerDropoutRates();
this.weightIndex = network.getWeightIndex();
this.layerOutput = network.getLayerOutput();
this.layerSums = network.getLayerSums();
this.layerFeedCounts = network.getLayerFeedCounts();
this.pair = BasicMLDataPair.createPair(network.getInputCount(), network
.getOutputCount());
}
/**
* @return The network being processed.
*/
public FlatNetwork getNetwork() {
return this.network;
}
/**
* @return The weights for this network.
*/
public double[] getWeights() {
return this.weights;
}
/**
* Process one training set element.
*
* @param pair the training data information
*/
public void process(final MLDataPair pair) {
this.network.compute(pair.getInputArray(), this.actual);
this.errorCalculation.updateError(this.actual, pair.getIdealArray(), pair.getSignificance());
// Calculate error for the output layer.
this.errorFunction.calculateError(
this.network.getActivationFunctions()[0], this.layerSums,this.layerOutput,
pair.getIdeal().getData(), this.actual, this.layerDelta, this.flatSpot[0],
pair.getSignificance());
// Apply regularization, if requested.
if( this.owner.getL1()>Encog.DEFAULT_DOUBLE_EQUAL
|| this.owner.getL1()>Encog.DEFAULT_DOUBLE_EQUAL ) {
double[] lp = new double[2];
calculateRegularizationPenalty(lp);
for(int i=0;i currentLevel && this.layerDropoutRates[currentLevel] != 0) {
dropoutRate = this.layerDropoutRates[currentLevel];
}
final int index = this.weightIndex[currentLevel];
final ActivationFunction activation = this.network
.getActivationFunctions()[currentLevel];
final double currentFlatSpot = this.flatSpot[currentLevel + 1];
// handle weights
// array references are made method local to avoid one indirection
final double[] layerDelta = this.layerDelta;
final double[] weights = this.weights;
final double[] gradients = this.gradients;
final double[] layerOutput = this.layerOutput;
final double[] layerSums = this.layerSums;
int yi = fromLayerIndex;
for (int y = 0; y < fromLayerSize; y++) {
final double output = layerOutput[yi];
double sum = 0;
int wi = index + y;
final int loopEnd = toLayerIndex+toLayerSize;
if(dropoutRate == 0 || dropoutRandomSource.nextDouble() > dropoutRate)
{
for (int xi = toLayerIndex; xi < loopEnd; xi++, wi += fromLayerSize) {
gradients[wi] += output * layerDelta[xi];
sum += weights[wi] * layerDelta[xi];
}
layerDelta[yi] = sum
* (activation.derivativeFunction(layerSums[yi], layerOutput[yi])+currentFlatSpot);
} else {
layerDelta[yi] = 0;
}
yi++;
}
}
/**
* Perform the gradient calculation for the specified index range.
*/
public final void run() {
try {
this.errorCalculation.reset();
for (int i = this.low; i <= this.high; i++) {
this.training.getRecord(i, this.pair);
process(pair);
}
final double error = this.errorCalculation.calculate();
this.owner.report(this.gradients, error, null);
EngineArray.fill(this.gradients, 0);
} catch (final Throwable ex) {
this.owner.report(null, 0, ex);
}
}
public final void run(int index) {
this.training.getRecord(index, this.pair);
process(pair);
this.owner.report(this.gradients, 0, null);
EngineArray.fill(this.gradients, 0);
}
public ErrorCalculation getErrorCalculation() {
return errorCalculation;
}
/**
* @return the gradients
*/
public double[] getGradients() {
return gradients;
}
public void calculateRegularizationPenalty(double[] l) {
for (int i = 0; i < network.getLayerCounts().length - 1; i++) {
layerRegularizationPenalty(i, l);
}
}
public void layerRegularizationPenalty(final int fromLayer, final double[] l) {
final int fromCount = network.getLayerTotalNeuronCount(fromLayer);
final int toCount = network.getLayerNeuronCount(fromLayer + 1);
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++) {
for (int toNeuron = 0; toNeuron < toCount; toNeuron++) {
double w = this.network.getWeight(fromLayer, fromNeuron, toNeuron);
l[0]+=Math.abs(w);
l[1]+=w*w;
}
}
}
}
Ads help maintain this website.