/*
 * Encog(tm) Core v3.4 - Java Version
 * http://www.heatonresearch.com/encog/
 * https://github.com/encog/encog-java-core
 
 * Copyright 2008-2016 Heaton Research, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *   
 * For more information on Heaton Research copyrights, licenses 
 * and trademarks visit:
 * http://www.heatonresearch.com/copyright
 */
package org.encog.neural.networks.training.propagation.sgd;

import org.encog.Encog;
import org.encog.EncogError;
import org.encog.engine.network.activation.ActivationFunction;
import org.encog.mathutil.error.ErrorCalculation;
import org.encog.mathutil.randomize.generate.GenerateRandom;
import org.encog.mathutil.randomize.generate.MersenneTwisterGenerateRandom;
import org.encog.ml.MLMethod;
import org.encog.ml.TrainingImplementationType;
import org.encog.ml.data.MLDataPair;
import org.encog.ml.data.MLDataSet;
import org.encog.ml.train.BasicTraining;
import org.encog.neural.error.CrossEntropyErrorFunction;
import org.encog.neural.error.ErrorFunction;
import org.encog.neural.flat.FlatNetwork;
import org.encog.neural.networks.ContainsFlat;
import org.encog.neural.networks.training.LearningRate;
import org.encog.neural.networks.training.Momentum;
import org.encog.neural.networks.training.propagation.TrainingContinuation;
import org.encog.neural.networks.training.propagation.sgd.update.AdamUpdate;
import org.encog.neural.networks.training.propagation.sgd.update.UpdateRule;
import org.encog.util.EngineArray;

public class StochasticGradientDescent extends BasicTraining implements Momentum,
		LearningRate {

	/**
	 * The learning rate.
	 */
	private double learningRate;

	/**
	 * The momentum.
	 */
	private double momentum;

    /**
     * The gradients.
     */
    private final double[] gradients;

    /**
     * The deltas for each layer.
     */
    private final double[] layerDelta;

    /**
     * L1 regularization.
     */
    private double l1;

    /**
     * L2 regularization.
     */
    private double l2;

    /**
     * The update rule to use.
     */
    private UpdateRule updateRule = new AdamUpdate();

    /**
	 * The last delta values.
	 */
	private double[] lastDelta;

    /**
     * A flat neural network.
     */
    private FlatNetwork flat;

    /**
     * The error function to use.
     */
    private ErrorFunction errorFunction = new CrossEntropyErrorFunction();

    /**
     * The error calculation.
     */
    private ErrorCalculation errorCalculation;

    private GenerateRandom rnd;

    private MLMethod method;

    public StochasticGradientDescent(final ContainsFlat network,
                                     final MLDataSet training) {
        this(network,training,new MersenneTwisterGenerateRandom());
    }

	public StochasticGradientDescent(final ContainsFlat network,
			final MLDataSet training, final GenerateRandom theRandom) {
        super(TrainingImplementationType.Iterative);

        setTraining(training);

        if( !(training instanceof BatchDataSet) ) {
            setBatchSize(25);
        }

        this.method = network;
        this.flat = network.getFlat();
        this.layerDelta = new double[this.flat.getLayerOutput().length];
        this.gradients = new double[this.flat.getWeights().length];
        this.errorCalculation = new ErrorCalculation();
        this.rnd = theRandom;
        this.learningRate = 0.001;
        this.momentum = 0.9;
    }

	public void process(final MLDataPair pair) {
        errorCalculation = new ErrorCalculation();

        double[] actual = new double[this.flat.getOutputCount()];

        flat.compute(pair.getInputArray(), actual);

		errorCalculation.updateError(actual, pair.getIdealArray(), pair.getSignificance());

		// Calculate error for the output layer.
		this.errorFunction.calculateError(
				flat.getActivationFunctions()[0], this.flat.getLayerSums(),this.flat.getLayerOutput(),
				pair.getIdeal().getData(), actual, this.layerDelta, 0,
				pair.getSignificance());

		// Apply regularization, if requested.
		if( this.l1> Encog.DEFAULT_DOUBLE_EQUAL
				|| this.l2>Encog.DEFAULT_DOUBLE_EQUAL  ) {
			double[] lp = new double[2];
			calculateRegularizationPenalty(lp);
			for(int i=0;i