// Catalano Machine Learning Library
// The Catalano Framework
//
// Copyright © Diego Catalano, 2012-2016
// diego.catalano at live.com
//
// Copyright © Andrew Kirillov, 2007-2008
// andrew.kirillov@gmail.com
//
//    This library is free software; you can redistribute it and/or
//    modify it under the terms of the GNU Lesser General Public
//    License as published by the Free Software Foundation; either
//    version 2.1 of the License, or (at your option) any later version.
//
//    This library is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//    Lesser General Public License for more details.
//
//    You should have received a copy of the GNU Lesser General Public
//    License along with this library; if not, write to the Free Software
//    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
//

package Catalano.MachineLearning.Exploration;

import java.util.Random;

/**
 * The class implements roulette whell exploration policy. Acording to the policy,
 * action a at state s is selected with the next probability:
 * 
 *                    Q( s, a )
 *  p( s, a ) = ------------------
 *               SUM( Q( s, b ) )
 *                b
 * 
 * where Q(s, a) is action's a estimation (usefulness) at state s.
 * 
 * The exploration policy may be applied only in cases, when action estimates (usefulness)
 * are represented with positive value greater then 0.
 * 
 * @author Diego Catalano
 */
public class RouletteWheelExploration implements IExplorationPolicy{
    private Random r = new Random();

    /**
     * Initializes a new instance of the RouletteWheelExploration class.
     */
    public RouletteWheelExploration() {
    }
    
    @Override
    public int ChooseAction(double[] actionEstimates){
        // actions count
        int actionsCount = actionEstimates.length;
        // actions sum
        double sum = 0, estimateSum = 0;

        for ( int i = 0; i < actionsCount; i++ )
        {
            estimateSum += actionEstimates[i];
        }

        // get random number, which determines which action to choose
        double actionRandomNumber = r.nextDouble( );

        for ( int i = 0; i < actionsCount; i++ )
        {
            sum += actionEstimates[i] / estimateSum;
            if ( actionRandomNumber <= sum )
                return i;
        }

        return actionsCount - 1;
    }
}