/*
* Copyright (C) 2015 Edward Raff
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see .
*/
package jsat.parameters;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import jsat.DataSet;
import jsat.classifiers.ClassificationDataSet;
import jsat.classifiers.ClassificationModelEvaluation;
import jsat.classifiers.Classifier;
import jsat.distributions.Distribution;
import jsat.exceptions.FailedToFitException;
import jsat.regression.RegressionDataSet;
import jsat.regression.RegressionModelEvaluation;
import jsat.regression.Regressor;
import jsat.utils.FakeExecutor;
import jsat.utils.concurrent.ParallelUtils;
import jsat.utils.random.RandomUtil;
import jsat.utils.random.XORWOW;
/**
* Random Search is a simple method for tuning the parameters of a
* classification or regression algorithm. Each parameter is given a
* distribution that represents the values of interest, and trials are done by
* randomly sampling each parameter from their respective distributions.
* Compared to {@link GridSearch} this method does better when lots of values
* are to be tested or when 2 or more parameters are to be evaluated.
* The model it takes must implement the {@link Parameterized} interface. By
* default, no parameters are selected for optimizations. This is because
* parameters value ranges are often algorithm specific. As such, the user must
* specify the parameters and the values to test using the addParameter
* methods.
*
* See : Bergstra, J., & Bengio, Y. (2012). Random Search for Hyper-Parameter Optimization. Journal ofMachine Learning Research, 13, 281–305.
* @author Edward Raff
*/
public class RandomSearch extends ModelSearch
{
private int trials = 25;
/**
* The matching list of distributions we will test.
*/
private List searchValues;
/**
* Creates a new GridSearch to tune the specified parameters of a regression
* model. The parameters still need to be specified by calling
* {@link #addParameter(jsat.parameters.DoubleParameter, double[]) }
*
* @param baseRegressor the regressor to tune the parameters of
* @param folds the number of folds of cross-validation to perform to
* evaluate each combination of parameters
* @throws FailedToFitException if the base regressor does not implement
* {@link Parameterized}
*/
public RandomSearch(Regressor baseRegressor, int folds)
{
super(baseRegressor, folds);
searchValues = new ArrayList();
}
/**
* Creates a new GridSearch to tune the specified parameters of a
* classification model. The parameters still need to be specified by
* calling {@link #addParameter(jsat.parameters.DoubleParameter, double[]) }
*
* @param baseClassifier the classifier to tune the parameters of
* @param folds the number of folds of cross-validation to perform to
* evaluate each combination of parameters
* @throws FailedToFitException if the base classifier does not implement
* {@link Parameterized}
*/
public RandomSearch(Classifier baseClassifier, int folds)
{
super(baseClassifier, folds);
searchValues = new ArrayList();
}
/**
* Copy constructor
* @param toCopy the object to copy
*/
public RandomSearch(RandomSearch toCopy)
{
super(toCopy);
this.trials = toCopy.trials;
this.searchValues = new ArrayList(toCopy.searchValues.size());
for (Distribution d : toCopy.searchValues)
this.searchValues.add(d.clone());
}
/**
* This method will automatically populate the search space with parameters
* based on which Parameter objects return non-null distributions.
*
* Note, using this method with Cross Validation has the potential for
* over-estimating the accuracy of results if the data set is actually used
* to for parameter guessing.
*
* It is possible for this method to return 0, indicating that no default
* parameters could be found. The intended interpretation is that there are
* no parameters that you need to tune to get good performance from
* the given model. Though there will be cases where the author has simply
* missed a class.
*
*
* @param data the data set to get parameter estimates from
* @return the number of parameters added
*/
public int autoAddParameters(DataSet data)
{
Parameterized obj;
if (baseClassifier != null)
obj = (Parameterized) baseClassifier;
else
obj = (Parameterized) baseRegressor;
int totalParms = 0;
for (Parameter param : obj.getParameters())
{
Distribution dist;
if (param instanceof DoubleParameter)
{
dist = ((DoubleParameter) param).getGuess(data);
if (dist != null)
{
addParameter((DoubleParameter) param, dist);
totalParms++;
}
}
else if (param instanceof IntParameter)
{
dist = ((IntParameter) param).getGuess(data);
if (dist != null)
{
addParameter((IntParameter) param, dist);
totalParms++;
}
}
}
return totalParms;
}
/**
* Sets the number of trials or samples that will be taken. This value is the number of models that will be trained and evaluated for their performance
* @param trials the number of models to build and evaluate
*/
public void setTrials(int trials)
{
if(trials < 1)
throw new IllegalArgumentException("number of trials must be positive, not " + trials);
this.trials = trials;
}
/**
*
* @return the number of models that will be built to evaluate
*/
public int getTrials()
{
return trials;
}
/**
* Adds a new double parameter to be altered for the model being tuned.
*
* @param param the model parameter
* @param initialSearchValues the distribution to sample from for this parameter
*/
public void addParameter(DoubleParameter param, Distribution dist)
{
if (param == null)
throw new IllegalArgumentException("null not allowed for parameter");
searchParams.add(param);
searchValues.add(dist.clone());
}
/**
* Adds a new double parameter to be altered for the model being tuned.
*
* @param param the model parameter
* @param initialSearchValues the distribution to sample from for this parameter
*/
public void addParameter(IntParameter param, Distribution dist)
{
if (param == null)
throw new IllegalArgumentException("null not allowed for parameter");
searchParams.add(param);
searchValues.add(dist.clone());
}
/**
* Adds a new parameter to be altered for the model being tuned.
*
* @param name the name of the parameter
* @param initialSearchValues the values to try for the specified parameter
*/
public void addParameter(String name, Distribution dist)
{
Parameter param = getParameterByName(name);
if(param instanceof DoubleParameter)
addParameter((DoubleParameter) param, dist);
else if(param instanceof IntParameter)
addParameter((IntParameter) param, dist);
else
throw new IllegalArgumentException("Parameter " + name + " is not for double or int values");
}
@Override
public void train(final ClassificationDataSet dataSet, final boolean parallel)
{
final PriorityQueue bestModels
= new PriorityQueue<>(folds, (ClassificationModelEvaluation t, ClassificationModelEvaluation t1) ->
{
double v0 = t.getScoreStats(classificationTargetScore).getMean();
double v1 = t1.getScoreStats(classificationTargetScore).getMean();
int order = classificationTargetScore.lowerIsBetter() ? 1 : -1;
return order * Double.compare(v0, v1);
});
/**
* Each model is set to have different combination of parameters. We
* then train each model to determine the best one.
*/
final List paramsToEval = new ArrayList();
Random rand = RandomUtil.getRandom();
for(int trial = 0; trial < trials; trial++)
{
for(int i = 0; i < searchParams.size(); i++)
{
double sampledValue = searchValues.get(i).invCdf(rand.nextDouble());
Parameter param = searchParams.get(i);
if(param instanceof DoubleParameter)
((DoubleParameter)param).setValue(sampledValue);
else if(param instanceof IntParameter)
((IntParameter)param).setValue((int) Math.round(sampledValue));
}
paramsToEval.add(baseClassifier.clone());
}
//if we are doing our CV splits ahead of time, get them done now
final List preFolded;
/**
* Pre-combine our training combinations so that any caching can be
* re-used
*/
final List trainCombinations;
if (reuseSameCVFolds)
{
preFolded = dataSet.cvSet(folds);
trainCombinations = new ArrayList<>(preFolded.size());
for (int i = 0; i < preFolded.size(); i++)
trainCombinations.add(ClassificationDataSet.comineAllBut(preFolded, i));
}
else
{
preFolded = null;
trainCombinations = null;
}
ParallelUtils.run(parallel && trainModelsInParallel, paramsToEval.size(), (indx)->
{
Classifier c = paramsToEval.get(indx);
ClassificationModelEvaluation cme = new ClassificationModelEvaluation(c, dataSet, !trainModelsInParallel && parallel);
cme.addScorer(classificationTargetScore.clone());
if (reuseSameCVFolds)
cme.evaluateCrossValidation(preFolded, trainCombinations);
else
cme.evaluateCrossValidation(folds);
synchronized (bestModels)
{
bestModels.add(cme);
}
});
Classifier bestClassifier = bestModels.peek().getClassifier();//Just re-train it on the whole set
if (trainFinalModel)
bestClassifier.train(dataSet, parallel);
trainedClassifier = bestClassifier;
}
@Override
public void train(final RegressionDataSet dataSet, final boolean parallel)
{
final PriorityQueue bestModels
= new PriorityQueue<>(folds, (RegressionModelEvaluation t, RegressionModelEvaluation t1) ->
{
double v0 = t.getScoreStats(regressionTargetScore).getMean();
double v1 = t1.getScoreStats(regressionTargetScore).getMean();
int order = regressionTargetScore.lowerIsBetter() ? 1 : -1;
return order * Double.compare(v0, v1);
});
/**
* Each model is set to have different combination of parameters. We
* then train each model to determine the best one.
*/
final List paramsToEval = new ArrayList<>();
Random rand = RandomUtil.getRandom();
for(int trial = 0; trial < trials; trial++)
{
for(int i = 0; i < searchParams.size(); i++)
{
double sampledValue = searchValues.get(i).invCdf(rand.nextDouble());
Parameter param = searchParams.get(i);
if(param instanceof DoubleParameter)
((DoubleParameter)param).setValue(sampledValue);
else if(param instanceof IntParameter)
((IntParameter)param).setValue((int) Math.round(sampledValue));
}
paramsToEval.add(baseRegressor.clone());
}
//if we are doing our CV splits ahead of time, get them done now
final List preFolded;
/**
* Pre-combine our training combinations so that any caching can be
* re-used
*/
final List trainCombinations;
if (reuseSameCVFolds)
{
preFolded = dataSet.cvSet(folds);
trainCombinations = new ArrayList<>(preFolded.size());
for (int i = 0; i < preFolded.size(); i++)
trainCombinations.add(RegressionDataSet.comineAllBut(preFolded, i));
}
else
{
preFolded = null;
trainCombinations = null;
}
ParallelUtils.run(parallel && trainModelsInParallel, paramsToEval.size(), (indx)->
{
Regressor r = paramsToEval.get(indx);
RegressionModelEvaluation cme = new RegressionModelEvaluation(r, dataSet, !trainModelsInParallel && parallel);
cme.addScorer(regressionTargetScore.clone());
if (reuseSameCVFolds)
cme.evaluateCrossValidation(preFolded, trainCombinations);
else
cme.evaluateCrossValidation(folds);
synchronized (bestModels)
{
bestModels.add(cme);
}
});
Regressor bestRegressor = bestModels.peek().getRegressor();//Just re-train it on the whole set
if (trainFinalModel)
bestRegressor.train(dataSet, parallel);
trainedRegressor = bestRegressor;
}
@Override
public RandomSearch clone()
{
return new RandomSearch(this);
}
}
Ads help maintain this website.