package jsat.regression;
import static java.lang.Math.*;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.logging.Level;
import java.util.logging.Logger;
import jsat.classifiers.*;
import jsat.datatransform.DataTransformProcess;
import jsat.exceptions.UntrainedModelException;
import jsat.math.OnLineStatistics;
import jsat.regression.evaluation.RegressionScore;
import jsat.utils.SystemInfo;
import jsat.utils.concurrent.ParallelUtils;
import jsat.utils.random.RandomUtil;
/**
* Provides a mechanism to quickly evaluate a regression model on a data set.
* This can be done by cross validation or with a separate testing set.
*
* @author Edward Raff
*/
public class RegressionModelEvaluation
{
private Regressor regressor;
private RegressionDataSet dataSet;
/**
* The source of threads
*/
private boolean parallel;
private OnLineStatistics sqrdErrorStats;
private long totalTrainingTime = 0, totalClassificationTime = 0;
private DataTransformProcess dtp;
private Map scoreMap;
private boolean keepModels = false;
/**
* This holds models for each index that will be kept. If using a test set,
* only index 0 is used.
*/
private Regressor[] keptModels;
/**
* This holds models for each fold index that will be used for warm starts.
* If using a test set, only index 0 is used.
*/
private Regressor[] warmModels;
/**
* Creates a new RegressionModelEvaluation that will perform parallel training.
* @param regressor the regressor model to evaluate
* @param dataSet the data set to train or perform cross validation from
* @param parallel {@code true} if the training should be done using
* multiple-cores, {@code false} for single threaded.
*/
public RegressionModelEvaluation(Regressor regressor, RegressionDataSet dataSet, boolean parallel)
{
this.regressor = regressor;
this.dataSet = dataSet;
this.parallel = parallel;
this.dtp =new DataTransformProcess();
scoreMap = new LinkedHashMap<>();
}
/**
* Creates a new RegressionModelEvaluation that will perform serial training
* @param regressor the regressor model to evaluate
* @param dataSet the data set to train or perform cross validation from
*/
public RegressionModelEvaluation(Regressor regressor, RegressionDataSet dataSet)
{
this(regressor, dataSet, false);
}
/**
* Set this to {@code true} in order to keep the trained models after
* evaluation. They can then be retrieved used the {@link #getKeptModels() }
* methods. The default value is {@code false}.
*
* @param keepModels {@code true} to keep the trained models after
* evaluation, {@code false} to discard them.
*/
public void setKeepModels(boolean keepModels)
{
this.keepModels = keepModels;
}
/**
* This will keep the models trained when evaluating the model. The models
* can be obtained after an evaluation from {@link #getKeptModels() }.
*
* @return {@code true} if trained models will be kept after evaluation.
*/
public boolean isKeepModels()
{
return keepModels;
}
/**
* Returns the models that were kept after the last evaluation. {@code null}
* will be returned instead if {@link #isKeepModels() } returns
* {@code false}, which is the default.
*
* @return the models that were kept after the last evaluation. Or
* {@code null} if if models are not being kept.
*/
public Regressor[] getKeptModels()
{
return keptModels;
}
/**
* Sets the models that will be used for warm starting training. If using
* cross-validation, the number of models given should match the number of
* folds. If using a test set, only one model should be given.
*
* @param warmModels the models to use for warm start training
*/
public void setWarmModels(Regressor... warmModels)
{
this.warmModels = warmModels;
}
/**
* Sets the data transform process to use when performing cross validation.
* By default, no transforms are applied
* @param dtp the transformation process to clone for use during evaluation
*/
public void setDataTransformProcess(DataTransformProcess dtp)
{
this.dtp = dtp.clone();
}
/**
* Performs an evaluation of the regressor using the training data set.
* The evaluation is done by performing cross validation.
* @param folds the number of folds for cross validation
* @throws UntrainedModelException if the number of folds given is less than 2
*/
public void evaluateCrossValidation(int folds)
{
evaluateCrossValidation(folds, RandomUtil.getRandom());
}
/**
* Performs an evaluation of the regressor using the training data set.
* The evaluation is done by performing cross validation.
* @param folds the number of folds for cross validation
* @param rand the source of randomness for generating the cross validation sets
* @throws UntrainedModelException if the number of folds given is less than 2
*/
public void evaluateCrossValidation(int folds, Random rand)
{
if(folds < 2)
throw new UntrainedModelException("Model could not be evaluated because " + folds + " is < 2, and not valid for cross validation");
List lcds = dataSet.cvSet(folds, rand);
evaluateCrossValidation(lcds);
}
/**
* Performs an evaluation of the regressor using the training data set,
* where the folds of the training data set are provided by the user. The
* folds do not need to be the same sizes, though it is assumed that they
* are all approximately the same size. It is the caller's responsibility to
* ensure that the folds are only from the original training data set.
*
* This method exists so that the user can provide very specific folds if
* they so desire. This can be useful when there is known bias in the data
* set, such as when caused by duplicate data point values. The caller can
* then manually make sure duplicate values all occur in the same fold to
* avoid over-estimating the accuracy of the model.
*
* @param lcds the training data set already split into folds
*/
public void evaluateCrossValidation(List lcds)
{
List trainCombinations = new ArrayList(lcds.size());
for (int i = 0; i < lcds.size(); i++)
trainCombinations.add(RegressionDataSet.comineAllBut(lcds, i));
evaluateCrossValidation(lcds, trainCombinations);
}
/**
* Note: Most people should never need to call this method. Make sure you
* understand what you are doing before you do.
*
* Performs an evaluation of the regressor using the training data set,
* where the folds of the training data set, and their combinations, are
* provided by the user. The folds do not need to be the same sizes, though
* it is assumed that they are all approximately the same size - and the the
* training combination corresponding to each index will be the sum of the
* folds in the other indices. It is the caller's responsibility to ensure
* that the folds are only from the original training data set.
*
* This method exists so that the user can provide very specific folds if
* they so desire, and when the same folds will be used multiple times.
* Doing so allows the algorithms called to take advantage of any potential
* caching of results based on the data set and avoid all possible excessive
* memory movement. (For example, {@link jsat.DataSet#getNumericColumns() } may
* get re-used and benefit from its caching)
* The same behavior of this method can be obtained by calling {@link #evaluateCrossValidation(java.util.List)
* }.
*
* @param lcds training data set already split into folds
* @param trainCombinations each index contains the training data sans the
* data stored in the fold associated with that index
*/
public void evaluateCrossValidation(List lcds, List trainCombinations)
{
sqrdErrorStats = new OnLineStatistics();
totalTrainingTime = totalClassificationTime = 0;
for(int i = 0; i < lcds.size(); i++)
{
RegressionDataSet trainSet = trainCombinations.get(i);
RegressionDataSet testSet = lcds.get(i);
evaluationWork(trainSet, testSet, i);
}
}
/**
* Performs an evaluation of the regressor using the initial data set to
* train, and testing on the given data set.
* @param testSet the data set to perform testing on
*/
public void evaluateTestSet(RegressionDataSet testSet)
{
sqrdErrorStats = new OnLineStatistics();
totalTrainingTime = totalClassificationTime = 0;
evaluationWork(dataSet, testSet, 0);
}
private void evaluationWork(RegressionDataSet trainSet, RegressionDataSet testSet, int index)
{
trainSet = trainSet.shallowClone();
DataTransformProcess curProccess = dtp.clone();
curProccess.learnApplyTransforms(trainSet);
long startTrain = System.currentTimeMillis();
final Regressor regressorTouse = regressor.clone();
if(warmModels != null && regressorTouse instanceof WarmRegressor)//train from the warm model
{
WarmRegressor wr = (WarmRegressor) regressorTouse;
wr.train(trainSet, warmModels[index], parallel);
}
else//do the normal thing
{
regressorTouse.train(trainSet, parallel);
}
totalTrainingTime += (System.currentTimeMillis() - startTrain);
if(keptModels != null)
keptModels[index] = regressorTouse;
//place to store the scores that may get updated by several threads
final Map scoresToUpdate = new HashMap<>();
for(Entry entry : scoreMap.entrySet())
{
RegressionScore score = entry.getKey().clone();
score.prepare();
scoresToUpdate.put(score, score);
}
// CountDownLatch latch;
// if(testSet.getSampleSize() < SystemInfo.LogicalCores || !parallel)
// {
// latch = new CountDownLatch(1);
// new Evaluator(testSet, curProccess, 0, testSet.getSampleSize(), scoresToUpdate, regressorTouse, latch).run();
// }
// else//go parallel!
// {
// latch = new CountDownLatch(SystemInfo.LogicalCores);
// final int blockSize = testSet.getSampleSize()/SystemInfo.LogicalCores;
// int extra = testSet.getSampleSize()%SystemInfo.LogicalCores;
//
// int start = 0;
// while(start < testSet.getSampleSize())
// {
// int end = start+blockSize;
// if(extra-- > 0)
// end++;
// parallel.submit(new Evaluator(testSet, curProccess, start, end, scoresToUpdate, regressorTouse, latch));
// start = end;
// }
// }
ParallelUtils.run(parallel, testSet.getSampleSize(), (start, end)->
{
//create a local set of scores to update
long localPredictionTime = 0;
OnLineStatistics localSqrdErrors = new OnLineStatistics();
Set localScores = new HashSet<>();
for (Entry entry : scoresToUpdate.entrySet())
localScores.add(entry.getKey().clone());
for (int i = start; i < end; i++)
{
DataPoint di = testSet.getDataPoint(i);
double trueVal = testSet.getTargetValue(i);
DataPoint tranDP = curProccess.transform(di);
long startTime = System.currentTimeMillis();
double predVal = regressorTouse.regress(tranDP);
localPredictionTime += (System.currentTimeMillis() - startTime);
double sqrdError = pow(trueVal - predVal, 2);
for (RegressionScore score : localScores)
score.addResult(predVal, trueVal, di.getWeight());
localSqrdErrors.add(sqrdError, di.getWeight());
}
synchronized (sqrdErrorStats)
{
sqrdErrorStats.add(localSqrdErrors);
totalClassificationTime += localPredictionTime;
for (RegressionScore score : localScores)
scoresToUpdate.get(score).addResults(score);
}
});
//accumulate score info
for (Entry entry : scoreMap.entrySet())
{
RegressionScore score = entry.getKey().clone();
score.prepare();
score.addResults(scoresToUpdate.get(score));
entry.getValue().add(score.getScore());
}
}
/**
* Adds a new score object that will be used as part of the evaluation when
* calling {@link #evaluateCrossValidation(int, java.util.Random) } or
* {@link #evaluateTestSet(jsat.regression.RegressionDataSet) }. The
* statistics for the given score are reset on every call, and the mean /
* standard deviation comes from multiple folds in cross validation.
*
* The score statistics can be obtained from
* {@link #getScoreStats(jsat.regression.evaluation.RegressionScore) }
* after one of the evaluation methods have been called.
*
* @param scorer the score method to keep track of.
*/
public void addScorer(RegressionScore scorer)
{
scoreMap.put(scorer, new OnLineStatistics());
}
/**
* Gets the statistics associated with the given score. If the score is not
* currently in the model evaluation {@code null} will be returned. The
* object passed in does not need to be the exact same object passed to
* {@link #addScorer(jsat.regression.evaluation.RegressionScore) },
* it only needs to be equal to the object.
*
* @param score the score type to get the result statistics
* @return the result statistics for the given score, or {@code null} if the
* score is not in th evaluation set
*/
public OnLineStatistics getScoreStats(RegressionScore score)
{
return scoreMap.get(score);
}
/**
* Prints out the classification information in a convenient format. If no
* additional scores were added via the
* {@link #addScorer(RegressionScore) }
* method, nothing will be printed.
*/
public void prettyPrintRegressionScores()
{
int nameLength = 10;
for(Entry entry : scoreMap.entrySet())
nameLength = Math.max(nameLength, entry.getKey().getName().length()+2);
final String pfx = "%-" + nameLength;//prefix
for(Entry entry : scoreMap.entrySet())
System.out.printf(pfx+"s %-5f (%-5f)\n", entry.getKey().getName(), entry.getValue().getMean(), entry.getValue().getStandardDeviation());
}
/**
* Returns the minimum squared error from all runs.
* @return the minimum observed squared error
*/
public double getMinError()
{
return sqrdErrorStats.getMin();
}
/**
* Returns the maximum squared error observed from all runs.
* @return the maximum observed squared error
*/
public double getMaxError()
{
return sqrdErrorStats.getMax();
}
/**
* Returns the mean squared error from all runs.
* @return the overall mean squared error
*/
public double getMeanError()
{
return sqrdErrorStats.getMean();
}
/**
* Returns the standard deviation of the error from all runs
* @return the overall standard deviation of the errors
*/
public double getErrorStndDev()
{
return sqrdErrorStats.getStandardDeviation();
}
/***
* Returns the total number of milliseconds spent training the regressor.
* @return the total number of milliseconds spent training the regressor.
*/
public long getTotalTrainingTime()
{
return totalTrainingTime;
}
/**
* Returns the total number of milliseconds spent performing regression on the testing set.
* @return the total number of milliseconds spent performing regression on the testing set.
*/
public long getTotalClassificationTime()
{
return totalClassificationTime;
}
/**
* Returns the regressor that was to be evaluated
* @return the regressor original given
*/
public Regressor getRegressor()
{
return regressor;
}
}
Ads help maintain this website.