package jsat.classifiers;

import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.logging.Level;
import java.util.logging.Logger;
import jsat.DataSet;
import jsat.classifiers.evaluation.ClassificationScore;
import jsat.datatransform.DataTransformProcess;
import jsat.exceptions.UntrainedModelException;
import jsat.math.OnLineStatistics;
import jsat.utils.SystemInfo;
import jsat.utils.concurrent.ParallelUtils;
import jsat.utils.random.RandomUtil;

/**
 * Provides a mechanism to quickly perform an evaluation of a model on a data set. 
 * This can be done with cross validation or with a testing set. 
 * 
 * @author Edward Raff
 */
public class ClassificationModelEvaluation
{
    /**
     * The model to evaluate
     */
    private Classifier classifier;
    /**
     * The data set to train with. 
     */
    private ClassificationDataSet dataSet;
    /**
     * The source of threads
     */
    private boolean parallel;
    private double[][] confusionMatrix;
    /**
     * The sum of all the weights for each data point that was used in testing. 
     */
    private double sumOfWeights;
    private long totalTrainingTime = 0, totalClassificationTime = 0;
    private DataTransformProcess dtp;
    private boolean keepPredictions;
    private CategoricalResults[] predictions;
    private int[] truths;
    private double[] pointWeights;
    private OnLineStatistics errorStats;
    private Map scoreMap;
    private boolean keepModels = false;
    /**
     * This holds models for each index that will be kept. If using a test set,
     * only index 0 is used.
     */
    private Classifier[] keptModels;
    /**
     * This holds models for each fold index that will be used for warm starts.
     * If using a test set, only index 0 is used.
     */
    private Classifier[] warmModels;
    
    /**
     * Constructs a new object that can perform evaluations on the model. 
     * The model will not be trained until evaluation time. 
     * 
     * @param classifier the model to train and evaluate
     * @param dataSet the training data set. 
     */
    public ClassificationModelEvaluation(Classifier classifier, ClassificationDataSet dataSet)
    {
        this(classifier, dataSet, false);
    }
    
    /**
     * Constructs a new object that can perform evaluations on the model. 
     * The model will not be trained until evaluation time. 
     * 
     * @param classifier the model to train and evaluate
     * @param dataSet the training data set. 
     * @param parallel {@code true} if the training should be done using
     * multiple-cores, {@code false} for single threaded.
     */
    public ClassificationModelEvaluation(Classifier classifier, ClassificationDataSet dataSet, boolean parallel)
    {
        this.classifier = classifier;
        this.dataSet = dataSet;
        this.parallel = parallel;
        this.dtp = new DataTransformProcess();
        keepPredictions = false;
        errorStats = new OnLineStatistics();
        scoreMap = new LinkedHashMap<>();
    }

    /**
     * Set this to {@code true} in order to keep the trained models after
     * evaluation. They can then be retrieved used the {@link #getKeptModels() }
     * methods. The default value is {@code false}.
     *
     * @param keepModels {@code true} to keep the trained models after
     * evaluation, {@code false} to discard them.
     */
    public void setKeepModels(boolean keepModels)
    {
        this.keepModels = keepModels;
    }

    /**
     * This will keep the models trained when evaluating the model. The models
     * can be obtained after an evaluation from {@link #getKeptModels() }.
     *
     * @return {@code true} if trained models will be kept after evaluation.
     */
    public boolean isKeepModels()
    {
        return keepModels;
    }

    /**
     * Returns the models that were kept after the last evaluation. {@code null}
     * will be returned instead if {@link #isKeepModels() } returns
     * {@code false}, which is the default.
     *
     * @return the models that were kept after the last evaluation. Or
     * {@code null} if if models are not being kept.
     */
    public Classifier[] getKeptModels()
    {
        return keptModels;
    }

    /**
     * Sets the models that will be used for warm starting training. If using
     * cross-validation, the number of models given should match the number of
     * folds. If using a test set, only one model should be given.
     *
     * @param warmModels the models to use for warm start training
     */
    public void setWarmModels(Classifier... warmModels)
    {
        this.warmModels = warmModels;
    }

    /**
     * Sets the data transform process to use when performing cross validation. 
     * By default, no transforms are applied
     * @param dtp the transformation process to clone for use during evaluation
     */
    public void setDataTransformProcess(DataTransformProcess dtp)
    {
        this.dtp = dtp.clone();
    }
    
    /**
     * Performs an evaluation of the classifier using the training data set. 
     * The evaluation is done by performing cross validation.
     * @param folds the number of folds for cross validation
     * @throws UntrainedModelException if the number of folds given is less than 2
     */
    public void evaluateCrossValidation(int folds)
    {
        evaluateCrossValidation(folds, RandomUtil.getRandom());
    }
    
    /**
     * Performs an evaluation of the classifier using the training data set. 
     * The evaluation is done by performing cross validation.
     * @param folds the number of folds for cross validation
     * @param rand the source of randomness for generating the cross validation sets
     * @throws UntrainedModelException if the number of folds given is less than 2
     */
    public void evaluateCrossValidation(int folds, Random rand)
    {
        if(folds < 2)
            throw new UntrainedModelException("Model could not be evaluated because " + folds + " is < 2, and not valid for cross validation");
        List lcds = dataSet.cvSet(folds, rand);
        evaluateCrossValidation(lcds);
    }
    
    /**
     * Performs an evaluation of the classifier using the training data set,
     * where the folds of the training data set are provided by the user. The
     * folds do not need to be the same sizes, though it is assumed that they
     * are all approximately the same size. It is the caller's responsibility to
     * ensure that the folds are only from the original training data set. 
*
* This method exists so that the user can provide very specific folds if * they so desire. This can be useful when there is known bias in the data * set, such as when caused by duplicate data point values. The caller can * then manually make sure duplicate values all occur in the same fold to * avoid over-estimating the accuracy of the model. * * @param lcds the training data set already split into folds */ public void evaluateCrossValidation(List lcds) { List trainCombinations = new ArrayList(lcds.size()); for (int i = 0; i < lcds.size(); i++) trainCombinations.add(ClassificationDataSet.comineAllBut(lcds, i)); evaluateCrossValidation(lcds, trainCombinations); } /** * Note: Most people should never need to call this method. Make sure you * understand what you are doing before you do.
*
* Performs an evaluation of the classifier using the training data set, * where the folds of the training data set, and their combinations, are * provided by the user. The folds do not need to be the same sizes, though * it is assumed that they are all approximately the same size - and the the * training combination corresponding to each index will be the sum of the * folds in the other indices. It is the caller's responsibility to ensure * that the folds are only from the original training data set.
*
* This method exists so that the user can provide very specific folds if * they so desire, and when the same folds will be used multiple times. * Doing so allows the algorithms called to take advantage of any potential * caching of results based on the data set and avoid all possible excessive * memory movement. (For example, {@link DataSet#getNumericColumns() } may * get re-used and benefit from its caching)
* The same behavior of this method can be obtained by calling {@link #evaluateCrossValidation(java.util.List) * }. * * @param lcds training data set already split into folds * @param trainCombinations each index contains the training data sans the * data stored in the fold associated with that index */ public void evaluateCrossValidation(List lcds, List trainCombinations) { int numOfClasses = dataSet.getClassSize(); sumOfWeights = 0.0; confusionMatrix = new double[numOfClasses][numOfClasses]; totalTrainingTime = 0; totalClassificationTime = 0; if(keepModels) keptModels = new Classifier[lcds.size()]; setUpResults(dataSet.getSampleSize()); int end = dataSet.getSampleSize(); for (int i = lcds.size() - 1; i >= 0; i--) { ClassificationDataSet trainSet = trainCombinations.get(i); ClassificationDataSet testSet = lcds.get(i); evaluationWork(trainSet, testSet, i); int testSize = testSet.getSampleSize(); if (keepPredictions) { System.arraycopy(predictions, 0, predictions, end - testSize, testSize); System.arraycopy(truths, 0, truths, end-testSize, testSize); System.arraycopy(pointWeights, 0, pointWeights, end-testSize, testSize); } end -= testSize; } } /** * Performs an evaluation of the classifier using the initial data set to train, and testing on the given data set. * @param testSet the data set to perform testing on */ public void evaluateTestSet(ClassificationDataSet testSet) { if(keepModels) keptModels = new Classifier[1]; int numOfClasses = dataSet.getClassSize(); sumOfWeights = 0.0; confusionMatrix = new double[numOfClasses][numOfClasses]; setUpResults(testSet.getSampleSize()); totalTrainingTime = totalClassificationTime = 0; evaluationWork(dataSet, testSet, 0); } private void evaluationWork(ClassificationDataSet trainSet, ClassificationDataSet testSet, int index) { DataTransformProcess curProcess = dtp.clone(); if (curProcess.getNumberOfTransforms() > 0) { trainSet = trainSet.shallowClone(); curProcess.learnApplyTransforms(trainSet); } final Classifier classifierToUse = classifier.clone(); long startTrain = System.currentTimeMillis(); if(warmModels != null && classifierToUse instanceof WarmClassifier)//train from the warm model { WarmClassifier wc = (WarmClassifier) classifierToUse; wc.train(trainSet, warmModels[index], parallel); } else//do the normal thing { classifierToUse.train(trainSet, parallel); } totalTrainingTime += (System.currentTimeMillis() - startTrain); if(keptModels != null) keptModels[index] = classifierToUse; CountDownLatch latch; final double[] evalErrorStats = new double[2];//first index is correct, 2nd is total //place to store the scores that may get updated by several threads final Map scoresToUpdate = new HashMap<>(); for(Entry entry : scoreMap.entrySet()) { ClassificationScore score = entry.getKey().clone(); score.prepare(dataSet.getPredicting()); scoresToUpdate.put(score, score); } ParallelUtils.run(parallel, testSet.getSampleSize(), (start, end) -> { //create a local set of scores to update double localCorrect = 0; double localSumOfWeights = 0; long localClassificationTime = 0; Set localScores = new HashSet<>(); for (Entry entry : scoresToUpdate.entrySet()) localScores.add(entry.getKey().clone()); for (int i = start; i < end; i++) { DataPoint dp = testSet.getDataPoint(i); dp = curProcess.transform(dp); long stratClass = System.currentTimeMillis(); CategoricalResults result = classifierToUse.classify(dp); localClassificationTime += (System.currentTimeMillis() - stratClass); for (ClassificationScore score : localScores) score.addResult(result, testSet.getDataPointCategory(i), dp.getWeight()); if (predictions != null) { predictions[i] = result; truths[i] = testSet.getDataPointCategory(i); pointWeights[i] = dp.getWeight(); } final int trueCat = testSet.getDataPointCategory(i); synchronized (confusionMatrix[trueCat]) { confusionMatrix[trueCat][result.mostLikely()] += dp.getWeight(); } if (trueCat == result.mostLikely()) localCorrect += dp.getWeight(); localSumOfWeights += dp.getWeight(); } synchronized (confusionMatrix) { totalClassificationTime += localClassificationTime; sumOfWeights += localSumOfWeights; evalErrorStats[0] += localSumOfWeights - localCorrect; evalErrorStats[1] += localSumOfWeights; for (ClassificationScore score : localScores) scoresToUpdate.get(score).addResults(score); } }); errorStats.add(evalErrorStats[0] / evalErrorStats[1]); //accumulate score info for (Entry entry : scoreMap.entrySet()) { ClassificationScore score = entry.getKey().clone(); score.prepare(dataSet.getPredicting()); score.addResults(scoresToUpdate.get(score)); entry.getValue().add(score.getScore()); } } /** * Adds a new score object that will be used as part of the evaluation when * calling {@link #evaluateCrossValidation(int, java.util.Random) } or * {@link #evaluateTestSet(jsat.classifiers.ClassificationDataSet) }. The * statistics for the given score are reset on every call, and the mean / * standard deviation comes from multiple folds in cross validation.
*
* The score statistics can be obtained from * {@link #getScoreStats(ClassificationScore) } * after one of the evaluation methods have been called. * * @param scorer the score method to keep track of. */ public void addScorer(ClassificationScore scorer) { scoreMap.put(scorer, new OnLineStatistics()); } /** * Gets the statistics associated with the given score. If the score is not * currently in the model evaluation {@code null} will be returned. The * object passed in does not need to be the exact same object passed to * {@link #addScorer(ClassificationScore) }, * it only needs to be equal to the object. * * @param score the score type to get the result statistics * @return the result statistics for the given score, or {@code null} if the * score is not in th evaluation set */ public OnLineStatistics getScoreStats(ClassificationScore score) { return scoreMap.get(score); } /** * Indicates whether or not the predictions made during evaluation should be * stored with the expected value for retrieval later. * @param keepPredictions true if space should be allocated to * store the predictions made */ public void keepPredictions(boolean keepPredictions) { this.keepPredictions = keepPredictions; } /** * * @return true if the predictions are being stored */ public boolean doseStoreResults() { return keepPredictions; } /** * If {@link #keepPredictions(boolean) } was set, this method will return * the array storing the predictions made by the classifier during * evaluation. These results may not be in the same order as the data set * they came from, but the order is paired with {@link #getTruths() } * * @return the array of predictions, or null */ public CategoricalResults[] getPredictions() { return predictions; } /** * If {@link #keepPredictions(boolean) } was set, this method will return * the array storing the target classes that should have been predicted * during evaluation. These results may not be in the same order as the data * set they came from, but the order is paired with {@link #getPredictions()} * * @return the array of target class values, or null */ public int[] getTruths() { return truths; } /** * If {@link #keepPredictions(boolean) } was set, this method will return * the array storing the weights for each of the points that were classified * * @return the array of data point weights, or null */ public double[] getPointWeights() { return pointWeights; } public double[][] getConfusionMatrix() { return confusionMatrix; } /** * Assuming that we are on the start of a new line, the confusion matrix will be pretty printed to {@link System#out System.out} */ public void prettyPrintConfusionMatrix() { CategoricalData predicting = dataSet.getPredicting(); int classCount = predicting.getNumOfCategories(); int nameLength = 10; for(int i = 0; i < classCount; i++) nameLength = Math.max(nameLength, predicting.getOptionName(i).length()+2); final String pfx = "%-" + nameLength;//prefix System.out.printf(pfx+"s ", "Matrix"); for(int i = 0; i < classCount-1; i++) System.out.printf(pfx+"s ", predicting.getOptionName(i).toUpperCase()); System.out.printf(pfx+"s\n", predicting.getOptionName(classCount-1).toUpperCase()); //Now the rows that have data! for(int i = 0; i entry : scoreMap.entrySet()) nameLength = Math.max(nameLength, entry.getKey().getName().length()+2); final String pfx = "%-" + nameLength;//prefix for(Entry entry : scoreMap.entrySet()) { OnLineStatistics stats = entry.getValue(); if(stats.getMax() == stats.getMin())//max = min = no varaince / 1 entry System.out.printf(pfx+"s %-5f\n", entry.getKey().getName(), stats.getMean()); else System.out.printf(pfx+"s %-5f (%-5f)\n", entry.getKey().getName(), stats.getMean(), stats.getStandardDeviation()); } } /** * Returns the total value of the weights for data points that were classified correctly. * @return the total value of the weights for data points that were classified correctly. */ public double getCorrectWeights() { double val = 0.0; for(int i = 0; i < confusionMatrix.length; i++) val += confusionMatrix[i][i]; return val; } /** * Returns the total value of the weights for all data points that were tested against * @return the total value of the weights for all data points that were tested against */ public double getSumOfWeights() { return sumOfWeights; } /** * Computes the weighted error rate of the classifier. If all weights of the data * points tested were equal, then the value returned is also the percent of data * points that the classifier erred on. * @return the weighted error rate of the classifier. */ public double getErrorRate() { return 1.0 - getCorrectWeights()/sumOfWeights; } /** * Returns the object that keeps track of the error on * individual evaluations. If cross-validation was used, * it is the statistics for the errors of each fold. If * not, it is for each time {@link #evaluateTestSet(jsat.classifiers.ClassificationDataSet) } was called. * @return the statistics for the error of all evaluation sets */ public OnLineStatistics getErrorRateStats() { return errorStats; } /*** * Returns the total number of milliseconds spent training the classifier. * @return the total number of milliseconds spent training the classifier. */ public long getTotalTrainingTime() { return totalTrainingTime; } /** * Returns the total number of milliseconds spent performing classification on the testing set. * @return the total number of milliseconds spent performing classification on the testing set. */ public long getTotalClassificationTime() { return totalClassificationTime; } /** * Returns the classifier that was original given for evaluation. * @return the classifier that was original given for evaluation. */ public Classifier getClassifier() { return classifier; } private void setUpResults(int resultSize) { if(keepPredictions) { predictions = new CategoricalResults[resultSize]; truths = new int[predictions.length]; pointWeights = new double[predictions.length]; } else { predictions = null; truths = null; pointWeights = null; } } }

Ads help maintain this website.