package jsat.classifiers.calibration;

import java.util.Collections;
import java.util.List;
import jsat.classifiers.ClassificationDataSet;
import jsat.classifiers.Classifier;
import jsat.classifiers.DataPoint;
import jsat.classifiers.DataPointPair;
import jsat.parameters.Parameter.ParameterHolder;
import jsat.parameters.Parameterized;

/**
 * This abstract class provides the frame work for an algorithm to perform 
 * probability calibration based on the outputs of a base learning algorithm for 
 * binary classification problems. 
 * 

* Calibration can be performed directly on output values, though it may cause * over-fitting. For this reason, the {@link CalibrationMode} may be set to an * alternative method. *

* The parameters include the calibration parameters, and any parameters that * would be returned by the base model. * * @author Edward Raff */ public abstract class BinaryCalibration implements Classifier, Parameterized { private static final long serialVersionUID = 2356311701854978890L; /** * The base classifier to train and calibrate the outputs of */ @ParameterHolder protected BinaryScoreClassifier base; /** * The number of CV folds */ protected int folds = 3; /** * The proportion of the data set to hold out for calibration */ protected double holdOut = 0.3; /** * The calibration mode to use */ protected CalibrationMode mode; /** * Creates a new Binary Calibration object * @param base the base learning algorithm * @param mode the calibration mode to use */ public BinaryCalibration(BinaryScoreClassifier base, CalibrationMode mode) { this.base = base; setCalibrationMode(mode); } /** * Controls how the scores are obtained for producing a "training set" to * calibrate the output of the underlying model. */ public static enum CalibrationMode { /** * The naive methods trains the classifier on the whole data set, and * then produces the scores for each training point. This may cause * over fitting. */ NAIVE, /** * The model will be trained by cross validation, using the specified * number of {@link #setCalibrationFolds(int) }. The default is 3 folds, * where the classifier will be trained on the folds not in the set, and * then produce scores for the unobserved test points in the held out * fold. */ CV, /** * The model will have a random {@link #setCalibrationHoldOut(double) * fraction} of the data set held out, and trained on the rest of the * data. The scores will then be produced for the held out data and used * for calibration. */ HOLD_OUT, } @Override public void train(ClassificationDataSet dataSet, boolean parallel) { double[] deci = new double[dataSet.getSampleSize()];//array of SVM decision values boolean[] label = new boolean[deci.length];//array of booleans: is the example labeled +1? int len = label.length; if (mode == CalibrationMode.CV) { List foldList = dataSet.cvSet(folds); int pos = 0; for(int i = 0; i < foldList.size(); i++) { ClassificationDataSet test = foldList.get(i); ClassificationDataSet train = ClassificationDataSet.comineAllBut(foldList, i); base.train(train, parallel); for(int j = 0; j < test.getSampleSize(); j++) { deci[pos] = base.getScore(test.getDataPoint(j)); label[pos] = test.getDataPointCategory(j) == 1; pos++; } } base.train(dataSet, parallel); } else if (mode == CalibrationMode.HOLD_OUT) { List> wholeSet = dataSet.getAsDPPList(); Collections.shuffle(wholeSet); int splitMark = (int) (wholeSet.size()*(1-holdOut)); ClassificationDataSet train = new ClassificationDataSet(wholeSet.subList(0, splitMark), dataSet.getPredicting()); ClassificationDataSet test = new ClassificationDataSet(wholeSet.subList(splitMark, wholeSet.size()), dataSet.getPredicting()); base.train(train, parallel); for(int i = 0; i < test.getSampleSize(); i++) { deci[i] = base.getScore(test.getDataPoint(i)); label[i] = test.getDataPointCategory(i) == 1; } len = test.getSampleSize(); base.train(dataSet, parallel); } else { base.train(dataSet, parallel); for (int i = 0; i < len; i++) { DataPoint dp = dataSet.getDataPoint(i); deci[i] = base.getScore(dp); label[i] = dataSet.getDataPointCategory(i) == 1; } } calibrate(label, deci, len); } /** * This method perform the model calibration on the outputs verse the class * labels. * @param label the set of labels, where {@code true} indicates the positive * class label, and {@code false} indicates the negative class label. * @param scores the score associated with each label from the learning * algorithm. * @param len the number of values (from zero) of the label and scores array * to use. This value may be less than the actual array size */ abstract protected void calibrate(boolean[] label, double[] scores, final int len); /** * If the calibration mode is set to {@link CalibrationMode#CV}, this * controls how many folds of cross validation will be used. The default is * 3. * @param folds the number of cross validation folds to perform */ public void setCalibrationFolds(int folds) { if(folds < 1) throw new IllegalArgumentException("Folds must be a positive value, not " + folds); this.folds = folds; } /** * Returns the number of cross validation folds to use * @return the number of cross validation folds to use */ public int getCalibrationFolds() { return folds; } /** * If the calibration mode is set to {@link CalibrationMode#HOLD_OUT}, this * what portion of the data set is randomly selected to be the hold out set. * The default is 0.3. * * @param holdOut the portion in (0, 1) to hold out */ public void setCalibrationHoldOut(double holdOut) { if(Double.isNaN(holdOut) || holdOut <= 0 || holdOut >= 1) throw new IllegalArgumentException("HoldOut must be in (0, 1), not " + holdOut); this.holdOut = holdOut; } /** * Returns the portion of the data set that will be held out for calibration * @return the portion of the data set that will be held out for calibration */ public double getCalibrationHoldOut() { return holdOut; } /** * Sets which calibration mode will be used during training * @param mode the calibration mode to use during training. */ public void setCalibrationMode(CalibrationMode mode) { this.mode = mode; } /** * Returns the calibration mode used during training * @return the calibration mode used during training */ public CalibrationMode getCalibrationMode() { return mode; } @Override abstract public BinaryCalibration clone(); }

Ads help maintain this website.