package jsat.classifiers.boosting;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jsat.classifiers.*;
import jsat.classifiers.linear.LinearBatch;
import jsat.linear.DenseVector;
import jsat.linear.Vec;
import jsat.regression.RegressionDataSet;
import jsat.regression.Regressor;
/**
* This provides an implementation of the Stacking ensemble method. Stacking
* learns several base classifiers and a top level classifier learns to predict
* the target based on the outputs of all the ensambled models. Historically a
* linear model (such as {@link LinearBatch}) is used, which translates to
* learning a weighted vote of the classifier outputs. However any classifier
* may be used so long as it supports the desired target type.
*
* Note, that Stacking tends to work best when the base classifiers produce
* reasonable probability estimates.
* Stacking supports {@link #supportsWeightedData() weighted data instances} if
* the aggregating model does.
*
* See: Wolpert, D. H. (1992). Stacked generalization. Neural Networks, 5, 241–259.
*
* @author Edward Raff
*/
public class Stacking implements Classifier, Regressor
{
private static final long serialVersionUID = -6173323872903232074L;
private int folds;
/**
* The number of weights needed per model
*/
private int weightsPerModel;
private Classifier aggregatingClassifier;
private List baseClassifiers;
private Regressor aggregatingRegressor;
private List baseRegressors;
public static final int DEFAULT_FOLDS = 3;
/**
* Creates a new Stacking classifier
* @param folds the number of cross validation folds for learning the base model
* @param aggregatingClassifier the classifier used to merge the results of all the input classifiers
* @param baseClassifiers the list of base classifiers to ensemble
*/
public Stacking(int folds, Classifier aggregatingClassifier, List baseClassifiers)
{
if(baseClassifiers.size() < 2)
throw new IllegalArgumentException("base classifiers must contain at least 2 elements, not " + baseClassifiers.size());
setFolds(folds);
this.aggregatingClassifier = aggregatingClassifier;
this.baseClassifiers = baseClassifiers;
boolean allRegressors = aggregatingClassifier instanceof Regressor;
for(Classifier cl : baseClassifiers)
if(!(cl instanceof Regressor))
allRegressors = false;
if(allRegressors)
{
aggregatingRegressor = (Regressor) aggregatingClassifier;
baseRegressors = (List) baseClassifiers;//ugly type easure exploitation...
}
}
/**
* Creates a new Stacking classifier
* @param folds the number of cross validation folds for learning the base model
* @param aggregatingClassifier the classifier used to merge the results of all the input classifiers
* @param baseClassifiers the array of base classifiers to ensemble
*/
public Stacking(int folds, Classifier aggregatingClassifier, Classifier... baseClassifiers)
{
this(folds, aggregatingClassifier, Arrays.asList(baseClassifiers));
}
/**
* Creates a new Stacking classifier that uses {@value #DEFAULT_FOLDS} folds of cross validation
* @param aggregatingClassifier the classifier used to merge the results of all the input classifiers
* @param baseClassifiers the list of base classifiers to ensemble
*/
public Stacking(Classifier aggregatingClassifier, List baseClassifiers)
{
this(DEFAULT_FOLDS, aggregatingClassifier, baseClassifiers);
}
/**
* Creates a new Stacking classifier that uses {@value #DEFAULT_FOLDS} folds of cross validation
* @param aggregatingClassifier the classifier used to merge the results of all the input classifiers
* @param baseClassifiers the array of base classifiers to ensemble
*/
public Stacking(Classifier aggregatingClassifier, Classifier... baseClassifiers)
{
this(DEFAULT_FOLDS, aggregatingClassifier, baseClassifiers);
}
/**
* Creates a new Stacking regressor
* @param folds the number of cross validation folds for learning the base model
* @param aggregatingRegressor the regressor used to merge the results of all the input classifiers
* @param baseRegressors the list of base regressors to ensemble
*/
public Stacking(int folds, Regressor aggregatingRegressor, List baseRegressors)
{
setFolds(folds);
this.aggregatingRegressor = aggregatingRegressor;
this.baseRegressors = baseRegressors;
boolean allClassifiers = aggregatingRegressor instanceof Classifier;
for(Regressor reg : baseRegressors)
if(!(reg instanceof Classifier))
allClassifiers = false;
if(allClassifiers)
{
aggregatingClassifier = (Classifier) aggregatingRegressor;
baseClassifiers = (List) baseRegressors;//ugly type easure exploitation...
}
}
/**
* Creates a new Stacking regressor
* @param folds the number of cross validation folds for learning the base model
* @param aggregatingRegressor the regressor used to merge the results of all the input classifiers
* @param baseRegressors the array of base regressors to ensemble
*/
public Stacking(int folds, Regressor aggregatingRegressor, Regressor... baseRegressors)
{
this(folds, aggregatingRegressor, Arrays.asList(baseRegressors));
}
/**
* Creates a new Stacking regressor that uses {@value #DEFAULT_FOLDS} folds of cross validation
* @param aggregatingRegressor the regressor used to merge the results of all the input classifiers
* @param baseRegressors the list of base regressors to ensemble
*/
public Stacking(Regressor aggregatingRegressor, List baseRegressors)
{
this(DEFAULT_FOLDS, aggregatingRegressor, baseRegressors);
}
/**
* Creates a new Stacking regressor that uses {@value #DEFAULT_FOLDS} folds of cross validation
* @param aggregatingRegressor the regressor used to merge the results of all the input classifiers
* @param baseRegressors the array of base regressors to ensemble
*/
public Stacking(Regressor aggregatingRegressor, Regressor... baseRegressors)
{
this(DEFAULT_FOLDS, aggregatingRegressor, baseRegressors);
}
/**
* Copy constructor
* @param toCopy the object to copy
*/
public Stacking(Stacking toCopy)
{
this.folds = toCopy.folds;
this.weightsPerModel = toCopy.weightsPerModel;
if(toCopy.aggregatingClassifier != null)
{
this.aggregatingClassifier = toCopy.aggregatingClassifier.clone();
this.baseClassifiers = new ArrayList(toCopy.baseClassifiers.size());
for(Classifier bc : toCopy.baseClassifiers)
this.baseClassifiers.add(bc.clone());
if(toCopy.aggregatingRegressor == toCopy.aggregatingClassifier)//supports both
{
aggregatingRegressor = (Regressor) aggregatingClassifier;
baseRegressors = (List) baseClassifiers;//ugly type easure exploitation...
}
}
else//we are doing with regressors only
{
this.aggregatingRegressor = toCopy.aggregatingRegressor.clone();
this.baseRegressors = new ArrayList(toCopy.baseRegressors.size());
for(Regressor br : toCopy.baseRegressors)
this.baseRegressors.add(br.clone());
}
}
/**
* Sets the number of folds of cross validation to use when creating the new
* set of weights that will be feed into the aggregating model.
* Note that the number of folds may be 1, and will run significantly
* faster since models do not need to be re-trained. However it will be more
* prone to overfitting.
* @param folds the number of cross validation folds to use
*/
public void setFolds(int folds)
{
if(folds < 1)
throw new IllegalArgumentException("Folds must be a positive integer, not " + folds);
this.folds = folds;
}
/**
*
* @return the number of CV folds used for training
*/
public int getFolds()
{
return folds;
}
@Override
public CategoricalResults classify(DataPoint data)
{
Vec w = new DenseVector(weightsPerModel*baseClassifiers.size());
if(weightsPerModel == 1)
for(int i = 0; i < baseClassifiers.size(); i++)
w.set(i, baseClassifiers.get(i).classify(data).getProb(0)*2-1);
else
{
for(int i = 0; i < baseClassifiers.size(); i++)
{
CategoricalResults pred = baseClassifiers.get(i).classify(data);
for(int j = 0; j < weightsPerModel; j++)
w.set(i*weightsPerModel+j, pred.getProb(j));
}
}
return aggregatingClassifier.classify(new DataPoint(w));
}
@Override
public void train(ClassificationDataSet dataSet, boolean parallel)
{
final int models = baseClassifiers.size();
final int C = dataSet.getClassSize();
weightsPerModel = C == 2 ? 1 : C;
ClassificationDataSet metaSet = new ClassificationDataSet(weightsPerModel*models, new CategoricalData[0], dataSet.getPredicting());
List dataFolds = dataSet.cvSet(folds);
//iterate in the order of the folds so we get the right dataum weights
for(ClassificationDataSet cds : dataFolds)
for(int i = 0; i < cds.getSampleSize(); i++)
metaSet.addDataPoint(new DenseVector(weightsPerModel*models), cds.getDataPointCategory(i), cds.getDataPoint(i).getWeight());
//create the meta training set
for(int c = 0; c < baseClassifiers.size(); c++)
{
Classifier cl = baseClassifiers.get(c);
int pos = 0;
for(int f = 0; f < dataFolds.size(); f++)
{
ClassificationDataSet train = ClassificationDataSet.comineAllBut(dataFolds, f);
ClassificationDataSet test = dataFolds.get(f);
cl.train(train, parallel);
for(int i = 0; i < test.getSampleSize(); i++)//evaluate and mark each point in the held out fold.
{
CategoricalResults pred = cl.classify(test.getDataPoint(i));
if(C == 2)
metaSet.getDataPoint(pos).getNumericalValues().set(c, pred.getProb(0)*2-1);
else
{
Vec toSet = metaSet.getDataPoint(pos).getNumericalValues();
for(int j = weightsPerModel*c; j < weightsPerModel*(c+1); j++)
toSet.set(j, pred.getProb(j-weightsPerModel*c));
}
pos++;
}
}
}
//train the meta model
aggregatingClassifier.train(metaSet, parallel);
//train the final classifiers, unless folds=1. In that case they are already trained
if(folds != 1)
{
for(Classifier cl : baseClassifiers)
cl.train(dataSet, parallel);
}
}
@Override
public boolean supportsWeightedData()
{
if(aggregatingClassifier != null)
return aggregatingClassifier.supportsWeightedData();
else
return aggregatingRegressor.supportsWeightedData();
}
@Override
public double regress(DataPoint data)
{
Vec w = new DenseVector(baseRegressors.size());
for (int i = 0; i < baseRegressors.size(); i++)
w.set(i, baseRegressors.get(i).regress(data));
return aggregatingRegressor.regress(new DataPoint(w));
}
@Override
public void train(RegressionDataSet dataSet, boolean parallel)
{
final int models = baseRegressors.size();
weightsPerModel = 1;
RegressionDataSet metaSet = new RegressionDataSet(models, new CategoricalData[0]);
List dataFolds = dataSet.cvSet(folds);
//iterate in the order of the folds so we get the right dataum weights
for(RegressionDataSet rds : dataFolds)
for(int i = 0; i < rds.getSampleSize(); i++)
metaSet.addDataPoint(new DataPoint(new DenseVector(weightsPerModel*models), rds.getDataPoint(i).getWeight()), rds.getTargetValue(i));
//create the meta training set
for(int c = 0; c < baseRegressors.size(); c++)
{
Regressor reg = baseRegressors.get(c);
int pos = 0;
for(int f = 0; f < dataFolds.size(); f++)
{
RegressionDataSet train = RegressionDataSet.comineAllBut(dataFolds, f);
RegressionDataSet test = dataFolds.get(f);
reg.train(train, parallel);
for(int i = 0; i < test.getSampleSize(); i++)//evaluate and mark each point in the held out fold.
{
double pred = reg.regress(test.getDataPoint(i));
metaSet.getDataPoint(pos++).getNumericalValues().set(c, pred);
}
}
}
//train the meta model
aggregatingRegressor.train(metaSet, parallel);
//train the final classifiers, unless folds=1. In that case they are already trained
if(folds != 1)
{
for(Regressor reg : baseRegressors)
reg.train(dataSet, parallel);
}
}
@Override
public Stacking clone()
{
return new Stacking(this);
}
}
Ads help maintain this website.