package jsat.datatransform;
import java.util.Arrays;
import jsat.DataSet;
import jsat.classifiers.CategoricalData;
import jsat.classifiers.DataPoint;
import jsat.distributions.Distribution;
import jsat.distributions.LogUniform;
import jsat.distributions.discrete.UniformDiscrete;
import jsat.linear.DenseVector;
import jsat.linear.Vec;
/**
* This transform converts numerical features into categorical ones via a simple
* histogram. Bins will be created for each numeric feature of equal sizes. Each
* numeric feature will be converted to the same number of bins.
* This transform will handle missing values by simply ignoring them, and
* leaving the value missing in the transformed categorical variable.
*
*
* @author Edward Raff
*/
public class NumericalToHistogram implements DataTransform
{
private static final long serialVersionUID = -2318706869393636074L;
private int n;
//First index is the vector index, 2nd index is the min value then the increment value
double[][] conversionArray;
CategoricalData[] newDataArray;
/**
* Creates a new transform which will use at most 25 bins when converting
* numeric features. This may not be optimal for any given dataset
*
*/
public NumericalToHistogram()
{
this(25);
}
/**
* Creates a new transform which will use O(sqrt(n)) bins for each numeric
* feature, where n is the number of data points in the dataset.
*
* @param dataSet the data set to create the transform from
*/
public NumericalToHistogram(DataSet dataSet)
{
this(dataSet, (int) Math.ceil(Math.sqrt(dataSet.getSampleSize())));
}
/**
* Creates a new transform which will use at most the specified number of bins
*
* @param n the number of bins to create
*/
public NumericalToHistogram(int n)
{
setNumberOfBins(n);
}
/**
* Creates a new transform which will use the specified number of bins for
* each numeric feature.
* @param dataSet the data set to create the transform from
* @param n the number of bins to create
*/
public NumericalToHistogram(DataSet dataSet, int n)
{
this(n);
fit(dataSet);
}
@Override
public void fit(DataSet dataSet)
{
conversionArray = new double[dataSet.getNumNumericalVars()][2];
double[] mins = new double[conversionArray.length];
double[] maxs = new double[conversionArray.length];
for(int i = 0; i < mins.length; i++)
{
mins[i] = Double.MAX_VALUE;
maxs[i] = Double.MIN_VALUE;
}
for(int i = 0; i < dataSet.getSampleSize(); i++)
{
Vec v = dataSet.getDataPoint(i).getNumericalValues();
for(int j = 0; j < mins.length; j++)
{
final double val = v.get(j);
if(Double.isNaN(val))
continue;
mins[j] = Math.min(mins[j], val);
maxs[j] = Math.max(maxs[j], val);
}
}
for(int i = 0; i < conversionArray.length; i++)
{
conversionArray[i][0] = mins[i];
conversionArray[i][1] = (maxs[i]-mins[i])/n;
}
newDataArray = new CategoricalData[dataSet.getNumNumericalVars() + dataSet.getNumCategoricalVars()];
for(int i = 0; i < dataSet.getNumNumericalVars(); i++)
newDataArray[i] = new CategoricalData(n);
System.arraycopy(dataSet.getCategories(), 0, newDataArray, dataSet.getNumNumericalVars(), dataSet.getNumCategoricalVars());
}
/**
* Sets the maximum number of histogram bins to use when creating the categorical version of numeric features.
* @param n the number of bins to create
*/
public void setNumberOfBins(int n)
{
if(n <= 0)
throw new RuntimeException("Must partition into a positive number of groups");
this.n = n;
}
/**
*
* @return the maximum number of bins to create
*/
public int getNumberOfBins()
{
return n;
}
/**
* Attempts to guess the number of bins to use
* @param data the dataset to be transforms
* @return a distribution of the guess
*/
public static Distribution guessNumberOfBins(DataSet data)
{
if(data.getSampleSize() < 20)
return new UniformDiscrete(2, data.getSampleSize()-1);
else if(data.getSampleSize() >= 1000000)
return new LogUniform(50, 1000);
int sqrt = (int) Math.sqrt(data.getSampleSize());
return new UniformDiscrete(Math.max(sqrt/3, 2), Math.min(sqrt*3, data.getSampleSize()-1));
}
/**
* Copy constructor
* @param other the transform to copy
*/
private NumericalToHistogram(NumericalToHistogram other)
{
this.n = other.n;
if(other.conversionArray != null)
{
this.conversionArray = new double[other.conversionArray.length][];
for(int i = 0; i < other.conversionArray.length; i++)
this.conversionArray[i] = Arrays.copyOf(other.conversionArray[i], other.conversionArray[i].length);
}
if(other.newDataArray != null)
{
this.newDataArray = new CategoricalData[other.newDataArray.length];
for(int i = 0; i < other.newDataArray.length; i++)
this.newDataArray[i] = other.newDataArray[i].clone();
}
}
@Override
public DataPoint transform(DataPoint dp)
{
int[] newCatVals = new int[newDataArray.length];
Vec v = dp.getNumericalValues();
for(int i = 0; i < conversionArray.length; i++)
{
double val = v.get(i) - conversionArray[i][0];
if(Double.isNaN(val))
{
newCatVals[i] = -1;//missing
continue;
}
int catVal = (int) Math.floor(val / conversionArray[i][1]);
if(catVal < 0)
catVal = 0;
else if(catVal >= n)
catVal = n-1;
newCatVals[i] = catVal;
}
System.arraycopy(dp.getCategoricalValues(), 0, newCatVals, conversionArray.length, dp.numCategoricalValues());
return new DataPoint(new DenseVector(0), newCatVals, newDataArray);
}
@Override
public NumericalToHistogram clone()
{
return new NumericalToHistogram(this);
}
}
Ads help maintain this website.