package jsat.datatransform;

import java.util.Arrays;
import java.util.List;
import jsat.DataSet;
import jsat.classifiers.DataPoint;
import jsat.linear.IndexValue;
import jsat.linear.Vec;
import jsat.math.IndexFunction;
import jsat.math.OnLineStatistics;
import jsat.utils.DoubleList;

/**
 * This transform applies a shifted Box-Cox transform for several fixed values 
 * of λ, and selects the one that provides the greatest reduction in the 
 * skewness of the distribution. This is done in an attempt to make the 
 * individual features appear more normal. The shifted values are done to
 * preserve zeros and keep sparse inputs sparse. This is done with two passes
 * through the data set, but requires only O(D #λ values) memory. 
 * 

* The default values of λ are -1, -1/2, 0, 1/2, 1. When using * negative λ values all zeros are skipped and left as zeros. λ = * 1 is an implicit value that is always included regardless of the input, as it * is equivalent to leaving the data unchanged when preserving zero values. * The stated default values include the log(x+1) and sqrt(x) * transforms that are commonly used for deskewing as special cases. *

* Skewness can be calculated by including zero, but by default ignores them as * "not-present" values. * * @author Edward Raff */ public class AutoDeskewTransform implements InPlaceTransform { private static final long serialVersionUID = -4894242802345656448L; private double[] finalLambdas; private double[] mins; private final IndexFunction transform = new IndexFunction() { private static final long serialVersionUID = -404316813485246422L; @Override public double indexFunc(double value, int index) { if(index < 0) return 0.0; return transform(value, finalLambdas[index], mins[index]); } }; private static final DoubleList defaultList = new DoubleList(7); static { defaultList.add(-1.0); defaultList.add(-0.5); defaultList.add(0.0); defaultList.add(0.5); defaultList.add(1.0); } private List lambdas; private boolean ignorZeros; /** * Creates a new AutoDeskew transform * */ public AutoDeskewTransform() { this(true, defaultList); } /** * Creates a new AutoDeskew transform * * @param lambdas the list of lambda values to evaluate */ public AutoDeskewTransform(final double... lambdas) { this(true, DoubleList.view(lambdas, lambdas.length)); } /** * Creates a new AutoDeskew transform * * @param lambdas the list of lambda values to evaluate */ public AutoDeskewTransform(final List lambdas) { this(true, lambdas); } /** * Creates a new AutoDeskew transform * * @param ignorZeros {@code true} to ignore zero values when calculating the * skewness, {@code false} to include them. * @param lambdas the list of lambda values to evaluate */ public AutoDeskewTransform(boolean ignorZeros, final List lambdas) { this.ignorZeros = ignorZeros; this.lambdas = lambdas; } /** * Creates a new deskewing object from the given data set * * @param dataSet the data set to deskew */ public AutoDeskewTransform(DataSet dataSet) { this(dataSet, defaultList); } /** * Creates a new deskewing object from the given data set * * @param dataSet the data set to deskew * @param lambdas the list of lambda values to evaluate */ public AutoDeskewTransform(DataSet dataSet, final List lambdas) { this(dataSet, true, lambdas); } /** * Creates a new deskewing object from the given data set * * @param dataSet the data set to deskew * @param ignorZeros {@code true} to ignore zero values when calculating the * skewness, {@code false} to include them. * @param lambdas the list of lambda values to evaluate */ public AutoDeskewTransform(DataSet dataSet, boolean ignorZeros, final List lambdas) { this(ignorZeros, lambdas); fit(dataSet); } @Override public void fit(DataSet dataSet) { //going to try leaving things alone nomatter what if (!lambdas.contains(1.0)) lambdas.add(1.0); OnLineStatistics[][] stats = new OnLineStatistics[lambdas.size()][dataSet.getNumNumericalVars()]; for (int i = 0; i < stats.length; i++) for (int j = 0; j < stats[i].length; j++) stats[i][j] = new OnLineStatistics(); mins = new double[dataSet.getNumNumericalVars()]; Arrays.fill(mins, Double.POSITIVE_INFINITY); boolean containsSparseVecs = false; //First pass, get min/max values for (int i = 0; i < dataSet.getSampleSize(); i++) { Vec x = dataSet.getDataPoint(i).getNumericalValues(); if (x.isSparse()) containsSparseVecs = true; for (IndexValue iv : x) { final int indx = iv.getIndex(); final double val = iv.getValue(); mins[indx] = Math.min(val, mins[indx]); } } if (containsSparseVecs) for (int i = 0; i < mins.length; i++)//done b/c we only iterated the non-zeros mins[i] = Math.min(0, mins[i]); //Second pass, find the best skew transform for (int i = 0; i < dataSet.getSampleSize(); i++) { Vec x = dataSet.getDataPoint(i).getNumericalValues(); double weight = dataSet.getDataPoint(i).getWeight(); int lastIndx = -1; for (IndexValue iv : x) { int indx = iv.getIndex(); double val = iv.getValue(); updateStats(lambdas, stats, indx, val, mins, weight); if (!ignorZeros)//we have to do this here instead of bulk insert at the end b/c of different weight value combinations for (int prevIndx = lastIndx + 1; prevIndx < indx; prevIndx++) updateStats(lambdas, stats, prevIndx, 0.0, mins, weight); lastIndx = indx; } //Catch trailing zero values if (!ignorZeros)//we have to do this here instead of bulk insert at the end b/c of different weight value combinations for (int prevIndx = lastIndx + 1; prevIndx < mins.length; prevIndx++) updateStats(lambdas, stats, prevIndx, 0.0, mins, weight); } //Finish by figureing out which did best finalLambdas = new double[mins.length]; int lambdaOneIndex = lambdas.indexOf(1.0); for (int d = 0; d < finalLambdas.length; d++) { double minSkew = Double.POSITIVE_INFINITY; double bestLambda = 1;//done this way incase a NaN slips in, we will leave data unchanged for (int k = 0; k < lambdas.size(); k++) { double skew = Math.abs(stats[k][d].getSkewness()); if (skew < minSkew) { minSkew = skew; bestLambda = lambdas.get(k); } } double origSkew = Math.abs(stats[lambdaOneIndex][d].getSkewness()); if (origSkew > minSkew * 1.05)//only change if there is a reasonable improvment finalLambdas[d] = bestLambda; else finalLambdas[d] = 1.0; } } /** * Copy constructor * * @param toCopy the object to copy */ protected AutoDeskewTransform(AutoDeskewTransform toCopy) { this.finalLambdas = Arrays.copyOf(toCopy.finalLambdas, toCopy.finalLambdas.length); this.mins = Arrays.copyOf(toCopy.mins, toCopy.mins.length); } private static double transform(final double val, final double lambda, final double min) { if (val == 0) return 0; //special cases if (lambda == 2) { return val * val; } if (lambda == 1) { return val; } else if (lambda == 0.5) { return Math.sqrt(val - min); } else if (lambda == 0) { return Math.log(val + 1 - min);//log(1) = 0 } else if (lambda == -0.5) { return 1 / (Math.sqrt(val - min)); } else if (lambda == -1) { return 1 / val; } else if (lambda == -2) { return 1 / (val * val); } else { //commented out case handled at top //if(lambda < 0 && val == 0) // return 0;//should be Inf, but we want to preserve sparsity return Math.pow(val, lambda) / lambda; } } @Override public DataPoint transform(DataPoint dp) { DataPoint newDP = dp.clone(); mutableTransform(newDP); return newDP; } @Override public void mutableTransform(DataPoint dp) { dp.getNumericalValues().applyIndexFunction(transform); } @Override public AutoDeskewTransform clone() { return new AutoDeskewTransform(this); } /** * Updates the online stats for each value of lambda * * @param lambdas the list of lambda values * @param stats the array of statistics trackers * @param indx the feature index to add to * @param val the value at the given feature index * @param mins the minimum value array * @param weight the weight to the given update */ private void updateStats(final List lambdas, OnLineStatistics[][] stats, int indx, double val, double[] mins, double weight) { for (int k = 0; k < lambdas.size(); k++) stats[k][indx].add(transform(val, lambdas.get(k), mins[indx]), weight); } @Override public boolean mutatesNominal() { return false; } }

Ads help maintain this website.