package jsat.classifiers.linear;

import java.util.Iterator;
import java.util.List;
import jsat.SingleWeightVectorModel;
import jsat.classifiers.CategoricalResults;
import jsat.classifiers.Classifier;
import jsat.exceptions.FailedToFitException;
import jsat.linear.IndexValue;
import jsat.linear.Vec;
import jsat.lossfunctions.LogisticLoss;
import jsat.lossfunctions.SquaredLoss;
import jsat.parameters.Parameter;
import jsat.parameters.Parameterized;
import jsat.regression.Regressor;

/**
 * This base class provides shared functionality and variables used by two 
 * different training algorithms for L1 regularized linear models. 
 * Both squared and log loss are supported, making the results equivalent to 
 * LASSO regression and Logistic regression respectively. 
*
* These algorithms requires all feature values to be in the range * [-1, 1]. The implementation can do implicit rescaling, but rescaling may * destroy sparsity. If the data set is sparse and all values are zero or * positive use the default [0,1] rescaling to perform efficient rescaling that * will not destroy sparsity.
*
* Both algorithms are from:
* Shalev-Shwartz, * S.,&Tewari, A. (2009). Stochastic Methods for L1-regularized * Loss Minimization. 26th International Conference on Machine Learning * (Vol. 12, pp. 929–936). * * @author Edward Raff */ public abstract class StochasticSTLinearL1 implements Classifier, Regressor, Parameterized, SingleWeightVectorModel { private static final long serialVersionUID = -6761456665014802608L; /** * The number of training iterations */ protected int epochs; /** * The regularization penalty */ protected double lambda; /** * The loss function to use */ protected Loss loss; /** * The final weight vector */ protected Vec w; /** * The bias term to add */ protected double bias; /** * The minimum observed value for each feature */ protected double[] obvMin; /** * The maximum observed value for each feature */ protected double[] obvMax; /** * Whether or not to perform feature rescaling */ protected boolean reScale; /** * The scaled minimum */ protected double minScaled = 0; /** * The scaled maximum */ protected double maxScaled = 1; public static final int DEFAULT_EPOCHS = 1000; public static final double DEFAULT_REG = 1e-14; public static final Loss DEFAULT_LOSS = Loss.SQUARED; @Override abstract public StochasticSTLinearL1 clone(); public static enum Loss { SQUARED { @Override public double loss(double a, double y) { return SquaredLoss.loss(a, y); } @Override public double deriv(double a, double y) { return SquaredLoss.deriv(a, y); } @Override public double beta() { return 1; } @Override public CategoricalResults classify(double a) { CategoricalResults cr = new CategoricalResults(2); a = (a+1)/2; if(a > 1) a = 1; else if(a < 0) a = 0; cr.setProb(1, a); cr.setProb(0, 1-a); return cr; } @Override public double regress(double a) { return a; } }, LOG { @Override public double loss(double a, double y) { return LogisticLoss.loss(a, y); } @Override public double deriv(double a, double y) { return LogisticLoss.deriv(a, y); } @Override public double beta() { return 1.0/4.0; } @Override public CategoricalResults classify(double a) { return LogisticLoss.classify(a); } @Override public double regress(double a) { return 1/(1+Math.exp(-a)); } }; /** * Returns the loss on the prediction * @param a the predicted value * @param y the target value * @return the loss */ abstract public double loss(double a, double y); /** * Returns the value of the derivative of the loss function * @param a the predicted value * @param y the target value * @return the derivative of the loss */ abstract public double deriv(double a, double y); /** * Returns an upper bound on the 2nd derivative for classification * @return an upper bound on the 2nd derivative for classification */ abstract public double beta(); /** * The categorical results for a classification problem * @param a the dot product of the weight vector and an input * @return the binary problem classification results */ abstract public CategoricalResults classify(double a); /** * The output value result for a regression problem * @param a the dot product of the weight vector and an input * @return the final regression output */ abstract public double regress(double a); } /** * Sets the number of iterations of training that will be performed. * @param epochs the number of iterations */ public void setEpochs(int epochs) { if(epochs < 1) throw new ArithmeticException("A positive amount of iterations must be performed"); this.epochs = epochs; } /** * Returns the number of iterations of updating that will be done * @return the number of iterations */ public double getEpochs() { return epochs; } /** * Sets the maximum value of any feature after scaling is applied. This * value can be no greater than 1. * @param maxFeature the maximum feature value after scaling */ public void setMaxScaled(double maxFeature) { if(Double.isNaN(maxFeature)) throw new ArithmeticException("NaN is not a valid feature value"); else if(maxFeature > 1) throw new ArithmeticException("Maximum possible feature value is 1, can not use " + maxFeature); else if(maxFeature <= minScaled) throw new ArithmeticException("Maximum feature value must be learger than the minimum"); this.maxScaled = maxFeature; } /** * Returns the maximum feature value after scaling * @return the maximum feature value after scaling */ public double getMaxScaled() { return maxScaled; } /** * Sets the minimum value of any feature after scaling is applied. This * value can be no smaller than -1 * @param minFeature the minimum feature value after scaling */ public void setMinScaled(double minFeature) { if(Double.isNaN(minFeature)) throw new ArithmeticException("NaN is not a valid feature value"); else if(minFeature < -1) throw new ArithmeticException("Minimum possible feature value is -1, can not use " + minFeature); else if(minFeature >= maxScaled) throw new ArithmeticException("Minimum feature value must be smaller than the maximum"); this.minScaled = minFeature; } /** * Returns the minimum feature value after scaling * @return the minimum feature value after scaling */ public double getMinScaled() { return minScaled; } /** * Sets the regularization constant used for learning. The regularization * must be positive, and the learning rate is proportional to the * regularization value. This means regularizations very near zero will * take a long time to converge. * * @param lambda the regularization to apply */ public void setLambda(double lambda) { if(Double.isInfinite(lambda) || Double.isNaN(lambda) || lambda <= 0) throw new ArithmeticException("A positive amount of regularization must be performed"); this.lambda = lambda; } /** * Returns the amount of regularization to used in training * @return the regularization parameter. */ public double getLambda() { return lambda; } /** * Sets the loss function to use. This should not be altered after training * unless the leaner is going to be trained again. * @param loss the loss function to use */ public void setLoss(Loss loss) { this.loss = loss; } /** * returns the loss function in use * @return the loss function in use */ public Loss getLoss() { return loss; } /** * Sets whether or not scaling should be applied on th feature values of the * training vectors. Scaling should be used intelligently, scaling can * destroy sparsity in the data set. If scaling is not applied, and a value * is not in the range [-1, 1], a {@link FailedToFitException} could occur. *
Rescaling does not alter the data points passed in. * @param reScale whether or not to rescale feature values */ public void setReScale(boolean reScale) { this.reScale = reScale; } /** * Returns if scaling is in use * @return true if feature values are rescaled during training. */ public boolean isReScale() { return reScale; } /** * Computes {@link #w}.{@link Vec#dot(jsat.linear.Vec) }x and does * so by rescaling x as needed automatically and efficiently, even * if x is sparse. * @param x the value to compute the dot product with * @return the dot produce of w and x with the bias term */ protected double wDot(Vec x) { double a; if (reScale) { a = bias; if(!w.isSparse())//w is dense, jsut iterate over x { for(IndexValue iv : x) { int j = iv.getIndex(); double xV = iv.getValue() - obvMin[j]; xV *= (maxScaled - minScaled) / (obvMax[j] - obvMin[j]); xV += minScaled; a += w.get(j)*xV; } return a; } //Compute the dot and rescale w/o extra spacein a sprase freindly way Iterator wIter = w.getNonZeroIterator(); Iterator xIter = x.getNonZeroIterator(); if(!wIter.hasNext() || !xIter.hasNext()) return a; IndexValue wIV = wIter.next(); IndexValue xIV = xIter.next(); do { if (wIV.getIndex() == xIV.getIndex()) { int j = xIV.getIndex(); double xV = xIV.getValue() - obvMin[j]; xV *= (maxScaled - minScaled) / (obvMax[j] - obvMin[j]); xV += minScaled; //Scaled, now add to result a += wIV.getValue() * xV; if (!wIter.hasNext() || !xIter.hasNext()) break; wIV = wIter.next(); xIV = xIter.next(); } else if (wIV.getIndex() < xIV.getIndex()) if (wIter.hasNext()) wIV = wIter.next(); else break; else if (wIV.getIndex() > xIV.getIndex()) if (xIter.hasNext()) xIV = xIter.next(); else break; } while (wIV != null && xIV != null); } else a = w.dot(x) + bias; return a; } /** * Returns the weight vector used to compute results via a dot product.
* Do not modify this value, or you will alter the results returned. * @return the learned weight vector for prediction */ public Vec getWRaw() { return w; } /** * Returns a copy of the weight vector used to compute results via a dot * product. * @return a copy of the learned weight vector for prediction */ public Vec getW() { if(w == null) return w; else return w.clone(); } @Override public Vec getRawWeight() { return w; } @Override public double getBias() { return bias; } @Override public Vec getRawWeight(int index) { if(index < 1) return getRawWeight(); else throw new IndexOutOfBoundsException("Model has only 1 weight vector"); } @Override public double getBias(int index) { if (index < 1) return getBias(); else throw new IndexOutOfBoundsException("Model has only 1 weight vector"); } @Override public int numWeightsVecs() { return 1; } }

Ads help maintain this website.