package jsat.regression;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
import jsat.classifiers.DataPoint;
import jsat.linear.*;
import jsat.parameters.Parameter;
import jsat.parameters.Parameterized;
import jsat.utils.FakeExecutor;
import jsat.utils.concurrent.ParallelUtils;
/**
* An implementation of Ridge Regression that finds the exact solution. Ridge
* Regression is equivalent to {@link MultipleLinearRegression} with an added
* L2 penalty for the weight vector.
* Two different methods of finding the solution can be used. This algorithm
* should be used only for small dimensions problems with a reasonable number of
* example points.
* For large dimension sparse problems, or dense problems with many data points
* (or both), use the {@link StochasticRidgeRegression}. For small data sets
* that pose non-linear problems, you can also use {@link KernelRidgeRegression}
*
* @author Edward Raff
*/
public class RidgeRegression implements Regressor, Parameterized
{
private static final long serialVersionUID = -4605757038780391895L;
private double lambda;
private Vec w;
private double bias;
private SolverMode mode;
/**
* Sets which solver to use
*/
public enum SolverMode
{
/**
* Solves by {@link CholeskyDecomposition}
*/
EXACT_CHOLESKY,
/**
* Solves by {@link SingularValueDecomposition}
*/
EXACT_SVD,
}
public RidgeRegression()
{
this(1e-2);
}
public RidgeRegression(double regularization)
{
this(regularization, SolverMode.EXACT_CHOLESKY);
}
public RidgeRegression(double regularization, SolverMode mode)
{
setLambda(regularization);
setSolverMode(mode);
}
/**
* Sets the regularization parameter used.
* @param lambda the positive regularization constant in (0, Inf)
*/
public void setLambda(double lambda)
{
if(Double.isNaN(lambda) || Double.isInfinite(lambda) || lambda <= 0)
throw new IllegalArgumentException("lambda must be a positive constant, not " + lambda);
this.lambda = lambda;
}
/**
* Returns the regularization constant in use
* @return the regularization constant in use
*/
public double getLambda()
{
return lambda;
}
/**
* Sets which solver is to be used
* @param mode the solver mode to use
*/
public void setSolverMode(SolverMode mode)
{
this.mode = mode;
}
/**
* Returns the solver in use
* @return the solver to use
*/
public SolverMode getSolverMode()
{
return mode;
}
@Override
public double regress(DataPoint data)
{
Vec x = data.getNumericalValues();
return w.dot(x)+bias;
}
@Override
public void train(RegressionDataSet dataSet, boolean parallel)
{
final int dim = dataSet.getNumNumericalVars()+1;
DenseMatrix X = new DenseMatrix(dataSet.getSampleSize(), dim);
for(int i = 0; i < dataSet.getSampleSize(); i++)
{
Vec from = dataSet.getDataPoint(i).getNumericalValues();
X.set(i, 0, 1.0);
for(int j = 0; j < from.length(); j++)
X.set(i, j+1, from.get(j));
}
final Vec Y = dataSet.getTargetValues();
final boolean serial = !parallel;
if(mode == SolverMode.EXACT_SVD)
{
SingularValueDecomposition svd = new SingularValueDecomposition(X);
double[] ridgeD;
ridgeD = Arrays.copyOf(svd.getSingularValues(), dim);
for(int i = 0; i < ridgeD.length; i++)
ridgeD[i] = 1 / (Math.pow(ridgeD[i], 2)+lambda);
Matrix U = svd.getU();
Matrix V = svd.getV();
// w = V (D^2 + lambda I)^(-1) D U^T y
Matrix.diagMult(V, DenseVector.toDenseVec(ridgeD));
Matrix.diagMult(V, DenseVector.toDenseVec(svd.getSingularValues()));
w = V.multiply(U.transpose()).multiply(Y);
}
else//cholesky
{
Matrix H = serial ? X.transposeMultiply(X) : X.transposeMultiply(X, ParallelUtils.CACHED_THREAD_POOL);
//H + I * reg equiv to H.mutableAdd(Matrix.eye(H.rows()).multiply(regularization));
for(int i = 0; i < H.rows(); i++)
H.increment(i, i, lambda);
CholeskyDecomposition cd = serial ? new CholeskyDecomposition(H) : new CholeskyDecomposition(H, ParallelUtils.CACHED_THREAD_POOL);
w = cd.solve(Matrix.eye(H.rows())).multiply(X.transpose()).multiply(Y);
}
//reformat w and seperate out bias term
bias = w.get(0);
Vec newW = new DenseVector(w.length()-1);
for(int i = 0; i < newW.length(); i++)
newW.set(i, w.get(i+1));
w = newW;
}
@Override
public boolean supportsWeightedData()
{
return false;
}
@Override
public RidgeRegression clone()
{
RidgeRegression clone = new RidgeRegression(lambda);
if(this.w != null)
clone.w = this.w.clone();
clone.bias = this.bias;
return clone;
}
}
Ads help maintain this website.