package jsat.classifiers.linear;
import java.util.List;
import java.util.concurrent.ExecutorService;
import jsat.classifiers.*;
import jsat.linear.DenseVector;
import jsat.linear.Vec;
import jsat.utils.IntList;
import jsat.utils.ListUtils;
import static java.lang.Math.*;
import java.util.*;
import jsat.DataSet;
import jsat.SimpleWeightVectorModel;
import jsat.SingleWeightVectorModel;
import jsat.distributions.Distribution;
import jsat.distributions.LogUniform;
import jsat.distributions.Uniform;
import jsat.exceptions.FailedToFitException;
import jsat.linear.*;
import jsat.lossfunctions.LogisticLoss;
import jsat.parameters.Parameter;
import jsat.parameters.Parameter.WarmParameter;
import jsat.parameters.Parameterized;
/**
* NewGLMNET is a batch method for solving Elastic Net regularized Logistic
* Regression problems of the form
* 0.5 * (1-α) ||w||2 + α * ||w||1 + C *
* ∑Ni=1 ℓ (wT xi + b, yi).
*
*
* For α = 1, this becomes pure Lasso / L1 regularized Logistic
* Regression. For α = 0, this becomes pure Ridge/ L2 regularized
* Logistic Regression, however better solvers such as
* {@link LogisticRegressionDCD} are faster if using α = 0.
* The default behavior is to use α=1, and includes the bias term.
* Including the bias term can take longer to train, but can also increase
* sparsity for some problems.
*
* This algorithm can be warm started from any classifier implementing the
* {@link SingleWeightVectorModel} interface.
*
*
* See:
*
* - Yuan, G., Ho, C.-H.,&Lin, C. (2012). An improved GLMNET for
* L1-regularized logistic regression. Journal of Machine Learning Research,
* 13, 1999–2030. doi:10.1145/2020408.2020421
* - King, R., Morgan, B. J. T., Gimenez, O., Brooks, S. P., Crc, H.,&Raton,
* B. (2010). Regularization Paths for Generalized Linear Models via
* Coordinate Descent. Journal of Statistical Software, 36(1), 1–22.
* - Zou, H.,&Hastie, T. (2005). Regularization and variable selection
* via the elastic net. Journal of the Royal Statistical Society, Series B,
* 67(2), 301–320. doi:10.1111/j.1467-9868.2005.00503.x
*
*
* @author Edward Raff
*/
public class NewGLMNET implements WarmClassifier, Parameterized, SingleWeightVectorModel
{
private static final long serialVersionUID = 4133368677783573518L;
//TODO make these other fields configurable as well
private static final double DEFAULT_BETA = 0.5;
private static final double DEFAULT_V = 1e-12;
private static final double DEFAULT_GAMMA = 0;
private static final double DEFAULT_SIGMA = 0.01;
/**
* The default tolerance for training is {@value #DEFAULT_EPS}.
*/
public static final double DEFAULT_EPS = 1e-2;
/**
* The default number of outer iterations of the training algorithm is
* {@value #DEFAULT_MAX_OUTER_ITER} .
*/
public static final int DEFAULT_MAX_OUTER_ITER = 100;
/**
* Weight vector
*/
private Vec w;
/**
* Bias term
*/
private double b;
private double beta = DEFAULT_BETA;
private double v = DEFAULT_V;
private double gamma = DEFAULT_GAMMA;
private double sigma = DEFAULT_SIGMA;
private double C;
private double alpha;
private int maxOuterIters = DEFAULT_MAX_OUTER_ITER;
private double e_out = DEFAULT_EPS;
private boolean useBias = true;
/**
* The maximum allowed line-search steps
*/
private int maxLineSearchSteps = 20;
/**
* Creates a new L1 regularized Logistic Regression solver with
* C = 1.
*/
public NewGLMNET()
{
this(1);
}
/**
* Creates a new L1 regularized Logistic Regression solver
* @param C the regularization term
*/
public NewGLMNET(double C)
{
this(C, 1);
}
/**
* Creates a new Elastic Net regularized Logistic Regression solver
* @param C the regularization term
* @param alpha the fraction of weight (in [0, 1]) to apply to L1
* regularization instead of L2 regularization.
*/
public NewGLMNET(double C, double alpha)
{
setC(C);
setAlpha(alpha);
}
/**
* Copy constructor
* @param toCopy the object to copy
*/
protected NewGLMNET(NewGLMNET toCopy)
{
if(toCopy.w !=null)
this.w = toCopy.w.clone();
this.b = toCopy.b;
this.beta = toCopy.beta;
this.v = toCopy.v;
this.gamma = toCopy.gamma;
this.sigma = toCopy.sigma;
this.C = toCopy.C;
this.e_out = toCopy.e_out;
this.maxOuterIters = toCopy.maxOuterIters;
this.alpha = toCopy.alpha;
this.useBias = toCopy.useBias;
}
/**
* Sets the regularization term, where smaller values indicate a larger
* regularization penalty.
*
* @param C the positive regularization term
*/
@WarmParameter(prefLowToHigh = true)
public void setC(double C)
{
if(C <= 0 || Double.isInfinite(C) || Double.isNaN(C))
throw new IllegalArgumentException("Regularization term C must be a positive value, not " + C);
this.C = C;
}
/**
*
* @return the regularization term
*/
public double getC()
{
return C;
}
/**
* Using α = 1 corresponds to pure L1 regularization, and
* α = 0 corresponds to pure L2 regularization. Any value
* in-between is then an Elastic Net regularization.
*
* @param alpha the value in [0, 1] for determining the regularization
* penalty's interpolation between pure L2 and L1
* regularization.
*/
public void setAlpha(double alpha)
{
if(alpha < 0 || alpha > 1 || Double.isNaN(alpha))
throw new IllegalArgumentException("alpha must be in [0, 1], not " + alpha);
this.alpha = alpha;
}
/***
*
* @return the fraction of weight (in [0, 1]) to apply to L1
* regularization instead of L2 regularization.
*/
public double getAlpha()
{
return alpha;
}
/**
* Sets the maximum number of training iterations for the algorithm,
* specifically the outer loop as mentioned in the original paper.
* {@value #DEFAULT_MAX_OUTER_ITER} is the default value used, and may need
* to be increased for more difficult problems.
*
* @param maxOuterIters the maximum number of outer iterations
*/
public void setMaxIters(int maxOuterIters)
{
if(maxOuterIters < 1)
throw new IllegalArgumentException("Number of training iterations must be positive, not " + maxOuterIters);
this.maxOuterIters = maxOuterIters;
}
/**
*
* @return the maximum number of training iterations
*/
public int getMaxIters()
{
return maxOuterIters;
}
/**
* Sets the tolerance parameter for convergence. Smaller values will be more
* exact, but larger values will converge faster. The default value is
* fairly exact at {@value #DEFAULT_EPS}, increasing it by an order of
* magnitude can often be done without hurting accuracy.
*
* @param e_out the tolerance parameter.
*/
public void setTolerance(double e_out)
{
if(e_out <= 0 || Double.isNaN(e_out))
throw new IllegalArgumentException("convergence tolerance paramter must be positive, not " + e_out);
this.e_out = e_out;
}
/**
*
* @return the convergence tolerance parameter
*/
public double getTolerance()
{
return e_out;
}
/**
* Controls whether or not an un-regularized bias term is added to the
* model. Using a bias term can increase runtime, especially in sparse data
* sets, as each data point will have work done for the implicit bias term.
* However the bias term is usually needed for small dimension problems, and
* can improve the sparsity of the solution for higher dimensional problems.
*
* @param useBias {@code true} if an un-regularized bias term should be used
* or {@code false} to not use any bias term.
*/
public void setUseBias(boolean useBias)
{
this.useBias = useBias;
}
/**
*
* @return {@code true} if an un-regularized bias term will be used
* or {@code false} to not use any bias term.
*/
public boolean isUseBias()
{
return useBias;
}
@Override
public CategoricalResults classify(DataPoint data)
{
return LogisticLoss.classify(w.dot(data.getNumericalValues())+b);
}
@Override
public void train(ClassificationDataSet dataSet, boolean parallel)
{
train(dataSet);
}
@Override
public void train(ClassificationDataSet dataSet, Classifier warmSolution, boolean parallel)
{
train(dataSet, warmSolution);
}
@Override
public void train(ClassificationDataSet dataSet, Classifier warmSolution)
{
if(warmSolution instanceof SimpleWeightVectorModel)
{
SimpleWeightVectorModel swv = (SimpleWeightVectorModel) warmSolution;
train(dataSet, swv.getRawWeight(0), swv.getBias(0), true);
}
else
throw new FailedToFitException("Warm solution is not of a");
}
@Override
public void train(ClassificationDataSet dataSet)
{
train(dataSet, null, 0, false);
}
private void train(ClassificationDataSet dataSet, Vec w_init, double b_init, boolean useInit)
{
/*
* The original NewGLMNET paper describes the algorithm as minimizing
* f(w) = ||w||_1 + L(w), where L(w) is the logistic loss summed over
* all the variables. To make adapation to elastic net easier, we define
* f(w) = alpha ||w||_1 + L(w), where L(w) = (1-alpha) ||w||_2 + loss sum.
* This way we keep all the framework for L_1 regularization and
* shrinking, and just update the appropriate terms where necessary.
*/
//paper uses n= #features so we will follow their lead
final int n = dataSet.getNumNumericalVars();
//l = # data points
final int l = dataSet.getSampleSize();
if(useInit)
{
w = new DenseVector(w_init);
b = useBias ? b_init : 0;
}
else
{
w = new DenseVector(n);
b = 0;
}
List X = dataSet.getDataVectors();
double first_M_bar = 0;
double e_in = 1.0;//set later when first_M_bar is set
double[] w_dot_x = new double[l];
double[] exp_w_dot_x = new double[l];
double[] exp_w_dot_x_plus_dx = new double[l];
/**
* Used in the linear search step at the end
*/
double[] d_dot_x = new double[l];
/**
* Contains the value 1/(1+e^(w^T x)). This is used in computing D and the partial derivatives.
*/
double[] D_part = new double[l];
double[] D = new double[l];
/**
* Stores the value Hkj,j computer at the start of each iteration
*/
double[] H = new double[n];
/**
* Stores the value Hkj,j computer at the start of
* each iteration for the bias term
*/
double H_bias = 0;
/**
* Stores the value &nambla; Lj
*/
double[] delta_L = new double[n];
/**
* The gradient value for the bias term
*/
double delta_L_bias = 0;
float[] y = new float[l];
double w_norm_1;
double w_norm_2;
if(useInit)
{
for(int i = 0; i < l; i++)
{
y[i] = dataSet.getDataPointCategory(i)*2-1;
w_dot_x[i] = w.dot(X.get(i))+b;
final double tmp = exp_w_dot_x_plus_dx[i] = exp_w_dot_x[i] = exp(w_dot_x[i]);
final double D_part_i = D_part[i]= 1/(1+tmp);
D[i] = tmp*D_part_i*D_part_i;
}
w_norm_1 = w.pNorm(1);
w_norm_2 = w.pNorm(2);
}
else//w = 0
{
for(int i = 0; i < l; i++)
{
y[i] = dataSet.getDataPointCategory(i)*2-1;
w_dot_x[i] = 0.0;
exp_w_dot_x_plus_dx[i] = exp_w_dot_x[i] = 1.0;
D_part[i]= 0.5;
D[i] = 0.25;
}
w_norm_1 = w.pNorm(1);
w_norm_2 = w.pNorm(2);
}
List columnsOfX = new ArrayList(Arrays.asList(dataSet.getNumericColumns()));
/**
* sum of all x_j values in the negative class. Used for ∇_j L in trick
* from LIBLINEAR eq(44)
*/
double[] col_neg_class_sum = new double[n];
for(int j = 0; j < n; j++)
{
Vec vec = columnsOfX.get(j);
for(IndexValue iv : vec)
if(y[iv.getIndex()] == -1)
col_neg_class_sum[j] += iv.getValue();
}
/**
* Sum of all x_j values in the negative class for the bias term.
*/
double col_neg_class_sum_bias = 0;
if(useBias)
{
for(int i = 0; i < l; i++)
if(y[i] == -1)
col_neg_class_sum_bias++;
}
/**
* weight for L_1 reg is alpha, so this will be the L_2 weight (1-alpha)
*/
final double l2w = (1-alpha);
// {
// double objVal = 0;
// for(int i = 0; i < l; i++)
// objVal += C*log(1+exp(-y[i]*w_dot_x[i]));
// objVal += alpha*w_norm_1 + l2w*w_norm_2;
// System.out.println("Start Obj Val: " + objVal);
// }
//algo 3
//Let M^out ← ∞
double M_out = Double.POSITIVE_INFINITY;
Vec d = new DenseVector(n);
double d_bias = 0;
boolean prevLineSearchFail = false;
for(int k = 0; k < maxOuterIters; k++)//For k = 1, 2, 3, . . .
{
//algo 3, Step 1.
IntList J = new IntList(n);
ListUtils.addRange(J, 0, n, 1);
double M = 0;
double M_bar = 0;
//algo 3, Step 2.
Iterator j_iter = J.iterator();
while(j_iter.hasNext())
{
int j = j_iter.next();
double w_j = w.get(j);
//2.1. Calculate H^k_{jj}, ∇_j L(w^k) and ∇^S_j f(w^k)
double delta_j_L = 0;
double deltaSqrd_L = 0;
for(IndexValue x_i : columnsOfX.get(j))
{
int i = x_i.getIndex();
double val = x_i.getValue();
delta_j_L += -val*D_part[i];
//eq(44) from LIBLINEAR paper , re-factored to avoid a division by using D_part
deltaSqrd_L += val*val*D[i];
}
delta_L[j] = delta_j_L = l2w*w_j + C*(delta_j_L + col_neg_class_sum[j]);
//H^k from eq (19)
/*
* regular is C X^T D X, L2 just adds + I , but we are alreayd
* doing + eps * I to make sure the gradient is there. So just
* do the max of v and lambda_2
*/
H[j] = C*deltaSqrd_L + max(v, l2w);
double deltaS_j_fw;
if(w_j > 0)
deltaS_j_fw = delta_j_L+alpha;
else if(w_j < 0)
deltaS_j_fw = delta_j_L-alpha;
else//w_j = 0
deltaS_j_fw = signum(delta_j_L)*max(abs(delta_j_L)-alpha, 0);
//done with step 2, we have all the info
//2.2. If w^k_j = 0 and |∇_j L(w^k)| < 1−M^out/l // outer-level shrinking
//then J ←J\{j}.
//else M ←max(M, |∇^S_j f(w^k)|) and M_bar ← M_bar +|∇^S_j f(w^k)|
if(w_j == 0 && abs(delta_j_L) < alpha-M_out/l)
j_iter.remove();
else
{
M = max(M, abs(deltaS_j_fw));
M_bar += abs(deltaS_j_fw);
}
}
if(useBias)
{
//2.1. Calculate H^k_{jj}, ∇_j L(w^k) and ∇^S_j f(w^k)
double delta_j_L = 0;
double deltaSqrd_L = 0;
for(int i = 0; i < l ; i++)//all have an implicit bias term
{
delta_j_L += -D_part[i];
//eq(44) from LIBLINEAR paper , re-factored to avoid a division by using D_part
deltaSqrd_L += D[i];
}
delta_L_bias = delta_j_L = C*(delta_j_L + col_neg_class_sum_bias);
//H^k from eq (19) , but dont need v * I since its the bias term
H_bias = C*deltaSqrd_L + v;
double deltaS_j_fw = delta_L_bias;
M = max(M, abs(deltaS_j_fw));
M_bar += abs(deltaS_j_fw);
}
if (k == 0)//first run
if (useInit)//we have some value of W already,
e_in = first_M_bar = getM_Bar_for_w0(n, l, columnsOfX, col_neg_class_sum, col_neg_class_sum_bias);
else//normal algo
e_in = first_M_bar = M_bar;
//algo 3, Step 3. 3. If M_bar ≤ eps_out , return w^k
if(M_bar <= e_out*first_M_bar)
break;
//algo 3, Step 4. Let M_out ←M
M_out = M;
//algo 3, Step 5. Run algo 4
//START: Algorithm 4 Inner iterations of NewGLMNET with shrinking
double M_in = Double.POSITIVE_INFINITY;
IntList T = new IntList(J);
d.zeroOut();
d_bias = 0;
/**
* Sometimes we see the |z| be very small over and over, so we stop
* if we see it too many times in a row (which means we really
* aren't making much progress)
*/
int smallZInARow = 0;
for(int p = 0; p < 1000; p++)// inner iterations
{
//step 1.
double m = 0, m_bar = 0;
/**
* Used to check if we aren't really making any progress
*/
double max_abs_z = 0;
Collections.shuffle(T);
Iterator T_iter = T.iterator();
final double dynRange = n*5.0/T.size();//used for dynamic clip, see below
while(T_iter.hasNext())//step 2.
{
final int j = T_iter.next();
final double w_j = w.get(j);
final double d_j = d.get(j);
//from eq(16)
//∇_j q^bar_k(d) = ∇_j L(w^k) + (∇^2 L(w^k) d)_j
//∇^2_jj q^bar_k(d)=∇^2_{jj} L(w^k)
double delta_qBar_j = 0;
//first compute the (∇^2 L(w^k) d)_j portion
//see after algo 2 before eq (17)
for(IndexValue iv : columnsOfX.get(j))
{
int i = iv.getIndex();
delta_qBar_j += iv.getValue()*D[i]*d_dot_x[i];
}
delta_qBar_j *= C;
//now add the part we know from before
delta_qBar_j += delta_L[j];
/*
* For L_2, use (A+B)C = AC + BC to modify ((lambda_2 * I + ∇_2 L(w))d)j
* so we need to add lambda_2 * I d^{p, j}_j to the final
* value. I * x = x, and we are taking the value of the j'th
* coordinate, so we just have to add lambda_2 d_j
*/
delta_qBar_j += l2w*d_j;
double deltaS_q_k_j;
if(w_j + d_j > 0)
deltaS_q_k_j = delta_qBar_j + alpha;
else if(w_j + d_j < 0)
deltaS_q_k_j = delta_qBar_j - alpha;
else //w_j + d_j == 0
deltaS_q_k_j = signum(delta_qBar_j)*max(abs(delta_qBar_j)-alpha, 0);
double deltaSqrd_q_jj = H[j];
if(w_j + d_j == 0 && abs(delta_qBar_j) < alpha - M_in/l)
{
T_iter.remove();//inner-level shrinking
}
else
{
m = max(m, abs(deltaS_q_k_j));
m_bar += abs(deltaS_q_k_j);
double z;
//find z by eq (9), our w_j is actuall w_j+d_j
if(delta_qBar_j+alpha <= deltaSqrd_q_jj*(w_j+d_j))
z = -(delta_qBar_j+alpha)/deltaSqrd_q_jj;
else if(delta_qBar_j-alpha >= deltaSqrd_q_jj*(w_j+d_j))
z = -(delta_qBar_j-alpha)/deltaSqrd_q_jj;
else
z = -(w_j+d_j);
if(abs(z) < 1e-11)
continue;
/*
* When everyone is active, clip the updates to a
* smaller range - as we are going to have a lot of
* changes going on and this might make steps far larger
* than it should. When there are fewer active
* dimensions, allow for more change
*/
z = min(max(z,-dynRange),dynRange);
max_abs_z = max(max_abs_z, abs(z));
d.increment(j, z);
//book keeping, see eq(17)
for(IndexValue iv : columnsOfX.get(j))
d_dot_x[iv.getIndex()] += z*iv.getValue();
}
}
if(useBias)
{
//from eq(16)
//∇_j q^bar_k(d) = ∇_j L(w^k) + (∇^2 L(w^k) d)_j
//∇^2_jj q^bar_k(d)=∇^2_{jj} L(w^k)
double delta_qBar_j = 0;
//first compute the (∇^2 L(w^k) d)_j portion
//see after algo 2 before eq (17)
for(int i = 0; i < l; i++)
delta_qBar_j += 1*D[i]*d_dot_x[i];//compiler will take out 1*, left just to remind us its the bias term
delta_qBar_j *= C;
//now add the part we know from before
delta_qBar_j += delta_L_bias;
double deltaS_q_k_j = delta_qBar_j;
double deltaSqrd_q_jj = H_bias;
m = max(m, abs(deltaS_q_k_j));
m_bar += abs(deltaS_q_k_j);
double z = -delta_qBar_j/(deltaSqrd_q_jj);
if (abs(z) > 1e-11)
{
z = min(max(z, -dynRange), dynRange);
max_abs_z = max(max_abs_z, abs(z));
d_bias += z;
//book keeping, see eq(17)
for(int i = 0; i < l ; i++)
d_dot_x[i] += z;
}
}
boolean breakInnerLoopAnyway = false;
if(max_abs_z == 0)
breakInnerLoopAnyway = true;
else if (max_abs_z <= 1e-6)
{
if(smallZInARow++ >= 3)//give it a few chances
breakInnerLoopAnyway = true;
}
else if(max_abs_z <= 1e-3)
{
if(smallZInARow++ >= 30)//give it a lot chances
breakInnerLoopAnyway = true;
}
else
smallZInARow = 0;//reset, we are making progress!
//step 3.
if(m_bar <= e_in || breakInnerLoopAnyway)
{
if(T.size() == J.size())
{
/*
* If at one outer iteration, the condition (26) holds
* after only one cycle of n CD steps, then we reduce
* e_in by 1/4.
* That is, the program automatically adjusts e_in if it
* finds that too few CD steps are conducted for
* minimizing qk(d)
*/
if(p == 0)
e_in /= 4;
break;
}
else
{
T.clear();
T.addAll(J);
M_in = Double.POSITIVE_INFINITY;
}
}
else
M_in = m;
}
//END: Algorithm 4 Inner iterations of NewGLMNET with shrinking
//algo 3, Step 6. Compute λ = max{1,β,β^2, . . . } such that λd satisfies (20)
//Use the form of eq(45) from Aug2014 LIBLINEAR paper
//get ||w+d||_1 and ∇L^T d in one loop together
double wPd_norm_1 = w_norm_1;
double wPd_norm_2 = w_norm_2;
double delta_L_dot_d = 0;
for(IndexValue iv: d)
{
final int j = iv.getIndex();
final double w_j = w.get(j);
final double d_j = iv.getValue();
wPd_norm_1 -= abs(w_j);
wPd_norm_1 += abs(w_j+d_j);
wPd_norm_2 -= w_j*w_j;
wPd_norm_2 += (w_j+d_j)*(w_j+d_j);
delta_L_dot_d += d_j*delta_L[j];
}
delta_L_dot_d += d_bias*delta_L_bias;
final double breakCondition = sigma*(delta_L_dot_d +
alpha*(wPd_norm_1-w_norm_1) +
l2w*(wPd_norm_2-w_norm_2) );
double lambda = 1;
int t = 0;
double wPlambda_d_norm_1 = wPd_norm_1;
double wPlambda_d_norm_2 = wPd_norm_2;
while(t < maxLineSearchSteps)//we may want to adjust this as beta changes
{
//"For line search, we use the following form of the sufficient decrease condition" eq(45) from LIBLINEAR paper Aug 2014
double newTerm = 0;
for(int i = 0; i < l; i++)
{
double exp_lamda_d_dot_x = exp(lambda*d_dot_x[i]);
exp_w_dot_x_plus_dx[i] = exp_w_dot_x[i]*exp_lamda_d_dot_x;
newTerm += log((exp_w_dot_x_plus_dx[i]+1)/(exp_w_dot_x_plus_dx[i]+exp_lamda_d_dot_x ));
if(y[i] == -1)
newTerm += lambda*d_dot_x[i];
}
newTerm = l2w*(wPlambda_d_norm_2 - w_norm_2) +//l2 reg
alpha*(wPlambda_d_norm_1 - w_norm_1) + //l1 reg
C*newTerm;//loss
if(newTerm <= lambda * breakCondition)
break;
//else
lambda = pow(beta, ++t);
//update norm
wPlambda_d_norm_1 = w_norm_1;
wPlambda_d_norm_2 = w_norm_2;
for(IndexValue iv: d)
{
final double w_j = w.get(iv.getIndex());
final double lambda_d_j = lambda*iv.getValue();
wPlambda_d_norm_1 -= abs(w_j);
wPlambda_d_norm_1 += abs(w_j+lambda_d_j);
wPlambda_d_norm_2 -= w_j*w_j;
wPlambda_d_norm_2 += (w_j+lambda_d_j)*(w_j+lambda_d_j);
}
}
//if line search fails twice in a row, just quit
if(t == maxLineSearchSteps)//this shouldn't happen unless we are having serious trouble improving our results
if (prevLineSearchFail)
break;//jsut finish.
else
prevLineSearchFail = true;
else
prevLineSearchFail = false;
//algo 3, Step 7. 7. w^{k+1} = w^k +λ d.
w.mutableAdd(lambda, d);
b += lambda * d_bias;
w_norm_1 = wPlambda_d_norm_1;
w_norm_2 = wPlambda_d_norm_2;
//and more book keeping
//val from last line search is new w
System.arraycopy(exp_w_dot_x_plus_dx, 0, exp_w_dot_x, 0, l);
//(w+lambda d)^T x = w^T x + d^T x
for(int i = 0; i < l; i++)
{
w_dot_x[i] += lambda*d_dot_x[i];
final double D_part_i = D_part[i]= 1/(1+exp_w_dot_x[i]);
D[i] = exp_w_dot_x[i]*D_part_i*D_part_i;
}
Arrays.fill(d_dot_x, 0.0);//new d = 0, always
// double objVal = 0;
// for(int i = 0; i < l; i++)
// objVal += C*log(1+exp(-y[i]*w_dot_x[i]));
// objVal += alpha*w_norm_1 + l2w*w_norm_2;
// System.out.println("Iter "+ k + " has New Obj Val: " + objVal);
}
}
/**
* When we perform a warm start, we want to train to the same point that we
* would have if we had not done a warm start. But our stopping point is
* based on the initial relative error. To get around that, this method
* computes what the error would have been for the zero weight vector
* @param n
* @param l
* @param columnsOfX
* @param col_neg_class_sum
* @param col_neg_class_sum_bias
* @return the error for M_bar that would have been computed if we were using the zero weight vector
*/
private double getM_Bar_for_w0(int n, int l, List columnsOfX, double[] col_neg_class_sum, double col_neg_class_sum_bias)
{
/**
* if w=0, then D_part[i] = 0.5 for all i
*/
final double D_part_i = 0.5;
//algo 3, Step 1.
double M_bar = 0;
//algo 3, Step 2.
for(int j = 0; j < n; j++)
{
final double w_j = 0;
//2.1. Calculate H^k_{jj}, ∇_j L(w^k) and ∇^S_j f(w^k)
double delta_j_L = -columnsOfX.get(j).sum()*0.5;
delta_j_L = /* (l2w * w_j) not needed b/c w_j=0*/ + C * (delta_j_L + col_neg_class_sum[j]);
double deltaS_j_fw;
//only the w_j = 0 case applies, b/c that is what this method is for!
//w_j = 0
deltaS_j_fw = signum(delta_j_L) * max(abs(delta_j_L) - alpha, 0);
//done with step 2, we have all the info
M_bar += abs(deltaS_j_fw);
}
if (useBias)
{
//2.1. Calculate H^k_{jj}, ∇_j L(w^k) and ∇^S_j f(w^k)
double delta_j_L = 0;
for (int i = 0; i < l; i++)//all have an implicit bias term
delta_j_L += -D_part_i;
delta_j_L = C * (delta_j_L + col_neg_class_sum_bias);
double deltaS_j_fw = delta_j_L;
M_bar += abs(deltaS_j_fw);
}
return M_bar;
}
@Override
public boolean supportsWeightedData()
{
return false;
}
@Override
public NewGLMNET clone()
{
return new NewGLMNET(this);
}
@Override
public Vec getRawWeight()
{
return w;
}
@Override
public double getBias()
{
return b;
}
@Override
public Vec getRawWeight(int index)
{
if(index < 1)
return getRawWeight();
else
throw new IndexOutOfBoundsException("Model has only 1 weight vector");
}
@Override
public double getBias(int index)
{
if(index < 1)
return getBias();
else
throw new IndexOutOfBoundsException("Model has only 1 weight vector");
}
@Override
public int numWeightsVecs()
{
return 1;
}
@Override
public boolean warmFromSameDataOnly()
{
return false;
}
/**
* Guess the distribution to use for the trade off term term
* {@link #setAlpha(double) (double) α} in Elastic Net regularization.
*
* @param d the data set to get the guess for
* @return the guess for the α parameter
*/
public static Distribution guessAlpha(DataSet d)
{
//Would do [0, .75], but if you are doing to be so close to full L2 reg you should really be using a different solver
return new Uniform(0.25, 0.75);
}
/**
* Guess the distribution to use for the regularization term
* {@link #setC(double) C} in Logistic Regression.
*
* @param d the data set to get the guess for
* @return the guess for the C parameter
*/
public static Distribution guessC(DataSet d)
{
double maxLambda = LinearTools.maxLambdaLogisticL1((ClassificationDataSet) d);
double minC = 1/(2*maxLambda*d.getSampleSize());
return new LogUniform(minC*10, minC*1000);
}
}
Ads help maintain this website.