package jsat.io;
import java.io.*;
import java.util.*;
import jsat.DataSet;
import jsat.classifiers.CategoricalData;
import jsat.classifiers.ClassificationDataSet;
import jsat.classifiers.DataPoint;
import jsat.datatransform.DenseSparceTransform;
import jsat.linear.*;
import jsat.regression.RegressionDataSet;
import jsat.utils.DoubleList;
import jsat.utils.StringUtils;
/**
* Loads a LIBSVM data file into a {@link DataSet}. LIVSM files do not indicate
* whether or not the target variable is supposed to be numerical or
* categorical, so two different loading methods are provided. For a LIBSVM file
* to be loaded correctly, it must match the LIBSVM spec without extensions.
*
* Each line should begin with a numeric value. This is either a regression
* target or a class label.
* Then, for each non zero value in the data set, a space should precede an
* integer value index starting from 1 followed by a colon ":" followed by a
* numeric feature value.
The single space at the beginning should be the
* only space. There should be no double spaces in the file.
*
* LIBSVM files do not explicitly specify the length of data vectors. This can
* be problematic if loading a testing and training data set, if the data sets
* do not include the same highest index as a non-zero value, the data sets will
* have incompatible vector lengths. To resolve this issue, use the loading
* methods that include the optional {@code vectorLength} parameter to specify
* the length before hand.
*
* @author Edward Raff
*/
public class LIBSVMLoader
{
private static boolean fastLoad = true;
private LIBSVMLoader()
{
}
/*
* LIBSVM format is sparse
* <1 based Index>:
*
*/
/**
* Loads a new regression data set from a LIBSVM file, assuming the label is
* a numeric target value to predict
*
* @param file the file to load
* @return a regression data set
* @throws FileNotFoundException if the file was not found
* @throws IOException if an error occurred reading the input stream
*/
public static RegressionDataSet loadR(File file) throws FileNotFoundException, IOException
{
return loadR(file, 0.5);
}
/**
* Loads a new regression data set from a LIBSVM file, assuming the label is
* a numeric target value to predict
*
* @param file the file to load
* @param sparseRatio the fraction of non zero values to qualify a data
* point as sparse
* @return a regression data set
* @throws FileNotFoundException if the file was not found
* @throws IOException if an error occurred reading the input stream
*/
public static RegressionDataSet loadR(File file, double sparseRatio) throws FileNotFoundException, IOException
{
return loadR(file, sparseRatio, -1);
}
/**
* Loads a new regression data set from a LIBSVM file, assuming the label is
* a numeric target value to predict
*
* @param file the file to load
* @param sparseRatio the fraction of non zero values to qualify a data
* point as sparse
* @param vectorLength the pre-determined length of each vector. If given a
* negative value, the largest non-zero index observed in the data will be
* used as the length.
* @return a regression data set
* @throws FileNotFoundException if the file was not found
* @throws IOException if an error occurred reading the input stream
*/
public static RegressionDataSet loadR(File file, double sparseRatio, int vectorLength) throws FileNotFoundException, IOException
{
return loadR(new FileReader(file), sparseRatio, vectorLength);
}
/**
* Loads a new regression data set from a LIBSVM file, assuming the label is
* a numeric target value to predict
*
* @param isr the input stream for the file to load
* @param sparseRatio the fraction of non zero values to qualify a data
* point as sparse
* @return a regression data set
* @throws IOException if an error occurred reading the input stream
*/
public static RegressionDataSet loadR(InputStreamReader isr, double sparseRatio) throws IOException
{
return loadR(isr, sparseRatio, -1);
}
/**
* Loads a new regression data set from a LIBSVM file, assuming the label is
* a numeric target value to predict.
*
* @param reader the reader for the file to load
* @param sparseRatio the fraction of non zero values to qualify a data
* point as sparse
* @param vectorLength the pre-determined length of each vector. If given a
* negative value, the largest non-zero index observed in the data will be
* used as the length.
* @return a regression data set
* @throws IOException
*/
public static RegressionDataSet loadR(Reader reader, double sparseRatio, int vectorLength) throws IOException
{
return (RegressionDataSet) loadG(reader, sparseRatio, vectorLength, false);
}
/**
* Loads a new classification data set from a LIBSVM file, assuming the
* label is a nominal target value
*
* @param file the file to load
* @return a classification data set
* @throws FileNotFoundException if the file was not found
* @throws IOException if an error occurred reading the input stream
*/
public static ClassificationDataSet loadC(File file) throws FileNotFoundException, IOException
{
return loadC(new FileReader(file), 0.5);
}
/**
* Loads a new classification data set from a LIBSVM file, assuming the
* label is a nominal target value
*
* @param file the file to load
* @param sparseRatio the fraction of non zero values to qualify a data
* point as sparse
* @return a classification data set
* @throws FileNotFoundException if the file was not found
* @throws IOException if an error occurred reading the input stream
*/
public static ClassificationDataSet loadC(File file, double sparseRatio) throws FileNotFoundException, IOException
{
return loadC(file, sparseRatio, -1);
}
/**
* Loads a new classification data set from a LIBSVM file, assuming the
* label is a nominal target value
*
* @param file the file to load
* @param sparseRatio the fraction of non zero values to qualify a data
* point as sparse
* @param vectorLength the pre-determined length of each vector. If given a
* negative value, the largest non-zero index observed in the data will be
* used as the length.
* @return a classification data set
* @throws FileNotFoundException if the file was not found
* @throws IOException if an error occurred reading the input stream
*/
public static ClassificationDataSet loadC(File file, double sparseRatio, int vectorLength) throws FileNotFoundException, IOException
{
return loadC(new FileReader(file), sparseRatio, vectorLength);
}
/**
* Loads a new classification data set from a LIBSVM file, assuming the
* label is a nominal target value
*
* @param isr the input stream for the file to load
* @param sparseRatio the fraction of non zero values to qualify a data
* point as sparse
* @return a classification data set
* @throws IOException if an error occurred reading the input stream
*/
public static ClassificationDataSet loadC(InputStreamReader isr, double sparseRatio) throws IOException
{
return loadC(isr, sparseRatio, -1);
}
/**
* Loads a new classification data set from a LIBSVM file, assuming the
* label is a nominal target value
*
* @param reader the input stream for the file to load
* @param sparseRatio the fraction of non zero values to qualify a data
* point as sparse
* @param vectorLength the pre-determined length of each vector. If given a
* negative value, the largest non-zero index observed in the data will be
* used as the length.
* @return a classification data set
* @throws IOException if an error occurred reading the input stream
*/
public static ClassificationDataSet loadC(Reader reader, double sparseRatio, int vectorLength) throws IOException
{
return (ClassificationDataSet) loadG(reader, sparseRatio, vectorLength, true);
}
/**
* Generic loader for both Classification and Regression interpretations.
* @param reader
* @param sparseRatio
* @param vectorLength
* @param classification {@code true} to treat as classification,
* {@code false} to treat as regression
* @return
* @throws IOException
*/
private static DataSet loadG(Reader reader, double sparseRatio, int vectorLength, boolean classification) throws IOException
{
StringBuilder processBuffer = new StringBuilder(20);
StringBuilder charBuffer = new StringBuilder(1024);
char[] buffer = new char[1024];
List sparceVecs = new ArrayList();
/**
* The category "label" for each value loaded in
*/
List labelVals = new DoubleList();
Map possibleCats = new HashMap();
int maxLen= 1;
STATE state = STATE.INITIAL;
int position = 0;
SparseVector tempVec = new SparseVector(1, 1);
/**
* The index that we have parse out of a non zero pair
*/
int indexProcessing = -1;
while(true)
{
while(charBuffer.length()-position <= 1)//make sure we have chars to handle
{
//move everything to the front
charBuffer.delete(0, position);
position = 0;
int read = reader.read(buffer);
if(read < 0)
break;
charBuffer.append(buffer, 0, read);
}
if(charBuffer.length()-position == 0)//EOF, no more chars
{
if(state == STATE.LABEL)//last line was empty
{
double label = Double.parseDouble(processBuffer.toString());
if (!possibleCats.containsKey(label) && classification)
possibleCats.put(label, possibleCats.size());
labelVals.add(label);
sparceVecs.add(new SparseVector(maxLen, 0));
}
else if(state == STATE.WHITESPACE_AFTER_LABEL)//last line was empty, but we have already eaten the label
{
sparceVecs.add(new SparseVector(maxLen, 0));
}
else if(state == STATE.FEATURE_VALUE || state == STATE.WHITESPACE_AFTER_FEATURE)//line ended after a value pair
{
//process the last value pair & insert into vec
double value = StringUtils.parseDouble(processBuffer, 0, processBuffer.length());
processBuffer.delete(0, processBuffer.length());
maxLen = Math.max(maxLen, indexProcessing+1);
tempVec.setLength(maxLen);
if (value != 0)
tempVec.set(indexProcessing, value);
sparceVecs.add(tempVec.clone());
}
else if(state == STATE.NEWLINE)
{
//nothing to do and everything already processed, just return
break;
}
else
throw new RuntimeException();
//we may have ended on a line, and have a sparse vec to add before returning
break;
}
char ch = charBuffer.charAt(position);
switch(state)
{
case INITIAL:
state = STATE.LABEL;
break;
case LABEL:
if (Character.isDigit(ch) || ch == '.' || ch == 'E' || ch == 'e' || ch == '-' || ch == '+')
{
processBuffer.append(ch);
position++;
}
else if (Character.isWhitespace(ch))//this gets spaces and new lines
{
double label = Double.parseDouble(processBuffer.toString());
if (!possibleCats.containsKey(label) && classification)
possibleCats.put(label, possibleCats.size());
labelVals.add(label);
//clean up and move to new state
processBuffer.delete(0, processBuffer.length());
if (ch == '\n' || ch == '\r')//empty line, so add a zero vector
{
tempVec.zeroOut();
sparceVecs.add(new SparseVector(maxLen, 0));
state = STATE.NEWLINE;
}
else//just white space
{
tempVec.zeroOut();
state = STATE.WHITESPACE_AFTER_LABEL;
}
}
else
throw new RuntimeException("Invalid LIBSVM file");
break;
case WHITESPACE_AFTER_LABEL:
if (Character.isDigit(ch))//move to next state
{
state = STATE.FEATURE_INDEX;
}
else if (Character.isWhitespace(ch))
{
if (ch == '\n' || ch == '\r')
{
tempVec.zeroOut();
sparceVecs.add(new SparseVector(maxLen, 0));///no features again, add zero vec
state = STATE.NEWLINE;
}
else//normal whie space
position++;
}
else
throw new RuntimeException();
break;
case FEATURE_INDEX:
if (Character.isDigit(ch))
{
processBuffer.append(ch);
position++;
}
else if(ch == ':')
{
indexProcessing = StringUtils.parseInt(processBuffer, 0, processBuffer.length())-1;
processBuffer.delete(0, processBuffer.length());
state = STATE.FEATURE_VALUE;
position++;
}
else
throw new RuntimeException();
break;
case FEATURE_VALUE:
//we need to accept all the values that may be part of a float value
if (Character.isDigit(ch) || ch == '.' || ch == 'E' || ch == 'e' || ch == '-' || ch == '+')
{
processBuffer.append(ch);
position++;
}
else
{
double value = StringUtils.parseDouble(processBuffer, 0, processBuffer.length());
processBuffer.delete(0, processBuffer.length());
maxLen = Math.max(maxLen, indexProcessing+1);
tempVec.setLength(maxLen);
if (value != 0)
tempVec.set(indexProcessing, value);
if (Character.isWhitespace(ch))
state = STATE.WHITESPACE_AFTER_FEATURE;
else
throw new RuntimeException();
}
break;
case WHITESPACE_AFTER_FEATURE:
if (Character.isDigit(ch))
state = STATE.FEATURE_INDEX;
else if (Character.isWhitespace(ch))
{
if (ch == '\n' || ch == '\r')
{
sparceVecs.add(tempVec.clone());
tempVec.zeroOut();
state = STATE.NEWLINE;
}
else
position++;
}
break;
case NEWLINE:
if (ch == '\n' || ch == '\r')
position++;
else
{
state = STATE.LABEL;
}
break;
}
}
if (vectorLength > 0)
if (maxLen > vectorLength)
throw new RuntimeException("Length given was " + vectorLength + ", but observed length was " + maxLen);
else
maxLen = vectorLength;
if(classification)
{
CategoricalData predicting = new CategoricalData(possibleCats.size());
//Give categories a unique ordering to avoid loading issues based on the order categories are presented
List allCatKeys = new DoubleList(possibleCats.keySet());
Collections.sort(allCatKeys);
for(int i = 0; i < allCatKeys.size(); i++)
possibleCats.put(allCatKeys.get(i), i);
ClassificationDataSet cds = new ClassificationDataSet(maxLen, new CategoricalData[0], predicting);
for(int i = 0; i < labelVals.size(); i++)
{
SparseVector vec = sparceVecs.get(i);
vec.setLength(maxLen);
cds.addDataPoint(vec, new int[0], possibleCats.get(labelVals.get(i)));
}
cds.applyTransform(new DenseSparceTransform(sparseRatio));
return cds;
}
else//regression
{
RegressionDataSet rds = new RegressionDataSet(maxLen, new CategoricalData[0]);
for(int i = 0; i < sparceVecs.size(); i++)
{
SparseVector sv = sparceVecs.get(i);
sv.setLength(maxLen);
rds.addDataPoint(sv, new int[0], labelVals.get(i));
}
rds.applyTransform(new DenseSparceTransform(sparseRatio));
return rds;
}
}
/**
* Writes out the given classification data set as a LIBSVM data file
* @param data the data set to write to a file
* @param os the output stream to write to. The stream will not be closed or
* flushed by this method
*/
public static void write(ClassificationDataSet data, OutputStream os)
{
PrintWriter writer = new PrintWriter(os);
for(int i = 0; i < data.getSampleSize(); i++)
{
int pred = data.getDataPointCategory(i);
Vec vals = data.getDataPoint(i).getNumericalValues();
writer.write(pred + " ");
for(IndexValue iv : vals)
{
double val = iv.getValue();
if(Math.rint(val) == val)//cast to long before writting to save space
writer.write((iv.getIndex()+1) + ":" + (long)val + " ");//+1 b/c 1 based indexing
else
writer.write((iv.getIndex()+1) + ":" + val + " ");//+1 b/c 1 based indexing
}
writer.write("\n");
}
writer.flush();
writer.close();
}
/**
* Writes out the given regression data set as a LIBSVM data file
* @param data the data set to write to a file
* @param os the output stream to write to. The stream will not be closed or
* flushed by this method
*/
public static void write(RegressionDataSet data, OutputStream os)
{
PrintWriter writer = new PrintWriter(os);
for(int i = 0; i < data.getSampleSize(); i++)
{
double pred = data.getTargetValue(i);
Vec vals = data.getDataPoint(i).getNumericalValues();
writer.write(pred + " ");
for(IndexValue iv : vals)
{
double val = iv.getValue();
if(Math.rint(val) == val)//cast to long before writting to save space
writer.write((iv.getIndex()+1) + ":" + (long)val + " ");//+1 b/c 1 based indexing
else
writer.write((iv.getIndex()+1) + ":" + val + " ");//+1 b/c 1 based indexing
}
writer.write("\n");
}
writer.flush();
writer.close();
}
/**
* Returns a DataWriter object which can be used to stream a set of
* arbitrary datapoints into the given output stream. This works in a thread
* safe manner.
* Categorical information dose not need to be specified since LIBSVM files can't store categorical features.
*
* @param out the location to store all the data
* @param dim information on how many numeric features exist
* @param type what type of data set (simple, classification, regression) to be written
* @return the DataWriter that the actual points can be streamed through
* @throws IOException
*/
public static DataWriter getWriter(OutputStream out, int dim, DataWriter.DataSetType type) throws IOException
{
DataWriter dw = new DataWriter(out, new CategoricalData[0], dim, type)
{
@Override
protected void writeHeader(CategoricalData[] catInfo, int dim, DataWriter.DataSetType type, OutputStream out)
{
//nothing to do, LIBSVM format has no header
}
@Override
protected void pointToBytes(DataPoint dp, double label, ByteArrayOutputStream byteOut)
{
PrintWriter writer = new PrintWriter(byteOut);
//write out label
if(this.type == DataSetType.REGRESSION)
writer.write(label + " ");
else if(this.type == DataSetType.CLASSIFICATION)
writer.write((int)label + " ");
else if(this.type == DataSetType.SIMPLE)
writer.write("0 ");
Vec vals = dp.getNumericalValues();
for(IndexValue iv : vals)
{
double val = iv.getValue();
if(Math.rint(val) == val)//cast to long before writting to save space
writer.write((iv.getIndex()+1) + ":" + (long)val + " ");//+1 b/c 1 based indexing
else
writer.write((iv.getIndex()+1) + ":" + val + " ");//+1 b/c 1 based indexing
}
writer.write("\n");
writer.flush();
}
};
return dw;
}
/**
* Simple state machine used to parse LIBSVM files
*/
private enum STATE
{
/**
* Initial state, doesn't actually do anything
*/
INITIAL,
LABEL,
WHITESPACE_AFTER_LABEL,
FEATURE_INDEX,
FEATURE_VALUE,
WHITESPACE_AFTER_FEATURE,
NEWLINE,
}
}
Ads help maintain this website.