/*
* Copyright (C) 2015 Edward Raff
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see .
*/
package jsat.io;
import java.io.*;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import jsat.*;
import jsat.classifiers.*;
import jsat.linear.*;
import jsat.regression.RegressionDataSet;
/**
* JSAT Data Loader provides a simple binary file format for storing and reading
* datasets. All datasets can always be read back in as a {@link SimpleDataSet},
* and {@link ClassificationDataSet} and {@link RegressionDataSet} datasets can
* be read back in as their original types.
*
* For well behaved datasets (where most numeric features are integer values),
* an uncompressed JSAT Dataset file may be larger than a similar
* {@link ARFFLoader ARFF} or {@link LIBSVMLoader LIBSVM} file. This is because
* JSAT always uses 32 or 64 bits (4 or 8 bytes) for every value, where values
* stored as a string could use as little as 1 or 2 bytes for simple values.
* However, JSAT's storage will be consistent - data which uses the floating
* point values (such as the number "0.25098039215686274") will use additional
* bytes in the human readable ARFF and LIBSVM formats that are not necessary,
* where the binary JSAT format will stay the same size.
*
* Even when JSAT produces larger files, since it is a simple binary format,
* reading and writing will usually be significantly faster.
*
* Additional storage savings can be obtained by using the
* {@link GZIPOutputStream} when storing a dataset, and then decompressed when
* read back in using {@link GZIPInputStream}.
*
*
*
*
* @author Edward Raff
*/
public class JSATData
{
private JSATData()
{
}
public static final byte[] MAGIC_NUMBER = new byte[]
{
'J', 'S', 'A', 'T', '_', '0', '0'
};
public static enum DatasetTypeMarker
{
STANDARD,
REGRESSION,
CLASSIFICATION;
}
public static enum FloatStorageMethod
{
AUTO
{
@Override
protected void writeFP(double value, DataOutputStream out) throws IOException
{
//Auto dosn't actually write! Auto just means figure out the best
throw new UnsupportedOperationException("Not supported .");
}
@Override
protected double readFP(DataInputStream in) throws IOException
{
//Auto dosn't actually write! Auto just means figure out the best
throw new UnsupportedOperationException("Not supported .");
}
@Override
protected boolean noLoss(double orig)
{
return true;
}
},
FP64
{
@Override
protected void writeFP(double value, DataOutputStream out) throws IOException
{
out.writeDouble(value);
}
@Override
protected double readFP(DataInputStream in) throws IOException
{
return in.readDouble();
}
@Override
protected boolean noLoss(double orig)
{
return true;
}
},
FP32
{
@Override
protected void writeFP(double value, DataOutputStream out) throws IOException
{
out.writeFloat((float) value);
}
@Override
protected double readFP(DataInputStream in) throws IOException
{
return in.readFloat();
}
@Override
protected boolean noLoss(double orig)
{
//below can only return true if ther eis no loss in storing these values as 32 bit floats instead of doubles
float f_o = (float) orig;
return Double.valueOf(f_o)-orig == 0.0;
}
},
SHORT
{
@Override
protected void writeFP(double value, DataOutputStream out) throws IOException
{
out.writeShort(Math.min(Math.max((int)value, Short.MIN_VALUE), Short.MAX_VALUE));
}
@Override
protected double readFP(DataInputStream in) throws IOException
{
return in.readShort();
}
@Override
protected boolean noLoss(double orig)
{
return Short.MIN_VALUE <= orig && orig <= Short.MAX_VALUE && orig == Math.rint(orig);
}
},
BYTE
{
@Override
protected void writeFP(double value, DataOutputStream out) throws IOException
{
out.writeByte(Math.min(Math.max((int)value, Byte.MIN_VALUE), Byte.MAX_VALUE));
}
@Override
protected double readFP(DataInputStream in) throws IOException
{
return in.readByte();
}
@Override
protected boolean noLoss(double orig)
{
return Byte.MIN_VALUE <= orig && orig <= Byte.MAX_VALUE && orig == Math.rint(orig);
}
},
U_BYTE
{
@Override
protected void writeFP(double value, DataOutputStream out) throws IOException
{
out.writeByte(Math.min(Math.max((int)value, 0), 255));
}
@Override
protected double readFP(DataInputStream in) throws IOException
{
return in.readByte() & 0xff;
}
@Override
protected boolean noLoss(double orig)
{
return 0 <= orig && orig <= 255 && orig == Math.rint(orig);
}
};
abstract protected void writeFP(double value, DataOutputStream out) throws IOException;
abstract protected double readFP(DataInputStream in) throws IOException;
abstract protected boolean noLoss(double orig);
static public > FloatStorageMethod getMethod(DataSet data, FloatStorageMethod method)
{
if (method == FloatStorageMethod.AUTO)//figure out what storage method to use!
{
EnumSet storageCandidates = EnumSet.complementOf(EnumSet.of(FloatStorageMethod.AUTO));
//loop through all the data and remove invalid candidates
for(int i = 0; i < data.getSampleSize(); i++)
{
DataPoint dp = data.getDataPoint(i);
for (IndexValue iv : dp.getNumericalValues())
{
Iterator iter = storageCandidates.iterator();
while (iter.hasNext())
{
if (!iter.next().noLoss(iv.getValue()))
iter.remove();
}
if (storageCandidates.size() == 1)
break;
}
Iterator iter = storageCandidates.iterator();
while (iter.hasNext())
{
if (!iter.next().noLoss(dp.getWeight()))
iter.remove();
}
if (storageCandidates.size() == 1)
break;
}
if(data instanceof RegressionDataSet)
{
for(IndexValue iv : ((RegressionDataSet)data).getTargetValues())
{
Iterator iter = storageCandidates.iterator();
while (iter.hasNext())
{
if (!iter.next().noLoss(iv.getValue()))
iter.remove();
}
if (storageCandidates.size() == 1)
break;
}
}
if(storageCandidates.contains(BYTE))
return BYTE;
else if(storageCandidates.contains(U_BYTE))
return U_BYTE;
else if(storageCandidates.contains(SHORT))
return SHORT;
else if(storageCandidates.contains(FP32))
return FP32;
return FP64;
}
else
return method;
}
}
public static final byte STRING_ENCODING_ASCII = 0;
public static final byte STRING_ENCODING_UTF_16 = 1;
/**
* This method writes out a JSAT dataset to a binary format that can be read
* in again later, and could be read in other languages.
*
* The format that is used will understand both
* {@link ClassificationDataSet} and {@link RegressionDataSet} datasets as
* special cases, and will store the target values in the binary file. When
* read back in, they can be returned as their original dataset type, or
* treated as normal fields as a {@link SimpleDataSet}.
* The storage format chosen for floating point values will chose a method
* that results in no loss of precision when reading the data back in.
*
* @param
* @param dataset the dataset to write out to a binary file
* @param outRaw the raw output stream, the caller should provide a buffered
* stream.
* @throws IOException
*/
public static > void writeData(DataSet dataset, OutputStream outRaw) throws IOException
{
writeData(dataset, outRaw, FloatStorageMethod.AUTO);
}
/**
* This method writes out a JSAT dataset to a binary format that can be read
* in again later, and could be read in other languages.
*
* The format that is used will understand both
* {@link ClassificationDataSet} and {@link RegressionDataSet} datasets as
* special cases, and will store the target values in the binary file. When
* read back in, they can be returned as their original dataset type, or
* treated as normal fields as a {@link SimpleDataSet}.
*
* @param
* @param dataset the dataset to write out to a binary file
* @param outRaw the raw output stream, the caller should provide a buffered
* stream.
* @param fpStore the storage method of storing floating point values, which
* may result in a loss of precision depending on the method chosen.
* @throws IOException
*/
public static > void writeData(DataSet dataset, OutputStream outRaw, FloatStorageMethod fpStore) throws IOException
{
fpStore = FloatStorageMethod.getMethod(dataset, fpStore);
DataWriter.DataSetType type;
CategoricalData predicting;
if(dataset instanceof ClassificationDataSet)
{
type = DataWriter.DataSetType.CLASSIFICATION;
predicting = ((ClassificationDataSet)dataset).getPredicting();
}
else if(dataset instanceof RegressionDataSet)
{
type = DataWriter.DataSetType.REGRESSION;
predicting = null;
}
else
{
type = DataWriter.DataSetType.SIMPLE;
predicting = null;
}
DataWriter dw = getWriter(outRaw, dataset.getCategories(), dataset.getNumNumericalVars(), predicting, fpStore, type);
//write out all the datapoints
for(int i = 0; i < dataset.getSampleSize(); i++)
{
double label = 0;
if (dataset instanceof ClassificationDataSet)
label = ((ClassificationDataSet) dataset).getDataPointCategory(i);
else if (dataset instanceof RegressionDataSet)
label = ((RegressionDataSet) dataset).getTargetValue(i);
dw.writePoint(dataset.getDataPoint(i), label);
}
dw.finish();
outRaw.flush();
}
/**
* Returns a DataWriter object which can be used to stream a set of arbitrary datapoints into the given output stream. This works in a thread safe manner.
*
* @param out the location to store all the data
* @param catInfo information about the categorical features to be written
* @param dim information on how many numeric features exist
* @param predicting information on the class label, may be {@code null} if not a classification dataset
* @param fpStore the format floating point values should be stored as
* @param type what type of data set (simple, classification, regression) to be written
* @return the DataWriter that the actual points can be streamed through
* @throws IOException
*/
public static DataWriter getWriter(OutputStream out, CategoricalData[] catInfo, int dim, final CategoricalData predicting, final FloatStorageMethod fpStore, DataWriter.DataSetType type) throws IOException
{
return new DataWriter(out, catInfo, dim, type)
{
@Override
protected void writeHeader(CategoricalData[] catInfo, int dim, DataWriter.DataSetType type, OutputStream out) throws IOException
{
DataOutputStream data_out = new DataOutputStream(out);
data_out.write(JSATData.MAGIC_NUMBER);
int numNumeric = dim;
int numCat = catInfo.length;
DatasetTypeMarker marker = DatasetTypeMarker.STANDARD;
if(type == type.REGRESSION)
{
numNumeric++;
marker = DatasetTypeMarker.REGRESSION;
}
if(type == type.CLASSIFICATION)
{
numCat++;
marker = DatasetTypeMarker.CLASSIFICATION;
}
data_out.writeByte(marker.ordinal());
data_out.writeByte(fpStore.ordinal());
data_out.writeInt(numNumeric);
data_out.writeInt(numCat);
data_out.writeInt(-1);//-1 used to indicate a potentially variable number of files
for(CategoricalData category : catInfo)
{
//first, whats the name of the i'th category
writeString(category.getCategoryName(), data_out);
data_out.writeInt(category.getNumOfCategories());//output the number of categories
for(int i = 0; i < category.getNumOfCategories(); i++)//the option names
writeString(category.getOptionName(i), data_out);
}
//extra for classification dataset
if(type == DataWriter.DataSetType.CLASSIFICATION)
{
CategoricalData category = predicting;
//first, whats the name of the i'th category
writeString(category.getCategoryName(), data_out);
data_out.writeInt(category.getNumOfCategories());//output the number of categories
for(int i = 0; i < category.getNumOfCategories(); i++)//the option names
writeString(category.getOptionName(i), data_out);
}
data_out.flush();
}
@Override
protected void pointToBytes(DataPoint dp, double label, ByteArrayOutputStream byteOut)
{
try
{
DataOutputStream data_out = new DataOutputStream(byteOut);
fpStore.writeFP(dp.getWeight(), data_out);
for(int val : dp.getCategoricalValues())
data_out.writeInt(val);
if(type == DataWriter.DataSetType.CLASSIFICATION)
data_out.writeInt((int) label);
Vec numericVals = dp.getNumericalValues();
data_out.writeBoolean(numericVals.isSparse());
if(numericVals.isSparse())
{
if(type == DataWriter.DataSetType.REGRESSION)
data_out.writeInt(numericVals.nnz()+1);//+1 for the target value, which may actually be zero...
else
data_out.writeInt(numericVals.nnz());
for(IndexValue iv : numericVals)
{
data_out.writeInt(iv.getIndex());
fpStore.writeFP(iv.getValue(), data_out);
}
}
else
{
for(int j = 0; j < numericVals.length(); j++)
fpStore.writeFP(numericVals.get(j), data_out);
}
//append the target value
if(type == DataWriter.DataSetType.REGRESSION)
{
/*
* if dense, we only need to just add the extra double. If
* sparse, we do the index and then the double.
*/
if (numericVals.isSparse())
data_out.writeInt(numericVals.length());
fpStore.writeFP(label, data_out);
}
data_out.flush();
}
catch (IOException ex)
{
Logger.getLogger(JSATData.class.getName()).log(Level.SEVERE, null, ex);
}
}
};
}
/**
* This loads a JSAT dataset from an input stream, and will not do any of
* its own buffering. The DataSet will be returned as either a
* {@link SimpleDataSet}, {@link ClassificationDataSet}, or
* {@link RegressionDataSet} depending on what type of dataset was
* originally written out.
*
* @param
* @param inRaw the input stream, caller should buffer it
* @return a dataset
* @throws IOException
*/
public static DataSet> load(InputStream inRaw) throws IOException
{
return load(inRaw, false);
}
/**
* Loads in a JSAT dataset as a {@link SimpleDataSet}. So long as the input
* stream is valid, this will not fail.
*
* @param inRaw the input stream, caller should buffer it
* @return a SimpleDataSet object
* @throws IOException
*/
public static SimpleDataSet loadSimple(InputStream inRaw) throws IOException
{
return (SimpleDataSet) load(inRaw, true);
}
/**
* Loads in a JSAT dataset as a {@link ClassificationDataSet}. An exception
* will be thrown if the original dataset in the file was not a
* {@link ClassificationDataSet}.
*
* @param inRaw the input stream, caller should buffer it
* @return a ClassificationDataSet object
* @throws IOException
* @throws ClassCastException if the original dataset was a not a ClassificationDataSet
*/
public static ClassificationDataSet loadClassification(InputStream inRaw) throws IOException
{
return (ClassificationDataSet) load(inRaw);
}
/**
* Loads in a JSAT dataset as a {@link RegressionDataSet}. An exception
* will be thrown if the original dataset in the file was not a
* {@link RegressionDataSet}.
*
* @param inRaw the input stream, caller should buffer it
* @return a RegressionDataSet object
* @throws IOException
* @throws ClassCastException if the original dataset was a not a RegressionDataSet
*/
public static RegressionDataSet loadRegression(InputStream inRaw) throws IOException
{
return (RegressionDataSet) load(inRaw);
}
/**
* This loads a JSAT dataset from an input stream, and will not do any of
* its own buffering. The DataSet will be returned as either a
* {@link SimpleDataSet}, {@link ClassificationDataSet}, or
* {@link RegressionDataSet} depending on what type of dataset was
* originally written out.
*
* This method supports forcing the load to return a {@link SimpleDataSet}.
*
* @param
* @param inRaw the input stream, caller should buffer it
* @param forceAsStandard {@code true} for for the dataset to be loaded as a
* {@link SimpleDataSet}, otherwise it will be determined based on the input
* streams contents.
* @return a dataset
* @throws IOException
*/
@SuppressWarnings("unchecked")
protected static DataSet> load(InputStream inRaw, boolean forceAsStandard) throws IOException
{
DataInputStream in = new DataInputStream(inRaw);
byte[] magic_number = new byte[MAGIC_NUMBER.length];
in.readFully(magic_number);
String magic = new String(magic_number, "US-ASCII");
if(!magic.startsWith("JSAT_"))
throw new RuntimeException("data does not contain magic number");
DatasetTypeMarker marker = DatasetTypeMarker.values()[in.readByte()];
FloatStorageMethod fpStore = FloatStorageMethod.values()[in.readByte()];
int numNumeric = in.readInt();
int numCat = in.readInt();
int N = in.readInt();
if(forceAsStandard)
marker = DatasetTypeMarker.STANDARD;
if(marker == DatasetTypeMarker.CLASSIFICATION)
numCat--;
else if(marker == DatasetTypeMarker.REGRESSION)
numNumeric--;
CategoricalData[] categories = new CategoricalData[numCat];
CategoricalData predicting = null;//may not be used
for(int i = 0; i < categories.length; i++)
{
//first, whats the name of the i'th category
String name = readString(in);
int k = in.readInt();//output the number of categories
categories[i] = new CategoricalData(k);
categories[i].setCategoryName(name);
for(int j = 0; j < k; j++)//the option names
categories[i].setOptionName(readString(in), j);
}
if(marker == DatasetTypeMarker.CLASSIFICATION)
{
//first, whats the name of the i'th category
String name = readString(in);
int k = in.readInt();//output the number of categories
predicting = new CategoricalData(k);
predicting.setCategoryName(name);
for(int j = 0; j < k; j++)//the option names
predicting.setOptionName(readString(in), j);
}
DataSet> data;
switch(marker)
{
case CLASSIFICATION:
data = new ClassificationDataSet(numNumeric, categories, predicting);
break;
case REGRESSION:
data = new RegressionDataSet(numNumeric, categories);
break;
default:
data = new SimpleDataSet(categories, numNumeric);
}
//read in all the data points
if(N < 0)
N = Integer.MAX_VALUE;
try
{
for(int i = 0; i < N; i++)
{
double weight = fpStore.readFP(in);//in.readDouble();
int[] catVals = new int[numCat];
double target = 0;
for(int j = 0; j < catVals.length; j++)
catVals[j] = in.readInt();
if(marker == DatasetTypeMarker.CLASSIFICATION)
{
//int can be stored losselessly in a double, so this is safe
target = in.readInt();
}
boolean sparse = in.readBoolean();
Vec numericVals;
if(sparse)
{
int nnz = in.readInt();
if(marker == DatasetTypeMarker.REGRESSION)
nnz--;//don't count the target value
int[] indicies = new int[nnz];
double[] values = new double[nnz];
for(int j = 0; j < nnz; j++)
{
indicies[j] = in.readInt();
values[j] = fpStore.readFP(in);
}
numericVals = new SparseVector(indicies, values, numNumeric, nnz);
}
else
{
numericVals = new DenseVector(numNumeric);
for(int j = 0; j < numNumeric; j++)
numericVals.set(j, fpStore.readFP(in));
}
//get the target value
if(marker == DatasetTypeMarker.REGRESSION)
{
/*
* if dense, we only need to just add the extra double. If
* sparse, we do the index and then the double.
*/
if (numericVals.isSparse())
in.readInt();//don't care, its the last index value - so its the target
target = fpStore.readFP(in);
}
DataPoint dp = new DataPoint(numericVals, catVals, categories, weight);
switch(marker)
{
case CLASSIFICATION:
((ClassificationDataSet) data).addDataPoint(dp, (int) target);
break;
case REGRESSION:
((RegressionDataSet) data).addDataPoint(dp, target);
break;
default:
((SimpleDataSet) data).add(dp);
}
}
}
catch (EOFException eo)
{
//No problem
}
in.close();
return data;
}
private static void writeString(String s, DataOutputStream out) throws IOException
{
boolean isAscii = true;
for(int i = 0; i < s.length() && isAscii; i++)
if(s.charAt(i) >= 256 || s.charAt(i) <= 0)
isAscii = false;
if(isAscii)
{
out.writeByte(STRING_ENCODING_ASCII);
out.writeInt(s.length());//number of bytes of the string
for(int i = 0; i < s.length(); i++)
out.writeByte(s.charAt(i));
}
else//write as UTF-8
{
byte[] bytes = s.getBytes("UTF-16");
out.writeByte(STRING_ENCODING_UTF_16);
out.writeInt(bytes.length);//number of bytes of the string
out.write(bytes);
}
}
private static String readString(DataInputStream in) throws IOException
{
StringBuilder builder = new StringBuilder();
byte encoding = in.readByte();
int bytesToRead = in.readInt();
switch(encoding)
{
case STRING_ENCODING_ASCII:
for (int i = 0; i < bytesToRead; i++)
builder.append(Character.toChars(in.readByte()));
return builder.toString();
case STRING_ENCODING_UTF_16:
byte[] bytes = new byte[bytesToRead];
in.readFully(bytes);
return new String(bytes, "UTF-16");
default:
throw new RuntimeException("Unkown string encoding value " + encoding);
}
}
}
Ads help maintain this website.