package jsat.datatransform;
import java.util.*;
import jsat.DataSet;
import jsat.classifiers.CategoricalData;
import jsat.classifiers.DataPoint;
import jsat.linear.*;
import jsat.utils.IntList;
import jsat.utils.IntSet;
/**
* This Data Transform allows the complete removal of specific features from the
* data set.
*
* @author Edward Raff
*/
public class RemoveAttributeTransform implements DataTransform
{
private static final long serialVersionUID = 2803223213862922734L;
/*
* Each index map maps the old indecies in the original data set to their
* new positions. The value in the array is old index, the index of the
* value is the index it would be when the attributes were removed.
* This means each is in sorted order, and is of the size of the resulting
* feature space
*/
protected int[] catIndexMap;
protected int[] numIndexMap;
protected CategoricalData[] newCatHeader;
private Set categoricalToRemove;
private Set numericalToRemove;
/**
* Empty constructor that may be used by extending classes. Transforms that
* extend this will need to call
* {@link #setUp(jsat.DataSet, java.util.Set, java.util.Set) } once the
* attributes to remove have been selected
*/
protected RemoveAttributeTransform()
{
}
/**
* Creates a new transform for removing specified features from a data set.
* Needs to still call {@link #fit(jsat.DataSet) } before ready to be used.
*
* @param categoricalToRemove the set of categorical attributes to remove,
* in the rage of [0, {@link DataSet#getNumCategoricalVars() }).
* @param numericalToRemove the set of numerical attributes to remove, in
* the rage of [0, {@link DataSet#getNumNumericalVars() }).
*/
public RemoveAttributeTransform(Set categoricalToRemove, Set numericalToRemove)
{
this.categoricalToRemove = categoricalToRemove;
this.numericalToRemove = numericalToRemove;
}
/**
* Creates a new transform for removing specified features from a data set
* @param dataSet the data set that this transform is meant for
* @param categoricalToRemove the set of categorical attributes to remove, in the rage of [0, {@link DataSet#getNumCategoricalVars() }).
* @param numericalToRemove the set of numerical attributes to remove, in the rage of [0, {@link DataSet#getNumNumericalVars() }).
*/
public RemoveAttributeTransform(DataSet dataSet, Set categoricalToRemove, Set numericalToRemove)
{
this.categoricalToRemove = categoricalToRemove;
this.numericalToRemove = numericalToRemove;
setUp(dataSet, categoricalToRemove, numericalToRemove);
}
/**
* Returns an unmodifiable list of the original indices of the numeric
* attributes that will be kept when this transform is applied.
* @return the numeric indices that are not removed by this transform
*/
public List getKeptNumeric()
{
return IntList.unmodifiableView(numIndexMap, numIndexMap.length);
}
/**
* Returns a mapping from the numeric indices in the transformed space back
* to their original indices
*
* @return a mapping from the transformed numeric space to the original one
*/
public Map getReverseNumericMap()
{
Map map = new HashMap();
for(int newIndex = 0; newIndex < numIndexMap.length; newIndex++)
map.put(newIndex, numIndexMap[newIndex]);
return map;
}
/**
* Returns an unmodifiable list of the original indices of the nominal
* attributes that will be kept when this transform is applied.
* @return the nominal indices that are not removed by this transform
*/
public List getKeptNominal()
{
return IntList.unmodifiableView(catIndexMap, catIndexMap.length);
}
/**
* Returns a mapping from the nominal indices in the transformed space back
* to their original indices
*
* @return a mapping from the transformed nominal space to the original one
*/
public Map getReverseNominalMap()
{
Map map = new HashMap();
for(int newIndex = 0; newIndex < catIndexMap.length; newIndex++)
map.put(newIndex, catIndexMap[newIndex]);
return map;
}
@Override
public void fit(DataSet data)
{
if (catIndexMap == null || numIndexMap == null)
setUp(data, categoricalToRemove, numericalToRemove);
}
/**
* Sets up the Remove Attribute Transform properly
*
* @param dataSet the data set to remove the attributes from
* @param categoricalToRemove the categorical attributes to remove
* @param numericalToRemove the numeric attributes to remove
*/
protected final void setUp(DataSet dataSet, Set categoricalToRemove, Set numericalToRemove)
{
for(int i : categoricalToRemove)
if (i >= dataSet.getNumCategoricalVars())
throw new RuntimeException("The data set does not have a categorical value " + i + " to remove");
for(int i : numericalToRemove)
if (i >= dataSet.getNumNumericalVars())
throw new RuntimeException("The data set does not have a numercal value " + i + " to remove");
catIndexMap = new int[dataSet.getNumCategoricalVars()-categoricalToRemove.size()];
newCatHeader = new CategoricalData[catIndexMap.length];
numIndexMap = new int[dataSet.getNumNumericalVars()-numericalToRemove.size()];
int k = 0;
for(int i = 0; i < dataSet.getNumCategoricalVars(); i++)
{
if(categoricalToRemove.contains(i))
continue;
newCatHeader[k] = dataSet.getCategories()[i].clone();
catIndexMap[k++] = i;
}
k = 0;
for(int i = 0; i < dataSet.getNumNumericalVars(); i++)
{
if(numericalToRemove.contains(i))
continue;
numIndexMap[k++] = i;
}
}
/**
* Copy constructor
* @param other the transform to copy
*/
protected RemoveAttributeTransform(RemoveAttributeTransform other)
{
if(other.categoricalToRemove != null)
this.categoricalToRemove = new IntSet(other.categoricalToRemove);
if(other.numericalToRemove != null)
this.numericalToRemove = new IntSet(other.numericalToRemove);
if(other.catIndexMap != null)
this.catIndexMap = Arrays.copyOf(other.catIndexMap, other.catIndexMap.length);
if(other.numIndexMap != null)
this.numIndexMap = Arrays.copyOf(other.numIndexMap, other.numIndexMap.length);
if(other.newCatHeader != null)
{
this.newCatHeader = new CategoricalData[other.newCatHeader.length];
for(int i = 0; i < this.newCatHeader.length; i++)
this.newCatHeader[i] = other.newCatHeader[i].clone();
}
}
/**
* A serious of Remove Attribute Transforms may be learned and applied
* sequentially to a single data set. Instead of keeping all the transforms
* around indefinitely, a sequential series of Remove Attribute Transforms
* can be consolidated into a single transform object.
* This method mutates the this transform by providing it with the
* transform that would have been applied before this current object. Once
* complete, this transform can be used two perform both removals in one
* step.
* Example:
* An initial set of features A is transformed into A' by
* transform t1
* A' is transformed into A'' by transform t2
* Instead, you can invoke t2.consolidate(t1).
* You can then transform A into A'' by using only transform
* t2
*
*
* @param preceding the DataTransform that immediately precedes this one in
* a sequential list of transforms
*/
public void consolidate(RemoveAttributeTransform preceding)
{
for(int i = 0; i < catIndexMap.length; i++)
catIndexMap[i] = preceding.catIndexMap[catIndexMap[i]];
for(int i = 0; i < numIndexMap.length; i++)
numIndexMap[i] = preceding.numIndexMap[numIndexMap[i]];
}
@Override
public DataPoint transform(DataPoint dp)
{
int[] catVals = dp.getCategoricalValues();
Vec numVals = dp.getNumericalValues();
CategoricalData[] newCatData = new CategoricalData[catIndexMap.length];
int[] newCatVals = new int[newCatData.length];
Vec newNumVals;
if (numVals.isSparse())
if (numVals instanceof SparseVector)
newNumVals = new SparseVector(numIndexMap.length, ((SparseVector) numVals).nnz());
else
newNumVals = new SparseVector(numIndexMap.length);
else
newNumVals = new DenseVector(numIndexMap.length);
for(int i = 0; i < catIndexMap.length; i++)
{
newCatVals[i] = catVals[catIndexMap[i]];
newCatData[i] = dp.getCategoricalData()[catIndexMap[i]];
}
int k = 0;
Iterator iter = numVals.getNonZeroIterator();
if (iter.hasNext())//if all values are zero, nothing to do
{
IndexValue curIV = iter.next();
for (int i = 0; i < numIndexMap.length; i++)//i is the old index
{
if (numVals.isSparse())//log(n) insert and loopups to avoid!
{
if (curIV == null)
continue;
if (numIndexMap[i] > curIV.getIndex())//We skipped a value that existed
while (numIndexMap[i] > curIV.getIndex() && iter.hasNext())
curIV = iter.next();
if (numIndexMap[i] < curIV.getIndex())//Index is zero, nothing to set
continue;
else if (numIndexMap[i] == curIV.getIndex())
{
newNumVals.set(i, curIV.getValue());
if (iter.hasNext())
curIV = iter.next();
else
curIV = null;
}
}
else//All dense, just set them all
newNumVals.set(i, numVals.get(numIndexMap[i]));
}
}
return new DataPoint(newNumVals, newCatVals, newCatData, dp.getWeight());
}
@Override
public RemoveAttributeTransform clone()
{
return new RemoveAttributeTransform(this);
}
}
Ads help maintain this website.