package jsat.datatransform;

import java.util.ArrayList;
import java.util.List;
import jsat.DataSet;
import jsat.classifiers.DataPoint;
import jsat.linear.Vec;
import jsat.parameters.Parameter;
import jsat.parameters.Parameter.ParameterHolder;
import jsat.parameters.Parameterized;

/**
 * Performing a transform on the whole data set before training a classifier can
 * add bias to the results. For proper evaluation, the transforms must be 
 * learned from the training set and not contain any knowledge from the testing 
 * set. A DataTransformProcess aids in this by providing a mechanism to contain 
 * several different transforms to learn and then apply. 
 * 

* The Parameters of the Data Transform Process are the parameters from the * individual transform factories that make up the whole process. The name * "DataTransformProcess" will not be prefixed to the parameter names. * * @author Edward Raff */ public class DataTransformProcess implements DataTransform, Parameterized { private static final long serialVersionUID = -2844495690944305885L; @ParameterHolder(skipSelfNamePrefix = true) private List transformSource; private List learnedTransforms; /** * Creates a new transform process that is empty. Transform factories must * be added using * {@link #addTransform(jsat.datatransform.DataTransformFactory) }. */ public DataTransformProcess() { transformSource = new ArrayList(); learnedTransforms = new ArrayList(); } /** * Creates a new transform process from the listed factories, which will be * applied in order by index. * * @param transforms the array of factories to apply as the data transform process */ public DataTransformProcess(DataTransform... transforms) { this(); for(DataTransform dt : transforms) this.addTransform(dt); } /** * Adds a transform to the list of transforms. Transforms are learned and * applied in the order in which they are added. * @param transform the factory for the transform to add */ public void addTransform(DataTransform transform) { transformSource.add(transform); } /** * * @return the number of transforms currently chained in this transform * process */ public int getNumberOfTransforms() { return transformSource.size(); } /** * Consolidates transformation objects when possible. Currently only works with {@link RemoveAttributeTransform} */ private void consolidateTransforms() { for(int i = 0; i < learnedTransforms.size()-1; i++) { DataTransform t1 = learnedTransforms.get(i); DataTransform t2 = learnedTransforms.get(i+1); if(!(t1 instanceof RemoveAttributeTransform && t2 instanceof RemoveAttributeTransform)) continue;//They are not both RATs RemoveAttributeTransform r1 = (RemoveAttributeTransform) t1; RemoveAttributeTransform r2 = (RemoveAttributeTransform) t2; r2.consolidate(r1); learnedTransforms.remove(i); i--; } } @Override public void fit(DataSet data) { learnApplyTransforms(data); } /** * Learns the transforms for the given data set. The data set will not be * altered. Once finished, this DataTransformProcess can be applied * to the dataSet to get the transformed data set. * * @param dataSet the data set to learn a series of transforms from */ public void leanTransforms(DataSet dataSet) { learnApplyTransforms(dataSet.shallowClone()); } /** * Learns the transforms for the given data set. The data set is then * altered after each transform is learned so the next transform can be * learned as well.
The results are equivalent to calling * {@link #learnApplyTransforms(jsat.DataSet) } on the data set and then * calling {@link DataSet#applyTransform(jsat.datatransform.DataTransform) } * with this DataTransformProces. * * @param dataSet the data set to learn a series of transforms from and * alter into the final transformed form */ public void learnApplyTransforms(DataSet dataSet) { learnedTransforms.clear(); //used to keep track if we can start using in place transforms boolean vecSafe = false; boolean catSafe = false; int iter = 0; //copy original references so we can check saftey of inplace mutation later Vec[] origVecs = new Vec[dataSet.getSampleSize()]; int[][] origCats = new int[dataSet.getSampleSize()][]; for (int i = 0; i < origVecs.length; i++) { DataPoint dp = dataSet.getDataPoint(i); origVecs[i] = dp.getNumericalValues(); origCats[i] = dp.getCategoricalValues(); } for (DataTransform dtf : transformSource) { DataTransform transform = dtf.clone(); transform.fit(dataSet); if(transform instanceof InPlaceTransform) { InPlaceTransform ipt = (InPlaceTransform) transform; //check if it is safe to apply mutations if(iter > 0 && !vecSafe || (ipt.mutatesNominal() && !catSafe)) { boolean vecClear = true, catClear = true; for (int i = 0; i < origVecs.length && (vecClear || catClear); i++) { DataPoint dp = dataSet.getDataPoint(i); vecClear = origVecs[i] != dp.getNumericalValues(); catClear = origCats[i] != dp.getCategoricalValues(); } vecSafe = vecClear; catSafe = catClear; } //Now we know if we can apply the mutations or not if(vecSafe && (!ipt.mutatesNominal() || catSafe)) dataSet.applyTransform(ipt, true); else//go back to normal dataSet.applyTransform(transform); } else dataSet.applyTransform(transform); learnedTransforms.add(transform); iter++; } consolidateTransforms(); } @Override public DataPoint transform(DataPoint dp) { final Vec origNum = dp.getNumericalValues(); final int[] origCat = dp.getCategoricalValues(); for(DataTransform dt : learnedTransforms) { if(dt instanceof InPlaceTransform) { InPlaceTransform it = (InPlaceTransform) dt; //check if we can safley mutableTransform instead of allocate if(origNum != dp.getNumericalValues() && (!it.mutatesNominal() || origCat != dp.getCategoricalValues())) { it.mutableTransform(dp); continue; } } dp = dt.transform(dp); } return dp; } @Override public DataTransformProcess clone() { DataTransformProcess clone = new DataTransformProcess(); for(DataTransform dtf : this.transformSource) clone.transformSource.add(dtf.clone()); for(DataTransform dt : this.learnedTransforms) clone.learnedTransforms.add(dt.clone()); return clone; } }

Ads help maintain this website.