[Home] DMelt member access. Java source code: jsat.text.TextDataLoader
package jsat.text;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import jsat.DataSet;
import jsat.SimpleDataSet;
import jsat.classifiers.CategoricalData;
import jsat.classifiers.DataPoint;
import jsat.datatransform.RemoveAttributeTransform;
import jsat.linear.SparseVector;
import jsat.linear.Vec;
import jsat.text.tokenizer.Tokenizer;
import jsat.text.wordweighting.WordWeighting;
import jsat.utils.IntList;
import jsat.utils.IntSet;
/**
* This class provides a framework for loading datasets made of Text documents
* as vectors. Text is broken up into a sequence of tokens using a
* {@link Tokenizer}, that must be provided. The weights used will be determined
* by some {@link WordWeighting word weighting scheme}.
* The user adds documents to the initial dataset using the {@link #addOriginalDocument(java.lang.String)
* } method. The {@link #finishAdding() } must be called when no more documents
* are left to add, at which point class will take care of calling the {@link WordWeighting#setWeight(java.util.List, java.util.List)
* } method to configure the word weighting used with the original data
* added.
*
* After the initial dataset is loaded, new strings can be converted to vectors
* using the {@link #newText(java.lang.String) } method. This should only be
* called after {@link #finishAdding() }.
*
* Instance of this class will keep a reference to all originally added vectors.
* To transform new texts into vectors without keeping references to all of the
* original vectors, the {@link #getTextVectorCreator() } will return an object
* that perform the transformation.
*
* @author Edward Raff
*/
public abstract class TextDataLoader implements TextVectorCreator
{
private static final long serialVersionUID = -657253682338792871L;
/**
* List of original vectors
*/
protected final List vectors;
/**
* Tokenizer to apply to input strings
*/
protected Tokenizer tokenizer;
/**
* Maps words to their associated index in an array
*/
protected ConcurrentHashMap wordIndex;
/**
* list of all word tokens encountered in order of first observation
*/
protected List allWords;
/**
* The map of integer counts of how many times each word token was seen. Key
* is the index of the word, value is the number of times it was seen. Using
* a map instead of a list so that it can be updated in a efficient thread
* safe way
*/
protected ConcurrentHashMap termDocumentFrequencys;
private WordWeighting weighting;
/**
* Temporary work space to use for tokenization
*/
protected ThreadLocal workSpace;
/**
* Temporary storage space to use for tokenization
*/
protected ThreadLocal> storageSpace;
/**
* Temporary space to use when creating vectors
*/
protected ThreadLocal
Ads help maintain this website.