package jsat.text;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import jsat.linear.SparseVector;
import jsat.linear.Vec;
import jsat.text.tokenizer.Tokenizer;
import jsat.text.wordweighting.WordWeighting;
/**
* Creates new text vectors from a dictionary of known tokens and a word
* weighting scheme.
*
* This object is generally intended to be constructed by a
* {@link TextDataLoader}, though can be used if you know all the words you will
* need (and can initialize the {@link WordWeighting}) before creating this
* object.
*
* @author Edward Raff
*/
public class BasicTextVectorCreator implements TextVectorCreator
{
private static final long serialVersionUID = -8620485679300539556L;
private final Tokenizer tokenizer;
private final Map wordIndex;
private final WordWeighting weighting;
/**
* Creates a new basic text vector creator
* @param tokenizer the tokenizer to apply to incoming strings
* @param wordIndex the map of each known word to its index, the size of the
* map indicating the maximum (exclusive) index
* @param weighting the weighting process to apply to each loaded document.
* This should have already been initialized, or be stateless.
*/
public BasicTextVectorCreator(Tokenizer tokenizer, Map wordIndex, WordWeighting weighting)
{
this.tokenizer = tokenizer;
this.wordIndex = wordIndex;
this.weighting = weighting;
}
@Override
public Vec newText(String text)
{
return newText(text, new StringBuilder(), new ArrayList());
}
@Override
public Vec newText(String input, StringBuilder workSpace, List storageSpace)
{
tokenizer.tokenize(input, workSpace, storageSpace);
SparseVector vec = new SparseVector(wordIndex.size());
for( String word : storageSpace)
{
if(wordIndex.containsKey(word))//Could also call retainAll on words before looping. Worth while to investigate
{
int index = wordIndex.get(word);
vec.increment(index, 1.0);
}
}
weighting.applyTo(vec);
return vec;
}
}
Ads help maintain this website.