package jsat.text;

import java.util.ArrayList;
import java.util.List;
import jsat.linear.SparseVector;
import jsat.linear.Vec;
import jsat.text.tokenizer.Tokenizer;
import jsat.text.wordweighting.BinaryWordPresent;
import jsat.text.wordweighting.WordWeighting;

/**
 * Hashed Text Vector Creator exists to convert a text string into a 
 * {@link Vec} using feature hashing. The {@link Tokenizer tokenization} and
 * {@link WordWeighting word weighting} method must be provided and already set
 * up. When constructed the user should make sure the 
 * {@link WordWeighting#setWeight(java.util.List, java.util.List) }
 * method has already been called, or is a stateless weighting (such as
 * {@link BinaryWordPresent}).
 *
 * @author Edward Raff
 */
public class HashedTextVectorCreator implements TextVectorCreator
{

    private static final long serialVersionUID = 1081388790985568192L;
    private int dimensionSize;
    private Tokenizer tokenizer;
    private WordWeighting weighting;

    /**
     * Creates a new text vector creator that works with hash-trick features
     * @param dimensionSize the dimension size of the feature space
     * @param tokenizer the tokenizer to apply to incoming strings
     * @param weighting the weighting process to apply to each loaded document. 
     */
    public HashedTextVectorCreator(int dimensionSize, Tokenizer tokenizer, WordWeighting weighting)
    {
        if(dimensionSize <= 1)
            throw new ArithmeticException("Vector dimension must be a positive value");
        this.dimensionSize = dimensionSize;
        this.tokenizer = tokenizer;
        this.weighting = weighting;
    }

    @Override
    public Vec newText(String input)
    {
        return newText(input, new StringBuilder(), new ArrayList());
    }

    @Override
    public Vec newText(String input, StringBuilder workSpace, List storageSpace)
    {
        tokenizer.tokenize(input, workSpace, storageSpace);
        SparseVector vec = new SparseVector(dimensionSize);
        for(String word : storageSpace)
        {
            //XXX This code generates a hashcode and then computes the absolute value of that hashcode. If the hashcode is Integer.MIN_VALUE, then the result will be negative as well (since Math.abs(Integer.MIN_VALUE) == Integer.MIN_VALUE). 
            vec.increment(Math.abs(word.hashCode())%dimensionSize, 1.0);
        }
        weighting.applyTo(vec);
        return vec;
    }
}

 

Ads help maintain this website.