package jsat.text.wordweighting;

import static java.lang.Math.log;
import java.util.List;
import jsat.linear.Vec;

/**
 * Applies Term Frequency Inverse Document Frequency (TF IDF) weighting to the 
 * word vectors. 
 * 
 * @author Edward Raff
 */
public class TfIdf implements WordWeighting
{

    private static final long serialVersionUID = 5749882005002311735L;

    public enum TermFrequencyWeight
    {
        /**
         * BOOLEAN only takes into account whether or not the word is present in
         * the document. 
* 1.0 if the count is non zero. */ BOOLEAN, /** * LOG returns a term weighting in [1, infinity) based on the log value * of the term frequency
* 1 + log(count) */ LOG, /** * DOC_NORMALIZED returns a term weighting in [0, 1] by normalizing the * frequency by the most common word in the document.
* count/(most Frequent word in document) * */ DOC_NORMALIZED; } private double totalDocuments; private List df; private double docMax = 0.0; private TermFrequencyWeight tfWeighting; /** * Creates a new TF-IDF document weighting scheme that uses * {@link TermFrequencyWeight#LOG LOG} weighting for term frequency. */ public TfIdf() { this(TermFrequencyWeight.LOG); } /** * Creates a new TF-IDF document weighting scheme that uses the specified * term frequency weighting * @param tfWeighting the weighting method to use for the term frequency * (tf) component */ public TfIdf(TermFrequencyWeight tfWeighting) { this.tfWeighting = tfWeighting; } @Override public void setWeight(List allDocuments, List df) { this.totalDocuments = allDocuments.size(); this.df = df; } @Override public double indexFunc(double value, int index) { if (index < 0 || value == 0.0) return 0.0; double tf;// = 1+log(value); switch(tfWeighting) { case BOOLEAN: tf = 1.0; break; case LOG: tf = 1+log(value); break; case DOC_NORMALIZED: tf = value/docMax; break; default: tf = value; } double idf = log(totalDocuments / df.get(index)); return tf * idf; } @Override public void applyTo(Vec vec) { if(df == null) throw new RuntimeException("TF-IDF weightings haven't been initialized, setWeight method must be called before first use."); if(tfWeighting == TermFrequencyWeight.DOC_NORMALIZED) docMax = vec.max(); vec.applyIndexFunction(this); } }

Ads help maintain this website.