/*
* Copyright (C) 2008-2015 by Holger Arndt
*
* This file is part of the Universal Java Matrix Package (UJMP).
* See the NOTICE file distributed with this work for additional
* information regarding copyright ownership and licensing.
*
* UJMP is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* UJMP is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with UJMP; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301 USA
*/
package org.ujmp.core.doublematrix.calculation.general.misc;
import org.ujmp.core.Matrix;
import org.ujmp.core.doublematrix.calculation.AbstractDoubleCalculation;
import org.ujmp.core.longmatrix.calculation.DocTerm;
import org.ujmp.core.util.MathUtil;
public class TfIdf extends AbstractDoubleCalculation {
private static final long serialVersionUID = 4262822624560201379L;
/**
* matrix with [documents x terms]
*/
private Matrix docTerm = null;
private Matrix sumPerDoc = null;
private Matrix sumPerTerm = null;
private boolean calculateTf = false;
private boolean calculateIdf = false;
private boolean normalize = false;
public TfIdf(Matrix matrix, boolean calculateTf, boolean calculateIdf, boolean normalize) {
super(matrix);
this.calculateTf = calculateTf;
this.calculateIdf = calculateIdf;
this.normalize = normalize;
if (normalize) {
throw new RuntimeException("not yet implemented");
}
}
public double getDouble(long... coordinates) {
if (docTerm == null) {
calculate();
}
double tf = docTerm.getAsDouble(coordinates);
double idf = 1.0;
double numDocs = docTerm.getRowCount();
if (calculateTf) {
tf = docTerm.getAsDouble(coordinates) / sumPerDoc.getAsDouble(coordinates[ROW], 0);
}
if (calculateIdf) {
idf = MathUtil.log10(numDocs / sumPerTerm.getAsDouble(0, coordinates[COLUMN]));
}
double result = tf * idf;
return MathUtil.isNaNOrInfinite(result) ? 0.0 : result;
}
private void calculate() {
docTerm = new DocTerm(getSource()).calcNew();
if (calculateTf) {
sumPerDoc = docTerm.sum(Ret.NEW, Matrix.COLUMN, true);
}
if (calculateIdf) {
sumPerTerm = docTerm.toBooleanMatrix().sum(Ret.NEW, Matrix.ROW, true);
}
}
public long[] getSize() {
if (docTerm == null) {
calculate();
}
return docTerm.getSize();
}
}
Ads help maintain this website.