[Home] DMelt member access . Java source code: com.datumbox.framework.core.machinelearning.modelselection.metrics.ClassificationMetrics
/**
* Copyright (C) 2013-2017 Vasilis Vryniotis
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.datumbox.framework.core.machinelearning.modelselection.metrics;
import com.datumbox.framework.core.common.dataobjects.Dataframe;
import com.datumbox.framework.core.common.dataobjects.Record;
import com.datumbox.framework.core.machinelearning.common.abstracts.modelselection.AbstractMetrics;
import java.util.*;
/**
* Estimates validation metrics for Classifiers.
*
* @author Vasilis Vryniotis
*/
public class ClassificationMetrics extends AbstractMetrics {
private static final long serialVersionUID = 1L;
/**
* Enum that stores the 4 possible Sensitivity Rates.
*/
public enum SensitivityRates {
/**
* True Positive.
*/
TRUE_POSITIVE,
/**
* True Negative.
*/
TRUE_NEGATIVE,
/**
* False Positive.
*/
FALSE_POSITIVE,
/**
* False Negative.
*/
FALSE_NEGATIVE;
}
//validation metrics
private double accuracy = 0.0;
private double macroPrecision = 0.0;
private double macroRecall = 0.0;
private double macroF1 = 0.0;
private final Map microPrecision = new HashMap<>(); //this is small. Size equal to 4*class numbers
private final Map microRecall = new HashMap<>(); //this is small. Size equal to 4*class numbers
private final Map microF1 = new HashMap<>(); //this is small. Size equal to 4*class numbers
private final Map, Double> contingencyTable = new HashMap<>(); //this is small. Size equal to 4*class numbers
/**
* Getter for Accuracy.
*
* @return
*/
public double getAccuracy() {
return accuracy;
}
/**
* Getter for Macro Precision.
*
* @return
*/
public double getMacroPrecision() {
return macroPrecision;
}
/**
* Getter for Macro Recall.
*
* @return
*/
public double getMacroRecall() {
return macroRecall;
}
/**
* Getter for Macro F1.
*
* @return
*/
public double getMacroF1() {
return macroF1;
}
/**
* Getter for Micro Precision.
*
* @return
*/
public Map getMicroPrecision() {
return microPrecision;
}
/**
* Getter for Micro Recall.
*
* @return
*/
public Map getMicroRecall() {
return microRecall;
}
/**
* Getter for Micro F1.
*
* @return
*/
public Map getMicroF1() {
return microF1;
}
/**
* Getter for Contingency Table.
*
* @return
*/
public Map, Double> getContingencyTable() {
return contingencyTable;
}
/**
* @param predictedData
* @see AbstractMetrics#AbstractMetrics(Dataframe)
*/
public ClassificationMetrics(Dataframe predictedData) {
super(predictedData);
//retrieve the classes from the dataset
Set classesSet = new HashSet<>();
for(Record r : predictedData) {
classesSet.add(r.getY());
classesSet.add(r.getYPredicted());
}
for(Object theClass : classesSet) {
contingencyTable.put(Arrays.asList(theClass, SensitivityRates.TRUE_POSITIVE), 0.0); //true possitive
contingencyTable.put(Arrays.asList(theClass, SensitivityRates.FALSE_POSITIVE), 0.0); //false possitive
contingencyTable.put(Arrays.asList(theClass, SensitivityRates.TRUE_NEGATIVE), 0.0); //true negative
contingencyTable.put(Arrays.asList(theClass, SensitivityRates.FALSE_NEGATIVE), 0.0); //false negative
}
int n = predictedData.size();
int c = classesSet.size();
int correctCount=0;
for(Record r : predictedData) {
Object yPred = r.getYPredicted();
if(yPred.equals(r.getY())) {
++correctCount;
for(Object cl : classesSet) {
if(cl.equals(yPred)) {
List tpk = Arrays.asList(cl, SensitivityRates.TRUE_POSITIVE);
contingencyTable.put(tpk, contingencyTable.get(tpk) + 1.0);
}
else {
List tpk = Arrays.asList(cl, SensitivityRates.TRUE_NEGATIVE);
contingencyTable.put(tpk, contingencyTable.get(tpk) + 1.0);
}
}
}
else {
for(Object cl : classesSet) {
if(cl.equals(yPred)) {
List tpk = Arrays.asList(cl, SensitivityRates.FALSE_POSITIVE);
contingencyTable.put(tpk, contingencyTable.get(tpk) + 1.0);
}
else if(cl.equals(r.getY())) {
List tpk = Arrays.asList(cl, SensitivityRates.FALSE_NEGATIVE);
contingencyTable.put(tpk, contingencyTable.get(tpk) + 1.0);
}
else {
List tpk = Arrays.asList(cl, SensitivityRates.TRUE_NEGATIVE);
contingencyTable.put(tpk, contingencyTable.get(tpk) + 1.0);
}
}
}
}
accuracy = correctCount/(double)n;
//Average Precision, Recall and F1: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.104.8244&rep=rep1&type=pdf
int activeClasses = c;
for(Object theClass : classesSet) {
double tp = contingencyTable.get(Arrays.asList(theClass, SensitivityRates.TRUE_POSITIVE));
double fp = contingencyTable.get(Arrays.asList(theClass, SensitivityRates.FALSE_POSITIVE));
double fn = contingencyTable.get(Arrays.asList(theClass, SensitivityRates.FALSE_NEGATIVE));
double classPrecision=0.0;
double classRecall=0.0;
double classF1=0.0;
if(tp>0.0) {
classPrecision = tp/(tp+fp);
classRecall = tp/(tp+fn);
classF1 = 2.0*classPrecision*classRecall/(classPrecision+classRecall);
}
else if(tp==0.0 && fp==0.0 && fn==0.0) {
//if this category did not appear in the dataset reduce the number of classes
--activeClasses;
}
microPrecision.put(theClass, classPrecision);
microRecall.put(theClass, classRecall);
microF1.put(theClass, classF1);
macroPrecision += classPrecision;
macroRecall += classRecall;
macroF1 += classF1;
}
macroPrecision /= activeClasses;
macroRecall /= activeClasses;
macroF1 /= activeClasses;
}
/**
* @param validationMetricsList
* @see AbstractMetrics#AbstractMetrics(List)
*/
public ClassificationMetrics(List validationMetricsList) {
super(validationMetricsList);
int k = validationMetricsList.size(); //number of samples
for(ClassificationMetrics vmSample : validationMetricsList) {
//fetch the classes from the keys of one of the micro metrics. This way if a class is not included in a fold, we don't get null exceptions
Set classesSet = vmSample.getMicroPrecision().keySet();
for(Object theClass : classesSet) {
Map, Double> ctEntryMap = vmSample.getContingencyTable();
//get the values of all SensitivityRates and average them
for(SensitivityRates sr : SensitivityRates.values()) {
List tpk = Arrays.asList(theClass, sr);
Double previousValue = contingencyTable.getOrDefault(tpk, 0.0);
contingencyTable.put(tpk, previousValue + ctEntryMap.get(tpk)/k);
}
//update micro metrics of class
Double previousPrecision = microPrecision.getOrDefault(theClass, 0.0);
microPrecision.put(theClass, previousPrecision + vmSample.getMicroPrecision().get(theClass)/k);
Double previousRecall = microRecall.getOrDefault(theClass, 0.0);
microRecall.put(theClass, previousRecall + vmSample.getMicroRecall().get(theClass)/k);
Double previousF1 = microF1.getOrDefault(theClass, 0.0);
microF1.put(theClass, previousF1 + vmSample.getMicroF1().get(theClass)/k);
}
//update macro metrics
accuracy += vmSample.getAccuracy()/k;
macroPrecision += vmSample.getMacroPrecision()/k;
macroRecall += vmSample.getMacroRecall()/k;
macroF1 += vmSample.getMacroF1()/k;
}
}
/** {@inheritDoc} */
@Override
public String toString() {
String sep = System.lineSeparator();
StringBuilder sb = new StringBuilder();
sb.append(getClass().getSimpleName()).append(":").append(sep);
sb.append("accuracy=").append(accuracy).append(sep);
sb.append("macroPrecision=").append(macroPrecision).append(sep);
sb.append("macroRecall=").append(macroRecall).append(sep);
sb.append("macroF1=").append(macroF1).append(sep);
sb.append("microPrecision=").append(microPrecision).append(sep);
sb.append("microRecall=").append(microRecall).append(sep);
sb.append("microF1=").append(microF1).append(sep);
sb.append("contingencyTable=").append(contingencyTable).append(sep);
return sb.toString();
}
}
Ads help maintain this website.