package jsat;

import java.io.*;
import java.util.*;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import jsat.classifiers.*;
import jsat.linear.DenseVector;
import jsat.linear.Vec;
import jsat.regression.RegressionDataSet;

/**
 * Class for loading ARFF files. ARFF is a human readable file format used by
 * Weka. The ARFF file formal allows for attributes that have missing
 * information, which are encoded in JSAT as {@link Double#NaN} for numeric
 * features, and a {@code -1} for categorical features.
 *
 * 
* About Weka * @author Edward Raff */ public class ARFFLoader { /** * Uses the given file path to load a data set from an ARFF file. * * @param file the path to the ARFF file to load * @return the data set from the ARFF file, or null if the file could not be loaded. */ public static SimpleDataSet loadArffFile(File file) { try { return loadArffFile(new FileReader(file)); } catch (FileNotFoundException ex) { Logger.getLogger(ARFFLoader.class.getName()).log(Level.SEVERE, null, ex); return null; } } /** * Uses the given reader to load a data set assuming it follows the ARFF * file format * @param input the reader to load the data set from * @return the data set from the stream, or null of the file could not be loaded */ public static SimpleDataSet loadArffFile(Reader input) { ArrayList list = new ArrayList(); BufferedReader br = new BufferedReader(input); int numOfVars = 0; int numReal = 0; List isReal = new ArrayList(); List variableNames = new ArrayList(); List> catVals = new ArrayList>(); String line = null; CategoricalData[] categoricalData = null; try { boolean atData = false; while( (line = br.readLine()) != null ) { if(line.startsWith("%") || line.trim().isEmpty()) continue;///Its a comment, skip line = line.trim(); if(line.startsWith("@") && !atData) { line = line.substring(1).toLowerCase(); if(line.toLowerCase().startsWith("data")) { categoricalData = new CategoricalData[numOfVars-numReal]; int k = 0; for(int i = 0; i < catVals.size(); i++) { if(catVals.get(i) != null) { categoricalData[k] = new CategoricalData(catVals.get(i).size()); categoricalData[k].setCategoryName(variableNames.get(i)); for(Entry entry : catVals.get(i).entrySet()) categoricalData[k].setOptionName(entry.getKey(), entry.getValue()); k++; } } atData = true; continue; } else if(!line.toLowerCase().startsWith("attribute")) continue; numOfVars++; line = line.substring("attribute".length()).trim();//Remove the space, it could be multiple spaces String variableName = null; line = line.replace("\t", " "); if(line.startsWith("'")) { Pattern p = Pattern.compile("'.+?'"); Matcher m = p.matcher(line); m.find(); variableName = nameTrim(m.group()); line = line.replaceFirst("'.+?'", "placeHolder"); } else variableName = nameTrim(line.trim().replaceAll("\\s+.*", "")); variableNames.add(variableName); String[] tmp = line.split("\\s+", 2); if(tmp[1].trim().equals("real") || tmp[1].trim().equals("numeric") || tmp[1].trim().startsWith("integer")) { numReal++; isReal.add(true); catVals.add(null); } else//Not correct, but we arent supporting anything other than real and categorical right now { isReal.add(false); String cats = tmp[1].replace("{", "").replace("}", "").trim(); if(cats.endsWith(",")) cats = cats.substring(0, cats.length()-1); String[] catValsRaw = cats.split(","); HashMap tempMap = new HashMap(); for(int i = 0; i < catValsRaw.length; i++) { catValsRaw[i] = nameTrim(catValsRaw[i]); tempMap.put(catValsRaw[i], i); } catVals.add(tempMap); } } else if(atData && !line.isEmpty()) { double weight = 1.0; String[] tmp = line.split(","); if(tmp.length != isReal.size()) { String s = tmp[isReal.size()]; if(tmp.length == isReal.size()+1)//{#} means the # is the weight { if(!s.matches("\\{\\d+(\\.\\d+)?\\}")) throw new RuntimeException("extra column must indicate a data point weigh in the form of \"{#}\", instead bad token " + s + " was found"); weight = Double.parseDouble(s.substring(1, s.length()-1)); } else { throw new RuntimeException("Column had " + tmp.length + " values instead of " + isReal.size()); } } DenseVector vec = new DenseVector(numReal); int[] cats = new int[numOfVars - numReal]; int k = 0;//Keeping track of position in cats for(int i = 0; i < isReal.size(); i++) { String val_string = tmp[i].trim(); if (isReal.get(i)) if (val_string.equals("?"))//missing value, indicated by NaN vec.set(i - k, Double.NaN); else vec.set(i - k, Double.parseDouble(val_string)); else//Categorical { tmp[i] = nameTrim(tmp[i]).trim().toLowerCase(); if(tmp[i].equals("?"))//missing value, indicated by -1 cats[k++] = -1; else cats[k++] = catVals.get(i).get(tmp[i]); } } list.add(new DataPoint(vec, cats, categoricalData, weight)); } } } catch (IOException ex) { } SimpleDataSet dataSet = new SimpleDataSet(list); int k = 0; for (int i = 0; i < isReal.size(); i++) if (isReal.get(i)) dataSet.setNumericName(variableNames.get(i), k++); return dataSet; } public static void writeArffFile(DataSet data, OutputStream os) { writeArffFile(data, os, "Default_Relation"); } /** * Writes out the dataset as an ARFF file to the given stream. This method * will automatically handle the target variable of * {@link ClassificationDataSet} and {@link RegressionDataSet}. * * @param data the dataset to write out * @param os the output stream to write too * @param relation the relation label to write out */ public static void writeArffFile(DataSet data, OutputStream os, String relation) { PrintWriter writer = new PrintWriter(os); //write out the relation tag writer.write(String.format("@relation %s\n", addQuotes(relation))); //write out attributes //first all categorical features CategoricalData[] catInfo = data.getCategories(); for( CategoricalData cate : catInfo) { writeCatVar(writer, cate); } //write out all numeric features for(int i = 0; i < data.getNumNumericalVars(); i++) { String name = data.getNumericName(i); writer.write("@attribute " + (name == null ? "num" + i : name.replaceAll("\\s+", "-")) + " NUMERIC\n"); } if(data instanceof ClassificationDataSet)//also write out class variable writeCatVar(writer, ((ClassificationDataSet)data).getPredicting()); if(data instanceof RegressionDataSet) writer.write("@ATTRIBUTE target NUMERIC\n"); writer.write("@DATA\n"); for(int row = 0; row < data.getSampleSize(); row++) { DataPoint dp = data.getDataPoint(row); boolean firstFeature = true; //cat vars first for(int i = 0; i < catInfo.length; i++) { if(!firstFeature) writer.write(","); firstFeature = false; int cat_val = dp.getCategoricalValue(i); if(cat_val < 0) writer.write("?"); else writer.write(addQuotes(catInfo[i].getOptionName(cat_val))); } //numeric vars Vec v = dp.getNumericalValues(); for(int i = 0; i < v.length(); i++) { if(!firstFeature) writer.write(","); firstFeature = false; double val = v.get(i); if(Double.isNaN(val))//missing value case writer.write("?"); else if(Math.rint(val) == val)//cast to long before writting to save space writer.write(Long.toString((long) val)); else writer.write(Double.toString(val)); } if (data instanceof ClassificationDataSet)//also write out class variable { if(!firstFeature) writer.write(","); firstFeature = false; ClassificationDataSet cdata = (ClassificationDataSet) data; writer.write(addQuotes(cdata.getPredicting().getOptionName(cdata.getDataPointCategory(row)))); } if (data instanceof RegressionDataSet) { if(!firstFeature) writer.write(","); firstFeature = false; writer.write(Double.toString(((RegressionDataSet)data).getTargetValue(row))); } writer.write("\n"); } writer.flush(); } private static String addQuotes(String string) { if(string.contains(" ")) return "\"" + string + "\""; else return string; } private static void writeCatVar(PrintWriter writer, CategoricalData cate) { writer.write("@ATTRIBUTE " + cate.getCategoryName().replaceAll("\\s+", "-") + " {" ); for(int i = 0; i < cate.getNumOfCategories(); i++) { if(i != 0) writer.write(","); writer.write(addQuotes(cate.getOptionName(i))); } writer.write("}\n"); } /** * Removes the quotes at the end and front of a string if there are any, as well as spaces at the front and end * @param in the string to trim * @return the white space and quote trimmed string */ private static String nameTrim(String in) { in = in.trim(); if(in.startsWith("'") || in.startsWith("\"")) in = in.substring(1); if(in.endsWith("'") || in.startsWith("\"")) in = in.substring(0, in.length()-1); return in.trim(); } }

Ads help maintain this website.