package jasext.flatfileserver;
import java.util.*; //Date
import java.io.*; // r/w of data file
import java.util.zip.*;
import java.text.*;
import javax.swing.BoundedRangeModel;
import javax.swing.DefaultBoundedRangeModel;
import hepjas.analysis.*; //EventData
import jas2.util.tree.*; // for TreeItem
import jas2.hist.*; //Rebinnable1DHistogram
import jas2.jds.module.*;
import java.lang.Math;
import jas2.util.FileReaderWithProgressBar;
/*
* Notes:
* a) Microsoft jvm does not throw NumberFormatException from Double.parse as long as the String to
* be parsed STARTS with a valid double
*
* b) Microsoft JVM throws a null pointer exception (!) if a date begins with a space
*
* c) JDK 1.1 DecimalFormat does not support exponential notation (although as of JDK1.1.6 code
* is there to do it, but it is commented out.
*/
public class FlatFileEventSource implements AnnotatedEventSource
{
//testing this filename is one set in parameter list
public final static int NO_STRING_DELIMITER = -4, WHITESPACE_DELIMITED = -4;
private DataColumn data[];
private int ith_event = 0;
private NTupleEventData event = new FlatFileEventData();
private Hashtable fieldNameHash = new Hashtable();
private int fieldCount;
private int rowCount;
private int quoteChar = NO_STRING_DELIMITER; // by default
private int delimChar = WHITESPACE_DELIMITED; // by default
private String m_name, m_nameOnTree;
private boolean m_containsHeaders = true, m_firstLineContainsTitle = false; // by default
private boolean m_gzip = false;
private BoundedRangeModel m_model = null;
private LocalJobBuilder m_ljb = null;
private String m_dateTimeFormat = "dd-MMM-yyyy hh:mm:ss";
private String m_timeFormat = "hh:mm:ss";
private String m_dateFormat = "dd-MMM-yyyy";
private Format[] formats = {
new IntegerFormat(),
new SimpleDateFormat(m_dateTimeFormat),
new SimpleDateFormat(m_timeFormat),
new SimpleDateFormat(m_dateFormat),
new ExponentialFormat(), // See note a above.
};
public FlatFileEventSource(String s) throws FlatFileException
{
m_name = s;
init(); // when this constuructor is used, we assume that defaults will be used, so we init() right away
}
public FlatFileEventSource()
{
/*
* After using this constructor, you must call init() once you have
* set up the parameters you wish to use.
*/
}
public Class getEventDataClass()
{
return NTupleEventData.class;
}
public void close()
{
}
public void init() throws FlatFileException
{
try
{
if (m_ljb != null) m_ljb.setMessage("Opening file "+ m_name);
BufferedReader input;
if (m_gzip)
{
if (m_model == null)
{
input = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(m_name))));
}
else
{
input = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileReaderWithProgressBar(m_name,m_model))));
}
}
else
{
if (m_model == null)
{
input = new BufferedReader(new FileReader(m_name));
}
else
{
input = new BufferedReader(new InputStreamReader(new FileReaderWithProgressBar(m_name, m_model)));
}
}
// Initially we treat all tokens as strings
StreamTokenizer in = new StreamTokenizer(input);
in.resetSyntax();
in.eolIsSignificant(true);
try
{
retrieveData(in);
}
catch (FlatFileSyntaxException e)
{
throw new FlatFileException("Error reading "+m_name+" at line "+in.lineno(),e);
}
catch (Throwable e)
{
throw new FlatFileException("Error while reading: "+m_name, e);
}
finally
{
try
{
input.close();
}
catch (IOException e) {}
}
}
catch(FileNotFoundException e)
{
throw new FlatFileException("Cannot open: "+m_name,e);
}
catch(IOException e)
{
throw new FlatFileException("Cannot open: "+m_name,e);
}
}
private void retrieveData(StreamTokenizer in) throws IOException, FlatFileSyntaxException
{
fieldCount = 0;
if (m_firstLineContainsTitle)
{
in.wordChars(0, 0xFFFF);
in.whitespaceChars('\n','\r');
// the first token will be everything up to the first '\n'
in.nextToken();
m_nameOnTree = in.sval; // that first line is the title
in.nextToken();
// in.ttype should be in.TT_EOL
}
else
m_nameOnTree = format(m_name);
// now that the title is read in we can start using regular char settings:
in.whitespaceChars(0,' ');
in.wordChars('\u0021','\u00ff');
if (quoteChar != NO_STRING_DELIMITER)
in.quoteChar(quoteChar);
if (delimChar != WHITESPACE_DELIMITED)
{
in.ordinaryChar(delimChar);
in.wordChars(' ', ' '); // spaces are now part of the word, not token separators
}
if (m_containsHeaders)
{
rowCount = 0;
// process first line of file -- that will contain the field names
Vector fieldNameList = new Vector();
in.nextToken();
while (in.ttype != in.TT_EOF && in.ttype != in.TT_EOL)
{
if (fieldCount != 0 && in.ttype == delimChar) in.nextToken();
if (in.ttype == delimChar || in.ttype == in.TT_EOL) // consecutive delim char indicate a null token
{
fieldNameList.addElement("");
fieldCount++;
}
else
{
if (in.ttype != in.TT_WORD && in.ttype != quoteChar) throw new BadFieldValue();
fieldNameList.addElement(in.sval.trim());
fieldCount++;
in.nextToken();
}
}
// create data structure to accommodate all the expected fields
data = new DataColumn[fieldCount];
for (int i=0; i < fieldCount; i++)
{
data[i] = new StringDataColumn((String) fieldNameList.elementAt(i));
fieldNameHash.put(fieldNameList.elementAt(i), new Integer(i));
}
}
else
{
rowCount = 1; // read in one to count fields
Vector firstRowData = new Vector();
// first row data will be stored in this Vector while columns are counted
in.nextToken();
while (in.ttype != in.TT_EOL && in.ttype != in.TT_EOF)
{
if (fieldCount != 0 && in.ttype == delimChar) in.nextToken();
if (in.ttype == delimChar || in.ttype == in.TT_EOL) // consecutive delim char indicate a null token
{
firstRowData.addElement(null);
}
else
{
if (in.ttype != in.TT_WORD && in.ttype != quoteChar) throw new BadFieldValue();
firstRowData.addElement(in.sval.trim());
in.nextToken();
}
fieldCount++;
}
data = new DataColumn[fieldCount];
String[] fieldNames = new String[fieldCount];
new jas2.util.ArbitraryNameGenerator().fillArray(fieldNames);
for (int i=0; i < fieldCount; i++)
{
data[i] = new StringDataColumn(fieldNames[i]);
((StringDataColumn) data[i]).addRow((String) firstRowData.elementAt(i)); // restore 1st row data
fieldNameHash.put(fieldNames[i], new Integer(i));
}
}
if (in.ttype == in.TT_EOF) throw new NoDataFound();
// read the rest of the file -- that will contain the field values
if (m_ljb != null) m_ljb.setMessage("Reading file "+ m_name);
while (in.ttype != in.TT_EOF)
{
in.nextToken();
if (in.ttype == in.TT_EOF) break;
if (in.ttype == in.TT_EOL) continue; // ignore blank lines
readDataRow(in);
rowCount++;
}
if (m_ljb != null)
{
m_ljb.setMessage("Converting "+ m_name);
m_model.setRangeProperties(0,0,0,fieldCount-1,false);
}
for (int i=0; i < fieldCount; i++)
{
data[i] = ((StringDataColumn) data[i]).convert(formats);
if (m_model != null) m_model.setValue(i);
}
if (m_ljb != null) m_ljb.setMessage("Processing complete: file "+ m_name);
}
private void readDataRow(StreamTokenizer in) throws IOException, BadFieldValue, AbsentFieldValue, ExtraFieldValue
{
int currentField = 0;
//read a number, put it into data construct
while (in.ttype != in.TT_EOL && in.ttype != in.TT_EOF)
{
if (currentField != 0 && in.ttype == delimChar) in.nextToken();
if (in.ttype == delimChar || in.ttype == in.TT_EOL) // consecutive delim char indicate a null token
{
if (currentField >= fieldCount) throw new ExtraFieldValue();
((StringDataColumn) data[currentField]).addRow(null);
currentField++;
}
else
{
if (in.ttype != in.TT_WORD && in.ttype != quoteChar) throw new BadFieldValue();
if (currentField >= fieldCount) throw new ExtraFieldValue();
((StringDataColumn) data[currentField]).addRow(in.sval.trim());
currentField++;
in.nextToken();
}
}
// check for too few values on a row
if (currentField < fieldCount) throw new AbsentFieldValue();
}
public String getDateTimeFormat()
{
return m_dateTimeFormat;
}
public void setDateTimeFormat(String s)
{
m_dateTimeFormat = s;
formats[1] = new SimpleDateFormat(s);
}
public String getTimeFormat()
{
return m_timeFormat;
}
public void setTimeFormat(String s)
{
m_timeFormat = s;
formats[2] = new SimpleDateFormat(s);
}
public String getDateFormat()
{
return m_dateFormat;
}
public void setDateFormat(String s)
{
m_dateFormat = s;
formats[3] = new SimpleDateFormat(s);
}
public int getDelim()
{
return delimChar;
}
public void setDelim(int c)
{
delimChar = c;
}
public int getQuoteChar()
{
return quoteChar;
}
public void setQuoteChar(int c)
{
quoteChar = c;
}
public String getFileName()
{
return m_name;
}
public void setFileName(String s)
{
m_name = s;
}
public boolean getHasHeaders()
{
return m_containsHeaders;
}
public void setHasHeaders(boolean b)
{
m_containsHeaders = b;
}
public boolean getFirstLineHasTitle()
{
return m_firstLineContainsTitle;
}
public void setFirstLineHasTitle(boolean b)
{
m_firstLineContainsTitle = b;
}
public boolean getGZIP()
{
return m_gzip;
}
public void setGZIP(boolean b)
{
m_gzip = b;
}
void setLocalJobBuilder(LocalJobBuilder ljb)
{
m_ljb = ljb;
if (m_ljb != null)
{
m_model = new DefaultBoundedRangeModel();
m_ljb.setProgressModel(m_model);
}
}
private String format(String fileName)
/*
* Formats a file name into a shorter form by
* taking the substring between the last separator
* and the last dot.
*/
{
try
{
String sep = File.separator;
int sepIndex = fileName.lastIndexOf(sep);
if (sepIndex >= 0) sepIndex += sep.length(); // go past the separator
else sepIndex = 0; // no separator in file name, so start from beginning
int dotIndex = fileName.lastIndexOf('.');
if (dotIndex > sepIndex)
return fileName.substring(sepIndex, dotIndex);
else
return fileName.substring(sepIndex);
}
catch (Exception e)
{
return fileName;
}
}
public TreeItem[] getItems()
{
// called by Data Server to get list of available data sets
// creating a tree with only two columns
TreeItem[] result = new TreeItem[fieldCount];
// one tree item will be the first column 'x'
for (int i = 0; i < fieldCount;i++)
{
result[i] = new TreeItem();
result[i].path = new String[1];
result[i].path[0] = data[i].getTitle();
result[i].type = data[i].getType();
}
return result;
}
public Object getItem(TreeItem i)
{
//given a particular data source, get a rebinnable histogram
// here the datasource is dat, which is actually a table of values
//extract column index from hash table of field names
int column_num = ((Integer) fieldNameHash.get(i.path[ i.path.length - 1])).intValue();
return data[column_num];
}
public int getTotalNumberOfEvents()
{
// returns number of events associated with data
return rowCount;
}
public void beforeFirstEvent()
{
//resets index to events
ith_event = -1;
}
public void afterLastEvent()
{
//used to do cleanup or report status of all events done
//suppose we try to initialise all the fields in the table to zero
}
public EventData getNextEvent() throws EndOfDataException
{
// returns event object
if (++ith_event >= rowCount) throw new EndOfDataException();
return event;
}
public String getName()
{
return m_nameOnTree;
}
// we are including an inline class that can implement an EventData interface.
// for efficiency reasons, we've included the class within the EventSource
// class so that the EventData class can have access to the Source's variables
private class FlatFileEventData extends AbstractNTupleEventData
{
public double getDouble(String name)
{
Integer column = (Integer) fieldNameHash.get(name);
if (column == null) throw new EventDataException("Can not find column "+name);
return data[column.intValue()].getDouble(ith_event);
}
public int getInt(String name)
{
Integer column = (Integer) fieldNameHash.get(name);
if (column == null) throw new EventDataException("Can not find column "+name);
return data[column.intValue()].getInt(ith_event);
}
public java.util.Date getDate(String name)
{
Integer column = (Integer) fieldNameHash.get(name);
if (column == null) throw new EventDataException("Can not find column "+name);
return data[column.intValue()].getDate(ith_event);
}
public String getString(String name)
{
Integer column = (Integer) fieldNameHash.get(name);
if (column == null) throw new EventDataException("Can not find column "+name);
return data[column.intValue()].getString(ith_event);
}
public double getDouble(final int column)
{
return data[column].getDouble(ith_event);
}
public int getInt(final int column)
{
return data[column].getInt(ith_event);
}
public java.util.Date getDate(final int column)
{
return data[column].getDate(ith_event);
}
public String getString(final int column)
{
return data[column].getString(ith_event);
}
public int findColumn(final String name)
{
Integer column = (Integer) fieldNameHash.get(name);
if (column == null) throw new EventDataException("Can not find column "+name);
return column.intValue();
}
}
}
/*
* DataColumn represents the histogrammable data.
*/
abstract class DataColumn implements Rebinnable1DHistogramData
{
private String title;
DataColumn(String title)
{
this.title = title;
}
DataColumn(DataColumn dc)
{
this.title = dc.getTitle();
}
public String getTitle()
{
return title;
}
int getInt(int n)
{
throw new FlatFileColumnException("data cannot be fetched using getInt");
}
double getDouble(int n)
{
throw new FlatFileColumnException("data cannot be fetched using getDouble");
}
String getString(int n)
{
throw new FlatFileColumnException("data cannot be fetched using getString");
}
Date getDate(int n)
{
throw new FlatFileColumnException("data cannot be fetched using getDate");
}
abstract String getType();
public String toString()
{
return "Column "+title+" ("+getType()+") min: "+getMin()+" max: "+getMax();
}
}
class StringDataColumn extends DataColumn
{
private Hashtable hashTable = new Hashtable();
private int valueCount = 0;
private int index;
private Vector data;
private String currentArray[];
private static final int MAXSIZE = 100;
private int nBins = 0;
StringDataColumn(String name)
{
super(name);
data = new Vector();
currentArray = new String[MAXSIZE];
index = 0;
valueCount = 0;
}
void addRow(String value)
{
//if current array is full, add it to the vector and start with a fresh array
if (index == MAXSIZE)
{
data.addElement(currentArray);
index = 0;
currentArray = new String[MAXSIZE];
}
currentArray[index++] = value;
valueCount++;
// We build a hashtable as we go
if (value != null) hashTable.put(value,this);
}
// Figure out the format of this column
private int getDataType(Format[] formats)
{
int type;
String s = null;
ParsePosition pp = new ParsePosition(0);
for (type = 0; type=0 && bin < bins) hist[bin]++;
}
// and return result
double[][] result = new double[1][];
result[0] = hist;
return result;
}
String getString(int n)
{
int ithArray = n / MAXSIZE;
if (ithArray >= data.size())
{
// value must exist in the current array
return currentArray[n % MAXSIZE];
}
else
{
// value must exist in vector
String a[] = (String[]) data.elementAt(ithArray);
return a[n % MAXSIZE];
}
}
int getSize()
{
return valueCount;
}
String getType()
{
return "string";
}
public jas2.hist.Statistics getStatistics()
{
return null;
}
}
class DoubleDataColumn extends DataColumn
{
private double min = 0;
private double max = 1;
private double[] data;
DoubleDataColumn(StringDataColumn source)
{
super(source);
data = new double[source.getSize()];
if (data.length > 0)
{
String s = source.getString(0);
min = max = (s==null) ? 0 : Double.valueOf(s).doubleValue();
}
for (int i=0; imax) max = d;
if (d= min && x < max) hist[(int) ( (x - min) / binWidth )]++;
}
double[][] result = new double[1][];
result[0] = hist;
return result;
}
public String[] getAxisLabels()
{
return null;
}
public boolean isRebinnable()
{
return true;
}
public int getAxisType()
{
return Rebinnable1DHistogramData.DOUBLE;
}
public double getMin()
{
return min;
}
public double getMax()
{
return max;
}
public int getBins()
{
return 40;
}
double getDouble(int n)
{
return data[n];
}
String getType()
{
return "float";
}
public jas2.hist.Statistics getStatistics()
{
return null;
}
}
class DateDataColumn extends DataColumn
{
private Date min = null;
private Date max = null;
private Date[] data;
DateDataColumn(StringDataColumn source, DateFormat df)
{
super(source);
data = new Date[source.getSize()];
try
{
for (int i=0; i=0 && bin 0)
{
s = source.getString(0);
min = max = (s==null) ? 0 : Integer.valueOf(s).intValue();
}
for (int i=0; imax) max = d;
if (d=0 && bin
Ads help maintain this website.