/*
This file is part of the software framework JStatCom,
a library for building applications for data based analysis.
Copyright (C) 2000-2005 Markus Kraetzig
This library is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation; either version 2.1 of the
License, or (at your option) any later version.
This library is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
USA
*/
package com.jstatcom.io;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import com.jstatcom.component.TopFrameReference;
import com.jstatcom.model.JSCData;
import com.jstatcom.model.JSCDate;
import com.jstatcom.model.JSCNArray;
import com.jstatcom.model.JSCSArray;
import com.jstatcom.model.JSCTypeDef;
import com.jstatcom.model.JSCTypes;
import com.jstatcom.model.SymbolTable;
import com.jstatcom.ts.TS;
import com.jstatcom.ts.TSDate;
import com.jstatcom.ts.TSDateRange;
import com.jstatcom.util.UStringArray;
/**
* Class for importing files from Gauss .dat data set files.
* There are 4 supported formats, see the Gauss reference manual for a
* specification of the binary formats:
*
* - Data Set v89
*
- Extended Data Set v89
*
- Data Set v92
*
- Data Set v96
*
*
* Version v96 is also supported for exporting data via a static method.
*
* @author Markus Kraetzig
*/
public final class TSGaussDATHandler implements DataHandler {
private boolean noDialog = false;
private static final TSGaussDATHandler handler = new TSGaussDATHandler();
private File selectedFile = null;
private TSDataDialog dialog = null;
private boolean canceled = false;
private final SymbolTable symbolTable = new SymbolTable(
"TSGaussDATHandler", false);
/**
* Gets the singleton instance of this handler.
*
* @return instance
*/
public static TSGaussDATHandler getInstance() {
return handler;
}
/**
* Creates a TSGaussDATHandler.
*/
private TSGaussDATHandler() {
super();
}
/**
* Initiates parsing of .dat Gauss data set file.
*
* @param file
* the file to read in
* @return true of import was ok, false if
* cancelled or erroneous
* @throws IllegalArgumentException
* if (file == null || !file.isFile())
* @throws RuntimeException
* when file reading fails
*/
public boolean importData(File file) {
return importData(file, true);
}
/**
* Initiates parsing of .dat Gauss data set file.
*
* @param file
* the file to read in *
* @param showError
* whether a dialog should be shown in case of an error
*
* @return true of import was ok, false if
* cancelled or erroneous
* @throws IllegalArgumentException
* if (file == null || !file.isFile())
* @throws RuntimeException
* when file reading fails
*/
@SuppressWarnings("unused")
public boolean importData(File file, boolean showError) {
if (file == null)
throw new IllegalArgumentException("Argument was null.");
if (!file.isFile())
throw new IllegalArgumentException("Invalid data file " + file
+ ".");
selectedFile = file;
canceled = false;
try {
// check whether .dht file exists
String dhtFile0 = selectedFile.getAbsolutePath();
String dhtFile1 = dhtFile0;
int index = dhtFile0.lastIndexOf(".");
if (index > -1) {
dhtFile0 = dhtFile0.substring(0, index) + ".dht";
dhtFile1 = dhtFile1.substring(1, index) + ".DHT";
}
File dht = new File(dhtFile0);
if (!dht.canRead())
dht = new File(dhtFile1);
// must be v89
if (dht.canRead()) {
DataInputStream dhtin = new DataInputStream(
new FileInputStream(dht));
// decide which parsing should be used
int size = dhtin.available();
byte[] header = new byte[size];
dhtin.read(header);
dhtin.close();
// small v89
if (header[0] == (byte) 0xDA && header[1] == (byte) 0xDA) {
DataInputStream in = new DataInputStream(
new FileInputStream(file));
dhtin = new DataInputStream(new FileInputStream(dht));
importSmallDATv89(in, dhtin);
in.close();
dhtin.close();
return !canceled;
}
// extended v89
if ((header[0] == (byte) 0xDA && header[1] == (byte) 0xEE)
|| header[0] == (byte) 0xEE && header[1] == (byte) 0xDA) {
DataInputStream in = new DataInputStream(
new FileInputStream(file));
dhtin = new DataInputStream(new FileInputStream(dht));
importExtendedDATv89(in, dhtin);
in.close();
dhtin.close();
return !canceled;
}
throw new RuntimeException(
"Unrecognized version from header file \""
+ dht.getName() + "\".");
}
DataInputStream in = new DataInputStream(new FileInputStream(file));
// decide which parsing should be used
int size = in.available();
byte[] header = new byte[size];
in.read(header);
in.close();
// to reset
in = new DataInputStream(new FileInputStream(file));
// v96
if (header[0] == (byte) 0xFF) {
importDATv96(in, header);
in.close();
return !canceled;
}
// v92
if (header[0] == (byte) 0x00) {
importDATv92(in, header);
in.close();
return !canceled;
}
throw new RuntimeException(
"Unrecognized Gauss data set version (neiter v89, v92, v96).");
} catch (Throwable t) {
throw new RuntimeException(t.getMessage());
}
}
/**
* Reads the small data set v89 .dat format.
*
* @param in
* stream to read from
* @param dhtIn
* header stream to read from
* @param header
* 128 byte array with header
*/
private void importSmallDATv89(DataInputStream in, DataInputStream dhtIn)
throws IOException {
// always littleEndian
dhtIn.skipBytes(6);
// cols
short cols = Short.reverseBytes(dhtIn.readShort());
dhtIn.skipBytes(4);
short type = Short.reverseBytes(dhtIn.readShort());
dhtIn.skipBytes(8);
byte[] flags = new byte[2];
dhtIn.read(flags);
boolean isComplex = flags[0] == (byte) 0x01;
dhtIn.skipBytes(104);
String[] names = new String[cols];
for (int i = 0; i < names.length; i++) {
byte[] nam = new byte[8];
dhtIn.read(nam);
int length = 0;
for (int j = 0; j < nam.length; j++) {
if (nam[j] == (byte) 0x00) {
length = j;
break;
}
}
names[i] = new String(nam, 0, length).trim();
}
// determine rows like in GAUSS
int rows = new Long(selectedFile.length() / (type * cols)).intValue();
double[][] val = new double[rows][cols];
for (int i = 0; i < rows; i++)
for (int j = 0; j < cols; j++) {
if (type == 2) {
val[i][j] = Double.longBitsToDouble(Short.reverseBytes(in
.readShort()));
}
if (type == 4) {
val[i][j] = Double.longBitsToDouble(Integer.reverseBytes(in
.readInt()));
}
if (type == 8) {
val[i][j] = Double.longBitsToDouble(Long.reverseBytes(in
.readLong()));
}
// skip complex part
if (isComplex)
in.skipBytes(type);
}
setDataToSymbolTable(val, names);
}
/**
* Reads the extended data set v89 .dat format.
*
* @param in
* stream to read from
* @param dhtIn
* header stream to read from
* @param header
* 128 byte array with header
*/
private void importExtendedDATv89(DataInputStream in, DataInputStream dhtIn)
throws IOException {
// always littleEndian
dhtIn.skipBytes(2);
short type = Short.reverseBytes(dhtIn.readShort());
dhtIn.skipBytes(4);
// cols
int cols = Integer.reverseBytes(dhtIn.readInt());
dhtIn.skipBytes(16);
byte[] flags = new byte[2];
dhtIn.read(flags);
boolean isComplex = flags[0] == (byte) 0x01;
dhtIn.skipBytes(98);
String[] names = new String[cols];
for (int i = 0; i < names.length; i++) {
byte[] nam = new byte[8];
dhtIn.read(nam);
int length = 0;
for (int j = 0; j < nam.length; j++) {
if (nam[j] == (byte) 0x00) {
length = j;
break;
}
}
names[i] = new String(nam, 0, length).trim();
}
// determine rows like in GAUSS
int rows = new Long(selectedFile.length() / (type * cols)).intValue();
double[][] val = new double[rows][cols];
for (int i = 0; i < rows; i++)
for (int j = 0; j < cols; j++) {
if (type == 2) {
val[i][j] = Double.longBitsToDouble(Short.reverseBytes(in
.readShort()));
}
if (type == 4) {
val[i][j] = Double.longBitsToDouble(Integer.reverseBytes(in
.readInt()));
}
if (type == 8) {
val[i][j] = Double.longBitsToDouble(Long.reverseBytes(in
.readLong()));
}
// skip complex part
if (isComplex)
in.skipBytes(type);
}
setDataToSymbolTable(val, names);
}
/**
* Reads the data set v92 .dat format.
*
* @param in
* stream to read from
* @param header
* 128 byte array with header
*/
private void importDATv92(DataInputStream in, byte[] header)
throws IOException {
boolean isLittleEndian = header[4] == (byte) 0xCA;
in.skipBytes(20);
// rows
int rows = in.readInt();
if (isLittleEndian)
rows = Integer.reverseBytes(rows);
// cols
int cols = in.readInt();
if (isLittleEndian)
cols = Integer.reverseBytes(cols);
int complex = in.readInt();
if (isLittleEndian)
complex = Integer.reverseBytes(complex);
int type = in.readInt();
if (isLittleEndian)
type = Integer.reverseBytes(type);
in.skipBytes(92);
String[] names = new String[cols];
for (int i = 0; i < names.length; i++) {
byte[] nam = new byte[8];
in.read(nam);
int length = 0;
for (int j = 0; j < nam.length; j++) {
if (nam[j] == (byte) 0x00) {
length = j;
break;
}
}
names[i] = new String(nam, 0, length).trim();
}
int[] selNum = new int[cols];
byte[] index = new byte[cols];
in.read(index);
for (int i = 0; i < cols; i++)
selNum[i] = index[i];
TSMatlabHandler.skipPaddingBytes(in, 8, cols);
// select only names with numerical data
names = UStringArray.selRowsIf(names, selNum);
double[][] val = new double[rows][names.length];
for (int i = 0; i < rows; i++) {
for (int j = 0, colCounter = 0; j < cols; j++) {
if (selNum[j] == 1) // number colums
if (isLittleEndian) {
if (type == 2) {
val[i][colCounter++] = Double
.longBitsToDouble(Short.reverseBytes(in
.readShort()));
}
if (type == 4) {
val[i][colCounter++] = Double
.longBitsToDouble(Integer.reverseBytes(in
.readInt()));
}
if (type == 8) {
val[i][colCounter++] = Double.longBitsToDouble(Long
.reverseBytes(in.readLong()));
}
} else {
if (type == 2) {
val[i][colCounter++] = in.readShort();
}
if (type == 4) {
val[i][colCounter++] = in.readFloat();
}
if (type == 8) {
val[i][colCounter++] = in.readDouble();
}
}
else
// skip columns with character data
in.skipBytes(type);
// because real and imag parts are interleaved
// ignore imag part
if (complex == 1)
in.skipBytes(type);
}
}
setDataToSymbolTable(val, names);
}
/**
* Reads the data set v96 .dat format.
*
* @param in
* stream to read from
* @param header
* 128 byte array with header
*/
private void importDATv96(DataInputStream in, byte[] header)
throws IOException {
boolean isLittleEndian = header[20] == (byte) 0x00;
in.skipBytes(48);
int type = in.readInt();
if (isLittleEndian)
type = Integer.reverseBytes(type);
// 0 - real matrix, 1 - complex matrix
int complex = in.readInt();
if (isLittleEndian)
complex = Integer.reverseBytes(complex);
// 1 - imaginary part of matrix follows real part (standard GAUSS style)
// 2 - imaginary part of each element immediately follows real part
// (FORTRAN style)
int imagStyle = in.readInt();
if (isLittleEndian)
imagStyle = Integer.reverseBytes(imagStyle);
in.skipBytes(4);
// 1 - row major ordering of elements, 2 - column major
int major = in.readInt();
if (isLittleEndian)
major = Integer.reverseBytes(major);
in.skipBytes(12);
// rows
int rows = in.readInt();
if (isLittleEndian)
rows = Integer.reverseBytes(rows);
// cols
int cols = in.readInt();
if (isLittleEndian)
cols = Integer.reverseBytes(cols);
in.skipBytes(40);
String[] names = new String[cols];
for (int i = 0; i < cols; i++) {
byte[] nam = new byte[32];
in.read(nam);
int length = 32;
for (int j = 0; j < nam.length; j++) {
if (nam[j] == (byte) 0x00) {
length = j;
break;
}
}
names[i] = new String(nam, 0, length).trim();
}
int[] selNum = new int[cols];
byte[] index = new byte[cols];
in.read(index);
for (int i = 0; i < cols; i++)
selNum[i] = index[i];
TSMatlabHandler.skipPaddingBytes(in, 8, cols);
// select only names with numerical data
names = UStringArray.selRowsIf(names, selNum);
double[][] val = new double[rows][names.length];
if (major == 1)
for (int i = 0; i < rows; i++) {
for (int j = 0, colCounter = 0; j < cols; j++) {
if (selNum[j] == 1) {// number colums
if (isLittleEndian) {
if (type == 2) {
val[i][colCounter++] = Double
.longBitsToDouble(Short.reverseBytes(in
.readShort()));
}
if (type == 4) {
val[i][colCounter++] = Double
.longBitsToDouble(Integer
.reverseBytes(in.readInt()));
}
if (type == 8) {
val[i][colCounter++] = Double
.longBitsToDouble(Long.reverseBytes(in
.readLong()));
}
} else {
if (type == 2) {
val[i][colCounter++] = in.readShort();
}
if (type == 4) {
val[i][colCounter++] = in.readFloat();
}
if (type == 8) {
val[i][colCounter++] = in.readDouble();
}
}
} else
// skip columns with character data
in.skipBytes(type);
// skip imaginary part if necessary
if (complex == 1 && imagStyle == 2)
in.skipBytes(type);
}
}
else
for (int j = 0, colCounter = 0; j < cols; j++) {
// number colums
if (selNum[j] == 1) {
for (int i = 0; i < rows; i++) {
if (isLittleEndian) {
if (type == 2) {
val[i][colCounter] = Double
.longBitsToDouble(Short.reverseBytes(in
.readShort()));
}
if (type == 4) {
val[i][colCounter] = Double
.longBitsToDouble(Integer
.reverseBytes(in.readInt()));
}
if (type == 8) {
val[i][colCounter] = Double
.longBitsToDouble(Long.reverseBytes(in
.readLong()));
}
} else {
if (type == 2) {
val[i][colCounter] = in.readShort();
}
if (type == 4) {
val[i][colCounter] = in.readFloat();
}
if (type == 8) {
val[i][colCounter] = in.readDouble();
}
}
// skip imaginary part if necessary
if (complex == 1 && imagStyle == 2)
in.skipBytes(type);
}
colCounter++;
} else
// skip columns with character data
in.skipBytes(rows * type);
}
setDataToSymbolTable(val, names);
}
private void setDataToSymbolTable(double[][] values, String[] names) {
getDataDialog().setVariableNames(names);
if (!noDialog) {
getDataDialog().setVisible(true);
if (getDataDialog().isCanceled()) {
canceled = true;
return;
}
}
symbolTable.set(new JSCNArray(TSImportTypes.DATA.name(), values));
symbolTable.set(new JSCDate(TSImportTypes.STARTDATE.name(),
getDataDialog().getTSDate()));
if (values.length > 0)
names = TSGaussFMTHandler.getNamesArray(selectedFile,
values[0].length, getDataDialog().getVariableNames());
symbolTable.set(new JSCSArray(TSImportTypes.VARNAMES.name(), names));
}
/**
* Gets the imported data according to an instance of
* TSImportTypes.
*
* @param type
* the elementtype of a time series dataset
* @return the data stored as JSCData
* @throws IllegalArgumentException
* if (type == null)
* @throws IllegalStateException
* if nothing was imported before
*/
public JSCData getData(ImportTypes type) {
if (symbolTable == null)
throw new IllegalStateException("There was nothing imported yet.");
if (type == null)
throw new IllegalArgumentException("Argument was null.");
return symbolTable.getJSCData(new JSCTypeDef(type.name(), type.type()));
}
/**
* Gets the dialog used for raw ASCII files.
*/
private TSDataDialog getDataDialog() {
if (dialog == null)
dialog = new TSDataDialog(TopFrameReference.getTopFrameRef(), true);
dialog.setLocationRelativeTo(TopFrameReference.getTopFrameRef());
return dialog;
}
public void exportTS(File file, TS[] ts, String description) {
if (file == null)
throw new IllegalArgumentException("File argument was null.");
if (ts == null)
throw new IllegalArgumentException("Time series argument was null.");
if (ts.length == 0)
return;
TSDateRange mergedRange = TSDateRange.getMergedRange(ts);
// data
String[] names = new String[ts.length];
for (int i = 0; i < names.length; i++)
names[i] = ts[i].name();
double[][] values = new double[mergedRange.numOfObs()][ts.length];
for (int i = 0; i < mergedRange.numOfObs(); i++) {
TSDate currentDate = mergedRange.dateForIndex(i);
for (int j = 0; j < ts.length; j++)
values[i][j] = ts[j].valueAt(currentDate);
}
export(values, names, file);
}
public void exportData(File file, JSCData data, String description) {
if (file == null)
throw new IllegalArgumentException("File argument was null.");
if (data == null)
throw new IllegalArgumentException("Data argument was null.");
if (data.type() != JSCTypes.NARRAY)
throw new IllegalArgumentException("Data of type \"" + data.type()
+ "\" cannot be stored in Excel format.");
if (data.isEmpty())
return;
String[] names = new String[] { data.name() };
JSCNArray narray = (JSCNArray) data;
if (narray.cols() > 1) {
names = new String[narray.cols()];
String nam = narray.name();
for (int i = 0; i < names.length; i++)
names[i] = nam + "_" + (i + 1);
}
export(narray.doubleArray(), names, file);
}
private void export(double[][] values, String[] names, File file) {
try {
DataOutputStream out = new DataOutputStream(new FileOutputStream(
file));
// write header 128 byte
byte[] toWrite = new byte[40];
for (int i = 0; i < 4; i++)
toWrite[i] = (byte) 0xFF;
for (int i = 4; i < 8; i++)
toWrite[i] = (byte) 0x00;
for (int i = 8; i < 12; i++)
toWrite[i] = (byte) 0xFF;
for (int i = 12; i < 16; i++)
toWrite[i] = (byte) 0x00;
for (int i = 16; i < 20; i++)
toWrite[i] = (byte) 0xFF;
for (int i = 20; i < 24; i++)
toWrite[i] = (byte) 0xFF;
for (int i = 24; i < 28; i++)
toWrite[i] = (byte) 0x00;
toWrite[28] = (byte) 0xAB;
toWrite[29] = (byte) 0xCD;
toWrite[30] = (byte) 0xEF;
toWrite[31] = (byte) 0x02;
toWrite[32] = (byte) 0x00;
toWrite[33] = (byte) 0x00;
toWrite[34] = (byte) 0x00;
toWrite[35] = (byte) 0x01;
out.write(toWrite);
out.writeInt(1);
out.writeInt(1008);
out.writeInt(8);
out.writeInt(0);
out.writeInt(1);
out.writeInt(2);
out.writeInt(1);
out.writeInt(0);
int info = names.length * 33;
int toSkip = TSMatlabHandler
.skipPaddingBytes(null, 8, names.length);
int headerSize = 128 + info + toSkip;
out.writeInt(headerSize);
toWrite = new byte[4];
out.write(toWrite);
out.writeInt(values.length);
out.writeInt(names.length);
toWrite = new byte[40];
out.write(toWrite);
// names
for (int i = 0; i < names.length; i++) {
byte[] sArray = names[i].getBytes();
byte[] aName = new byte[32];
for (int j = 0; j < sArray.length && j < aName.length; j++)
aName[j] = sArray[j];
out.write(aName);
}
// variable type flags
for (int i = 0; i < names.length; i++)
out.writeByte(1);
toWrite = new byte[toSkip];
out.write(toWrite);
// data
for (int i = 0; i < values.length; i++)
for (int j = 0; j < values[0].length; j++)
out.writeDouble(values[i][j]);
out.close();
} catch (Throwable ex) {
throw new RuntimeException(ex.getMessage());
}
}
/**
* Sets this handler in test mode. No user dialog is shown, default values
* are used instead. This is helpful for running automated unit tests.
*
* @param noDialog
*/
public void setTestMode(boolean noDialog) {
this.noDialog = noDialog;
}
}
Ads help maintain this website.