/*
This file is part of the software framework JStatCom,
a library for building applications for data based analysis.
Copyright (C) 2000-2005 Markus Kraetzig
This library is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation; either version 2.1 of the
License, or (at your option) any later version.
This library is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
USA
*/
package com.jstatcom.io;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.zip.Deflater;
import java.util.zip.Inflater;
import org.apache.log4j.Logger;
import com.jstatcom.component.CompSettings;
import com.jstatcom.component.TopFrameReference;
import com.jstatcom.model.JSCConstants;
import com.jstatcom.model.JSCData;
import com.jstatcom.model.JSCDate;
import com.jstatcom.model.JSCNArray;
import com.jstatcom.model.JSCSArray;
import com.jstatcom.model.JSCString;
import com.jstatcom.model.JSCTypeDef;
import com.jstatcom.model.JSCTypes;
import com.jstatcom.model.SymbolTable;
import com.jstatcom.ts.TS;
import com.jstatcom.ts.TSDate;
import com.jstatcom.ts.TSDateRange;
import com.jstatcom.util.UMatrix;
/**
* Handles export and import of Matlab .mat files Level 5, according to the
* specification document "MAT-File Format Version 7" by Mathworks. This class
* should be used as a Singleton.
*
* @author Markus Kraetzig
*/
public final class TSMatlabHandler implements DataHandler {
private static final Logger log = Logger.getLogger(TSMatlabHandler.class);
private boolean noDialog = false;
private static final TSMatlabHandler handler = new TSMatlabHandler();
// supported MATLAB data types
private static final int miINT8 = 1;
private static final int miUINT8 = 2;
private static final int miINT16 = 3;
private static final int miUINT16 = 4;
private static final int miINT32 = 5;
private static final int miUINT32 = 6;
private static final int miSINGLE = 7;
private static final int miDOUBLE = 9;
private static final int miINT64 = 12;
private static final int miMATRIX = 14;
private static final int miCOMPRESSED = 15;
// supported MATLAB array classes
private static final int mxDOUBLE_CLASS = 6;
private static final int mxUINT32_CLASS = 13;
private TSDataDialog dialog = null;
private boolean cancelled = false;
private boolean isLittleEndian = false;
private final List namesList = new ArrayList();
private final List dataList = new ArrayList();
private final SymbolTable symbolTable = new SymbolTable("TSMatlabHandler",
false);
/**
* Gets the singleton instance of this handler.
*
* @return instance
*/
public static TSMatlabHandler getInstance() {
return handler;
}
/**
* Initiates parsing of .mat Matlab file.
*
* @param file
* the file to read in
* @return true of import was ok, false if
* cancelled or erroneous
* @throws IllegalArgumentException
* if (file == null || !file.isFile())
* @throws RuntimeException
* when file reading fails
*/
public boolean importData(File file) {
if (file == null)
throw new IllegalArgumentException("Argument was null.");
if (!file.isFile())
throw new IllegalArgumentException("Invalid data file " + file
+ ".");
cancelled = false;
namesList.clear();
dataList.clear();
try {
DataInputStream in = new DataInputStream(new FileInputStream(file));
// 1 - 116 description
byte[] toRead = new byte[116];
in.read(toRead);
if (toRead[0] == 0x00 && toRead[2] == 0x00 || toRead[3] == 0x00
|| toRead[4] == 0x00) {
throw new RuntimeException(
"Cannot read Matlab Level 4 files. Please use newer Level 5 format.");
}
String description = new String(toRead).trim();
// skip subsys data and 0x0100
in.skip(10);
// endian check MI or IM
toRead = new byte[2];
in.read(toRead);
String endianTest = new String(toRead);
isLittleEndian = endianTest.equalsIgnoreCase("IM");
// now read each data element, ignore unsupported types
int skippedBytes = 128;
while (in.available() > 0) {
// test small data format
toRead = new byte[4];
in.read(toRead);
// small data check tests whether 2 2-Byte chunks contain each
// at least one nonzero byte, which is he case for small data
// but not for normal data; test is independent of endianness
boolean isSmallData = !(toRead[0] == 0x00 && toRead[1] == 0x00)
&& !(toRead[2] == 0x00 && toRead[3] == 0x00);
in.close();
in = new DataInputStream(new FileInputStream(file));
in.skip(skippedBytes);
if (isSmallData) { // might never happen
in.skip(8);
skippedBytes += 8;
} else
skippedBytes += readData(in);
}
in.close();
// get names and NARRAY data from list
if (!namesList.isEmpty())
setDataToSymbolTable(description);
else {
throw new RuntimeException(
"Import failed because none of the stored Matlab data types is supported.");
}
return !cancelled;
} catch (Throwable t) {
throw new RuntimeException(t.getMessage());
}
}
/**
* Reads a data element. The stream must be at the beginning of the data
* tag. When the method return, the stream points to the beginning of the
* next data element or EOF.
*/
private int readData(DataInputStream in) throws Exception {
// data type
int type = in.readInt();
if (isLittleEndian)
type = Integer.reverseBytes(type);
// number of bytes
int numOfBytes = in.readInt();
if (isLittleEndian)
numOfBytes = Integer.reverseBytes(numOfBytes);
// decompress and call this method with a new stream
if (type == miCOMPRESSED) {
DataInputStream inflatedIn = miCompressed(in, numOfBytes);
// recursive call of this method but now with the decompressed
// stream for a data element
readData(inflatedIn);
inflatedIn.close();
return 8 + numOfBytes; // no padding of compressed
} // matrix type
if (type == miMATRIX) {
byte[] data = new byte[numOfBytes];
in.read(data);
miMatrix(data);
} else {
// supported type to be read directly, fill lists
// ignored type, just skip
in.skipBytes(numOfBytes);
}
return 8 + numOfBytes + skipPaddingBytes(in, 8, numOfBytes);
}
private void miMatrix(byte[] data) throws IOException {
ByteArrayInputStream inBytes = new ByteArrayInputStream(data);
DataInputStream inDat = new DataInputStream(inBytes);
// ARRAY FLAGS
inDat.skip(8); // tags
byte[] flags_class = new byte[4];
inDat.read(flags_class);
int array_class = flags_class[3];
if (isLittleEndian)
array_class = flags_class[0];
inDat.skip(4); // undefined
// do nothing if character or other type
if (array_class < mxDOUBLE_CLASS || array_class > mxUINT32_CLASS) {
inDat.close();
inBytes.close();
return;
}
// DIMENSIONS
inDat.skip(4);
int n = 0;
int nBytes = inDat.readInt();
if (isLittleEndian)
nBytes = Integer.reverseBytes(nBytes);
n = nBytes / 4;
if (n > 2)
return;
int rows = inDat.readInt();
if (isLittleEndian)
rows = Integer.reverseBytes(rows);
int cols = inDat.readInt();
if (isLittleEndian)
cols = Integer.reverseBytes(cols);
nBytes += skipPaddingBytes(inDat, 8, nBytes);
int index = 24 + nBytes;
// NAME
boolean isSmallData = !(data[index] == 0x00 && data[index + 1] == 0x00)
&& !(data[index + 2] == 0x00 && data[index + 3] == 0x00);
String name = "";
if (isSmallData) {
inDat.skip(4);
byte[] nam = new byte[4];
inDat.read(nam);
name = new String(nam).trim().toLowerCase();
index += 8;
} else {
inDat.skip(4);
int nameBytes = inDat.readInt();
if (isLittleEndian)
nameBytes = Integer.reverseBytes(nameBytes);
byte[] nam = new byte[nameBytes];
inDat.read(nam);
name = new String(nam).trim().toLowerCase();
index += 8 + nameBytes + skipPaddingBytes(inDat, 8, nameBytes);
}
for (int i = 0; i < cols; i++) {
String toAdd = name;
if (cols > 1)
toAdd = name + "_" + (i + 1);
if (JSCConstants.isValidName(toAdd) != null
|| namesList.contains(toAdd))
toAdd = "var_" + (namesList.size() + 1);
namesList.add(toAdd);
}
// REAL PART
isSmallData = !(data[index] == 0x00 && data[index + 1] == 0x00)
&& !(data[index + 2] == 0x00 && data[index + 3] == 0x00);
int type = 0;
if (!isSmallData) {
type = inDat.readInt();
if (isLittleEndian)
type = Integer.reverseBytes(type);
int realPartBytes = inDat.readInt();
if (isLittleEndian)
realPartBytes = Integer.reverseBytes(realPartBytes);
} else {
// data type and number of bytes is a 4 byte chunk
if (isLittleEndian) {
short typeShort = Short.reverseBytes(inDat.readShort());
type = typeShort;
inDat.skip(2);
} else {
inDat.skip(2);
short typeShort = inDat.readShort();
type = typeShort;
}
}
double[][] dataArray = new double[rows][cols];
for (int i = 0; i < cols; i++)
for (int j = 0; j < rows; j++)
dataArray[j][i] = readNumber(inDat, type, isLittleEndian);
String arrayName = "realPart";
if (JSCConstants.isValidName(name) == null)
arrayName = name;
JSCNArray realPart = new JSCNArray(arrayName, dataArray);
dataList.add(realPart);
// padding is not needed
inDat.close();
inBytes.close();
}
/**
* Reads a number from the stream according to the given type and whether it
* is little endian or not.
*
* @param inDat
* @param mLabNumberType
* @param littleEndian
* @return
* @throws IOException
*/
private double readNumber(DataInputStream inDat, int mLabNumberType,
boolean littleEndian) throws IOException {
double number = Double.NaN;
if (littleEndian) {
switch (mLabNumberType) {
case miDOUBLE:
number = Double.longBitsToDouble(Long.reverseBytes(inDat
.readLong()));
break;
case miSINGLE:
number = Float.intBitsToFloat(Integer.reverseBytes(inDat
.readInt()));
break;
case miINT8:
number = inDat.readByte();
break;
case miUINT8:
number = inDat.readUnsignedByte();
break;
case miINT16:
short snum = Short.reverseBytes(inDat.readShort());
number = snum;
break;
case miUINT16:
byte[] numBytesS = new byte[2];
inDat.read(numBytesS);
int uinum16 = ((numBytesS[1] & 0xFF) << 8)
| (numBytesS[0] & 0xFF);
number = uinum16;
break;
case miINT32:
int inum = Integer.reverseBytes(inDat.readInt());
number = inum;
break;
case miUINT32:
byte[] numBytes = new byte[4];
inDat.read(numBytes);
long uinum32 = ((((long) (numBytes[3] & 0xFF)) << 24)
| (((long) (numBytes[2] & 0xFF)) << 16)
| (((long) (numBytes[1] & 0xFF)) << 8) | (numBytes[0] & 0xFF));
number = Double.longBitsToDouble(uinum32);
break;
case miINT64:
number = Double.longBitsToDouble(Long.reverseBytes(inDat
.readLong()));
break;
default:
// just ignore
break;
}
} else {
switch (mLabNumberType) {
case miDOUBLE:
number = inDat.readDouble();
break;
case miSINGLE:
number = inDat.readFloat();
break;
case miINT8:
number = inDat.readByte();
break;
case miUINT8:
number = inDat.readUnsignedByte();
break;
case miINT16:
number = inDat.readShort();
break;
case miUINT16:
number = inDat.readUnsignedShort();
break;
case miINT32:
number = inDat.readInt();
break;
case miUINT32:
byte[] numBytes = new byte[4];
inDat.read(numBytes);
long num = ((((long) (numBytes[0] & 0xFF)) << 24)
| (((long) (numBytes[1] & 0xFF)) << 16)
| (((long) (numBytes[2] & 0xFF)) << 8) | (numBytes[3] & 0xFF));
number = Double.longBitsToDouble(num);
break;
case miINT64:
number = inDat.readLong();
break;
default:
// just ignore
break;
}
}
return number;
}
/**
* Skips the bytes remaining to fill a padding -byte boundary
* after writing bytes. It returns the number of skipped
* bytes.
*
* @param in
* stream to skip bytes on, can be null (then only
* skipped bytes are computed)
* @param padding
* boundary that must be filled
* @param bytes
* actual number of bytes that should be appended with padding
* bytes to fill the boundary
* @return number of bytes skipped
* @throws IOException
*/
public static int skipPaddingBytes(DataInputStream in, int padding,
int bytes) throws IOException {
int rest = (bytes % padding);
int toSkip = 0;
if (rest > 0)
toSkip = padding - rest;
if (in != null)
in.skipBytes(toSkip);
return toSkip;
}
private DataInputStream miCompressed(DataInputStream in,
int numOfCompressedBytes) throws Exception {
byte[] readCompressed = new byte[numOfCompressedBytes];
in.read(readCompressed);
// decompress
Inflater decompresser = new Inflater();
decompresser.setInput(readCompressed, 0, numOfCompressedBytes);
byte[] dataTags = new byte[8];
decompresser.inflate(dataTags);
decompresser.end();
ByteArrayInputStream inBytes = new ByteArrayInputStream(dataTags);
DataInputStream inDat = new DataInputStream(inBytes);
// data type
int type = 0;
int numOfBytes = 0;
try {
type = inDat.readInt();
if (isLittleEndian)
type = Integer.reverseBytes(type);
// number of bytes
numOfBytes = inDat.readInt();
if (isLittleEndian)
numOfBytes = Integer.reverseBytes(numOfBytes);
} catch (Throwable t) {
log.error("Reading bytes failed.", t);
}
inBytes.close();
inDat.close();
// second run, get the whole data element
decompresser = new Inflater();
decompresser.setInput(readCompressed, 0, numOfCompressedBytes);
byte[] data = new byte[8 + numOfBytes];
decompresser.inflate(data);
decompresser.end();
inBytes = new ByteArrayInputStream(data);
inDat = new DataInputStream(inBytes);
return inDat;
}
/**
* Gets the imported data according to an instance of
* TSImportTypes.
*
* @param type
* the elementtype of a time series dataset
* @return the data stored as JSCData
* @throws IllegalArgumentException
* if (type == null)
* @throws IllegalStateException
* if nothing was imported before
*/
public JSCData getData(ImportTypes type) {
if (symbolTable == null)
throw new IllegalStateException("There was nothing imported yet.");
if (type == null)
throw new IllegalArgumentException("Argument was null.");
return symbolTable.getJSCData(new JSCTypeDef(type.name(), type.type()));
}
/**
* Gets the dialog used for raw ASCII files.
*/
private TSDataDialog getDataDialog() {
if (dialog == null)
dialog = new TSDataDialog(TopFrameReference.getTopFrameRef(), true);
dialog.setLocationRelativeTo(TopFrameReference.getTopFrameRef());
return dialog;
}
//
private void setDataToSymbolTable(String description) {
String[] namArray = new String[namesList.size()];
for (int i = 0; i < namArray.length; i++)
namArray[i] = namesList.get(i) + "";
getDataDialog().setVariableNames(namArray);
if (!noDialog) {
getDataDialog().setVisible(true);
if (getDataDialog().isCanceled()) {
cancelled = true;
return;
}
}
int allCols = 0;
int maxRows = 0;
JSCNArray[] arrays = new JSCNArray[dataList.size()];
for (int i = 0; i < dataList.size(); i++) {
arrays[i] = dataList.get(i);
allCols += arrays[i].cols();
int rows = arrays[i].rows();
if (rows > maxRows)
maxRows = rows;
}
double[][] values = UMatrix.multiply(new double[maxRows][allCols],
Double.NaN);
for (int i = 0, colIndex = 0; i < arrays.length; i++)
for (int j = 0; j < arrays[i].cols(); j++, colIndex++)
for (int k = 0; k < arrays[i].rows(); k++)
values[k][colIndex] = arrays[i].doubleAt(k, j);
symbolTable.set(new JSCNArray(TSImportTypes.DATA.name(), values));
symbolTable.set(new JSCDate(TSImportTypes.STARTDATE.name(),
getDataDialog().getTSDate()));
symbolTable.set(new JSCString(TSImportTypes.DESCRIPTION.name(),
description));
StringTokenizer st = new StringTokenizer(getDataDialog()
.getVariableNames());
String[] stringArray = new String[st.countTokens()];
for (int i = 0; i < stringArray.length; i++)
stringArray[i] = st.nextToken();
symbolTable.set(new JSCSArray(TSImportTypes.VARNAMES.name(),
stringArray));
}
public void exportTS(File file, TS[] ts, String description) {
if (file == null)
throw new IllegalArgumentException("File argument was null.");
if (ts == null)
throw new IllegalArgumentException("Time series argument was null.");
if (ts.length == 0)
return;
TSDateRange mergedRange = TSDateRange.getMergedRange(ts);
// data
String[] names = new String[ts.length];
for (int i = 0; i < names.length; i++)
names[i] = ts[i].name();
double[][] values = new double[mergedRange.numOfObs()][ts.length];
for (int i = 0; i < mergedRange.numOfObs(); i++) {
TSDate currentDate = mergedRange.dateForIndex(i);
for (int j = 0; j < ts.length; j++)
values[i][j] = ts[j].valueAt(currentDate);
}
export(values, names, description, file);
}
public void exportData(File file, JSCData data, String description) {
if (file == null)
throw new IllegalArgumentException("File argument was null.");
if (data == null)
throw new IllegalArgumentException("Data argument was null.");
if (data.type() != JSCTypes.NARRAY)
throw new IllegalArgumentException("Data of type \"" + data.type()
+ "\" cannot be stored in Excel format.");
if (data.isEmpty())
return;
JSCNArray narray = (JSCNArray) data;
export(narray.doubleArray(), new String[] { narray.name() },
description, file);
}
/**
* Writes standard data elements that belong to each Matlab
* miMatrix type.
*
* @param dataOut
* @param rows
* @param cols
* @param name
* @throws IOException
*/
private void writeDataHeader(DataOutputStream dataOut, int rows, int cols,
String name) throws IOException {
dataOut.writeInt(miMATRIX);
int numOfChars = name.length();
int skip = skipPaddingBytes(null, 8, numOfChars);
int arraySize = 8 * rows * cols;
int numOfBytes = 40 + numOfChars + skip + 8 + arraySize;
dataOut.writeInt(numOfBytes);
// flags
dataOut.writeInt(miUINT32);
dataOut.writeInt(8);
byte[] toWrite = new byte[2];
dataOut.write(toWrite);
// dataOut.writeByte(1 << 2);
dataOut.writeByte(0);
// class
dataOut.writeByte(mxDOUBLE_CLASS);
toWrite = new byte[4];
dataOut.write(toWrite);
// dimension
dataOut.writeInt(miINT32);
dataOut.writeInt(8);
dataOut.writeInt(rows);
dataOut.writeInt(cols);
// name
dataOut.writeInt(miINT8);
dataOut.writeInt(numOfChars);
dataOut.write(name.getBytes());
toWrite = new byte[skip];
dataOut.write(toWrite);
// data
dataOut.writeInt(miDOUBLE);
dataOut.writeInt(arraySize);
}
/**
* Compresses byte array held in byteOut and writes it to
* out.
*
* @param byteOut
* @param out
* @throws IOException
*/
private void deflateDataAndWrite(ByteArrayOutputStream byteOut,
DataOutputStream out) throws IOException {
// compress
Deflater compresser = new Deflater();
compresser.setInput(byteOut.toByteArray());
compresser.finish();
int numOfCompressedBytes = 0;
byte[] compressed = new byte[0];
for (int k = 0; !compresser.finished(); k++) {
if (k > 0) {
compresser = new Deflater();
compresser.setInput(byteOut.toByteArray());
compresser.finish();
}
compressed = new byte[byteOut.size() + 100 * k];
numOfCompressedBytes = compresser.deflate(compressed);
}
out.writeInt(miCOMPRESSED);
out.writeInt(numOfCompressedBytes);
out.write(compressed, 0, numOfCompressedBytes);
}
private void export(double[][] values, String[] names, String description,
File file) {
try {
DataOutputStream out = new DataOutputStream(new FileOutputStream(
file));
byte[] toWrite = new byte[116];
if (description == null)
description = "";
String des = "MATLAB 5.0 MAT-file, Created with JStatCom: "
+ CompSettings.getDateString() + ",\n" + description;
// only 116
byte[] sArray = des.getBytes();
for (int j = 0; j < sArray.length && j < toWrite.length; j++)
toWrite[j] = sArray[j];
out.write(toWrite);
toWrite = new byte[8];
out.write(toWrite);
out.writeByte(1);
out.writeByte(0);
out.writeByte('M');
out.writeByte('I');
// compute num of bytes
// in case of length 1 there is no difference
if (values[0].length == names.length) {// timeseries
for (int i = 0; i < names.length; i++) {
ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(byteOut);
int rows = values.length;
writeDataHeader(dataOut, rows, 1, names[i]);
for (int j = 0; j < rows; j++)
dataOut.writeDouble(values[j][i]);
deflateDataAndWrite(byteOut, out);
dataOut.close();
byteOut.close();
}
} else { // matrix
ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(byteOut);
int rows = values.length;
int cols = values[0].length;
writeDataHeader(dataOut, rows, cols, names[0]);
for (int i = 0; i < cols; i++)
for (int j = 0; j < rows; j++)
dataOut.writeDouble(values[j][i]);
deflateDataAndWrite(byteOut, out);
dataOut.close();
byteOut.close();
}
out.close();
} catch (Throwable ex) {
throw new RuntimeException(ex.getMessage());
}
}
/**
* Sets this handler in test mode. No user dialog is shown, default values
* are used instead. This is helpful for running automated unit tests.
*
* @param noDialog
*/
public void setTestMode(boolean noDialog) {
this.noDialog = noDialog;
}
}
Ads help maintain this website.