/*
* Encog(tm) Core v3.4 - Java Version
* http://www.heatonresearch.com/encog/
* https://github.com/encog/encog-java-core
* Copyright 2008-2016 Heaton Research, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* For more information on Heaton Research copyrights, licenses
* and trademarks visit:
* http://www.heatonresearch.com/copyright
*/
package org.encog.ml.data.buffer;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
/**
* Used to access an Encog Binary Training file (*.EGB).
*
*/
public class EncogEGBFile {
/**
* The size of a double.
*/
public static final int DOUBLE_SIZE = Double.SIZE / 8;
/**
* The size of the file header.
*/
public static final int HEADER_SIZE = EncogEGBFile.DOUBLE_SIZE * 3;
/**
* The file that we are working with.
*/
private final File file;
/**
* The number of input values per record.
*/
private int inputCount;
/**
* The number of ideal values per record.
*/
private int idealCount;
/**
* The underlying file.
*/
private RandomAccessFile raf;
/**
* The file channel used.
*/
private FileChannel fc;
/**
* A byte buffer to hold the header.
*/
private final ByteBuffer headerBuffer;
/**
* A byte buffer to hold the records.
*/
private ByteBuffer recordBuffer;
/**
* The number of values in a record, this is the input and ideal combined.
*/
private int recordCount;
/**
* The size of a record.
*/
private int recordSize;
/**
* The number of records int he file.
*/
private int numberOfRecords;
/**
* Construct an EGB file.
*
* @param theFile
* The file.
*/
public EncogEGBFile(final File theFile) {
this.file = theFile;
this.headerBuffer = ByteBuffer.allocate(EncogEGBFile.HEADER_SIZE);
}
/**
* Add a column.
*
* @param col
* THe column to add.
* @param isInput
* Is this an input column?
*/
public void addColumn(final int col, final boolean isInput) {
try {
// process the file
// allocate buffers
final ByteBuffer readBuffer = ByteBuffer
.allocate(EncogEGBFile.DOUBLE_SIZE * 1024);
final ByteBuffer writeBuffer = ByteBuffer
.allocate(EncogEGBFile.DOUBLE_SIZE * 1024);
readBuffer.clear();
writeBuffer.clear();
readBuffer.order(ByteOrder.LITTLE_ENDIAN);
writeBuffer.order(ByteOrder.LITTLE_ENDIAN);
long readLocation = EncogEGBFile.HEADER_SIZE;
long writeLocation = EncogEGBFile.HEADER_SIZE;
int recordOffset = 0;
this.fc.position(readLocation);
this.fc.read(readBuffer);
readLocation = this.fc.position();
readBuffer.rewind();
boolean done = false;
int count = 0;
do {
// if there is more to read, then process it
if (readBuffer.hasRemaining()) {
final double d = readBuffer.getDouble();
// If this is the column to insert, add a zero
if (recordOffset == col) {
// do we need to cycle the write buffer?
writeLocation = checkWrite(writeBuffer, writeLocation);
writeBuffer.putDouble(0);
}
// write the existing value
writeLocation = checkWrite(writeBuffer, writeLocation);
writeBuffer.putDouble(d);
// keep track of where we are in a record.
recordOffset++;
if (recordOffset >= this.recordCount) {
recordOffset = 0;
count++;
// are we done?
if (count >= this.numberOfRecords) {
done = true;
}
}
} else {
// read more
readBuffer.clear();
readBuffer.order(ByteOrder.LITTLE_ENDIAN);
this.fc.position(readLocation);
this.fc.read(readBuffer);
readLocation = this.fc.position();
readBuffer.rewind();
}
} while (!done);
// write any remaining data in the write buffer
if (writeBuffer.position() > 0) {
writeBuffer.flip();
this.fc.write(writeBuffer, writeLocation);
}
// does it fall inside of input or ideal?
if (isInput) {
this.inputCount++;
this.recordCount++;
} else {
this.idealCount++;
this.recordCount++;
}
this.recordCount = this.inputCount + this.idealCount + 1;
this.recordSize = this.recordCount * EncogEGBFile.DOUBLE_SIZE;
// adjust file size
this.raf.setLength((this.numberOfRecords * this.recordSize)
+ EncogEGBFile.HEADER_SIZE);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Add a row.
*
* @param row
* Where to add the row.
*/
public void addRow(final int row) {
try {
this.numberOfRecords++;
this.raf.setLength((this.numberOfRecords * this.recordSize)
+ EncogEGBFile.HEADER_SIZE);
for (int i = this.numberOfRecords - 1; i >= row; i--) {
final int s = EncogEGBFile.HEADER_SIZE + (this.recordSize * i);
final int t = EncogEGBFile.HEADER_SIZE + (this.recordSize * i)
+ this.recordSize;
clear();
this.fc.read(this.recordBuffer, s);
this.recordBuffer.flip();
this.fc.write(this.recordBuffer, t);
}
clear();
for (int i = 0; i < this.recordCount-1; i++) {
this.recordBuffer.putDouble(0);
}
this.recordBuffer.putDouble(1.0);
this.recordBuffer.flip();
this.fc.write(this.recordBuffer, EncogEGBFile.HEADER_SIZE
+ (this.recordSize * row));
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Calculate the index for the specified row.
*
* @param row
* The row to calculate for.
* @return The index.
*/
private long calculateIndex(final long row) {
return (long)EncogEGBFile.HEADER_SIZE + (row * (long)this.recordSize);
}
/**
* Read a row and column.
*
* @param row
* The row, or record, to read.
* @param col
* The column to read.
* @return THe value read.
*/
private int calculateIndex(final int row, final int col) {
return EncogEGBFile.HEADER_SIZE + (row * this.recordSize)
+ (col * EncogEGBFile.DOUBLE_SIZE);
}
/**
* Check a write, make sure there is enough room to write.
*
* @param writeBuffer
* The buffer.
* @param inWriteLocation
* The write location.
* @return The new write location.
* @throws IOException
* If an IO error occurs.
*/
private long checkWrite(final ByteBuffer writeBuffer,
final long inWriteLocation) throws IOException {
long writeLocation = inWriteLocation;
if (!writeBuffer.hasRemaining()) {
this.fc.position(writeLocation);
writeBuffer.flip();
this.fc.write(writeBuffer);
writeLocation = this.fc.position();
writeBuffer.clear();
writeBuffer.order(ByteOrder.LITTLE_ENDIAN);
}
return writeLocation;
}
/**
* Clear the record buffer.
*/
private void clear() {
this.recordBuffer.clear();
this.recordBuffer.order(ByteOrder.LITTLE_ENDIAN);
}
/**
* Close the file.
*/
public void close() {
try {
if (this.raf != null) {
this.raf.close();
this.raf = null;
}
if (this.fc != null) {
this.fc.close();
this.fc = null;
}
System.gc();
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Create a new RGB file.
*
* @param theInputCount
* The input count.
* @param theIdealCount
* The ideal count.
*/
public void create(final int theInputCount, final int theIdealCount) {
try {
this.inputCount = theInputCount;
this.idealCount = theIdealCount;
final double[] input = new double[inputCount];
final double[] ideal = new double[idealCount];
this.file.delete();
this.raf = new RandomAccessFile(this.file, "rw");
this.raf.setLength(0);
this.fc = this.raf.getChannel();
this.headerBuffer.clear();
this.headerBuffer.order(ByteOrder.LITTLE_ENDIAN);
this.headerBuffer.put((byte) 'E');
this.headerBuffer.put((byte) 'N');
this.headerBuffer.put((byte) 'C');
this.headerBuffer.put((byte) 'O');
this.headerBuffer.put((byte) 'G');
this.headerBuffer.put((byte) '-');
this.headerBuffer.put((byte) '0');
this.headerBuffer.put((byte) '0');
this.headerBuffer.putDouble(input.length);
this.headerBuffer.putDouble(ideal.length);
this.numberOfRecords = 0;
this.recordCount = this.inputCount + this.idealCount + 1;
this.recordSize = this.recordCount * EncogEGBFile.DOUBLE_SIZE;
this.recordBuffer = ByteBuffer.allocate(this.recordSize);
this.headerBuffer.flip();
this.fc.write(this.headerBuffer);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Delete a column.
*
* @param col
* The column to delete.
*/
public void deleteCol(final int col) {
try {
// process the file
// allocate buffers
final ByteBuffer readBuffer = ByteBuffer
.allocate(EncogEGBFile.DOUBLE_SIZE * 1024);
final ByteBuffer writeBuffer = ByteBuffer
.allocate(EncogEGBFile.DOUBLE_SIZE * 1024);
readBuffer.clear();
writeBuffer.clear();
readBuffer.order(ByteOrder.LITTLE_ENDIAN);
writeBuffer.order(ByteOrder.LITTLE_ENDIAN);
long readLocation = EncogEGBFile.HEADER_SIZE;
long writeLocation = EncogEGBFile.HEADER_SIZE;
int recordOffset = 0;
this.fc.position(readLocation);
this.fc.read(readBuffer);
readLocation = this.fc.position();
readBuffer.rewind();
boolean done = false;
int count = 0;
do {
// if there is more to read, then process it
if (readBuffer.hasRemaining()) {
final double d = readBuffer.getDouble();
// skip the specified column, as we write
if (recordOffset != col) {
writeLocation = checkWrite(writeBuffer, writeLocation);
writeBuffer.putDouble(d);
}
// keep track of where we are in a record.
recordOffset++;
if (recordOffset >= this.recordCount) {
recordOffset = 0;
count++;
// are we done?
if (count >= this.numberOfRecords) {
done = true;
}
}
} else {
// read more
readBuffer.clear();
readBuffer.order(ByteOrder.LITTLE_ENDIAN);
this.fc.position(readLocation);
this.fc.read(readBuffer);
readLocation = this.fc.position();
readBuffer.rewind();
}
} while (!done);
// write any remaining data in the write buffer
if (writeBuffer.position() > 0) {
writeBuffer.flip();
this.fc.write(writeBuffer, writeLocation);
}
// does it fall inside of input or ideal?
if (col < this.inputCount) {
this.inputCount--;
this.recordCount--;
} else {
this.idealCount--;
this.recordCount--;
}
this.recordCount = this.inputCount + this.idealCount + 1;
this.recordSize = this.recordCount * EncogEGBFile.DOUBLE_SIZE;
// adjust file size
this.raf.setLength((this.numberOfRecords * this.recordSize)
+ EncogEGBFile.HEADER_SIZE);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Delete a row.
*
* @param row
* The row to delete.
*/
public void deleteRow(final int row) {
try {
for (int i = row; i < this.numberOfRecords - 1; i++) {
final int s = EncogEGBFile.HEADER_SIZE + (this.recordSize * i)
+ this.recordSize;
final int t = EncogEGBFile.HEADER_SIZE + (this.recordSize * i);
clear();
this.fc.read(this.recordBuffer, s);
this.recordBuffer.flip();
this.fc.write(this.recordBuffer, t);
}
this.numberOfRecords--;
this.raf.setLength((this.numberOfRecords * this.recordSize)
+ EncogEGBFile.HEADER_SIZE);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* @return the fc
*/
public FileChannel getFc() {
return this.fc;
}
/**
* @return the file
*/
public File getFile() {
return this.file;
}
/**
* @return the headerBuffer
*/
public ByteBuffer getHeaderBuffer() {
return this.headerBuffer;
}
/**
* @return the idealCount
*/
public int getIdealCount() {
return this.idealCount;
}
/**
* @return the inputCount
*/
public int getInputCount() {
return this.inputCount;
}
/**
* @return the numberOfRecords
*/
public int getNumberOfRecords() {
return this.numberOfRecords;
}
/**
* @return the raf
*/
public RandomAccessFile getRaf() {
return this.raf;
}
/**
* @return the recordBuffer
*/
public ByteBuffer getRecordBuffer() {
return this.recordBuffer;
}
/**
* @return the recordCount
*/
public int getRecordCount() {
return this.recordCount;
}
/**
* @return the recordSize
*/
public int getRecordSize() {
return this.recordSize;
}
/**
* OPen an existing EGB file.
*/
public void open() {
try {
this.raf = new RandomAccessFile(this.file, "rw");
this.fc = this.raf.getChannel();
this.headerBuffer.clear();
this.headerBuffer.order(ByteOrder.LITTLE_ENDIAN);
boolean isEncogFile = true;
this.fc.read(this.headerBuffer);
this.headerBuffer.position(0);
isEncogFile = isEncogFile ? this.headerBuffer.get() == 'E' : false;
isEncogFile = isEncogFile ? this.headerBuffer.get() == 'N' : false;
isEncogFile = isEncogFile ? this.headerBuffer.get() == 'C' : false;
isEncogFile = isEncogFile ? this.headerBuffer.get() == 'O' : false;
isEncogFile = isEncogFile ? this.headerBuffer.get() == 'G' : false;
isEncogFile = isEncogFile ? this.headerBuffer.get() == '-' : false;
if (!isEncogFile) {
throw new BufferedDataError(
"File is not a valid Encog binary file:"
+ this.file.toString());
}
final char v1 = (char) this.headerBuffer.get();
final char v2 = (char) this.headerBuffer.get();
final String versionStr = "" + v1 + v2;
try {
final int version = Integer.parseInt(versionStr);
if (version > 0) {
throw new BufferedDataError(
"File is from a newer version of Encog than is currently in use.");
}
} catch (final NumberFormatException ex) {
throw new BufferedDataError("File has invalid version number.");
}
this.inputCount = (int) this.headerBuffer.getDouble();
this.idealCount = (int) this.headerBuffer.getDouble();
this.recordCount = this.inputCount + this.idealCount + 1;
this.recordSize = this.recordCount * EncogEGBFile.DOUBLE_SIZE;
if( this.recordSize==0 ) {
this.numberOfRecords = 0;
} else {
this.numberOfRecords
= (int) ((this.file.length() - EncogEGBFile.HEADER_SIZE)
/ this.recordSize);
}
this.recordBuffer = ByteBuffer.allocate(this.recordSize);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Read a single double.
*
* @return The double read.
*/
public double read() {
try {
clear();
this.recordBuffer.limit(EncogEGBFile.DOUBLE_SIZE);
this.fc.read(this.recordBuffer);
this.recordBuffer.position(0);
return this.recordBuffer.getDouble();
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Read an array of doubles.
*
* @param d
* The array to read into.
*/
public void read(final double[] d) {
try {
clear();
this.recordBuffer.limit(EncogEGBFile.DOUBLE_SIZE * d.length);
this.fc.read(this.recordBuffer);
this.recordBuffer.position(0);
for (int i = 0; i < d.length; i++) {
d[i] = this.recordBuffer.getDouble();
}
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Read a double array at the specified record.
*
* @param row
* The record to read.
* @param d
* The array to read into.
*/
public void read(final int row, final double[] d) {
try {
clear();
this.recordBuffer.limit(EncogEGBFile.DOUBLE_SIZE * d.length);
this.fc.read(this.recordBuffer, calculateIndex(row));
this.recordBuffer.position(0);
for (int i = 0; i < this.recordCount; i++) {
d[i] = this.recordBuffer.getDouble();
}
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Read a row and column.
*
* @param row
* The row to read.
* @param col
* The column to read.
* @return The value read.
*/
public double read(final int row, final int col) {
try {
clear();
this.recordBuffer.limit(EncogEGBFile.DOUBLE_SIZE);
this.fc.read(this.recordBuffer, calculateIndex(row, col));
this.recordBuffer.position(0);
return this.recordBuffer.getDouble(0);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Set the current location to the specified row.
*
* @param row
* The row.
*/
public void setLocation(final int row) {
try {
this.fc.position(calculateIndex(row));
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Write a byte.
*
* @param b
* The byte to write.
*/
public void write(final byte b) {
try {
clear();
this.recordBuffer.put(b);
this.recordBuffer.flip();
this.fc.write(this.recordBuffer);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Write an array.
*
* @param v
* The array to write.
*/
public void write(final double[] v) {
try {
clear();
for (final double element : v) {
this.recordBuffer.putDouble(element);
}
this.recordBuffer.flip();
this.fc.write(this.recordBuffer);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Write an array at the specified record.
*
* @param row
* The record to write.
* @param v
* The array to write.
*/
public void write(final int row, final double[] v) {
try {
clear();
for (final double element : v) {
this.recordBuffer.putDouble(element);
}
this.recordBuffer.flip();
this.fc.write(this.recordBuffer);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
/**
* Write the specified row and column.
*
* @param row
* The row.
* @param col
* The column.
* @param v
* The value.
*/
public void write(final int row, final int col, final double v) {
try {
clear();
this.recordBuffer.putDouble(v);
this.recordBuffer.flip();
this.fc.write(this.recordBuffer, calculateIndex(row, col));
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
public void write(double d) {
try {
clear();
this.recordBuffer.putDouble(d);
this.recordBuffer.flip();
this.fc.write(this.recordBuffer);
} catch (final IOException ex) {
throw new BufferedDataError(ex);
}
}
}
Ads help maintain this website.