/*
 * Copyright (C) 2017 edraff
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see .
 */
package jsat.io;

import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import jsat.classifiers.CategoricalData;
import jsat.classifiers.DataPoint;

/**
 * This interface defines a contract by which datapoints may be written out to a
 * dataset file (such as a CSV) in an incremental fashion. This object is thread
 * safe for all methods.
 *
 * @author Edward Raff
 */
public abstract class DataWriter implements Closeable
{
    /**
     * Use a 10MB local buffer for each thread
     */
    private static final int LOCAL_BUFFER_SIZE = 1024*1024*10;
    /**
     * The list of all local buffers, used to make sure all data makes it out when {@link #finish() } is called. 
     */
    protected List all_buffers = Collections.synchronizedList(new ArrayList());
    /**
     * The destination to ultimately write the dataset to
     */
    protected final OutputStream out;
    /**
     * The type of dataset to be written out
     */
    public final DataSetType type;
    /**
     * the categorical feature information for the whole corpus
     */
    public final CategoricalData[] catInfo;
    /**
     * the number of numeric features for the whole corpus
     */
    public final int dim;
    
    public DataWriter(OutputStream out, CategoricalData[] catInfo, int dim, DataSetType type) throws IOException
    {
        this.out = out;
        this.type = type;
        this.catInfo = catInfo;
        this.dim = dim;
        writeHeader(catInfo, dim, type, out);
    }
    
    
    /**
     * The local buffers that writing will be done to. 
     */
    protected ThreadLocal local_baos = new ThreadLocal()
    {
        @Override
        protected ByteArrayOutputStream initialValue()
        {
            ByteArrayOutputStream baos = new ByteArrayOutputStream(LOCAL_BUFFER_SIZE);
            all_buffers.add(baos);
            return baos;
        }
    };
    
    
    abstract protected void writeHeader(CategoricalData[] catInfo, int dim, DataSetType type, OutputStream out) throws IOException;
    
    /**
     * Write out the given data point to the output stream
     * @param dp the data point to write to the file
     * @param label The associated label for this dataum. If {@link #type} is a
     * {@link DataSetType#SIMPLE} set, this value will be ignored. If
     * {@link DataSetType#CLASSIFICATION}, the value will be assumed to be an
     * integer class label.
     */
    public void writePoint(DataPoint dp, double label) throws IOException
    {
        ByteArrayOutputStream baos = local_baos.get();
        pointToBytes(dp, label, baos);
        if(baos.size() >= LOCAL_BUFFER_SIZE)//We've got a big chunk of data, lets dump it
            synchronized(out)
            {
                baos.writeTo(out);
                baos.reset();
            }
    }
    
    /**
     * This method converts a datapoint into the sequence of bytes used by the underlying file format. 
     * @param dp the data point to be converted to set of bytes
     * @param label the label of the point to convert to the set of bytes
     * @param byteOut the location to write the bytes to.
     */
    abstract protected void pointToBytes(DataPoint dp, double label, ByteArrayOutputStream byteOut);
    
    /**
     * To be called after all threads are done calling {@link #writePoint(jsat.classifiers.DataPoint, double) }. 
     */
    public synchronized void finish() throws IOException
    {
        synchronized(out)
        {
            for(ByteArrayOutputStream baos : all_buffers)
            {
                baos.writeTo(out);
                baos.reset();
            }
            out.flush();
        }
        
    }

    @Override
    public void close() throws IOException
    {
        finish();
        out.close();
    }
    
    public static enum DataSetType
    {
        SIMPLE, CLASSIFICATION, REGRESSION
    }
}

 

Ads help maintain this website.