"""
     * Example of how to use directly the algorithms of the framework in order to
     * perform regression of complex data.
     * - datumbox.configuration.properties: It defines for the default storage engine (required)
     * - datumbox.concurrencyconfiguration.properties: It controls the concurrency levels (required)
     * - datumbox.inmemoryconfiguration.properties: It contains the configurations for the InMemory storage engine (required)
     * - datumbox.mapdbconfiguration.properties: It contains the configurations for the MapDB storage engine (optional)
     * - logback.xml: It contains the configuration file for the logger (optional)
"""

# get data
from jhplot import *
http="http://datamelt.org/examples/data/"
print Web.get(http+"longley.csv")

from com.datumbox.framework.common import Configuration
from com.datumbox.framework.core.common.dataobjects import Dataframe
from com.datumbox.framework.core.common.dataobjects import Record
from com.datumbox.framework.common.dataobjects import TypeInference
from com.datumbox.framework.common.utilities import RandomGenerator
from com.datumbox.framework.core.machinelearning import MLBuilder
from com.datumbox.framework.core.machinelearning.preprocessing import StandardScaler;
from com.datumbox.framework.core.machinelearning.featureselection import PCA;
from com.datumbox.framework.core.machinelearning.modelselection.metrics import LinearRegressionMetrics
from com.datumbox.framework.core.machinelearning.regression import MatrixLinearRegression
from java.io import *
from java.util import Map,LinkedHashMap

print "Create some configuration files.."
c=open("datumbox.configuration.default.properties","w")
s = """
    # The full package name of the Storage Engine. This determines the default storage engine:
    configuration.storageConfiguration=com.datumbox.framework.storage.inmemory.InMemoryConfiguration
    """
c.write(s)
c.close()



c=open("datumbox.concurrencyconfiguration.default.properties","w")
s = """
    # Whether the concurrent execution of tasks is allowed (options: true/false):
    concurrencyConfiguration.parallelized=true
    # The maximum number of Threads that can be executed concurrently for each task: 
    #   - Use 0 for setting it equal to the number of CPUs on the system.
    #   - Use 1 to turn off concurrency (same as concurrencyConfiguration.parallelized=false).
    #   - Any other positive value acts as a limit on the concurrency level, provided that the concurrencyConfiguration.parallelized=true.
    concurrencyConfiguration.maxNumberOfThreadsPerTask=0
    """
c.write(s)
c.close()


c=open("datumbox.inmemoryconfiguration.default.properties","w")
s="""# The relative or absolute path for the directory where the models are stored (if not specified the temporary directory is used):
    inMemoryConfiguration.directory=
  """
c.write(s)
c.close()

print "Initialization.." 
RandomGenerator.setGlobalSeed(42L)               # optionally set a specific seed for all Random objects
# configuration =Configuration(None,None)
configuration = Configuration.getConfiguration() # default configuration based on properties file
#configuration.setStorageConfiguration(InMemoryConfiguration())
# configuration.setStorageConfiguration(new InMemoryConfiguration()) //use In-Memory engine (default)
# configuration.setStorageConfiguration(new MapDBConfiguration()) //use MapDB engine
# configuration.getConcurrencyConfiguration().setParallelized(true) //turn on/off the parallelization
# configuration.getConcurrencyConfiguration().setMaxNumberOfThreadsPerTask(4) //set the concurrency level
        
print "Reading Data .."
fileReader = InputStreamReader(FileInputStream("longley.csv"))
headerDataTypes = LinkedHashMap()
headerDataTypes.put("Employed", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("GNP.deflator", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("GNP", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("Unemployed", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("Armed.Forces", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("Population", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("Year", TypeInference.DataType.NUMERICAL);
trainingDataframe = Dataframe.Builder.parseCSVFile(fileReader, "Employed", headerDataTypes, ',', '"', "\r\n", None, None, configuration);
testingDataframe = trainingDataframe.copy();

# Transform Dataframe
# Scale continuous variables
nsParams = StandardScaler.TrainingParameters();
nsParams.setScaleResponse(True);
numericalScaler = MLBuilder.create(nsParams, configuration);
numericalScaler.fit_transform(trainingDataframe);
numericalScaler.save("LaborStatistics");


# Feature Selection
# Perform dimensionality reduction using PCA
featureSelectionParameters = PCA.TrainingParameters();
featureSelectionParameters.setMaxDimensions(trainingDataframe.xColumnSize()-1); # remove one dimension
featureSelectionParameters.setWhitened(False);
featureSelectionParameters.setVariancePercentageThreshold(0.99999995);

featureSelection = MLBuilder.create(featureSelectionParameters, configuration);
featureSelection.fit_transform(trainingDataframe);
featureSelection.save("LaborStatistics");

# Fit the regressor
param = MatrixLinearRegression.TrainingParameters();
regressor = MLBuilder.create(param, configuration);
regressor.fit(trainingDataframe);
regressor.save("LaborStatistics");
regressor.close(); # close the regressor, we will use it again later

# Use the regressor
# Apply the same numerical scaling on testingDataframe
numericalScaler.transform(testingDataframe);
        
# Apply the same featureSelection transformations on testingDataframe
featureSelection.transform(testingDataframe);

# Load again the regressor
regressor = MLBuilder.load(MatrixLinearRegression, "LaborStatistics", configuration);
regressor.predict(testingDataframe);

# Get validation metrics on the training set
vm = LinearRegressionMetrics(testingDataframe);

print ("Results:")
for entry in testingDataframe.entries():
        rId = entry.getKey()
        r = entry.getValue()
        print "Record ",rId," - Real Y: ",r.getY(),", Predicted Y: ",r.getYPredicted()
print "Regressor Rsquare: ", vm.getRSquare()
              
# Clean up
numericalScaler.delete()
featureSelection.delete();
regressor.delete();
trainingDataframe.close();
testingDataframe.close();