"""
     * Example of how to use directly the algorithms of the framework in order to
     * perform classification. A similar approach can be used to perform clustering,
     * regression, build recommender system or perform topic modeling and dimensionality
     * reduction.
     *     
     * - datumbox.configuration.properties: It defines for the default storage engine (required)
     * - datumbox.concurrencyconfiguration.properties: It controls the concurrency levels (required)
     * - datumbox.inmemoryconfiguration.properties: It contains the configurations for the InMemory storage engine (required)
     * - datumbox.mapdbconfiguration.properties: It contains the configurations for the MapDB storage engine (optional)
     * - logback.xml: It contains the configuration file for the logger (optional)
"""


# get data
from jhplot import *
http="http://datamelt.org/examples/data/"
print Web.get(http+"diabetes.tsv.gz")


from com.datumbox.framework.common import Configuration
from com.datumbox.framework.core.common.dataobjects import Dataframe
from com.datumbox.framework.core.common.dataobjects import Record
from com.datumbox.framework.common.dataobjects import TypeInference
from com.datumbox.framework.common.utilities import RandomGenerator
from com.datumbox.framework.core.machinelearning import MLBuilder
from com.datumbox.framework.core.machinelearning.classification import SoftMaxRegression
from com.datumbox.framework.core.machinelearning.featureselection import PCA
from com.datumbox.framework.core.machinelearning.modelselection.metrics import ClassificationMetrics
from com.datumbox.framework.core.machinelearning.modelselection.splitters import ShuffleSplitter
from com.datumbox.framework.core.machinelearning.preprocessing import MinMaxScaler
from java.io import *
from java.util import Map,LinkedHashMap
from java.util.zip import GZIPInputStream

print "Create some configuration files.."

c=open("datumbox.configuration.default.properties","w")
s = """
    # The full package name of the Storage Engine. This determines the default storage engine:
    configuration.storageConfiguration=com.datumbox.framework.storage.inmemory.InMemoryConfiguration
    """
c.write(s)
c.close()

c=open("datumbox.concurrencyconfiguration.default.properties","w")
s = """
    # Whether the concurrent execution of tasks is allowed (options: true/false):
    concurrencyConfiguration.parallelized=true
    # The maximum number of Threads that can be executed concurrently for each task: 
    #   - Use 0 for setting it equal to the number of CPUs on the system.
    #   - Use 1 to turn off concurrency (same as concurrencyConfiguration.parallelized=false).
    #   - Any other positive value acts as a limit on the concurrency level, provided that the concurrencyConfiguration.parallelized=true.
    concurrencyConfiguration.maxNumberOfThreadsPerTask=0
    """
c.write(s)
c.close()


c=open("datumbox.inmemoryconfiguration.default.properties","w")
s="""# The relative or absolute path for the directory where the models are stored (if not specified the temporary directory is used):
    inMemoryConfiguration.directory=
  """
c.write(s)
c.close()

print "Initialization.." 
RandomGenerator.setGlobalSeed(42L)               # optionally set a specific seed for all Random objects
# configuration =Configuration(None,None)
configuration = Configuration.getConfiguration() # default configuration based on properties file
#configuration.setStorageConfiguration(InMemoryConfiguration())
# configuration.setStorageConfiguration(new InMemoryConfiguration()) //use In-Memory engine (default)
# configuration.setStorageConfiguration(new MapDBConfiguration()) //use MapDB engine
# configuration.getConcurrencyConfiguration().setParallelized(true) //turn on/off the parallelization
# configuration.getConcurrencyConfiguration().setMaxNumberOfThreadsPerTask(4) //set the concurrency level
               
print "Reading Data .."
     
fileReader = BufferedReader(InputStreamReader(GZIPInputStream(FileInputStream("diabetes.tsv.gz"))))
headerDataTypes = LinkedHashMap()
headerDataTypes.put("pregnancies", TypeInference.DataType.NUMERICAL)
headerDataTypes.put("plasma glucose", TypeInference.DataType.NUMERICAL)
headerDataTypes.put("blood pressure", TypeInference.DataType.NUMERICAL)
headerDataTypes.put("triceps thickness", TypeInference.DataType.NUMERICAL)
headerDataTypes.put("serum insulin", TypeInference.DataType.NUMERICAL)
headerDataTypes.put("bmi", TypeInference.DataType.NUMERICAL)
headerDataTypes.put("dpf", TypeInference.DataType.NUMERICAL)
headerDataTypes.put("age", TypeInference.DataType.NUMERICAL)
headerDataTypes.put("test result", TypeInference.DataType.CATEGORICAL)

data = Dataframe.Builder.parseCSVFile(fileReader, "test result", headerDataTypes, '\t', '"', "\r\n", None, None, configuration)
     
# Spit into train and test datasets
split = ShuffleSplitter(0.8, 1).split(data).next()
trainingDataframe = split.getTrain()
testingDataframe = split.getTest()
        
        
# Transform Dataframe
# Scale continuous variables
nsParams = MinMaxScaler.TrainingParameters()
numericalScaler = MLBuilder.create(nsParams, configuration)
numericalScaler.fit_transform(trainingDataframe)
numericalScaler.save("Diabetes")
        

# Feature Selection
# Perform dimensionality reduction using PCA
featureSelectionParameters = PCA.TrainingParameters()
featureSelectionParameters.setMaxDimensions(trainingDataframe.xColumnSize()-1)  # remove one dimension
featureSelectionParameters.setWhitened(False)
featureSelectionParameters.setVariancePercentageThreshold(0.99999995)

featureSelection = MLBuilder.create(featureSelectionParameters, configuration)
featureSelection.fit_transform(trainingDataframe)
featureSelection.save("Diabetes")
        
        
# Fit the classifier
param = SoftMaxRegression.TrainingParameters()
param.setTotalIterations(200)
param.setLearningRate(0.1)

classifier = MLBuilder.create(param, configuration)
classifier.fit(trainingDataframe)
classifier.save("Diabetes")
        
        
# Use the classifier
# Apply the same numerical scaling on testingDataframe
numericalScaler.transform(testingDataframe)
        
# Apply the same featureSelection transformations on testingDataframe
featureSelection.transform(testingDataframe)

# Use the classifier to make predictions on the testingDataframe
classifier.predict(testingDataframe)
        
# Get validation metrics on the test set
vm = ClassificationMetrics(testingDataframe)
        
print ("Results:")
for entry in testingDataframe.entries():
        rId = entry.getKey()
        r = entry.getValue()
        print "Record ",rId," - Real Y: ",r.getY(),", Predicted Y: ",r.getYPredicted()
    
              
# Clean up
# Delete scaler, featureselector and classifier.
numericalScaler.delete()
featureSelection.delete()
classifier.delete()
        
# Close Dataframes.
trainingDataframe.close()
testingDataframe.close()