"""
     * Example of how to use directly the algorithms of the framework in order to
     * perform Cluster Analysis. 
     * - datumbox.configuration.properties: It defines for the default storage engine (required)
     * - datumbox.concurrencyconfiguration.properties: It controls the concurrency levels (required)
     * - datumbox.inmemoryconfiguration.properties: It contains the configurations for the InMemory storage engine (required)
     * - datumbox.mapdbconfiguration.properties: It contains the configurations for the MapDB storage engine (optional)
     * - logback.xml: It contains the configuration file for the logger (optional)
"""


# get data
from jhplot import *
http="http://datamelt.org/examples/data/"
print Web.get(http+"heart.csv")


from com.datumbox.framework.common import Configuration
from com.datumbox.framework.core.common.dataobjects import Dataframe
from com.datumbox.framework.core.common.dataobjects import Record
from com.datumbox.framework.common.dataobjects import TypeInference
from com.datumbox.framework.common.utilities import RandomGenerator
from com.datumbox.framework.core.machinelearning.clustering import Kmeans;
from com.datumbox.framework.core.machinelearning import MLBuilder
from com.datumbox.framework.core.machinelearning.modelselection.metrics import ClusteringMetrics
from com.datumbox.framework.core.machinelearning.preprocessing import MinMaxScaler
from com.datumbox.framework.core.machinelearning.preprocessing import OneHotEncoder
from java.io import *
from java.util import Map,LinkedHashMap

print "Create some configuration files.."

c=open("datumbox.configuration.default.properties","w")
s = """
    # The full package name of the Storage Engine. This determines the default storage engine:
    configuration.storageConfiguration=com.datumbox.framework.storage.inmemory.InMemoryConfiguration
    """
c.write(s)
c.close()



c=open("datumbox.concurrencyconfiguration.default.properties","w")
s = """
    # Whether the concurrent execution of tasks is allowed (options: true/false):
    concurrencyConfiguration.parallelized=true
    # The maximum number of Threads that can be executed concurrently for each task: 
    #   - Use 0 for setting it equal to the number of CPUs on the system.
    #   - Use 1 to turn off concurrency (same as concurrencyConfiguration.parallelized=false).
    #   - Any other positive value acts as a limit on the concurrency level, provided that the concurrencyConfiguration.parallelized=true.
    concurrencyConfiguration.maxNumberOfThreadsPerTask=0
    """
c.write(s)
c.close()


c=open("datumbox.inmemoryconfiguration.default.properties","w")
s="""# The relative or absolute path for the directory where the models are stored (if not specified the temporary directory is used):
    inMemoryConfiguration.directory=
  """
c.write(s)
c.close()




print "Initialization.." 
RandomGenerator.setGlobalSeed(42L)               # optionally set a specific seed for all Random objects
# configuration =Configuration(None,None)
configuration = Configuration.getConfiguration() # default configuration based on properties file
#configuration.setStorageConfiguration(InMemoryConfiguration())
# configuration.setStorageConfiguration(new InMemoryConfiguration()) //use In-Memory engine (default)
# configuration.setStorageConfiguration(new MapDBConfiguration()) //use MapDB engine
# configuration.getConcurrencyConfiguration().setParallelized(true) //turn on/off the parallelization
# configuration.getConcurrencyConfiguration().setMaxNumberOfThreadsPerTask(4) //set the concurrency level
        
        
print "Reading Data .."
  
fileReader = InputStreamReader(FileInputStream("heart.csv"))
headerDataTypes = LinkedHashMap()
headerDataTypes.put("Age", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("Sex", TypeInference.DataType.CATEGORICAL);
headerDataTypes.put("ChestPain", TypeInference.DataType.CATEGORICAL);
headerDataTypes.put("RestBP", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("Cholesterol", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("BloodSugar", TypeInference.DataType.BOOLEAN);
headerDataTypes.put("ECG", TypeInference.DataType.CATEGORICAL); 
headerDataTypes.put("MaxHeartRate", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("Angina", TypeInference.DataType.BOOLEAN);
headerDataTypes.put("OldPeak", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("STSlope", TypeInference.DataType.ORDINAL);
headerDataTypes.put("Vessels", TypeInference.DataType.NUMERICAL);
headerDataTypes.put("Thal", TypeInference.DataType.CATEGORICAL);
headerDataTypes.put("Class", TypeInference.DataType.CATEGORICAL);
            
trainingDataframe = Dataframe.Builder.parseCSVFile(fileReader, "Class", headerDataTypes, ',', '"', "\r\n", None, None, configuration);
     


# Store data and load them back
trainingDataframe.save("HeartDeseaseDataset");
testingDataframe = Dataframe.Builder.load("HeartDeseaseDataset", configuration);
        
        
# Transform Dataframe
# Convert Categorical variables to dummy variables (boolean) and scale continuous variables
nsParams =MinMaxScaler.TrainingParameters()
numericalScaler = MLBuilder.create(nsParams, configuration)

numericalScaler.fit_transform(trainingDataframe);
numericalScaler.save("HeartDesease");

ceParams = OneHotEncoder.TrainingParameters();
categoricalEncoder = MLBuilder.create(ceParams, configuration);

categoricalEncoder.fit_transform(trainingDataframe);
categoricalEncoder.save("HeartDesease");
        
        
        
# Fit the clusterer
param = Kmeans.TrainingParameters();
param.setK(2);
param.setMaxIterations(200);
param.setInitializationMethod(Kmeans.TrainingParameters.Initialization.FORGY);
param.setDistanceMethod(Kmeans.TrainingParameters.Distance.EUCLIDIAN);
param.setWeighted(False);
param.setCategoricalGamaMultiplier(1.0);
param.setSubsetFurthestFirstcValue(2.0);

clusterer = MLBuilder.create(param, configuration);
clusterer.fit(trainingDataframe);
clusterer.save("HeartDesease");
        
        
# Use the clusterer
# Apply the same scaling and encoding on testingDataframe
numericalScaler.transform(testingDataframe);
categoricalEncoder.transform(testingDataframe);

# Make predictions on the test set
clusterer.predict(testingDataframe);
        
# Get validation metrics on the test set
vm=ClusteringMetrics(testingDataframe);
        
print ("Results:")
for entry in testingDataframe.entries():
        rId = entry.getKey()
        r = entry.getValue()
        print "Record ",rId," - Real Y: ",r.getY(),", Predicted Y: ",r.getYPredicted()
    
              
# Clean up
# Delete scaler, featureselector and classifier.
numericalScaler.delete()
clusterer.delete();
 
# Close Dataframes.
trainingDataframe.close()
testingDataframe.close()