# see http://cs.stanford.edu/people/karpathy/recurrentjs/
# This demo shows usage of the recurrentjs library that allows you to train deep Recurrent Neural Networks (RNN) and Long Short-Term Memory Networks (LSTM)


from java.util import Random

from recunn.datasets import TextGenerationUnbroken 
from recunn.datastructs import DataSet
from recunn.model import Model
from recunn.trainer import Trainer
from recunn.util import NeuralNetworkHelper
from jhplot import *


rng = Random()
totalSequences = 2000
initFromSaved = True # set this to false to start with a fresh model
sequenceMinLength = 10
sequenceMaxLength = 100
textSource = "LilDicky"
http="http://datamelt.org/examples/data/"
print Web.get(http+"/text/"+textSource+".txt")
data = TextGenerationUnbroken(textSource+".txt", totalSequences, sequenceMinLength, sequenceMaxLength, rng)
savePath = textSource+".ser"
overwriteSaved = True
TextGenerationUnbroken.reportSequenceLength = 500
bottleneckSize = 10 # one-hot input is squeezed through this
hiddenDimension = 200
hiddenLayers = 1
learningRate = 0.001
initParamsStdDev = 0.08

lstm = NeuralNetworkHelper.makeLstmWithInputBottleneck(
       data.inputDimension, bottleneckSize,
       hiddenDimension, hiddenLayers,
       data.outputDimension, data.getModelOutputUnitToUse(),
       initParamsStdDev, rng)

reportEveryNthEpoch = 10
trainingEpochs = 1000

Trainer.train(trainingEpochs, learningRate, lstm, data, reportEveryNthEpoch, initFromSaved, overwriteSaved, savePath, rng)