package cc.mallet.topics;

import cc.mallet.optimize.LimitedMemoryBFGS;
import cc.mallet.optimize.OptimizationException;

import cc.mallet.util.*;
import cc.mallet.types.*;
import cc.mallet.classify.MaxEnt;
import cc.mallet.pipe.Pipe;
import cc.mallet.pipe.Noop;

import java.io.*;
import java.util.concurrent.*;
import java.util.Arrays;
import java.util.ArrayList;

import java.text.NumberFormat;

public class DMRTopicModel extends ParallelTopicModel {

	static CommandOption.String inputFile = new CommandOption.String(DMRTopicModel.class, "input", "FILENAME", true, null,
		  "Filename for DMR topic model. Each instance should have features in its Target field.", null);

	static CommandOption.String outputModelFilename = new CommandOption.String(DMRTopicModel.class, "output-model", "FILENAME", true, null,
		  "The filename in which to write the binary topic model at the end of the iterations.  " +
		 "By default this is null, indicating that no file will be written.", null);

	static CommandOption.String outputParametersFilename = new CommandOption.String(DMRTopicModel.class, "output-params", "FILENAME", true, "parameters.tsv",
		  "The filename in which to write the DMR parameters.", null);

	static CommandOption.String inputModelFilename = new CommandOption.String(DMRTopicModel.class, "input-model", "FILENAME", true, null,
		  "The filename from which to read the binary topic model to which the --input will be appended, " +
		  "allowing incremental training.  " +
		 "By default this is null, indicating that no file will be read.", null);

	static CommandOption.String inferencerFilename = new CommandOption.String(DMRTopicModel.class, "inferencer-filename", "FILENAME", true, null,
		  "A topic inferencer applies a previously trained topic model to new documents.  " +
		 "By default this is null, indicating that no file will be written.", null);

	static CommandOption.String evaluatorFilename = new CommandOption.String(DMRTopicModel.class, "evaluator-filename", "FILENAME", true, null,
		  "A held-out likelihood evaluator for new documents.  " +
		 "By default this is null, indicating that no file will be written.", null);

	static CommandOption.String stateFile = new CommandOption.String(DMRTopicModel.class, "output-state", "FILENAME", true, null,
		  "The filename in which to write the Gibbs sampling state after at the end of the iterations.  " +
		 "By default this is null, indicating that no file will be written.", null);

	static CommandOption.String topicKeysFile = new CommandOption.String(DMRTopicModel.class, "output-topic-keys", "FILENAME", true, null,
		  "The filename in which to write the top words for each topic and any Dirichlet parameters.  " +
		 "By default this is null, indicating that no file will be written.", null);

	static CommandOption.String docTopicsFile = new CommandOption.String(DMRTopicModel.class, "output-doc-topics", "FILENAME", true, null,
		  "The filename in which to write the topic proportions per document, at the end of the iterations.  " +
		 "By default this is null, indicating that no file will be written.", null);

	static CommandOption.Double docTopicsThreshold = new CommandOption.Double(DMRTopicModel.class, "doc-topics-threshold", "DECIMAL", true, 0.0,
		  "When writing topic proportions per document with --output-doc-topics, " +
		 "do not print topics with proportions less than this threshold value.", null);

	static CommandOption.Integer docTopicsMax = new CommandOption.Integer(DMRTopicModel.class, "doc-topics-max", "INTEGER", true, -1,
		  "When writing topic proportions per document with --output-doc-topics, " +
		  "do not print more than INTEGER number of topics.  "+
		 "A negative value indicates that all topics should be printed.", null);

	static CommandOption.Integer outputStateIntervalOption = new CommandOption.Integer(DMRTopicModel.class, "output-state-interval", "INTEGER", true, 0,
		  "The number of iterations between writing the sampling state to a text file.  " +
		 "You must also set the --output-state to use this option, whose argument will be the prefix of the filenames.", null);

	static CommandOption.Integer outputParametersIntervalOption = new CommandOption.Integer(DMRTopicModel.class, "output-params-interval", "INTEGER", true, 0,
		  "The number of iterations between writing the DMR parameters to a text file.  " +
		 "You must also set the --output-params to use this option, whose argument will be the prefix of the filenames.", null);

	static CommandOption.Integer numTopicsOption = new CommandOption.Integer(DMRTopicModel.class, "num-topics", "INTEGER", true, 10,
		 "The number of topics to fit.", null);

	static CommandOption.Integer numThreadsOption = new CommandOption.Integer(DMRTopicModel.class, "num-threads", "INTEGER", true, 1,
		 "The number of threads to run in parallel.", null);

	static CommandOption.Integer numIterationsOption = new CommandOption.Integer(DMRTopicModel.class, "num-iterations", "INTEGER", true, 1000,
		 "The number of iterations of Gibbs sampling.", null);

	static CommandOption.Integer randomSeedOption = new CommandOption.Integer(DMRTopicModel.class, "random-seed", "INTEGER", true, 0,
		 "The random seed for the Gibbs sampler.  Default is 0, which will use the clock.", null);

	static CommandOption.Integer topWordsOption = new CommandOption.Integer(DMRTopicModel.class, "num-top-words", "INTEGER", true, 20,
		 "The number of most probable words to print for each topic after model estimation.", null);

	static CommandOption.Integer showTopicsIntervalOption = new CommandOption.Integer(DMRTopicModel.class, "show-topics-interval", "INTEGER", true, 50,
		 "The number of iterations between printing a brief summary of the topics so far.", null);

	static CommandOption.Integer optimizeIntervalOption = new CommandOption.Integer(DMRTopicModel.class, "optimize-interval", "INTEGER", true, 20,
		 "The number of iterations between reestimating DMR parameters.", null);

	static CommandOption.Integer optimizeBurnInOption = new CommandOption.Integer(DMRTopicModel.class, "optimize-burn-in", "INTEGER", true, 20,
		 "The number of iterations to run before first estimating DMR parameters.", null);

	static CommandOption.Double alphaOption = new CommandOption.Double(DMRTopicModel.class, "alpha", "DECIMAL", true, 50.0,
		 "Alpha parameter: initial smoothing over topic distribution.",null);

	static CommandOption.Double betaOption = new CommandOption.Double(DMRTopicModel.class, "beta", "DECIMAL", true, 0.01,
		 "Beta parameter: smoothing over unigram distribution.",null);

	MaxEnt dmrParameters = null;
    int numFeatures;
    int defaultFeatureIndex;

    Pipe parameterPipe = null;
	
	double[][] alphaCache;
    double[] alphaSumCache;
	
	double initialAlpha = 0.1;
	
	public String parametersFilename = "dmr_parameters.txt";
	public int saveParametersInterval = 0;

	public DMRTopicModel(int numberOfTopics) {
		super(numberOfTopics);
	}

	public void addInstances (InstanceList training) {

		alphabet = training.getDataAlphabet();
		numTypes = alphabet.size();
		
		betaSum = beta * numTypes;

		Randoms random = null;
		if (randomSeed == -1) {
			random = new Randoms();
		}
		else {
			random = new Randoms(randomSeed);
		}

		alphaCache = new double[ training.size() ][numTopics];
		alphaSumCache = new double[ training.size() ];
		
		int index = 0;
		for (Instance instance : training) {
			FeatureSequence tokens = (FeatureSequence) instance.getData();
			LabelSequence topicSequence =
				new LabelSequence(topicAlphabet, new int[ tokens.size() ]);
			
			int[] topics = topicSequence.getFeatures();
			for (int position = 0; position < topics.length; position++) {

				int topic = random.nextInt(numTopics);
				topics[position] = topic;
				
			}

			TopicAssignment t = new TopicAssignment(instance, topicSequence);
			data.add(t);
			
			Arrays.fill(alphaCache[index], initialAlpha);
			alphaSumCache[index] = numTopics * initialAlpha;
			
			index++;
		}
		
		alphaCache = new double[ data.size() ][numTopics];
		alphaSumCache = new double[ data.size() ];
		
		buildInitialTypeTopicCounts();
	
		// Create a "fake" pipe with the features in the data and 
		//  an int-int hashmap of topic counts in the target.
	
		parameterPipe = new Noop();
		Alphabet featureAlphabet = training.getTargetAlphabet();
		numFeatures = featureAlphabet.size() + 1;

		parameterPipe.setDataAlphabet(featureAlphabet);
		parameterPipe.setTargetAlphabet(topicAlphabet);

		dmrParameters = new MaxEnt(parameterPipe, new double[numFeatures * numTopics]);
		defaultFeatureIndex = dmrParameters.getDefaultFeatureIndex();
		double logInitialAlpha = Math.log(initialAlpha);
		for (int topic=0; topic < numTopics; topic++) {
			dmrParameters.setParameter(topic, defaultFeatureIndex, logInitialAlpha);
		}
		
		for (int doc=0; doc < data.size(); doc++) {
			cacheAlphas(data.get(doc).instance, doc);
		}
		
		totalTokens = 0;
		for (int doc = 0; doc < data.size(); doc++) {
			FeatureSequence fs = (FeatureSequence) data.get(doc).instance.getData();
			totalTokens += fs.getLength();
		}
	}


	public void estimate () throws IOException {

		long startTime = System.currentTimeMillis();

		DMRRunnable[] runnables = new DMRRunnable[numThreads];

		int docsPerThread = data.size() / numThreads;
		int offset = 0;

		if (numThreads > 1) {
		
			for (int thread = 0; thread < numThreads; thread++) {
				int[] runnableTotals = new int[numTopics];
				System.arraycopy(tokensPerTopic, 0, runnableTotals, 0, numTopics);
				
				int[][] runnableCounts = new int[numTypes][];
				for (int type = 0; type < numTypes; type++) {
					int[] counts = new int[typeTopicCounts[type].length];
					System.arraycopy(typeTopicCounts[type], 0, counts, 0, counts.length);
					runnableCounts[type] = counts;
				}
				
				// some docs may be missing at the end due to integer division
				if (thread == numThreads - 1) {
					docsPerThread = data.size() - offset;
				}
				
				Randoms random = null;
				if (randomSeed == -1) {
					random = new Randoms();
				}
				else {
					random = new Randoms(randomSeed);
				}

				runnables[thread] = new DMRRunnable(numTopics, this, beta, random, data, runnableCounts, runnableTotals, offset, docsPerThread);
				
				offset += docsPerThread;
			
			}
		}
		else {
			
			// If there is only one thread, copy the typeTopicCounts
			//  arrays directly, rather than allocating new memory.

			Randoms random = null;
			if (randomSeed == -1) {
				random = new Randoms();
			}
			else {
				random = new Randoms(randomSeed);
			}

			runnables[0] = new DMRRunnable(numTopics, this, beta, random, data, typeTopicCounts, tokensPerTopic, offset, docsPerThread);

			// If there is only one thread, we 
			//  can avoid communications overhead.
			// This switch informs the thread not to 
			//  gather statistics for its portion of the data.
			runnables[0].makeOnlyThread();
		}

		ExecutorService executor = Executors.newFixedThreadPool(numThreads);
	
		for (int iteration = 1; iteration <= numIterations; iteration++) {

			long iterationStart = System.currentTimeMillis();

			if (showTopicsInterval != 0 && iteration != 0 && iteration % showTopicsInterval == 0) {
				double[] parameters = dmrParameters.getParameters();
				for (int topic = 0; topic < numTopics; topic++) {
					alpha[topic] = Math.exp(parameters[topic*numFeatures + defaultFeatureIndex]);
				}
				logger.info("\n" + displayTopWords (wordsPerTopic, false));
			}

			if (saveStateInterval != 0 && iteration % saveStateInterval == 0) {
				this.printState(new File(stateFilename + '.' + iteration));
			}

			if (saveModelInterval != 0 && iteration % saveModelInterval == 0) {
				this.write(new File(modelFilename + '.' + iteration));
			}

			if (saveParametersInterval != 0 && iteration % saveParametersInterval == 0) {
				this.writeParameters(new File(parametersFilename + '.' + iteration));
			}

			if (numThreads > 1) {
			
				// Submit runnables to thread pool
				
				for (int thread = 0; thread < numThreads; thread++) {					
					logger.fine("submitting thread " + thread);
					executor.submit(runnables[thread]);
				}
				
				// I'm getting some problems that look like 
				//  a thread hasn't started yet when it is first
				//  polled, so it appears to be finished. 
				// This only occurs in very short corpora.
				try {
					Thread.sleep(20);
				} catch (InterruptedException e) {
					
				}
				
				boolean finished = false;
				while (! finished) {
					
					try {
						Thread.sleep(10);
					} catch (InterruptedException e) {
						
					}
					
					finished = true;
					
					// Are all the threads done?
					for (int thread = 0; thread < numThreads; thread++) {
						//logger.info("thread " + thread + " done? " + runnables[thread].isFinished);
						finished = finished && runnables[thread].isFinished;
					}
					
				}
				
				//System.out.print("[" + (System.currentTimeMillis() - iterationStart) + "] ");
				
				sumTypeTopicCounts(runnables);
				
				//System.out.print("[" + (System.currentTimeMillis() - iterationStart) + "] ");
				
				for (int thread = 0; thread < numThreads; thread++) {
					int[] runnableTotals = runnables[thread].getTokensPerTopic();
					System.arraycopy(tokensPerTopic, 0, runnableTotals, 0, numTopics);
					
					int[][] runnableCounts = runnables[thread].getTypeTopicCounts();
					for (int type = 0; type < numTypes; type++) {
						int[] targetCounts = runnableCounts[type];
						int[] sourceCounts = typeTopicCounts[type];
						
						int index = 0;
						while (index < sourceCounts.length) {
							
							if (sourceCounts[index] != 0) {
								targetCounts[index] = sourceCounts[index];
							}
							else if (targetCounts[index] != 0) {
								targetCounts[index] = 0;
							}
							else {
								break;
							}
							
							index++;
						}
						//System.arraycopy(typeTopicCounts[type], 0, counts, 0, counts.length);
					}
				}
			}
			else {
				runnables[0].run();
			}

			long elapsedMillis = System.currentTimeMillis() - iterationStart;
			if (elapsedMillis < 1000) {
				logger.fine(elapsedMillis + "ms ");
			}
			else {
				logger.fine((elapsedMillis/1000) + "s ");
			}

			if (iteration > burninPeriod && optimizeInterval != 0 &&
				iteration % optimizeInterval == 0) {

				// Optimize DMR parameters
				learnParameters();
				
				logger.fine("[O " + (System.currentTimeMillis() - iterationStart) + "] ");
			}
			
			if (iteration % 10 == 0) {
				if (printLogLikelihood) {
					logger.info ("<" + iteration + "> LL/token: " + formatter.format(modelLogLikelihood() / totalTokens));
				}
				else {
					logger.info ("<" + iteration + ">");
				}
			}
		}

		executor.shutdownNow();
	
		long seconds = Math.round((System.currentTimeMillis() - startTime)/1000.0);
		long minutes = seconds / 60;	seconds %= 60;
		long hours = minutes / 60;	minutes %= 60;
		long days = hours / 24;	hours %= 24;

		StringBuilder timeReport = new StringBuilder();
		timeReport.append("\nTotal time: ");
		if (days != 0) { timeReport.append(days); timeReport.append(" days "); }
		if (hours != 0) { timeReport.append(hours); timeReport.append(" hours "); }
		if (minutes != 0) { timeReport.append(minutes); timeReport.append(" minutes "); }
		timeReport.append(seconds); timeReport.append(" seconds");
		
		logger.info(timeReport.toString());
	}

	/**
	 *  Set alpha based on features in an instance
	 */
	public void cacheAlphas(Instance instance, int docIndex) {
		
		// we can't use the standard score functions from MaxEnt,
		//  since our features are currently in the Target.
		FeatureVector features = (FeatureVector) instance.getTarget();
		double[] parameters = dmrParameters.getParameters();
		
		alphaSumCache[docIndex] = 0.0;
		
		for (int topic = 0; topic < numTopics; topic++) {
			alphaCache[docIndex][topic] = parameters[topic*numFeatures + defaultFeatureIndex]
				+ MatrixOps.rowDotProduct (parameters, numFeatures, topic, features, defaultFeatureIndex, null);
			
			alphaCache[docIndex][topic] = Math.exp(alphaCache[docIndex][topic]);
			alphaSumCache[docIndex] += alphaCache[docIndex][topic];
		}
	}
	

	public void learnParameters() {

		// Create a "fake" pipe with the features in the data and 
		//  an int-int hashmap of topic counts in the target.
		
		if (parameterPipe == null) {
			parameterPipe = new Noop();

			parameterPipe.setDataAlphabet(data.get(0).instance.getTargetAlphabet());
			parameterPipe.setTargetAlphabet(topicAlphabet);
		}

		InstanceList parameterInstances = new InstanceList(parameterPipe);

		if (dmrParameters == null) {
			dmrParameters = new MaxEnt(parameterPipe, new double[numFeatures * numTopics]);
		}

		for (int doc=0; doc < data.size(); doc++) {
			
			if (data.get(doc).instance.getTarget() == null) {
				continue;
			}

			FeatureCounter counter = new FeatureCounter(topicAlphabet);

			for (int topic : data.get(doc).topicSequence.getFeatures()) {
				counter.increment(topic);
			}

			// Put the real target in the data field, and the
			//  topic counts in the target field
			parameterInstances.add( new Instance(data.get(doc).instance.getTarget(), counter.toFeatureVector(), null, null) );

		}

		DMROptimizable optimizable = new DMROptimizable(parameterInstances, dmrParameters);
		optimizable.setRegularGaussianPriorVariance(0.5);
		optimizable.setInterceptGaussianPriorVariance(100.0);

		LimitedMemoryBFGS optimizer = new LimitedMemoryBFGS(optimizable);

		// Optimize once
		try {
			optimizer.optimize();
		} catch (OptimizationException e) {
			// step size too small
		}

		// Restart with a fresh initialization to improve likelihood
		try {
			optimizer.optimize();
		} catch (OptimizationException e) {
			// step size too small
		}
		dmrParameters = optimizable.getClassifier();
		
		for (int doc=0; doc < data.size(); doc++) {
			cacheAlphas(data.get(doc).instance, doc);
		}
	}

	public void printTopWords (PrintStream out, int numWords, boolean usingNewLines) {
		super.printTopWords(out, numWords, usingNewLines);
	}

	public void writeParameters(File parameterFile) throws IOException {
		if (dmrParameters != null) {
			PrintStream out = new PrintStream(parameterFile);
			dmrParameters.print(out);
			out.close();
		}
	}
	
	public double modelLogLikelihood() {
		double logLikelihood = 0.0;
		int nonZeroTopics;

		// The likelihood of the model is a combination of a 
		// Dirichlet-multinomial for the words in each topic
		// and a Dirichlet-multinomial for the topics in each
		// document.

		// The likelihood function of a dirichlet multinomial is
		//	 Gamma( sum_i alpha_i )	 prod_i Gamma( alpha_i + N_i )
		//	prod_i Gamma( alpha_i )	  Gamma( sum_i (alpha_i + N_i) )

		// So the log likelihood is 
		//	logGamma ( sum_i alpha_i ) - logGamma ( sum_i (alpha_i + N_i) ) + 
		//	 sum_i [ logGamma( alpha_i + N_i) - logGamma( alpha_i ) ]

		// Do the documents first

		int[] topicCounts = new int[numTopics];
		int[] docTopics;

		for (int doc=0; doc < data.size(); doc++) {
			LabelSequence topicSequence =	(LabelSequence) data.get(doc).topicSequence;
			FeatureVector features = (FeatureVector) data.get(doc).instance.getTarget();
		
			double[] parameters = dmrParameters.getParameters();

			docTopics = topicSequence.getFeatures();

			for (int token=0; token < docTopics.length; token++) {
				topicCounts[ docTopics[token] ]++;
			}

			for (int topic=0; topic < numTopics; topic++) {
				if (topicCounts[topic] > 0) {
					logLikelihood += (Dirichlet.logGammaStirling(alphaCache[doc][topic] + topicCounts[topic]) -
									  Dirichlet.logGammaStirling(alphaCache[doc][topic]));
				}
			}

			// subtract the (count + parameter) sum term
			logLikelihood -= Dirichlet.logGammaStirling(alphaSumCache[doc] + docTopics.length) + Dirichlet.logGammaStirling(alphaSumCache[doc]);

			Arrays.fill(topicCounts, 0);
		}

		// And the topics

		// Count the number of type-topic pairs that are not just (logGamma(beta) - logGamma(beta))
		int nonZeroTypeTopics = 0;

		for (int type=0; type < numTypes; type++) {
			// reuse this array as a pointer

			topicCounts = typeTopicCounts[type];

			int index = 0;
			while (index < topicCounts.length &&
				   topicCounts[index] > 0) {
				int topic = topicCounts[index] & topicMask;
				int count = topicCounts[index] >> topicBits;
				
				nonZeroTypeTopics++;
				logLikelihood += Dirichlet.logGammaStirling(beta + count);

				if (Double.isNaN(logLikelihood)) {
					logger.warning("NaN in log likelihood calculation");
					return 0;
				}
				else if (Double.isInfinite(logLikelihood)) {
					logger.warning("infinite log likelihood");
					return 0;
				}

				index++;
			}
		}
	
		for (int topic=0; topic < numTopics; topic++) {
			logLikelihood -= 
				Dirichlet.logGammaStirling( (beta * numTypes) +
											tokensPerTopic[ topic ] );

			if (Double.isNaN(logLikelihood)) {
				logger.info("NaN after topic " + topic + " " + tokensPerTopic[ topic ]);
				return 0;
			}
			else if (Double.isInfinite(logLikelihood)) {
				logger.info("Infinite value after topic " + topic + " " + tokensPerTopic[ topic ]);
				return 0;
			}

		}
	
		// logGamma(|V|*beta) for every topic
		logLikelihood += 
			Dirichlet.logGammaStirling(beta * numTypes) * numTopics;

		// logGamma(beta) for all type/topic pairs with non-zero count
		logLikelihood -=
			Dirichlet.logGammaStirling(beta) * nonZeroTypeTopics;
		
		if (Double.isNaN(logLikelihood)) {
			logger.info("at the end");
		}
		else if (Double.isInfinite(logLikelihood)) {
			logger.info("Infinite value beta " + beta + " * " + numTypes);
			return 0;
		}

		return logLikelihood;
	}
	
	/** Return a tool for estimating topic distributions for new documents */
	public DMRInferencer getInferencer() {
		return new DMRInferencer(typeTopicCounts, tokensPerTopic, dmrParameters, alphabet, beta, betaSum);
	}

	// Serialization

	private static final long serialVersionUID = 1;
	private static final int CURRENT_SERIAL_VERSION = 0;
	private static final int NULL_INTEGER = -1;

	private void writeObject (ObjectOutputStream out) throws IOException {
		out.writeInt (CURRENT_SERIAL_VERSION);
		
		out.writeObject(dmrParameters);

		out.writeObject(data);
		out.writeObject(alphabet);
		out.writeObject(topicAlphabet);

		out.writeInt(numTopics);

		out.writeInt(topicMask);
		out.writeInt(topicBits);

		out.writeInt(numTypes);

		out.writeObject(alpha);
		out.writeDouble(alphaSum);
		out.writeDouble(beta);
		out.writeDouble(betaSum);

		out.writeObject(typeTopicCounts);
		out.writeObject(tokensPerTopic);

		out.writeObject(docLengthCounts);
		out.writeObject(topicDocCounts);

		out.writeInt(numIterations);
		out.writeInt(burninPeriod);
		out.writeInt(saveSampleInterval);
		out.writeInt(optimizeInterval);
		out.writeInt(showTopicsInterval);
		out.writeInt(wordsPerTopic);

		out.writeInt(saveStateInterval);
		out.writeObject(stateFilename);

		out.writeInt(saveModelInterval);
		out.writeObject(modelFilename);

		out.writeInt(randomSeed);
		out.writeObject(formatter);
		out.writeBoolean(printLogLikelihood);

		out.writeInt(numThreads);
	}

	private void readObject (ObjectInputStream in) throws IOException, ClassNotFoundException {
		
		int version = in.readInt ();

		dmrParameters = (MaxEnt) in.readObject();
		numFeatures = dmrParameters.getAlphabet().size();
		defaultFeatureIndex = dmrParameters.getDefaultFeatureIndex();

		data = (ArrayList) in.readObject ();
		alphabet = (Alphabet) in.readObject();
		topicAlphabet = (LabelAlphabet) in.readObject();
		
		numTopics = in.readInt();
		
		topicMask = in.readInt();
		topicBits = in.readInt();
		
		numTypes = in.readInt();
		
		alpha = (double[]) in.readObject();
		alphaSum = in.readDouble();
		beta = in.readDouble();
		betaSum = in.readDouble();
		
		typeTopicCounts = (int[][]) in.readObject();
		tokensPerTopic = (int[]) in.readObject();
		
		docLengthCounts = (int[]) in.readObject();
		topicDocCounts = (int[][]) in.readObject();
	
		numIterations = in.readInt();
		burninPeriod = in.readInt();
		saveSampleInterval = in.readInt();
		optimizeInterval = in.readInt();
		showTopicsInterval = in.readInt();
		wordsPerTopic = in.readInt();

		saveStateInterval = in.readInt();
		stateFilename = (String) in.readObject();
		
		saveModelInterval = in.readInt();
		modelFilename = (String) in.readObject();
		
		randomSeed = in.readInt();
		formatter = (NumberFormat) in.readObject();
		printLogLikelihood = in.readBoolean();

		numThreads = in.readInt();
	}

	public void write (File serializedModelFile) {
		try {
			ObjectOutputStream oos = new ObjectOutputStream (new FileOutputStream(serializedModelFile));
			oos.writeObject(this);
			oos.close();
		} catch (IOException e) {
			System.err.println("Problem serializing DMRTopicModel to file " +
							   serializedModelFile + ": " + e);
		}
	}

	public static DMRTopicModel read (File f) throws Exception {

		DMRTopicModel topicModel = null;

		ObjectInputStream ois = new ObjectInputStream (new FileInputStream(f));
		topicModel = (DMRTopicModel) ois.readObject();
		ois.close();

		return topicModel;
	}
	
	public static void main (String[] args) throws IOException {

		CommandOption.setSummary (DMRTopicModel.class,
								  "A tool for estimating, saving and printing diagnostics for topic models with arbitrary document features.");
		CommandOption.process (DMRTopicModel.class, args);

		if (! inputFile.wasInvoked()) {
			System.err.println("You must specify a data set with the --input option.");
			System.exit(1);
		}

		DMRTopicModel topicModel = null;

		if (inputModelFilename.value != null) {

			try {
				topicModel = DMRTopicModel.read(new File(inputModelFilename.value));
			} catch (Exception e) {
				System.err.println("Unable to restore saved topic model " + 
								   inputModelFilename.value + ": " + e);
				System.exit(1);
			}
		}
		else {
			InstanceList training = InstanceList.load(new File(inputFile.value));

			System.out.println ("Data loaded.");
		
			// For historical reasons we currently only support FeatureSequence data,
			//  not the FeatureVector, which is the default for the input functions.
			//  Provide a warning to avoid ClassCastExceptions.
			if (training.size() > 0 &&
				training.get(0) != null) {
				Object data = training.get(0).getData();
				if (! (data instanceof FeatureSequence)) {
					System.err.println("Topic modeling currently only supports feature sequences: use --keep-sequence option when importing data.");
					System.exit(1);
				}
			}
			
			topicModel = new DMRTopicModel (numTopicsOption.value);
			if (randomSeedOption.value != 0) {
				topicModel.setRandomSeed(randomSeedOption.value);
			}
			
			topicModel.addInstances(training);
		}

		topicModel.setNumThreads(numThreadsOption.value);
		topicModel.setTopicDisplay(showTopicsIntervalOption.value, topWordsOption.value);

		topicModel.setNumIterations(numIterationsOption.value);
		topicModel.setOptimizeInterval(optimizeIntervalOption.value);
		topicModel.setBurninPeriod(optimizeBurnInOption.value);

		if (outputStateIntervalOption.value != 0) {
			topicModel.setSaveState(outputStateIntervalOption.value, stateFile.value);
		}
		
		topicModel.parametersFilename = outputParametersFilename.value;
		topicModel.saveParametersInterval = outputParametersIntervalOption.value;

		topicModel.estimate();

		topicModel.writeParameters(new File(outputParametersFilename.value));

		if (topicKeysFile.value != null) {
			topicModel.printTopWords(new File(topicKeysFile.value), topWordsOption.value, false);
		}

		if (stateFile.value != null) {
			topicModel.printState (new File(stateFile.value));
		}

		if (docTopicsFile.value != null) {
			PrintWriter out = new PrintWriter (new FileWriter ((new File(docTopicsFile.value))));
			topicModel.printDocumentTopics(out, docTopicsThreshold.value, docTopicsMax.value);
			out.close();
		}

		if (inferencerFilename.value != null) {
			DMRInferencer inferencer = topicModel.getInferencer();
			
			try {
				ObjectOutputStream oos =
					new ObjectOutputStream(new FileOutputStream(inferencerFilename.value));
				oos.writeObject(topicModel.getInferencer());
				oos.close();
			} catch (Exception e) {
				System.err.println(e.getMessage());
			}
		}

		if (outputModelFilename.value != null) {
			assert (topicModel != null);
			
			topicModel.write(new File(outputModelFilename.value));
		}

	}
}
 

Ads help maintain this website.