package cc.mallet.topics;
import cc.mallet.util.Randoms;
import cc.mallet.util.CommandOption;
import cc.mallet.types.*;
import java.util.*;
import java.io.*;
import java.util.concurrent.*;
public class WordEmbeddings {
static CommandOption.String inputFile = new CommandOption.String(WordEmbeddings.class, "input", "FILENAME", true, null,
"The filename from which to read the list of training instances. Use - for stdin. " +
"The instances must be FeatureSequence or FeatureSequenceWithBigrams, not FeatureVector", null);
static CommandOption.String outputFile = new CommandOption.String(WordEmbeddings.class, "output", "FILENAME", true, "weights.txt",
"The filename to write text-formatted word vectors.", null);
static CommandOption.String outputContextFile = new CommandOption.String(WordEmbeddings.class, "output-context", "FILENAME", true, "NONE",
"The filename to write text-formatted context vectors.", null);
static CommandOption.Integer numDimensions = new CommandOption.Integer(WordEmbeddings.class, "num-dimensions", "INTEGER", true, 50,
"The number of dimensions to fit.", null);
static CommandOption.Integer windowSizeOption = new CommandOption.Integer(WordEmbeddings.class, "window-size", "INTEGER", true, 5,
"The number of adjacent words to consider.", null);
static CommandOption.Integer numThreads = new CommandOption.Integer(WordEmbeddings.class, "num-threads", "INTEGER", true, 1,
"The number of threads for parallel training.", null);
static CommandOption.Integer numIterationsOption = new CommandOption.Integer(WordEmbeddings.class, "num-iters", "INTEGER", true, 3,
"The number of passes through the training data.", null);
static CommandOption.Double samplingFactorOption = new CommandOption.Double(WordEmbeddings.class, "frequency-factor", "NUMBER", true, 0.0001,
"Down-sample words that account for more than ~2.5x this proportion or the corpus.", null);
static CommandOption.Integer numSamples = new CommandOption.Integer(WordEmbeddings.class, "num-samples", "INTEGER", true, 5,
"The number of negative samples to use in training.", null);
static CommandOption.String exampleWord = new CommandOption.String(WordEmbeddings.class, "example-word", "STRING", true, null,
"If defined, periodically show the closest vectors to this word.", null);
static CommandOption.String orderingOption = new CommandOption.String(WordEmbeddings.class, "ordering", "STRING", true, "linear",
"\"linear\" reads documents in order, \"shuffled\" reads in random order, \"random\" selects documents at random and may repeat/drop documents", null);
public static final int LINEAR_ORDERING = 0;
public static final int SHUFFLED_ORDERING = 1;
public static final int RANDOM_ORDERING = 2;
Alphabet vocabulary;
int numWords;
int numColumns;
double[] weights;
double[] negativeWeights;
int stride;
int numIterations;
int[] wordCounts;
double[] retentionProbability;
double[] samplingDistribution;
int[] samplingTable;
int samplingTableSize = 100000000;
double samplingSum = 0.0;
int totalWords = 0;
double maxExpValue = 6.0;
double minExpValue = -6.0;
double[] sigmoidCache;
int sigmoidCacheSize = 1000;
int windowSize = 5;
IDSorter[] sortedWords = null;
int orderingStrategy = LINEAR_ORDERING;
public int getMinDocumentLength() {
return minDocumentLength;
}
public void setMinDocumentLength(int minDocumentLength) {
if (minDocumentLength <= 0) {
throw new IllegalArgumentException("Minimum document length must be at least 1.");
}
this.minDocumentLength = minDocumentLength;
}
private int minDocumentLength = 10;
public void setNumIterations(int i) { numIterations = i; }
public String getQueryWord() {
return queryWord;
}
public void setQueryWord(String queryWord) {
this.queryWord = queryWord;
}
String queryWord = "the";
Randoms random = new Randoms();
public WordEmbeddings() { }
public WordEmbeddings(Alphabet a, int numColumns, int windowSize) {
vocabulary = a;
numWords = vocabulary.size();
System.out.format("Vocab size: %d\n", numWords);
this.numColumns = numColumns;
this.stride = numColumns;
weights = new double[numWords * stride];
negativeWeights = new double[numWords * stride];
for (int word = 0; word < numWords; word++) {
for (int col = 0; col < numColumns; col++) {
weights[word * stride + col] = (random.nextDouble() - 0.5f) / numColumns;
negativeWeights[word * stride + col] = 0.0;
}
}
wordCounts = new int[numWords];
samplingDistribution = new double[numWords];
retentionProbability = new double[numWords];
samplingTable = new int[samplingTableSize];
this.windowSize = windowSize;
sigmoidCache = new double[sigmoidCacheSize + 1];
for (int i = 0; i < sigmoidCacheSize; i++) {
double value = ((double) i / sigmoidCacheSize) * (maxExpValue - minExpValue) + minExpValue;
sigmoidCache[i] = 1.0 / (1.0 + Math.exp(-value));
}
}
public void initializeSortables() {
sortedWords = new IDSorter[numWords];
for (int word = 0; word < numWords; word++) {
sortedWords[word] = new IDSorter(word, 0.0);
}
}
public void countWords(InstanceList instances, double samplingFactor) {
for (Instance instance: instances) {
FeatureSequence tokens = (FeatureSequence) instance.getData();
int length = tokens.getLength();
for (int position = 0; position < length; position++) {
int type = tokens.getIndexAtPosition(position);
wordCounts[type]++;
}
totalWords += length;
}
for (int word = 0; word < numWords; word++) {
// Word2Vec style sampling weight
double frequencyScore = (double) wordCounts[word] / (samplingFactor * totalWords);
retentionProbability[word] = Math.min((Math.sqrt(frequencyScore) + 1) / frequencyScore, 1.0);
}
if (sortedWords == null) {
initializeSortables();
}
for (int word = 0; word < numWords; word++) {
sortedWords[word].set(word, wordCounts[word]);
}
Arrays.sort(sortedWords);
samplingDistribution[0] = Math.pow(wordCounts[ sortedWords[0].getID() ], 0.75);
for (int word = 1; word < numWords; word++) {
samplingDistribution[word] = samplingDistribution[word-1] + Math.pow(wordCounts[ sortedWords[word].getID() ], 0.75);
}
samplingSum = samplingDistribution[numWords-1];
int order = 0;
for (int i = 0; i < samplingTableSize; i++) {
samplingTable[i] = sortedWords[order].getID();
while (samplingSum * i / samplingTableSize > samplingDistribution[order]) {
order++;
}
}
System.out.println("done counting: " + totalWords);
}
public void train(InstanceList instances, int numThreads, int numSamples) {
ExecutorService executor = Executors.newFixedThreadPool(numThreads);
WordEmbeddingRunnable[] runnables = new WordEmbeddingRunnable[numThreads];
for (int thread = 0; thread < numThreads; thread++) {
runnables[thread] = new WordEmbeddingRunnable(this, instances, numSamples, numThreads, thread);
runnables[thread].setOrdering(orderingStrategy);
executor.submit(runnables[thread]);
}
long startTime = System.currentTimeMillis();
double difference = 0.0;
boolean finished = false;
while (! finished) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
}
int wordsSoFar = 0;
// Are all the threads done?
boolean anyRunning = false;
for (int thread = 0; thread < numThreads; thread++) {
if (runnables[thread].shouldRun) { anyRunning = true; }
wordsSoFar += runnables[thread].wordsSoFar;
//System.out.format("%.3f ", runnables[thread].getMeanError());
}
long runningMillis = System.currentTimeMillis() - startTime;
System.out.format("%d\t%d\t%fk w/s %f avg\n", wordsSoFar, runningMillis, (double) wordsSoFar / runningMillis,
averageAbsWeight());
//variances();
difference = 0.0;
if (! anyRunning || wordsSoFar > numIterations * totalWords) {
finished = true;
for (int thread = 0; thread < numThreads; thread++) {
runnables[thread].shouldRun = false;
}
}
if (queryWord != null && vocabulary.contains(queryWord)) {
findClosest(copy(queryWord));
}
}
executor.shutdownNow();
}
public void findClosest(double[] targetVector) {
if (sortedWords == null) {
initializeSortables();
}
double targetSquaredSum = 0.0;
for (int col = 0; col < numColumns; col++) {
targetSquaredSum += targetVector[col] * targetVector[col];
}
double targetNormalizer = 1.0 / Math.sqrt(targetSquaredSum);
for (int word = 0; word < numWords; word++) {
double innerProduct = 0.0;
double wordSquaredSum = 0.0;
for (int col = 0; col < numColumns; col++) {
wordSquaredSum += weights[word * stride + col] * weights[word * stride + col];
}
double wordNormalizer = 1.0 / Math.sqrt(wordSquaredSum);
for (int col = 0; col < numColumns; col++) {
innerProduct += targetVector[col] * weights[word * stride + col];
}
innerProduct *= targetNormalizer * wordNormalizer;
sortedWords[word].set(word, innerProduct);
}
Arrays.sort(sortedWords);
for (int i = 0; i < 10; i++) {
System.out.format("%f\t%d\t%s\n", sortedWords[i].getWeight(), sortedWords[i].getID(), vocabulary.lookupObject(sortedWords[i].getID()));
}
}
public double averageAbsWeight() {
double sum = 0.0;
for (int word = 0; word < numWords; word++) {
for (int col = 0; col < numColumns; col++) {
sum += Math.abs(weights[word * stride + col]);
}
}
return sum / (numWords * numColumns);
}
public double[] variances() {
double[] means = new double[numColumns];
for (int word = 0; word < numWords; word++) {
for (int col = 0; col < numColumns; col++) {
means[col] += weights[word * stride + col];
}
}
for (int col = 0; col < numColumns; col++) {
means[col] /= numWords;
}
double[] squaredSums = new double[numColumns];
double diff;
for (int word = 0; word < numWords; word++) {
for (int col = 0; col < numColumns; col++) {
diff = weights[word * stride + col] - means[col];
squaredSums[col] += diff * diff;
}
}
for (int col = 0; col < numColumns; col++) {
squaredSums[col] /= (numWords - 1);
System.out.format("%f\t", squaredSums[col]);
}
System.out.println();
return squaredSums;
}
public void write(PrintWriter out) {
for (int word = 0; word < numWords; word++) {
Formatter buffer = new Formatter(Locale.US);
buffer.format("%s", vocabulary.lookupObject(word));
for (int col = 0; col < numColumns; col++) {
buffer.format(" %.6f", weights[word * stride + col]);
}
out.println(buffer);
}
}
public void writeContext(PrintWriter out) {
for (int word = 0; word < numWords; word++) {
Formatter buffer = new Formatter(Locale.US);
buffer.format("%s", vocabulary.lookupObject(word));
for (int col = 0; col < numColumns; col++) {
buffer.format(" %.6f", negativeWeights[word * stride + col]);
}
out.println(buffer);
}
}
public double[] copy(String word) {
return copy(vocabulary.lookupIndex(word));
}
public double[] copy(int word) {
double[] result = new double[numColumns];
for (int col = 0; col < numColumns; col++) {
result[col] = weights[word * stride + col];
}
return result;
}
public double[] add(double[] result, String word) {
return add(result, vocabulary.lookupIndex(word));
}
public double[] add(double[] result, int word) {
for (int col = 0; col < numColumns; col++) {
result[col] += weights[word * stride + col];
}
return result;
}
public double[] subtract(double[] result, String word) {
return subtract(result, vocabulary.lookupIndex(word));
}
public double[] subtract(double[] result, int word) {
for (int col = 0; col < numColumns; col++) {
result[col] -= weights[word * stride + col];
}
return result;
}
public static void main (String[] args) throws Exception {
// Process the command-line options
CommandOption.setSummary (WordEmbeddings.class,
"Train continuous word embeddings using the skip-gram method with negative sampling.");
CommandOption.process (WordEmbeddings.class, args);
InstanceList instances = InstanceList.load(new File(inputFile.value));
WordEmbeddings matrix = new WordEmbeddings(instances.getDataAlphabet(), numDimensions.value, windowSizeOption.value);
matrix.queryWord = exampleWord.value;
matrix.setNumIterations(numIterationsOption.value);
matrix.countWords(instances, samplingFactorOption.value);
if (orderingOption.value != null) {
if (orderingOption.value.startsWith("s")) { matrix.orderingStrategy = SHUFFLED_ORDERING; }
else if (orderingOption.value.startsWith("l")) { matrix.orderingStrategy = LINEAR_ORDERING; }
else if (orderingOption.value.startsWith("r")) { matrix.orderingStrategy = RANDOM_ORDERING; }
else {
System.err.println("Unrecognized ordering: " + orderingOption.value + ", using linear.");
}
}
matrix.train(instances, numThreads.value, numSamples.value);
PrintWriter out = new PrintWriter(outputFile.value);
matrix.write(out);
out.close();
if (outputContextFile.value != null) {
out = new PrintWriter(outputContextFile.value);
matrix.writeContext(out);
out.close();
}
}
}
Ads help maintain this website.