package cc.mallet.topics;
import java.util.*;
import cc.mallet.types.*;
public class WordEmbeddingRunnable implements Runnable {
public WordEmbeddings model;
public InstanceList instances;
public int numSamples;
public boolean shouldRun = true;
double residual = 0.0;
int numUpdates = 0;
int numThreads;
int threadID;
int iteration = 0;
int stride, docID;
public Random random;
int numColumns;
int orderingStrategy = WordEmbeddings.LINEAR_ORDERING;
public long wordsSoFar = 0;
private int minDocumentLength;
public WordEmbeddingRunnable(WordEmbeddings model, InstanceList instances, int numSamples, int numThreads, int threadID) {
this.model = model;
this.stride = model.stride;
this.instances = instances;
this.numSamples = numSamples;
this.numThreads = numThreads;
this.threadID = threadID;
random = new Random();
numColumns = model.numColumns;
minDocumentLength = model.getMinDocumentLength();
}
public void setRandomSeed(int seed) {
random = new Random(seed);
}
public void setOrdering(int strategy) {
this.orderingStrategy = strategy;
}
public double getMeanError() {
if (numUpdates == 0) { return docID; }
double result = residual / numUpdates;
residual = 0.0;
numUpdates = 0;
return result;
}
public void run() {
long previousWordsSoFar = 0;
long wordsConsidered = 0;
long wordsSampled = 0;
int numDocuments = instances.size();
int inputTypeOffset, outputTypeOffset, sampledTypeOffset;
int inputType, outputType, sampledType;
int start, end, subWindow;
double innerProduct, weightedResidual;
double learningRate = 0.025;
double gradientSum = 0.0;
double[] gradient = new double[numColumns];
int minDoc = threadID * (numDocuments / numThreads);
int maxDoc = (threadID + 1) * (numDocuments / numThreads);
int numDocs = maxDoc - minDoc;
int[] agenda = new int[numDocs];
if (orderingStrategy == WordEmbeddings.SHUFFLED_ORDERING) {
for (int i = 0; i < numDocs; i++) {
agenda[i] = i;
}
for (int i = 0; i < numDocs; i++) {
int swapIndex = i + random.nextInt(numDocs - i);
int temp = agenda[swapIndex];
agenda[swapIndex] = agenda[i];
agenda[i] = temp;
}
}
else if (orderingStrategy == WordEmbeddings.RANDOM_ORDERING) {
for (int i = 0; i < numDocs; i++) {
agenda[i] = minDoc + random.nextInt(numDocs);
}
}
else { // Default: linear ordering
for (int i = 0; i < numDocs; i++) {
agenda[i] = minDoc + i;
}
}
docID = 0;
int maxDocID = (threadID + 1) * (numDocuments / numThreads);
if (maxDocID > numDocuments) {
maxDocID = numDocuments;
}
double cacheScale = (double) model.sigmoidCacheSize / (model.maxExpValue - model.minExpValue);
int[] tokenBuffer = new int[100000];
while (shouldRun) {
Instance instance = instances.get( agenda[docID] );
docID++;
if (docID == numDocs) {
// start over at the beginning
docID = 0;
iteration++;
if (iteration >= model.numIterations) {
shouldRun = false;
return;
}
}
if (wordsSoFar - previousWordsSoFar > 10000) {
learningRate = Math.max(0.025 * 0.0001, 0.025 * (1.0 - (double) numThreads * wordsSoFar / (model.numIterations * model.totalWords)));
previousWordsSoFar = wordsSoFar;
//System.out.format("%f\t%f\t%d\t%d\t%d\t%f\t%f\n", learningRate, 100.0 * wordsSoFar / model.totalWords, wordsSoFar, wordsConsidered, updates, (double) updates / wordsConsidered, gradientSum / updates);
}
double[] weights = model.weights;
double[] negativeWeights = model.negativeWeights;
FeatureSequence tokens = (FeatureSequence) instance.getData();
int originalLength = tokens.getLength();
int length = 0;
// Subsample the document, dropping frequent words frequently.
for (int inputPosition = 0; inputPosition < originalLength; inputPosition++) {
inputType = tokens.getIndexAtPosition(inputPosition);
wordsSoFar++;
if (random.nextDouble() < model.retentionProbability[inputType]) {
tokenBuffer[length] = inputType;
length++;
wordsSampled++;
}
}
// Skip short documents
assert minDocumentLength > 0;
if (length < minDocumentLength) { continue; }
for (int inputPosition = 0; inputPosition < length; inputPosition++) {
wordsConsidered++;
inputType = tokenBuffer[inputPosition];
inputTypeOffset = inputType * stride;
subWindow = random.nextInt(model.windowSize) + 1;
start = Math.max(0, inputPosition - subWindow);
end = Math.min(length - 1, inputPosition + subWindow);
for (int outputPosition = start; outputPosition <= end; outputPosition++) {
if (inputPosition == outputPosition) { continue; }
outputType = tokenBuffer[outputPosition];
//if (inputType == outputType) { continue; }
outputTypeOffset = outputType * stride;
innerProduct = 0.0;
for (int col = 0; col < numColumns; col++) {
innerProduct += negativeWeights[inputTypeOffset + col] * weights[outputTypeOffset + col];
}
if (innerProduct < model.minExpValue) {
weightedResidual = learningRate; // (1.0 - 0.0)
}
else if (innerProduct > model.maxExpValue) {
weightedResidual = 0.0; // (1.0 - 1.0)
}
else {
weightedResidual = learningRate * (1.0 - model.sigmoidCache[ (int) Math.floor( (innerProduct - model.minExpValue) * cacheScale ) ]);
}
for (int col = 0; col < numColumns; col++) {
gradient[col] = weightedResidual * negativeWeights[inputTypeOffset + col];
negativeWeights[inputTypeOffset + col] += weightedResidual * weights[outputTypeOffset + col];
}
double meanNegativePrediction = 0.0;
for (int sample = 0; sample < numSamples; sample++) {
sampledType = model.samplingTable[ random.nextInt(model.samplingTableSize) ];
if (sampledType == inputType) { continue; }
sampledTypeOffset = sampledType * stride;
innerProduct = 0.0;
for (int col = 0; col < numColumns; col++) {
innerProduct += negativeWeights[sampledTypeOffset + col] * weights[outputTypeOffset + col];
}
if (innerProduct < model.minExpValue) {
weightedResidual = 0.0; // (0.0 - 0.0)
}
else if (innerProduct > model.maxExpValue) {
weightedResidual = -learningRate; // (0.0 - 1.0)
}
else {
weightedResidual = learningRate * -model.sigmoidCache[ (int) Math.floor( (innerProduct - model.minExpValue) * cacheScale ) ];
}
for (int col = 0; col < numColumns; col++) {
gradient[col] += weightedResidual * negativeWeights[sampledTypeOffset + col];
negativeWeights[sampledTypeOffset + col] += weightedResidual * weights[outputTypeOffset + col];
}
}
//residual += prediction - meanNegativePrediction;
numUpdates++;
for (int col = 0; col < numColumns; col++) {
weights[outputTypeOffset + col] += gradient[col];
}
}
}
}
}
}
Ads help maintain this website.