package jsat.text.stemming;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* Implements Porter's stemming algorithm http://tartarus.org/~martin/PorterStemmer/def.txt .
* Implemented for ease of understanding and legibility rather than performance.
* @author Edward Raff
*/
public class PorterStemmer extends Stemmer
{
private static final long serialVersionUID = -3809291457988435043L;
static final Map step2_endings = new LinkedHashMap();
static
{
step2_endings.put("ational", "ate");
step2_endings.put("tional", "tion");
step2_endings.put("enci", "ence");
step2_endings.put("anci", "ance");
step2_endings.put("izer", "ize");
step2_endings.put("abli", "able");
step2_endings.put("alli", "al");
step2_endings.put("entli", "ent");
step2_endings.put("eli", "e");
step2_endings.put("ousli", "ous");
step2_endings.put("ization", "ize");
step2_endings.put("ation", "ate");
step2_endings.put("ator", "ate");
step2_endings.put("alsim", "al");
step2_endings.put("iveness", "ive");
step2_endings.put("fulness", "ful");
step2_endings.put("ousness", "ous");
step2_endings.put("aliti", "al");
step2_endings.put("iviti", "ive");
step2_endings.put("biliti", "ble");
}
static final Map step3_endings = new LinkedHashMap();
static
{
step3_endings.put("icate", "ic");
step3_endings.put("ative", "");
step3_endings.put("alize", "al");
step3_endings.put("iciti", "ic");
step3_endings.put("ical", "ic");
step3_endings.put("ful", "");
step3_endings.put("ness", "");
}
static final Map step4_endings = new LinkedHashMap();
static
{
step4_endings.put("al", "");
step4_endings.put("ance", "");
step4_endings.put("ence", "");
step4_endings.put("er", "");
step4_endings.put("ic", "");
step4_endings.put("able", "");
step4_endings.put("ible", "");
step4_endings.put("ant", "");
step4_endings.put("ement", "");
step4_endings.put("ment", "");
step4_endings.put("ent", "");
step4_endings.put("ion", "");
step4_endings.put("ou", "");
step4_endings.put("ism", "");
step4_endings.put("ate", "");
step4_endings.put("iti", "");
step4_endings.put("ous", "");
step4_endings.put("ive", "");
step4_endings.put("ize", "");
}
@Override
public String stem(String s)
{
String tmp;
//Step 1a
if (s.endsWith("sses"))
s = s.replaceAll("sses$", "ss");
else if (s.endsWith("ies"))
s = s.replaceAll("ies$", "i");
else if(s.endsWith("ss"))
{
//Do nothing
}
else if(s.endsWith("s"))
s = s.substring(0, s.length()-1);
//Step 1b
boolean step1b_specialCase = false;//If the second or third of the rules in Step 1b is successful
if (s.endsWith("eed"))
{
tmp = s.replaceAll("eed$", "ee");
if(measure(tmp) > 0)
s = tmp;
}
else if (s.endsWith("ed"))
{
tmp = s.replaceAll("ed$", "");
if(containsVowel(tmp))
{
s = tmp;
step1b_specialCase = true;
}
}
else if (s.endsWith("ing"))
{
tmp = s.replaceAll("ing$", "");
if(containsVowel(tmp))
{
s = tmp;
step1b_specialCase = true;
}
}
if (step1b_specialCase)
{
if (s.endsWith("at"))
s = s.concat("e");
else if (s.endsWith("bl"))
s = s.concat("e");
else if (s.endsWith("iz"))
s = s.concat("e");
else if(doubleConstant(s, 'l', 's', 'z'))
s = s.substring(0, s.length()-1);//remove last letter
else if(oRule(s) && measure(s) == 1)
s = s.concat("e");
}
//Step 1c
if(s.endsWith("y") && containsVowel(s.substring(0, s.length()-1)))
s = s.substring(0, s.length()-1).concat("i");
//Step 2
for (Map.Entry entry : step2_endings.entrySet())
if (s.endsWith(entry.getKey()))
{
tmp = s.replaceAll(entry.getKey() + "$", entry.getValue());
if (measure(tmp) > 0)
{
s = tmp;
break;
}
}
//Step 3
for (Map.Entry entry : step3_endings.entrySet())
if (s.endsWith(entry.getKey()))
{
tmp = s.replaceAll(entry.getKey() + "$", entry.getValue());
if (measure(tmp) > 0)
{
s = tmp;
break;
}
}
//Step 4
for (Map.Entry entry : step4_endings.entrySet())
if (s.endsWith(entry.getKey()))
{
if(s.endsWith("ion") && !(s.length() >= 4 && (s.charAt(s.length()-4) == 's' || s.charAt(s.length()-4) == 't')))
continue;//special case on ion, and they didn't match
tmp = s.replaceAll(entry.getKey() + "$", entry.getValue());
if (measure(tmp) > 1)
{
s = tmp;
break;
}
}
//Step 5a
if (s.endsWith("e"))
{
tmp = s.substring(0, s.length() - 1);
if(measure(tmp) > 1)
s = tmp;
else if(measure(tmp) == 1 && !oRule(tmp))
s = tmp;
}
//Step 5b
int lp = s.length()-1;
if(lp < 1)
return s;
if(s.charAt(lp) == s.charAt(lp-1) && s.charAt(lp) == 'l')
{
tmp = s.substring(0, s.length() - 1);
if(measure(tmp) > 1)
s = tmp;
}
return s;
}
private static int measure(String s)
{
return measure(s, 0, s.length());
}
private static int measure(String c, int start, int length)
{
//[C](VC){m}[V]
//Measure == the value of m in the above exprsion
int pos = start;
int m = 0;
//Move past first C, now we are detecing (VC){m}[V]
while(!isVowel(c, pos) && pos < (length - start))
pos++;
boolean vFollowedByC = false;
do
{
vFollowedByC = false;
while (isVowel(c, pos)&& pos < (length-start))
pos++;
while (!isVowel(c, pos) && pos < (length-start))
{
pos++;
vFollowedByC = true;
}
m++;
}
while (pos < (length - start) && vFollowedByC);
if(vFollowedByC)//VC <- endded like that, it counts
return m;
else//V <- ended in V, dosnt count
return m-1;
}
private static boolean isVowel(String s, int pos)
{
/*
* A \consonant\ in a word is a letter other than A, E, I, O or U, and other
* than Y preceded by a consonant.
*/
if (pos >= s.length())
return false;
switch (s.charAt(pos))
{
case 'a':
case 'e':
case 'i':
case 'o':
case 'u':
return true;
case 'y':
if (pos == s.length() - 1)//end of the array
return true;
return isVowel(s, pos + 1);//Y preceded by a constant is a Vowel
default:
return false;
}
}
/**
* *o - the stem ends cvc, where the second c is not W, X or Y (e.g. -WIL, -HOP).
*/
private static boolean oRule(String s)
{
int pos = s.length()-1;
if(pos < 2)
return false;
if (!isVowel(s, pos) && isVowel(s, pos - 1) && !isVowel(s, pos - 2))
{
switch (s.charAt(pos))
{
case 'w':
case 'x':
case 'y':
return false;
default:
return true;
}
}
return false;
}
private static boolean containsVowel(String s)
{
for (int i = 0; i < s.length(); i++)
if (isVowel(s, i))
return true;
return false;
}
private static boolean doubleConstant(String s, char... except)
{
if (s.length() <= 1)
return false;
char c;
if ((c = s.charAt(s.length() - 1)) == s.charAt(s.length() - 2))
{
for (char e : except)
if (c == e)
return false;
return true;
}
return false;
}
}
Ads help maintain this website.