package jsat.text.stemming;
import java.util.HashMap;
/**
* Implements Lovins' stemming algorithm described here
* http://snowball.tartarus.org/algorithms/lovins/stemmer.html
*
* @author Edward Raff
*/
public class LovinsStemmer extends Stemmer
{
private static final long serialVersionUID = -3229865664217642197L;
//There are 11 ending hash maps, each postfixed with the number of characters
private static final HashMap ending11 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = 4064350307133685150L;
{
put("alistically", "B"); put("arizability", "A"); put("izationally", "B");
}};
private static final HashMap ending10 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -5247798032923242997L;
{
put("antialness", "A"); put("arisations", "A"); put("arizations", "A");
put("entialness", "A");
}};
private static final HashMap ending9 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -9153017770847287495L;
{
put("allically", "C"); put("antaneous", "A"); put("antiality", "A");
put("arisation", "A"); put("arization", "A"); put("ationally", "B");
put("ativeness", "A"); put("eableness", "E"); put("entations", "A");
put("entiality", "A"); put("entialize", "A"); put("entiation", "A");
put("ionalness", "A"); put("istically", "A"); put("itousness", "A");
put("izability", "A"); put("izational", "A");
}};
private static final HashMap ending8 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = 3671522347706544570L;
{
put("ableness", "A"); put("arizable", "A"); put("entation", "A");
put("entially", "A"); put("eousness", "A"); put("ibleness", "A");
put("icalness", "A"); put("ionalism", "A"); put("ionality", "A");
put("ionalize", "A"); put("iousness", "A"); put("izations", "A");
put("lessness", "A");
}};
private static final HashMap ending7 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -4697823317524161452L;
{
put("ability", "A"); put("aically", "A"); put("alistic", "B");
put("alities", "A"); put("ariness", "E"); put("aristic", "A");
put("arizing", "A"); put("ateness", "A"); put("atingly", "A");
put("ational", "B"); put("atively", "A"); put("ativism", "A");
put("elihood", "E"); put("encible", "A"); put("entally", "A");
put("entials", "A"); put("entiate", "A"); put("entness", "A");
put("fulness", "A"); put("ibility", "A"); put("icalism", "A");
put("icalist", "A"); put("icality", "A"); put("icalize", "A");
put("ication", "G"); put("icianry", "A"); put("ination", "A");
put("ingness", "A"); put("ionally", "A"); put("isation", "A");
put("ishness", "A"); put("istical", "A"); put("iteness", "A");
put("iveness", "A"); put("ivistic", "A"); put("ivities", "A");
put("ization", "F"); put("izement", "A"); put("oidally", "A");
put("ousness", "A");
}};
private static final HashMap ending6 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -7030401064572348271L;
{
put("aceous", "A"); put("acious", "B"); put("action", "G");
put("alness", "A"); put("ancial", "A"); put("ancies", "A");
put("ancing", "B"); put("ariser", "A"); put("arized", "A");
put("arizer", "A"); put("atable", "A"); put("ations", "B");
put("atives", "A"); put("eature", "Z"); put("efully", "A");
put("encies", "A"); put("encing", "A"); put("ential", "A");
put("enting", "C"); put("entist", "A"); put("eously", "A");
put("ialist", "A"); put("iality", "A"); put("ialize", "A");
put("ically", "A"); put("icance", "A"); put("icians", "A");
put("icists", "A"); put("ifully", "A"); put("ionals", "A");
put("ionate", "D"); put("ioning", "A"); put("ionist", "A");
put("iously", "A"); put("istics", "A"); put("izable", "E");
put("lessly", "A"); put("nesses", "A"); put("oidism", "A");
}};
private static final HashMap ending5 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -5282435864116373834L;
{
put("acies", "A"); put("acity", "A"); put("aging", "B");
put("aical", "A"); put("alism", "B"); put("ality", "A");
put("alize", "A"); put("allic", "b"); put("anced", "B");
put("ances", "B"); put("antic", "C"); put("arial", "A");
put("aries", "A"); put("arily", "A"); put("arity", "B");
put("arize", "A"); put("aroid", "A"); put("ately", "A");
put("ating", "I"); put("ation", "B"); put("ative", "A");
put("ators", "A"); put("atory", "A"); put("ature", "E");
put("early", "Y"); put("ehood", "A"); put("eless", "A");
put("ement", "A"); put("enced", "A"); put("ences", "A");
put("eness", "E"); put("ening", "E"); put("ental", "A");
put("ented", "C"); put("ently", "A"); put("fully", "A");
put("ially", "A"); put("icant", "A"); put("ician", "A");
put("icide", "A"); put("icism", "A"); put("icist", "A");
put("icity", "A"); put("idine", "I"); put("iedly", "A");
put("ihood", "A"); put("inate", "A"); put("iness", "A");
put("ingly", "B"); put("inism", "J"); put("inity", "c");
put("ional", "A"); put("ioned", "A"); put("ished", "A");
put("istic", "A"); put("ities", "A"); put("itous", "A");
put("ively", "A"); put("ivity", "A"); put("izers", "F");
put("izing", "F"); put("oidal", "A"); put("oides", "A");
put("otide", "A"); put("ously", "A");
}};
private static final HashMap ending4 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -7293777277850278026L;
{
put("able", "A"); put("ably", "A"); put("ages", "B");
put("ally", "B"); put("ance", "B"); put("ancy", "B");
put("ants", "B"); put("aric", "A"); put("arly", "K");
put("ated", "I"); put("ates", "A"); put("atic", "B");
put("ator", "A"); put("ealy", "Y"); put("edly", "E");
put("eful", "A"); put("eity", "A"); put("ence", "A");
put("ency", "A"); put("ened", "E"); put("enly", "E");
put("eous", "A"); put("hood", "A"); put("ials", "A");
put("ians", "A"); put("ible", "A"); put("ibly", "A");
put("ical", "A"); put("ides", "L"); put("iers", "A");
put("iful", "A"); put("ines", "M"); put("ings", "N");
put("ions", "B"); put("ious", "A"); put("isms", "B");
put("ists", "A"); put("itic", "H"); put("ized", "F");
put("izer", "F"); put("less", "A"); put("lily", "A");
put("ness", "A"); put("ogen", "A"); put("ward", "A");
put("wise", "A"); put("ying", "B"); put("yish", "A");
}};
private static final HashMap ending3 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -5629841014950478203L;
{
put("acy", "A"); put("age", "B"); put("aic", "A");
put("als", "b"); put("ant", "B"); put("ars", "O");
put("ary", "F"); put("ata", "A"); put("ate", "A");
put("eal", "Y"); put("ear", "Y"); put("ely", "E");
put("ene", "E"); put("ent", "C"); put("ery", "E");
put("ese", "A"); put("ful", "A"); put("ial", "A");
put("ian", "A"); put("ics", "A"); put("ide", "L");
put("ied", "A"); put("ier", "A"); put("ies", "P");
put("ily", "A"); put("ine", "M"); put("ing", "N");
put("ion", "Q"); put("ish", "C"); put("ism", "B");
put("ist", "A"); put("ite", "a"); put("ity", "A");
put("ium", "A"); put("ive", "A"); put("ize", "F");
put("oid", "A"); put("one", "R"); put("ous", "A");
}};
private static final HashMap ending2 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -8894812965945848256L;
{
put("ae", "A"); put("al", "b"); put("ar", "X");
put("as", "B"); put("ed", "E"); put("en", "F");
put("es", "E"); put("ia", "A"); put("ic", "A");
put("is", "A"); put("ly", "B"); put("on", "S");
put("or", "T"); put("um", "U"); put("us", "V");
put("yl", "R"); put("s\'", "A"); put("\'s", "A");
}};
private static final HashMap ending1 = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -7536643426902207427L;
{
put("a", "A"); put("e", "A"); put("i", "A");
put("o", "A"); put("s", "W"); put("y", "B");
}};
private static final HashMap endings = new HashMap()
{/**
*
*/
private static final long serialVersionUID = -8057392854617089310L;
{
putAll(ending11); putAll(ending10); putAll(ending9); putAll(ending8); putAll(ending7); putAll(ending6);
putAll(ending5); putAll(ending4); putAll(ending3); putAll(ending2); putAll(ending1);
}};
private static String removeEnding(String word)
{
//The stem must contain at least 2 characters, so word-2 is the min
for(int i = Math.min(11, word.length()-2); i > 0; i--)
{
String ending = word.substring(word.length()-i);
String condition = endings.get(ending);
if(condition == null)
continue;
String stem = word.substring(0, word.length()-i);
switch(condition.charAt(0))
{
case 'A'://No restrictions on stem
return stem;
case 'B': //Minimum stem length = 3
if(stem.length() >= 3)
return stem;
break;
case 'C': //Minimum stem length = 4
if(stem.length() >= 4)
return stem;
break;
case 'D'://Minimum stem length = 5
if(stem.length() >= 5)
return stem;
break;
case 'E'://Do not remove ending after e
if(stem.endsWith("e"))
break;
return stem;
case 'F'://Minimum stem length = 3 and do not remove ending after e
if(stem.endsWith("e") || stem.length() < 3)
break;
return stem;
case 'G'://Minimum stem length = 3 and remove ending only after f
if(stem.endsWith("f") && stem.length() >= 3)
return stem;
break;
case 'H'://Remove ending only after t or ll
if(stem.endsWith("t") || stem.endsWith("ll"))
return stem;
break;
case 'I'://Do not remove ending after o or e
if(stem.endsWith("o") || stem.endsWith("e"))
break;
return stem;
case 'J': //Do not remove ending after a or e
if(stem.endsWith("a") || stem.endsWith("e"))
break;
return stem;
case 'K'://Minimum stem length = 3 and remove ending only after l, i or u*e
if(stem.length() >= 3 && stem.matches(".*(i|u.e|l)$"))
return stem;
break;
case 'L'://Do not remove ending after u, x or s, unless s follows o
if(stem.endsWith("os"))
return stem;
else if(stem.matches(".*(u|x|s)$"))
break;
return stem;
case 'M'://Do not remove ending after a, c, e or m
if(stem.endsWith("a") || stem.endsWith("c") || stem.endsWith("e") || stem.endsWith("m"))
break;
else
return stem;
case 'N'://Minimum stem length = 4 after s**, elsewhere = 3
if (stem.matches(".*s..$"))
if (stem.length() >= 4)
return stem;
else
break;
else if (stem.length() >= 3)
return stem;
break;
case 'O'://Remove ending only after l or i
if(stem.endsWith("l") || stem.endsWith("i"))
return stem;
break;
case 'P'://Do not remove ending after c
if(stem.endsWith("e"))
break;
return stem;
case 'Q'://Minimum stem length = 3 and do not remove ending after l or n
if(stem.length() < 3 || stem.endsWith("l") || stem.endsWith("n"))
break;
return stem;
case 'R'://Remove ending only after n or r
if(stem.endsWith("n") || stem.endsWith("r"))
return stem;
break;
case 'S'://Remove ending only after dr or t, unless t follows t
if(stem.endsWith("dr") || (stem.endsWith("t") && !stem.endsWith("tt")))
return stem;
break;
case 'T'://Remove ending only after s or t, unless t follows o
if(stem.endsWith("s") || (stem.endsWith("t") && !stem.endsWith("ot")))
return stem;
break;
case 'U'://Remove ending only after l, m, n or r
if(stem.endsWith("l") || stem.endsWith("m") || stem.endsWith("n") || stem.endsWith("r"))
return stem;
break;
case 'V'://Remove ending only after c
if(stem.endsWith("c"))
return stem;
break;
case 'W'://Do not remove ending after s or u
if(stem.endsWith("s") || stem.endsWith("u"))
break;
return stem;
case 'X'://Remove ending only after l, i or u*e
if(stem.matches(".*(l|i|u.e)$"))
return stem;
break;
case 'Y'://Remove ending only after in
if(stem.endsWith("in"))
return stem;
break;
case 'Z'://Do not remove ending after f
if(stem.endsWith("f"))
break;
return stem;
case 'a'://AA: Remove ending only after d, f, ph, th, l, er, or, es or t
if(stem.matches(".*(d|f|ph|th|l|er|or|es|t)$"))
return stem;
break;
case 'b'://BB: Minimum stem length = 3 and do not remove ending after met or ryst
if(stem.length() < 3 || stem.endsWith("met") || stem.endsWith("ryst"))
break;
return stem;
case 'c'://CC: Remove ending only after l
if(stem.endsWith("l"))
return stem;
break;
}
}
return word;
}
private static String fixStem(String stem)
{
//Rule 1 remove one of double b, d, g, l, m, n, p, r, s, t
char lastChar = stem.charAt(stem.length()-1);
stem = stem.replaceFirst("(dd|bb|gg|ll|mm|nn|pp|rr|ss|tt)$", "" + lastChar);
//Rule 2
stem = stem.replaceFirst("iev$", "ief");
//Rule 3
stem = stem.replaceFirst("uct$", "uc");
//Rule 4
stem = stem.replaceFirst("umpt$", "um");
//Rule 5
stem = stem.replaceFirst("rpt$", "rb");
//Rule 6
stem = stem.replaceFirst("urs$", "ur");
//Rule 7
stem = stem.replaceFirst("istr$", "ister");
//Rule 7a
stem = stem.replaceFirst("metr$", "meter");
//Rule 8
stem = stem.replaceFirst("olv$", "olut");
//Rule 9
if(stem.endsWith("ul") && !stem.endsWith("aoiul"))
stem = stem.replaceFirst("[^aoi]ul$", "l");
//Rule 10
stem = stem.replaceFirst("bex$", "bic");
//Rule 11
stem = stem.replaceFirst("dex$", "dic");
//Rule 12
stem = stem.replaceFirst("pex$", "pic");
//Rule 13
stem = stem.replaceFirst("tex$", "tic");
//Rule 14
stem = stem.replaceFirst("ax$", "ac");
//Rule 15
stem = stem.replaceFirst("ex$", "ec");
//Rule 16
stem = stem.replaceFirst("ix$", "ic");
//Rule 17
stem = stem.replaceFirst("lux$", "luc");
//Rule 18
stem = stem.replaceFirst("uad$", "uas");
//Rule 19
stem = stem.replaceFirst("vad$", "vas");
//Rule 20
stem = stem.replaceFirst("cid$", "cis");
//Rule 21
stem = stem.replaceFirst("lid$", "lis");
//Rule 22
stem = stem.replaceFirst("erid$", "eris");
//Rule 23
stem = stem.replaceFirst("pand$", "pans");
//Rule 24
if(stem.endsWith("end") && !stem.endsWith("send"))
stem = stem.replaceFirst("[^s]end$", "ens");
//Rule 25
stem = stem.replaceFirst("ond$", "ons");
//Rule 26
stem = stem.replaceFirst("lud$", "lus");
//Rule 27
stem = stem.replaceFirst("rud$", "rus");
//Rule 28
stem = stem.replaceFirst("[^pt]her$", "hes");
//Rule 29
stem = stem.replaceFirst("mit$", "mis");
//Rule 30
if(stem.endsWith("ent") && !stem.endsWith("ment"))
stem = stem.replaceFirst("[^m]ent$", "ens");
//Rule 31
stem = stem.replaceFirst("ert$", "ers");
//Rule 32
if(stem.endsWith("et") && !stem.endsWith("net"))
stem = stem.replaceFirst("et$", "es");
//Rule 33
stem = stem.replaceFirst("yt$", "ys");
//Rule 34
stem = stem.replaceFirst("yz$", "ys");
return stem;
}
public String stem(String word)
{
return fixStem(removeEnding(word));
}
}
Ads help maintain this website.