package jsat.text.stemming;

/**
 * Provides an implementation of the Paice Husk stemmer as described in: 
* Paice, C. D. (1990). Another Stemmer. ACM SIGIR Forum, 4(3), 56–61. * @author Edward Raff */ public class PaiceHuskStemmer extends Stemmer { private static final long serialVersionUID = -5949389288166850651L; static private class Rule { /** * The ending to try and match */ public final String ending; /** * How many characters from the end of the string to remove */ public final int toRemove; /** * The string to append the ending with */ public final String newEnding; /** * Indicates that this rule may only be applied if the word has not been * modified before */ public final boolean virgin; /** * Indicates that the stemming process should exit if this rule is applied */ public final boolean terminal; public Rule(String ending, int toRemove, String newEnding, boolean virgin, boolean terminal) { this.ending = ending; this.toRemove = toRemove; this.newEnding = newEnding; this.virgin = virgin; this.terminal = terminal; } /** * If valid, returns the modified input string based on this rule. If no * modification was done, the exact same string that was passed in is * returned. An object comparison can then be done to check if the * string was modified.
* Appropriate handling of virgin and terminal flags is up to the user * @param input the unstemmed input * @return the stemmed output */ public String apply(String input) { if(input.endsWith(ending)) { if(isVowel(input.charAt(0))) { //Stats with a vowel, stemmed result must be at least 2 chars long if(input.length()-toRemove+newEnding.length() < 2) return input; } else//Starts with a consonant, 3 lets must remain { if(input.length()-toRemove+newEnding.length() < 3) return input;//Not long enought //Result must also contain at least one vowel boolean noVowels = true; for(int i = 0; i < input.length()-toRemove && noVowels; i++) if(isVowel(input.charAt(i)) || input.charAt(i)== 'y') noVowels = false; for(int i = 0; i < newEnding.length() && noVowels; i++) if(isVowel(newEnding.charAt(i)) || newEnding.charAt(i)== 'y') noVowels = false; if(noVowels) return input;//No vowels left, stemmin is not valid to aply } //We made it, we can apply the stem and return a new string if(toRemove == 0)//Proctected word, return a new string explicitly to be super sure return new String(input); return input.substring(0, input.length()-toRemove) + newEnding; } return input; } } /* * Oreded alphabetically by ending, meaning the rules should be attempted in * the order they are presented in the array */ private static final Rule[] ARules = new Rule[] { new Rule("ia", 2, "", true, true), //ai*2. new Rule("a", 1, "", true, true), //a*1. }; private static final Rule[] BRules = new Rule[] { new Rule("bb", 1, "", false, true), //bb1. }; private static final Rule[] CRules = new Rule[] { new Rule("ytic", 3, "s", false, true),//city3s. new Rule("ic", 2, "", false, false), //ci2> new Rule("nc", 1, "t", false, false), //cn1t> }; private static final Rule[] DRules = new Rule[] { new Rule("dd", 1, "", false, true), //dd1. new Rule("ied", 3, "y", false, false),//dei3y> new Rule("ceed", 2, "ss", false, true), //deec2ss. new Rule("eed", 1, "", false, true), //dee1. new Rule("ed", 2, "", false, false), //de2> new Rule("hood", 4, "", false, false), //dooh4> }; private static final Rule[] ERules = new Rule[] { new Rule("e", 1, "", false, false), //e1> }; private static final Rule[] FRules = new Rule[] { new Rule("lief", 1, "v", false, true), //feil1v. new Rule("if", 2, "", false, true), //fi2> }; private static final Rule[] GRules = new Rule[] { new Rule("ing", 3, "", false, false), //gni3> new Rule("iag", 3, "y", false, true), //gai3y. new Rule("ag", 2, "", false, false), //ga2> new Rule("gg", 1, "", false, true), //gg1. }; private static final Rule[] HRules = new Rule[] { new Rule("th", 2, "", true, true), //ht*2. new Rule("guish", 5, "ct", false, true), //hsiug5ct. new Rule("ish", 3, "", false, false), //hsi3> }; private static final Rule[] IRules = new Rule[] { new Rule("i", 1, "", true, true), //i*1. new Rule("i", 1, "y", false, false), //i1y> }; private static final Rule[] JRules = new Rule[] { new Rule("ij", 1, "d", false, true), //ji1d. new Rule("fuj", 1, "S", false, true), //juf1s. new Rule("uj", 1, "d", false, true), //ju1d. new Rule("oj", 1, "d", false, true), //jo1d. new Rule("hej", 1, "r", false, true), //jeh1r. new Rule("verj", 1, "t", false, true), //jrev1t. new Rule("misj", 2, "t", false, true), //jsim2t. new Rule("nj", 1, "d", false, true), //jn1d. new Rule("j", 1, "s", false, true), //j1s. }; private static final Rule[] LRules = new Rule[] { new Rule("ifiabl", 6, "", false, true), //lbaifi6. new Rule("iabl", 4, "y", false, true), //lbai4y. new Rule("abl", 3, "", false, false), //lba3> new Rule("ibl", 3, "", false, true), //lbi3. new Rule("bil", 2, "l", false, false), //lib2l> new Rule("cl", 1, "", false, true), //lc1. new Rule("iful", 4, "y", false, true), //lufi4y. new Rule("ful", 3, "", false, false), //luf3> new Rule("uf", 2, "", false, true), //lu2. new Rule("ial", 3, "", false, false), //lai3> new Rule("ual", 3, "", false, false), //lau3> new Rule("al", 2, "", false, false), //la2> new Rule("ll", 1, "", false, true), //ll1. }; private static final Rule[] MRules = new Rule[] { new Rule("ium", 3, "", false, true), //mui3. new Rule("mu", 2, "", true, true), //mu*2. new Rule("ism", 3, "", false, false), //msi3> new Rule("mm", 1, "", false, true), //mm1. }; private static final Rule[] NRules = new Rule[] { new Rule("sion", 4, "j", false, false), //nois4j> new Rule("xion", 4, "ct", false, true), //noix4ct. new Rule("ion", 3, "", false, false), //noi3> new Rule("ian", 3, "", false, false), //nai3> new Rule("an", 2, "", false, false), //na2> new Rule("een", 0, "", false, true), //nee0. new Rule("en", 2, "", false, false), //ne2> new Rule("nn", 1, "", false, true), //nn1. }; private static final Rule[] PRules = new Rule[] { new Rule("ship", 4, "", false, false), //pihs4> new Rule("pp", 1, "", false, true), //pp1. }; private static final Rule[] RRules = new Rule[] { new Rule("er", 2, "", false, false), //re2> new Rule("ear", 0, "", false, true), //rea0. new Rule("ar", 2, "", false, true), //ra2. new Rule("or", 2, "", false, false), //ro2> new Rule("ur", 2, "", false, false), //ru2> new Rule("rr", 1, "", false, true), //rr1. new Rule("tr", 1, "", false, false), //rt1> new Rule("ier", 3, "y", false, false), //rei3y> }; private static final Rule[] SRules = new Rule[] { new Rule("ies", 3, "y", false, false), //sei3y> new Rule("sis", 2, "", false, true), //sis2. new Rule("ness", 4, "", false, false), //ssen4> new Rule("ss", 0, "", false, true), //ss0. new Rule("ous", 3, "", false, false), //suo3> new Rule("us", 2, "", true, true), //su*2. new Rule("s", 1, "", true, false), //s*1> new Rule("s", 0, "", false, true), //s0. }; private static final Rule[] TRules = new Rule[] { new Rule("plicat", 4, "y", false, true), //tacilp4y. new Rule("at", 2, "", false, false), //ta2> new Rule("ment", 4, "", false, false), //tnem4> new Rule("ent", 3, "", false, false), //tne3> new Rule("ant", 3, "", false, false), //tna3> new Rule("ript", 2, "b", false, true), //tpir2b. new Rule("orpt", 2, "b", false, true), //tpro2b. new Rule("duct", 1, "", false, true), //tcud1. new Rule("sumpt", 2, "", false, true), //tpmus2. new Rule("cept", 2, "iv", false, true), //tpec2iv. new Rule("olut", 2, "v", false, true), //tulo2v. new Rule("sist", 0, "", false, true), //tsis0. new Rule("ist", 3, "", false, false), //tsi3> new Rule("tt", 1, "", false, true), //tt1. }; private static final Rule[] URules = new Rule[] { new Rule("iqu", 3, "", false, true), //uqi3. new Rule("ogu", 1, "", false, true), //ugo1. }; private static final Rule[] VRules = new Rule[] { new Rule("siv", 3, "j", false, false), //vis3j> new Rule("iev", 0, "", false, true), //vie0. new Rule("iv", 2, "", false, false), //vi2> }; private static final Rule[] YRules = new Rule[] { new Rule("bly", 1, "", false, false), //ylb1> new Rule("ily", 3, "y", false, false), //yli3y> new Rule("ply", 0, "", false, true), //ylp0. new Rule("ly", 2, "", false, false), //yl2> new Rule("ogy", 1, "", false, true), //ygo1. new Rule("phy", 1, "", false, true), //yhp1. new Rule("omy", 1, "", false, true), //ymo1. new Rule("opy", 1, "", false, true), //ypo1. new Rule("ity", 3, "", false, false), //yti3> new Rule("ety", 3, "", false, false), //yte3> new Rule("lty", 2, "", false, true), //ytl2. new Rule("istry", 5, "", false, true), //yrtsi5. new Rule("ary", 3, "", false, false), //yra3> new Rule("ory", 3, "", false, false), //yro3> new Rule("ify", 3, "", false, true), //yfi3. new Rule("ncy", 2, "t", false, false), //ycn2t> new Rule("acy", 3, "", false, false), //yca3> }; private static final Rule[] ZRules = new Rule[] { new Rule("iz", 2, "", false, false), //zi2> new Rule("yz", 1, "s", false, true), //zy1s. }; private static final Rule[] NoRules = new Rule[0]; /** * The rules for the */ private static final Rule[][] rules = new Rule[][] { ARules, BRules, CRules, DRules, ERules, FRules, GRules, HRules, IRules, JRules, NoRules, LRules, MRules, NRules, NoRules, PRules, NoRules, RRules, SRules, TRules, URules, VRules, NoRules, NoRules, YRules, ZRules }; private static boolean isVowel(char letter) { return letter == 'a' || letter == 'e' || letter == 'i' || letter == 'o' || letter == 'u'; } @Override public String stem(String word) { boolean virginRound = true; boolean stop; int charOffset = "a".charAt(0); do { stop = true; int ruleIndex = word.charAt(word.length()-1)-charOffset; if(ruleIndex < 0 || ruleIndex > rules.length) continue; for(Rule rule : rules[ruleIndex]) { if(rule.virgin && !virginRound) continue; String test = rule.apply(word); if(test != word)//Rule was applied, is it acceptable? { word = test; stop = false; if(rule.terminal) return word; else break; } } virginRound = false; } while(!stop); return word; } }

Ads help maintain this website.