package jsat.text.tokenizer;

import java.util.ArrayList;
import java.util.List;

/**
 *
 * A simple tokenizer. It converts everything to lower case, and splits on white
 * space. Anything that is not a letter, digit, or space, is treated as white 
 * space. This behavior can be altered slightly, and allows for setting a 
 * minimum and maximum allowed length for tokens. This can be useful when 
 * dealing with noisy documents, and removing small words. 
* * @author Edward Raff */ public class NaiveTokenizer implements Tokenizer { private static final long serialVersionUID = -2112091783442076933L; private boolean useLowerCase; private boolean otherToWhiteSpace = true; private boolean noDigits = false; private int minTokenLength = 1; private int maxTokenLength = Integer.MAX_VALUE; /** * Creates a new naive tokenizer that converts words to lower case */ public NaiveTokenizer() { this(true); } /** * Creates a new naive tokenizer * * @param useLowerCase {@code true} to convert everything to lower, * {@code false} to leave the case as is */ public NaiveTokenizer(boolean useLowerCase) { this.useLowerCase = useLowerCase; } /** * Sets whether or not characters are made to be lower case or not * @param useLowerCase */ public void setUseLowerCase(boolean useLowerCase) { this.useLowerCase = useLowerCase; } /** * Returns {@code true} if letters are converted to lower case, * {@code false} for case sensitive * @return {@code true} if letters are converted to lower case, */ public boolean isUseLowerCase() { return useLowerCase; } /** * Sets whether or not all non letter and digit characters are treated as * white space, or ignored completely. If ignored, the tokenizer parses the * string as if all non letter, digit, and whitespace characters did not * exist in the original string.
*
* Setting this to {@code false} can result in a lower feature count, * especially for noisy documents. * @param otherToWhiteSpace {@code true} to treat all "other" characters as * white space, {@code false} to ignore them */ public void setOtherToWhiteSpace(boolean otherToWhiteSpace) { this.otherToWhiteSpace = otherToWhiteSpace; } /** * Returns whether or not all other illegal characters are treated as * whitespace, or ignored completely. * @return {@code true} if all other characters are treated as whitespace */ public boolean isOtherToWhiteSpace() { return otherToWhiteSpace; } @Override public List tokenize(String input) { ArrayList toRet = new ArrayList(); StringBuilder sb = new StringBuilder(input.length()/10); tokenize(input, sb, toRet); return toRet; } @Override public void tokenize(String input, StringBuilder workSpace, List storageSpace) { for(int i = 0; i < input.length(); i++) { char c = input.charAt(i); if(Character.isLetter(c)) if (useLowerCase) workSpace.append(Character.toLowerCase(c)); else workSpace.append(c); else if (!noDigits && Character.isDigit(c)) workSpace.append(c); else if(!otherToWhiteSpace && !Character.isWhitespace(c)) continue; else //end of token { if(workSpace.length() >= minTokenLength && workSpace.length() <= maxTokenLength) storageSpace.add(workSpace.toString()); workSpace.setLength(0); } } if(workSpace.length() >= minTokenLength && workSpace.length() <= maxTokenLength) storageSpace.add(workSpace.toString()); } /** * Sets the maximum allowed length for any token. Any token discovered * exceeding the length will not be accepted and skipped over. The default * is unbounded. * * @param maxTokenLength the maximum token length to accept as a valid token */ public void setMaxTokenLength(int maxTokenLength) { if(maxTokenLength < 1) throw new IllegalArgumentException("Max token length must be positive, not " + maxTokenLength); if(maxTokenLength <= minTokenLength) throw new IllegalArgumentException("Max token length must be larger than the min token length"); this.maxTokenLength = maxTokenLength; } /** * Returns the maximum allowed token length * @return the maximum allowed token length */ public int getMaxTokenLength() { return maxTokenLength; } /** * Sets the minimum allowed token length. Any token discovered shorter than * the minimum length will not be accepted and skipped over. The default * is 0. * @param minTokenLength the minimum length for a token to be used */ public void setMinTokenLength(int minTokenLength) { if(minTokenLength < 0) throw new IllegalArgumentException("Minimum token length must be non negative, not " + minTokenLength); if(minTokenLength > maxTokenLength) throw new IllegalArgumentException("Minimum token length can not exced the maximum token length"); this.minTokenLength = minTokenLength; } /** * Returns the minimum allowed token length * @return the maximum allowed token length */ public int getMinTokenLength() { return minTokenLength; } /** * Sets whether digits will be accepted in tokens or treated as "other" (not * white space and not character).
* The default it to allow digits. * * @param noDigits {@code true} to disallow numeric digits, {@code false} to * allow digits. */ public void setNoDigits(boolean noDigits) { this.noDigits = noDigits; } /** * Returns {@code true} if digits are not allowed in tokens, {@code false} * otherwise. * @return {@code true} if digits are not allowed in tokens, {@code false} * otherwise. */ public boolean isNoDigits() { return noDigits; } }

Ads help maintain this website.