package jsat.text.tokenizer;
import java.io.Serializable;
import java.util.List;
/**
* Interface for taking the text of a document and breaking it up into features.
* For example "This doc" might become "this" and "doc"
*
* @author Edward Raff
*/
public interface Tokenizer extends Serializable
{
/**
* Breaks the input string into a series of tokens that may be used as
* features for a classifier. The returned tokens must be either new string
* objects or interned strings. If a token is returned that is backed by
* the original document, memory may get leaked by processes consuming the
* token.
* This method should be thread safe
*
* @param input the string to tokenize
* @return an already allocated list to place the tokens into
*/
public List tokenize(String input);
/**
* Breaks the input string into a series of tokens that may be used as
* features for a classifier. The returned tokens must be either new string
* objects or interned strings. If a token is returned that is backed by
* the original document, memory may get leaked by processes consuming the
* token.
* This method should be thread safe
*
* @param input the string to tokenize
* @param workSpace an already allocated (but empty) string builder than can
* be used as a temporary work space.
* @param storageSpace an already allocated (but empty) list to place the
* tokens into
*/
public void tokenize(String input, StringBuilder workSpace, List storageSpace);
}
Ads help maintain this website.