package jsat.text.tokenizer;

import java.util.List;
import jsat.text.stemming.Stemmer;

/**
 *
 * @author Edward Raff
 */
public class StemmingTokenizer implements Tokenizer
{

	private static final long serialVersionUID = 2883247633791522390L;
	private Stemmer stemmer;
    private Tokenizer baseTokenizer;

    public StemmingTokenizer(Stemmer stemmer, Tokenizer baseTokenizer)
    {
        this.stemmer = stemmer;
        this.baseTokenizer = baseTokenizer;
    }
    
    @Override
    public List tokenize(String input)
    {
        List tokens = baseTokenizer.tokenize(input);
        stemmer.applyTo(tokens);
        return tokens;
    }

    @Override
    public void tokenize(String input, StringBuilder workSpace, List storageSpace)
    {
        baseTokenizer.tokenize(input, workSpace, storageSpace);
        stemmer.applyTo(storageSpace);
    }
    
}

 

Ads help maintain this website.