/* Copyright (C) 2003 Univ. of Massachusetts Amherst, Computer Science Dept.
This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
http://www.cs.umass.edu/~mccallum/mallet
This software is provided under the terms of the Common Public License,
version 1.0, as published by http://www.opensource.org. For further
information, see the file `LICENSE' included with this distribution. */
package cc.mallet.extract;
import java.util.regex.Pattern;
import java.util.*;
import cc.mallet.types.Label;
import cc.mallet.types.LabelAlphabet;
import cc.mallet.types.Sequence;
/**
* Tokenization filter that will create nested spans based on a hierarchical labeling of the data.
* The labels should be of the form LBL1[|LBLk]*. For example,
*
* A A|B A|B|C A|B|C A|B A A
* w1 w2 w3 w4 w5 w6 w7
*
* will result in LabeledSpans like
* <A>w1 <B>w2 <C>w3 w4</C> w5</B> w6 w7</A>
*
* Also, labels of the form <B-field> will force a new instance of the field to begin,
* even if it is already active. And prefixes of I- are ignored so you can use BIO labeling.
*
* Created: Nov 12, 2004
*
* @author
Ads help maintain this website.