defaulttokenizationfilter.java

来自「mallet是自然语言处理、机器学习领域的一个开源项目。」· Java 代码 · 共 62 行

/* Copyright (C) 2003 Univ. of Massachusetts Amherst, Computer Science Dept.
   This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
   http://www.cs.umass.edu/~mccallum/mallet
   This software is provided under the terms of the Common Public License,
   version 1.0, as published by http://www.opensource.org.  For further
   information, see the file `LICENSE' included with this distribution. */
package edu.umass.cs.mallet.base.extract;

import edu.umass.cs.mallet.base.types.LabelAlphabet;
import edu.umass.cs.mallet.base.types.Label;
import edu.umass.cs.mallet.base.types.Sequence;

import java.io.Serializable;
/**
 * Created: Nov 12, 2004
 *
 * @author <A HREF="mailto:[e-mail]>[e-mail]</A>
 * @version $Id: DefaultTokenizationFilter.java,v 1.4 2005/11/08 23:01:30 culotta Exp $
 */
public class DefaultTokenizationFilter implements TokenizationFilter, Serializable {

  public LabeledSpans constructLabeledSpans (LabelAlphabet dict, Object document, Label backgroundTag,
                                                    Tokenization input, Sequence seq)
  {
    LabeledSpans labeled = new LabeledSpans (document);
    addSpansFromTags (labeled, input, seq, dict, backgroundTag);
    return labeled;
  }
  

  private void addSpansFromTags (LabeledSpans labeled, Tokenization input, Sequence tags, LabelAlphabet dict,
                                 Label backgroundTag)
   {
     int i = 0;
     int docidx = 0;
     while (i < tags.size()) {
       Label thisTag = dict.lookupLabel (tags.get(i).toString());
       int startTokenIdx = i;
       while (i < tags.size()) {
         Label nextTag = dict.lookupLabel (tags.get(i).toString ());
         if (thisTag != nextTag) break;
         i++;
       }
       int endTokenIdx = i;
       Span span = input.subspan(startTokenIdx, endTokenIdx);
       addBackgroundIfNecessary (labeled, (StringSpan) span, docidx, backgroundTag);
       docidx = ((StringSpan) span).getEndIdx ();
       labeled.add (new LabeledSpan (span, thisTag, thisTag == backgroundTag));
     }
   }

   private void addBackgroundIfNecessary (LabeledSpans labeled, StringSpan span, int docidx, Label background)
   {
     int nextIdx = span.getStartIdx ();
     if (docidx < nextIdx) {
       Span newSpan = new StringSpan ((CharSequence) span.getDocument (), docidx, nextIdx);
       labeled.add (new LabeledSpan (newSpan, background, true));
     }
   }

}