biotokenizationfilter.java

来自「mallet是自然语言处理、机器学习领域的一个开源项目。」· Java 代码 · 共 128 行

/* Copyright (C) 2003 Univ. of Massachusetts Amherst, Computer Science Dept.
   This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
   http://www.cs.umass.edu/~mccallum/mallet
   This software is provided under the terms of the Common Public License,
   version 1.0, as published by http://www.opensource.org.  For further
   information, see the file `LICENSE' included with this distribution. */
package edu.umass.cs.mallet.base.extract;

import edu.umass.cs.mallet.base.types.Label;
import edu.umass.cs.mallet.base.types.LabelAlphabet;
import edu.umass.cs.mallet.base.types.Sequence;

import java.io.Serializable;
import java.io.ObjectOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;

/**
 * Created: Nov 12, 2004
 *
 * @author <A HREF="mailto:[e-mail]>[e-mail]</A>
 * @version $Id: BIOTokenizationFilter.java,v 1.7 2005/11/03 20:02:07 ghuang Exp $
 */
public class BIOTokenizationFilter implements TokenizationFilter, Serializable {

  public LabeledSpans constructLabeledSpans (LabelAlphabet dict, Object document, Label backgroundTag,
                                             Tokenization input, Sequence seq)
  {
    LabeledSpans labeled = new LabeledSpans (document);
    addSpansFromTags (labeled, input, seq, dict, backgroundTag);
    return labeled;
  }


  private void addSpansFromTags (LabeledSpans labeled, Tokenization input, Sequence tags, LabelAlphabet dict,
                                 Label backgroundTag)
  {
    int i = 0;
    int docidx = 0;
    while (i < tags.size ()) {
      Label thisTag = dict.lookupLabel (tags.get (i).toString ());
      int startTokenIdx = i;
      while (++i < tags.size ()) {
        Label nextTag = dict.lookupLabel (tags.get (i).toString ());
        if (isBeginTag (nextTag) || !tagsMatch (thisTag, nextTag)) break;
      }
      int endTokenIdx = i;
      Span span = createSpan (input, startTokenIdx, endTokenIdx);
      addBackgroundIfNecessary (labeled, (StringSpan) span, docidx, backgroundTag);
      docidx = ((StringSpan) span).getEndIdx ();

      if (isBeginTag (thisTag) || isInsideTag (thisTag)) {
        thisTag = trimTag (dict, thisTag);
      }
      labeled.add (new LabeledSpan (span, thisTag, thisTag == backgroundTag));
    }
  }

  protected Span createSpan (Tokenization input, int startTokenIdx, int endTokenIdx)
  {
    return input.subspan (startTokenIdx, endTokenIdx);
  }


  private Label trimTag (LabelAlphabet dict, Label tag)
  {
    String name = (String) tag.getEntry ();
    return dict.lookupLabel (name.substring (2));
  }


  private boolean tagsMatch (Label tag1, Label tag2)
  {
    String name1 = (String) tag1.getEntry ();
    String name2 = (String) tag2.getEntry ();

    if (isBeginTag (tag1) || isInsideTag (tag1)) {
      name1 = name1.substring (2);
    }
    if (isInsideTag (tag2)) {
      name2 = name2.substring (2);
    }

    return name1.equals (name2);
  }


  private boolean isBeginTag (Label lbl)
  {
    String name = (String) lbl.getEntry ();
    return name.startsWith ("B-");
  }

  private boolean isInsideTag (Label lbl)
  {
    String name = (String) lbl.getEntry ();
    return name.startsWith ("I-");
  }

  private void addBackgroundIfNecessary (LabeledSpans labeled, StringSpan span, int docidx, Label background)
  {
    int nextIdx = span.getStartIdx ();
    if (docidx < nextIdx) {
      Span newSpan = new StringSpan ((CharSequence) span.getDocument (), docidx, nextIdx);
      labeled.add (new LabeledSpan (newSpan, background, true));
    }
  }

  // Serialization garbage

  private static final long serialVersionUID = -8726127297313150023L;
  private static final int CURRENT_SERIAL_VERSION = 1;
  
  private void writeObject (ObjectOutputStream out) throws IOException
  {
	  out.defaultWriteObject ();
	  out.writeInt (CURRENT_SERIAL_VERSION);
  }
  
  
  private void readObject (ObjectInputStream in) throws IOException, ClassNotFoundException
  {
	  in.defaultReadObject ();
	  in.readInt (); // read version
  }

}