tokenization.java

来自「mallet是自然语言处理、机器学习领域的一个开源项目。」· Java 代码 · 共 45 行

/* Copyright (C) 2002 Univ. of Massachusetts Amherst, Computer Science Dept.
   This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
   http://www.cs.umass.edu/~mccallum/mallet
   This software is provided under the terms of the Common Public License,
   version 1.0, as published by http://www.opensource.org.  For further
   information, see the file `LICENSE' included with this distribution. */

/** 
   @author Andrew McCallum <a href="mailto:[e-mail]">[e-mail]</a>
 */

package edu.umass.cs.mallet.base.extract;

import edu.umass.cs.mallet.base.types.*;

public interface Tokenization extends Sequence //??
{
  /**
   * Returns the document of which this is a tokenization.
   * @return
   */
	public Object getDocument ();

  public Span getSpan (int i);


  /** Returns a span formed by concatenating the spans from start to end.
   *  In more detail:
   *  <ul>
   *   <li>The start of the new span will be the start index of <tt>getSpan(start)</tt>.
   *   <li>The end of the new span will be the start index of <tt>getSpan(end)</tt>.
   *   <li>Unless <tt>start == end</tt>, the new span will completely include <tt>getSpan(start)</tt>.
   *   <li>The new span will never intersect <tt>getSpan(end)</tt>
   *   <li>If <tt>start == end</tt>, then the new span contains no text.
   *  </ul>
   *
   * @param start The index of the first token in the new span (inclusive).
   *   This is an index of a token, *not* an index into the document.
   * @param end The index of the first token in the new span (exclusive).
   *   This is an index of a token, *not* an index into the document.
   * @return A span into this tokenization's document
   */
  Span subspan (int start, int end);
}