retokenizer.java

来自「人力资源管理系统主要包括：人员管理、招聘管理、培训管理、奖惩管理和薪金管理五大管」· Java 代码 · 共 85 行

package net.sf.hibern8ide.highlighter;
/*
=====================================================================

  RETokenizer.java
  
  Created by Claude Duguay
  Copyright (c) 2002
  
=====================================================================
*/

import java.util.regex.Matcher;

public class RETokenizer
{
  protected RETypes types;
  protected Matcher matcher;
  
  public RETokenizer(RETypes types, String text)
  {
    this.types = types;
    matcher = types.getMatcher(text);
  }
  
  protected Token getToken(int pos)
  {
    int count = types.getTypeCount();
    for (int i = 1; i <= count; i++)
    {
      String token = matcher.group(i);
      if (token != null)
      {
        String type = types.getName(i - 1);
        return new Token(token, type, pos);
      }
    }
    return null;
  }
  
  public Token nextToken()
  {
    if (matcher.find())
    {
      return getToken(matcher.start());
    }
    return null;
  }
  
  public static class Token
  {
    public String token;
    public String type;
    protected int pos;
    
    public Token(String token, String type, int pos)
    {
      this.token = token;
      this.type = type;
      this.pos = pos;
    }
    
    public String getText()
    {
      return token;
    }
    
    public String getType()
    {
      return type;
    }
    
    public int getPos()
    {
      return pos;
    }
    
    public String toString()
    {
      return type + "(" + token + ", " + pos + ')';
    }
  }
}