code5.txt

我做了几个英文分词的程序

http://www.sunjinfei.com/netfeifei/article/show.asp?id=12879


请教C#中有没代替java的stringtokenizer的类

如题 

1楼  

use   System.String   's   Split   or   System.Text.RegularExpressions.Regex   's   Split   method 

2楼  

多谢，搞定     
    
  public   class   Tokenizer   
  {   
    
  private   System.Collections.ArrayList   elements;   
  private   string   source;   
  private   string   delimiters   =   "   \t\n\r";   
    
  public   Tokenizer(string   source)   
  {   
  this.elements   =   new   System.Collections.ArrayList();   
  this.elements.AddRange(source.Split(this.delimiters.ToCharArray()));   
  this.RemoveEmptyStrings();   
  this.source   =   source;   
  }   
  public   Tokenizer(string   source,   string   delimiters)   
  {   
  this.elements   =   new   System.Collections.ArrayList();   
  this.delimiters   =   delimiters;   
  this.elements.AddRange(source.Split(this.delimiters.ToCharArray()));   
  this.RemoveEmptyStrings();   
  this.source   =   source;   
  }   
    
    
  public   int   Count   
  {   
  get   
  {   
  return   (this.elements.Count);   
  }   
  }   
    
  public   bool   HasMoreTokens()   
  {   
  return   (this.elements.Count   >   0);   
  }   
    
  public   string   NextToken()   
  {   
  string   result;   
  if   (source   ==   "")   throw   new   System.Exception();   
  else   
  {   
  this.elements   =   new   System.Collections.ArrayList();   
  this.elements.AddRange(this.source.Split(delimiters.ToCharArray()));   
  RemoveEmptyStrings();   
  result   =   (string)   this.elements[0];   
  this.elements.RemoveAt(0);   
  this.source   =   this.source.Remove(this.source.IndexOf(result),result.Length);   
  this.source   =   this.source.TrimStart(this.delimiters.ToCharArray());   
  return   result;   
  }   
  }   
    
    
  public   string   NextToken(string   delimiters)   
  {   
  this.delimiters   =   delimiters;   
  return   NextToken();   
  }   
    
  private   void   RemoveEmptyStrings()   
  {   
  for   (int   index=0;   index   <   this.elements.Count;   index++)   
  if   ((string)this.elements[index]==   "")   
  {   
  this.elements.RemoveAt(index);   
  index--;   
  }   
  }   
  }