lex.java

包含两个简单的词法分析器代码

//import java.util.Scanner;
import java.io.*;

class ParserException extends Exception
{
 public ParserException(String message)
 {
  super(message);
 }
}


/////////////////////////////////////////////////////////////


class Token
{
 public final static int INVALID = -1;
 public final static int LEFTPARENTHESIS = 27;//左括号'('
 public final static int RIGHTPARENTHESIS = 28;//右括号')'
 public final static int ADD = 13;//'+'
 public final static int SUB = 14;//'-'
 public final static int MUL = 15;//'*'
 public final static int DIV = 16;//'/'
 public final static int MAO = 17;//':'
 public final static int MAO_DENG = 18;//':='
 public final static int LESS = 20;//'<'
 public final static int LESS_EQU = 21;//'<='
 public final static int UNEQU = 22;//'<>'
 public final static int MORE = 23;//'>'
 public final static int MORE_EQU = 24;//'>='
 public final static int EQU= 25;//'='
 public final static int FENGHAO = 26;//';'
 public final static int NUM = 11;//数字类别码
 public final static int ID=10;//ID类别码
 public final static int JIN=0;//'#'
 public final static String[] keywords={"begin","if","then","while","do","end","printf","int"};//关键字
 private String content;
 private int type;
 public Token(String content, int type)
 {
  this.content = content;
  this.type = type;
 }
 public String getContent()//得到标识符的值
 {
  return content;
 }
 public double getDoubleValue()//将标识符转换为double类型
 {
  return Double.parseDouble(content);
 }
 public int getType()//得到种别码
 {
  return type;
 }
}

/////////////////////////////////////////////////////////


public class Lex
{
 private String buffer;
 private int colNum = 0;
 private char curChar;
 public Lex(String input)
 {
  this.buffer = input;
  curChar = getChar();
 }
 private char getChar()   //得到一个字符
 {
  char ch = '#';
  while(buffer!=null && colNum<buffer.length())
  {
   ch = buffer.charAt(colNum);
   colNum++;
   break;
  }
  return ch;
 }
 private void skipBlank()  //跳过空格
 {
  while(curChar == ' ')
   curChar = getChar();
 }
 public Token getToken() throws ParserException  //识别出数字、ID、关键字、符号
 {
  Token tk = null;
  if(curChar == ' ')
   skipBlank();
  if((curChar>='a'&&curChar<='z')||(curChar>='A'&&curChar<='Z'))  // 将ID识别出来
	 {
       String strf="";
	   strf=strf+curChar;
	   curChar=getChar();
	   boolean equ=false;
	   int j=0;
       while((curChar>='a'&&curChar<='z')||(curChar>='A'&&curChar<='Z')||(curChar>='0'&&curChar<='9'))
		 {
		    strf=strf+curChar;
			curChar=getChar();
		 }
	   for (int i=0;i<8;i++ )//将关键字识别出来
	   {
		   if(strf.equals(Token.keywords[i]))
		   {	
			   equ=true;
			   strf=Token.keywords[i];
			   j=i;
		   }
	   }
	   if(equ)
		   tk=new Token(strf,j+1);
	   else
		 {
		   String id="\'"+strf+"\'";
	       tk = new Token(id,Token.ID);
		 }
	 }
  else  //识别数值、特殊符号
  {switch(curChar)
   {
   case '(':
    tk = new Token("(",Token.LEFTPARENTHESIS);
    curChar = getChar();
    break;
   case ')':
    tk = new Token(")",Token.RIGHTPARENTHESIS);
    curChar = getChar();
    break;
   case ':':
	int c=colNum;
    curChar=getChar();
    if(curChar=='=')
	  {
		tk = new Token(":=",Token.MAO_DENG);
        curChar = getChar();
	  }
	else
	  {
	   colNum=c;
	   tk = new Token(":",Token.MAO);
       curChar = getChar();
	  }
	  break;
   case '+':
    tk = new Token("+",Token.ADD);
    curChar = getChar();
    break;
   case '-':
    tk = new Token("-",Token.SUB);
    curChar = getChar();
    break;
   case '*':
    tk = new Token("*",Token.MUL);
    curChar = getChar();
    break;
   case '/':
    tk = new Token("/",Token.DIV);
    curChar = getChar();
    break;
   case '=':
	tk = new Token("=",Token.EQU);
    curChar = getChar();
    break;
   case '<':
	int a=colNum;
    curChar=getChar();
    if(curChar=='=')
	  {
		tk = new Token("<=",Token.LESS_EQU);
        curChar = getChar();
	  }
	else if (curChar=='>')
	{
		tk = new Token("<=",Token.LESS_EQU);
        curChar = getChar();
	}
	else {
	   colNum=a;
	   tk = new Token("<",Token.LESS);
       curChar = getChar();
	  }
	  break;
   case '>':
	int b=colNum;
    curChar=getChar();
    if(curChar=='=')
	  {
		tk = new Token(">=",Token.MORE_EQU);
        curChar = getChar();
	  }
	else
	  {
	   colNum=b;
	   tk = new Token(">",Token.MORE);
       curChar = getChar();
	  }
	  break;
   case ';':
	tk = new Token(";",Token.FENGHAO);
    curChar = getChar();
    break;
   case '0':
   case '1':
   case '2':
   case '3':
   case '4':
   case '5':
   case '6':
   case '7':
   case '8':
   case '9':
   case '.':
    tk = parseNumber();
    break;
   case '#':
    tk = new Token("#",Token.JIN);
    tk = null;
    break;
   default:
    tk = new Token("Invalid character",Token.INVALID);
    curChar = getChar();
    break;
  }
  }
  return tk;
 }
 private Token parseNumber() throws ParserException  //进行digit=digit digit*
 {
  int dotNum = 0;
  boolean key = true;
  StringBuffer buf = new StringBuffer();
  buf.append(curChar);
  if(curChar == '.') dotNum++;
  while(key)
  {
   curChar = getChar();
   switch(curChar)
   {
   case '0':
   case '1':
   case '2':
   case '3':
   case '4':
   case '5':
   case '6':
   case '7':
   case '8':
   case '9':
    buf.append(curChar);
    continue;
   case '.':
    dotNum++;
    if(dotNum > 1)  //判断是否有多于一个'.'的情况,可以判断一个实数
     throw new ParserException("the string inputed error at column:" + colNum);
    buf.append('.');
    continue;
   default:
    key = false;
    continue;
   }
  }
  return new Token(buf.toString(),Token.NUM);
 }
 
 public static void main(String[] args) throws IOException{
   String[] str=new String[15];
   //Scanner scan=new Scanner(System.in);
   //BufferedReader br=new BufferedReader(new InputStreamReader(System.in));
   BufferedReader br=new BufferedReader(new FileReader(new File("1.txt")));
   int k=0;
   char ch;
   //System.out.println("Please input simple code ,input a \'#\' to end:");
   do{
	  // System.out.print("");
	   //str[k]=scan.nextLine();
           str[k]=br.readLine();
           ch=str[k].charAt(0);
	   k++;
   }while(ch!='#');
   for (int i=0;i<str.length;i++ )
   {
   
	try {
		Lex lex = new Lex(str[i]);
		while(true) {
				Token tk = lex.getToken();
				if(tk == null) {
					break;
				}
				else
					System.out.println("line"+(i+1)+": ("+tk.getType()+","+tk.getContent()+")");
		}
		}catch(Exception e) {
			e.printStackTrace();
		}
	}
 }
}