scanner.java

java语言开发的基于tiny语言的编译器

package jeex.tiny;

import java.io.*;
/**
 * A class for lexical analyse.
 */
class Scanner implements Tokens {	
	private char[] buf ; // hold the file in buf wholly
	private int bp = 0; // buffer pointer 
	
	int token; // current token recognized
	int line ;   // current line of file
	int column ; // current column of file
	String identName; // the name of the current identifier,if the current token is an identifier.
	Object literalValue;// the value of the literal, if the current token is a literal.
	
	Scanner(InputStream in) throws IOException {
		int bufsize = in.available() + 1;
		buf = new char[bufsize];
		InputStreamReader r = new InputStreamReader(in);
		r.read(buf,0,bufsize - 1);
		buf[bufsize - 1] = '#'; // to indicate the end of the buffer
		line = 1; column = 1;
		nextToken();
	}
	
	void nextToken() {
		skipWhiteSpace();
		switch(buf[bp]) {
			case '<': { token = LT; bp ++; return; }
			case '=': { token = EQ; bp ++; return; }
			case '*': { token = MUL; bp ++; return; }
			case '/': { token = DIV; bp ++; return; }
			case '+': { token = PLUS; bp ++; return; }
			case '-': { token = MINUS; bp ++; return; }
			case '(': { token = LPAREN; bp ++; return; } 
			case ')': { token = RPAREN; bp ++; return; }
			case ';': { token = SEMI; bp ++; return; }
			case '#': { token = EOF; bp ++; return; }
			case '.': { token = DOT; bp ++; return; }
			case ':': {	
				bp ++; 
				if (buf[bp] != '=') {
					lexError(bp - 1); 
					bp ++;
					token = ERROR;
					return;
				}
				else {
					bp ++;
					token = ASSIGN;
					return;
				}
			}
			default : {
				if (Character.isLetter(buf[bp])) {
					ident();
					return ;
				} else if (Character.isDigit(buf[bp])) {
					number();
					return;
				} else {
					bp ++;
					token = ERROR; 
					return;
				}
			}			
			
		}
	}
		
	private void ident() {
		int beginbp = bp;
		while(Character.isLetterOrDigit(buf[++bp])) 
			;
		identName = new String(buf,beginbp,bp - beginbp);
		
		KeyStringToken ks = searchKey(identName);
		if (ks == null) token = IDENT;
		else token = ks.token;
	}
	
	private void number() {
		int beginbp = bp;
		while(Character.isDigit(buf[++bp]))
			;
		if(Character.isLetter(buf[bp])) {
			token = ERROR;
			return;
		}
		String temp = new  String(buf,beginbp,bp - beginbp);
		literalValue = temp;
		token = NUMBER;
	}
	
	private void skipWhiteSpace() {
		while(buf[bp] == '\t'||buf[bp] == '\n'||buf[bp] == ' '||buf[bp] == '\r') {
			if (buf[bp] == '\n') line ++;
			bp ++;
		}
	}
	
	private void lexError(int pos) {
		Log.error("Lex error at line " + line);
	}
	
	private static class KeyStringToken {
		String key;
		int token;
		KeyStringToken(String key,int token) {
			this.key = key;
			this.token = token;
		}
	}
	
	private static KeyStringToken keyMap[] = {
		new KeyStringToken("if",IF),
		new KeyStringToken("then",THEN),
		new KeyStringToken("else",ELSE),
		new KeyStringToken("end",END),
		new KeyStringToken("repeat",REPEAT),
		new KeyStringToken("until",UNTIL),
		new KeyStringToken("read",READ),
		new KeyStringToken("write",WRITE)
	};
	
	private KeyStringToken searchKey(String key) {
		for(int i = 0; i < keyMap.length; i++) {
			if (keyMap[i].key.equals(key)) return keyMap[i];
		}
		return null;
	}
	
	public static void main(String[] args) throws IOException {
		if (args.length <= 0) {
			System.out.println("Usage: Scanner filename");
			System.exit(1);
		}
		FileInputStream in = new FileInputStream(args[0]);
		Scanner s = new Scanner(in);
		while(s.token != EOF) {
			if (s.token == IDENT) {
				System.out.println(" ID:" + s.identName);
			} else if (s.token == NUMBER) {
				System.out.println(" NUMBER:" + s.literalValue);
			} else if (s.token == ERROR) {
				System.out.println("ERROR");
			} else {
				System.out.println(TokenUtil.tokenToString(s.token));			
			}
			s.nextToken();
		}
	}
}