consumer.java

java 词法分析器,用于一般的C,C++,VB,PS/SQL 语句的翻译

package fri.patterns.interpreter.parsergenerator.lexer;

import java.util.*;
import java.io.IOException;
import java.io.Serializable;
import fri.patterns.interpreter.parsergenerator.Token;
import fri.patterns.interpreter.parsergenerator.syntax.Rule;

/**
	Consuming characters (or bytes) means reading from an input stream as long
	as the read characters match the pattern the consumer was built for.
	A consumer can be built with a fixed string, a character set, other
	character consumers, or a mixed sequence of those. It can be a list of
	alternating (parallel) consumers. It can be set repeatable and nullable.
	A character consumer succeeds when he could apply all consumers or
	patterns/sets stored in its sequence when reading input. It fails and
	unreads all characters when one non-nullable consumer in its sequence fails.
	
	@author (c) 2002, Fritz Ritzberger
*/

class Consumer implements
	Comparable,
	Serializable
{
	private List sequence = new ArrayList();
	private List constraints;
	private boolean nullable = false;
	private boolean repeatable = false;
	protected Rule rule;
	protected int fixedLength = -1, startLength = -1, variance = -1;
	
	
	/** Empty consumer knowing its rule. This could become a Lexer toplevel consumer. */
	Consumer(Rule rule)	{
		this.rule = rule;
	}

	/** Consumer with some start character or fixed string, without rule. */
	Consumer(String charOrString)	{
		append(charOrString);
	}

	/** Internal constructor needed in LexerBuilder. */
	protected Consumer()	{
	}


	/** Append a String or character to sequence. */
	public void append(String charOrString)	{
		Object o = sequence.size() > 0 ? sequence.get(sequence.size() - 1) : null;	// check preceding
		if (o instanceof String)	{	// has a string preceding
			String s = (String)o;
			s = s + charOrString;	// append to it
			sequence.set(sequence.size() - 1, s);
		}
		else	{
			sequence.add(charOrString);
		}
	}

	/** Add a character consumer reference. */
	public void append(Reference subConsumer)	{
		sequence.add(subConsumer);
	}

	/** Add a character consumer. */
	public void append(Consumer subConsumer)	{
		sequence.add(subConsumer);
	}

	/** Add a character set by its high character. Low character is previously appended one. */
	public void appendSet(String high)
		throws LexerException
	{
		String low = (String) sequence.get(sequence.size() - 1);	// throws ClassCastException if not String
		if (low.length() > 1)	{	// low character was appended to previous string
			int i = low.length() - 1;
			sequence.set(sequence.size() - 1, low.substring(0, i));	// cut last character
			sequence.add(new CharacterSet(""+low.charAt(i), high));	// append set
		}
		else	{
			sequence.set(sequence.size() - 1, new CharacterSet(low, high));
		}
	}

	/** Passed Consumer will constrain every character of this consumer. */
	public void subtract(Consumer constraint)	{
		if (constraints == null)
			constraints = new ArrayList();
		constraints.add(constraint);
	}
	
	/** Passed reference will constrain every character of this consumer. */
	public void subtract(Reference constraint)	{
		if (constraints == null)
			constraints = new ArrayList();
		constraints.add(constraint);
	}
	

	/** Resolve all references to real consumers after build. */
	public void resolveConsumerReferences(Map charConsumers, Map doneList)
		throws LexerException
	{
		if (doneList.get(this) != null)
			return;
		doneList.put(this, this);
		
		List [] varr = new List[] { sequence, getAlternatives(), constraints };
		for (int j = 0; j < varr.length; j++)	{
			List v = varr[j];
			
			if (v != null)	{
				for (int i = 0; v != null && i < v.size(); i++)	{
					Object o = v.get(i);
					
					if (o instanceof Reference)	{
						//System.err.println("Found consumer reference "+o+" within "+rule);
						Reference cr = (Reference)o;
						Object cc = charConsumers.get(cr.nonterminal);
						
						if (cc == null)
							throw new LexerException("Consumer-Reference not found: "+cr.nonterminal);
						
						v.set(i, cc);
					}
					else
					if (o instanceof Consumer)	{
						Consumer cc = (Consumer)o;
						cc.resolveConsumerReferences(charConsumers, doneList);
					}
				}
			}
		}
	}
	
	
	public void setNullable()	{
		nullable = true;
	}
	
	public boolean isNullable()	{
		return nullable;
	}
	
	public void setRepeatable()	{
		repeatable = true;
	}
	
	public boolean isRepeatable()	{
		return repeatable;
	}

	/** Always returns null as this consumer holds no alternatives. */
	public List getAlternatives()	{
		return null;
	}
	
	
	/** Implements Comparable: Sort alternatives by precedence: - getStartVariance(), + getStartLength(), + getFixedLength(). */
	public int compareTo(Object o)	{
		Consumer cc = (Consumer)o;
		int i;
		i = getStartVariance() - cc.getStartVariance();	// be as exact as possible
		if (i != 0)
			return i;
		i = cc.getStartLength() - getStartLength();	// be as significant as possible
		if (i != 0)
			return i;
		i = cc.getFixedLength() - getFixedLength();	// be long as possible
		if (i != 0)
			return i;
		return 0;
	}

	/** Returns the first character of this consumer, or null if starting with a set. */
	public Character getStartCharacter()	{
		Object o = sequence.get(0);
		//System.err.println("getStartCharacter from sequence "+o);
		if (o instanceof Consumer)
			return ((Consumer)o).getStartCharacter();
		else
		if (o instanceof CharacterSet)
			return null;
		else
			return new Character(((String)o).charAt(0));
	}
	

	/** Returns the count of possible start characters. For fixed start character this is 1. */
	public int getStartVariance()	{
		if (this.variance > 0)
			return this.variance;
			
		if (getStartCharacter() != null)
			return this.variance = 1;

		int variance;
		Object o = sequence.get(0);
		if (o instanceof Consumer)
			variance = ((Consumer)o).getStartVariance();
		else
		if (o instanceof CharacterSet)
			variance = ((CharacterSet)o).getVariance();
		else
			throw new IllegalStateException("No fixed start character, no character set, where am i?");
			
		for (int i = 0; constraints != null && i < constraints.size(); i++)
			variance -= ((Consumer) constraints.get(i)).getStartVariance();
			
		return this.variance = variance;
	}
	
	/** The fixed length ends before the first found character set (like "0..9") or repeatable or nullable consumer. */
	public int getFixedLength()	{
		if (fixedLength >= 0)	// never call toString() before all sequences have arrived
			return fixedLength;
		fixedLength = getSomeLength(false, new ArrayList());
		return fixedLength;
	}
	
	/** The start length ends before the first found repeatable or nullable consumer (like "chars*"), but not before a character set. */
	public int getStartLength()	{
		if (startLength >= 0)	// never call toString() before all sequences have arrived
			return startLength;
		List reason = new ArrayList();
		startLength = getSomeLength(true, reason);
		//System.err.println("found start length "+startLength+" for rule "+rule+", sequence "+sequence+", reason "+reason);
		return startLength;
	}


	protected int getSomeLength(boolean exploreStartLength, List breakIndicator)	{
		int len = 0;
		
		for (int i = 0; breakIndicator.size() <= 0 && i < sequence.size(); i++)	{
			Object o = sequence.get(i);
			
			if (o instanceof Consumer)	{
				Consumer cc = (Consumer)o;

				if (cc.isNullable())	{	// fixed start length ends here
					breakIndicator.add("nullable");	// make parent terminate
					return len;
				}
				else
				if (cc.isRepeatable())	{	// fixed start length ends here
					len += cc.getSomeLength(exploreStartLength, breakIndicator);
					breakIndicator.add("repeatable");	// make parent terminate
					return len;
				}
				
				len += cc.getSomeLength(exploreStartLength, breakIndicator);
			}
			else
			if (o instanceof CharacterSet)	{
				if (exploreStartLength)	{
					breakIndicator.add("set");	// make parent terminate
					return len;
				}
				len += 1;	// would match exactly one character
			}
			else	{	// must be String
				len += ((String)o).length();	// matches a string of same length
			}
		}
		
		return len;
	}



	/** Return contained consumer when having only one and no constraints. This is called immediately after construction. */
	Consumer optimize()	{
		if (constraints == null && sequence.size() == 1 && sequence.get(0) instanceof Consumer)	{
			// give up this formal-only consumer, take rule to sub-consumer
			Consumer cc = (Consumer) sequence.get(0);
			cc.rule = rule;
			return cc;
		}
		return this;
	}

	/**
		Returns true if the rule of this consumer matches the passed left recursive rule.
		E.g. passing "nonterm ::= nonterm something" would match "nonterm ::= something".
	*/
	boolean matchesRepeatableRule(Rule rule)	{
		if (rule.rightSize() != this.rule.rightSize() + 1)
			return false;