html-lex.sml

这是我们参加06年全国开源软件的竞赛作品

\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\050\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\
\\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049\049"
),
 (51, 
"\000\000\000\000\000\000\000\000\000\052\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\052\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
\\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
),
(0, "")]
fun f x = x 
val s = map f (rev (tl (rev s))) 
exception LexHackingError 
fun look ((j,x)::r, i) = if i = j then x else look(r, i) 
  | look ([], i) = raise LexHackingError
fun g {fin=x, trans=i} = {fin=x, trans=look(s,i)} 
in Vector.fromList(map g 
[{fin = [], trans = 0},
{fin = [], trans = 1},
{fin = [], trans = 1},
{fin = [], trans = 3},
{fin = [], trans = 3},
{fin = [], trans = 5},
{fin = [], trans = 5},
{fin = [], trans = 7},
{fin = [], trans = 7},
{fin = [(N 79),(N 81)], trans = 9},
{fin = [(N 79)], trans = 9},
{fin = [(N 81)], trans = 11},
{fin = [(N 3)], trans = 12},
{fin = [], trans = 13},
{fin = [], trans = 14},
{fin = [(N 32)], trans = 0},
{fin = [], trans = 16},
{fin = [], trans = 17},
{fin = [], trans = 18},
{fin = [(N 37)], trans = 0},
{fin = [(N 79),(N 81)], trans = 20},
{fin = [(N 79)], trans = 21},
{fin = [(N 72),(N 79)], trans = 9},
{fin = [(N 79)], trans = 23},
{fin = [(N 79)], trans = 24},
{fin = [(N 61),(N 79)], trans = 9},
{fin = [(N 79)], trans = 26},
{fin = [(N 67),(N 79)], trans = 9},
{fin = [(N 76),(N 79),(N 81)], trans = 9},
{fin = [(N 74),(N 79)], trans = 9},
{fin = [(N 44)], trans = 0},
{fin = [(N 44)], trans = 31},
{fin = [(N 40)], trans = 0},
{fin = [(N 42)], trans = 0},
{fin = [(N 55)], trans = 0},
{fin = [(N 49),(N 55)], trans = 0},
{fin = [(N 55)], trans = 36},
{fin = [(N 47)], trans = 0},
{fin = [(N 53),(N 55)], trans = 0},
{fin = [(N 51)], trans = 0},
{fin = [(N 25)], trans = 0},
{fin = [(N 13),(N 25)], trans = 41},
{fin = [(N 13)], trans = 41},
{fin = [(N 5),(N 25)], trans = 0},
{fin = [(N 15),(N 25)], trans = 0},
{fin = [(N 25)], trans = 45},
{fin = [], trans = 45},
{fin = [(N 23)], trans = 0},
{fin = [(N 25)], trans = 48},
{fin = [], trans = 48},
{fin = [(N 19)], trans = 0},
{fin = [(N 10),(N 25)], trans = 51},
{fin = [(N 10)], trans = 51},
{fin = [(N 7)], trans = 0}])
end
structure StartStates =
	struct
	datatype yystartstate = STARTSTATE of int

(* start state definitions *)

val COM1 = STARTSTATE 3;
val COM2 = STARTSTATE 5;
val INITIAL = STARTSTATE 1;
val STAG = STARTSTATE 7;

end
type result = UserDeclarations.lexresult
	exception LexerError (* raised if illegal leaf action tried *)
end

fun makeLexer yyinput =
let	val yygone0=1
	val yylineno = ref 0

	val yyb = ref "\n" 		(* buffer *)
	val yybl = ref 1		(*buffer length *)
	val yybufpos = ref 1		(* location of next character to use *)
	val yygone = ref yygone0	(* position in file of beginning of buffer *)
	val yydone = ref false		(* eof found yet? *)
	val yybegin = ref 1		(*Current 'start state' for lexer *)

	val YYBEGIN = fn (Internal.StartStates.STARTSTATE x) =>
		 yybegin := x

fun lex (yyarg as (errorFn, file)) =
let fun continue() : Internal.result = 
  let fun scan (s,AcceptingLeaves : Internal.yyfinstate list list,l,i0) =
	let fun action (i,nil) = raise LexError
	| action (i,nil::l) = action (i-1,l)
	| action (i,(node::acts)::l) =
		case node of
		    Internal.N yyk => 
			(let val yytext = substring(!yyb,i0,i-i0)
			     val yypos = i0+ !yygone
			val _ = yylineno := CharVector.foldl
				(fn (#"\n", n) => n+1 | (_, n) => n) (!yylineno) yytext
			open UserDeclarations Internal.StartStates
 in (yybufpos := i; case yyk of 

			(* Application actions *)

  10 => (addStr yytext; continue())
| 13 => (addStr yytext; continue())
| 15 => (addStr yytext; continue())
| 19 => (addStr yytext; continue())
| 23 => (addStr yytext; continue())
| 25 => (addStr yytext; continue())
| 3 => (addStr yytext; YYBEGIN STAG; continue())
| 32 => (case Elems.endTag file (yytext, !yylineno, !yylineno)
	     of NONE => continue()
	      | (SOME tag) => tag
	    (* end case *))
| 37 => (YYBEGIN COM1; continue())
| 40 => (YYBEGIN COM2; continue())
| 42 => (continue())
| 44 => (continue())
| 47 => (YYBEGIN COM1; continue())
| 49 => (YYBEGIN INITIAL; continue())
| 5 => (addStr yytext;
	    YYBEGIN INITIAL;
	    case Elems.startTag file (getStr(), !yylineno, !yylineno)
	     of NONE => continue()
	      | (SOME tag) => tag
	    (* end case *))
| 51 => (continue())
| 53 => (continue())
| 55 => (errorFn("bad comment syntax", !yylineno, !yylineno+1);
	    YYBEGIN INITIAL;
	    continue())
| 61 => (
(** At some point, we should support &#SPACE; and &#TAB; **)
	    continue())
| 67 => (T.CHAR_REF(yytext, !yylineno, !yylineno))
| 7 => (addStr " "; continue())
| 72 => (T.ENTITY_REF(yytext, !yylineno, !yylineno))
| 74 => (continue())
| 76 => (continue())
| 79 => (T.PCDATA(yytext, !yylineno, !yylineno))
| 81 => (errorFn(concat[
		"bogus character #\"", Char.toString(String.sub(yytext, 0)),
		"\" in PCDATA\n"
	      ], !yylineno, !yylineno+1);
	    continue())
| _ => raise Internal.LexerError

		) end )

	val {fin,trans} = Vector.sub(Internal.tab, s)
	val NewAcceptingLeaves = fin::AcceptingLeaves
	in if l = !yybl then
	     if trans = #trans(Vector.sub(Internal.tab,0))
	       then action(l,NewAcceptingLeaves
) else	    let val newchars= if !yydone then "" else yyinput 1024
	    in if (size newchars)=0
		  then (yydone := true;
		        if (l=i0) then UserDeclarations.eof yyarg
		                  else action(l,NewAcceptingLeaves))
		  else (if i0=l then yyb := newchars
		     else yyb := substring(!yyb,i0,l-i0)^newchars;
		     yygone := !yygone+i0;
		     yybl := size (!yyb);
		     scan (s,AcceptingLeaves,l-i0,0))
	    end
	  else let val NewChar = Char.ord(String.sub(!yyb,l))
		val NewState = Char.ord(String.sub(trans,NewChar))
		in if NewState=0 then action(l,NewAcceptingLeaves)
		else scan(NewState,NewAcceptingLeaves,l+1,i0)
	end
	end
(*
	val start= if substring(!yyb,!yybufpos-1,1)="\n"
then !yybegin+1 else !yybegin
*)
	in scan(!yybegin (* start *),nil,!yybufpos,!yybufpos)
    end
in continue end
  in lex
  end
end