htmlparser.java

一个Web爬虫（机器人

                        ++bufptr;
                    }
                    break;

                case CMT_DASH:
                    if (c == '-') {
                        ++bufptr;
                        state = CMT_DASHDASH;
                    }
                    else {
                        ++bufptr;
                        state = CMT;
                    }
                    break;

                case CMT_DASHDASH:
                    if (c == '>') {
                        ++bufptr;
                        tag = new Tag (page, tagStart, bufbase+bufptr, Tag.COMMENT, true);
                        tokens.addElement (tag);
                        state = START;
                    }
                    else if (c == '-') {
                        ++bufptr;
                        state = CMT_DASHDASH;
                    }
                    else {
                        ++bufptr;
                        state = CMT;
                    }
                    break;

                case DIRECTIVE:
                    if (c == '>') {
                        ++bufptr;
                        tag = new Tag (page, tagStart, bufbase+bufptr, Tag.COMMENT, true);
                        tokens.addElement (tag);
                        state = START;
                    }
                    else {
                        ++bufptr;
                    }
                    break;

                // Tags
                case STAG:
                    if (c == '>' || isWhitespace(c)) {
                        tag = new Tag (page, tagStart, bufbase+bufptr, // tag doesn't really end here
                                                                       // -- we'll fix it up when we actually see it
                                       tagName.toString (), true);
                        tokens.addElement (tag);
                        attrs.setSize (0);
                        state = ATTR;
                    }
                    else {
                        tagName.append (c);
                        ++bufptr;
                        // state == STAG;
                    }
                    break;

                case ETAG:
                    if (c == '>') {
                        ++bufptr;
                        tag = new Tag (page, tagStart, bufbase+bufptr, tagName.toString (), false);
                        tokens.addElement (tag);
                        state = START;
                    }
                    else {
                        tagName.append (c);
                        ++bufptr;
                        // state == ETAG
                    }
                    break;

                // Attributes
                case ATTR:
                    if (isWhitespace(c))
                        ++bufptr;
                    else if (c == '>') {
                        ++bufptr;
                        tag.end = bufbase+bufptr;
                        if (attrs.size() > 0) {
                            tag.htmlAttributes = new String[attrs.size()];
                            attrs.copyInto (tag.htmlAttributes);
                        }
                        state = START;
                    }
                    else {
                        attrName.setLength (0);
                        state = ATTRNAME;
                    }
                    break;

                case ATTRNAME:
                    if (c == '>' || c == '=' || isWhitespace(c)) {
                        state = EQ;
                    }
                    else {
                        attrName.append (c);
                        ++bufptr;
                        // state == ATTRNAME;
                    }
                    break;

                case EQ:
                    if (isWhitespace(c))
                        ++bufptr;
                    else if (c == '=') {
                        ++bufptr;
                        state = AFTEREQ;
                    }
                    else {
                        String name = Tag.toHTMLAttributeName (attrName.toString());
                        tag.setLabel (name);
                        attrs.addElement (name);
                        state = ATTR;
                    }
                    break;

                case AFTEREQ:
                    if (isWhitespace (c))
                        ++bufptr;
                    else
                        switch (c) {
                            case '>':
                            {
                                String name = Tag.toHTMLAttributeName (attrName.toString());
                                tag.setLabel (name);
                                attrs.addElement (name);
                                state = ATTR;
                                break;
                            }
                            case '\'':
                                ++bufptr;
                                attrVal.setLength (0);
                                state = ATTRVAL_SQ;
                                break;
                            case '"':
                                ++bufptr;
                                attrVal.setLength (0);
                                state = ATTRVAL_DQ;
                                break;
                            default:
                                attrVal.setLength (0);
                                state = ATTRVAL;
                                break;
                        }
                    break;

                case ATTRVAL:
                    if (c == '>' || isWhitespace(c)) {
                        String name = Tag.toHTMLAttributeName (attrName.toString());
                        tag.setLabel (name, attrVal.toString());
                        attrs.addElement (name);
                        state = ATTR;
                    }
                    else if (c == '&') {
                        ++bufptr;
                        postEntityState = ATTRVAL;
                        entityTargetBuf = attrVal;
                        state = ENTITY;
                    }
                    else {
                        ++bufptr;
                        attrVal.append (c);
                        // state == ATTRVAL;
                    }
                    break;

                case ATTRVAL_SQ:
                    if (c=='\'') {
                        ++bufptr;
                        String name = Tag.toHTMLAttributeName (attrName.toString());
                        tag.setLabel (name, attrVal.toString());
                        attrs.addElement (name);
                        state = ATTR;
                    }
                    else if (c == '&') {
                        ++bufptr;
                        postEntityState = ATTRVAL_SQ;
                        entityTargetBuf = attrVal;
                        state = ENTITY;
                    }
                    else {
                        ++bufptr;
                        attrVal.append (c);
                        // state == ATTRVAL_SQ;
                    }
                    break;

                case ATTRVAL_DQ:
                    if (c=='"') {
                        ++bufptr;
                        String name = Tag.toHTMLAttributeName (attrName.toString());
                        tag.setLabel (name, attrVal.toString());
                        attrs.addElement (name);
                        state = ATTR;
                    }
                    else if (c == '&') {
                        ++bufptr;
                        postEntityState = ATTRVAL_DQ;
                        entityTargetBuf = attrVal;
                        state = ENTITY;
                    }
                    else {
                        ++bufptr;
                        attrVal.append (c);
                        // state == ATTRVAL_DQ;
                    }
                    break;

                default:
                    throw new RuntimeException ("HtmlTokenizer entered illegal state " + state);
            }
        }

        // EOF
        switch (state) {
            case INWORD:
                // EOF terminated some text -- save the text
                tokens.addElement (new Text (page, wordStart, bufbase+bufptr, wordBuf.toString ()));
                ++nWords;
                break;

            default:
                // EOF in the middle of tags is illegal
                // don't try to recover
                break;
        }

        int nTotal = tokens.size ();

        page.tokens = new Region[nTotal];
        tokens.copyInto (page.tokens);

        page.words = new Text[nWords];
        int textnum = 0;
        page.tags = new Tag[nTotal - nWords];
        int tagnum = 0;

        for (int i=0; i < nTotal; ++i) {
            if (page.tokens[i] instanceof Tag)
                page.tags[tagnum++] = (Tag)page.tokens[i];
            else
                page.words[textnum++] = (Text)page.tokens[i];
        }

        if (saveContent) {
            page.content = contentBuf.toString ();
            page.start = 0;
            page.end = contentBuf.length();
        }
    }

    private static boolean isWhitespace (char c) {
//#ifdef JDK1.1 
        return Character.isWhitespace (c);
//#endif JDK1.1
/*#ifdef JDK1.0
        return Character.isSpace (c);
#endif JDK1.0*/
    }
        

    private static Hashtable entities = new Hashtable2()
          .add ("quot", new Character ((char)34))
          .add ("amp", new Character ((char)38))
          .add ("lt", new Character ((char)60))
          .add ("gt", new Character ((char)62))
          .add ("nbsp", new Character ((char)160))
          .add ("iexcl", new Character ((char)161))
          .add ("cent", new Character ((char)162))
          .add ("pound", new Character ((char)163))
          .add ("curren", new Character ((char)164))
          .add ("yen", new Character ((char)165))
          .add ("brvbar", new Character ((char)167))
          .add ("sect", new Character ((char)167))
          .add ("uml", new Character ((char)168))
          .add ("copy", new Character ((char)169))
          .add ("ordf", new Character ((char)170))
          .add ("laquo", new Character ((char)171))
          .add ("not", new Character ((char)172))
          .add ("shy", new Character ((char)173))
          .add ("reg", new Character ((char)174))
          .add ("macr", new Character ((char)175))
          .add ("deg", new Character ((char)176))
          .add ("plusmn", new Character ((char)177))
          .add ("sup2", new Character ((char)178))
          .add ("sup3", new Character ((char)179))
          .add ("acute", new Character ((char)180))
          .add ("micro", new Character ((char)181))
          .add ("para", new Character ((char)182))
          .add ("middot", new Character ((char)183))
          .add ("cedil", new Character ((char)184))
          .add ("sup1", new Character ((char)185))
          .add ("ordm", new Character ((char)186))
          .add ("raquo", new Character ((char)187))
          .add ("frac14", new Character ((char)188))
          .add ("frac12", new Character ((char)189))
          .add ("frac34", new Character ((char)190))
          .add ("iquest", new Character ((char)191))
          .add ("Agrave", new Character ((char)192))
          .add ("Aacute", new Character ((char)193))
          .add ("Acirc", new Character ((char)194))
          .add ("Atilde", new Character ((char)195))
          .add ("Auml", new Character ((char)196))
          .add ("Aring", new Character ((char)197))
          .add ("AElig", new Character ((char)198))
          .add ("Ccedil", new Character ((char)199))
          .add ("Egrave", new Character ((char)200))
          .add ("Eacute", new Character ((char)201))
          .add ("Ecirc", new Character ((char)202))
          .add ("Euml", new Character ((char)203))
          .add ("Igrave", new Character ((char)204))
          .add ("Iacute", new Character ((char)205))
          .add ("Icirc", new Character ((char)206))
          .add ("Iuml", new Character ((char)207))
          .add ("ETH", new Character ((char)208))
          .add ("Ntilde", new Character ((char)209))
          .add ("Ograve", new Character ((char)210))
          .add ("Oacute", new Character ((char)211))
          .add ("Ocirc", new Character ((char)212))
          .add ("Otilde", new Character ((char)213))
          .add ("Ouml", new Character ((char)214))
          .add ("times", new Character ((char)215))
          .add ("Oslash", new Character ((char)216))
          .add ("Ugrave", new Character ((char)217))
          .add ("Uacute", new Character ((char)218))
          .add ("Ucirc", new Character ((char)219))
          .add ("Uuml", new Character ((char)220))
          .add ("Yacute", new Character ((char)221))
          .add ("THORN", new Character ((char)222))
          .add ("szlig", new Character ((char)223))
          .add ("agrave", new Character ((char)224))
          .add ("aacute", new Character ((char)225))
          .add ("acirc", new Character ((char)226))
          .add ("atilde", new Character ((char)227))
          .add ("auml", new Character ((char)228))
          .add ("aring", new Character ((char)229))
          .add ("aelig", new Character ((char)230))
          .add ("ccedil", new Character ((char)231))
          .add ("egrave", new Character ((char)232))
          .add ("eacute", new Character ((char)233))
          .add ("ecirc", new Character ((char)234))
          .add ("euml", new Character ((char)235))
          .add ("igrave", new Character ((char)236))
          .add ("iacute", new Character ((char)237))
          .add ("icirc", new Character ((char)238))
          .add ("iuml", new Character ((char)239))
          .add ("eth", new Character ((char)240))
          .add ("ntilde", new Character ((char)241))
          .add ("ograve", new Character ((char)242))
          .add ("oacute", new Character ((char)243))
          .add ("ocirc", new Character ((char)244))
          .add ("otilde", new Character ((char)245))
          .add ("ouml", new Character ((char)246))
          .add ("divide", new Character ((char)247))
          .add ("oslash", new Character ((char)248))
          .add ("ugrave", new Character ((char)249))
          .add ("uacute", new Character ((char)250))
          .add ("ucirc", new Character ((char)251))
          .add ("uuml", new Character ((char)252))
          .add ("yacute", new Character ((char)253))
          .add ("thorn", new Character ((char)254))
          .add ("yuml", new Character ((char)255))
          ;

    private static Character lookupEntityRef (String name) {
        return (Character) entities.get (name);
    }

    /*
     *  Parser (constructs a canonical tree of elements)
     *
     */

    Vector vElements = new Vector ();
    Vector vLinks = new Vector ();

    StringBuffer text = new StringBuffer ();

    // elements with no content: e.g., IMG, BR, HR.  End tags for these elements
    // are simply ignored.
    private static Hashtable empty = new Hashtable2 ()
          .add (Tag.AREA)
          .add (Tag.BASE)
          .add (Tag.BASEFONT)
          .add (Tag.BGSOUND)
          .add (Tag.BR)
          .add (Tag.COL)
          .add (Tag.COLGROUP)
          .add (Tag.COMMENT) // actually <!-- ... -->
          .add (Tag.HR)
          .add (Tag.IMG)
          .add (Tag.INPUT)
          .add (Tag.ISINDEX)
          .add (Tag.LINK)
          .add (Tag.META)
          .add (Tag.NEXTID)
          .add (Tag.PARAM)
          .add (Tag.SPACER)
          .add (Tag.WBR)
          ;

    // elements that close <P> (correspond to "%block" entity in HTML 3.2 DTD)
    static Hashtable blocktag = new Hashtable2()
          .add (Tag.P)
          .add (Tag.UL)
          .add (Tag.OL)
          .add (Tag.DIR)
          .add (Tag.MENU)
          .add (Tag.PRE)
          .add (Tag.XMP)
          .add (Tag.LISTING)
          .add (Tag.DL)
          .add (Tag.DIV)
          .add (Tag.CENTER)
          .add (Tag.BLOCKQUOTE)
          .add (Tag.FORM)
          .add (Tag.ISINDEX)
          .add (Tag.HR)
          .add (Tag.TABLE)