htmlparser.java

java写的crawler

                    else {
                        tagName.append (c);
                        ++bufptr;
                        // state == ETAG
                    }
                    break;

                // Attributes
                case ATTR:
                    if (isWhitespace(c))
                        ++bufptr;
                    else if (c == '>') {
                        ++bufptr;
                        tag.end = bufbase+bufptr;
                        if (attrs.size() > 0) {
                            tag.htmlAttributes = new String[attrs.size()];
                            attrs.copyInto (tag.htmlAttributes);
                        }
                        state = START;
                    }
                    else {
                        attrName.setLength (0);
                        state = ATTRNAME;
                    }
                    break;

                case ATTRNAME:
                    if (c == '>' || c == '=' || isWhitespace(c)) {
                        state = EQ;
                    }
                    else {
                        attrName.append (c);
                        ++bufptr;
                        // state == ATTRNAME;
                    }
                    break;

                case EQ:
                    if (isWhitespace(c))
                        ++bufptr;
                    else if (c == '=') {
                        ++bufptr;
                        state = AFTEREQ;
                    }
                    else {
                        String name = Tag.toHTMLAttributeName (attrName.toString());
                        tag.setLabel (name);
                        attrs.addElement (name);
                        state = ATTR;
                    }
                    break;

                case AFTEREQ:
                    if (isWhitespace (c))
                        ++bufptr;
                    else
                        switch (c) {
                            case '>':
                            {
                                String name = Tag.toHTMLAttributeName (attrName.toString());
                                tag.setLabel (name);
                                attrs.addElement (name);
                                state = ATTR;
                                break;
                            }
                            case '\'':
                                ++bufptr;
                                attrVal.setLength (0);
                                state = ATTRVAL_SQ;
                                break;
                            case '"':
                                ++bufptr;
                                attrVal.setLength (0);
                                state = ATTRVAL_DQ;
                                break;
                            default:
                                attrVal.setLength (0);
                                state = ATTRVAL;
                                break;
                        }
                    break;

                case ATTRVAL:
                    if (c == '>' || isWhitespace(c)) {
                        String name = Tag.toHTMLAttributeName (attrName.toString());
                        tag.setLabel (name, attrVal.toString());
                        attrs.addElement (name);
                        state = ATTR;
                    }
                    else if (c == '&') {
                        ++bufptr;
                        postEntityState = ATTRVAL;
                        entityTargetBuf = attrVal;
                        state = ENTITY;
                    }
                    else {
                        ++bufptr;
                        attrVal.append (c);
                        // state == ATTRVAL;
                    }
                    break;

                case ATTRVAL_SQ:
                    if (c=='\'') {
                        ++bufptr;
                        String name = Tag.toHTMLAttributeName (attrName.toString());
                        tag.setLabel (name, attrVal.toString());
                        attrs.addElement (name);
                        state = ATTR;
                    }
                    else if (c == '&') {
                        ++bufptr;
                        postEntityState = ATTRVAL_SQ;
                        entityTargetBuf = attrVal;
                        state = ENTITY;
                    }
                    else {
                        ++bufptr;
                        attrVal.append (c);
                        // state == ATTRVAL_SQ;
                    }
                    break;

                case ATTRVAL_DQ:
                    if (c=='"') {
                        ++bufptr;
                        String name = Tag.toHTMLAttributeName (attrName.toString());
                        tag.setLabel (name, attrVal.toString());
                        attrs.addElement (name);
                        state = ATTR;
                    }
                    else if (c == '&') {
                        ++bufptr;
                        postEntityState = ATTRVAL_DQ;
                        entityTargetBuf = attrVal;
                        state = ENTITY;
                    }
                    else {
                        ++bufptr;
                        attrVal.append (c);
                        // state == ATTRVAL_DQ;
                    }
                    break;

                default:
                    throw new RuntimeException ("HtmlTokenizer entered illegal state " + state);
            }
        }

        // EOF
        switch (state) {
            case INWORD:
                // EOF terminated some text -- save the text
                tokens.addElement (new Text (page, wordStart, bufbase+bufptr, wordBuf.toString ()));
                ++nWords;
                break;

            default:
                // EOF in the middle of tags is illegal
                // don't try to recover
                break;
        }

        int nTotal = tokens.size ();

        page.tokens = new Region[nTotal];
        tokens.copyInto (page.tokens);

        page.words = new Text[nWords];
        int textnum = 0;
        page.tags = new Tag[nTotal - nWords];
        int tagnum = 0;

        for (int i=0; i < nTotal; ++i) {
            if (page.tokens[i] instanceof Tag)
                page.tags[tagnum++] = (Tag)page.tokens[i];
            else
                page.words[textnum++] = (Text)page.tokens[i];
        }
    }

    private static boolean isWhitespace (char c) {
//#ifdef JDK1.1 
        return Character.isWhitespace (c);
//#endif JDK1.1
/*#ifdef JDK1.0
        return Character.isSpace (c);
#endif JDK1.0*/
    }
        

    private static Hashtable entities = new Hashtable2()
          .add ("quot", new Character ((char)34))
          .add ("amp", new Character ((char)38))
          .add ("lt", new Character ((char)60))
          .add ("gt", new Character ((char)62))
          .add ("nbsp", new Character ((char)160))
          .add ("iexcl", new Character ((char)161))
          .add ("cent", new Character ((char)162))
          .add ("pound", new Character ((char)163))
          .add ("curren", new Character ((char)164))
          .add ("yen", new Character ((char)165))
          .add ("brvbar", new Character ((char)167))
          .add ("sect", new Character ((char)167))
          .add ("uml", new Character ((char)168))
          .add ("copy", new Character ((char)169))
          .add ("ordf", new Character ((char)170))
          .add ("laquo", new Character ((char)171))
          .add ("not", new Character ((char)172))
          .add ("shy", new Character ((char)173))
          .add ("reg", new Character ((char)174))
          .add ("macr", new Character ((char)175))
          .add ("deg", new Character ((char)176))
          .add ("plusmn", new Character ((char)177))
          .add ("sup2", new Character ((char)178))
          .add ("sup3", new Character ((char)179))
          .add ("acute", new Character ((char)180))
          .add ("micro", new Character ((char)181))
          .add ("para", new Character ((char)182))
          .add ("middot", new Character ((char)183))
          .add ("cedil", new Character ((char)184))
          .add ("sup1", new Character ((char)185))
          .add ("ordm", new Character ((char)186))
          .add ("raquo", new Character ((char)187))
          .add ("frac14", new Character ((char)188))
          .add ("frac12", new Character ((char)189))
          .add ("frac34", new Character ((char)190))
          .add ("iquest", new Character ((char)191))
          .add ("Agrave", new Character ((char)192))
          .add ("Aacute", new Character ((char)193))
          .add ("Acirc", new Character ((char)194))
          .add ("Atilde", new Character ((char)195))
          .add ("Auml", new Character ((char)196))
          .add ("Aring", new Character ((char)197))
          .add ("AElig", new Character ((char)198))
          .add ("Ccedil", new Character ((char)199))
          .add ("Egrave", new Character ((char)200))
          .add ("Eacute", new Character ((char)201))
          .add ("Ecirc", new Character ((char)202))
          .add ("Euml", new Character ((char)203))
          .add ("Igrave", new Character ((char)204))
          .add ("Iacute", new Character ((char)205))
          .add ("Icirc", new Character ((char)206))
          .add ("Iuml", new Character ((char)207))
          .add ("ETH", new Character ((char)208))
          .add ("Ntilde", new Character ((char)209))
          .add ("Ograve", new Character ((char)210))
          .add ("Oacute", new Character ((char)211))
          .add ("Ocirc", new Character ((char)212))
          .add ("Otilde", new Character ((char)213))
          .add ("Ouml", new Character ((char)214))
          .add ("times", new Character ((char)215))
          .add ("Oslash", new Character ((char)216))
          .add ("Ugrave", new Character ((char)217))
          .add ("Uacute", new Character ((char)218))
          .add ("Ucirc", new Character ((char)219))
          .add ("Uuml", new Character ((char)220))
          .add ("Yacute", new Character ((char)221))
          .add ("THORN", new Character ((char)222))
          .add ("szlig", new Character ((char)223))
          .add ("agrave", new Character ((char)224))
          .add ("aacute", new Character ((char)225))
          .add ("acirc", new Character ((char)226))
          .add ("atilde", new Character ((char)227))
          .add ("auml", new Character ((char)228))
          .add ("aring", new Character ((char)229))
          .add ("aelig", new Character ((char)230))
          .add ("ccedil", new Character ((char)231))
          .add ("egrave", new Character ((char)232))
          .add ("eacute", new Character ((char)233))
          .add ("ecirc", new Character ((char)234))
          .add ("euml", new Character ((char)235))
          .add ("igrave", new Character ((char)236))
          .add ("iacute", new Character ((char)237))
          .add ("icirc", new Character ((char)238))
          .add ("iuml", new Character ((char)239))
          .add ("eth", new Character ((char)240))
          .add ("ntilde", new Character ((char)241))
          .add ("ograve", new Character ((char)242))
          .add ("oacute", new Character ((char)243))
          .add ("ocirc", new Character ((char)244))
          .add ("otilde", new Character ((char)245))
          .add ("ouml", new Character ((char)246))
          .add ("divide", new Character ((char)247))
          .add ("oslash", new Character ((char)248))
          .add ("ugrave", new Character ((char)249))
          .add ("uacute", new Character ((char)250))
          .add ("ucirc", new Character ((char)251))
          .add ("uuml", new Character ((char)252))
          .add ("yacute", new Character ((char)253))
          .add ("thorn", new Character ((char)254))
          .add ("yuml", new Character ((char)255))
          ;

    private static Character lookupEntityRef (String name) {
        return (Character) entities.get (name);
    }

    /*
     *  Parser (constructs a canonical tree of elements)
     *
     */

    Vector vElements = new Vector ();
    Vector vLinks = new Vector ();

    StringBuffer text = new StringBuffer ();

    // elements with no content: e.g., IMG, BR, HR.  End tags for these elements
    // are simply ignored.
    private static Hashtable empty = new Hashtable2 ()
          .add (Tag.AREA)
          .add (Tag.BASE)
          .add (Tag.BASEFONT)
          .add (Tag.BGSOUND)
          .add (Tag.BR)
          .add (Tag.COL)
          .add (Tag.COLGROUP)
          .add (Tag.COMMENT) // actually <!-- ... -->
          .add (Tag.HR)
          .add (Tag.IMG)
          .add (Tag.INPUT)
          .add (Tag.ISINDEX)
          .add (Tag.LINK)
          .add (Tag.META)
          .add (Tag.NEXTID)
          .add (Tag.PARAM)
          .add (Tag.SPACER)
          .add (Tag.WBR)
          ;

    // elements that close <P> (correspond to "%block" entity in HTML 3.2 DTD)
    static Hashtable blocktag = new Hashtable2()
          .add (Tag.P)
          .add (Tag.UL)
          .add (Tag.OL)
          .add (Tag.DIR)
          .add (Tag.MENU)
          .add (Tag.PRE)
          .add (Tag.XMP)
          .add (Tag.LISTING)
          .add (Tag.DL)
          .add (Tag.DIV)
          .add (Tag.CENTER)
          .add (Tag.BLOCKQUOTE)
          .add (Tag.FORM)
          .add (Tag.ISINDEX)
          .add (Tag.HR)
          .add (Tag.TABLE)
          .add (Tag.H1)
          .add (Tag.H2)
          .add (Tag.H3)
          .add (Tag.H4)
          .add (Tag.H5)
          .add (Tag.H6)
          .add (Tag.ADDRESS)
          ;

    // maps elements which force closure to the elements that they close, e.g.,
    // LI maps to LI, DT maps to DD,DT, and all block-level tags map to P.
    private static Hashtable forcesClosed = new Hashtable2 ()
          .add (Tag.DD, new Hashtable2 () .add (Tag.DD) .add (Tag.DT))
          .add (Tag.DT, new Hashtable2 () .add (Tag.DD) .add (Tag.DT))
          .add (Tag.LI, new Hashtable2 () .add (Tag.LI))
          .add (Tag.OPTION, new Hashtable2 () .add (Tag.OPTION))
          .add (Tag.TR, new Hashtable2 () .add (Tag.TR))
          .add (Tag.TD, new Hashtable2 () .add (Tag.TD) .add (Tag.TH))
          .add (Tag.TH, new Hashtable2 () .add (Tag.TD) .add (Tag.TH))
          ;
    static {
        Hashtable p = new Hashtable2 () .add (Tag.P);
        Enumeration enum = blocktag.keys ();
        while (enum.hasMoreElements ())
            union (forcesClosed, enum.nextElement(), p);
    }

    // union of forcesClosed plus the tag's possible containers.  For instance,
    // LI maps to LI, OL, UL, MENU, DIR.  When a forcesClosed tag like LI is
    // encountered, the parser looks upward for the first context tag.
    // Having the tag's container element included in the search ensures that
    // LI in a nested list won't close its parent LI.
    static Hashtable context = new Hashtable2 ()
          .add (Tag.DD, new Hashtable2 () .add (Tag.DL))
          .add (Tag.DT, new Hashtable2 () .add (Tag.DL))
          .add (Tag.LI, new Hashtable2 () .add (Tag.OL) .add (Tag.UL) .add (Tag.MENU) .add (Tag.DIR))
          .add (Tag.OPTION, new Hashtable2 () .add (Tag.SELECT))
          .add (Tag.TR, new Hashtable2 () .add (Tag.TABLE))