tokens.h

harvest是一个下载html网页得机器人

/*
 * Structure to hold all the tokens
 * 
 */

typedef struct state {
  FILE *infile;
  FILE *outfile;
  struct WPToken *lastToken;
  char *data;
} state;

struct WPToken {
  enum token_class {
    reserved,
    literal,
    singleByte,
    fixedLength,
    varLength,
  } wp_type;
  int  wp_code;
  int  wp_intdata;
  char *wp_strdata;
  int  wp_size;        /* for fixed length types */
  int  (*wp_handleFunction)(state *st,struct WPToken *wpt,void *data);
};

extern struct WPToken wp_tokens[];

struct VariableCode {
  char func;
  char subFunc;
  short len;
  char data[1];
};

/************************************************************************/
/* Remembering the expansions                                           */
/************************************************************************/

/* The array "expansion" contains the expansions for everything.
 * Everything is initialized to NULL.
 *
 * We set up things as follows:
 *  expansion[0..255]  contain the expansions for the possible characters.
 *  expansion[256...]  contain the expansions for the special codes.
 *
 * Make sure this table is kept in parallel with the names[] array
 *
 *
 */

/*      name   value          When is it expanded? */
/*      ----    ---           -------------------- */
enum token_type {          /* Some compilers do not like unnamed enums */
        typeout = 256,     /* Typed out as soon as it is encountered */
        BEGIN        ,     /* Before the first character of the file */
        END          ,     /* After the last character of the file   */
        Comment      ,     /* For wp2x-generated comments            */
        eComment     ,

        PageNo       ,     /* Current page number */
        RomanPage    ,     /* Set page number (to roman numerals) */
        ArabicPage   ,     /* Set page number (to arabic) */

        HSpace       ,     /* unbreakable space (`Hard space') */

        Tab          ,     /* Tab character */
        BeginTabs    ,     /* Begin tab settings */

        /* DO NOT CHANGE THE RELATIVE ORDER OF THESE FOUR TOKENS */
        SetTab       ,     /* Set normal tabstop at %d */
        SetTabCenter ,     /* Set center tabstop at %d */
        SetTabRight  ,     /* Set right-justified tab at %d */
        SetTabDecimal,     /* Set decimal tab at %d */

        EndTabs      ,     /* End tab settings */

        HPg          ,     /* Hard page break */
        CondEOP      ,     /* Conditional end-of-page */
        HRt          ,     /* Hard return */
        SRt          ,     /* Soft return */

        NHyph        ,     /* Normal hyphen */
        NHyphE       ,     /* Normal hyphen at the end of a line */
        HHyph        ,     /* Hard (nonbreakable) hyphen */
        DHyph        ,     /* Discretionary hyphen */
        DHyphE       ,     /* Discretionary hyphen at the end of a line */
        NoHyphWord   ,     /* Do not hyphenate this word */

        Marg         ,     /* Margin settings */
        TopMarg      ,     /* Set top margin */
        PageLength   ,     /* Set page length */

        SS           ,     /* Single spacing */
        DS           ,     /* Double spacing */
        OHS          ,     /* 1.5 spacing (One and a Half Spacing) */
        TS           ,     /* Triple spacing */
        LS           ,     /* Generic line spacing */
        LPI          ,     /* set 6 or 8 LPI */

        Bold         ,     /* Begin boldface */
        eBold        ,     /* End boldface */
        Und          ,     /* Begin underline */
        eUnd         ,     /* End underline */
        DoubleUnd    ,     /* Begin double underline */
        eDoubleUnd   ,     /* End double underline */
        Red          ,     /* Begin redline */
        eRed         ,     /* End redline */
        Strike       ,     /* Begin strikeout */
        eStrike      ,     /* End strikeout */
        Rev          ,     /* Begin reverse video */
        eRev         ,     /* End reverse video */
	Outline      ,     /* Begin outline mode */
	eOutline     ,     /* End outline mode */
	Fine         ,     /* Begin fine mode */
	eFine        ,     /* End fine mode */

        Over         ,     /* Overstrike */
        eOver        ,     /* [mythical "end overstroke" code] */
        Sup          ,     /* Superscript */
        eSup         ,     /* [mythical "end superscript" code] */
        Sub          ,     /* Subscript */
        eSub         ,     /* [mythical "end subscript" code] */

	Large        ,     /* Large */
	eLarge       ,     /* end Large */
	Small        ,     /* Small */
	eSmall       ,     /* end Small */
	VeryLarge    ,     /* VeryLarge */
	eVeryLarge   ,     /* end very large */
	ExtraLarge   ,     /* extra large */
	eExtraLarge  ,     /* end extra large */
	Italics      ,     /* Italics */
	eItalics     ,     /* end Italics */
	Shadow       ,     /* Shadow */
	eShadow      ,     /* end shadow */
	SmallCaps    ,     /* start small caps */
	eSmallCaps   ,     /* end small caps */

        UpHalfLine   ,     /* Advance printer up 1/2 line */
        DownHalfLine ,     /* Advance printer down 1/2 line */
        AdvanceToHalfLine, /* Advance to absolute vertical position */

        Indent       ,     /* Indented paragraph */
        DIndent      ,     /* Left-and-right-indented paragraph */
        eIndent      ,     /* End indented paragraph */
        eDIndent      ,    /* End left-and-right paragraph */
        MargRel      ,     /* Margin release (unknown argument) */

        Center       ,     /* Center current line */
        eCenter      ,     /* End centering */
        CenterHere   ,     /* Center line around current column */
        eCenterHere  ,     /* End centering */

        Align        ,     /* Begin alignment */
        eAlign       ,     /* End alignment */
        AlignChar    ,     /* Set alignment character */
        FlushRight   ,     /* Begin flush right */
        eFlushRight  ,     /* End flush right */

        Math         ,     /* Begin math mode */
        eMath        ,     /* End math mode */
        MathCalc     ,     /* Begin math calc mode */
        MathCalcColumn,    /* Math calc column */

        SubTtl       ,     /* Do subtotal */
        IsSubTtl     ,     /* Subtotal entry */
        Ttl          ,     /* Do total */
        IsTtl        ,     /* Total entry */
        GrandTtl     ,     /* Do grand total */
        NegateTotal  ,     /* Negate current total */

        Col          ,     /* Begin column mode */
        eCol         ,     /* End column mode */

        Fn           ,     /* Begin footnote */
        eFn          ,     /* End footnote */
        En           ,     /* Begin endnote */
        eEn          ,     /* End endnote */
        SetFnNum     ,     /* Set footnote number */
        FNoteNum     ,     /* Footnote number */
        ENoteNum     ,     /* Endnote number */
	FigureNum    ,     /* Figure number */
        TableMarker  ,     /* Table of contents or whatever marker */

        Hyph         ,     /* Hyphenation on */
        eHyph        ,     /*             off */
        Just         ,     /* Justification on */
        eJust        ,     /*               off */
        Wid          ,     /* Widow/orphan protection on */
        eWid         ,     /*                         off */
        HZone        ,     /* Hyphenation zone */
        DAlign       ,     /* Decimal alignment character */

        Header       ,     /* Begin header text */
        eHeader      ,     /* End header text */
        Footer       ,     /* Begin footer text */
        eFooter      ,     /* End footer text */

        Supp         ,     /* Suppress formatting for one page */
        CtrPg        ,     /* Center page vertically */

        SetFont      ,     /* Change pitch or font */
        SetBin       ,     /* Select paper bin (0, 1, ...) */

        PN           ,     /* Page number position (PN+0 through PN+8) */

/* Internal tokens for unsupported operations */
        UnsupportedPlaceHolder = PN + 9,
        SetPageNumberColumn,
        SetTabs,
        SetUnderlineMode,
        DefineColumn,
        SetFootnoteAttributes,
        SetParagraphNumberingStyle,
        NumberedParagraph,
        BeginMarkedText,
        EndMarkedText,
        DefineMarkedText,
        DefineIndexMark,
        DefineMathColumns,
        Obsolete,
        ReservedCode,
        UnknownCode,
	MergeCode,
        LastToken
};

    
#define process0(st,c)     process(st,c,0,0)
#define process1(st,c,a)   process(st,c,a,0)
#define process2(st,c,a,b) process(st,c,a,b)