tokens.c

harvest是一个下载html网页得机器人

/*
 * Definitions for each WP token.
 * 
 *
 */

#include <stdio.h>

#include "tokens.h"
#include "charsets.h"

extern int silent;

int popEnvironment(state *st, struct WPToken *wpt, void *data)
{
  extern void leave_environment(state *, int);
  leave_environment(st,wpt->wp_intdata);
  return 1;
}

int hardReturn(state *st, struct WPToken *wpt, void *data)
{
  process0(st,HRt);
  return 1;
}

int softPage(state *st, struct WPToken *wpt, void *data)
{
  return 1;
}

int hardReturnSoftPage(state *st, struct WPToken *wpt, void *data)
{
  hardReturn(st,wpt,data);
  softPage(st,wpt,data);
  return 1;
}

int ignoreByte(state *st, struct WPToken *wpt, void *data)
{
  if(!silent) {
    fprintf(stderr,"Ignoring byte [%02x]\n",wpt->wp_code);
  }
  return 1;
}

int ignoreFixed(state *st, struct WPToken *wpt, void *data)
{
  if(!silent) {
    fprintf(stderr,"Ignoring fixed [%02x]\n",wpt->wp_code);
  }
  return(1);
}

int ignoreVar(state *st, struct WPToken *wpt, void *data) 
{
  struct VariableCode *vc=data;
  if(!silent) {
    fprintf(stderr,"Ignoring variable [%02x] sub [%02x] length: %d\n",
	    vc->func, vc->subFunc, vc->len);
  }
  return 1;
}

int hardPage(state *st, struct WPToken *wpt, void *data)
{
  process0(st,HPg);
  return 1;
}

int softReturn(state *st, struct WPToken *wpt, void *data)
{
  process0(st,SRt);
  return 1;
}

int outputToken(state *st, struct WPToken *wpt, void *data)
{
  process0(st,wpt->wp_intdata);
  return 1;
}

int extendedChar(state *st, struct WPToken *wpt, void *data)
{
  char *bytes=data;
  int  char_code=bytes[1];
  int  char_set=bytes[2];

  if ( char_set >= 0 && char_set <= 8 ) 
    fputs(isolatin1_chars[char_set][char_code],st->outfile);
  else
    fprintf( st->outfile, "<!-- CS=%d, CC=%d -->", char_set, char_code );

  return 1;
}
  
int attributeOn(state *st, struct WPToken *wpt, void *data)
{
  char *bytes=data;

  char attribute=bytes[1];

  switch(attribute) {
  case 0: /* extra large */
    process0(st,ExtraLarge);
    break;
  case 1: /* very large */
    process0(st,VeryLarge);
    break;
  case 2: /* large */
    process0(st,Large);
    break;
  case 3: /* small */
    process0(st,Small);
    break;
  case 4: /* fine */
    process0(st,Fine);
    break;
  case 5: /* superscript */
    process0(st,Sup);
    break;
  case 6: /* subscript */
    process0(st,Sub);
    break;
  case 7: /* outline */
    process0(st,Outline);
    break;
  case 8: /* italics */
    process0(st,Italics);
    break;
  case 9: /* shadow */
    process0(st,Shadow);
    break;
  case 10: /* redline */
    process0(st,Red);
    break;
  case 11: /* double underline */
    process0(st,DoubleUnd);
    break;
  case 12: /* bold */
    process0(st,Bold);
    break;
  case 13: /* strikeout */
    process0(st,Strike);
    break;
  case 14: /* underline */
    process0(st,Und);
    break;
  case 15: /* small caps */
    process0(st,SmallCaps);
    break;
  }
  return 1;
}
  
int attributeOff(state *st, struct WPToken *wpt, void *data)
{
  char *bytes=data;
  char attribute=bytes[1];

  switch(attribute) {
  case 0: /* extra large */
    process0(st,eExtraLarge);
    break;
  case 1: /* very large */
    process0(st,eVeryLarge);
    break;
  case 2: /* large */
    process0(st,eLarge);
    break;
  case 3: /* small */
    process0(st,eSmall);
    break;
  case 4: /* fine */
    process0(st,eFine);
    break;
  case 5: /* superscript */
    process0(st,eSup);
    break;
  case 6: /* subscript */
    process0(st,eSub);
    break;
  case 7: /* outline */
    process0(st,eOutline);
    break;
  case 8: /* italics */
    process0(st,eItalics);
    break;
  case 9: /* shadow */
    process0(st,eShadow);
    break;
  case 10: /* redline */
    process0(st,eRed);
    break;
  case 11: /* double underline */
    process0(st,eDoubleUnd);
    break;
  case 12: /* bold */
    process0(st,eBold);
    break;
  case 13: /* strikeout */
    process0(st,eStrike);
    break;
  case 14: /* underline */
    process0(st,eUnd);
    break;
  case 15: /* small caps */
    process0(st,eSmallCaps);
    break;
  }

  return 1;
}

int endIndent(state *st, struct WPToken *wpt, void *data)
{
  /* do nothing */
  return 1;
}

int startIndent(state *st, struct WPToken *wpt, void *data)
{
  char *bytes=data;
  
  if(bytes[1]&1) {
    /* left/right indent */
    process0(st,DIndent);
  }
  else {
    process0(st,Indent);
  }
  do {
    if(process_token(st)==0) return 0;
  } while(!(st->lastToken->wp_handleFunction==endIndent));

  if(bytes[1]&1) {
    /* left/right indent */
    process0(st,eDIndent);
  }
  else {
    process0(st,eIndent);
  }
  return 1;
}



struct WPToken wp_tokens[]={
/* wp_type     wp_code  wp_intdata wp_strdata wp_size    wp_handleFunction */
  {reserved,   0x00,    0,         NULL,      0,         NULL},
  {singleByte, 0x01,    0,         NULL,      0,         ignoreByte},
  {singleByte, 0x02,    0,         NULL,      0,         ignoreByte},
  {singleByte, 0x03,    0,         NULL,      0,         ignoreByte},
  {singleByte, 0x04,    0,         NULL,      0,         ignoreByte},
  {singleByte, 0x05,    0,         NULL,      0,         ignoreByte},
  {singleByte, 0x06,    0,         NULL,      0,         ignoreByte},