context.c

harvest是一个下载html网页得机器人

            current token is repeatable; 0 if it is not.
*/
int testend(mod, pos, andoptsw, newtknsw)
struct thdr mod[];            /* Model of current open element. */
struct mpos pos[];            /* Position in open element's model. */
int andoptsw;                 /* 1=test optional AND members; 0=ignore. */
int newtknsw;                 /* 1=new token test; 0=end element test. */
{
     int rc = 0;              /* Return code: RCNREQ RCHIT RCMISS RCEND */

     while (!rc) {
          TRACEEND("TRACEEND", mod, pos, rc, andoptsw);
          /* TESTMISS:
             If we've hit no tokens yet in the current group, and
             the current token is the last unhit one in the group we can test,
             we will end the group (it may never really have started!)
             because we might be able to try the token that follows it.
             In any group, a token is the last testable unhit token if it
             is the last sequential one, as the GI was already tested against
             the preceding unhit tokens.  In addition,
             in a SEQ group, it is the last testable unhit token if it isn't
             optional, because we can't skip past it to the following ones.
             If we end the group, before popping the level, set M to G, as this
             level`s group header will be the next level's current token.
          */
          if (!ANYHIT(H) && (T==GNUM
			     || (GTYPE==TTSEQ && BITOFF(TOCC, TOPT)))) {
               M = G; --P;
               if (P<=1) {
                    if (BITON(TOCC, TOPT) || TOKENHIT) rc = RCEND;
                    else                               rc = RCMISS;
               }
               continue;
          }
          /* TESTHIT:
             See if we've hit all the non-optional tokens in the group.
             If so, pop to the previous level and set the group's hit bit.
             If we were called from NEWTOKEN we are trying to find the token
             to test against the next start-tag, so if the group is repeatable,
             process it again.  (If not, we were called from ECONTEXT and
             are testing whether the element can be ended.)
             Otherwise, if we are at the first level, the element is over.
          */
          if ((GTYPE==TTOR  && TOKENHIT)
	      || (GTYPE==TTSEQ && T==(UNCH)GNUM
		  && (TOKENHIT || BITON(TOCC, TOPT)))
	      || (GTYPE==TTAND && allhit(&GHDR, H, 0, andoptsw))) {
               M = G;
	       --P;
	       HITSET(H, T);
	       Tstart = T;
               if (newtknsw && BITON(TOCC, TREP)) rc = RCHIT;
               else if (P<=1)                     rc = RCEND;
	       /* If we are looking for a new token to test against the next
		  start-tag, then we need to consider optional and members
		  in this group, even if we didn't need to consider them
		  in the group that we just ended because that group had
		  wrapped. */
	       else if (newtknsw) andoptsw = 1;
               /* Else loop to test new outer group. */
          }
          else rc = RCNREQ;   /* No group ended this time, so return. */
     }
     TRACEEND("ENDFOUND", mod, pos, rc, andoptsw);
     return(rc);
}
/* TOKENOPT: Return 1 if current token is contextually optional;
             otherwise, return 0.
*/
int tokenopt(mod, pos)
struct thdr mod[];            /* Model of current open element. */
struct mpos pos[];            /* Position in open element's model. */
{
     TRACEEND("TOKENOPT", mod, pos, 0, 0);
     return (BITON(TOCC, TOPT) /* Inherently optional. */
	     || TOKENHIT      /* Was hit (handles "plus" suffix case). */
	     || (!ANYHIT(H) && groupopt(mod, pos)));
			      /* In optional group with no hits. */
}
/* GROUPOPT: Temporarily makes the current group be the current token so that
             TOKENOPT() can be applied to it.  Returns the value returned
             by TOKENOPT.
*/
int groupopt(mod, pos)
struct thdr mod[];            /* Model of current open element. */
struct mpos pos[];            /* Position in open element's model. */
{
     UNCH saveM;              /* Save M when testing if group is not required.*/
     int rc;                  /* 1=contextually optional; 0=not. */

     if (P==1) return(BITON(GOCC, TOPT) || TOKENHIT);
     saveM = M; M = G; --P;
     rc = tokenopt(mod, pos);
     ++P; G = M; M = saveM;
     return(rc);
}
/* TOKENREQ: Returns RCREQ if the current token is "contextually required".
             That is, it is not contextually optional and
                 1) it is a member of a "seq" group that is either required
                    or has at least 1 hit token.
                 2) it is a member of an "and" group in which all other
                    tokens were hit.
                          Optional tokens are not counted
                          if GI is ETDCDATA, as we are looking for an
                          omitted start-tag.  Otherwise, they are counted,
                          as the GI might match one of them.
             Returns RCNREQ if the current token is "not required".
*/
int tokenreq(gi, mod, pos)
struct etd *gi;               /* ETD of new GI. */
struct thdr mod[];            /* Model of current open element. */
struct mpos pos[];            /* Position in open element's model. */
{
     TRACEGI("TOKENREQ", gi, mod, pos);
     return( tokenopt(mod, pos) ? RCNREQ
            : ( GTYPE==TTSEQ && (ANYHIT(H) || groupreq(gi, mod, pos)==RCREQ)
#if 0
	       || (GTYPE==TTAND && allhit(&GHDR, H, T, \*gi!=ETDCDATA*\ 1))
#endif
	       )
                ? RCREQ : RCNREQ );
}
/* GROUPREQ: Temporarily makes the current group be the current token so that
             TOKENREQ() can be applied to it.  Returns the value returned
             by TOKENREQ.
*/
int groupreq(gi, mod, pos)
struct etd *gi;               /* ETD of new GI. */
struct thdr mod[];            /* Model of current open element. */
struct mpos pos[];            /* Position in open element's model. */
{
     UNCH saveM;              /* Save M when testing if group is not required.*/
     int rc;                  /* Return code: RCREQ RCNREQ */

     if (P==1) return(BITOFF(GOCC, TOPT) ? RCREQ : RCNREQ);
     saveM = M; M = G; --P;
     rc = tokenreq(gi, mod, pos);
     ++P; G = M; M = saveM;
     return(rc);
}
/* GRPSZ: Returns the number of tokens spanned by a group in the model (M),
          from the group's start (G) to a specified index within the group (T).
          M = 0, plus 1 for each token in the group, plus the size of
          any subgroups (gotten by calling GRPSZ recursively).  On entry,
          M must be equal to G at the current level.
*/
int grpsz(g, t)
struct thdr *g;               /* mod[G]: Ptr to group in the model. */
int t;                        /* T: Index of last token in the group. */
{
     struct thdr *p = g;      /* Ptr to current token in the model. */
     int m = 0;               /* Size of group (including nested groups). */
     int i = 0;               /* Number of group members (loop counter). */
     UNS type;                /* Token type (without TOREP bits). */

     while (++i<=t) {
          ++p; ++m;
          type = GET(p->ttype, TTMASK);
          if (type==TTOR || type==TTSEQ || type==TTAND) {
               m += grpsz(p, p->tu.tnum);
               p = g+m;
          }
     }
     return(m);
}
/* ALLHIT: Returns 1 if all hit bits for the specified group are turned on,
           (other than those that correspond to optional tokens if "opt" is
           0) and the "but" bit (all bits if "but" bit is zero).  Otherwise,
           returns 0.  GRPSZ is used to skip past subgroup tokens.
*/
int allhit(p, hits, but, opt)
struct thdr *p;               /* mod[G]: Ptr to group in the model. */
unsigned long *hits;	      /* H: Hit bits to be tested. */
int but;                      /* Index of bit to ignore; 0=test all. */
int opt;                      /* 1=optional tokens must be hit; 0=ignore. */
{
     int b = 0;               /* Index of bit being tested in hits. */
     int e = p->tu.tnum;      /* Ending index (number of bits to test). */
     unsigned type;           /* Token type (without TOREP bits). */

     while (++p, ++b<=e) {
          if (HITOFF(hits,b) && (opt || BITOFF(p->ttype,TOPT)) && b!=but)
               return 0;
          if ((type = GET(p->ttype,TTMASK))==TTOR || type==TTSEQ || type==TTAND)
               p += grpsz(p, p->tu.tnum);
     }
     return 1;
}
/* OFFBIT: Returns the index of the first unset bit after (i.e., not including)
           the caller's "first" bit. If all bits through the
           specified last bit are on, it returns 0.
*/
int offbit(bits, first, last)
unsigned long *bits;	      /* Bits to be tested. */
int first;                    /* Index of first bit to be tested in bits. */
int last;                     /* Index of last bit to be tested in bits. */
{
     while (++first <= last)
          if (HITOFF(bits, first))
	       return first;
     return 0;
}

/* ANYHIT: Return 1 if any bit is set. */

int anyhit(bits)
unsigned long *bits;
{
     int i;
     for (i = 0; i < grplongs; i++)
	  if (bits[i] != 0)
	       return 1;
     return 0;
}

/*
Local Variables:
c-indent-level: 5
c-continued-statement-offset: 5
c-brace-offset: -5
c-argdecl-indent: 0
c-label-offset: -5
comment-column: 30
End:
*/