tokens.c

c语言编译器

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cpp.h"

static char wbuf[2*OBS];
static char *wbp = wbuf;

/*
 * 1 for tokens that don't need whitespace when they get inserted
 * by macro expansion
 */
static const char wstab[] = {
	0,	/* END */
	0,	/* UNCLASS */
	0,	/* NAME */
	0,	/* NUMBER */
	0,	/* STRING */
	0,	/* CCON */
	1,	/* NL */
	0,	/* WS */
	0,	/* DSHARP */
	0,	/* EQ */
	0,	/* NEQ */
	0,	/* LEQ */
	0,	/* GEQ */
	0,	/* LSH */
	0,	/* RSH */
	0,	/* LAND */
	0,	/* LOR */
	0,	/* PPLUS */
	0,	/* MMINUS */
	0,	/* ARROW */
	1,	/* SBRA */
	1,	/* SKET */
	1,	/* LP */
	1,	/* RP */
	0,	/* DOT */
	0,	/* AND */
	0,	/* STAR */
	0,	/* PLUS */
	0,	/* MINUS */
	0,	/* TILDE */
	0,	/* NOT */
	0,	/* SLASH */
	0,	/* PCT */
	0,	/* LT */
	0,	/* GT */
	0,	/* CIRC */
	0,	/* OR */
	0,	/* QUEST */
	0,	/* COLON */
	0,	/* ASGN */
	1,	/* COMMA */
	0,	/* SHARP */
	1,	/* SEMIC */
	1,	/* CBRA */
	1,	/* CKET */
	0,	/* ASPLUS */
 	0,	/* ASMINUS */
 	0,	/* ASSTAR */
 	0,	/* ASSLASH */
 	0,	/* ASPCT */
 	0,	/* ASCIRC */
 	0,	/* ASLSH */
	0,	/* ASRSH */
 	0,	/* ASOR */
 	0,	/* ASAND */
	0,	/* ELLIPS */
	0,	/* DSHARP1 */
	0,	/* NAME1 */
	0,	/* DEFINED */
	0,	/* UMINUS */
};

void
maketokenrow(int size, Tokenrow *trp)
{
	trp->max = size;
	if (size>0)
		trp->bp = (Token *)domalloc(size*sizeof(Token));
	else
		trp->bp = NULL;
	trp->tp = trp->bp;
	trp->lp = trp->bp;
}

Token *
growtokenrow(Tokenrow *trp)
{
	int ncur = trp->tp - trp->bp;
	int nlast = trp->lp - trp->bp;

	trp->max = 3*trp->max/2 + 1;
	trp->bp = (Token *)realloc(trp->bp, trp->max*sizeof(Token));
	if (trp->bp == NULL)
		error(FATAL, "Out of memory from realloc");
	trp->lp = &trp->bp[nlast];
	trp->tp = &trp->bp[ncur];
	return trp->lp;
}

/*
 * Compare a row of tokens, ignoring the content of WS; return !=0 if different
 */
int
comparetokens(Tokenrow *tr1, Tokenrow *tr2)
{
	Token *tp1, *tp2;

	tp1 = tr1->tp;
	tp2 = tr2->tp;
	if (tr1->lp-tp1 != tr2->lp-tp2)
		return 1;
	for (; tp1<tr1->lp ; tp1++, tp2++) {
		if (tp1->type != tp2->type
		 || (tp1->wslen==0) != (tp2->wslen==0)
		 || tp1->len != tp2->len
		 || strncmp((char*)tp1->t, (char*)tp2->t, tp1->len)!=0)
			return 1;
	}
	return 0;
}

/*
 * replace ntok tokens starting at dtr->tp with the contents of str.
 * tp ends up pointing just beyond the replacement.
 * Canonical whitespace is assured on each side.
 */
void
insertrow(Tokenrow *dtr, int ntok, Tokenrow *str)
{
	int nrtok = rowlen(str);

	dtr->tp += ntok;
	adjustrow(dtr, nrtok-ntok);
	dtr->tp -= ntok;
	movetokenrow(dtr, str);
	makespace(dtr);
	dtr->tp += nrtok;
	makespace(dtr);
}

/*
 * make sure there is WS before trp->tp, if tokens might merge in the output
 */
void
makespace(Tokenrow *trp)
{
	uchar *tt;
	Token *tp = trp->tp;

	if (tp >= trp->lp)
		return;
	if (tp->wslen) {
		if (tp->flag&XPWS
		 && (wstab[tp->type] || trp->tp>trp->bp && wstab[(tp-1)->type])) {
			tp->wslen = 0;
			return;
		}
		tp->t[-1] = ' ';
		return;
	}
	if (wstab[tp->type] || trp->tp>trp->bp && wstab[(tp-1)->type])
		return;
	tt = newstring(tp->t, tp->len, 1);
	*tt++ = ' ';
	tp->t = tt;
	tp->wslen = 1;
	tp->flag |= XPWS;
}

/*
 * Copy an entire tokenrow into another, at tp.
 * It is assumed that there is enough space.
 *  Not strictly conforming.
 */
void
movetokenrow(Tokenrow *dtr, Tokenrow *str)
{
	int nby;

	/* nby = sizeof(Token) * (str->lp - str->bp); */
	nby = (char *)str->lp - (char *)str->bp;
	memmove(dtr->tp, str->bp, nby);
}

/*
 * Move the tokens in a row, starting at tr->tp, rightward by nt tokens;
 * nt may be negative (left move).
 * The row may need to be grown.
 * Non-strictly conforming because of the (char *), but easily fixed
 */
void
adjustrow(Tokenrow *trp, int nt)
{
	int nby, size;

	if (nt==0)
		return;
	size = (trp->lp - trp->bp) + nt;
	while (size > trp->max)
		growtokenrow(trp);
	/* nby = sizeof(Token) * (trp->lp - trp->tp); */
	nby = (char *)trp->lp - (char *)trp->tp;
	if (nby)
		memmove(trp->tp+nt, trp->tp, nby);
	trp->lp += nt;
}

/*
 * Copy a row of tokens into the destination holder, allocating
 * the space for the contents.  Return the destination.
 */
Tokenrow *
copytokenrow(Tokenrow *dtr, Tokenrow *str)
{
	int len = rowlen(str);

	maketokenrow(len, dtr);
	movetokenrow(dtr, str);
	dtr->lp += len;
	return dtr;
}

/*
 * Produce a copy of a row of tokens.  Start at trp->tp.
 * The value strings are copied as well.  The first token
 * has WS available.
 */
Tokenrow *
normtokenrow(Tokenrow *trp)
{
	Token *tp;
	Tokenrow *ntrp = new(Tokenrow);
	int len;

	len = trp->lp - trp->tp;
	if (len<=0)
		len = 1;
	maketokenrow(len, ntrp);
	for (tp=trp->tp; tp < trp->lp; tp++) {
		*ntrp->lp = *tp;
		if (tp->len) {
			ntrp->lp->t = newstring(tp->t, tp->len, 1);
			*ntrp->lp->t++ = ' ';
			if (tp->wslen)
				ntrp->lp->wslen = 1;
		}
		ntrp->lp++;
	}
	if (ntrp->lp > ntrp->bp)
		ntrp->bp->wslen = 0;
	return ntrp;
}

/*
 * Debugging
 */
void
peektokens(Tokenrow *trp, char *str)
{
	Token *tp;

	tp = trp->tp;
	flushout();
	if (str)
		fprintf(stderr, "%s ", str);
	if (tp<trp->bp || tp>trp->lp)
		fprintf(stderr, "(tp offset %d) ", tp-trp->bp);
	for (tp=trp->bp; tp<trp->lp && tp<trp->bp+32; tp++) {
		if (tp->type!=NL) {
			int c = tp->t[tp->len];
			tp->t[tp->len] = 0;
			fprintf(stderr, "%s", tp->t);
			tp->t[tp->len] = c;
		}
		if (tp->type==NAME) {
			fprintf(stderr, tp==trp->tp?"{*":"{");
			prhideset(tp->hideset);
			fprintf(stderr, "} ");
		} else
			fprintf(stderr, tp==trp->tp?"{%x*} ":"{%x} ", tp->type);
	}
	fprintf(stderr, "\n");
	fflush(stderr);
}

void
puttokens(Tokenrow *trp)
{
	Token *tp;
	int len;
	uchar *p;

	if (verbose)
		peektokens(trp, "");
	tp = trp->bp;
	for (; tp<trp->lp; tp++) {
		len = tp->len+tp->wslen;
		p = tp->t-tp->wslen;
		while (tp<trp->lp-1 && p+len == (tp+1)->t - (tp+1)->wslen) {
			tp++;
			len += tp->wslen+tp->len;
		}
		if (len>OBS/2) {		/* handle giant token */
			if (wbp > wbuf)
				fwrite(wbuf, 1, wbp-wbuf, stdout);
			fwrite((char *)p, 1, len, stdout);
			wbp = wbuf;
		} else {	
			memcpy(wbp, p, len);
			wbp += len;
		}
		if (wbp >= &wbuf[OBS]) {
			fwrite(wbuf, 1, OBS, stdout);
			if (wbp > &wbuf[OBS])
				memcpy(wbuf, wbuf+OBS, wbp - &wbuf[OBS]);
			wbp -= OBS;
		}
	}
	trp->tp = tp;
	if (cursource->fd==stdin)
		flushout();
}

void
flushout(void)
{
	if (wbp>wbuf) {
		fwrite(wbuf, 1, wbp-wbuf, stdout);
		fflush(stdout);
		wbp = wbuf;
	}
}

/*
 * turn a row into just a newline
 */
void
setempty(Tokenrow *trp)
{
	trp->tp = trp->bp;
	trp->lp = trp->bp+1;
	*trp->bp = nltoken;
}

/*
 * generate a number
 */
char *
outnum(char *p, int n)
{
	if (n>=10)
		p = outnum(p, n/10);
	*p++ = n%10 + '0';
	return p;
}

/*
 * allocate and initialize a new string from s, of length l, at offset o
 * Null terminated.
 */
uchar *
newstring(uchar *s, int l, int o)
{
	uchar *ns = (uchar *)domalloc(l+o+1);

	ns[l+o] = '\0';
	return (uchar*)strncpy((char*)ns+o, (char*)s, l) - o;
}