token.cpp

这是一个能够自动生成文档的程序

#include "stdafx.h"

#include "Token.h"
#include "CPGlobals.h"

#ifdef _DEBUG
#undef THIS_FILE
static char THIS_FILE[]=__FILE__;
#define new DEBUG_NEW
#endif

BEGIN_TEST_DUMP(CToken)
    //TEST_DUMP(mLine)
    //TEST_DUMP(mColumn)
    TEST_DUMP(mpContent)
	//TEST_DUMP(mType)
END_TEST_DUMP()

int CToken::sPrecedence[];

CToken::CToken(int iLine, int iColumn, UINT iType/*=TT_EMPTY*/, BOOL iForeSpace/*=0*/)
:
mLine(iLine),
mColumn(iColumn),
mType(iType),
mSubType(TTS_NON),
mpContent(NULL),
mForeSpace(iForeSpace)
{
}

CToken::CToken(int iLine, int iColumn, CToken* ipOther)
:
mLine(iLine),
mColumn(iColumn),
mpContent(NULL)
{
	ASSERT(ipOther != NULL);
	mType = ipOther->GetType();
	mSubType = ipOther->GetSubType();
	mForeSpace = ipOther->IsForeSpace();
	
	if(ipOther->GetContent() != NULL)
	{
		int len = strlen(ipOther->GetContent());
		CopyContent(ipOther->GetContent(), len);
	}
}

CToken::CToken(CToken* ipOther)
{
	ASSERT(ipOther != NULL);
    
	mLine = ipOther->GetLine();
	mColumn = ipOther->GetColumn();
	mType = ipOther->GetType();
	mSubType = ipOther->GetSubType();
	mForeSpace = ipOther->IsForeSpace();
	mpContent = NULL;

	if(ipOther->GetContent() != NULL)
	{
		int len = strlen(ipOther->GetContent());
		CopyContent(ipOther->GetContent(), len);
	}
}


CToken::~CToken()
{
    delete [] mpContent;
}

BOOL CToken::IsBeginOfWord(char ch)
{
    if(ch >= 'a' && ch <= 'z')
		return TRUE;

	if(ch >= 'A' && ch <= 'Z')
		return TRUE;

	if(ch == '_')
		return TRUE;

	return FALSE;
}

BOOL CToken::IsWordPart(char ch)
{
	if(ch >= 'a' && ch <= 'z')
		return TRUE;
	if(ch >= 'A' && ch <= 'Z')
		return TRUE;
	if(ch >= '0' && ch <= '9')
		return TRUE;
	if(ch == '_')
        return TRUE;

	return FALSE;
}

BOOL CToken::IsDigit(char ch)
{
	if(ch >= '0' && ch <= '9')
		return TRUE;

	return FALSE;
}

#define SKIP_CONNECT_LINE() \
if(ch == '\\') \
{ \
    char next = *pTemp; \
    if(next == '\n') \
{ \
    LINE_INCREASE(1); \
	COLUMN_RESET(); \
	len+=2;   \
	pTemp++; \
	ch = 0; \
    continue; \
} \
} \

int CToken::ReadWord(LPCTSTR ipCode)
{
    ASSERT(ipCode != NULL);
	LPCTSTR pTemp = ipCode;

	static char buf[1024] = {0};
	int len   = 0; //指针移到的字节数
	int count = 0; //有效的字符数
	while(TRUE)
	{
		char ch = *pTemp++;
		if(!IsWordPart(ch))
		{
			SKIP_CONNECT_LINE();
			break;
		}

		len++;
		buf[count++] = ch;
	}

	CopyContent(buf, count);
	return len;
}

int CToken::ReadDigit(LPCTSTR ipCode)
{
    ASSERT(ipCode != NULL);
	LPCTSTR pTemp = ipCode;

	mSubType = TTS_INT;
	char pre = 0;
	char ch = 0;
	BOOL isE = FALSE; //科学记数法
	BOOL isH = FALSE; //八进制或16进制
	BOOL isU = FALSE; //是否无符号
	static char buf[1024] = {0};
	int len = 0;
	int count = 0;
	while(TRUE)
	{
		pre = ch;
		ch = *pTemp++;

		if(pre == 0 && ch == '0')
			isH = TRUE;
		else if(ch == 'l' || ch == 'L')
			mSubType = TTS_LONG;
		else if(ch == 'f' || ch == 'F')
		    mSubType = TTS_FLOAT;
		else if(ch == '.')
			mSubType = TTS_DOUBLE;
		else if(ch == 'u' || ch == 'U')
			isU = TRUE;

		if(IsWordPart(ch) || ch == '.')
		{
			buf[count++] = ch;
			len++;
            if(!isH && (ch == 'e' || ch == 'E'))
			{
				mSubType = TTS_DOUBLE;
				isE = TRUE;
			}
		}
        else if(ch == '-' && isE)
		{
			buf[count++] = ch;
			len++;
			isE = FALSE;
		}
		else
		{
			SKIP_CONNECT_LINE();
			break;
		}
	}
	
	CopyContent(buf, count);
	if(isU) mSubType += 1; //无符号数

	TEST_TRACE(this);
	TEST_TRACE(ch);

	//检查数字是否合法
	return len;
}

int CToken::ReadSingleLineComment(LPCTSTR ipCode)
{
    ASSERT(ipCode != NULL);
	LPCTSTR pTemp = ipCode;
	
	int len = 0; 
	while(TRUE)
	{
		char ch = *pTemp++;
        SKIP_CONNECT_LINE();
		if(ch == '\0' || ch == '\n') 
			break;
		len++;
	}

	CopyContent(ipCode, len);
	return len;
}

int CToken::ReadMulLineComment(LPCTSTR ipCode)
{
    ASSERT(ipCode != NULL);
	LPCTSTR pTemp = ipCode;

	char pre = 0;
	char ch = 0;
	int len = 0;
	while(TRUE)
	{
		pre = ch;
		ch = *pTemp++;
		if(ch == '\0') break;
		if(ch == '\n') 
		{
			LINE_INCREASE(1);
			COLUMN_RESET();
		}

		len++;
        if(ch == '/' && pre == '*')
			break;
	}

	CopyContent(ipCode, len);
	return len;
}

int CToken::ReadString(LPCTSTR ipCode)
{
    ASSERT(ipCode != NULL);
	LPCTSTR pTemp = ipCode;

	char pre = 0;
	char ch = 0;
	int len = 0;
	while(TRUE)
	{
		if(pre == '\\' && ch == '\\')
			pre = '\0';
		else 
			pre = ch;

		ch = *pTemp++;
		if(ch == '\0') break;
        if(ch == '\n')
		{
			LINE_INCREASE(1);
			COLUMN_RESET();
		}

		len++;
        if(ch == '\"' && pre != '\\')
			break;
	}

	CopyContent(ipCode, len, "\"");
	return len+1;
}

int CToken::ReadWString(LPCTSTR ipCode)
{
    ASSERT(ipCode != NULL);
	LPCTSTR pTemp = ipCode;

	char pre = 0;
	char ch = 0;
	int len = 0;
	while(TRUE)
	{
		if(pre == '\\' && ch == '\\')
			pre = '\0';
		else 
			pre = ch;

		ch = *pTemp++;
		if(ch == '\0') break;
		if(ch == '\n')
		{
			LINE_INCREASE(1);
			COLUMN_RESET();
		}

		len++;
        if(ch == '\"' && pre != '\\')
			break;
	}

	CopyContent(ipCode, len, "L\"");
	return len+2;
}

int CToken::ReadChar(LPCTSTR ipCode)
{
    ASSERT(ipCode != NULL);
	LPCTSTR pTemp = ipCode;

	static char buf[4] = {0};
	int count = 0;
	int len = 0;
	char pre = '\0';
	char ch = '\0';
	while(count<3)
	{
		if(pre != '\\') pre = ch;
		else pre = '\0';

	    ch = *pTemp++;
		SKIP_CONNECT_LINE();
	    if(ch == '\0') break;
	    len++;
		buf[count++] = ch;
		if(ch == '\'' && pre != '\\')
			break;
	}

	for(; count>0; count--)
	{
		if(buf[count-1] != '\'')
		{
			len--;
			buf[count] = '\0';
		}
		else
		{
			break;
		}
	}

	CopyContent(buf, count, "\'");

	mType = TT_DIGIT; //改为数字类型
	mSubType = TTS_CHAR; //子类型为char

	return len+1;
}

int CToken::ReadWChar(LPCTSTR ipCode)
{
    ASSERT(ipCode != NULL);
	LPCTSTR pTemp = ipCode;
    
	static char buf[4] = {0};
	int count = 0;
	int len = 0;
	char pre = '\0';
	char ch = '\0';
	while(count<3)
	{
		if(pre != '\\') pre = ch;
		else pre = '\0';

	    ch = *pTemp++;
		SKIP_CONNECT_LINE();
	    if(ch == '\0') break;
	    len++;
		buf[count++] = ch;
		if(ch == '\'' && pre != '\\')
			break;
	}

	CopyContent(buf, count, "L\'");

	mType = TT_DIGIT; //改为数字类型
	mSubType = TTS_SHORT; //子类型为short

	return len+2;
}

void CToken::CopyContent(LPCTSTR ipCode, int iLen)
{
	ASSERT(mpContent == NULL);
    mpContent = new char[iLen+1];
	for(int i=0; i<iLen; i++)
	{
		mpContent[i] = *ipCode++;
	}
	mpContent[iLen] = '\0';
}

void CToken::CopyContent(LPCTSTR ipCode, int iLen, LPCTSTR iHead)
{
	ASSERT(mpContent == NULL);
	
	int headLen = 0;
	if(iHead != NULL) headLen = strlen(iHead);
    mpContent = new char[iLen+headLen+1];

	int i=0;
	for(; i<headLen; i++)
	{
		mpContent[i] = *iHead++;
	}

	for(; i<iLen+headLen; i++)
	{
		mpContent[i] = *ipCode++;
	}

	mpContent[iLen+headLen] = '\0';
}

void CToken::UpdateContent(LPCTSTR iNewContent)
{
	delete [] mpContent;
	mpContent = NULL;
	CopyContent(iNewContent, strlen(iNewContent));
}

void CToken::CatContent(LPCTSTR iCat)
{
	int len = 0;
	if(iCat == NULL) 
		return;
	else
		len +=  strlen(iCat);
	if(mpContent != NULL) len += strlen(mpContent);
	
	char* pNew = new char[len+1];
	::memset(pNew, 0, len+1);
	if(mpContent != NULL) 
		strcpy(pNew, mpContent);
	if(iCat != NULL)
		strcat(pNew, iCat);

	delete [] mpContent;
	mpContent = pNew;
}

void CToken::CleanupComment(CString& oComment)
{
	ASSERT(IsType(TT_SINCOMMENT) || IsType(TT_MULCOMMENT));
	ASSERT(mpContent != NULL);      

	int len = strlen(mpContent);
	if(IsType(TT_SINCOMMENT)) //忽略最前面的//及连行符\\\n
	{
        char ch = 0;
		char pre = 0;
		BOOL readed = FALSE; //是否读到有效内容
	    for(int i=2; i<len; i++)
		{
			pre = ch;
			ch = mpContent[i];
			if(pre == '\\' && ch == '\n' && !oComment.IsEmpty())
			{
				oComment.Delete(oComment.GetLength()-1);
			}
            else
			{
				if(!readed && ch != '/')
					readed = TRUE;
				if(readed)
				    oComment += ch;
			}
		}
	}
	else if(IsType(TT_MULCOMMENT)) //忽略前后的/**/及多余的**
	{      
        char ch = 0;
		char pre = 0;
		BOOL lineHead = TRUE; //位于一行的开头
	    for(int i=2; i<len-2; i++)
		{
			pre = ch;
			ch = mpContent[i];

			if(lineHead) //忽略每一行前面的空格及*
			{
				while(ch == '*' || ch == ' ' || ch == '\t' || 
					ch == '\r' || ch == '\n' || ch == '/')
				{
					pre = ch;
					ch = mpContent[++i];
				}

				lineHead = FALSE;
			}
			else if(ch == '\n') //换行
			{
				lineHead = TRUE;
				//输出结果中是否换行，如果一行以句号结束则加换行符，否则不加
				oComment.TrimRight();
				if(oComment.GetLength() >2 && 
					(oComment.Right(2) == "。" || oComment.Right(1) == "."))
					oComment += "<br>";
				continue;
			}

			oComment += ch;
		}
	}
}