appemail.cpp

linux 上http email 协议分析程序 主要能够处理大数据量的主干网的应用

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <netinet/in.h>
#include <arpa/inet.h>

#include <Config.h>
#include <BaseConst.h>
#include <UTF8Util.h>

#include "AppEmail.h"
#include "Basic.h"

/************************************************************
 * Function:					run
 * Author:                      wzy
 * Version:                             V01.00.000
 * Date:                2003.03.01
 * Description:                
 *                                             
 *                       
 ************************************************************/
void CAppEmail::run()
{
	//printf("to getstream \n");
	m_theEmail->clear();

	chkDate();

	#ifdef FROMFILE
		m_ilenrcv=readFile(m_szrcv);
	#else
		m_ilenrcv=m_pshm->read(m_szrcv);
		
	#endif
	//m_ilenrcv=0;
	if(m_ilenrcv<sizeof(m_theHead))	
	{
		
		sleep(1);
		m_iidletime++;
		if(m_iidletime>=60)
		{
			system("kill $(ps -ea | grep cbtcphash | awk '{print $1}')");
			TLog::toErr("%s:%s",TUTime::YYYYMMDDHHMMSS(),"kill-tcp");
			m_iidletime=0;
		}
		//printf("no email data\n");
		return;
	}
	m_iidletime=0;
	
	//if(m_ilenrcv>sizeof(m_theHead))	
	//		traceFile(m_szrcv, m_ilenrcv,"../emaillog");
	
	#ifdef _TRACE
		printf("recv from shm data len=[%d] \n",m_ilenrcv);
		if(m_ilenrcv>sizeof(m_theHead))	
			traceFile(m_szrcv, m_ilenrcv,"../emaillog");
	#else
		
	#endif 
	
	m_bret=true;
	//if(m_ilenrcv>sizeof(m_theHead))	
	//		saveFile(m_szrcv, m_ilenrcv,"../emaillog");	
	if(m_ilenrcv>sizeof(m_theHead))
	{
		memcpy(&m_theHead,m_szrcv,sizeof(m_theHead));
		logHead(&m_theHead);
		
		if(isspmon(&m_theHead))
			m_bsp=true;
		else
			m_bsp=false;
		if(m_bsp)
		{
			TLog::toLog("this is spemail\n");
			//traceFile(m_szrcv, m_ilenrcv,"../emaillog");
		}
		int isp=ntohs(m_theHead.src_port);
		int idp=ntohs(m_theHead.dst_port);
	
		m_theStream->init(m_ilenrcv,m_szrcv+sizeof(m_theHead));
		if(idp==25)
		{
			//目标端口是SMTP，用户发送命令，邮件报文
			printf("-smtpi-\n");
			m_clientip=m_theHead.src_ip;
			m_bret=doSmtpInq();
		}	
		
		else if(isp==25)
		{	
			//源端口是SMTP，邮件服务器返回状态
			printf("-smtpr-\n");
			m_clientip=m_theHead.dst_ip;
			m_bret=doSmtpRsp();
		}
		else if(isp==110)
		{
			//源端口是POP3，邮件服务器返回邮件报文
			printf("-pop3r-\n");
			m_clientip=m_theHead.dst_ip;
			m_bret=doPop3Rsp();
		}
		else if(idp==110)
		{
			//目标端口是POP3，用户发送认证信息
			printf("-pop3i-\n");
			m_clientip=m_theHead.src_ip;
			m_bret=doPop3Inq();
		}
		if(m_bret==false)
			traceFile(m_szrcv, m_ilenrcv,"../emaillog");

		//printf("end of processing protocol\n");
	}
		
}

void CAppEmail::chkDate()
{
	if(m_pTblEmail->chkDate())
		commitFile();

}

//可以根据报文大小来判断是不是邮件
//与服务器间的命令不用解释
/************************************************************
 * Function:                    doSmtpInq
 * Author:                       wzy
 * Version:                             V01.00.000
 * Date:                2004.03.01
 * Description:       目标端口是SMTP，用户发送命令，邮件报文         
 *                                             
 *                       
 ************************************************************/
bool CAppEmail::doSmtpInq()
{
	//如果一个连接中发多个邮件，要加一个循环
	
	//char *pend=m_theStream->getEndPos();
	
	while(1)
	{
		char szbuf[1024];
		if(m_theStream->getLine(szbuf)>0)
		{
			if(!strcasecmp(szbuf,"DATA"))
				break;
		}
		else
			return true;
	}
	
	m_pbegin_1mail=m_theStream->getCurPos();
	
	m_pend_1mail=m_theStream->getEndPos();
	
	m_theEmail->clearField();	
	if(!m_theEmail->profHeader(m_theStream))
		return false;
			
	if(m_theEmail->availField()<=0)
		return true;

	
	getAddInfo();
	
	if(isSpAdd())
	{
		m_bsp=true;
	
		TLog::toLog("this is spemail spadd\n");
	}			
	saveFileInfo();	
	if(m_bsp)
		savespFileInfo();
	if(m_bsp)
			LogMail(m_pbegin_1mail,m_pend_1mail);
	if(m_ptheFile->appFileBuffer(m_pbegin_1mail,m_pend_1mail-m_pbegin_1mail,m_pSeqEmail->getfkS_TR_T_MAILFILE())>EMAILFILE_COMPRESS_SIZE)
	{
		commitFile();
	}	
	
	//char *pv=m_theEmail->getHv("From");
	
	//saveNA(PT_SMTP,NA_SNDMAIL,pv,m_bsp);
	m_pSeqEmail->getS_TR_T_MAILINFO();
	if(m_bsp)
		m_pSeqEmail->getS_TR_T_SPEMAILINFO();
	
	saveEmailInfo(SMTP);
	if(m_bsp)
		savespEmailInfo(SMTP);
	saveAddInfo();
	if(m_bsp)
	{
		savespAddInfo();
		saveObject2Info();
	}
	
	
	#ifdef DEBUG
	int id=0;
	#endif
	while(1)
	{
		
		#ifdef DEBUG
			printf("-----------no%d part-email --------------\n",id++);
		#endif
		char szbd[128];
		m_pbegin_1part=m_theStream->getCurPos();
		if(!m_theEmail->getStackBoudary(szbd))
			strcpy(szbd,END_BOUNDARY);

		m_pend_1part=m_theStream->gotoMarker(szbd);
		
		if(m_pend_1part==NULL)
		{
			return true;
		}
		int len=m_pend_1part-m_pbegin_1part-2;
		if(len>0)
		{
			memset(m_szbuf,0,sizeof(m_szbuf));
			switch(m_theEmail->getEncodingType())
			{
			case encode_base64:
				
				m_lendata=m_pBase64->Decode(m_pbegin_1part,len,m_szbuf,m_lenbuf);
				break;

			case encode_qp:
				
				m_lendata=m_pQprint->Decode(m_pbegin_1part,len,m_szbuf,m_lenbuf);
				break;

			case encode_7bit:
			case encode_8bit:
			case encode_binary:
			case encode_unknown:
			default:
				if(len<m_lenbuf)
					m_lendata=len;
				else
					m_lendata=m_lenbuf;
				
				memcpy(m_szbuf,m_pbegin_1part,m_lendata);
				break;
			}
			checkKey();
			saveEmailText();	
			if(m_bsp)
				savespEmailText();		
		}
		
		int iret=m_theEmail->profBoundary(m_theStream);
		if(iret<0)
			return false;
		else if(iret==0)
			return true;

		m_theEmail->clearField();
			
		if(!m_theEmail->profHeader(m_theStream))
			return false;
			
		//getchar();
	}
	return true;	
}


//SMTP 服务器返回状态码信息
bool CAppEmail::doSmtpRsp()
{
	return true;
}


//用户的账户/密码资料
bool CAppEmail::doPop3Inq()
{
	char szuser[128];
	char szpass[128];
	while(1)
	{
		char szbuf[1024];
		char szatom[1024];
		char *pval;
		if(m_theStream->getLine(szbuf)<=0)
			return true;
		
		pval=szbuf;
		if( Cval::getNextAtom(pval,szatom))
		{
			if(!strcasecmp(szatom,"USER"))
			{
				if(Cval::getNextAtom(pval,szatom))
				{
					#ifdef DEBUG
						printf("USER NAME=[%s]\n",szatom);
					#endif
					if(strlen(szatom)>=128)
						return true;
					strcpy(szuser,szatom);
						break;
				}
			}
		}
	}
	
	while(1)
	{
		char szbuf[1024];
		char szatom[1024];
		char *pval;
		if(m_theStream->getLine(szbuf)<=0)
			return true;
		
		pval=szbuf;
		if( Cval::getNextAtom(pval,szatom))
		{
			if(!strcasecmp(szatom,"PASS"))
			{
				if(Cval::getNextAtom(pval,szatom))
				{
					#ifdef DEBUG
						printf("USER PASS=[%s]\n",szatom);
					#endif
					if(strlen(szatom)>=128)
						return true;
					strcpy(szpass,szatom);
						break;
					
				}
			}
		}
	}
	
	//saveNA(PT_POP3,NA_LOGIN,szuser,m_bsp);

	STRU_TR_T_MAILACCOUNT mailAccount;
	mailAccount.MAC_ID=m_pSeqEmail->getS_TR_T_MAILACCOUNT();
	strcpy(mailAccount.MAC_USERID,szuser);
	strcpy(mailAccount.MAC_PASSWORD,szpass);

	struct in_addr inaddr;
		
	inaddr.s_addr = m_theHead.dst_ip;
	sprintf(mailAccount.MAC_SERVERIP,"%s",inet_ntoa(inaddr));
		
	strcpy(mailAccount.MAC_DOMAINNAME,"-");
	mailAccount.MAC_SERVICE=POP3;
	strcpy(mailAccount.MAC_CAPTIME,TUTime::YYYYMMDDHHMMSS());

	int iret;
	iret=m_pTblEmail->insRecord(&mailAccount);
	
		
	#ifdef DEBUG
		printf("USER:(%s) PASS=(%s)\n",szuser,szpass);
	#endif
	
	return true;
}

//
//POP3服务器返回邮件体
bool CAppEmail::doPop3Rsp()
{
	#ifdef DEBUG
		int imailid=0;
	#endif
	
	m_pbegin_1mail=m_theStream->getCurPos();

	m_theEmail->clearField();		
	
	if(!m_theEmail->profHeader(m_theStream))
	{
		#ifdef DEBUG
			printf("to profHeader break\n");
		#endif
		return true;
	}
	
	if(m_theEmail->availField()<=0)
		return true;
	
	getAddInfo();
	
	if(isSpAdd())
	{
		m_bsp=true;
	
		TLog::toLog("this is spemail spadd\n");
	}			
	saveFileInfo();
	if(m_bsp)
		savespFileInfo();
	//char *pv=m_theEmail->getHv("From");
	//saveNA(PT_POP3,NA_RCVMAIL,pv,m_bsp);
		
	m_pSeqEmail->getS_TR_T_MAILINFO();	
	if(m_bsp)
		m_pSeqEmail->getS_TR_T_SPEMAILINFO();

	reverseHead(&m_theHead);
	saveEmailInfo(POP3);
	if(m_bsp)
		savespEmailInfo(POP3);
	reverseHead(&m_theHead);
	saveAddInfo();
	if(m_bsp)
	{
		savespAddInfo();
		saveObject2Info();
	}

	
	while(1)
	{
		#ifdef DEBUG
			printf("--------------------------------------------------------------------\n");
			printf("to prcess no(%d) mail\n",imailid++);
			printf("--------------------------------------------------------------------\n");
		#endif
							
		char szbd[128];
		//处理段落的头部
		/*if(!m_theEmail->profHeader(m_theStream))
		{
			#ifdef DEBUG
				printf("to prof email partHeader break\n");
			#endif
			break;
		}*/
		m_pbegin_1part=m_theStream->getCurPos();
		if(!m_theEmail->getStackBoudary(szbd))
		{
			printf("szbd is end boundary\n");
			strcpy(szbd,END_BOUNDARY);
		}
		printf("boundary=[%s]\n",szbd);
		m_pend_1part=m_theStream->gotoMarker(szbd);
		if(m_pend_1part==NULL )
		{
			if(strcmp(szbd,END_BOUNDARY))
				return false;
			else
				return true;
		}
		int len=m_pend_1part-m_pbegin_1part-2;
		if(len>0)
		{
			memset(m_szbuf,0,sizeof(m_szbuf));
			switch(m_theEmail->getEncodingType())
			{
			case encode_base64:
				m_lendata=m_pBase64->Decode(m_pbegin_1part,len,m_szbuf,m_lenbuf);
				break;

			case encode_qp:
				m_lendata=m_pQprint->Decode(m_pbegin_1part,len,m_szbuf,m_lenbuf);
				break;

			case encode_7bit:
			case encode_8bit:
			case encode_binary:
			case encode_unknown:
				if(len<m_lenbuf)
					m_lendata=len;
				else
					m_lendata=m_lenbuf;
				memcpy(m_szbuf,m_pbegin_1part,m_lendata);
				break;
			}
			checkKey();
			saveEmailText();
			if(m_bsp)
				savespEmailText();	
		}
		
		int id=m_theEmail->profBoundary(m_theStream);
		if(id<0)
			return false;
		if(id!=0)
		{
			if(!m_theEmail->profHeader(m_theStream))
			{
				#ifdef DEBUG
					printf("to profpartHeader break\n");
				#endif
				return false;
			}
		}
		else //if(id==0)
		{
			m_pend_1mail=m_theStream->getCurPos();
			
			if(m_bsp)
				LogMail(m_pbegin_1mail,m_pend_1mail);
			
			if(m_ptheFile->appFileBuffer(m_pbegin_1mail,m_pend_1mail-m_pbegin_1mail,m_pSeqEmail->getfkS_TR_T_MAILFILE())>EMAILFILE_COMPRESS_SIZE)
			{
				commitFile();
			}
			
			
		
			m_theStream->gotoNextLine();
			m_pbegin_1mail=m_theStream->getCurPos();
			m_theEmail->clearField();
			if(!m_theEmail->profHeader(m_theStream))
			{
				#ifdef DEBUG
					printf("to profHeader break\n");
				#endif
				return false;
			}
			
		
			#ifdef DEBUG
				printf("end of one email\n");
			#endif
			if(m_theEmail->availField()>0)
			{
				getAddInfo();
				if(isSpAdd())
					m_bsp=true;
				else
					m_bsp=false;
								
				saveFileInfo();
				if(m_bsp)
					savespFileInfo();
					
				m_pSeqEmail->getS_TR_T_MAILINFO();
				if(m_bsp)
					m_pSeqEmail->getS_TR_T_SPEMAILINFO();
				
				reverseHead(&m_theHead);
				saveEmailInfo(POP3);
				if(m_bsp)
					savespEmailInfo(POP3);
				reverseHead(&m_theHead);
				
				saveAddInfo();
				if(m_bsp)
				{
					savespAddInfo();
					saveObject2Info();
				}
				#ifdef DEBUG
				imailid=0;
				#endif
			}
		
		}
	
	}
	return true;
}

void CAppEmail::LogMail(char *m_pbegin_1mail,char *m_pend_1mail)
{
	char szbuf[31];
	int len;
	memcpy(szbuf,m_pbegin_1mail,30);
	szbuf[30]=0;
	TLog::toLog("begin_mail=(%s)",szbuf);
	len=m_pend_1mail-m_pbegin_1mail;
	TLog::toLog("len=(%d)",len);

	memcpy(szbuf,m_pend_1mail-30,30);
	szbuf[30]=0;
	TLog::toLog("end_mail=(%s)",szbuf);

}
	

CAppEmail::CAppEmail(char *filename)
{
	m_iidletime=0;
	m_ipk_mailfiles=0;
	m_ipk_spmailfiles=0;	
	
	m_lenbuf=1024*1024*32;
	int ishmid,ishmsize;
	char buf[128];
	char buf1[128];

	CConfig cfg;
	if(filename==NULL)
	{
		if (cfg.ReadConfig("../etc/config.ini") != 0)
		{
			if (cfg.ReadConfig("config.ini") != 0)
			{
				printf("Read config.ini  Failed!\n");
				exit(-1);
			//return -1;
			}
		}
	}
	else
	{
		if (cfg.ReadConfig(filename) != 0)
		{
			printf("Read %s Failed!\n",filename);
			exit(-1);
		//return -1;
		}

	}

	if(cfg.GetItemValue("DB", "name", buf)!=0)
	{
		printf("Read DB:id Failed!\n");
		exit(-1);
	}

	if(cfg.GetItemValue("DB", "passwd", buf1)!=0)
	{
		printf("Read DB:id Failed!\n");
		exit(-1);
	}
	//int isizebuf=10*1024*1024;
	
	COra *db=new COra(buf,buf1);

	
	if(cfg.GetItemValue("DYNIPOFFTIME", "shorttime", buf)!=0)
	{
		printf("Read DYNIPOFFTIME:shorttime Failed!\n");
		exit(-1);
	}
	m_icleartime=atoi(buf);
	
	if(cfg.GetItemValue("COMBINESHM", "key_email", buf)!=0)
	{
		printf("Read SHM:id Failed!\n");
		exit(-1);
	}
	ishmid=atoi(buf);


	if(cfg.GetItemValue("COMBINESHM", "size_email", buf)!=0)
	{
		printf("Read SHM:size Failed!\n");
		exit(-1);
	}
	ishmsize=atoi(buf)*1024*1024;
	
	
	m_pshm=new CShmApply();
	m_pshm->init(ishmid,ishmsize, SHM_APPPERM);