apphttp.cpp

linux 上http email 协议分析程序 主要能够处理大数据量的主干网的应用

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <BaseConst.h>

#include <Config.h>
#include <UTF8Util.h>
#include "AppHttp.h"
#include "Basic.h"
#include "DataInfo.h"
#include "Consts.h"

#define SIMULATE_SPE
/************************************************************
 * Function:					run
 * Author:                      wzy
 * Version:                             V01.00.000
 * Date:                2004.03.01
 * Description:                
 * Date:				2005.03.28                                            
 * Description:
 *						只处理特控//
 ************************************************************/
void CAppHttp::setNullInq() 
{
 		strcpy(m_szmethod,"GET");
		
		strcpy(m_szdir,"/html/lostinqhtml/");
		strcpy(m_szhost,"/html/lostinqhtml/");
		strcpy(m_szurl,"/html/lostinqhtml/");
		
		*m_szpostpara=0;
		*m_pszpostdata=0;
		m_ilenpostdata=0;
		
		struct in_addr inaddr;
		
		char filename[128];
		inaddr.s_addr = m_theHead.src_ip;
		sprintf(filename,"%s::%d--",inet_ntoa(inaddr),ntohs(m_theHead.src_port));
	
		inaddr.s_addr = m_theHead.dst_ip;
		sprintf(filename+strlen(filename),"%s::%d--",inet_ntoa(inaddr),ntohs(m_theHead.dst_port));
	
		strcat(filename,".");
		strcat(filename,TUTime::YYYYMMDDHHMMSS());
	
		strcat(m_szdir,filename);
		strcat(m_szhost,filename);
		strcat(m_szurl,filename);
		m_idataflow=3;
}

void CAppHttp::setNullRet() 
{
 		m_pszretData=NULL;
		m_iretDataLen=0;
		m_idataflow=2;
}

void CAppHttp::timeOut() 
{
	
	int itot=m_theHash->clear(m_punit,16*1024);
	//printf();
	if(itot>0)
	{
		struct ST_TCP_STREAM theHeadtmp;
		TUnit_v * punit;
		punit=m_punit;
		
		//memcpy((void*)&theHeadtmp,(void*)&m_theHead,sizeof(m_theHead));
		
		TLog::toLog("timeOut tot=[%d]\n",itot);
		for(int i=0;i<itot;i++)
		{
			m_theHead.protocol=punit->key.protocol;
			m_theHead.src_ip=punit->key.src_ip;
			m_theHead.src_port=punit->key.src_port;
			m_theHead.dst_ip=punit->key.dst_ip;
			m_theHead.dst_port=punit->key.dst_port;
			
			if(ntohs(m_theHead.src_port)==80)
			{//源端口是HTTP，WEB服务器返回网页	
				reverseHead(&m_theHead);
				m_iretlen=punit->len;
				memcpy(m_pszret,punit->pval,m_iretlen	);
				m_theretStream->init(m_iretlen,m_pszret);
				initDir();
				while(!m_theretStream->endofStream())
				{
					setNullInq();
					if(!profoneRet())
						break;
					TLog::toLog("onlyret");
					if(isspmon(&m_theHead))
					{
						registerspPage(m_szdir);
						registerspWatchedObj();
					}
					else if(posUrl() )
					{
						registerspPage(m_szdir);
						registerWatchedObj();
					}
				}	
					
			}
			else
			{
				m_iinqlen=punit->len;
				memcpy(m_pszinq,punit->pval,m_iinqlen	);
				m_theinqStream->init(m_iinqlen,m_pszinq);
				
				initDir();
				while(!m_theinqStream->endofStream())
				{
					if(!setinqHead())
						break;
					if(!profoneInqData())
						break;
						
					setNullRet();
					TLog::toLog("onlyinq");
					if(isspmon(&m_theHead))
					{
						registerspPage(m_szdir);
						registerspWatchedObj();
						registerspContent();
					}
					else if(posUrl() )
					{
						registerPage();
						registerWatchedObj();
						registerContent();
					}
				}	
			}
										
			m_theMem->free((char *)(punit->pval));
			punit++;
		}
		//memcpy((void*)&m_theHead,(void*)&theHeadtmp,sizeof(m_theHead));
	}
	
} 
 
void CAppHttp::run()
{
	//m_theHash->clear(m_theMem);
	
	//m_theHash->clear(m_punit,);
	
	#ifdef FROMFILE
		m_ilenrcv=readFile(m_szrcv);
	#else
		m_ilenrcv=m_pshm->read(m_szrcv);
	#endif
	if(m_ilenrcv<sizeof(m_theHead))
	{
		sleep(1);
		m_iidletime++;
		if(m_iidletime>=60)
		{
			//system("kill $(ps -ea | grep cbtcphash | awk '{print $1}')");
			TLog::toErr("%s:%s",TUTime::YYYYMMDDHHMMSS(),"kill-tcp");
			m_iidletime=0;
		}
		printf("no http data\n");
		return;
	}
	m_iidletime=0;
	logHead(&m_theHead);
	//printf("recv from shm data len=[%d] \n",m_ilenrcv);
	#ifdef _TRACE
		//printf("recv from shm data len=[%d] \n",m_ilenrcv);
		if(m_ilenrcv>sizeof(m_theHead))	
		{
			traceFile(m_szrcv, m_ilenrcv,"../tracelog");
			
		}
	#else
		if(m_ilenrcv>sizeof(m_theHead))	
			saveFile(m_szrcv, m_ilenrcv,"../tracelog");
	#endif 

	memcpy(&m_theHead,m_szrcv,sizeof(m_theHead));
	//#ifdef _LOGHEAD
	
	//#endif
	if(ntohs(m_theHead.src_port)==80)
	{//源端口是HTTP，WEB服务器返回网页	
		printf("------------------received http return ----------------\n");
		{
			reverseHead(&m_theHead);
		
			m_iretlen=m_ilenrcv-sizeof(m_theHead);
			m_pszret=m_szrcv+sizeof(m_theHead);

			m_pszinq=m_szsaved+sizeof(m_theHead);
			m_iinqlen=findHtml(&m_theHead,m_pszinq);
			
			if(m_iinqlen>0)
			{
				#ifdef _TRACEMATCH
				
					printf("trace file.........\n");
					traceFile(m_pszret-sizeof(m_theHead),m_iretlen+sizeof(m_theHead),"../httplog");
					
					memcpy((m_pszinq-sizeof(m_theHead)),(char *)(void *)&m_theHead,sizeof(m_theHead));
					
					traceFile(m_pszinq-sizeof(m_theHead),m_iinqlen+sizeof(m_theHead),"../httplog");
				#else
					memcpy((m_pszinq-sizeof(m_theHead)),(char *)(void *)&m_theHead,sizeof(m_theHead));
				#endif 
				
				profHtml();
			}
			else
			{
				saveHtml(&m_theHead,m_pszret,m_iretlen);
			}
		}		
	}
	else if(ntohs(m_theHead.dst_port)==80)
	{//目标端口是HTTP，用户请求网页
		printf("------------------received http inq ----------------\n");
		m_iinqlen=m_ilenrcv-sizeof(m_theHead);
		m_pszinq=m_szrcv+sizeof(m_theHead);
		m_theinqStream->init(m_iinqlen,m_pszinq);
		
		/*if(!isspmon(&m_theHead))
		{
			if(setinqHead())
				if(m_phpadd->searchAdd(m_szurl)>=0)
					saveNA(PT_HTTP,NA_OTHER,m_szurl);
		}
		*/
		
		
		//if(isspmon(&m_theHead))
		{
			m_iinqlen=m_ilenrcv-sizeof(m_theHead);
			m_pszinq=m_szrcv+sizeof(m_theHead);
		
			m_pszret=m_szsaved+sizeof(m_theHead);
			m_iretlen=findHtml(&m_theHead,m_pszret);

			if(m_iretlen>0)
			{
				#ifdef _TRACEMATCH
			
					traceFile(m_pszinq-sizeof(m_theHead),m_iinqlen+sizeof(m_theHead),"../httplog");
				
					reverseHead(&m_theHead);
					memcpy((m_pszret-sizeof(m_theHead)),(char *)(void *)&m_theHead,sizeof(m_theHead));
					traceFile(m_pszret-sizeof(m_theHead),m_iretlen+sizeof(m_theHead),"../httplog");
					reverseHead(&m_theHead);
				#else
					reverseHead(&m_theHead);
					memcpy((m_pszret-sizeof(m_theHead)),(char *)(void *)&m_theHead,sizeof(m_theHead));
					reverseHead(&m_theHead);
				#endif 

				profHtml();
			}
			else
			{
				printf("save inq html\n");
				saveHtml(&m_theHead,m_pszinq,m_iinqlen);
			}
		}
	}
	else if(ntohs(m_theHead.dst_port)==443)
	{//https
		https();
	}
	
}

void CAppHttp::https()
{
	struct in_addr inaddr;
	STRU_TR_T_SSLWEBPAGE rec;
	
	rec.SWP_ID=m_pseqssl->getS_TR_T_SSLWEBPAGE();
	inaddr.s_addr = m_theHead.src_ip;
	sprintf(rec.SWP_CLIENTIP,"%s",inet_ntoa(inaddr));
	inaddr.s_addr = m_theHead.dst_ip;
	sprintf(rec.SWP_SERVERIP,"%s",inet_ntoa(inaddr));
	strcpy(rec.SWP_CAPTIME,TUTime::YYYYMMDDHHMMSS());
	
	m_pInfo->getAccount(m_theHead.src_ip, rec.SWP_CLNTACCOUNT,rec.SWP_CLNTCALLID);
	
	m_pInfo->getLocation(m_theHead.src_ip, rec.SWP_CLNTLOCATION);
	TLog::toErr("get https\n");	
	m_ptblssl->insRecord(&rec);
}

//查询匹配的HTML
int CAppHttp::findHtml(struct ST_TCP_STREAM *phead,char * _szhtml)
{
	TUnit_v aunit; 
	
	aunit.key.protocol=phead->protocol;
	aunit.key.src_ip=phead->src_ip;
	aunit.key.src_port=phead->src_port;
	aunit.key.dst_ip=phead->dst_ip;
	aunit.key.dst_port=phead->dst_port;
			
	if(!m_theHash->find(&aunit))
	{
		return -1;
	}
	else
	{
		memcpy(_szhtml,aunit.pval,aunit.len);
		m_theMem->free((char *)aunit.pval);
		m_theHash->remove(&aunit);
		return aunit.len;
	}

}

bool CAppHttp::saveHtml(struct ST_TCP_STREAM *phead,char *_szhtml,int _ihtmllen)
{
	TUnit_v aunit; 
	aunit.key.protocol=phead->protocol;
	aunit.key.src_ip=phead->src_ip;
	aunit.key.src_port=phead->src_port;
	aunit.key.dst_ip=phead->dst_ip;
	aunit.key.dst_port=phead->dst_port;
			
	char *pbuffer=m_theMem->maloc(_ihtmllen);

	if(pbuffer==NULL)
	{
		TLog::toErr("alloc httpmem notenough [%d]",_ihtmllen);
		exit(0);
		return false;
	}
	memcpy(pbuffer,_szhtml,_ihtmllen);
	
	aunit.len=_ihtmllen;
	aunit.pval=pbuffer;
	aunit.pnext=NULL;

	if(!m_theHash->add(&aunit))
	{
		m_theMem->free(pbuffer);
		TLog::toErr("alloc httpmemhead notenough");
		exit(0);
		return false;
	}
	
	return true;
	
}

void CAppHttp::profHtml()
{
	printf("to exec profhtml\n");
	
	m_ptblweb->chkDate();

	m_theinqStream->init(m_iinqlen,m_pszinq);

	m_theretStream->init(m_iretlen,m_pszret);

	initDir();
	while(!m_theinqStream->endofStream())
	{
		
		if(!setinqHead())
			return;
		if(!profoneInqData())
			return;
		
		if(!profoneRet())
			return;
		
								
		if(availContType())
		{
			#ifdef SIMULATE_SPE
			if(	!isspmon(&m_theHead))
			#else

			if(	isspmon(&m_theHead))
			#endif
			{
				registerspPage(m_szdir);
				m_theHtmlFile->writeFile(m_szdir,m_pszretData,m_iretDataLen);	
				checkspKey();
				registerspWatchedObj();
				registerspContent();
			}
			else if(posUrl())
			{
				registerPage();
				m_theHtmlFile->writeFile(m_szdir,m_pszretData,m_iretDataLen);	
				checkKey();
				registerWatchedObj();
				registerContent();
			}
		}
	}	
}


/*
	这里告警现在都假设是特控
	否则，应该根据特泛控来决定是否arec.ALE_INFOID=
*/
void CAppHttp::checkspKey()
{
	
	char *ptr;
	char szbuf[128];

	if( m_pszretData==NULL|| m_iretDataLen==0)
		return;
	ptr=m_prdsip->searchKeyword(2, m_pszretData,m_iretDataLen,szbuf); 
	
	if(ptr!=NULL)	
	{
		STRU_TR_T_ALARMEVENT arec;
		TLog::toLog("(searchedkey ok[%s])\n",szbuf );
		
		arec.ALE_ID=m_pSeqEvent->getS_TR_T_ALARMEVENT(); 
		arec.AMG_ID=0;
		arec.ALE_ALARMTYPE =2;
		arec.ALE_ALARMSTATUS =1 ;
		arec.ALE_INFOTYPE=0;   
    	arec.ALE_INFOID= m_pspseqweb->getfkS_TR_T_SPESENPAGE();
		memset(arec.ALE_ALARMCONTENT,0,sizeof(arec.ALE_ALARMCONTENT));
		//strncpy(arec.ALE_ALARMCONTENT,ptr,sizeof(arec.ALE_ALARMCONTENT)-1);
		strcpy(arec.ALE_ALARMCONTENT,szbuf);

		strcpy(arec.ALE_ALARMTIME,TUTime::YYYYMMDDHHMMSS());
	  
		strcpy(arec.ALE_ALARMTITLE ,"HTTP KEY");
		arec.ALE_FORWARDSTATUS=0;
	
		m_pEvent->insRecord(&arec);
	}
}		

void CAppHttp::checkKey()
{
	
	char *ptr;
	char szbuf[128];

	if( m_pszretData==NULL|| m_iretDataLen==0)
		return;
	ptr=m_prdsip->searchKeyword(2, m_pszretData,m_iretDataLen,szbuf); 
	
	if(ptr!=NULL)	
	{
		STRU_TR_T_ALARMEVENT arec;
		TLog::toLog("(searchedkey ok[%s])\n",szbuf );
		
		arec.ALE_ID=m_pSeqEvent->getS_TR_T_ALARMEVENT(); 
		arec.AMG_ID=0;
		arec.ALE_ALARMTYPE =2;
		arec.ALE_ALARMSTATUS =1 ;
		arec.ALE_INFOTYPE=0;   
    	arec.ALE_INFOID= m_pseqweb->getfkS_TR_T_SENPAGE();
		memset(arec.ALE_ALARMCONTENT,0,sizeof(arec.ALE_ALARMCONTENT));
		//strncpy(arec.ALE_ALARMCONTENT,ptr,sizeof(arec.ALE_ALARMCONTENT)-1);
		strcpy(arec.ALE_ALARMCONTENT,szbuf);

		strcpy(arec.ALE_ALARMTIME,TUTime::YYYYMMDDHHMMSS());
	  
		strcpy(arec.ALE_ALARMTITLE ,"HTTP KEY");
		arec.ALE_FORWARDSTATUS=0;
	
		m_pEvent->insRecord(&arec);
	}
}		



void CAppHttp::setPageRecord(STRU_TR_T_SENPAGE &recpage)
{
	
	struct in_addr inaddr;
	
	recpage.ACT_ID=0;
	recpage.WAP_ID=0;
	recpage.SPG_ID=m_pseqweb->getS_TR_T_SENPAGE();
		
	if(strlen(m_szdir)<sizeof(recpage.SPG_PAGEPOSITION))
		strcpy(recpage.SPG_PAGEPOSITION,m_szdir);
	else
	{
		memcpy(recpage.SPG_PAGEPOSITION,m_szdir,sizeof(recpage.SPG_PAGEPOSITION)-1);
		recpage.SPG_PAGEPOSITION[sizeof(recpage.SPG_PAGEPOSITION)-1]=0;
	}
	if(strlen(m_szurl)<sizeof(recpage.SPG_DOMAINNAME))
		strcpy(recpage.SPG_DOMAINNAME,m_szurl);
	else
	{
		memcpy(recpage.SPG_DOMAINNAME,m_szurl,sizeof(recpage.SPG_DOMAINNAME)-1);
		recpage.SPG_DOMAINNAME[sizeof(recpage.SPG_DOMAINNAME)-1]=0;
	}
	inaddr.s_addr = m_theHead.src_ip;
	sprintf(recpage.SPG_CLIENTIP,"%s",inet_ntoa(inaddr));
	inaddr.s_addr = m_theHead.dst_ip;
	sprintf(recpage.SPG_SERVERIP,"%s",inet_ntoa(inaddr));
	
	strcpy(recpage.SPG_CAPTIME,TUTime::YYYYMMDDHHMMSS());
	
	recpage.SPG_DATAFLOW=m_idataflow;
	/*
	if(strcasecmp(m_szmethod,"POST")==0)
		recpage.SPG_DATAFLOW=0;
	else
		recpage.SPG_DATAFLOW=1;
	*/
	recpage.SPG_RESOURCELENGTH=m_iretDataLen;	
	
	strcpy(recpage.SPG_CLNTCALLID,"-");
	m_pInfo->getAccount(m_theHead.src_ip, recpage.SPG_CLNTACCOUNT,recpage.SPG_CLNTCALLID);
	//TLog::toErr("[%s] act=[%s][%s]\n",recpage.SPG_CLIENTIP, recpage.SPG_CLNTACCOUNT,recpage.SPG_CLNTCALLID);
	m_pInfo->getLocation(m_theHead.src_ip, recpage.SPG_CLNTLOCATION);
		
	recpage.SPG_HASSUBMIT=0;
	recpage.SPG_SPEINFOSTATUS=1;
}

bool CAppHttp::posUrl()
{
	return m_pappweb->setcurUrlID(m_szurl);