template.cpp

概述：数据的纵向收集

#include "stdafx.h"
#include "Template.h"
#include "IniUtil.h"
#include "TemplateEx.h"

//////////////////////////////////////////////////////////////////////////
CTemplate* CTemplate::s_pTemplate = NULL;

//////////////////////////////////////////////////////////////////////////
CTemplate::CTemplate()
:m_nSid(9999),	//默认是9999
m_nConnect(5),	//默认同时对一个站点5个链接
m_nThread(10),	//默认是开10个工作线程
m_nCTimeOut(10),//默认链接一个站点的时间（秒），不然啊就fail
m_nUtf8(0),
m_nLog(0),
m_nNULLInBody(0)
{
}

CTemplate::~CTemplate()
{
	s_pTemplate = NULL;
}

CTemplate* CTemplate::Instance()
{
	if(s_pTemplate==NULL)
	{
		//build & proxy pattern
		s_pTemplate = new CTemplate();
	}
	return s_pTemplate;
}
void CTemplate::Release()
{
	if(s_pTemplate!=NULL)
	{
		delete s_pTemplate;
	}
	s_pTemplate=NULL;
}

//////////////////////////////////////////////////////////////////////////
int CTemplate::GetSid()
{
	return m_nSid;
}

int CTemplate::GetThread()
{
	return m_nThread;
}

int CTemplate::GetConnect()
{
	return m_nConnect;
}

int CTemplate::GetCTimeOut()
{
	return m_nCTimeOut;
}

int CTemplate::GetUtf8()
{
	return m_nUtf8;
}

CString CTemplate::GetUserAgent()
{
	return m_strUserAgent;
}
CString CTemplate::GetCookie()
{
	return m_Cookie;
}

int CTemplate::GetNULLInBody()
{
	return m_nNULLInBody;
}

const std::vector<CString>& CTemplate::GetContentTypeArray()
{
	return m_arrContentType;
}
const std::vector<CSeedItem>& CTemplate::GetSeedArray()
{
	return m_arrSeed;
}
const std::vector<CFilterItem>& CTemplate::GetFilterArray()
{
	return m_arrFilter;
}
const std::vector<CFingerItem>& CTemplate::GetContentFingerArray()
{
	return m_arrFingerFilter;
}
//////////////////////////////////////////////////////////////////////////
void CTemplate::SetPath(const CString& strPath)
{
	m_strPath=strPath;
}

void CTemplate::Parse()
{
	if(m_strPath.Find(DEFAULT_TEMPLATE_FILE_EX)>=0)
	{
		DebugString("[CTemplate::Parse]","template:(%s)",m_strPath);
		CTemplateEx::Load(m_strPath);
	}
	else
	{
		DebugString("[CTemplate::Parse]","error path:(%s)",m_strPath);
	}
}
//////////////////////////////////////////////////////////////////////////
CString CTemplate::GetContentTypeX()
{
	CString strAccept="";
	const std::vector<CString>& arrContentType = CTemplate::Instance()->GetContentTypeArray();
	for(int i=0;i<(int)arrContentType.size();i++)
	{
		strAccept.Append(arrContentType[i]);
		if(i!=arrContentType.size()-1)
			strAccept.Append(", ");
	}
	return strAccept;
}
DWORD CTemplate::GetTaskTypeX(const CString& strUrl)
{
	return GetTaskTypeX(strUrl,strUrl.GetLength());
}

DWORD CTemplate::GetTaskTypeX(LPCSTR lpszUrl,int nLength)
{
	DWORD dwType = 0;

	const std::vector<CFilterItem> & arrFilter = CTemplate::Instance()->GetFilterArray();
	for (int i=0;i < (int)arrFilter.size();i++)
	{
		FILTER_TYPE ftType=arrFilter[i].nType;
		const CString& strPattern =arrFilter[i].strPattern;

		const char* lpszError = NULL;
		int nErrorOffset=0;
		int rc=0;
		int ovector[30]={0};/* should be a multiple of 3 */

		pcre* re = pcre_compile(strPattern,0,&lpszError,&nErrorOffset,NULL);
		if(re)
		{
			rc = pcre_exec(re,NULL,lpszUrl,nLength,0,0,ovector,30);
			if(rc<0)
			{
				switch(rc)
				{
				case PCRE_ERROR_NOMATCH:
					break;
				default:
					break;
				}
			}
			else
			{
				switch(ftType)
				{
				case FT_PARSE:
					dwType|=TT_PARSE;
					break;
				case FT_SKIP:
					dwType|=TT_SKIP;
					break;
				case FT_STORE:
					dwType|=TT_STORE;
					break;
				case FT_REPLACE:
					dwType|=TT_REPLACE;
					break;
				default:
					break;
				}
			}

			if(lpszError)
			{
				pcre_free_substring(lpszError);
			}
			pcre_free(re);
		}
	}

	return dwType;
}