crawler.h

概述：数据的纵向收集

//[e-mail]
#pragma once 

#include <atlstr.h>
#include <map>
#include "iocs.h"
#include "PageStat.h"
#include "HttpCrawler.h"

class CCrawler
{
//给CrawlerMgr调用的接口，负责处理Skip Parse Store三种的任务并且处理网页超级链接分析
public:
	static BOOL HandleTask(CPageStat& objTask);

//CCrawlerMgr停止和释放时候调用
public:
	static void FreeConnectAll();

//curl & hlink的回调函数
public:
	static int OnFindLink(const char *elem, const char *attr, struct uri *uri, void *arg);
	static void DownloadAndRegexPage(char* szURL,char* szCookie,char* szPattern,CString& szResult);
	static void DownloadAndRegexHeader(char* szURL,char* szCookie,char* szPattern,CString& szResult);

private:
	static CIOCriticalSection s_csLinkParse;

//处理函数
protected:
	static void DoTaskStore(const CString& strUrl,const CString& strHeader,const CString& strContent);
	//static void DoTaskStore(const CString& strUrl,const CString& strContent);
	static void DoTaskParse(const CString& strUrl,const CString& strContent);
	static bool CheckContent(CPageStat& objTask,CString& strContent);

//thread<->curl共享，如果出错就重新分配一个
protected:	
	static CHttpCrawler crawler;
};