class_httpretriever.php

软件的主要特点: 1.专门针对有"验证码"要求的论坛. 2.可以完成如下功能: 自动发布新帖,自动回复指定帖,自动注册论坛帐号 (所有操作不能同时针对一个网站,因为有"验证码"要求的网站是每次打开一个

							$time_firstdata = $this->getmicrotime();
							$process_time = $time_firstdata - $time_connected;
							$time_connected = 0;
						}
						$this->received_bytes += strlen($line);
						if ($iterations % 20 == 0) {
							$this->update_transfer_counters();
						}
					}

					
					// some dumbass webservers don't respect Connection: close and just
					// leave the connection open, so we have to be diligent about
					// calculating the content length so we can disconnect at the end of
					// the response
					if ( (!$headers_received) && (trim($line)=="") ) {
						$headers_received = true;
						$this->progress(HRP_DEBUG,"Got headers: {$this->response}");

						if (preg_match('/^Content-Length: ([0-9]+)/im',$this->response,$matches)) {
							$data_length = (int) $matches[1];
							$this->progress(HRP_DEBUG,"Content length is $data_length");
							$this->expected_bytes = $data_length;
							$this->update_transfer_counters();
						} else {
							$this->progress(HRP_DEBUG,"No data length specified");
						}
						if (preg_match("/^Transfer-Encoding: chunked/im",$this->response,$matches)) {
							$chunked = true;
							$this->progress(HRP_DEBUG,"Chunked transfer encoding requested");
						} else {
							$this->progress(HRP_DEBUG,"CTE not requested");
						}
						
						if (preg_match_all("/^Set-Cookie: ((.*?)\=(.*?)(?:;\s*(.*))?)$/im",$this->response,$cookielist,PREG_SET_ORDER)) {
							foreach ($cookielist as $k=>$cookie) $this->cookie_headers .= $cookie[0]."\n";
							
							// get the path for which cookies will be valid if no path is specified
							$cookiepath = preg_replace('/\/{2,}/','',$urldata['path']);
							if (substr($cookiepath,-1)!='/') {
								$cookiepath = explode('/',$cookiepath);
								array_pop($cookiepath);
								$cookiepath = implode('/',$cookiepath) . '/';
							}
							// process each cookie
							foreach ($cookielist as $k=>$cookiedata) {
								list(,$rawcookie,$name,$value,$attributedata) = $cookiedata;
								$attributedata = explode(';',trim($attributedata));
								$attributes = array();

								$cookie = array(
									'value'=>$value,
									'raw'=>trim($rawcookie),
								);
								foreach ($attributedata as $k=>$attribute) {
									list($attrname,$attrvalue) = explode('=',trim($attribute));
									$cookie[$attrname] = $attrvalue;
								}

								if (!isset($cookie['domain']) || !$cookie['domain']) $cookie['domain'] = $urldata['host'];
								if (!isset($cookie['path']) || !$cookie['path']) $cookie['path'] = $cookiepath;
								if (isset($cookie['expires']) && $cookie['expires']) $cookie['expires'] = strtotime($cookie['expires']);
								
								if (!$this->validate_response_cookie($cookie,$urldata['host'])) continue;
								
								// do not store expired cookies; if one exists, unset it
								if ( isset($cookie['expires']) && ($cookie['expires']<time()) ) {
									unset($this->response_cookies[ $name ][ $cookie['path'] ]);
									continue;
								}
								
								$this->response_cookies[ $name ][ $cookie['path'] ] = $cookie;
							}
						}
					}
					
					if ($this->result_close) {
						if (preg_match_all("/HTTP\/([0-9\.]+) ([0-9]+) (.*?)[\r\n]/",$this->response,$matches)) {
							$resultcodes = $matches[2];
							foreach ($resultcodes as $k=>$code) {
								if ($code!=100) {
									$this->progress(HRP_INFO,'HTTP result code received; closing connection');

									$this->result_code = $code;
									$this->result_text = $matches[3][$k];
									fclose($fp);
					
									return ($this->result_code==200);
								}
							}
						}
					}
				}
				if (feof($fp)) $this->progress(HRP_DEBUG,'EOF on socket');
				@fclose ($fp);
				
				$this->update_transfer_counters();
				
				if (is_array($this->response_cookies)) {
					// make sure paths are sorted in the order in which they should be applied
					// when setting response cookies
					foreach ($this->response_cookies as $name=>$paths) {
						ksort($this->response_cookies[$name]);
					}
				}
				$this->progress(HRP_INFO,'Request complete');
			} else {
				$this->error = strtoupper($urldata["scheme"])." connection to ".$hostname." port ".$urldata["port"]." failed";
				$this->progress(HRP_ERROR,$this->error);
				return false;
			}

		// perform an HTTP/HTTPS request using CURL
		} elseif ( !$this->disable_curl && ( ($urldata["scheme"]=="https") || ($this->force_curl) ) ) {
			$this->progress(HRP_INFO,'Passing HTTP request for $url to CURL');
			$curl_mode = true;
			if (!$this->_curl_request($url)) return false;
			
		// unknown protocol
		} else {
			$this->error = "Unsupported protocol: ".$urldata["scheme"];
			$this->progress(HRP_ERROR,$this->error);
			return false;
		}
		
		$this->raw_response = $this->response;

		$totallength = strlen($this->response);
		
		do {
			$headerlength = strpos($this->response,"\r\n\r\n");

			$response_headers = explode("\r\n",substr($this->response,0,$headerlength));
			$http_status = trim(array_shift($response_headers));
			foreach ($response_headers as $line) {
				list($k,$v) = explode(":",$line,2);
				$this->response_headers[trim($k)] = trim($v);
			}
			$this->response = substr($this->response,$headerlength+4);
	
			/* // Handled in-transfer now
			if (($this->response_headers['Transfer-Encoding']=="chunked") && (!$curl_mode)) {
				$this->remove_chunkiness();
			}
			*/
		
			if (!preg_match("/^HTTP\/([0-9\.]+) ([0-9]+) (.*?)$/",$http_status,$matches)) {
				$matches = array("",$this->version,0,"HTTP request error");
			}
			list (,$response_version,$this->result_code,$this->result_text) = $matches;

			// skip HTTP result code 100 (Continue) responses
		} while (($this->result_code==100) && ($headerlength));
		
		// record some statistics, roughly compatible with CURL's curl_getinfo()
		if (!$curl_mode) {
			$total_time = $this->getmicrotime() - $time_request_start;
			$transfer_time = $total_time - $connect_time;
			$this->stats = array(
				"total_time"=>$total_time,
				"connect_time"=>$connect_time,	// time between connection request and connection established
				"process_time"=>$process_time,	// time between HTTP request and first data (non-headers) received
				"url"=>$url,
				"content_type"=>$this->response_headers["Content-Type"],
				"http_code"=>$this->result_code,
				"header_size"=>$headerlength,
				"request_size"=>$totallength,
				"filetime"=>strtotime($this->response_headers["Date"]),
				"pretransfer_time"=>$connect_time,
				"size_download"=>$totallength,
				"speed_download"=>$transfer_time > 0 ? round($totallength / $transfer_time) : 0,
				"download_content_length"=>$totallength,
				"upload_content_length"=>0,
				"starttransfer_time"=>$connect_time,
			);
		}
		
		
		$ok = ($this->result_code==200);
		if ($ok) {
			// if a page preprocessor is defined, call it to process the page contents
			if (is_callable($this->page_preprocessor)) $this->response = call_user_func($this->page_preprocessor,$this,$this->response);
			
			// if caching is enabled, save the page
			if ($this->caching) $this->_cache_store($cachetoken,$url);
		}

		return $ok;
	}
	
	function validate_response_cookie($cookie,$actual_hostname) {
		// make sure the cookie can't be set for a TLD, eg: '.com'		
		$cookiehost = $cookie['domain'];
		$p = strrpos($cookiehost,'.');
		if ($p===false) return false;
		
		$tld = strtolower(substr($cookiehost,$p+1));
		$special_domains = array("com", "edu", "net", "org", "gov", "mil", "int");
		$periods_required = in_array($tld,$special_domains) ? 1 : 2;
		
		$periods = substr_count($cookiehost,'.');
		if ($periods<$periods_required) return false;
		
		if (substr($actual_hostname,0,1)!='.') $actual_hostname = '.'.$actual_hostname;
		if (substr($cookiehost,0,1)!='.') $cookiehost = '.'.$cookiehost;
		$domain_match = (
			($actual_hostname==$cookiehost) ||
			(substr($actual_hostname,-strlen($cookiehost))==$cookiehost)
		);
		
		return $domain_match;

	}
	
	function build_headers() {
		$headers = "";
		foreach ($this->headers as $name=>$value) {
			$value = trim($value);
			if (empty($value)) continue;
			$headers .= "{$name}: {$value}\r\n";
		}

		if (isset($this->request_cookies) && is_array($this->request_cookies)) {
			$cookielist = array();
			foreach ($this->request_cookies as $name=>$value) {
				$cookielist[] = "{$name}={$value}";
			}
			if (count($cookielist)) $headers .= "Cookie: ".implode('; ',$cookielist)."\r\n";
		}
		
		
		$headers .= "\r\n";
		
		return $headers;
	}
	
	// opposite of parse_url()
	function rebuild_url($urlinfo) {
		$url = $urlinfo['scheme'].'://';
		
		if ($urlinfo['user'] || $urlinfo['pass']) {
			$url .= $urlinfo['user'];
			if ($urlinfo['pass']) {
				if ($urlinfo['user']) $url .= ':';
				$url .= $urlinfo['pass'];
			}
			$url .= '@';
		}
		
		$url .= $urlinfo['host'];
		if ($urlinfo['port']) $url .= ':'.$urlinfo['port'];
		
		$url .= $urlinfo['path'];
		
		if ($urlinfo['query']) $url .= '?'.$urlinfo['query'];
		if ($urlinfo['fragment']) $url .= '#'.$urlinfo['fragment'];
		
		return $url;
	}
	
	function _replace_hostname(&$url,$new_hostname) {
		$parts = parse_url($url);
		$old_hostname = $parts['host'];
		
		$parts['host'] = $new_hostname;
		
		$url = $this->rebuild_url($parts);
				
		return $old_hostname;
	}
	
	function _curl_request($url) {
		$this->error = false;

		// if a direct connection IP address was specified,	replace the hostname
		// in the URL with the IP address, and set the Host: header to the
		// original hostname
		if ($this->connect_ip) {
			$old_hostname = $this->_replace_hostname($url,$this->connect_ip);
			$this->headers["Host"] = $old_hostname;
		}
		

		unset($this->headers["Content-Length"]);
		$headers = explode("\n",$this->build_headers());
		
		$ch = curl_init();
		curl_setopt($ch,CURLOPT_URL, $url); 
		curl_setopt($ch,CURLOPT_USERAGENT, $this->headers["User-Agent"]); 
		curl_setopt($ch,CURLOPT_HEADER, 1); 
		curl_setopt($ch,CURLOPT_RETURNTRANSFER, 1); 
//		curl_setopt($ch,CURLOPT_FOLLOWLOCATION, 1); // native method doesn't support this yet, so it's disabled for consistency
		curl_setopt($ch,CURLOPT_TIMEOUT, 10);
		if ($this->curl_proxy) {
			curl_setopt($ch,CURLOPT_PROXY,$this->curl_proxy);
		}
		curl_setopt($ch,CURLOPT_HTTPHEADER, $headers);
		
		if ($this->method=="POST") {
			curl_setopt($ch,CURLOPT_POST,1);
			curl_setopt($ch,CURLOPT_POSTFIELDS,$this->post_data);
		}
		if ($this->insecure_ssl) {
			curl_setopt($ch,CURLOPT_SSL_VERIFYPEER,0);
		}
		if ($this->ignore_ssl_hostname) {
			curl_setopt($ch,CURLOPT_SSL_VERIFYHOST,1);
		}
		
		$this->response = curl_exec ($ch);
		if (curl_errno($ch)!=0) {
			$this->error = "CURL error #".curl_errno($ch).": ".curl_error($ch);
		}
		
		$this->stats = curl_getinfo($ch);
		curl_close($ch);
		
		return ($this->error === false);
	}
	
	function progress($level,$msg) {
		if (is_callable($this->progress_callback)) call_user_func($this->progress_callback,$level,$msg);
	}
	
	// Gets any available HTTPRetriever error message (including both internal
	// errors and HTTP errors)
	function get_error() {
		return $this->error ? $this->error : 'HTTP ' . $this->result_code.': '.$this->result_text;
	}
	
	function get_content_type() {
		if (!$ctype = $this->response_headers['Content-Type']) {
			$ctype = $this->response_headers['Content-type'];
		}
		list($ctype,) = explode(';',$ctype);
		
		return strtolower($ctype);
	}
	
	function update_transfer_counters() {
		if (is_callable($this->transfer_callback)) call_user_func($this->transfer_callback,$this->received_bytes,$this->expected_bytes);
	}

	function set_transfer_display($enabled = true) {
		if ($enabled) {
			$this->transfer_callback = array(&$this,'default_transfer_callback');
		} else {
			unset($this->transfer_callback);
		}
	}
	
	function set_progress_display($enabled = true) {
		if ($enabled) {
			$this->progress_callback = array(&$this,'default_progress_callback');
		} else {
			unset($this->progress_callback);
		}
	}
	
	function default_progress_callback($severity,$message) {
		$severities = array(
			HRP_DEBUG=>'debug',
			HRP_INFO=>'info',
			HRP_ERROR=>'error',
		);
		
		echo date('Y-m-d H:i:sa').' ['.$severities[$severity].'] '.$message."\n";
		flush();
	}

	function default_transfer_callback($transferred,$expected) {
		$msg = "Transferred " . round($transferred/1024,1);
		if ($expected>=0) $msg .= "/" . round($expected/1024,1);
		$msg .=	"KB";
		if ($expected>0) $msg .= " (".round($transferred*100/$expected,1)."%)";
		echo date('Y-m-d H:i:sa')." $msg\n";
		flush();
	}	
	
	function getmicrotime() { 
		list($usec, $sec) = explode(" ",microtime()); 
		return ((float)$usec + (float)$sec); 
	}	
}
?>