class_httpretriever.php

软件的主要特点: 1.专门针对有"验证码"要求的论坛. 2.可以完成如下功能: 自动发布新帖,自动回复指定帖,自动注册论坛帐号 (所有操作不能同时针对一个网站,因为有"验证码"要求的网站是每次打开一个

				$this->progress(HRP_DEBUG,"Request used query string; not caching");
				return;
			} else {
				$this->progress(HRP_DEBUG,"Request appears to be static and cacheable");
			}
		}

		$values = array(
			"stats"=>$this->stats,
			"result_code"=>$this->result_code,
			"result_text"=>$this->result_text,
			"version"=>$this->version,
			"response"=>$this->response,
			"response_headers"=>$this->response_headers,
			"response_cookies"=>$this->response_cookies,
			"raw_response"=>$this->raw_response,
		);
		$values = serialize($values);

		$cache_dir = $this->cache_path;
		if (substr($cache_dir,-1)!='/') $cache_dir .= '/';
		
		if ($this->caching_highvolume) {
			$cache_dir .= substr($token,0,2) . '/';
			if (!is_dir($cache_dir)) @mkdir($cache_dir);
		}
		
		$filename = $cache_dir.$token.'.tmp';

		$fp = @fopen($filename,"w");
		if (!$fp) {
			$this->progress(HRP_DEBUG,"Unable to create cache file");
			return false;
		}
		fwrite($fp,$values);
		fclose($fp);

		$this->progress(HRP_DEBUG,"HTTP response stored to cache");
	}
	
	// (internal) fetch a page from the cache
	function _cache_fetch($token) {
		$this->cache_hit = false;
		$this->progress(HRP_DEBUG,"Checking for cached page value");

		$cache_dir = $this->cache_path;
		if (substr($cache_dir,-1)!='/') $cache_dir .= '/';
		
		if ($this->caching_highvolume) $cache_dir .= substr($token,0,2) . '/';
		
		$filename = $cache_dir.$token.'.tmp';
		if (!file_exists($filename)) {
			$this->progress(HRP_DEBUG,"Page not available in cache");
			return false;
		}
		
		if (time()-filemtime($filename)>$this->caching) {
			$this->progress(HRP_DEBUG,"Page in cache is expired");
			@unlink($filename);
			return false;
		}
		
		if ($values = file_get_contents($filename)) {
			$values = unserialize($values);
			if (!$values) {
				$this->progress(HRP_DEBUG,"Invalid cache contents");
				return false;
			}
			
			$this->stats = $values["stats"];
			$this->result_code = $values["result_code"];
			$this->result_text = $values["result_text"];
			$this->version = $values["version"];
			$this->response = $values["response"];
			$this->response_headers = $values["response_headers"];
			$this->response_cookies = $values["response_cookies"];
			$this->raw_response = $values["raw_response"];
			
			$this->progress(HRP_DEBUG,"Page loaded from cache");
			$this->cache_hit = true;
			return true;
		} else {
			$this->progress(HRP_DEBUG,"Error reading cache file");
			return false;
		}
	}
	
	function parent_path($path) {
		if (substr($path,0,1)=='/') $path = substr($path,1);
		if (substr($path,-1)=='/') $path = substr($path,0,strlen($path)-1);
		$path = explode('/',$path);
		array_pop($path);
		return count($path) ? ('/' . implode('/',$path)) : '';
	}
	
	// $cookies should be an array in one of two formats.
	//
	// Either: $cookies[ 'cookiename' ] = array (
	//		'/path/'=>array(
	//			'expires'=>time(),
	//			'domain'=>'yourdomain.com',
	//			'value'=>'cookievalue'
	//		)
	// );
	//
	// Or, a more simplified format:
	//	$cookies[ 'cookiename' ] = 'value';
	//
	// The former format will automatically check to make sure that the path, domain,
	// and expiration values match the HTTP request, and will only send the cookie if
	// they do match.  The latter will force the cookie to be set for the HTTP request
	// unconditionally.
	// 	
	function response_to_request_cookies($cookies,$urlinfo) {
		
		// check for simplified cookie format (name=value)
		$cookiekeys = array_keys($cookies);
		if (!count($cookiekeys)) return;
		
		$testkey = array_pop($cookiekeys);
		if (!is_array($cookies[ $testkey ])) {
			foreach ($cookies as $k=>$v) $this->request_cookies[$k] = $v;
			return;
		}
		
		// must not be simplified format, so parse as complex format:
		foreach ($cookies as $name=>$paths) {
			foreach ($paths as $path=>$values) {
				// make sure the cookie isn't expired
				if ( isset($values['expires']) && ($values['expires']<time()) ) continue;
				
				$cookiehost = $values['domain'];
				$requesthost = $urlinfo['host'];
				// make sure the cookie is valid for this host
				$domain_match = (
					($requesthost==$cookiehost) ||
					(substr($requesthost,-(strlen($cookiehost)+1))=='.'.$cookiehost)
				);				
				
				// make sure the cookie is valid for this path
				$cookiepath = $path; if (substr($cookiepath,-1)!='/') $cookiepath .= '/';
				$requestpath = $urlinfo['path']; if (substr($requestpath,-1)!='/') $requestpath .= '/';
				if (substr($requestpath,0,strlen($cookiepath))!=$cookiepath) continue;
				
				$this->request_cookies[$name] = $values['value'];
			}
		}
	}					
	
	// Execute the request for a particular URL, and transparently follow
	// HTTP redirects if enabled.  If $cookies is specified, it is assumed
	// to be an array received from $this->response_cookies and will be
	// processed to determine which cookies are valid for this host/URL.
	function _execute_request($url,$cookies = false) {
		// valid codes for which we transparently follow a redirect
		$redirect_codes = array(301,302,303,307);
		// valid methods for which we transparently follow a redirect
		$redirect_methods = array('GET','HEAD');

		$request_result = false;
		
		$this->followed_redirect = false;
		$this->response_cookies = array();
		$this->cookie_headers = '';

		$previous_redirects = array();
		do {
			// send the request
			$request_result = $this->_send_request($url,$cookies);
			$lasturl = $url;
			$url = false;

			// see if a redirect code was received
			if ($this->follow_redirects && in_array($this->result_code,$redirect_codes)) {
				
				// only redirect on a code 303 or if the method was GET/HEAD
				if ( ($this->result_code==303) || in_array($this->method,$redirect_methods) ) {
					
					// parse the information from the OLD URL so that we can handle
					// relative links
					$oldurlinfo = parse_url($lasturl);
					
					$url = $this->response_headers['Location'];
					
					// parse the information in the new URL, and fill in any blanks
					// using values from the old URL
					$urlinfo = parse_url($url);
					foreach ($oldurlinfo as $k=>$v) {
						if (!$urlinfo[$k]) $urlinfo[$k] = $v;
					}
					
					// create an absolute path
					if (substr($urlinfo['path'],0,1)!='/') {
						$baseurl = $oldurlinfo['path'];
						if (substr($baseurl,-1)!='/') $baseurl = $this->parent_path($url) . '/';
						$urlinfo['path'] = $baseurl . $urlinfo['path'];
					}
					
					// rebuild the URL
					$url = $this->rebuild_url($urlinfo);

					$this->method = "GET";
					$this->post_data = "";
					
					$this->progress(HRP_INFO,'Redirected to '.$url);
				}
			}
			
			if ( $url && strlen($url) ) {
				
				if (isset($previous_redirects[$url])) {
					$this->error = "Infinite redirection loop";
					$request_result = false;
					break;
				}
				if ( is_numeric($this->follow_redirects) && (count($previous_redirects)>$this->follow_redirects) ) {
					$this->error = "Exceeded redirection limit";
					$request_result = false;
					break;
				}

				$previous_redirects[$url] = true;
			}

		} while ($url && strlen($url));

		// clear headers that shouldn't persist across multiple requests
		$per_request_headers = array('Host','Content-Length');
		foreach ($per_request_headers as $k=>$v) unset($this->headers[$v]);
		
		if (count($previous_redirects)>1) $this->followed_redirect = array_keys($previous_redirects);
		
		return $request_result;
	}
	
	// private - sends an HTTP request to $url
	function _send_request($url,$cookies = false) {
		$this->progress(HRP_INFO,"Initiating {$this->method} request for $url");
		if ($this->caching) {
			$cachetoken = md5($url.'|'.$this->post_data);
			if ($this->_cache_fetch($cachetoken)) return true;
		}
		
		$time_request_start = $this->getmicrotime();
		
		$urldata = parse_url($url);
		$this->urldata = &$urldata;
		$http_host = $urldata['host'] . (isset($urldata['port']) ? ':'.$urldata['port'] : '');
		
		if (!isset($urldata["port"]) || !$urldata["port"]) $urldata["port"] = ($urldata["scheme"]=="https") ? 443 : 80;
		if (!isset($urldata["path"]) || !$urldata["path"]) $urldata["path"] = '/';
		
		if (!empty($urldata['user'])) $this->auth_username = $urldata['user'];
		if (!empty($urldata['pass'])) $this->auth_password = $urldata['pass'];
		
		//echo "Sending HTTP/{$this->version} {$this->method} request for ".$urldata["host"].":".$urldata["port"]." page ".$urldata["path"]."<br>";
		
		if ($this->version>"1.0") $this->headers["Host"] = $http_host;
		if ($this->method=="POST") {
			$this->headers["Content-Length"] = strlen($this->post_data);
			if (!isset($this->headers["Content-Type"])) $this->headers["Content-Type"] = "application/x-www-form-urlencoded";
		}
		
		if ( !empty($this->auth_username) || !empty($this->auth_password) ) {
			$this->headers['Authorization'] = 'Basic '.base64_encode($this->auth_username.':'.$this->auth_password);
		} else {
			unset($this->headers['Authorization']);
		}
		
		if (is_array($cookies)) {
			$this->response_to_request_cookies($cookies,$urldata);
		}
		
		if (!empty($urldata["query"])) $urldata["path"] .= "?".$urldata["query"];
		$request = $this->method." ".$urldata["path"]." HTTP/".$this->version."\r\n";
		$request .= $this->build_headers();
		$request .= $this->post_data;
		
		$this->response = "";
		
		// clear headers that shouldn't persist across multiple requests
		// (we can do this here as we've already built the request, including headers, above)
		$per_request_headers = array('Host','Content-Length');
		foreach ($per_request_headers as $k=>$v) unset($this->headers[$v]);
		
		// Native SSL support requires the OpenSSL extension, and was introduced in PHP 4.3.0
		$php_ssl_support = extension_loaded("openssl") && version_compare(phpversion(),"4.3.0")>=0;
		
		// if this is a plain HTTP request, or if it's an HTTPS request and OpenSSL support is available,
		// natively perform the HTTP request
		if ( ( ($urldata["scheme"]=="http") || ($php_ssl_support && ($urldata["scheme"]=="https")) ) && (!$this->force_curl) ) {
			$curl_mode = false;

			$hostname = $this->connect_ip ? $this->connect_ip : $urldata['host'];
			if ($urldata["scheme"]=="https") $hostname = 'ssl://'.$hostname;
			
			$time_connect_start = $this->getmicrotime();

			$this->progress(HRP_INFO,'Opening socket connection to '.$hostname.' port '.$urldata['port']);

			$this->expected_bytes = -1;
			$this->received_bytes = 0;
			
			$fp = @fsockopen ($hostname,$urldata["port"],$errno,$errstr,$this->connect_timeout);
			$time_connected = $this->getmicrotime();
			$connect_time = $time_connected - $time_connect_start;
			if ($fp) {
				if ($this->stream_timeout) stream_set_timeout($fp,$this->stream_timeout);
				$this->progress(HRP_INFO,"Connected; sending request");
				
				$this->progress(HRP_DEBUG,$request);
				fputs ($fp, $request);
				$this->raw_request = $request;
				
				if ($this->stream_timeout) {
					$meta = socket_get_status($fp);
					if ($meta['timed_out']) {
						$this->error = "Exceeded socket write timeout of ".$this->stream_timeout." seconds";
						$this->progress(HRP_ERROR,$this->error);
						return false;
					}
				}
				
				$this->progress(HRP_INFO,"Request sent; awaiting reply");
				
				$headers_received = false;
				$data_length = false;
				$chunked = false;
				$iterations = 0;
				while (!feof($fp)) {
					if ($data_length>0) {
						$line = fread($fp,$data_length);
						$this->progress(HRP_DEBUG,"[DL] Got a line: [{$line}] " . gettype($line));

						if ($line!==false) $data_length -= strlen($line);
					} else {
						$line = @fgets($fp,10240);
						$this->progress(HRP_DEBUG,"[NDL] Got a line: [{$line}] " . gettype($line));
						
						if ( ($chunked) && ($line!==false) ) {
							$line = trim($line);
							if (!strlen($line)) continue;
							
							list($data_length,) = explode(';',$line,2);
							$data_length = (int) hexdec(trim($data_length));
							
							if ($data_length==0) {
								$this->progress(HRP_DEBUG,"Done");
								// end of chunked data
								break;
							}
							$this->progress(HRP_DEBUG,"Chunk length $data_length (0x$line)");
							continue;
						}
					}
					
					if ($line===false) {
						$meta = socket_get_status($fp);
						if ($meta['timed_out']) {
							if ($this->stream_timeout) {
								$this->error = "Exceeded socket read timeout of ".$this->stream_timeout." seconds";
							} else {
								$this->error = "Exceeded default socket read timeout";
							}
							$this->progress(HRP_ERROR,$this->error);
							return false;
						} else {
							$this->progress(HRP_ERROR,'No data but not timed out');
						}
						continue;
					}					

					// check time limits if requested
					if ($this->max_time>0) {
						if ($this->getmicrotime() - $time_request_start > $this->max_time) {
							$this->error = "Exceeded maximum transfer time of ".$this->max_time." seconds";
							$this->progress(HRP_ERROR,$this->error);
							return false;
							break;
						}
					}

					$this->response .= $line;
					
					$iterations++;
					if ($headers_received) {
						if ($time_connected>0) {