httpsocket.cpp

功能强大的网络蜘蛛软件

#include "HttpSocket.h"

HttpSocket::HttpSocket(string domain, unsigned int port)
{
    this->domain = domain;
    this->port = port;
    this->encode="";
    this->ipaddr="";
    this->path="";
    this->reponse_number="";
    this->content_length=0;
    this->header_info="";
}

HttpSocket::~HttpSocket()
{

}
int HttpSocket::GetUrlDomain(string &path)
{
    size_t domain_end_pos = 0;
    string domain_tmp = "";
    domain_end_pos = path.find("/", 7);
    if(domain_end_pos != string::npos)
    {
        domain_tmp = path.substr(7, domain_end_pos-7);
        this->path = path.substr(domain_end_pos + 1);
    }else{
        domain_tmp = path.substr(7);
        this->path = "";
    }
    this->domain = domain_tmp;
    this->port = 80;
    size_t port_pos = 0;
    port_pos = domain_tmp.find(":");
    if (port_pos!=string::npos)
    {
        this->domain = domain_tmp.substr(0, port_pos );
        this->port = atoi(domain_tmp.substr(port_pos + 1).c_str());
    }
    return 0;
}
int HttpSocket::analysis_url(string url)
{
    if (url.substr(0, 4) == "http")
    {
        this->GetUrlDomain(url);
    }
    else
    {
        this->path = url;
    }
    if (this->GetIPaddrByName(this->domain, this->ipaddr))
    {
        return 1;//get ipaddr error
    }
    return 0;
}
int HttpSocket::Request(string &content, string request_url,  string addition )
{
    string response="";
    int result=0;
    int sockfd=0;
    int len=0;
    struct sockaddr_in address;
    content.erase();

    if (this->analysis_url(request_url))
    {
        return 1;
    }

    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    address.sin_family = AF_INET;
    address.sin_addr.s_addr = inet_addr(this->ipaddr.c_str());
    address.sin_port = htons(this->port);
    len = sizeof(address);

    result = connect(sockfd, (struct sockaddr *)&address, len);

    if (result == -1)
    {
        string errstr = strerror(errno);
        return 2;
    }
    string sendBuffer = "GET /" + this->path + " HTTP/1.1\r\n";
    sendBuffer += "Host:" + this->domain +"\r\n";
    sendBuffer += addition;
    sendBuffer += "User-Agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\r\n";
    sendBuffer += "Connection: close\r\n\r\n";
    int w_rs=0;
    w_rs = write(sockfd,sendBuffer.c_str(),sendBuffer.size());
    char RecvBuffer[6001];

    while (true)
    {
        result = read(sockfd,&RecvBuffer,6000);
        if (result <= 0)
        {
            break;
        }
        RecvBuffer[result] = '\0';
        //cout << RecvBuffer << endl;
        response.append(RecvBuffer);
    }
    close(sockfd);
    if (response.empty())
    {
        return 3;
    }

    this->analysis(response);
    response = response.substr(response.size()-this->content_length);

    size_t pos=0;
    int length=0;
    int now_pos = 0;
    while (true)
    {
        pos = response.find("\r\n", now_pos + 2);
        if(pos > 2)
        {
            content = response;
            break;
        }

        if (pos==string::npos)
        {
            if (now_pos==0)
            {
                content=response;
            }
            break;
        }

        length = Functions::Hex2Decimal(response.substr(now_pos, pos - now_pos ));
        content.append(response.substr(pos + 2, length));
        now_pos = pos + length + 2;
    }

    return 0;
}

int HttpSocket::SendPostRequest(string &post_data, string request_url)
{

    int result;
    int sockfd;
    int len;
    string content;
    struct sockaddr_in address;
    if (this->analysis_url(request_url))
    {
        return 1;
    }

    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    address.sin_family = AF_INET;
    address.sin_addr.s_addr = inet_addr(this->ipaddr.c_str());
    address.sin_port = htons(this->port);
    len = sizeof(address);

    result = connect(sockfd, (struct sockaddr *)&address, len);

    if (result == -1)
    {
        perror("oops: client1");
        return 2;
    }

    char c_length[200];
    unsigned int length = post_data.size();
    sprintf(c_length, "%u", length);
    string s_length=c_length;
    string response;

    string headerBuffer = "POST /" + this->path + " HTTP/1.1\r\n";
    headerBuffer += "Host:" + this->domain +"\r\n";
    headerBuffer += "Content-Type: application/x-www-form-urlencoded\r\n";
    headerBuffer += "Content-Length: " + s_length + "\r\n";
    headerBuffer += "Connection: close\r\n\r\n";

    write(sockfd,headerBuffer.c_str(),strlen(headerBuffer.c_str()));
    char RecvBuffer[1025];
    while (read(sockfd,&RecvBuffer,1024))
    {
        RecvBuffer[result] = '\0';
        content.append(RecvBuffer);
    }
    close(sockfd);

    return 0;
}
string HttpSocket::getHeader()
{
    return this->header_info;
}
string HttpSocket::getStringByPos(string text, string::size_type pos)
{
    string::size_type i=0;
    string::size_type endPos = text.find(" ");
    while (++i<pos)
    {
        text = text.substr(text.find(" ")+1);
        endPos = text.find(" ");
    }

    return text.substr(0, endPos);
}
void HttpSocket::analysis(string &response)
{
    string::size_type pos = 0;
    string::size_type content_start_pos = 0;
    string::size_type charset_pos = 0;
    string::size_type charset_end_pos = 0;

    //返回头的结束位置
    pos = response.find("\r\n\r\n");
    if(pos==string::npos)
    {
        return ;
    }
    //头信息
    this->header_info = response.substr(0, pos + strlen("\r\n\r\n"));
    //内容开始的位置
    content_start_pos = pos+strlen("\r\n\r\n");
    //内容的长度
    this->content_length = response.size()-content_start_pos;
    //请求的内容
    pos = this->header_info.find("\r\n");
    if(pos==string::npos)
    {
        return ;
    }
    this->reponse_number = this->getStringByPos(this->header_info.substr(0, pos), 2);

    charset_pos = this->header_info.find("charset=");
    if(charset_pos==string::npos)
    {
        return ;
    }
    charset_end_pos = this->header_info.find("\r\n", charset_pos);
    if(charset_end_pos==string::npos)
    {
        return ;
    }
    this->encode = this->header_info.substr(charset_pos + strlen("charset="), charset_end_pos - charset_pos - strlen("charset="));
}
int HttpSocket::GetIPaddrByName(string domain, string &ipaddr)
{
    char **addrs=0;
    char ip[16]={0};
    struct hostent *hostinfo=0;

    hostinfo = gethostbyname(domain.c_str());

    if (!hostinfo)
    {
        return 1;
    }

    if (hostinfo->h_addrtype != AF_INET)
    {
        return 2;
    }

    addrs = hostinfo->h_addr_list;
    sprintf(ip, "%s", inet_ntoa(*(struct in_addr *)*addrs));
    ipaddr = ip;

    return 0;
}
int HttpSocket::GetFile(string request_url, string &filename)
{
    int result=0;
    int sockfd=0;
    int len=0;
    struct sockaddr_in address;

    if (this->analysis_url(request_url))
    {
        return 1;
    }

    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    address.sin_family = AF_INET;
    address.sin_addr.s_addr = inet_addr(this->ipaddr.c_str());
    address.sin_port = htons(this->port);
    len = sizeof(address);

    result = connect(sockfd, (struct sockaddr *)&address, len);

    if (result == -1)
    {
        string errstr = strerror(errno);
        return 2;
    }
    string sendBuffer = "GET /" + this->path + " HTTP/1.1\r\n";
    sendBuffer += "Host:" + this->domain +"\r\n";
    sendBuffer += "User-Agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\r\n";
    sendBuffer += "Accept: text/html; image/gif; image/jpeg; */*\r\n";
    sendBuffer += "Connection: close\r\n\r\n";
    int w_rs=0;
    //w_rs = send(sockfd,sendBuffer.c_str(),sendBuffer.size(),0);
    w_rs = write(sockfd,sendBuffer.c_str(),sendBuffer.size());

    string fileext = this->path;
    int rpos = this->path.rfind("/");
    if(rpos >= 0)
    {
        fileext = this->path.substr(rpos);
    }
    rpos = fileext.rfind(".");
    if(rpos >= 0)
    {
        fileext = fileext.substr(rpos);
    }

    struct stat statbuf;
    mode_t modes;

    stat("./images", &statbuf);
    modes = statbuf.st_mode;
    if(!S_ISDIR(modes))
    {
        mkdir("./images",0777);
    }

    struct timeval tv;
    struct timezone tz;
    gettimeofday (&tv , &tz);

    char file_name[155];
    sprintf(file_name, "%d%d", (unsigned int)tv.tv_sec,(unsigned int)tv.tv_usec);
    filename.append("./images/");
    filename.append(file_name);
    filename.append(fileext);
    if(filename=="")
    {
        cout << file_name << endl;
    }
    FILE *fp;
    char *offset=0;
    fp = fopen(filename.c_str(),"wb");

    char RecvBuffer[6001];

    while (true)
    {
        //memset(RecvBuffer, 0, sizeof(RecvBuffer));
        //result = recv(sockfd,&RecvBuffer,6000,0);
        result = read(sockfd,RecvBuffer,6000);
        if (result <= 0)
        {
            break;
        }
        RecvBuffer[result] = '\0';
        offset = strstr(RecvBuffer, "\r\n\r\n");

        if(offset == NULL)
        {
            offset = &RecvBuffer[0];
        }else{
            offset = offset + 4;