http-retr.c

linux下的网络下载工具prozilla的源码

/* Functions to help with  retreiving HTTP files
   Copyright (C) 2000 Kalum Somaratna 
   
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
   
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
   
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */


#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif				/*
				 * HAVE_CONFIG_H 
				 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <ctype.h>
#include <netdb.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <assert.h>

#ifdef HAVE_NCURSES_H
#include <ncurses.h>
#else
#include <curses.h>
#endif				/*
				 * HAVE_CURSES 
				 */

#include "connect.h"
#include "runtime.h"
#include "url.h"
#include "misc.h"
#include "resume.h"
#include "main.h"
#include "mainconf.h"
#include "connection.h"
#include "http.h"
#include "http-retr.h"
#include "debug.h"

void clean_httpsock(void *cdata)
{
    int flags;
    connection_data *connection = (connection_data *) cdata;

    debug_prz("in clean http sock\n");

    if (connection->http_sock > 0)
    {
	flags = fcntl(connection->http_sock, F_GETFD, 0);
	if (flags == -1)
	{
	    debug_prz("sock invalid\n");
	} else
	    close(connection->http_sock);
    }

}

uerr_t http_get_file_chunk(connection_data * connection)
{
    char *req = NULL;
    char *user, *passwd, *wwwauth, *referer=0;
    int tos = IPTOS_THROUGHPUT;
    netrc_entry *netrc_ent;
    uerr_t err;
    /*
     * did we get a filename? 
     */

    assert(connection->localfile != NULL);
    assert(connection->file_mode != NULL);

    /*clear the socks */
    connection->http_sock = 0;
    /*
     * set the thread attributes 
     */
    pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);

    connection->hs.len = 0L;
    connection->hs.contlen = -1;
    connection->hs.res = -1;
    connection->hs.newloc = NULL;
    connection->hs.remote_time = NULL;
    connection->hs.error = NULL;


    /*
     * if there is nothing to download then return 
     */
    if (connection->status == COMPLETED)
    {
	gettimeofday(&connection->time_begin, NULL);
	return HOK;
    }

    /*
     * Lets test and see wether the file length to be got is zero  
     * If so then there is no point in connecting.. 
     * just create the file (zero length) and return  
     * have to resort to a ugly method to detect this :( 
     */

    if (connection->remote_startpos == 0 && connection->remote_endpos < 0)
    {
	if (!
	    (connection->fp =
	     fopen(connection->localfile, connection->file_mode)))
	{
	    message("Error opening file %s for writing: %s",
		    connection->localfile, strerror(errno));
	    connection->status = LOCALFATAL;
	    return WRITEERR;
	}
	fclose(connection->fp);
	connection->status = COMPLETED;
	return HOK;
    }

    connection->status = CONNECTING;
    err =
	connect_to_server(&(connection->http_sock), connection->u.host,
			  connection->u.port, rt.timeout);
    if (err != NOCONERROR)
    {
	message("Error connecting to %s", connection->u.host);
	connection->status = REMOTEFATAL;
	return err;
    }


    /* Authentification code */
    user = connection->u.user;
    passwd = connection->u.passwd;

    /*
     * Use .netrc if asked to do so. 
     */
    if (rt.use_netrc == TRUE)
    {
	netrc_ent = search_netrc(rt.netrc_list, connection->u.host);
	if (netrc_ent != NULL)
	{
	    user = netrc_ent->account;
	    passwd = netrc_ent->password;
	}
    }

    user = user ? user : "";
    passwd = passwd ? passwd : "";

    if (strlen(user) || strlen(passwd))
    {
	/* Construct the necessary header */
	wwwauth = get_basic_auth_str(user, passwd);
	message("Authenticating as user %s (%s password)", user,
		(strlen(passwd))?"using":"no");

	debug_prz("Authentification string=%s\n", wwwauth);
    } else
	wwwauth = 0;

    if(connection->u.referer)
      {
	referer= alloca(13+strlen(connection->u.referer));
	sprintf(referer, "Referer: %s\r\n", connection->u.referer);
      }

    /*
     * get the headers by sending GET 
     */

    {
      const char reqfmt[] =
	"GET %s HTTP/1.0\r\n"
	"User-Agent: %s\r\n"
	"Host: %s\r\n"
	"Accept: */*\r\n"
	"Range: bytes=%Ld-%Ld\r\n"
	"%s%s\r\n";
      int reqlen;

      reqlen = snprintf(NULL, 0,
			reqfmt,
			connection->u.path, USER_AGENT,
			connection->u.host,
			(long long) connection->remote_startpos,
			(long long) connection->remote_endpos,
			referer ? referer : "",
			wwwauth ? wwwauth : "");

      if (reqlen <= 0)
	die("Unable to calculate buffer length for HTTP GET request\n");

      reqlen++; /* nul */
      req = kmalloc(reqlen);

      snprintf(req, reqlen,
	       reqfmt,
	       connection->u.path, USER_AGENT,
	       connection->u.host,
	       connection->remote_startpos, connection->remote_endpos,
	       referer ? referer : "",
	       wwwauth ? wwwauth : "");
    }

    debug_prz("HTTP request= %s\n", req);

    err =
	http_fetch_headers(connection->http_sock, &(connection->u),
			   &(connection->hs), req);

    kfree(req);
    if (wwwauth)
	free(wwwauth);

    if (err != HOK)
    {
	/*Check if we authenticated using any user or password and if we 
	   were kicked out, if so return HAUTHFAIL */
	if (err == HAUTHREQ && (strlen(user) || strlen(passwd)))
	    err = HAUTHFAIL;
	/*
	 * a error occured druing the process 
	 */
	close(connection->http_sock);
	connection->status = REMOTEFATAL;
	return err;
    }

    if (!
	(connection->fp =
	 fopen(connection->localfile, connection->file_mode)))
    {
	message("Error opening file %s for writing: %s",
		connection->localfile, strerror(errno));
	close(connection->http_sock);
	connection->status = LOCALFATAL;
	return FOPENERR;
    }

    /*
     * prioritize packets 
     */
    setsockopt(connection->http_sock, IPPROTO_IP, IP_TOS, (char *) &tos,
	       sizeof(tos));

    /*
     * Make sure all writes go directly to the file 
     */
    setvbuf(connection->fp, NULL, _IONBF, 0);
    connection->status = DOWNLOADING;
    /*
     * lets store the start time in the connections structure
     * if this is the first time we have started 
     */
    if ((connection->time_begin.tv_sec == 0)
	&& (connection->time_begin.tv_usec == 0))
	gettimeofday(&connection->time_begin, NULL);

    err =
	http_retr_fsize_known(connection, connection->http_sock,
			      connection->fp);
    close(connection->http_sock);

    return err;

}


/* This function will get the complete file*/
/* Used when the server doesn't support reteving the file in chunks */

uerr_t http_get_complete_file(connection_data * connection)
{
    char *req = NULL;
    char *user, *passwd, *wwwauth, *referer=0;
    int tos = IPTOS_THROUGHPUT;
    netrc_entry *netrc_ent;
    uerr_t err;

    /* did we get a filename? */

    assert(connection->localfile != NULL);
    assert(connection->file_mode != NULL);

    /*clear the socks */
    connection->http_sock = 0;

    /*
     * set the thread attributes 
     */
    pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);

    connection->hs.len = 0L;
    connection->hs.contlen = -1;
    connection->hs.res = -1;
    connection->hs.newloc = NULL;
    connection->hs.remote_time = NULL;
    connection->hs.error = NULL;

    /*
     * if there is nothing to download then return 
     */
    if (connection->status == COMPLETED)
    {
	gettimeofday(&(connection->time_begin), NULL);
	return HOK;
    }

    connection->status = CONNECTING;
    err =
	connect_to_server(&(connection->http_sock), connection->u.host,
			  connection->u.port, rt.timeout);
    if (err != NOCONERROR)
    {
	message("Error connecting to %s", connection->u.host);
	connection->status = REMOTEFATAL;
	return err;
    }

    /* Authentification code */
    user = connection->u.user;
    passwd = connection->u.passwd;

    /*
     * Use .netrc if asked to do so. 
     */
    if (rt.use_netrc == TRUE)
    {
	netrc_ent = search_netrc(rt.netrc_list, connection->u.host);
	if (netrc_ent != NULL)
	{
	    user = netrc_ent->account;
	    passwd = netrc_ent->password;
	}
    }


    user = user ? user : "";
    passwd = passwd ? passwd : "";

    if (strlen(user) || strlen(passwd))
    {
	/*Construct the necessary header */
	wwwauth = get_basic_auth_str(user, passwd);
	message("Authenticating as user %s password %s", user, passwd);

	debug_prz("Authentification string=%s\n", wwwauth);
    } else
	wwwauth = 0;

   if(connection->u.referer)
      {
	referer= alloca(13+strlen(connection->u.referer));
	sprintf(referer, "Referer: %s\r\n", connection->u.referer);
      }


    /*
     * lets fetch the headers and since we want to retreive the file 
     */

    /*
     * we will use the GET command 
     */

    {
      const char reqfmt[] =
	"GET %s HTTP/1.0\r\n"
	"User-Agent: %s\r\n"
	"Host: %s\r\n"
	"Accept: */*\r\n"
	"%s%s\r\n";
      int reqlen;

      reqlen = snprintf(NULL, 0,
			reqfmt,
			connection->u.path, USER_AGENT,
			connection->u.host, referer ? referer : "",
			wwwauth ? wwwauth : "");

      if (reqlen <= 0)
	die("Unable to calculate buffer length for HTTP GET request\n");

      reqlen++; /* nul */
      req = kmalloc(reqlen);

      snprintf(req, reqlen,
	       reqfmt,
	       connection->u.path, USER_AGENT,
	       connection->u.host, referer ? referer : "",
	       wwwauth ? wwwauth : "");
    }

    debug_prz("HTTP request = %s", req);

    err =
	http_fetch_headers(connection->http_sock, &(connection->u),
			   &(connection->hs), req);

    kfree(req);
    if (wwwauth)
	free(wwwauth);

    if (err != HOK)
    {
	/*Check if we authenticated using any user or password and if we 
	   were kicked out, if so return HAUTHFAIL */
	if (err == HAUTHREQ && (strlen(user) || strlen(passwd)))
	    err = HAUTHFAIL;
	/*
	 * a error occured druing the process 
	 */
	connection->status = REMOTEFATAL;
	close(connection->http_sock);
	return err;
    }

    if (!
	(connection->fp =
	 fopen(connection->localfile, connection->file_mode)))
    {
	message("Error opening file %s for writing: %s",
		connection->localfile, strerror(errno));
	close(connection->http_sock);
	connection->status = LOCALFATAL;
	return FOPENERR;
    }

    /*
     * prioritize packets 
     */
    setsockopt(connection->http_sock, IPPROTO_IP, IP_TOS, (char *) &tos,
	       sizeof(tos));

    /*Make sure all writes go directly to the file */
    setvbuf(connection->fp, NULL, _IONBF, 0);
    connection->remote_bytes_received = 0;
    connection->status = DOWNLOADING;

    /*
     * lets store the start time in the connections structure
     * if this is the first time we have started 
     */
    if ((connection->time_begin.tv_sec == 0)
	&& (connection->time_begin.tv_usec == 0))
	gettimeofday(&connection->time_begin, NULL);

    if (connection->main_file_size != -1)
    {
	err =
	    http_retr_fsize_known(connection, connection->http_sock,
				  connection->fp);
    } else
    {
	err =
	    http_retr_fsize_notknown(connection, connection->http_sock,
				     connection->fp);
    }

    close(connection->http_sock);

    return err;
}


/* 
 * This function attemps to retrieve the requested file, if a error occurs it retries    
 * until either the file is retrieved properly or the max number of retry attemps are
 * exceeded
 */

void http_loop(connection_data * connection)
{
    uerr_t err = HERR;
    int first_attempt = TRUE;

    assert(rt.try_attempts >= 0);
    assert(connection->try_attempts >= 0);

    pthread_cleanup_push(clean_httpsock, (void *) (connection));

    do
    {
	if (first_attempt != TRUE)
	{