main.c

linux下的网络下载工具prozilla的源码

/* The main file 
   Copyright (C) 2000 Kalum Somaratna 
   
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
   
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
   
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif

#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <time.h>
#include <errno.h>
#include <ctype.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <sys/time.h>
#include <signal.h>
#include <setjmp.h>
#include <assert.h>
#include <pthread.h>
#include <sys/param.h>
#include <sys/stat.h>

#ifdef HAVE_NCURSES_H
#include <ncurses.h>
#else
#include <curses.h>
#endif
#include <pwd.h>
#include <ctype.h>

#include "connect.h"
#include "ftp.h"
#include "url.h"
#include "misc.h"
#include "http.h"
#include "main.h"
#include "mainconf.h"
#include "connection.h"
#include "interface.h"
#include "resume.h"
#include "http-retr.h"
#include "ftp-retr.h"
#include "runtime.h"
#include "getopt.h"
#include "netrc.h"
#include "logfile.h"
#include "ftpsearch.h"
#include "debug.h"
#include "init.h"


#define RETRY_LOGIN_TIME 300


/*
 * pointer to array of connections which is allocated in allocate_connections 
 * according to the value of int num_connections above
 */

connection_data *connections = NULL;
/*
 * the threads 
 */
pthread_t *threads = NULL;
struct runtime rt;

/* Mutex for the changing of the state */
pthread_mutex_t status_change_mutex = PTHREAD_MUTEX_INITIALIZER;

/* Mutex for the setting of the throttling of the  rates per connection */
pthread_mutex_t compute_throttle_mutex = PTHREAD_MUTEX_INITIALIZER;

/* Condition which is broadcast when a thread changes its status 
 *to connecting 
 */
pthread_cond_t connecting_cond = PTHREAD_COND_INITIALIZER;

/*
 * structure for options parsing
 */

struct option long_opts[] = {
    /*
     * { name  has_arg  *flag  val } 
     */
    {"resume", no_argument, NULL, 'r'},
/*    {"connections", required_argument, NULL, 'c'},*/
    {"license", no_argument, NULL, 'L'},
    {"help", no_argument, NULL, 'h'},
    {"gtk", no_argument, NULL, 'g'},
    {"no-netrc", no_argument, NULL, 'n'},
    {"tries", required_argument, NULL, 't'},
    {"force", no_argument, NULL, 'f'},
    {"version", no_argument, NULL, 'v'},
    {"directory-prefix", required_argument, NULL, 'P'},
    {"use-port", no_argument, NULL, 129},
    {"retry-delay", required_argument, NULL, 130},
    {"timeout", required_argument, NULL, 131},
    {"no-getch", no_argument, NULL, 132},
    {"debug", no_argument, NULL, 133},
    {"ftpsearch", no_argument, NULL, 's'},
    {"no-search", no_argument, NULL, 135},
    {"pt", required_argument, NULL, 136},
    {"pao", required_argument, NULL, 137},
    {"max-ftps-servers", required_argument, NULL, 138},
    {"max-bps", required_argument, NULL, 139},
    {0, 0, 0, 0}
};


/*
 * func prototypes 
 */


/* creates the threads */
interface_ret do_downloads(ftp_mirror * mirrors, int num_servers);
/* The following funcs display the license and help*/
void help(void);
void license(void);

/* Checks wether a file exists and if it does 
 * get the users input
 */
int query_overwrite_target(char *fname);
void query_resume_old_download(urlinfo * u);
void delete_file_portions(urlinfo * u);
/*Routines to handle the logfile */
void do_log_file_normal(urlinfo * u);
void do_log_file_resume(urlinfo * u);


/* displays the software license */
void license(void)
{
    fprintf(stderr,
	    "   Copyright (C) 2000 Kalum Somaratna\n"
	    "\n"
	    "   This program is free software; you can redistribute it and/or modify\n"
	    "   it under the terms of the GNU General Public License as published by\n"
	    "   the Free Software Foundation; either version 2, or (at your option)\n"
	    "   any later version.\n"
	    "\n"
	    "   This program is distributed in the hope that it will be useful,\n"
	    "   but WITHOUT ANY WARRANTY; without even the implied warranty of\n"
	    "   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n"
	    "   GNU General Public License for more details.\n"
	    "\n"
	    "   You should have received a copy of the GNU General Public License\n"
	    "   along with this program; if not, write to the Free Software\n"
	    "   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.\n");
}


/* displays the help message */

void help(void)
{
    fprintf(stderr,
	    "Usage: proz [OPTIONS] file_url\n"
	    "\n"
	    "Ex: proz http://gnu.org/gnu.jpg\n"
	    "\n"
	    "Options:\n"
	    "      -h, --help        Give this help\n"
	    "      -r, --resume      Resume an interrupted download\n"
	    "      -f, --force       Never prompt the user when overwriting files\n"
	    "      -1                Force a single connection only\n"
	    "      -g, --gtk         Use GTK interface, instead of curses (broken for now)\n"
	    "      -n, --no-netrc    Don't use .netrc, get the user/password\n"
	    "                        from the command line,otherwise use the\n"
	    "                        anonymous login for FTP sessions\n"
	    "      --no-getch        Instead of waiting for the user pressing a key,\n"
	    "                        print the error to stdout and quit\n"
	    "      --debug           Log debugging info to a file (default is debug.log)\n"
	    "\n"
	    "Directories:\n"
	    "      -P,  --directory-prefix=DIR  save the generated file to DIR/\n"
	    "\n"
	    "FTP Options:\n"
	    "      --use-port        Force usage of PORT insted of PASV (default)\n"
	    "                        for ftp transactions\n"
	    "\n"
	    "Download Options:\n"
	    "      -s,  --ftpsearch  Do a ftpsearch for faster mirrors\n"
	    "      --no-search       Do a direct download (no ftpsearch)\n"
	    "      -k=n              Use n connections instead of the default(4)\n"
	    "      --timeout=n       Set the timeout for connections to n seconds\n"
	    "                        (default 180)\n"
	    "      -t, --tries=n     Set number of attempts to n (default(200), 0=unlimited)\n"
	    "      --retry-delay=n   Set the time between retrys to n seconds\n"
	    "                        (default 15 seconds)\n"
	    "      --max-bps=n       Limit bandwith consumed to n bps (0=unlimited)\n"
	    "\n"
	    "FTP Search Options:\n"
	    "      --pt=n            Wait 2*n seconds for a server response (default 2*4)\n"
	    "      --pao=n           Ping n servers at once(default 5 servers at once)\n"
	    "      --max-ftps-servers=n  Request a max of n servers from ftpsearch (default 40)\n"
	    "\n"
	    "      -L, --license     Display software license\n"
	    "      -v, --version     Display version number\n"
	    "\n"
	    "ProZilla homepage: http://prozilla.genesys.ro\n"
	    "Please report bugs to <[e-mail]>\n");
}


/* Displays the version */

void version(void)
{
    fprintf(stderr, "%s. Version: %s\n", PACKAGE_NAME, PACKAGE_VERSION);
}


/*
 * This function allocates the FTP connections   
 */

void ftp_allocate_connections(urlinfo * u, off_t file_length,
			      char *file_io_mode)
{
    int i;
    off_t bytes_per_connection;
    off_t bytes_left;

    /*
     * Find the max path in GNU C
     */
    char buffer[MAXPATHLEN];

    connections = (connection_data *) kmalloc(sizeof(connection_data) *
					      rt.num_connections);

    if (file_length == -1)
    {
	assert(rt.num_connections == 1);
	bytes_per_connection = -1;
	bytes_left = -1;

    } else
    {
	bytes_per_connection = file_length / rt.num_connections;
	bytes_left = file_length % rt.num_connections;
    }

    for (i = 0; i < rt.num_connections; i++)
    {
	memset(&connections[i], 0, sizeof(connection_data));
	memcpy(&(connections[i].u), u, sizeof(urlinfo));
	if (snprintf(buffer, sizeof(buffer),
		     "%s%s%d",
		     u->file, DEFAULT_FILE_EXT, i) >= sizeof(buffer))
	{
	  die("Error: Filename for %s, part %d is too long\n", u->file, i);
	}
	connections[i].localfile = kstrdup(buffer);
	connections[i].file_mode = kstrdup(file_io_mode);
	connections[i].retry = TRUE;

	if (file_length == -1)
	{
	    connections[i].main_file_size = -1;
	    connections[i].remote_startpos = 0;
	    connections[i].remote_endpos = -1;
	} else
	{
	    connections[i].main_file_size = file_length;
	    connections[i].remote_startpos = i * bytes_per_connection;
	    connections[i].remote_endpos =
		i * bytes_per_connection + bytes_per_connection;
	}

	connections[i].local_startpos = 0;
	connections[i].orig_local_startpos = 0;
	/*
	 * set the number of times a connection will be retried 
	 */
	connections[i].status = IDLE;
    }
    /*
     * add the remaining bytes to the last connection 
     */
    connections[--i].remote_endpos += bytes_left;

    if (rt.run_mode == RESUME)
    {
	if (resume_modify_ftp_connections(connections, rt.num_connections)
	    != 0)
	    die("A file error while accessing the DL'ed files %s",
		strerror(errno));
    }
}

/*
 * This function allocates the HTTP connections   
 */

void http_allocate_connections(urlinfo * u, off_t file_length,
			       char *file_io_mode)
{
    int i;
    off_t bytes_per_connection;
    off_t bytes_left;

    /*
     * Find the max path in GNU C
     */
    char buffer[MAXPATHLEN];


    connections = (connection_data *) kmalloc(sizeof(connection_data) *
					      rt.num_connections);

    if (file_length == -1)
    {
	assert(rt.num_connections == 1);
	bytes_per_connection = -1;
	bytes_left = -1;
    } else
    {
	bytes_per_connection = file_length / rt.num_connections;
	bytes_left = file_length % rt.num_connections;
    }

    for (i = 0; i < rt.num_connections; i++)
    {
	memset(&connections[i], 0, sizeof(connection_data));
	memcpy(&(connections[i].u), u, sizeof(urlinfo));

	if (snprintf(buffer, sizeof(buffer),
		     "%s%s%d",
		     u->file, DEFAULT_FILE_EXT, i) >= sizeof(buffer))
	{
	  die("Error: Filename for %s, part %d is too long\n", u->file, i);
	}
	connections[i].localfile = kstrdup(buffer);
	connections[i].file_mode = kstrdup(file_io_mode);
	connections[i].retry = TRUE;

	if (file_length == -1)
	{
	    connections[i].main_file_size = -1;
	    connections[i].remote_startpos = 0;
	    connections[i].remote_endpos = -1;
	} else
	{
	    connections[i].main_file_size = file_length;
	    connections[i].remote_startpos = i * bytes_per_connection;
	    connections[i].remote_endpos =
		i * bytes_per_connection + bytes_per_connection - 1;
	}
	connections[i].local_startpos = 0;
	connections[i].orig_local_startpos = 0;
	connections[i].status = IDLE;
    }
    /*
     * add the remaining bytes to the last connection 
     */
    connections[--i].remote_endpos += bytes_left;

    if (rt.run_mode == RESUME)
    {
	if (resume_modify_http_connections(connections, rt.num_connections)
	    != 0)
	    die("A file error while accessing the DL'ed files %s",
		strerror(errno));
    }
}


/*
 * The function that handles the creation and handling of the threads 
 */


interface_ret do_downloads(ftp_mirror * mirrors, int num_servers)
{
    int i;
    interface_ret ret;

    /* set the download start time to zero */

    memset(&(rt.dl_start_time), 0, sizeof(struct timeval));

    if (threads == NULL)
	threads =
	    (pthread_t *) kmalloc(sizeof(pthread_t) * rt.num_connections);


    for (i = 0; i < rt.num_connections; i++)
    {
	switch (connections[i].u.proto)
	{
	case URLFTP:
	    if (pthread_create(&threads[i], NULL,
			       (void *) &ftp_loop,
			       (void *) (&connections[i])) != 0)
		die("Error: Not enough system resources");

	    break;

	case URLHTTP:
	    if (pthread_create(&threads[i], NULL,
			       (void *) &http_loop,
			       (void *) (&connections[i])) != 0)
		die("Error: Not enough system resources"
		    "to create thread!\n");

	    break;

	default:
	    die("Error: Unsupported Protocol was specified");
	    break;
	}
    }

#ifdef HAVE_GTK
    if (rt.use_gtk_option == TRUE)
    {
	rt.display_mode = DISPLAY_GTK;
	gtk_do_interface(file_size);
    } else
    {
#endif
	rt.in_curses_display_loop = TRUE;

	ret =
	    curses_do_interface(connections, rt.num_connections, mirrors,
				num_servers);
	rt.in_curses_display_loop = FALSE;
#ifdef HAVE_GTK
    }
#endif

    for (i = 0; i < rt.num_connections; i++)
    {
	pthread_join(threads[i], NULL);
    }

    free(threads);
    threads = NULL;

    return ret;
}


/* A important function which the interface routines call
 * This function will handle the state of the threads while
 * they are doing the work 
 */

void handle_threads(void)
{
    int i;
    static int max_simul_conns = 0;


    /*
     *If the server disallows a ftp connection, *
     * The reason maybe that the server has being asked to disallow *
     * more than a certain number of logins per ip address (the f**ers!) *
     * so then when a thread returns with LOGINFAIL then we will wait until *
     * another thread has finished before attempting 
     *to reconnect by restarting the failed thread 
     */

    for (i = 0; i < rt.num_connections; i++)
    {