download.c

prozgui是一款Linxu下著名的下载工具

/******************************************************************************
 libprozilla - a download accelerator library
 Copyright (C) 2001 Kalum Somaratna

 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
******************************************************************************/

/* Download routines. */

/* $Id: download.c,v 1.33 2001/09/30 23:13:50 kalum Exp $ */


#include "common.h"
#include "download.h"
#include "ftp-retr.h"
#include "http-retr.h"
#include "logfile.h"
#include "debug.h"
#include "ftpsearch.h"

/*This will delete the joining file if the download is cancelled */
void cleanup_joining_thread(void *cdata);

/******************************************************************************
 Initialize the download.
******************************************************************************/
download_t *proz_download_init(urlinfo * u)
{

/*  pthread_mutexattr_t attr; */
  download_t *download = kmalloc(sizeof(download_t));

  /*  attr.__mutexkind = PTHREAD_MUTEX_RECURSIVE_NP;  */

  memset(download, 0, sizeof(download_t));

  /*FIXME   pthread_mutex_init(&download->status_change_mutex, &attr); */

  pthread_mutex_init(&download->status_change_mutex, NULL);
  pthread_mutex_init(&download->access_mutex, NULL);

  memcpy(&download->u, u, sizeof(urlinfo));

  download->dl_dir = kmalloc(PATH_MAX);
  download->output_dir = kmalloc(PATH_MAX);
  download->log_dir = kmalloc(PATH_MAX);
  strcpy(download->dl_dir, libprozrtinfo.dl_dir);
  strcpy(download->output_dir, libprozrtinfo.output_dir);
  strcpy(download->log_dir, libprozrtinfo.log_dir);
  download->resume_mode = FALSE;
  download->max_simul_connections = 0;

  download->max_allowed_bps = libprozrtinfo.max_bps_per_dl;
  download->file_build_msg = (char *) kmalloc(MAX_MSG_SIZE + 1);
  download->using_ftpsearch = FALSE;

  return download;
}

/******************************************************************************
 This will setup a download based on the connection info, and will attempt 
 to setup req_connections, but it might not be possible  if we dont know the 
 file size, returns the number of connections allocated. 
******************************************************************************/
int proz_download_setup_connections_no_ftpsearch(download_t * download,
						 connection_t * connection,
						 int req_connections)
{
  int num_connections, i;
  long bytes_per_connection;
  long bytes_left;
  FILE *fp;
  char *out_file;
  struct stat stat_buf;

  download->main_file_size = connection->main_file_size;
  download->resume_support = connection->resume_support;

  if (download->main_file_size == -1)
    {
      num_connections = 1;
      bytes_per_connection = -1;
      bytes_left = -1;

    } else
      {
	if (connection->resume_support == FALSE)
	  num_connections = 1;
	else
	  num_connections = req_connections;

	bytes_per_connection = connection->main_file_size / num_connections;
	bytes_left = connection->main_file_size % num_connections;
      }

  download->pconnections = kmalloc(sizeof(connection_t *) * num_connections);
  download->num_connections = num_connections;


 

  out_file=kmalloc(PATH_MAX);
  snprintf(out_file, PATH_MAX, "%s/%s.prozilla",
	   download->dl_dir, connection->u.file);


  //First see if the file exists then we dont create a new one else we do

  if (stat(out_file, &stat_buf) == -1)
    {
      /* the call failed */
      /* if the error is due to the file not been present then there is no 
	 need to do anything..just continue, otherwise return error (-1)
      */
      if (errno == ENOENT)
	{
	  //File not exists so create it
	  if (!
	      (fp =
	       fopen(out_file, "w+")))
	    {
	      download_show_message(download,
				    _
				    ("Unable to open file %s: %s!"),
				    out_file, strerror(errno));
	      return -1;
	    }

	}
      else
	return -1;
    }
  else
    {
      //TODO: File exists :  if it doesnt match file size warna boput it...
      if (!
	  (fp =
	   fopen(out_file, "r+")))
	{
	  download_show_message(download,
				_
				("Unable to open file %s: %s!"),
				out_file, strerror(errno));
	  return -1;
	}

    }


  //TRY setting the offset;
  if (download->main_file_size != -1)
    {

      if(fseek(fp, download->main_file_size, SEEK_SET)!=0)
	return -1;
    }

  /*Make sure all writes go directly to the file */
   setvbuf(fp, NULL, _IONBF, 0);

  for (i = 0; i < num_connections; i++)
    {
      download->pconnections[i]=proz_connection_init(&download->u,
						     &download->status_change_mutex);

      /*Copy somethings we need from the original connection */
      download->resume_support = download->pconnections[i]->resume_support =
	connection->resume_support;
      memcpy(&download->pconnections[i]->hs, &connection->hs,
	     sizeof(http_stat_t));


      download->pconnections[i]->localfile = kmalloc(PATH_MAX);
      strcpy(out_file, download->pconnections[i]->localfile);

    
      download->pconnections[i]->fp=fp;

      download->pconnections[i]->retry = TRUE;

      if (connection->main_file_size == -1)
	{
	  download->pconnections[i]->main_file_size = -1;
	  download->pconnections[i]->remote_startpos = 0;
	  download->pconnections[i]->orig_remote_startpos = 0;
	  download->pconnections[i]->remote_endpos = -1;


	  download->pconnections[i]->local_startpos = 0;
	} else
	  {
	    download->pconnections[i]->main_file_size = connection->main_file_size;
	    download->pconnections[i]->orig_remote_startpos = download->pconnections[i]->remote_startpos = i * bytes_per_connection;
	    download->pconnections[i]->remote_endpos =
	      i * bytes_per_connection + bytes_per_connection;

	    //Changing things here.....
	    download->pconnections[i]->local_startpos =    download->pconnections[i]->remote_startpos;
	  }



      /*Set the connections message to be download->msg_proc calback */
      proz_connection_set_msg_proc(download->pconnections[i],
				   download->msg_proc, download->cb_data);
    }

  /* Add the remaining bytes to the last connection   */

  download->pconnections[--i]->remote_endpos += bytes_left;
  download->using_ftpsearch = FALSE;


  /*NOTE:  Should we check for previously started downloads here and adjust 
    the local_startpos, accordingly or check for resumes later in another function 
    which is called after this?
  */

  return num_connections;
}


/* This will for each connection setup the local_startpos and file write mode if any prior download exists returns 1 on success, and -1 on a error.*/
int proz_download_load_resume_info(download_t * download)
{
  int i;
  int ret = 1;
  logfile lf;
  if(proz_log_read_logfile(&lf, download, TRUE)==1)
     proz_debug("sucessfully loaded resume info");


  for (i = 0; i < download->num_connections; i++)
  {

    if(download->pconnections[i]->remote_endpos - download->pconnections[i]->remote_startpos == download->pconnections[i]->remote_bytes_received)
      {

	connection_change_status(download->pconnections[i], COMPLETED);
	continue;
      }

            download->pconnections[i]->remote_startpos +=download->pconnections[i]->remote_bytes_received;


  }

  download->resume_mode = TRUE;
  return ret;
}

/* This will create the threads and start the downloads, 
   if resume is true it will load the resume info too if the download supports it
*/

void proz_download_start_downloads(download_t * download,
				   boolean resume_mode)
{

  int i;
 
  if (resume_mode)
  {
    /*Does this download suport resume? */
    if (download->resume_support == TRUE)
      proz_download_load_resume_info(download);
  }
else
  {
 /*Create the log file */
  if (log_create_logfile
      (download->num_connections, download->main_file_size,
       download->u.url, download) != 1)
  {
    download_show_message(download,
			  _("Warning! Unable to create logfile!"));
  }
}

  /* Allocate number of threads */
  download->threads =
      (pthread_t *) kmalloc(sizeof(pthread_t) * download->num_connections);

  /*Create them */
  for (i = 0; i < download->num_connections; i++)
  {
    switch (download->pconnections[i]->u.proto)
    {
    case URLHTTP:
      /*   http_loop(&download->connections[i]); */
      if (pthread_create(&download->threads[i], NULL,
			 (void *) &http_loop,
			 (void *) (download->pconnections[i])) != 0)

	proz_die(_("Error: Not enough system resources"));

      break;
    case URLFTP:

      /*  ftp_loop(&download->connections[i]);  */


      if (pthread_create(&download->threads[i], NULL,
			 (void *) &ftp_loop,
			 (void *) (download->pconnections[i])) != 0)
	proz_die(_("Error: Not enough system resources"));

      break;
    default:
      proz_die(_("Error: Unsupported Protocol was specified"));
    }
  }

  download_show_message(download, _("All threads created"));
}

void proz_download_stop_downloads(download_t * download)
{
  int i;
  /*Stop the threads */

  for (i = 0; i < download->num_connections; i++)
  {
    pthread_cancel(download->threads[i]);
    pthread_join(download->threads[i], NULL);
  }
}

/* returns  one of  DLINPROGRESS, DLERR, DLDONE, DLREMOTEFATAL, DLLOCALFATAL*/
uerr_t proz_download_handle_threads(download_t * download)
{


  //Create logfile everytime this is callaed
  log_create_logfile(download->num_connections, download->main_file_size,download->u.url, download);

  if (download->using_ftpsearch == TRUE)
    return download_handle_threads_ftpsearch(download);
  else
    return download_handle_threads_no_ftpsearch(download);
}

/* returns  one of  DLINPROGRESS, DLERR, DLDONE, DLREMOTEFATAL, DLLOCALFATAL*/
uerr_t download_handle_threads_no_ftpsearch(download_t * download)
{
  int i;

  for (i = 0; i < download->num_connections; i++)
    {
      /*Set the DL start time if it is not done so */
      pthread_mutex_lock(download->pconnections[i]->status_change_mutex);
      if (download->pconnections[i]->status == DOWNLOADING
	  && download->start_time.tv_sec == 0
	  && download->start_time.tv_usec == 0)
	{
	  gettimeofday(&download->start_time, NULL);
	}
      pthread_mutex_unlock(download->pconnections[i]->status_change_mutex);
    }


  /*If all the connections are completed then end them, and return complete */
  if ((proz_download_all_dls_status(download, COMPLETED)) == TRUE)
    {
      char * out_filename;
      char * orig_filename;
      download_show_message(download,
			    "All the conenctions have retreived the file"
			    "..waiting for them to end");
      proz_download_wait_till_all_end(download);
      download_show_message(download, "All the threads have being ended.");

      /*Close and rename file to original */
      flockfile(download->pconnections[0]->fp);
      fclose(download->pconnections[0]->fp);
      funlockfile(download->pconnections[0]->fp);

      out_filename=kmalloc(PATH_MAX);
      orig_filename=kmalloc(PATH_MAX);

      snprintf(orig_filename, PATH_MAX, "%s/%s",
	       download->dl_dir, download->pconnections[0]->u.file);
      snprintf(out_filename, PATH_MAX, "%s/%s.prozilla",
	       download->dl_dir, download->pconnections[0]->u.file);
      if(rename(out_filename, orig_filename)==-1)
	{
	  download_show_message(download, "Error While attempting to rename the file: %s", strerror(errno));
	}

      download_show_message(download, "Successfully renamed file");
      /*Delete the logfile as we dont need it now */
      if(proz_log_delete_logfile(download)!=1)
	download_show_message(download, "Error: Unable to delete the logfile: %s", strerror(errno));

      return DLDONE;
    }

  /*TODO handle restartable connections */
  for (i = 0; i < download->num_connections; i++)
    {
      dl_status status;
      uerr_t connection_err;

      pthread_mutex_lock(download->pconnections[i]->status_change_mutex);
      status = download->pconnections[i]->status;
      pthread_mutex_unlock(download->pconnections[i]->status_change_mutex);

      pthread_mutex_lock(&download->pconnections[i]->access_mutex);
      connection_err = download->pconnections[i]->err;
      pthread_mutex_unlock(&download->pconnections[i]->access_mutex);


      switch (status)
	{
	case MAXTRYS:
	  break;

	case REMOTEFATAL:
	  /* handle the CANTRESUME err code */

	  if (connection_err == CANTRESUME)
	    {
	      /*Terminate the connections */
	      proz_download_stop_downloads(download);
	      /*FIXME Do we delete any downloaded portions here ? */
	      return CANTRESUME;
	    } else /*Handle the file not being found on the server */
	      if (connection_err == FTPNSFOD || connection_err == HTTPNSFOD)
		{
		  if (proz_download_all_dls_filensfod(download) == TRUE)
		    {