ftpsearch.c

linux下的网络下载工具prozilla的源码

/* Functions to help with  retreiving and parsing mirror information 
   Copyright (C) 2000 Kalum Somaratna 
   
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
   
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
   
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <unistd.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <errno.h>
#include <string.h>
#include <sys/select.h>
#include <sys/time.h>
#include <ctype.h>
#include <pthread.h>
#include <assert.h>
#ifdef HAVE_NCURSES_H
#include <ncurses.h>
#else
#include <curses.h>
#endif				/*
				 * HAVE_CURSES 
				 */
#include "main.h"
#include "http.h"
#include "netrc.h"
#include "connect.h"
#include "runtime.h"
#include "url.h"
#include "misc.h"
#include "ftpsearch.h"
#include "ping.h"
#include "debug.h"


typedef struct {
    struct ftp_mirror *ftp_mirrors;
    int num_servers;
    int ping_run_done;
} ping_mthread_data;


uerr_t get_mirror_info(urlinfo * u, http_stat_t * hs, char **ret_buf);
void ping_the_list(ping_mthread_data * ping_mt_data);

char *find_ahref(char *buf)
{

    return (strstr(buf, "<A HREF="));

}

char *find_end(char *buf)
{
    return (strstr(buf, ">"));
}

char *find_closed_a(char *buf)
{
    return (strstr(buf, "</A"));
}

char *get_string_ahref(char *buf, char *out)
{
    char *p1, *p2, *p3;

    p1 = find_ahref(buf);
    assert(p1 != NULL);

    p2 = find_end(p1);
    assert(p2 != NULL);

    p3 = find_closed_a(p2);
    assert(p3 != NULL);

    strncpy(out, p2 + 1, p3 - p2 - 1);
    out[p3 - p2 - 1] = 0;
    return p3;

}


char *prepare_lycos_url(urlinfo * url_data, char * ftps_loc)
{
#define LYCOS_URL "?form=advanced&query=%s&doit=Search&type=Exact+search&hits=%d&matches=&hitsprmatch=&limdom=&limpath=&limsize1=%Ld&limsize2=%Ld&f1=Host&f2=Path&f3=Size&f4=-&f5=-&f6=-&header=none&sort=none&trlen=20"

    char *lycos_url_buf;
    int lycos_url_len;

    /*    int len,len1;
       char *p, *p1,*lycos_dir;
       int num_dirs, use_file=0, use_dir=0;
     */


    /* Okay lets now construct the URL we want to do lycos */
    lycos_url_len = snprintf(NULL, 0,
			     "%s" LYCOS_URL,
			     ftps_loc, url_data->file, rt.ftps_mirror_req_n,
			    (long long) url_data->file_size, 
			    (long long) url_data->file_size);
    if (lycos_url_len <= 0)
      die("Unable to calculate buffer length for Lycos search\n");

    lycos_url_len++; /* nul */
    lycos_url_buf = kmalloc(lycos_url_len);

    snprintf(lycos_url_buf, lycos_url_len,
	     "%s" LYCOS_URL,
	     ftps_loc, url_data->file, rt.ftps_mirror_req_n,
	     (long long) url_data->file_size, 
	     (long long) url_data->file_size);

#undef LYCOS_URL

/* if we have a dir
   if(strcmp(url_data->dir,"pub")==0)
   {
   use_file=TRUE;
   }
   p=kstrdup(url_data->dir);
   p1=strtok(p,"/");
*/

    /*    while(p=kstrdup(url_data->dir))
       {

       }
       p=strdup(url_data->dir);
       p1=strtok(p,"/");

       sprintf(lycos_url_buf,"localhost/search3.html"); 
     */
    /*     sprintf(lycos_url_buf,"localhost/search3.html"); */

    debug_prz("ftpsearch url= %s\n", lycos_url_buf);
    return lycos_url_buf;
}

uerr_t parse_html_mirror_list(urlinfo * url_data,char *p,struct ftp_mirror ** pmirrors, int *num_servers)
{

   char  *p1, *p2, *i = 0, *j;
    ftp_mirror *ftp_mirrors;
    int k, num_ah = 0, num_pre = 0;
    char buf[1024];


    if (strstr(p, "No hits") != 0)
    {
	*num_servers = 0;
	return MIRINFOK;
    }

/*Check the number of PRE tags */
    p1 = p;
    while (((p1 = strstr(p1, "<PRE>")) != NULL) && p1)
    {
	num_pre++;
	p1 += 5;
    }

    debug_prz("Number of PRE tags found = %d\n", num_pre);

    if (num_pre == 1)
    {

	if ((i = strstr(p, "<PRE>")) == NULL)
	{
	    debug_prz("nomatches found");
	    return MIRPARSEFAIL;
	}

	debug_prz("match at %d found", i - p);

	if ((j = strstr(p, "</PRE>")) == NULL)
	{
	    debug_prz("nomatches found");
	    return MIRPARSEFAIL;
	}
    } else
    {
	/*search for the reported hits text */
	char *rep_hits;
	int prior_pres = 0;

	if ((rep_hits = strstr(p, "reported hits")) == NULL)
	{
	    debug_prz("no reported hits found");
	    return MIRPARSEFAIL;
	}

	/* Okay so we got the position after the results, lets see how many PRE tags were there before it */

	p1 = p;
	while (((p1 = strstr(p1, "<PRE>")) < rep_hits) && p1)
	{
	    prior_pres++;
	    p1 += 5;
	}
	/* now get the location of the PRE before the output */

	p1 = p;
	i = 0;
	while (prior_pres--)
	{
	    p1 = strstr(p1, "<PRE>");
	    p1 += 5;
	}
	i = p1 - 5;

	/*now find the </PRE>  tag which is after the results */
	j = strstr(i, "</PRE>");

	if (j == NULL)
	{
	    debug_prz("The expected </PRE> tag was not found!\n");
	    return MIRPARSEFAIL;
	}
    }

    p1 = kmalloc((j - i - 5) + 100);
    strncpy(p1, i + 5, j - i - 5);

    debug_prz("\nstring len= %ld", strlen(p1));
    p1[j - i - 5 + 1] = 0;

    p2 = p1;

    while ((i = strstr(p1, "<A HREF=")) != NULL)
    {
	num_ah++;
	p1 = i + 8;
    }

    debug_prz("\n%d ahrefs found\n", num_ah);

    if (num_ah == 0)
    {
	*num_servers = 0;
	return MIRINFOK;
    }



    *num_servers = num_ah / 3;
    debug_prz("%d servers found\n", *num_servers);

    /* Allocate +1 because we need to add the user specified server as well
     */
    ftp_mirrors = kmalloc(sizeof(ftp_mirror) * ((*num_servers) + 1));
    assert(ftp_mirrors != 0);

    for (k = 0; k < *num_servers; k++)
    {
	memset(&(ftp_mirrors[k]), 0, sizeof(ftp_mirror));
	p2 = get_string_ahref(p2, buf);
	ftp_mirrors[k].server_name = kstrdup(buf);
	p2 = get_string_ahref(p2, buf);

	/*Strip any leading slash in the path name if preent */
	if (*buf == '/')
	    ftp_mirrors[k].path = kstrdup(buf + 1);
	else
	    ftp_mirrors[k].path = kstrdup(buf);

	p2 = get_string_ahref(p2, buf);
	ftp_mirrors[k].file_name = kstrdup(buf);
    }

    /* add the users server to the end if it is a ftp server */
    if (url_data->proto == URLFTP)
    {
	memset(&(ftp_mirrors[k]), 0, sizeof(ftp_mirror));
	ftp_mirrors[k].server_name = kstrdup(url_data->host);
	ftp_mirrors[k].path = kstrdup(url_data->dir);
	ftp_mirrors[k].file_name = kstrdup(url_data->file);
	*num_servers += 1;
    }

    debug_prz("%d servers found\n", *num_servers);

    for (k = 0; k < *num_servers; k++)
    {

	ftp_mirrors[k].full_name =
	    kmalloc(strlen(ftp_mirrors[k].server_name) +
		    strlen(ftp_mirrors[k].path) +
		    strlen(ftp_mirrors[k].file_name) + 12);
	sprintf(ftp_mirrors[k].full_name, "ftp://%s:21/%s/%s",
		ftp_mirrors[k].server_name, ftp_mirrors[k].path,
		ftp_mirrors[k].file_name);

	debug_prz("%s\n", ftp_mirrors[k].full_name);
    }


    *pmirrors = ftp_mirrors;
    return MIRINFOK;

}

uerr_t get_complete_mirror_list(urlinfo * url_data,
				struct ftp_mirror ** pmirrors,
				int *num_servers)
{
  uerr_t err=HERR;
  char *lycos_url_buf;
  urlinfo lycos_url_data;
  char *p;
  http_stat_t hs;

  char *ftps_servers[]={
    LYCOS_FTPSEARCH_LOC,
    "http://ftpsearch.uniovi.es:8000/ftpsearch"
  };
  int num_ftps_servers=2;
  int scount;

  for(scount=0;scount<num_ftps_servers+1;scount++)
    {
	
      if(scount==0)
	{
	  /*Lets first try the server specified at runtime */ 
	  lycos_url_buf = prepare_lycos_url(url_data, rt.ftpsearch_url);
	  if (!lycos_url_buf)
	    {
	      die("error in preparing URL %s",rt.ftpsearch_url);
	    }

	}
      else
	{
	  /* Well it appears that the default ftp server did not work 
	     well lets fall back on the ones listed above */

	  /* If the originally tried url was the same as the ones specified above 
	     then skip it
	  */
	  if(strcmp(ftps_servers[scount-1],rt.ftpsearch_url)==0)
	    continue;
	  else
	    {
	      lycos_url_buf = prepare_lycos_url(url_data, ftps_servers[scount-1]);
	      if (!lycos_url_buf)
		{
		  die("error in preparing URL %s",ftps_servers[scount-1]);
		}
	    }

	}

      memset(&lycos_url_data, 0, sizeof(lycos_url_data));

      err = parseurl(lycos_url_buf, &lycos_url_data, 0);
      if (err != URLOK)
	{
	  message("problem with URL\n");
	  continue;
	}

      err = get_mirror_info(&lycos_url_data, &hs, &p);
      if (err != HOK)
	{
	  message("A error occured while talking to %s\n",lycos_url_data.host);
	  freeurl(&lycos_url_data, 0);
   
	  if(scount==0)
	    {
	      napms(1000);
	      message("Will try backup ftpsearch servers instead");
	      napms(1000);
	    }

	  continue;
	}

  
      /*free the lycos url data struct */
      freeurl(&lycos_url_data, 0);

      err=parse_html_mirror_list(url_data,p,pmirrors, num_servers); 
      if (err==MIRINFOK)
	break;

      if(scount==0)
	    {
	      message("Unable to Parse the output of the server will try backup servers instead");
	      napms(1000);
	    }

    }

  return err;
}

void ping_one_server(ftp_mirror * server)
{
    uerr_t err;


    debug_prz("Pinging = %s....\n", server->server_name);

    err = myftp_ping(server->server_name, &(server->tv));

    debug_prz("err returned in pos =%d", err);

    if (err != FTPOK)
    {
	server->status = NORESPONSE;
/*	    message("%s No response/Error\n", server->server_name); */
    } else
    {
	/* ok it succeded */
	server->status = RESPONSEOK;
	server->milli_secs =
	    ((server->tv.tv_sec * 1000000) + server->tv.tv_usec) / 1000;
/*	    message("%s %dms\n",server->server_name, server->milli_secs); */
	debug_prz("%s %dms\n", server->server_name, server->milli_secs);
    }

}


void ping_the_list(ping_mthread_data * ping_mt_data)
{
    int k = 0, i, j;
    int num_iter;
    int num_left;
    pthread_t *ping_threads;

    pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
    ping_threads =
	(pthread_t *) kmalloc(sizeof(pthread_t) * rt.max_simul_pings);