ftpsearch.c

prozgui是一款Linxu下著名的下载工具

/******************************************************************************
 libprozilla - a download accelerator library
 Copyright (C) 2001 Kalum Somaratna

 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
******************************************************************************/

#include "common.h"
#include "prozilla.h"
#include "misc.h"
#include "connect.h"
#include "debug.h"
#include "http.h"
#include "ftpsearch.h"


urlinfo *prepare_lycos_url(ftps_request_t * request, char *ftps_loc,
			   int num_req_mirrors);
urlinfo *prepare_filesearching_url(ftps_request_t * request, char *ftps_loc,
				   int num_req_mirrors);

uerr_t parse_lycos_html_mirror_list(ftps_request_t * request, char *p);
uerr_t parse_filesearching_html_mirror_list(ftps_request_t * request, char *p);

uerr_t get_mirror_info(connection_t * connection, char **ret_buf);

uerr_t parse_html_mirror_list(ftps_request_t * request, char *p);
char *get_string_ahref(char *buf, char *out);
char *find_ahref(char *buf);
char *find_end(char *buf);
char *find_closed_a(char *buf);
char *get_string_ahref(char *buf, char *out);
char *grow_buffer(char *buf_start, char *cur_pos, int *buf_len,
		  int data_len);
uerr_t get_complete_mirror_list(ftps_request_t * request);
ftp_mirror_t *reprocess_mirror_list(ftp_mirror_t * mirrors,
				    int *num_servers);


ftps_request_t * proz_ftps_request_init(
			       urlinfo * requested_url, long file_size,
			       char *ftps_loc,
			       ftpsearch_server_type_t server_type,
			       int num_req_mirrors)
{
  urlinfo *url;
  ftps_request_t * request;

  assert(requested_url);
  assert(requested_url->file);

  request=kmalloc(sizeof(ftps_request_t));
  memset(request, 0, sizeof(ftps_request_t));
  request->file_name = strdup(requested_url->file);
  request->requested_url = proz_copy_url(requested_url);
  request->file_size = file_size;
  request->server_type = server_type;
  pthread_mutex_init(&request->access_mutex, 0);

  switch (server_type)
  {
  case LYCOS:
    url = prepare_lycos_url(request, ftps_loc, num_req_mirrors);
    if (url == 0)
      proz_die("Bad URl specification");
 
    /*NOTE  pasing zero as the status change mutes as we dont need it here */
      request->connection=proz_connection_init(url,0);


    break;
  case FILESEARCH_RU:
    url = prepare_filesearching_url(request, ftps_loc, num_req_mirrors);
    if (url == 0)
      proz_die("Bad URl specification");
 
    /*NOTE  pasing zero as the status change mutes as we dont need it here */
      request->connection=proz_connection_init(url,0);
    break;
  default:
    proz_debug("Unsupported FTP search server type");
    proz_die("Unsupported FTP search server type");
  }

  return request;
}


urlinfo *prepare_lycos_url(ftps_request_t * request, char *ftps_loc,
			   int num_req_mirrors)
{
  urlinfo *url;
  uerr_t err;
  char *lycos_url_buf;
  int lycos_url_len = strlen(ftps_loc) +
      strlen
      ("?form=advanced&query=%s&doit=Search&type=Exact+search&hits=%d&matches=&hitsprmatch=&limdom=&limpath=&limsize1=%d&limsize2=%d&limtime1=&limtime2=&f1=Host&f2=Path&f3=Size&f4=-&f5=-&f6=-&header=none&sort=none&trlen=20");


  assert(request->file_name);

  url = (urlinfo *) kmalloc(sizeof(urlinfo));

  /* Okay lets now construct the URL we want to do lycos */
  lycos_url_buf =
      (char *) kmalloc(lycos_url_len + strlen(request->file_name) + 300);

  sprintf(lycos_url_buf,
	  "%s?form=advanced&query=%s&doit=Search&type=Exact+search&hits=%d&matches=&hitsprmatch=&limdom=&limpath=&limsize1=%ld&limsize2=%ld&f1=Host&f2=Path&f3=Size&f4=-&f5=-&f6=-&header=none&sort=none&trlen=20",
	  ftps_loc, request->file_name, num_req_mirrors,
	  request->file_size, request->file_size);

  /* Debugging purposes */
  /*sprintf(lycos_url_buf,"localhost/search.html");    */


  proz_debug("ftpsearch url= %s\n", lycos_url_buf);

  err = proz_parse_url(lycos_url_buf, url, 0);

  if (err != URLOK)
    return 0;
  else
    return url;
}



urlinfo *prepare_filesearching_url(ftps_request_t * request, char *ftps_loc,
			   int num_req_mirrors)
{
  urlinfo *url;
  uerr_t err;
  char *filesearching_url_buf;
  int filesearching_url_len = strlen(ftps_loc) +
      strlen
      ("?q=ddd-3.3.tar.bz2&l=en&t=f&e=on&m=20&o=n&s=on&s1=4811576&s2=4811576&d=&p=&p2=&x=10&y=14");


  assert(request->file_name);

  url = (urlinfo *) kmalloc(sizeof(urlinfo));

  /* Okay lets now construct the URL we want to do lycos */
  filesearching_url_buf =
      (char *) kmalloc(filesearching_url_len + strlen(request->file_name) + 300);

  sprintf(filesearching_url_buf,
	  "%s?q=%s&l=en&t=f&e=on&m=%d&o=n&s=on&s1=%ld&s2=%ld&d=&p=&p2=&x=10&y=14",
	  ftps_loc, request->file_name, num_req_mirrors,
	  request->file_size, request->file_size);

  /* Debugging purposes */
  /*  sprintf(filesearching_url_buf,"localhost/fs.html");   */


  proz_debug("ftpsearch url= %s\n", filesearching_url_buf);

  err = proz_parse_url(filesearching_url_buf, url, 0);

  if (err != URLOK)
    return 0;
  else
    return url;
}



uerr_t parse_html_mirror_list(ftps_request_t * request, char *p)
{

  switch (request->server_type)
  {
  case LYCOS:
    return parse_lycos_html_mirror_list(request, p);
    break;
  case FILESEARCH_RU:
    return parse_filesearching_html_mirror_list(request, p);
    break;
  default:
    proz_debug("Unsupported FTP search server type");
    proz_die("Unsupported FTP search server type");
  }
  return MIRPARSEFAIL;
}

uerr_t parse_lycos_html_mirror_list(ftps_request_t * request, char *p)
{

  struct ftp_mirror **pmirrors = &request->mirrors;
  int *num_servers = &request->num_mirrors;
  char *p1, *p2, *i = 0, *j;
  ftp_mirror_t *ftp_mirrors;
  int k, num_ah = 0, num_pre = 0;
  char buf[1024];


  if (strstr(p, "No hits") != 0)
  {
    *num_servers = 0;
    return MIRINFOK;
  }

/*Check the number of PRE tags */
  p1 = p;
  while (((p1 = strstr(p1, "<PRE>")) != NULL) && p1)
  {
    num_pre++;
    p1 += 5;
  }

  proz_debug("Number of PRE tags found = %d\n", num_pre);

  if (num_pre == 1)
  {

    if ((i = strstr(p, "<PRE>")) == NULL)
    {
      proz_debug("nomatches found");
      return MIRPARSEFAIL;
    }

    proz_debug("match at %d found", i - p);

    if ((j = strstr(p, "</PRE>")) == NULL)
    {
      proz_debug("nomatches found");
      return MIRPARSEFAIL;
    }
  } else
  {
    /*search for the reported hits text */
    char *rep_hits;
    int prior_pres = 0;

    if ((rep_hits = strstr(p, "reported hits")) == NULL)
    {
      proz_debug("no reported hits found");
      return MIRPARSEFAIL;
    }

    /* Okay so we got the position after the results, lets see how many PRE tags were there before it */

    p1 = p;
    while (((p1 = strstr(p1, "<PRE>")) < rep_hits) && p1)
    {
      prior_pres++;
      p1 += 5;
    }
    /* now get the location of the PRE before the output */

    p1 = p;
    i = 0;
    while (prior_pres--)
    {
      p1 = strstr(p1, "<PRE>");
      p1 += 5;
    }
    i = p1 - 5;

    /*now find the </PRE>  tag which is after the results */
    j = strstr(i, "</PRE>");

    if (j == NULL)
    {
      proz_debug("The expected </PRE> tag was not found!\n");
      return MIRPARSEFAIL;
    }
  }

  p1 = kmalloc((j - i - 5) + 100);
  strncpy(p1, i + 5, j - i - 5);

  proz_debug("\nstring len= %ld", strlen(p1));
  p1[j - i - 5 + 1] = 0;

  p2 = p1;

  while ((i = strstr(p1, "<A HREF=")) != NULL)
  {
    num_ah++;
    p1 = i + 8;
  }

  proz_debug("\n%d ahrefs found\n", num_ah);

  if (num_ah == 0)
  {
    *num_servers = 0;
    return MIRINFOK;
  }



  *num_servers = num_ah / 3;
  proz_debug("%d servers found\n", *num_servers);

  /* Allocate +1 because we need to add the user specified server as well
   */
  ftp_mirrors =
      (ftp_mirror_t *) kmalloc(sizeof(ftp_mirror_t) *
			       ((*num_servers) + 1));

  for (k = 0; k < *num_servers; k++)
  {
    memset(&(ftp_mirrors[k]), 0, sizeof(ftp_mirror_t));
    p2 = get_string_ahref(p2, buf);
    ftp_mirrors[k].server_name = kstrdup(buf);
    p2 = get_string_ahref(p2, buf);

    ftp_mirrors[k].paths = kmalloc(sizeof(mirror_path_t));

    //      ftp_mirrors[k].paths=kmalloc (sizeof (char *));
    ftp_mirrors[k].num_paths = 1;

    /*Strip any leading slash in the path name if preent */
    if (*buf == '/')
      ftp_mirrors[k].paths[0].path = kstrdup(buf + 1);
    else
      ftp_mirrors[k].paths[0].path = kstrdup(buf);

    p2 = get_string_ahref(p2, buf);
    ftp_mirrors[k].file_name = kstrdup(buf);
  }

  /* add the users server to the end if it is a ftp server */
  if (request->requested_url->proto == URLFTP)
  {
    memset(&(ftp_mirrors[k]), 0, sizeof(ftp_mirror_t));
    ftp_mirrors[k].server_name = kstrdup(request->requested_url->host);

    ftp_mirrors[k].paths = kmalloc(sizeof(mirror_path_t));
    ftp_mirrors[k].num_paths = 1;

    if (*(request->requested_url->dir))
      ftp_mirrors[k].paths[0].path = kstrdup(request->requested_url->dir);
    else
      ftp_mirrors[k].paths[0].path = kstrdup("");

    ftp_mirrors[k].file_name = kstrdup(request->requested_url->file);
    *num_servers += 1;
  }

  proz_debug("%d servers found\n", *num_servers);

  for (k = 0; k < *num_servers; k++)
  {

    ftp_mirrors[k].full_name =
	(char *) kmalloc(strlen(ftp_mirrors[k].server_name) +
			 strlen(ftp_mirrors[k].paths[0].path) +
			 strlen(ftp_mirrors[k].file_name) + 13);
    sprintf(ftp_mirrors[k].full_name, "%s%s:21/%s%s%s", "ftp://",
	    ftp_mirrors[k].server_name, ftp_mirrors[k].paths[0].path, "/",
	    ftp_mirrors[k].file_name);

    proz_debug("%s\n", ftp_mirrors[k].full_name);
  }


  *pmirrors = reprocess_mirror_list(ftp_mirrors, num_servers);
  /*    *pmirrors = ftp_mirrors; */
  return MIRINFOK;
}


uerr_t parse_filesearching_html_mirror_list(ftps_request_t * request, char *p)
{

  struct ftp_mirror **pmirrors = &request->mirrors;
  int *num_servers = &request->num_mirrors;
  char *p1, *p2, *i = 0, *j;
  ftp_mirror_t *ftp_mirrors;
  int k, num_ah = 0, num_pre = 0;
  char buf[1024];


  if (strstr(p, "not found") != 0)
  {
    *num_servers = 0;
    return MIRINFOK;
  }

/*Check the number of PRE tags */
  p1 = p;
  while (((p1 = strstr(p1, "<pre")) != NULL) && p1)
  {
    num_pre++;
    p1 += 4;
  }

  proz_debug("Number of PRE tags found = %d\n", num_pre);

  if (num_pre == 1)
  {

    if ((i = strstr(p, "<pre class=list>")) == NULL)
    {
      proz_debug("nomatches found");
      return MIRPARSEFAIL;
    }

    proz_debug("match at %d found", i - p);

    if ((j = strstr(p, "</pre>")) == NULL)
    {
      proz_debug("nomatches found");
      return MIRPARSEFAIL;
    }
  } else
  {
    /*search for the reported hits text */
    char *rep_hits;
    int prior_pres = 0;

    if ((rep_hits = strstr(p, "reported hits")) == NULL)
    {
      proz_debug("no reported hits found");
      return MIRPARSEFAIL;
    }

    /* Okay so we got the position after the results, lets see how many PRE tags were there before it */

    p1 = p;
    while (((p1 = strstr(p1, "<pre")) < rep_hits) && p1)
    {
      prior_pres++;
      p1 += 5;
    }
    /* now get the location of the PRE before the output */

    p1 = p;
    i = 0;
    while (prior_pres--)
    {
      p1 = strstr(p1, "<pre class=list>");
      p1 += 5;
    }
    i = p1 - 5;

    /*now find the </PRE>  tag which is after the results */
    j = strstr(i, "</pre>");

    if (j == NULL)
    {
      proz_debug("The expected </PRE> tag was not found!\n");
      return MIRPARSEFAIL;
    }
  }

  p1 = kmalloc((j - i - 16) + 100);
  strncpy(p1, i + 16, j - i - 16);

  proz_debug("\nstring len= %ld", strlen(p1));
  proz_debug("\nstring value= %s", p1);

  p1[j - i - 16 + 1] = 0;

  p2 = p1;

  while ((i = strstr(p1, "<a href=")) != NULL)
  {
    num_ah++;
    p1 = i + 8;
  }

  proz_debug("\n%d ahrefs found\n", num_ah);

  if (num_ah == 0)
  {
    *num_servers = 0;
    return MIRINFOK;
  }



  *num_servers = num_ah / 3;
  proz_debug("%d servers found\n", *num_servers);

  /* Allocate +1 because we need to add the user specified server as well
   */
  ftp_mirrors =
      (ftp_mirror_t *) kmalloc(sizeof(ftp_mirror_t) *
			       ((*num_servers) + 1));

  for (k = 0; k < *num_servers; k++)
  {
    memset(&(ftp_mirrors[k]), 0, sizeof(ftp_mirror_t));
    p2 = get_string_ahref(p2, buf);
    ftp_mirrors[k].server_name = kstrdup(buf);
    p2 = get_string_ahref(p2, buf);
    ftp_mirrors[k].paths = kmalloc(sizeof(mirror_path_t));

    //      ftp_mirrors[k].paths=kmalloc (sizeof (char *));
    ftp_mirrors[k].num_paths = 1;

    /*Strip any trailing slash */
    if(buf[strlen(buf)-1]=='/')
      buf[strlen(buf)-1]=0;

    /*Strip any leading slash in the path name if preent */
    if (*buf == '/')
      ftp_mirrors[k].paths[0].path = kstrdup(buf + 1);
    else
      ftp_mirrors[k].paths[0].path = kstrdup(buf);

    p2 = get_string_ahref(p2, buf);
    ftp_mirrors[k].file_name = kstrdup(buf);
  }

  /* add the users server to the end if it is a ftp server */
  if (request->requested_url->proto == URLFTP)
  {
    memset(&(ftp_mirrors[k]), 0, sizeof(ftp_mirror_t));
    ftp_mirrors[k].server_name = kstrdup(request->requested_url->host);

    ftp_mirrors[k].paths = kmalloc(sizeof(mirror_path_t));
    ftp_mirrors[k].num_paths = 1;

    if (*(request->requested_url->dir))
      ftp_mirrors[k].paths[0].path = kstrdup(request->requested_url->dir);
    else
      ftp_mirrors[k].paths[0].path = kstrdup("");

    ftp_mirrors[k].file_name = kstrdup(request->requested_url->file);
    *num_servers += 1;
  }

  proz_debug("%d servers found\n", *num_servers);

  for (k = 0; k < *num_servers; k++)
  {

    ftp_mirrors[k].full_name =
	(char *) kmalloc(strlen(ftp_mirrors[k].server_name) +
			 strlen(ftp_mirrors[k].paths[0].path) +
			 strlen(ftp_mirrors[k].file_name) + 13);
    sprintf(ftp_mirrors[k].full_name, "%s%s:21/%s%s%s", "ftp://",
	    ftp_mirrors[k].server_name, ftp_mirrors[k].paths[0].path, "/",
	    ftp_mirrors[k].file_name);