ftpsearch.c

prozgui是一款Linxu下著名的下载工具

    proz_debug("%s\n", ftp_mirrors[k].full_name);
  }


  *pmirrors = reprocess_mirror_list(ftp_mirrors, num_servers);
  /*    *pmirrors = ftp_mirrors; */
  return MIRINFOK;
}


uerr_t get_mirror_info(connection_t * connection, char **ret_buf)
{
  uerr_t err;
  int remote_port_len;
  char *user, *passwd, *www_auth = NULL, *proxy_auth = NULL, *location =
      NULL, *referer = NULL, *pragma_no_cache = NULL;
  char *request, *remote_port;
  netrc_entry *netrc_ent;
  char buffer[HTTP_BUFFER_SIZE];
  /*The http stats that were returned after the call with GET */
  http_stat_t hs_after_get;

  char *p, *p1, *p2;
  int p_len, ret, total;

  /* we want it to terminate immediately */
  pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);


  /*clear the socks */
  connection->data_sock = 0;
  memset(&hs_after_get, 0, sizeof(hs_after_get));


  connection_change_status(connection, CONNECTING);

  if (http_use_proxy(connection))
  {
    connection_show_message(connection, _("Connecting to %s"),
			    connection->http_proxy->proxy_url.host);
    err = connect_to_server(&connection->data_sock,
			    connection->http_proxy->proxy_url.host,
			    connection->http_proxy->proxy_url.port,
			    &connection->xfer_timeout);
    if (err != NOCONERROR)
    {
      proz_debug(_("Error connecting to %s"),
		 connection->http_proxy->proxy_url.host);
      connection_change_status(connection, REMOTEFATAL);
      return err;
    }
  } else
  {
    connection_show_message(connection, _("Connecting to %s"),
			    connection->u.host);

    err = connect_to_server(&connection->data_sock, connection->u.host,
			    connection->u.port, &connection->xfer_timeout);
    if (err != NOCONERROR)
    {
      proz_debug(_("Error connecting to %s"), connection->u.host);
      connection_change_status(connection, REMOTEFATAL);
      return err;
    }
  }

  user = connection->u.user;
  passwd = connection->u.passwd;

  /* Use .netrc if asked to do so. */
  if (connection->use_netrc == TRUE)
  {
    netrc_ent = search_netrc(libprozrtinfo.netrc_list, connection->u.host);

    if (netrc_ent != NULL)
    {
      user = netrc_ent->account;
      passwd = netrc_ent->password;
    }
  }

  user = user ? user : "";
  passwd = passwd ? passwd : "";

  if (strlen(user) || strlen(passwd))
  {
    /* Construct the necessary header. */
    www_auth = get_basic_auth_str(user, passwd, "Authorization");
    proz_debug(_("Authenticating as user %s password %s"), user, passwd);
    proz_debug(_("Authentification string=%s"), www_auth);
  } else
    www_auth = 0;

  if (http_use_proxy(connection))
  {
    if (strlen(connection->http_proxy->username)
	|| strlen(connection->http_proxy->passwd))
      proxy_auth =
	  get_basic_auth_str(connection->http_proxy->username,
			     connection->http_proxy->passwd,
			     "Proxy-Authorization");
  }

  if (connection->u.port == 80)
  {
    remote_port = NULL;
    remote_port_len = 0;
  } else
  {
    remote_port = (char *) alloca(64);
    remote_port_len = sprintf(remote_port, ":%d", connection->u.port);
  }

  if (connection->u.referer)
  {
    referer = (char *) alloca(13 + strlen(connection->u.referer));
    sprintf(referer, "Referer: %s\r\n", connection->u.referer);
  }

  /* If we go through a proxy the request for the URL is different */
  if (http_use_proxy(connection))
  {
    location = (char *) alloca(strlen(connection->u.url) + 1);
    strcpy(location, connection->u.url);
  } else
  {
    location = (char *) alloca(strlen(connection->u.path) + 1);
    strcpy(location, connection->u.path);
  }

  /*Use no-cache directive for proxy servers, yes by default here as we dont want ftpsearch rsults which can change soon to be cached */
  if (http_use_proxy(connection))
  {
    pragma_no_cache = (char *) alloca(21);
    sprintf(pragma_no_cache, "Pragma: no-cache\r\n");
  }

  request = (char *) alloca(strlen(location)
			    + strlen(connection->user_agent)
			    + strlen(connection->u.host) + remote_port_len
			    + (referer ? strlen(referer) : 0)
			    + (www_auth ? strlen(www_auth) : 0)
			    + (proxy_auth ? strlen(proxy_auth) : 0)
			    + 64
			    +
			    (pragma_no_cache ? strlen(pragma_no_cache) :
			     0));

  /* TODO Add referrer tag. */
  sprintf(request,
	  "GET %s HTTP/1.0\r\nUser-Agent: %s\r\nHost: %s%s\r\nAccept: */*\r\n%s%s%s%s\r\n",
	  location, connection->user_agent, connection->u.host,
	  remote_port ? remote_port : "",
	  referer ? referer : "",
	  www_auth ? www_auth : "", proxy_auth ? proxy_auth : "",
	  pragma_no_cache ? pragma_no_cache : "");

  proz_debug("HTTP request = %s", request);

  connection_show_message(connection, _("Sending HTTP request"));
  err = http_fetch_headers(connection, &hs_after_get, request);

  /* What hapenned ? */
  if (err != HOK)
  {
    /*Check if we authenticated using any user or password and if we 
       were kicked out, if so return HAUTHFAIL */
    if (err == HAUTHREQ && (strlen(user) || strlen(passwd)))
      err = HAUTHFAIL;
    /*
     * a error occured druing the process 
     */
    close_sock(&connection->data_sock);
    connection_change_status(connection, REMOTEFATAL);
    return err;
  }


  /* Ok start fetching the data */
  p1 = p = (char *) kmalloc(HTTP_BUFFER_SIZE + 1);
  p_len = HTTP_BUFFER_SIZE + 1;
  total = 0;


  do
  {

    ret =
	krecv(connection->data_sock, buffer, sizeof(buffer), 0,
	      &connection->xfer_timeout);
    if (ret > 0)
    {
      p2 = grow_buffer(p, p1, &p_len, ret);
      memcpy(p2 + (p1 - p), buffer, ret);
      p1 = (p1 - p) + ret + p2;
      p = p2;
    }
    total += ret;
  }
  while (ret > 0);

  if (ret == -1)
  {
    if (errno == ETIMEDOUT)
    {
      close(connection->data_sock);
      return READERR;
    }
    close(connection->data_sock);
    return READERR;
  }

  p[total] = 0;
  *ret_buf = p;


  close_sock(&connection->data_sock);
  return HOK;
}



char *find_ahref(char *buf)
{

  return (strcasestr(buf, "<A HREF="));
}

char *find_end(char *buf)
{
  return (strcasestr(buf, ">"));
}

char *find_closed_a(char *buf)
{
  return (strcasestr(buf, "</A"));
}

char *get_string_ahref(char *buf, char *out)
{
  char *p1, *p2, *p3;

  p1 = find_ahref(buf);
  assert(p1 != NULL);

  p2 = find_end(p1);
  assert(p2 != NULL);

  p3 = find_closed_a(p2);
  assert(p3 != NULL);

  strncpy(out, p2 + 1, p3 - p2 - 1);
  out[p3 - p2 - 1] = 0;
  return p3;
}



char *grow_buffer(char *buf_start, char *cur_pos, int *buf_len,
		  int data_len)
{
  const int INIT_SIZE = 4048;
  int bytes_left;
  char *p;

  /* find how many bytes are left */
  bytes_left = *buf_len - (cur_pos - buf_start);
  assert(bytes_left >= 0);
  assert(data_len <= INIT_SIZE);

  if (bytes_left < data_len + 1)
  {
    /* time to realloc the buffer buffer */
    p = krealloc(buf_start, *buf_len + INIT_SIZE);
    *buf_len += INIT_SIZE;
    return p;
  } else
  {
    return buf_start;
  }
}



void proz_get_complete_mirror_list(ftps_request_t * request)
{
  request->info_running = TRUE;
  /*      get_complete_mirror_list(request); */

  if (pthread_create(&request->info_thread, NULL,
		     (void *) &get_complete_mirror_list,
		     (void *) request) != 0)
    proz_die(_("Error: Not enough system resources"));



}


void proz_cancel_mirror_list_request(ftps_request_t * request)
{
 request->info_running = FALSE;
 pthread_cancel(request->info_thread);
 pthread_join(request->info_thread,0);
}

uerr_t get_complete_mirror_list(ftps_request_t * request)
{
  char *data_buf;

  pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);

  do
  {
    pthread_mutex_lock(&request->access_mutex);
    request->info_running = TRUE;
    pthread_mutex_unlock(&request->access_mutex);

    pthread_cleanup_push(cleanup_socks, (void *) request->connection);
    request->err = get_mirror_info(request->connection, &data_buf);
    pthread_cleanup_pop(0);

    if (request->err == NEWLOCATION)
    {
      char *constructed_newloc;
      /*DONE : handle relative urls too */
      constructed_newloc =
	  uri_merge(request->connection->u.url,
		    request->connection->hs.newloc);

      proz_debug("Redirected to %s, merged URL = %s",
		 request->connection->hs.newloc, constructed_newloc);

      proz_free_url(&request->connection->u, 0);
      request->err =
	  proz_parse_url(constructed_newloc, &request->connection->u, 0);


      if (request->err != URLOK)
      {
	connection_show_message(request->connection,
				_
				("The server returned location is wrong: %s!"),
				constructed_newloc);
	pthread_mutex_lock(&request->connection->access_mutex);
	request->info_running = FALSE;
	pthread_mutex_unlock(&request->connection->access_mutex);
	kfree(constructed_newloc);
	pthread_mutex_lock(&request->access_mutex);
	request->info_running = FALSE;
	pthread_mutex_unlock(&request->access_mutex);
	return (request->err = HERR);
      } else
	connection_show_message(request->connection,
				_("Redirected to => %s"),
				constructed_newloc);

      kfree(constructed_newloc);
      request->err = NEWLOCATION;
    }
  }
  while (request->err == NEWLOCATION);

  /*TODO handle and process the redirection here */
  if (request->err != HOK)
  {
    pthread_mutex_lock(&request->access_mutex);
    request->info_running = FALSE;
    pthread_mutex_unlock(&request->access_mutex);
    return request->err;
  }

  request->err = parse_html_mirror_list(request, data_buf);
  /*TODO  see if can give further info */
  pthread_mutex_lock(&request->access_mutex);
  request->info_running = FALSE;
  pthread_mutex_unlock(&request->access_mutex);
  return request->err;

}


boolean proz_request_info_running(ftps_request_t * request)
{
  boolean ret;
  pthread_mutex_lock(&request->access_mutex);
  ret = request->info_running;
  pthread_mutex_unlock(&request->access_mutex);
  return ret;
}

boolean proz_request_mass_ping_running(ftps_request_t * request)
{
  boolean ret;
  pthread_mutex_lock(&request->access_mutex);
  ret = request->mass_ping_running;
  pthread_mutex_unlock(&request->access_mutex);
  return ret;
}



ftp_mirror_t *reprocess_mirror_list(ftp_mirror_t * mirrors,
				    int *num_servers)
{

  ftp_mirror_t *ftp_mirrors;
  int i, j;
  int num_new_servers = 0;

  ftp_mirrors =
      (ftp_mirror_t *) kmalloc(sizeof(ftp_mirror_t) * ((*num_servers)));

  for (i = 0; i < *num_servers; i++)
  {
    if (mirrors[i].copied != 1)
    {
      num_new_servers++;
      memset(ftp_mirrors + num_new_servers - 1, 0, sizeof(ftp_mirror_t));
      memcpy(ftp_mirrors + num_new_servers - 1, mirrors + i,
	     sizeof(ftp_mirror_t));

      /*For the moment assume that all the mirrors support resume */
      ftp_mirrors[num_new_servers - 1].resume_supported=TRUE;

      for (j = i + 1; j < *num_servers; j++)
      {
	if ((strcasecmp
	     (mirrors[i].server_name,
	      mirrors[j].server_name) == 0) && mirrors[j].copied != 1)
	{
	  /*found a  match */
	  ftp_mirrors[num_new_servers - 1].num_paths++;
	  ftp_mirrors[num_new_servers - 1].paths =
	      krealloc(ftp_mirrors[num_new_servers - 1].paths,
		       (sizeof(mirror_path_t) *
			ftp_mirrors[num_new_servers - 1].num_paths));

	  //              ftp_mirrors[num_new_servers-1].paths = krealloc(ftp_mirrors[num_new_servers-1].paths,ftp_mirrors[num_new_servers-1].num_paths );

	  ftp_mirrors[num_new_servers -
		      1].paths[ftp_mirrors[num_new_servers - 1].num_paths -
			       1].path = strdup(mirrors[j].paths[0].path);

	  ftp_mirrors[num_new_servers -
		      1].paths[ftp_mirrors[num_new_servers - 1].num_paths -
			       1].valid = TRUE;

	  mirrors[j].copied = 1;
	}
      }
    }
  }

  *num_servers = num_new_servers;


  proz_debug("Displaying the reparsed list \n");
  for (i = 0; i < num_new_servers; i++)
  {
    proz_debug("%s\n", ftp_mirrors[i].full_name);
    for (j = 0; j < ftp_mirrors[i].num_paths; j++)
      proz_debug("\t%s\n", ftp_mirrors[i].paths[j].path);
  }
  proz_debug("End display reparsed list\n");
  /*TODO free the mirros struct which we will not use now */
  return ftp_mirrors;
}



/*fixme do something about this, move to a better file rather than main.c  */

int compare_two_servers(const void *a, const void *b)
{
  const ftp_mirror_t *ma = (const ftp_mirror_t *) a;
  const ftp_mirror_t *mb = (const ftp_mirror_t *) b;

  int milli_sec_a;
  int milli_sec_b;

  if (ma->status != RESPONSEOK && (mb->status != RESPONSEOK))
    return 1000000;


  milli_sec_a = ma->milli_secs;

  if (ma->status != RESPONSEOK)
  {
    milli_sec_a = 1000000;
  }


  milli_sec_b = mb->milli_secs;

  if (mb->status != RESPONSEOK)
  {
    milli_sec_b = 1000000;
  }


  return (milli_sec_a - milli_sec_b);
}


void proz_sort_mirror_list(ftp_mirror_t * mirrors, int num_servers)
{
  int i;
  qsort(mirrors, num_servers, sizeof(ftp_mirror_t), compare_two_servers);
  for (i = 0; i < num_servers; i++)
    proz_debug("Mirror = %s, time =%d", mirrors[i].server_name,
	       mirrors[i].milli_secs);
}




int ftpsearch_get_server_position(ftps_request_t * request, char *server)
{
  int i;
  for (i = 0; i < request->num_mirrors; i++)
  {

    if (strcmp(request->mirrors[i].server_name, server) == 0)
      return i;
  }
  return -1;
}


int ftpsearch_get_path_position(ftps_request_t * request, char *server,
				char *path)
{
  int i, pos;

  pos = ftpsearch_get_server_position(request, server);
  assert(pos != -1);

  proz_debug("num avail paths %d", request->mirrors[pos].num_paths);

  for (i = 0; i < request->mirrors[pos].num_paths; i++)
  {
    proz_debug("avail path is %s", request->mirrors[pos].paths[i].path);
    proz_debug("path to check is %s", path);
    if (strcmp(request->mirrors[pos].paths[i].path, path) == 0)
      return i;

  }
  return -1;

}