url.c

网络爬虫程序

  case URLT_FTP:
  case URLT_FTPS:
    dst.p.ftp.host = tl_strdup(src->p.ftp.host);
    dst.p.ftp.user = tl_strdup(src->p.ftp.user);
    dst.p.ftp.password = tl_strdup(src->p.ftp.password);
    dst.p.ftp.path = tl_strdup(src->p.ftp.path);
    dst.p.ftp.anchor_name = tl_strdup(src->p.ftp.anchor_name);
    dst.p.ftp.port = src->p.ftp.port;
    dst.p.ftp.dir = src->p.ftp.dir;
    if(src->extension)
      dst.extension = ftp_url_ext_dup(src->extension);
    break;
  case URLT_HTTP:
  case URLT_HTTPS:
    dst.p.http.host = tl_strdup(src->p.http.host);
    dst.p.http.port = src->p.http.port;
    dst.p.http.document = tl_strdup(src->p.http.document);
    dst.p.http.searchstr = tl_strdup(src->p.http.searchstr);
    dst.p.http.anchor_name = tl_strdup(src->p.http.anchor_name);
    dst.p.http.user = tl_strdup(src->p.http.user);
    dst.p.http.password = tl_strdup(src->p.http.password);
    if(src->extension && (src->status & URL_FORM_ACTION))
      dst.extension = form_info_dup(src->extension);
    break;
  case URLT_GOPHER:
    dst.p.gopher.host = tl_strdup(src->p.gopher.host);
    dst.p.gopher.port = src->p.gopher.port;
    dst.p.gopher.selector = tl_strdup(src->p.gopher.selector);
    break;
  case URLT_FROMPARENT:        /* This is a 'can't happen'. */
    assert(0);
  case URLT_UNKNOWN:
    dst.p.unsup.urlstr = tl_strdup(src->p.unsup.urlstr);
    break;
  }

  return new_url(&dst);
}

/* convert any URL string to absolute path */
char *url_to_absolute_url(char *base, char *baset, url * parent, char *act)
{
  char *psp = NULL;
  url *purl;
  char *pom;
  int pomlen;

  if(act[0] == 0)
    return 0;
  if(act[0] == '#')
    return 0;

  pomlen = strlen(url_to_filename(parent, TRUE)) + strlen(priv_cfg.cache_dir)
  + strlen(baset) + strlen(act);
  pom = _malloc(pomlen);

  if((act[0] == '/' && act[1] == '/') && parent->type != URLT_FILE)
  {
    /* we should handle it like net_path */
    snprintf(pom, pomlen, "%s:%s", prottable[parent->type].urlid, act);
    psp = tl_strdup(pom);
    purl = url_parse(act);
  }
  else
  {
    purl = url_parse(act);
  }

  if(purl->type == URLT_FROMPARENT)
  {
    purl->type = parent->type;
    url_finishpath(purl);
  }
  assert(purl->type != URLT_FROMPARENT);

  if(purl->type == URLT_FILE && (parent->type == URLT_FILE))
  {
    if(!(*purl->p.file.filename))
    {
      strcpy(pom, baset);
    }
    else
    {
      if(*(purl->p.file.filename) != '/')
      {
        strcpy(pom, base);
        strcat(pom, purl->p.file.filename);

        free(purl->p.file.filename);
        purl->p.file.filename = tl_strdup(pom);
      }
      else
        snprintf(pom, pomlen, "%s%s",
          prottable[purl->type].typestr, purl->p.file.filename);
    }
    psp = tl_strdup(pom);
  }
  else if((purl->type == URLT_FILE) &&
    (cfg.base_level == 0 || cfg.enable_info) &&
    (parent->status & URL_REDIRECT || parent->status & URL_ISLOCAL))
  {
    char *p1, *p;
    url *pomurl;

    if(*purl->p.file.filename == '/')
      strcpy(pom, purl->p.file.filename);
    else
    {
      int l;
      p = url_to_filename(parent, TRUE);
      strcpy(pom, p);
      if(*purl->p.file.filename)
      {
        p1 = strrchr(pom, '/');
        if(p1)
          *(p1 + 1) = '\0';
        strcat(pom, purl->p.file.filename);
      }
      /* remove any dynamic stuff to get base name */
      for(l = strlen(pom); l > 0 && pom[l] != '/' && pom[l] != '?'; --l)
        ;
      if(pom[l] == '?')
        pom[l] = '\0';
      /* now fix for index-name files */
      l = strlen(pom)-strlen(priv_cfg.index_name);
      if(l > 0 && !strcmp(pom+l, priv_cfg.index_name) && pom[l-1] == '/')
        pom[l] = '\0';
    }
    if(purl->p.file.searchstr)
    {
      strcat(pom, "?");
      strcat(pom, purl->p.file.searchstr);
    }

    if(purl->p.file.anchor_name)
    {
      strcat(pom, "#");
      strcat(pom, purl->p.file.anchor_name);
    }

    p = get_abs_file_path(pom);
    pomurl = filename_to_url(p);
    _free(p);
    if(pomurl)
    {
      psp = url_to_urlstr(pomurl, TRUE);
      free_deep_url(pomurl);
      _free(pomurl);
    }
  }
  if((!psp && purl->type == URLT_FILE) &&
    (parent->type == URLT_HTTP ||
      parent->type == URLT_HTTPS ||
      parent->type == URLT_FTPS || parent->type == URLT_FTP))
  {
    char *ri;
    if(*(purl->p.file.filename) == '/')
    {
      char *idx;

      strcpy(pom, base);
      idx = strfindnchr(pom, '/', 3);
      if(idx)
        strcpy(idx - 1, purl->p.file.filename);
      else
        strcat(pom, purl->p.file.filename);

      if(purl->p.file.searchstr)
      {
        strcat(pom, "?");
        strcat(pom, purl->p.file.searchstr);
      }

      if(purl->p.file.anchor_name)
      {
        strcat(pom, "#");
        strcat(pom, purl->p.file.anchor_name);
      }
    }
    else if(!(*purl->p.file.filename) && !purl->p.file.searchstr)
    {
      if(purl->p.file.anchor_name)
      {
        /* Problem; we just have "#anchor" and unfortunately
           baset might be the parent directory, not the
           actual parent. (Nor is that found in "parent"
           necessarily).
         */

        if(*baset && baset[strlen(baset) - 1] != '/')
        {
          strcpy(pom, baset);
          strcat(pom, "#");
          strcat(pom, purl->p.file.anchor_name);
        }
        else                    /* What to do? Just hope to ignore this altogeher */
          strcpy(pom, "");

      }
    }
    else
    {
      strcpy(pom, base);
      if(!*purl->p.file.filename && purl->p.file.searchstr && parent->type == URLT_HTTP)
      {
        ri = strrchr(pom, '/');
        if(ri)
          strcpy(ri, parent->p.http.document);
        else
          strcat(pom, parent->p.http.document);
      }
      else
      {
        ri = strrchr(pom, '/');
        if(ri)
          strcpy(ri + 1, purl->p.file.filename);
        else
          strcat(pom, purl->p.file.filename);
      }

      if((parent->status & URL_REDIRECT) &&
        (strlen(purl->p.file.filename) >= strlen(priv_cfg.index_name)) &&
        !strcmp(priv_cfg.index_name,
          purl->p.file.filename + strlen(purl->p.file.filename) -
          strlen(priv_cfg.index_name)))
      {
        *(pom + strlen(pom) - strlen(priv_cfg.index_name)) = '\0';
      }

      if(purl->p.file.searchstr)
      {
        strcat(pom, "?");
        strcat(pom, purl->p.file.searchstr);
      }

      if(purl->p.file.anchor_name)
      {
        strcat(pom, "#");
        strcat(pom, purl->p.file.anchor_name);
      }
    }
    psp = tl_strdup(pom);
  }
  else if(!psp)
  {
    psp = tl_strdup(act);
  }

  free_deep_url(purl);
  _free(purl);

  if(psp && *psp)
  {
    purl = url_parse(psp);
    if(purl->type == URLT_FROMPARENT)
    {
      purl->type = parent->type;
      url_finishpath(purl);
    }
    url_path_abs(purl);
    if(prottable[purl->type].supported)
    {
      free(psp);
      psp = url_to_urlstr(purl, TRUE);
    }
    free_deep_url(purl);
    _free(purl);
  }

  _free(pom);

  return psp;
}

/**************************************/
/* encode unsafe characters with      */
/* url-encoded encoding               */
/**************************************/
static char *url_encode_str_real(char *urlstr, char *unsafe, int safety)
{
  char *res, *p, *r;

  if(urlstr == NULL)
    return NULL;

  if(cfg.noencode)
  {
    return strdup(urlstr);
  }

  res = _malloc(strlen(urlstr) * 3 + 1);

  for(p = urlstr, r = res; *p; p++, r++)
  {
    if(safety && *p == '%' && tl_ascii_isxdigit(p[1]) &&
      tl_ascii_isxdigit(p[2]))
    {
      *r = *p;
    }
    else if(strchr(unsafe, *p) ||
      ((unsigned char) *p > 0x7f) || ((unsigned char) *p < 0x20))
    {
      *r = '%';
      r++;
      *r = hexa[((unsigned char)*p) >> 4];
      r++;
      *r = hexa[((unsigned char)*p) % 16];
    }
    else
    {
      *r = *p;
    }
  }
  *r = '\0';

  return res;
}

char *url_encode_str(char *urlstr, char *unsafe)
{
  return url_encode_str_real(urlstr, unsafe, FALSE);
}

static char *url_encode_str_safe(char *urlstr, char *unsafe)
{
  return url_encode_str_real(urlstr, unsafe, TRUE);
}

/* Convert the HTML entities to direct characters, size is ignored at the
moment, res returns the encoded character, the return value is the number
of encoded bytes. Currently only &amp; is handled! */
static int fix_html_entity(const char *str, int size, char *res)
{
  if(size >= 5 && (!strncmp(str, "&amp;", 5) || !strncmp(str, "&#38;", 5)))
  {
    *res = '&';
    return 5;
  }
  return 0;
}

/*****************************************/
/* dekodovanie zakodovanych znakov z URL */
/* FIXME: Translate me                   */
/*****************************************/
char *url_decode_str(const char *urlstr, int len)
{
  char *res, *r;
  int i;

  if(urlstr == NULL)
    return NULL;

  res = tl_strndup(urlstr, len);

  for(i = 0, r = res; i < len; r++, i++)
  {
    if(urlstr[i] == '%' && urlstr[i + 1] && urlstr[i + 2] &&
      tl_ascii_isxdigit(urlstr[i + 1]) && tl_ascii_isxdigit(urlstr[i + 2]))
    {
      *r = HEX2CHAR(urlstr + i);
      i += 2;
    }
    else if(urlstr[i] == '&')
    {
      int s;
      if((s = fix_html_entity(urlstr+i, len-i, r)))
        i += s-1;
      else
        *r = urlstr[i]; /* copy the & */
    }
    else
    {
      *r = urlstr[i];
    }
  }
  *r = '\0';

  return res;
}

static char *url_decode_html(const char *urlstr, int len)
{
  char *res, *r;
  int i;

  if(urlstr == NULL)
    return NULL;

  res = tl_strndup(urlstr, len);

  for(i = 0, r = res; i < len; r++, i++)
  {
    if(urlstr[i] == '&')
    {
      int s;
      if((s = fix_html_entity(urlstr+i, len-i, r)))
        i += s-1;
      else
        *r = urlstr[i]; /* copy the & */
    }
    else
    {
      *r = urlstr[i];
    }
  }
  *r = '\0';

  return res;
}


/*************************************/
/* uvolnenie pamate po strukture URL */
/* FIXME: Translate me!              */
/*************************************/
void free_deep_url(url * urlp)
{
  if(urlp->local_name)
  {
    url_remove_from_file_hash_tab(urlp);
    _free(urlp->local_name);
  }

  switch (urlp->type)
  {
  case URLT_FILE:
    _free(urlp->p.file.filename);
    _free(urlp->p.file.searchstr);
    _free(urlp->p.file.anchor_name);
    break;
  case URLT_HTTP:
  case URLT_HTTPS:
    _free(urlp->p.http.host);
    _free(urlp->p.http.document);
    _free(urlp->p.http.searchstr);
    _free(urlp->p.http.anchor_name);
    _free(urlp->p.http.password);
    _free(urlp->p.http.user);
    if(urlp->status & URL_FORM_ACTION)
    {
      form_info *fi = (form_info *) urlp->extension;
      dllist *ptr;

      _free(fi->text);
      _free(fi->action);
      ptr = fi->infos;
      while(ptr)
      {
        form_field *ff = (form_field *) ptr->data;

        _free(ff->value);
        _free(ff->name);
        _free(ff);

        ptr = dllist_remove_entry(ptr, ptr);
      }
    }
    break;
  case URLT_FTP:
  case URLT_FTPS:
    _free(urlp->p.ftp.host);
    _free(urlp->p.ftp.user);
    _free(urlp->p.ftp.password);
    _free(urlp->p.ftp.anchor_name);
    _free(urlp->p.ftp.path);
    if(urlp->extension)
      ftp_url_ext_free(urlp->extension);
    break;
  case URLT_GOPHER:
    _free(urlp->p.gopher.host);
    _free(urlp->p.gopher.selector);
  case URLT_FROMPARENT:
  default:
    _free(urlp->p.unsup.urlstr);
    break;
  }

  dllist_free_all(urlp->parent_url);


#ifdef WITH_TREE
#ifdef I_FACE
  _free(urlp->tree_nfo);

  if(urlp->prop)
  {
    _free(urlp->prop->type);
    free(urlp->prop);
  }

#endif
#endif

#ifdef HAVE_MT
  pthread_mutex_destroy(&urlp->lock);
#endif
}

void cat_links_to_url_list(dllist * l1)
{
  dllist *p = l1;
  url *same;
  dllist *reg = NULL, *inl = NULL;
  int nadd = 0;
  cond_info_t condp;

  condp.level = 1;
  condp.urlnr = 0;
  condp.size = 0;
  condp.time = 0L;
  condp.mimet = NULL;
  condp.full_tag = NULL;
  condp.params = NULL;
  condp.html_doc = NULL;
  condp.html_doc_offset = 0;
  condp.tag = NULL;
  condp.attrib = NULL;

  while(p)
  {
    if(url_append_condition((url *) p->data, &condp))
    {
      url_clear_anchor((url *) p->data);
      if((same = url_was_befor((url *) p->data)))
      {
        link_url_in_list(same, (url *) p->data);
        free_deep_url((url *) p->data);
        free((url *)p->data);
      }
      else
      {
        url *urlp = (url *) p->data;

        nadd++;
        LOCK_TCNT;
        cfg.total_cnt++;
        UNLOCK_TCNT;

        urlp->ref_cnt = 1;

#ifdef WITH_TREE
#ifdef I_FACE
        if(cfg.xi_face)
        {
          urlp->tree_nfo = _malloc(sizeof(GUI_TREE_RTYPE));
          urlp->tree_nfo[0] = gui_tree_make_entry(urlp);
        }
#endif
#endif

        url_add_to_url_hash_tab(urlp);

        switch (cfg.scheduling_strategie)
        {
        case SSTRAT_DO_SIRKY:
        case SSTRAT_DO_HLBKY:
          reg = dllist_append(reg, (dllist_t)p->data);
          break;
        case SSTRAT_DO_SIRKY_I:
        case SSTRAT_DO_HLBKY_I:
          if(urlp->status & URL_INLINE_OBJ)