ftp.c

来自「网络爬虫程序」· C语言 代码 · 共 2,200 行 · 第 1/4 页

  if(!docp->ftp_control)
  {
    if(priv_cfg.ftp_proxy && cfg.ftp_dirtyp)
    {
      if(!(docp->datasock =
          bufio_sock_fdopen(net_connect(priv_cfg.ftp_proxy,
              cfg.ftp_proxy_port, docp))))
      {
        if(_h_errno_ != 0)
          xherror(priv_cfg.ftp_proxy);
        else
          xperror("net_connect");

        docp->errcode = ERR_PROXY_CONNECT;
        return NULL;
      }

      if(http_dumy_proxy_connect(docp, host, port, priv_cfg.ftp_proxy,
          cfg.ftp_proxy_port))
      {
        docp->errcode = ERR_PROXY_CONNECT;
        return NULL;
      }
      docp->ftp_control = docp->datasock;
      docp->datasock = NULL;
    }
    else
    {
      if(!(docp->ftp_control =
          bufio_sock_fdopen(net_connect(host, port, docp))))
      {
        if(_h_errno_ != 0)
          xherror(host);
        else
          xperror("net_connect");

        docp->errcode = ERR_FTP_CONNECT;
        docp->ftp_fatal_err = TRUE;
        return NULL;
      }
    }
    if(ftp_get_response(docp, NULL, TRUE) >= 400)
    {
      CLOSE_CONTROL;
      docp->ftp_fatal_err = TRUE;
      docp->errcode = ERR_FTP_CONNECT;
      return NULL;
    }
#ifdef USE_SSL
    if(docp->doc_url->type == URLT_FTPS)
    {
      char buf[2048];
      int reply;
      bufio *ssl_sock;

      sprintf(buf, "AUTH SSL\r\n");
      ftp_control_write(docp, buf, strlen(buf));
      DEBUG_PROTOC("%s", buf);

      reply = ftp_get_response(docp, NULL, TRUE);

      /*** 234 code is used by SafeGossip ***/
      /*** thank to Martijn ...           ***/
      if(reply != 334 && reply != 234)
      {
        CLOSE_CONTROL;
        docp->ftp_fatal_err = (reply == 600);
        docp->errcode = ERR_FTPS_UNSUPORTED;
        return NULL;
      }

      ssl_sock = my_ssl_do_connect(docp, docp->ftp_control, NULL);

      if(!ssl_sock)
      {
        if(!docp->errcode)
          docp->errcode = ERR_FTPS_CONNECT;
        CLOSE_CONTROL;
        return NULL;
      }
      else
      {
        docp->ftp_control = ssl_sock;
      }
    }
#endif

    if(ftp_do_login_handshake(docp, ruser, user, password))
    {
      CLOSE_CONTROL;
      return NULL;
    }

#ifdef IP_TOS
#ifdef IPTOS_LOWDELAY
    if(docp->ftp_control)
    {
      int v = IPTOS_LOWDELAY;
      if(setsockopt(bufio_getfd(docp->ftp_control),
          IPPROTO_IP, IP_TOS, (char *) &v, sizeof(v)))
      {
        perror("ftp: setsockopt TOS - LOWDELAY failed");
      }
    }
#endif
#endif

  }
#ifdef DEBUG
  else
  {
    if(cfg.debug)
      xprintf(1, gettext("Re-using established FTP control connection\n"));
  }
#endif

  return docp->ftp_control;
}

static int ftp_set_transfer_type(doc * docp, char *type)
{
  char buf[20];
  int reply;

  snprintf(buf, sizeof(buf), "TYPE %s\r\n", type);
  ftp_control_write(docp, buf, strlen(buf));
  DEBUG_PROTOC("%s", buf);

  if((reply = ftp_get_response(docp, NULL, TRUE)) >= 400)
  {
    CLOSE_CONTROL;
    docp->ftp_fatal_err = (reply == 600);
    docp->errcode = ERR_FTP_UNKNOWN;
    return -1;
  }

  return 0;
}

static int ftp_send_list_command(doc * docp, char *filename)
{
  char buf[2148];
  int l;

  l = strlen(filename) - 1;
  if(l)
  {
    if(tl_is_dirname(filename + 1) && strcmp(filename, "//"))
      l--;
  }

  if(cfg.ftp_list)
    strcpy(buf, "LIST");
  else
    strcpy(buf, "NLST");

  if(cfg.ftp_list_options)
  {
    strcat(buf, " ");
    strncat(buf, cfg.ftp_list_options, sizeof(buf) - strlen(buf));
    buf[sizeof(buf) - 1] = '\0';
  }

  if(l)
  {
    strncat(buf, " ", sizeof(buf) - strlen(buf));
    buf[sizeof(buf) - 1] = '\0';
    strncat(buf, filename + 1, sizeof(buf) - strlen(buf));
    buf[sizeof(buf) - 1] = '\0';
  }

  strncat(buf, "\r\n", sizeof(buf) - strlen(buf));
  buf[sizeof(buf) - 1] = '\0';

  ftp_control_write(docp, buf, strlen(buf));
  DEBUG_PROTOC("%s", buf);

  return 0;
}

static int ftp_check_directory_existence(doc * docp, char *filename)
{
  int rv = 0;

  if(cfg.ftp_list && cfg.fix_wuftpd)
  {
    int l;
    char buf[2148];
    char *mbuf;

    l = strlen(filename) - 1;
    if(l)
    {
      if(tl_is_dirname(filename + 1) && strcmp(filename, "//"))
        l--;
    }

    strcpy(buf, "STAT -d1 ");
    if(l)
    {
      strncat(buf, filename + 1, sizeof(buf) - strlen(buf));
      buf[sizeof(buf) - 1] = '\0';
      strncat(buf, "\r\n", sizeof(buf) - strlen(buf));
      buf[sizeof(buf) - 1] = '\0';
    }
    else
    {
      /* we can assume that login directory exists */
      return 0;
    }

    ftp_control_write(docp, buf, strlen(buf));
    DEBUG_PROTOC("%s", buf);

    mbuf = NULL;
    if(ftp_get_response(docp, &mbuf, TRUE) == 213)
    {
      char *p;

      p = mbuf + strcspn(mbuf, "\r\n");
      p += strspn(p, "\r\n");
      if(strncmp(filename + 1, p, strcspn(p, "\r\n")))
        rv = -1;
    }

    _free(mbuf);
  }

  return rv;
}

/****************************************************************/
/* otvorenie spojenia na FTP server, prihlasenie + zahajenie    */
/* prenosu suboru / adresara                                    */
/* FIXME: Translate me!                                         */
/****************************************************************/
static bufio *ftp_connect(doc * docp, char *host, int port, char *ruser,
  char *user, char *password, char *filename, bool_t * is_dir)
{
  char buf[2148];
  int reply;
  char *mbuf;

  docp->errcode = ERR_NOERROR;
  docp->ftp_fatal_err = FALSE;
  docp->ftp_respc = 0;

  if(!(docp->ftp_control =
      ftp_open_control_connection(docp, host, port, ruser, user, password)))
    return NULL;

  if(!tl_is_dirname(filename) && cfg.mode != MODE_FTPDIR)
  {
    if(docp->doc_url->extension &&
      ((ftp_url_extension *) docp->doc_url->extension)->size > 0)
    {
      docp->totsz = ((ftp_url_extension *) docp->doc_url->extension)->size;
      reply = 213;
      DEBUG_PROTOC("taking stored size\n");
    }
    else
    {
      snprintf(buf, sizeof(buf), "SIZE %s\r\n", filename + 1);
      ftp_control_write(docp, buf, strlen(buf));
      DEBUG_PROTOC("%s", buf);


      mbuf = NULL;
      if((reply = ftp_get_response(docp, &mbuf, TRUE)) == 213)
      {
        int sz;

        sscanf(mbuf, "%d %d", &reply, &sz);
        docp->totsz = sz;
      }
      else
      {
        if(reply == 600)
        {
          docp->errcode = ERR_READ;
          docp->ftp_fatal_err = TRUE;
          _free(mbuf);
          return NULL;
        }
      }

      _free(mbuf);
    }

    if(reply != 550)  /*** file doesn't exist or not plain file  == 550 ***/
    {
      if(!cfg.always_mdtm &&
        docp->doc_url->extension &&
        ((ftp_url_extension *) docp->doc_url->extension)->time != 0)
      {
        docp->dtime = ((ftp_url_extension *) docp->doc_url->extension)->time;
        reply = 213;
        DEBUG_PROTOC("taking stored MDTM %d\n", (int)
          ((ftp_url_extension *) docp->doc_url->extension)->time);
      }
      else
      {
        snprintf(buf, sizeof(buf), "MDTM %s\r\n", filename + 1);
        ftp_control_write(docp, buf, strlen(buf));
        DEBUG_PROTOC("%s", buf);

        mbuf = NULL;
        if((reply = ftp_get_response(docp, &mbuf, TRUE)) == 213)
        {
          sscanf(mbuf, "%d %2147s", &reply, buf);
          if(strlen(buf) == 14)
          {
            docp->dtime = time_ftp_scn(buf);
          }
        }
        else
        {
          if(reply == 600)
          {
            docp->ftp_fatal_err = TRUE;
            docp->errcode = ERR_READ;
            _free(mbuf);
            return NULL;
          }
        }
        _free(mbuf);
      }

      DEBUG_PROTOC("Size: file %d, remote %d; time: file %d, remote %d\n",
        docp->origsize, docp->totsz, docp->origtime, docp->dtime);

      if(docp->origsize && (docp->totsz >= 0)
        && (docp->origsize != docp->totsz))
      {
        xprintf(1, gettext("Size differs, regeting whole\n"));
        docp->rest_pos = 0;
      }
      else if(docp->origtime && docp->dtime)
      {
        if((difftime(docp->dtime, docp->origtime) > 0))
        {
          if(docp->rest_pos)
          {
            docp->rest_pos = 0;
            xprintf(1,
              gettext("Modified from last download - regeting whole\n"));
          }
        }
        else if((cfg.mode == MODE_SYNC || cfg.mode == MODE_MIRROR)
          && !docp->rest_pos)
        {
          docp->errcode = ERR_FTP_ACTUAL;
          return NULL;
        }
      }

      if(docp->check_limits)
      {
        cond_info_t condp;
        int r;

        condp.level = 3;
        condp.urlnr = docp->doc_nr;
        condp.size = docp->totsz;
        condp.time = docp->dtime;
        condp.mimet = NULL;
        condp.full_tag = NULL;
        condp.params = NULL;
        condp.html_doc = NULL;
        condp.html_doc_offset = 0;
        condp.tag = NULL;
        condp.attrib = NULL;

        r = url_append_condition(docp->doc_url, &condp);

        if(!r)
        {
          switch (condp.reason)
          {
          case CONDT_MAX_SIZE:
            docp->errcode = ERR_BIGGER;
            break;
          case CONDT_MIN_SIZE:
            docp->errcode = ERR_SMALLER;
            break;
          case CONDT_NEWER_THAN:
          case CONDT_OLDER_THAN:
            docp->errcode = ERR_OUTTIME;
            break;
          case CONDT_USER_CONDITION:
            docp->errcode = ERR_SCRIPT_DISABLED;
            break;
          default:
            docp->errcode = ERR_RDISABLED;
            break;
          }
          return NULL;
        }
      }

      if(ftp_set_transfer_type(docp, "I"))
      {
        docp->ftp_fatal_err = TRUE;
        docp->errcode = ERR_FTP_DATACON;
        return NULL;
      }

      if(ftp_open_data_init(docp))
      {
        bufio_close(docp->ftp_control);
        docp->ftp_control = NULL;
        docp->errcode = ERR_FTP_DATACON;
        return NULL;
      }

      if(docp->rest_pos)
      {
        char *rsp;

        sprintf(buf, "REST %ld\r\n", (long) docp->rest_pos);
        ftp_control_write(docp, buf, strlen(buf));
        DEBUG_PROTOC("%s", buf);
        if((reply = ftp_get_response(docp, &rsp, TRUE)) >= 400)
        {
          _free(rsp);
          if(reply == 600)
          {
            docp->ftp_fatal_err = TRUE;
            bufio_close(docp->datasock);
            docp->datasock = NULL;
            docp->errcode = ERR_READ;
            return NULL;
          }
          docp->errcode = ERR_FTP_NOREGET;
        }
        else
        {
          sprintf(buf, "%ld", (long) docp->rest_pos);
          if(!strstr(rsp + 3, buf))
          {
            xprintf(1,
              gettext
              ("Warning: FTP server understands REST command, but can't handle it properly.\n"));
            docp->errcode = ERR_FTP_NOREGET;
          }
          _free(rsp);
        }
      }

      snprintf(buf, sizeof(buf), "RETR %s\r\n", filename + 1);
      ftp_control_write(docp, buf, strlen(buf));
      DEBUG_PROTOC("%s", buf);

        /** check control connection if there is available  **/
        /** server response. This is required to properly   **/
        /** establish support active data connections, also **/
        /** in case of error                                **/
      if(!cfg.ftp_activec ||
        tl_selectr(bufio_getfd(docp->ftp_control), 10) == 0)
      {
        if(!ftp_open_data_finish(docp))
        {
          docp->ftp_fatal_err = TRUE;
          if(!docp->errcode)
            docp->errcode = ERR_FTP_DATACON;
          return NULL;
        }
      }

      if(!((reply = ftp_get_response(docp, NULL, TRUE)) >= 400))
      {
        docp->errcode = ERR_NOERROR;

        if(!ftp_open_data_finish(docp))
        {
          docp->ftp_fatal_err = TRUE;
          if(!docp->errcode)
            docp->errcode = ERR_FTP_DATACON;
          return NULL;
        }

        return docp->datasock;
      }

      if(reply == 600)
      {
        docp->ftp_fatal_err = TRUE;
        bufio_close(docp->datasock);
        docp->datasock = NULL;
        docp->errcode = ERR_READ;
        return NULL;
      }
      docp->errcode = ERR_FTP_GET;
    }
    else
    {
      docp->errcode = ERR_FTP_GET;
    }
  }

  if(((docp->doc_url->extension &&
        ((ftp_url_extension *) docp->doc_url->extension)->type == FTP_TYPE_D)
      || !docp->doc_url->extension) && ((docp->check_limits
        && cfg.condition.ftpdir) || !docp->check_limits))
  {
    if(ftp_check_directory_existence(docp, filename))
    {
      docp->errcode = ERR_FTP_NODIR;
      docp->ftp_respc = 550;
      if(docp->datasock)
      {
        bufio_close(docp->datasock);
        docp->datasock = NULL;
      }
      return NULL;
    }

    if(ftp_set_transfer_type(docp, "A"))
    {
      return NULL;
    }

    if(!docp->datasock)
    {
      if(ftp_open_data_init(docp))
      {
        bufio_close(docp->ftp_control);
        docp->ftp_control = NULL;
        docp->errcode = ERR_FTP_DATACON;
        return NULL;
      }
    }

    ftp_send_list_command(docp, filename);

    /** check control connection if there is available  **/
    /** server response. This is required to properly   **/
    /** establish support active data connections, also **/
    /** in case of error                                **/
    if(!cfg.ftp_activec ||
      tl_selectr(bufio_getfd(docp->ftp_control), 10) == 0)
    {
      if(!ftp_open_data_finish(docp))
      {
        docp->ftp_fatal_err = TRUE;
        if(!docp->errcode)
          docp->errcode = ERR_FTP_DATACON;
        return NULL;
      }
    }