sws.c

harvest是一个下载html网页得机器人

      sprintf(logbuf, "Found test number %d in path", test_no);
      logmsg(logbuf);

      if(strstr(reqbuf, "Authorization: Digest")) {
        /* If the client is passing this Digest-header, we set the part number
           to 1000. Not only to spice up the complexity of this, but to make
           Digest stuff to work in the test suite. */
        *part += 1000;
        logmsg("Received Digest request, sending back data %d", *part);
      }
      else if(strstr(reqbuf, "Authorization: NTLM TlRMTVNTUAAD")) {
        /* If the client is passing this type-3 NTLM header */
        *part += 1002;
        logmsg("Received NTLM type-3, sending back data %d", *part);
      }
      else if(strstr(reqbuf, "Authorization: NTLM TlRMTVNTUAAB")) {
        /* If the client is passing this type-1 NTLM header */
        *part += 1001;
        logmsg("Received NTLM type-1, sending back data %d", *part);
      }

      if(strstr(reqbuf, "Connection: close"))
        *open = FALSE; /* close connection after this request */
    }
    else {
      if(sscanf(reqbuf, "CONNECT %" MAXDOCNAMELEN_TXT "s HTTP/%d.%d",
                doc,
                &prot_major, &prot_minor) == 3) {
        sprintf(logbuf, "Receiced a CONNECT %s HTTP/%d.%d request", 
                doc, prot_major, prot_minor);
        logmsg(logbuf);

        if(prot_major*10+prot_minor == 10)
          *open = FALSE; /* HTTP 1.0 closes connection by default */

        if(!strncmp(doc, "bad", 3))
          /* if the host name starts with bad, we fake an error here */
          test_no = DOCNUMBER_BADCONNECT;
        else
          test_no = DOCNUMBER_CONNECT;
      }
      else {
        logmsg("Did not find test number in PATH");
        test_no = DOCNUMBER_404;
      }
    }

    return test_no;
  }
  
  logmsg("Got illegal request");
  fprintf(stderr, "Got illegal request\n");
  return DOCNUMBER_404;
}

/* returns -1 on failure */
static int send_doc(int sock,
                    int doc,
                    int part_no,
                    int *alive) /* keep the connection alive or not */
{
  int written;
  int count;
  const char *buffer;
  char *ptr;
  FILE *stream;
  char *cmd=NULL;
  int cmdsize=0;
  FILE *dump;
  int persistant = TRUE;

  static char weare[256];

  char filename[256];
  char partbuf[80]="data";

  *alive = FALSE;

  if(doc < 0) {
    switch(doc) {
    case DOCNUMBER_QUIT:
      logmsg("Replying to QUIT");
      buffer = docquit;
      break;
    case DOCNUMBER_WERULEZ:
      /* we got a "friends?" question, reply back that we sure are */
      logmsg("Identifying ourselves as friends");
      sprintf(weare, "HTTP/1.1 200 OK\r\n\r\nWE ROOLZ: %d\r\n",
              (int)getpid());
      buffer = weare;
      break;
    case DOCNUMBER_INTERNAL:
      logmsg("Bailing out due to internal error");
      return -1;
    case DOCNUMBER_CONNECT:
      logmsg("Replying to CONNECT");
      buffer = docconnect;
      break;
    case DOCNUMBER_BADCONNECT:
      logmsg("Replying to a bad CONNECT");
      buffer = docbadconnect;
      break;
    case DOCNUMBER_404:
    default:
      logmsg("Replying to with a 404");
      buffer = doc404;
      break;
    }
    ptr = NULL;
    stream=NULL;

    count = strlen(buffer);
  }
  else {
    logmsg("Fetch response data, test %d part %d", doc, part_no);

    if(0 != part_no)
      sprintf(partbuf, "data%d", part_no);

    sprintf(filename, TEST_DATA_PATH, doc);

    stream=fopen(filename, "rb");
    if(!stream) {
      logmsg("Couldn't open test file");
      return 0;
    }
    else {
      buffer = spitout(stream, "reply", partbuf, &count);
      ptr = (char *)buffer;
      fclose(stream);
    }

    /* re-open the same file again */
    stream=fopen(filename, "rb");
    if(!stream) {
      logmsg("Couldn't open test file");
      return 0;
    }
    else {    
      /* get the custom server control "commands" */
      cmd = (char *)spitout(stream, "reply", "postcmd", &cmdsize);
      fclose(stream);
    }
  }

  dump = fopen(RESPONSE_DUMP, "ab"); /* b is for windows-preparing */
  if(!dump) {
    logmsg("couldn't create logfile: " RESPONSE_DUMP);
    return -1;
  }

  /* If the word 'swsclose' is present anywhere in the reply chunk, the
     connection will be closed after the data has been sent to the requesting
     client... */
  if(strstr(buffer, "swsclose") || !count) {
    persistant = FALSE;
    logmsg("connection close instruction swsclose found in response");
  }

  do {
    written = swrite(sock, buffer, count);
    if (written < 0) {
      logmsg("Sending response failed and we bailed out!");
      return -1;
    }
    /* write to file as well */
    fwrite(buffer, 1, written, dump);

    count -= written;
    buffer += written;
  } while(count>0);

  fclose(dump);

  logmsg("Response sent!");

  if(ptr)
    free(ptr);

  if(cmdsize > 0 ) {
    char command[32];
    int num;
    char *ptr=cmd;
    do {
      if(2 == sscanf(ptr, "%31s %d", command, &num)) {
        if(!strcmp("wait", command))
          sleep(num); /* wait this many seconds */
        else
          logmsg("Unknown command in reply command section");
      }
      ptr = strchr(ptr, '\n');
      if(ptr)
        ptr++;
      else
        ptr = NULL;
    } while(ptr && *ptr);
  }
  if(cmd)
    free(cmd);

  *alive = persistant;

  return 0;
}

int main(int argc, char *argv[])
{
  struct sockaddr_in me;
  int sock, msgsock, flag;
  unsigned short port = DEFAULT_PORT;
  const char *logfile = DEFAULT_LOGFILE;
  int part_no;
  FILE *pidfile;
  
  if(argc>1)
    port = atoi(argv[1]);

  logfp = fopen(logfile, "a");
  if (!logfp) {
    perror(logfile);
    exit(1);
  }

#ifdef HAVE_SIGNAL
  signal(SIGPIPE, sigpipe_handler);
#endif
#ifdef HAVE_SIGINTERRUPT
  siginterrupt(SIGPIPE, 1);
#endif

  sock = socket(AF_INET, SOCK_STREAM, 0);
  if (sock < 0) {
    perror("opening stream socket");
    fprintf(logfp, "Error opening socket -- aborting\n");
    fclose(logfp);
    exit(1);
  }

  flag = 1;
  if (setsockopt
      (sock, SOL_SOCKET, SO_REUSEADDR, (const void *) &flag,
       sizeof(int)) < 0) {
    perror("setsockopt(SO_REUSEADDR)");
  }

  me.sin_family = AF_INET;
  me.sin_addr.s_addr = INADDR_ANY;
  me.sin_port = htons(port);
  if (bind(sock, (struct sockaddr *) &me, sizeof me) < 0) {
    perror("binding stream socket");
    fprintf(logfp, "Error binding socket -- aborting\n");
    fclose(logfp);
    exit(1);
  }

  pidfile = fopen(".http.pid", "w");
  if(pidfile) {
    fprintf(pidfile, "%d\n", (int)getpid());
    fclose(pidfile);
  }
  else
    fprintf(stderr, "Couldn't write pid file\n");

  /* start accepting connections */
  listen(sock, 5);

  while (1) {
    int doc;
    int open;
    int alive;
    
    msgsock = accept(sock, NULL, NULL);
    
    if (msgsock == -1)
      continue;
    
    logmsg("New client connected");

    do {

      doc = get_request(msgsock, &part_no, &open);
      logmsg("Received request, now send response number %d part %d",
             doc, part_no);
      send_doc(msgsock, doc, part_no, &alive);

      if((doc < 0) && (doc != DOCNUMBER_CONNECT)) {
        logmsg("special request received, no persistancy");
        break;
      }
      if(!alive) {
        logmsg("instructed to close connection after server-reply");
        break;
      }

      if(open)
        logmsg("persistant connection, awaits new request");
      /* if we got a CONNECT, loop and get another request as well! */
    } while(open || (doc == DOCNUMBER_CONNECT));

    logmsg("Closing client connection");
    sclose(msgsock);

    if (doc == DOCNUMBER_QUIT)
      break;
  }
  
  sclose(sock);
  fclose(logfp);
  
  return 0;
}