main.c

harvest是一个下载html网页得机器人

					if (argc <= 1) {
						fprintf(stderr, "%s: the server port must follow the -C option\n", GProgname);
#if	ISSERVER
						RETURNMAIN(usageS());
#else	/*ISSERVER*/
						RETURNMAIN(usage());
#endif	/*ISSERVER*/
					}
					argv ++;
					SERV_PORT = atoi(argv[0]);
					argc --;
				}
				else {
					SERV_PORT = atoi(p+1);
				}
				if ((SERV_PORT < MIN_SERV_PORT) || (SERV_PORT > MAX_SERV_PORT)) {
					fprintf(stderr, "Bad server port %d: must be in [%d, %d]: using default %d\n",
						SERV_PORT, MIN_SERV_PORT, MAX_SERV_PORT, DEF_SERV_PORT);
					SERV_PORT = DEF_SERV_PORT;
				}
				quitwhile = ON;
				break;

#if	ISSERVER
#if	SFS_COMPAT
			case 'R' :
				RemoteFiles = ON;
				break;

			case 'Z' :
				/* No op */
				break;
#endif
			/* server cannot recognize any other option */
			default :
				fprintf(stderr, "%s: server cannot recognize option: '%s'\n", GProgname, p);
				RETURNMAIN(usageS());
#else	/*ISSERVER*/

			/* These have 1 argument each, so must do quitwhile */
			case 'd' :
			case 'e' :
			case 'f' :
			case 'k' :
			case 'D' :
			case 'F' : 
			case 'I' :
			case 'L' :
			case 'R' :
			case 'S' :
			case 'T' :
			case 'Y' :
			case 'p' :
				if (argv[0][2] == '\0') {/* space after - option */
					if(argc <= 1) {
						fprintf(stderr, "%s: the '-%c' option must have an argument\n", GProgname, c);
						RETURNMAIN(usage());
					}
					argv++;
					argc--;
				}
				quitwhile = ON;
				break;

			/* These are illegal */
			case 'm' :
			case 'v' :
				fprintf(stderr, "%s: illegal option: '-%c'\n", GProgname, c);
				RETURNMAIN(usage());

			/* They can't be patterns and filenames since they start with a -, these don't have arguments */
			case '!' :
			case 'a' :
			case 'b' :
			case 'c' :
			case 'h' :
			case 'i' :
			case 'j' :
			case 'l' :
			case 'n' :
			case 'o' :
			case 'q' :
			case 'r' :
			case 's' :
			case 't' :
			case 'u' :
			case 'g' :
			case 'w' :
			case 'x' :
			case 'y' :
			case 'z' :
			case 'A' :
			case 'B' :
			case 'E' :
			case 'G' :
			case 'M' :
			case 'N' :
			case 'O' :
			case 'P' :
			case 'Q' :
			case 'U' :
			case 'W' :
			case 'X' :
			case 'Z' :
				break;

			case 'C':
				CONTACT_SERVER = 1;
				break;

			case 'V' :
				printf("\nThis is glimpse version %s, %s.\n\n", GLIMPSE_VERSION, GLIMPSE_DATE);
				RETURNMAIN(0);

			default :
				if (isdigit(c)) quitwhile = ON;
				else {
					fprintf(stderr, "%s: illegal option: '-%c'\n", GProgname, c);
					RETURNMAIN(usage());
				}
				break;
#endif	/*ISSERVER*/
			} /* switch(c) */
			p ++;
		}
	}
#else
	CONTACT_SERVER = 1;
	argc=0;
#endif

#if	!ISSERVER
	/* Next arg must be the pattern: Check if the user wants to run the client as agrep, or doesn't want to contact the server */
	if ((argc > 1) || (!CONTACT_SERVER)) goto doityourself;
#endif	/*!ISSERVER*/

	argv = oldargv;
	argc = oldargc;
#endif	/*CLIENTSERVER*/

#if	ISSERVER && CLIENTSERVER
	if (-1 == read_index(indexdir)) RETURNMAIN(ret);

	/* Install signal handlers so that glimpseserver doesn't continue to run when sockets get broken, etc. */
	for (i=0; i<32; i++)
		if (ignore_signal[i]) signal(i, SIG_IGN);
	signal(SIGHUP, cleanup);
	signal(SIGINT, cleanup);
	if (((void (*)())-1 == signal(SIGPIPE, quitrequest)) ||
	    ((void (*)())-1 == signal(SIGUSR1, quitrequest)) ||
#ifndef	SCO
	    ((void (*)())-1 == signal(SIGURG, quitrequest)) ||
#endif
	    ((void (*)())-1 == signal(SIGUSR2, reinitialize)) ||
	    ((void (*)())-1 == signal(SIGHUP, reinitialize))) {
		/* Check for return values here since they ensure reliability */
		fprintf(stderr, "glimpseserver: Unable to install signal-handlers.\n");
		RETURNMAIN(-1);
	}

#if	USE_UNIXDOMAIN
	if ((sockfd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
		fprintf(stderr, "server cannot open socket for communication.\n");
		RETURNMAIN(-1);
	}
	char TMP_FILE_NAME[256];
	strcpy(TMP_FILE_NAME,TEMP_DIR) ;
	strcat(TMP_FILE_NAME,"/.glimpse_server");
	unlink(TMP_FILE_NAME);
	memset((char *)&serv_addr, '\0', sizeof(serv_addr));
	serv_addr.sun_family = AF_UNIX;
	strcpy(serv_addr.sun_path, TMP_FILE_NAME);	/* < 108 ! */
	len = strlen(serv_addr.sun_path) + sizeof(serv_addr.sun_family);
#else	/*USE_UNIXDOMAIN*/
	if ((sockfd = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
		perror("glimpseserver: Cannot create socket");
		RETURNMAIN(-1);
	}
	memset((char *)&serv_addr, '\0', sizeof(serv_addr));
        serv_addr.sin_family = AF_INET;
	serv_addr.sin_port = htons(SERV_PORT);
#if	0
	/* use host-names not internet style d.d.d.d notation */
	serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
#else
	/* 
	 * We only want to accept connections from glimpse clients 
	 * on the SERV_HOST, do not use INADDR_ANY!
	 */
	if ((hp = gethostbyname(SERV_HOST)) == NULL) {
		perror("glimpseserver: Cannot resolve host");
		RETURNMAIN(-1);
	}
	memcpy((caddr_t)&serv_addr.sin_addr, hp->h_addr, hp->h_length);
#endif	/*0*/
	len = sizeof(serv_addr);
#endif	/*USE_UNIXDOMAIN*/
	/* test code for glimpse server, get it to realse socket when it dies: contribution by Sheldon Smoker <[e-mail]> */
	if((setsockopt(sockfd,SOL_SOCKET,SO_REUSEADDR,(char *)&one,sizeof(one))) == -1) {
		fprintf(stderr,"glimpseserver: could not set socket option\n");
		perror("setsockopt");
		exit(1);
	}
	/* end test code */

	if (bind(sockfd, (struct sockaddr *)&serv_addr, len) < 0) {
		perror("glimpseserver: Cannot bind to socket");
		RETURNMAIN(-1);
	}
	listen(sockfd, SERVER_QUEUE_SIZE);

	printf("glimpseserver: On-line (pid = %d, port = %d) waiting for request...\n", getpid(), SERV_PORT);
	fflush(stdout);	/* must fflush to print on server stdout */
	while (1) {
		/*
		 *  Spin until sockfd is ready to do a non-blocking accept(2).
		 *  We only wait for 15 seconds, because SunOS may
		 *  swap us out if we block for 20 seconds or more.
		 *  -- Courtesy: Darren Hardy, [e-mail]
		 */
		if ((ret = do_select(sockfd, 15)) == 0) {
			if ((errno == EINTR) && glimpse_reinitialize) {
				glimpse_reinitialize = 0;
				CLEANUP;
				close(sockfd);
				sleep(IC_PORTRELEASE);
				reinitialize_server(oldargc, oldargv);
			}
			continue;
		}
		else if (ret != 1) continue;

		/* get parameters */
		ret = 0;
		clargc = 0;
		clargv = NULL;
		cli_len = sizeof(cli_addr);
		if ((newsockfd = accept(sockfd, &cli_addr, &cli_len)) < 0) continue;
		if (getreq(newsockfd, glimpse_reqbuf, &clstdin, &clstdout, &clstderr, &clargc, &clargv, &clpid) < 0) {
			ret = -1;
#if	DEBUG
			printf("getreq errno: %d\n", errno);
#endif	/*DEBUG*/
			goto end_process;
		}

#if	DEBUG
		printf("server processing request on %x\n", newsockfd);
#endif	/*DEBUG*/
		/*
		 * Server doesn't wait for response, no point using
		svstdin = dup(0);
		close(0);
		dup(clstdin);
		close(clstdin);
		 */
		/*
		 * This is wrong since clstderr == clstdout!
		svstdout = dup(1);
		close(1);
		dup(clstdout);
		close(clstdout);
		svstderr = dup(2);
		close(2);
		dup(clstderr);
		close(clstderr);
		*/
		svstdout = dup(1);
		svstderr = dup(2);
		close(1);
		close(2);
		dup(clstdout);
		dup(clstderr);
		close(clstdout);
		close(clstderr);

                        /*
			 * IMPORTANT: Unbuffered I/O to the client!
                         * Done for Harvest since partial results might be
                         * needed and fflush will not flush partial results
                         * to the client if we type ^C and kill it: it puts
                         * them into /dev/null. This way, output is unbuffered
                         * and the client sees at least some results if killed.
                         */
                        setbuf(stdout, NULL);
                        setbuf(stderr, NULL);

			glimpse_call = 0;
			glimpse_clientdied = 0;
			ret = process_query(clargc, clargv, newsockfd);
		/*
		 * Server doesn't wait for response, no point using
		close(0);
		dup(svstdin);
		close(svstdin);
		svstdin = 0;
		 */
		if (glimpse_clientdied) {
			/*
			 * This code is *ONLY* used as a safety net now.  
			 * The old problem was that users would see portions 
			 * of previous (and usually) unrelated queries!
			 * glimpseserver now uses unbuffered I/O to the
			 * client so all previous fwrite's to now are
			 * gone.  But since this is such a nasty problem
			 * we flush stdout to /dev/null just in case.
			 */
			clout = open("/dev/null", O_WRONLY);
			close(1);
			dup(clout);
			close(clout);
			fflush(stdout);
		} 

		/* Restore svstdout and svstdout to stdout/stderr */
		close(1);
		dup(svstdout);
		close(svstdout);
		svstdout = 1;
		close(2);
		dup(svstderr);
		close(svstderr);
		svstderr = 2;

	end_process:
#if	USE_MSGHDR
		/* send reply and cleanup */
		array[0] = (ret & 0xff000000) >> 24;
		array[1] = (ret & 0xff0000) >> 16;
		array[2] = (ret & 0xff00) >> 8;
		array[3] = (ret & 0xff);
		writen(newsockfd, array, 4);
#endif	/*USE_MSGHDR*/
#if	DEBUG
		write(1, "done\n", 5);
#endif	/*DEBUG*/
		for (i=0; i<clargc; i++)
			if (clargv[i] != NULL) my_free(clargv[i], 0);
		if (clargv != NULL) my_free(clargv, 0);
		close(newsockfd);	/* if !USE_MSGHDR, client directly reads from socket and writes onto stdout until EOF */
	}
#else	/*ISSERVER && CLIENTSERVER*/

#if	CLIENTSERVER
trynewsocket:
#if	USE_UNIXDOMAIN
	if ((sockfd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
		perror("socket");
		goto doityourself;
	}
	memset((char *)&serv_addr, '\0', sizeof(serv_addr));
	serv_addr.sun_family = AF_UNIX;
	char TMP_FILE_NAME[256];
	strcpy(TMP_FILE_NAME,TEMP_DIR) ;
	strcat(TMP_FILE_NAME,"/.glimpse_server");
	strcpy(serv_addr.sun_path, TMP_FILE_NAME);	/* < 108 ! */
	len = strlen(serv_addr.sun_path) + sizeof(serv_addr.sun_family);
#else	/*USE_UNIXDOMAIN*/
	if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
		perror("socket");
		goto doityourself;
	}
	serv_addr.sin_family = AF_INET;
	serv_addr.sin_port = htons(SERV_PORT);
#if	0
	/* use host-names not internet style d.d.d.d notation */
	serv_addr.sin__addr.s_addr = inet_addr(SERV_HOST);
#else	/*0*/
	if ((hp = gethostbyname(SERV_HOST)) == NULL) {
		fprintf(stderr, "gethostbyname (%s) failed\n", SERV_HOST);
		goto doityourself;
	}
	memcpy((caddr_t)&serv_addr.sin_addr, hp->h_addr, hp->h_length);
#endif	/*0*/
	len = sizeof(serv_addr);
#endif	/*USE_UNIXDOMAIN*/

	if (connect(sockfd, (struct sockaddr *)&serv_addr, len) < 0) {
                char errbuf[4096];
                sprintf(errbuf, "glimpse: Cannot contact glimpseserver: %s, port %d:", SERV_HOST, SERV_PORT);
                perror(errbuf);
		/* perror(SERV_HOST); */
#if	DEBUG
		printf("connect errno: %d\n", errno);
#endif	/*DEBUG*/
		close(sockfd);
		if ((errno == ECONNREFUSED) && (tried < 4)) {
			tried ++;
			goto trynewsocket;
		}
		goto doityourself;
	}

	if (sendreq(sockfd, glimpse_reqbuf, fileno(stdin), fileno(stdout), fileno(stderr), argc, argv, getpid()) < 0) {
		perror("sendreq");
#if	DEBUG
		printf("sendreq errno: %d\n", errno);
#endif	/*DEBUG*/
		close(sockfd);
		goto doityourself;
	}

#if	USE_MSGHDR
	if (readn(sockfd, array, 4) != 4) {
		close(sockfd);
		goto doityourself;
	}
	ret = (array[0] << 24) + (array[1] << 16) + (array[2] << 8) + array[3];
#else	/*USE_MSGHDR*/
{
	/* 
	 *  Dump everything the server writes into the socket onto 
	 *  stdout until EOF/error.  Do this in a way so that *everything*
	 *  the server sends is dumped to stdout by the client.  The