ccache_ftp.c

harvest是一个下载html网页得机器人

					break;

				default:
					break;
				}
			}
			/* if at last line of server reply (or not a multi-line
			 * reply) read until end of line reached 
			 */
			else if ((!isMultiLine) || (lastLine)) {
				if (inChar == CARRG_RET)
					hasCR = TRUE;

				else if ((hasCR) && (inChar == LINE_FEED))
					done = TRUE;

				else
					hasCR = FALSE;
			}
		}
	}
}


/******************************************************************************
** function name: CheckServReply()
**
** preconditions: msgType must indicate a valid type of message to check for
**	errors on, servReply is preallocated and contains the FTP
**	server reply
**
** postconditions: error code returned to caller indicating whether or not
**	servReply is an error message for msgType.
**
** author/credits: Dave Merkel, 4/94
**
******************************************************************************/

int CheckServReply(msgType, servReply)
     int msgType;		/* type of message to check for errors on */
     char *servReply;		/* message to check */
{
	char locReply[4];	/* temp reply storage */
	u_long reply;		/* for cast to long so switch may be used */

	/* if servReply exists, cast to long for quick compares */
	if (servReply != NULL) {
		strncpy(locReply, servReply, 3);
		locReply[3] = '\0';
		reply = htonl(*(u_long *) locReply);
	} else
		reply = 0;

	/* check the message types */
	switch (msgType) {
	case CONNECT_CHK:
	case REINIT_CHK:
		if (CONNECT_EST == reply)
			return (NO_ERROR);
		return (SERV_REPLY_ERROR);

	case USER_CHK:
		switch (reply) {
		case SEND_PASS:
			return (NO_ERROR);
		case USER_LOGIN:
			return (NO_PASS_REQ);
		default:
			break;
		}
		return (SERV_REPLY_ERROR);

	case PASSWD_CHK:
		if (USER_LOGIN == reply)
			return (NO_ERROR);
		return (SERV_REPLY_ERROR);

	case TYPE_CHK:
	case MODE_CHK:
	case PORT_CHK:
		if (CMD_OKAY == reply)
			return (NO_ERROR);
		return (SERV_REPLY_ERROR);

	case RETRIEVE_CHK:
		switch (reply) {
		case DATA_CONN_OPEN:
		case START_TRANS:
		case TRANS_SUCCESS:
			return (NO_ERROR);
		default:
			break;
		}
		return (SERV_REPLY_ERROR);

	case DISCONNECT_CHK:
		if (CLOSING == reply)
			return (NO_ERROR);
		return (SERV_REPLY_ERROR);

	default:
		return (NO_ERROR);
	}

	return (NO_ERROR);
}


/******************************************************************************
** function name: PrepareDataConnect()
**
** preconditions: theSocket is FTP control connection, sendPort indicates 
**	whether or not the PORT command will be used
**
** postconditions: a socket descriptor is returned to caller for an FTP
**	data connection
**
** author/credits: Dave Merkel, 4/94
**	Code for reusing existing socket for data connection taken from
**	ftp client source, Regents of the University of California, with
**	permission as indicated in source files
**
******************************************************************************/

int PrepareDataConnect(theSocket, sendPort)
     int theSocket;		/* control connection */
     Boolean sendPort;		/* PORT command */
{
	extern int gethostname();	/* unix system call */
	/* for use with PORT */
	struct hostent *theEntry;
	struct sockaddr_in theAddr;
	char hostName[HOST_NAME_LENGTH];
	int on = 1;
	int addrLen = sizeof(theAddr);

	/* data connection to return */
	int newSocket;

	newSocket = socket(AF_INET, SOCK_STREAM, 0);
	if (newSocket < 0)
		return (GET_SOCKET_ERR);

	memset(&theAddr, '\0', addrLen);

	/* if not using PORT, data connection will be control connection,
	 * so we must reuse theSocket
	 */
	if (!sendPort) {
		/* code taken from University of California ftp source code */
		if (setsockopt(newSocket, SOL_SOCKET, SO_REUSEADDR,
			(char *) &on, sizeof(on)) < 0)
			return (SET_SOCKOPT_ERR);
		if (getsockname(theSocket, (struct sockaddr *) &theAddr, &addrLen) < 0)
			return (GET_SOCKNAME_ERR);
	}
	/* we're using the port command, so get information to prepare new 
	 * socket 
	 */
	else {
		if (gethostname(hostName, HOST_NAME_LENGTH) < 0)
			return (GET_HOSTNAME_ERR);
		if (!(theEntry = gethostbyname(hostName)))
			return (GET_HOSTBYNAME_ERR);
		theAddr.sin_family = theEntry->h_addrtype;
		theAddr.sin_port = 0;
	}

	/* bind the new socket */
	if (bind(newSocket, (struct sockaddr *) &theAddr, addrLen) < 0)
		return (BIND_ERR);

	/* listen for data connection */
	listen(newSocket, BACK_LOG);

	return (newSocket);
}


/******************************************************************************
** function name: RetrieveFile()
**
** preconditions: theSocket is data connection for FTP session, data is 
**	preallocated by caller
**
** postconditions: file returned to caller inside of data struct
**
** author/credits: Dave Merkel, 4/94
**
******************************************************************************/

int RetrieveFile(theSocket, data)
     int theSocket;		/* FTP data connection */
     DataReturn *data;		/* struct to return file in */
{
	struct sockaddr_in theAddr;	/* for accepting data connect */
	struct timeval timeout;	/* for read timeouts */
	Boolean done = FALSE;	/* loop control */
	fd_set dataDetect;	/* select read detects */

	/* data and filename storage buffers */
	char *theData, *tmpFile;

	/* indices into array storage */
	long index = 0, readLength, maxIndex = 0;

	int addrLen = sizeof(theAddr);
	int dataSocket = 0, selectResult, i;
	int flags, mode, theFile = 0;

	/* get sockaddr_in information for accept calls */
	if (getsockname(theSocket, (struct sockaddr *) &theAddr, &addrLen) < 0)
		return (GET_SOCKNAME_ERR);

	timeout.tv_sec = 0;
	timeout.tv_usec = 0;
	FD_ZERO(&dataDetect);
	FD_SET(dataSocket, &dataDetect);

	/* prepare to accept data connect, timeout after ACCEPT_TIMEOUT
	 * seconds
	 */
	for (i = 0; i < ACCEPT_TIMEOUT; i++) {
		/* poll for connection */
		selectResult = select(dataSocket + 1, &dataDetect, (fd_set *) 0,
		    (fd_set *) 0, &timeout);

		/* connection requested */
		if (selectResult == 0)
			dataSocket = accept(theSocket, (struct sockaddr *)
			    &theAddr, &addrLen);

		if (dataSocket >= 0)
			break;

		if (i == 0)
			timeout.tv_sec = 1;
	}

	/* no accept received, timed out */
	if (i == ACCEPT_TIMEOUT)
		return (ACCEPT_TIMEOUT_ERR);

	/* if we should return data in memory */
	if (data->inMemory) {
		theData = (char *) malloc(INIT_FILE_SIZE);
		maxIndex = INIT_FILE_SIZE - 1;
	}
	/* otherwise stored in file */
	else {
		theData = (char *) malloc(BUFFER_SIZE);
		flags = O_TRUNC | O_RDWR | O_CREAT | O_NDELAY;
		mode = INIT_PERMISSION;

		/* if we should store in temp file */
		if (data->useTempFile) {
			tmpFile = xstrdup(tempnam(NULL, "ccurl"));
			strcpy(data->fileName, tmpFile);
			if (tmpFile == NULL) {
				close(dataSocket);
				return (CANT_GET_TMPNAME);
			}
		}
		/* open storage file */
		theFile = open(data->fileName, flags, mode);
		if (theFile < 0) {
			close(dataSocket);
			return (FILE_OPEN_ERR);
		}
		theData = (char *) malloc(BUFFER_SIZE);
	}


	/* if we couldn't allocate buffers */
	if (!theData) {
		close(dataSocket);
		if (!data->inMemory)
			close(theFile);
		return (MEMORY_ERROR);
	}
	i = 0;
	while ((!done) && (i < READ_TIMEOUT)) {
		/* start read by doing select on dataSocket, when
		 * select is given, but a read brings 0 bytes, 
		 * dataSocket is closed by server (in STREAM mode)
		 * and file is complete
		 */

		FD_ZERO(&dataDetect);
		FD_SET(dataSocket, &dataDetect);

		timeout.tv_sec = 0;

		/* poll for arrival of data */
		selectResult = select(dataSocket + 1, &dataDetect,
		    (fd_set *) 0, (fd_set *) 0,
		    &timeout);

		/* data arrived */
		if (FD_ISSET(dataSocket, &dataDetect)) {
			/* if returning data in memory */
			if (data->inMemory) {
				readLength = read(dataSocket,
				    &theData[index], maxIndex - index);

				index += readLength;

				/* if we have filled memory buffer, try
				 * to realloc space
				 */
				if (index >= maxIndex) {
					theData = (char *) realloc(theData, maxIndex
					    + 1 + REALLOC_BLOCK);

					/* couldn't allocate more space */
					if (!theData) {
						data->buffer = theData;
						close(dataSocket);
						return (MEMORY_ERROR);
					}
					/* possible realloc problem, need to 
					 * point to possibly moved new block
					 */
					maxIndex += REALLOC_BLOCK;
				}
			}
			/* if we are using file storage */
			else {
				/* read data and write to theFile */
				readLength = read(dataSocket, theData,
				    BUFFER_SIZE);

				index += readLength;
				if (write(theFile, theData, readLength) <
				    readLength) {
					close(dataSocket);
					close(theFile);
					return (WRITE_FILE_ERR);
				}
			}

			/* if we read 0 bytes from successful select poll,
			 * the data connection is closed and EOF has
			 * been reached
			 */
			if (readLength == 0)
				done = TRUE;

			i = 0;
		}
		/* no data present, so wait 1 second and continue */
		else {
			timeout.tv_sec = 1;
			selectResult = select(dataSocket + 1, &dataDetect,
			    (fd_set *) 0, (fd_set *) 0,
			    &timeout);
			i++;
		}
	}

	/* close data connection, point to new data if inMemory, otherwise
	 * close data file
	 */
	close(dataSocket);
	if (data->inMemory)
		data->buffer = theData;
	else
		close(theFile);

	/* return size of retrieved file */
	data->fileSize = index;

	if (i >= READ_TIMEOUT)
		return (READ_TIMEOUT_ERR);

	return (NO_ERROR);
}