rdfile.c

harvest是一个下载html网页得机器人

#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "machdr.h"
#include "rdfile.h"
#include "rdfileopt.h"
#include <dirent.h>
#define dirstruct dirent
#include "../util/curtime.h"
#include "../util/masks.h"
#include "../util/util.h"

#ifdef AUFSPLUS
#define AUFS
#endif /* AUFSPLUS */
#ifdef AUFS
#define APPLESHARE
#endif /* AUFS */
#ifdef APPLEDOUBLE
#define APPLESHARE
#endif /* APPLEDOUBLE */

#define NOTFOUND	0
#define ISFILE		1
#define	INFOFILE	2
#define INFOEXT		3
#define	SKIPFILE	4
#define	MACBINARY	5
#define	DIRECTORY	6
#ifdef APPLESHARE
#define	SHAREFILE	7
#endif /* APPLESHARE */

#define DATA_FORMAT	1
#define RSRC_FORMAT	2
#define UNIX_FORMAT	3

extern char *malloc();
extern char *realloc();
#if 0
extern char *strcpy();
extern char *strncpy();
#endif
extern char *strcat();

static void check_files();
static void read_file();
static void enter_dir();
static void exit_dir();
static int get_stdin_file();

char file_info[INFOBYTES];
char *data_fork, *rsrc_fork;
int data_size, rsrc_size;
static int max_data_size, max_rsrc_size;

typedef struct filelist {
    int nfiles;
    char **files;
    int *kind;
    struct filelist *previous;
    int current;
#ifdef APPLESHARE
    int shared_dir;
#endif /* APPLESHARE */
} filelist;

static int data_only;
static int no_recurse;
static int read_stdin;
static filelist global_files;
static filelist *current_files;
static char f_auth[5];
static char f_type[5];
static char f_name[] = ".foldername";
#ifdef APPLESHARE
#ifdef AUFS
#include "aufs.h"
static char infodir[] = ".finderinfo";
static char rsrcdir[] = ".resource";
static void read_aufs_info();
#endif /* AUFS */
#ifdef APPLEDOUBLE
#include "appledouble.h"
static char infodir[] = ".AppleDouble";
static void read_appledouble_info();
#endif /* APPLEDOUBLE */
#endif /* APPLESHARE */
static char filename[255];
static int filekind;

void setup(argc, argv)
int argc;
char **argv;
{
    if(argc == 0) {
	read_stdin = 1;
    } else {
	read_stdin = 0;
	global_files.previous = NULL;
	global_files.nfiles = argc;
	global_files.files = argv;
	global_files.current = 0;
	current_files = &global_files;
	check_files(1);
    }
}

static void check_files(initial)
int initial;
{
    struct stat stbuf;
    int i, j, n;
    char filename[255], filename1[255];

    /* Check the method to read the file */
    current_files->current = 0;
    /* Set initially to NOTFOUND or DIRECTORY */
    n = current_files->nfiles;
    current_files->kind = (int *)malloc((unsigned)n * sizeof(int));
    if(current_files->kind == NULL) {
	(void)fprintf(stderr, "Insufficient memory\n");
	exit(1);
    }
    for(i = 0; i < n; i++) {
	current_files->kind[i] = NOTFOUND;
	if(stat(current_files->files[i], &stbuf) >= 0) {
	    if((stbuf.st_mode & S_IFMT) == S_IFDIR) {
		/* We found a directory */
		current_files->kind[i] = DIRECTORY;
		continue;
	    }
	    current_files->kind[i] = ISFILE;
	}
    }
    /* That is enough for data_only mode */
    if(data_only) {
	return;
    }
#ifdef APPLESHARE
    /* First check whether we are in a folder on a shared volume */
    i = 1;
#ifdef AUFS
    if(stat(rsrcdir,&stbuf) < 0) {
	i = 0;
    } else {
	if((stbuf.st_mode & S_IFMT) != S_IFDIR) {
	    i = 0;
	}
    }
    if(stat(infodir,&stbuf) < 0) {
	i = 0;
    } else {
	if((stbuf.st_mode & S_IFMT) != S_IFDIR) {
	    i = 0;
	}
    }
#endif /* AUFS */
#ifdef APPLEDOUBLE
    if(stat(infodir,&stbuf) < 0) {
	i = 0;
    } else {
	if((stbuf.st_mode & S_IFMT) != S_IFDIR) {
	    i = 0;
	}
    }
#endif /* APPLEDOUBLE */
    current_files->shared_dir = i;
#endif /* APPLESHARE */
    for(i = 0; i < n; i++) {
	if(current_files->kind[i] == NOTFOUND) {
	    j = 0;
	} else if(current_files->kind[i] == ISFILE) {
	    /* Check whether the file is special */
#ifdef APPLESHARE
	    if(!current_files->shared_dir &&
	       !strcmp(current_files->files[i], f_name)) {
		current_files->kind[i] = SKIPFILE;
		continue;
	    }
#else /* APPLESHARE */
	    if(!strcmp(current_files->files[i], f_name)) {
		current_files->kind[i] = SKIPFILE;
		continue;
	    }
#endif /* APPLESHARE */
	    j = 1;
	} else if(current_files->kind[i] == SKIPFILE) {
	    continue;
	} else if(!initial) { /* DIRECTORY */
	    /* Check whether the directory is special */
	    if(!strcmp(current_files->files[i], ".") ||
	       !strcmp(current_files->files[i], "..")) {
		current_files->kind[i] = SKIPFILE;
	    }
#ifdef APPLESHARE
#ifdef AUFS
	    if(current_files->shared_dir &&
	       (!strcmp(current_files->files[i], infodir) ||
		!strcmp(current_files->files[i], rsrcdir))) {
		current_files->kind[i] = SKIPFILE;
	    }
#endif /* AUFS */
#ifdef APPLEDOUBLE
	    if(current_files->shared_dir &&
	       !strcmp(current_files->files[i], infodir)) {
		current_files->kind[i] = SKIPFILE;
	    }
#endif /* APPLEDOUBLE */
#endif /* APPLESHARE */
	    continue;
	} else { /* Take all directories from command line! */
	    continue;
	}
#ifdef APPLESHARE
	/* Check whether file is in shared format */
	if(j & current_files->shared_dir) {
	    j = 0;
	    filename[0] = 0;
	    (void)strcat(filename, infodir);
	    (void)strcat(filename, "/");
	    (void)strcat(filename, current_files->files[i]);
	    /* There ought to be an associated file in the info direcory */
	    if(stat(filename, &stbuf) >= 0) {
		current_files->kind[i] = SHAREFILE;
		continue;
	    }
	}
#endif /* APPLESHARE */
	/* If file not found check for the same with .info extension */
	if(!j) {
	    filename[0] = 0;
	    (void)strcat(filename, current_files->files[i]);
	    (void)strcat(filename, ".info");
	    /* Found a .info file, else no such file found */
	    if(stat(filename, &stbuf) >= 0) {
		current_files->kind[i] = INFOEXT;
	    }
	    continue;
	}
	/* Now we found the file.  Check first whether the name ends with
	   .info */
	j = strlen(current_files->files[i]) - 5;
	if(!strncmp(current_files->files[i] + j, ".info", 5)) {
	    /* This is a .info file.  Set as INFOFILE */
	    current_files->kind[i] = INFOFILE;
	    /* Now remove from list of files the same with .data or .rsrc
	       extension */
	    filename[0] = 0;
	    (void)strcat(filename, current_files->files[i]);
	    filename[j] = 0;
	    (void)strcpy(filename1, filename);
	    (void)strcat(filename, ".data");
	    (void)strcat(filename1, ".rsrc");
	    for(j = i + 1; j < n; j++) {
		if(!strcmp(filename, current_files->files[j])) {
		    /* Associated .data file */
		    current_files->kind[j] = SKIPFILE;
		    continue;
		}
		if(!strcmp(filename1, current_files->files[j])) {
		    /* Associated .rsrc file */
		    current_files->kind[j] = SKIPFILE;
		}
	    }
	    continue;
	}
	if(!strncmp(current_files->files[i] + j, ".data", 5) ||
	   !strncmp(current_files->files[i] + j, ".rsrc", 5)) {
	    /* .data or .rsrc file found.  Check whether there is an
	       associated .info file in the filelist */
	    filename[0] = 0;
	    (void)strcat(filename, current_files->files[i]);
	    filename[j] = 0;
	    (void)strcat(filename, ".info");
	    for(j = i + 1; j < n; j++) {
		if(!strcmp(filename, current_files->files[j])) {
		    /* Found an associated .info file! */
		    current_files->kind[i] = SKIPFILE;
		    break;
		}
	    }
	    if(j < n) {
		continue;
	    }
	}
	/* Finally nothing special */
	current_files->kind[i] = MACBINARY;
    }
}

int nextfile()
{
    int i;

    if(read_stdin) {
	return get_stdin_file();
    }
    i = current_files->current;
again:
    if(i == current_files->nfiles) {
	if(current_files->previous == NULL) {
	    return ISATEND;
	} else {
	    exit_dir();
	    current_files->current++;
	    return ENDDIR;
	}
    }
    filename[0] = 0;
    (void)strcat(filename, current_files->files[i]);
    filekind = current_files->kind[i];
    switch(filekind) {
    case DIRECTORY:
	if(no_recurse) {
	    (void)fprintf(stderr, "Directory %s skipped.\n", filename);
	    i++;
	    current_files->current = i;
	    goto again;
	}
	enter_dir();
	return ISDIR;
    case SKIPFILE:
	i++;
	current_files->current = i;
	goto again;
    case NOTFOUND:
	(void)fprintf(stderr, "File %s not found.\n", filename);
	exit(1);
    default:
	read_file();
	current_files->current = i + 1;
	return ISFILE;
    }
}

static void read_file()
{
    FILE *fd;
    int c, j, lname, skip;
    struct stat stbuf;
#ifdef APPLESHARE
    char filename1[255];
#endif /* APPLESHARE */

    switch(filekind) {
    case ISFILE:
	if(stat(filename, &stbuf) < 0) {
	    (void)fprintf(stderr, "Cannot stat file %s\n", filename);
	    exit(1);
	}
	for(j = 0; j < INFOBYTES; j++) {
	    file_info[j] = 0;
	}
	(void)strcpy(file_info + I_NAMEOFF + 1, filename);
	file_info[I_NAMEOFF] = strlen(filename);
	put4(file_info + I_CTIMOFF, (unsigned long)stbuf.st_ctime + TIMEDIFF);
	put4(file_info + I_MTIMOFF, (unsigned long)stbuf.st_mtime + TIMEDIFF);
	if(data_only == RSRC_FORMAT) {
	    rsrc_size = stbuf.st_size;
	    data_size = 0;
	    if(rsrc_size > max_rsrc_size) {
		if(rsrc_fork == NULL) {
		    rsrc_fork = malloc((unsigned)rsrc_size);
		} else {
		    rsrc_fork = realloc(rsrc_fork, (unsigned)rsrc_size);
		}
		max_rsrc_size = rsrc_size;
	    }
	    if(f_type[0] == 0) {
		(void)strncpy(file_info + I_TYPEOFF, "RSRC", 4);
	    } else {
		(void)strncpy(file_info + I_TYPEOFF, f_type, 4);
	    }
	    if(f_auth[0] == 0) {
		(void)strncpy(file_info + I_AUTHOFF, "RSED", 4);
	    } else {
		(void)strncpy(file_info + I_AUTHOFF, f_auth, 4);
	    }
	    put4(file_info + I_RLENOFF, (unsigned long)rsrc_size);
	    if((fd = fopen(filename, "r")) == NULL) {
		(void)fprintf(stderr, "Cannot open file %s\n", filename);
		exit(1);
	    }
	    if(fread(rsrc_fork, 1, rsrc_size, fd) != rsrc_size) {
		(void)fprintf(stderr, "Short file %s\n", filename);
		exit(1);
	    }
	    (void)fclose(fd);
	} else {
	    data_size = stbuf.st_size;
	    rsrc_size = 0;
	    if(data_size > max_data_size) {
		if(data_fork == NULL) {
		    data_fork = malloc((unsigned)data_size);
		} else {
		    data_fork = realloc(data_fork, (unsigned)data_size);
		}
		max_data_size = data_size;
	    }
	    if(f_type[0] == 0) {
		(void)strncpy(file_info + I_TYPEOFF, "TEXT", 4);
	    } else {
		(void)strncpy(file_info + I_TYPEOFF, f_type, 4);
	    }
	    if(f_auth[0] == 0) {
		(void)strncpy(file_info + I_AUTHOFF, "MACA", 4);
	    } else {
		(void)strncpy(file_info + I_AUTHOFF, f_auth, 4);
	    }
	    put4(file_info + I_DLENOFF, (unsigned long)data_size);
	    if((fd = fopen(filename, "r")) == NULL) {
		(void)fprintf(stderr, "Cannot open file %s\n", filename);
		exit(1);
	    }
	    if(fread(data_fork, 1, data_size, fd) != data_size) {
		(void)fprintf(stderr, "Short file %s\n", filename);
		exit(1);
	    }
	    (void)fclose(fd);
	    if(data_only == UNIX_FORMAT) {
		for(j = 0; j < data_size; j++) {
		    c = data_fork[j];
		    if(c == '\012' || c == '\015') {
			data_fork[j] = '\027' -c;
		    }
		}
	    }
	}
	break;
    case INFOEXT:
	(void)strcat(filename, ".info");
    case INFOFILE:
	lname = strlen(filename) - 5;
	if((fd = fopen(filename, "r")) == NULL) {
	    (void)fprintf(stderr, "Cannot open file %s\n", filename);
	    exit(1);
	}
	if(fread(file_info, 1, INFOBYTES, fd) != INFOBYTES) {
	    (void)fprintf(stderr, "Cannot read info header %s\n", filename);
	}
	(void)fclose(fd);
	data_size = get4(file_info + I_DLENOFF);
	rsrc_size = get4(file_info + I_RLENOFF);
	if(data_size > max_data_size) {
	    if(data_fork == NULL) {
		data_fork = malloc((unsigned)data_size);
	    } else {
		data_fork = realloc(data_fork, (unsigned)data_size);
	    }
	    max_data_size = data_size;
	}
	if(rsrc_size > max_rsrc_size) {
	    if(rsrc_fork == NULL) {
		rsrc_fork = malloc((unsigned)rsrc_size);
	    } else {
		rsrc_fork = realloc(rsrc_fork, (unsigned)rsrc_size);
	    }
	    max_rsrc_size = rsrc_size;
	}
	if(data_size != 0) {
	    filename[lname] = 0;
	    (void)strcat(filename, ".data");
	    if((fd = fopen(filename, "r")) == NULL) {
		(void)fprintf(stderr, "Cannot open data fork %s\n", filename);
		exit(1);
	    }
	    if(fread(data_fork, 1, data_size, fd) != data_size) {
		(void)fprintf(stderr, "Premature EOF on %s\n", filename);
	    }
	    (void)fclose(fd);
	}
	if(rsrc_size != 0) {
	    filename[lname] = 0;
	    (void)strcat(filename, ".rsrc");
	    if((fd = fopen(filename, "r")) == NULL) {
		(void)fprintf(stderr, "Cannot open rsrc fork %s\n", filename);
		exit(1);
	    }
	    if(fread(rsrc_fork, 1, rsrc_size, fd) != rsrc_size) {
		(void)fprintf(stderr, "Premature EOF on %s\n", filename);
	    }
	    (void)fclose(fd);
	}
	break;
    case MACBINARY:
	if((fd = fopen(filename, "r")) == NULL) {
	    (void)fprintf(stderr, "Cannot open file %s\n", filename);
	    exit(1);
	}
	if(fread(file_info, 1, INFOBYTES, fd) != INFOBYTES) {
	    (void)fprintf(stderr, "Short file %s\n", filename);
	    exit(1);
	}
	if(file_info[0] != 0) {
	    (void)fprintf(stderr, "File is not MacBinary: %s\n", filename);
	    exit(1);
	}
	data_size = get4(file_info + I_DLENOFF);
	rsrc_size = get4(file_info + I_RLENOFF);
	if(file_info[I_LOCKOFF] & 1) {
	    file_info[I_FLAGOFF + 1] = PROTCT_MASK;