broker.h

harvest是一个下载html网页得机器人

	/* Object Identifier information: MANDATORY */
	fd_t FD;		/* The OID Number */
	char *url;		/* The URL */
	num32 urls;		/* cache of URL size */
	time_t update_time;	/* The Update-Time */
	int GID;		/* GathererID Identifier */

	/* Extra information about the Object */
	char *md5;		/* The MD5: OPTIONAL */
	num32 md5s;		/* cache of md5 size */
	time_t ttl;		/* The Time-to-Live */
	time_t lmt;		/* The Last-Modification-Time */
	time_t refresh_rate;	/* The Refresh-Rate */
	num32 flag;		/* Flag for misc information */
	off_t rec_off;		/* Offset in the Registry file */
	char *desc;		/* One-line description of object */
	num32 descs;		/* cache of desc size */

	struct REG_T *next, *prev;	/* Doubly-linked list */
} reg_t;

#define LONGLEN 	sizeof(long)
#define INTLEN 		sizeof(int)
#ifndef TIMELEN
#define TIMELEN 	sizeof(time_t)
#endif
#define OFFLEN 		sizeof(off_t)
#define FDLEN 		sizeof(fd_t)


/* ------------ Registry Storage Manager Types and Macros ------- */
#define REGISTRY_MAGIC		0x1f3a2bec
#define REGISTRY_VERSION	1
#define SET_DELETED(x)		(x) |= 0x01
#define IS_DELETED(x)		((x) & 0x01)

#define ENTRY_DELETED		2
#define REGISTRY_EOF		3

typedef struct reghdr {
	num32 magic;
	num32 version;
	num32 nrecords;
	num32 nrecords_deleted;
	num32 nrecords_valid;
} REGISTRY_HEADER;

typedef struct regrechdr {
	num32 record_size;
	num32 magic;
	num32 flag;
} RECORD_HEADER;

/* ============ Query Manager Data Types ======== */
/* Types */
#define LOGICAL 	0
#define SELECT  	1
#define FUNCTION  	2
#define STRING		3

/* Ops */
#define AND     	10
#define OR      	11
#define NOT     	12
#define EXCEPT  	13

#define EXACT   	20
#define REGEX   	21
#define GTT     	22
#define LST     	23

/* query types */
#define UQUERY     	30
#define QBULK      	31
#define ABULK      	32
#define ADMIN      	33
#define QDELETE    	34

/*action types */
#define COLLECTION 	40
#define CLEANING   	41
#define RCOMP      	42

/* Admin ops */
#define AD_COLLECT  	50
#define AD_CLEAN    	51
#define AD_REFRESH  	52
#define AD_UNZIP    	53

typedef struct QLIST {
	int type;
	int op;
	char *rlist;
	char *llist;
} qlist_t;

typedef struct ARG {
	int		type;
	void		*ptr;
	struct ARG	*next;
} arg_t;

/* ============ Event Manager Data Types ======== */
#define NUM_PRTY 4

typedef struct EVENT {
	int type;
	char queri[MAX_QUERY];
	int sock;
	time_t qtime;
	struct EVENT *next;
} event_t;

typedef struct PRTY {
	event_t *top;
	event_t *tail;
	int qnum;
} prty_t;

typedef struct EVNTQ {
	prty_t pri[NUM_PRTY];
} eventq_t;

#define USEREV    	"#USER"
#define BULKEV    	"#BULK"
#define ADMINEV   	"#ADMIN"
#define USEREV_S  	5
#define BULKEV_S  	5
#define ADMINEV_S 	6

/*  ========== BROKER RESULT CODES ========== */

#define BR_USER_MSG		101
#define BR_END_RESULTS		103
#define BR_URL			120
#define BR_OPAQUE_DATA		122
#define BR_DESCRIPTION		124
#define BR_SUMMARY_URL		125
#define BR_BROKER_URL		126
#define BR_ATTRIBUTE		127
#define BR_END_OBJECT		130

/* ========== ERROR Messages ========== */
#define ERR_MSG "111 - Indexer ERROR: The Indexer does not support all queries.\n"
#define ERR_SIZE 63

#define FILEONLY_WARN "101 - Warning: The Indexer does not file only listings.\n"
#define FILEONLY_WARN_S strlen(FILEONLY_WARN)

#define IND_FAIL "111 - Indexer ERROR: Could not complete query.\n"
#define IND_FAIL_S 47

#define LOADERR "111 - Broker is too heavily loaded.  Please try again later.\n"
#define LOADERR_S 60

#define PARSERR "111 - PARSE ERROR: Invalid Query.\n"
#define PARSERR_S 34

#define ADMINERR "Last Command was UNsuccessful.\n"
#define ADMINERR_S 31

#define ADMINSUC "Last Command was Successful.\n"
#define ADMINSUC_S 29

#define BULK_SUC  "000 - Bulk Object Transfer Successfully Initiated.\n"
#define BULK_END  "099 - Bulk Object Transfer Completed.\n"
#define BULK_ERR  "400 - Bulk Object Transfer Error.\n"

#define QMVERSION "200 - Harvest Broker Interface Version 0.5.\n"
#define PIPECHK   "200 - Testing the socket to the client.\n"

/* ============ INDEXER ============== */
struct indexing_routines {
	char *indexer_type;
	int (*IND_Index_Start) ();
	int (*IND_Index_Flush) ();
	int (*IND_New_Object) ();
	int (*IND_Destroy_Obj) ();
	int (*IND_Index_Full) ();
	int (*IND_Index_Incremental) ();
	int (*IND_initialize) ();
	int (*IND_config) ();
	int (*IND_do_query) ();
	void (*IND_Init_Flags) ();
	void (*IND_Set_Flags) ();
};


/* ============ Prototyping ============= */

#ifndef _PARAMS
#if defined(__STDC__) || defined(__cplusplus) || defined(__STRICT_ANSI__)
#define _PARAMS(ARGS) ARGS
#else				/* Traditional C */
#define _PARAMS(ARGS) ()
#endif				/* __STDC__ */
#endif				/* _PARAMS */

/* stor_man.c - Storage Manager */
int 	SM_Init _PARAMS((void));		/* Initalize */
fd_t 	SM_Create_Obj _PARAMS((void));		/* Create Object */
int 	SM_Destroy_Obj _PARAMS((fd_t));		/* Delete Object */
int 	SM_Exist_Obj _PARAMS((fd_t));		/* Exist Object? */
FILE *	SM_Read_Obj _PARAMS((fd_t));		/* Read Object Data */
FILE *	SM_Write_Obj _PARAMS((fd_t));		/* Write Object Data */
char *  SM_Get_Obj_URL _PARAMS((fd_t));		/* URL to Read Object Data */

/* main.c */
FILE *StMgr_Get_Object _PARAMS((fd_t, char *));
void sigdie _PARAMS((int));
void sigreap _PARAMS((int));
int Config_Broker _PARAMS((char *));
int Initialize_Com _PARAMS((void));
int set_str _PARAMS((char *, char *));
int set_rate _PARAMS((char *, int));
int Broker_Shutdown _PARAMS((void));
int Do_Compression _PARAMS((void));
int Do_Collection _PARAMS((void));
int Do_Cleaning _PARAMS((void));

/* registry.c */
int RG_Compress _PARAMS((void));
int RG_FD_Exists _PARAMS((fd_t));
int RG_Init _PARAMS((void));
int RG_Register _PARAMS((reg_t *));
int RG_Unregister _PARAMS((reg_t *));
int RG_Free_Entry _PARAMS((reg_t *));
int RG_List_Reg _PARAMS((void));
int RG_Build_Registry _PARAMS((void));
int RG_Cleaner _PARAMS((void));
int RG_Clean_Entry _PARAMS((reg_t *));
int RG_Count_Reg _PARAMS((void));
int RG_bulk_query _PARAMS((int, time_t));
int RG_Comp_Reg _PARAMS((void));
int RG_Free_Registry _PARAMS((void));
char *RG_Get_URL _PARAMS((fd_t));
reg_t *RG_Object_Search_Entry _PARAMS((reg_t *));
reg_t *RG_Object_Search_ByURL _PARAMS((char *));
reg_t *RG_Get_Entry _PARAMS((fd_t));
void RG_Registry_Shutdown _PARAMS((void));
int RG_Sync_Registry _PARAMS((void));

/* gid.c */
int RG_gid_register _PARAMS((GathererID *));
void RG_gid_init _PARAMS((void));
GathererID *RG_gid _PARAMS((int));
GathererID *RG_gid_new _PARAMS((char *, char *, char*));
void RG_gid_free _PARAMS((GathererID *));
void RG_gid_destroy _PARAMS((void));

/* collector.c */
FILE *COL_UPD_Obj_begin _PARAMS((reg_t *));
int COL_Fill_Entry _PARAMS((reg_t *));
int COL_UPD_Obj_end _PARAMS((reg_t *));
int COL_DEL_Obj _PARAMS((reg_t *));
int COL_REF_Obj _PARAMS((reg_t *));
int COL_Save_Att _PARAMS((AVPair *, reg_t *));
int COL_RM_Dup_Obj _PARAMS((reg_t *));
int COL_Do_Collection _PARAMS((void));
int COL_reset_update _PARAMS((char *, int, time_t));
time_t COL_get_last_update _PARAMS((char *, int));
char *COL_Normalize_Name _PARAMS((char *));
char *COL_Init_Connect _PARAMS((char *, int, int, time_t, char *));
char *COL_Get_UInput _PARAMS((int, time_t));
char *COL_Get_CInput _PARAMS((int, time_t));
char *COL_Get_BInput _PARAMS((int, char *, int, time_t));
FILE *COL_Create_Read_Pipe _PARAMS((char *));
void COL_Close_Read_Pipe _PARAMS((FILE *));

/* parser.y */
int P_parse_input _PARAMS((char *, int));
int P_parse_command _PARAMS((FILE *, int));
int P_parse_object _PARAMS((int));
int P_get_next_char _PARAMS((FILE *));

/* log.c */
int LOG_Init _PARAMS((void));
int LOG_rotate _PARAMS((void));
int LOG_turn_on _PARAMS((unsigned int));
int LOG_turn_off _PARAMS((unsigned int));
int LOG_reset_master _PARAMS((void));
int LOG_flush_log _PARAMS((void));
int LOG_close_log _PARAMS((void));
FILE *LOG_fptr _PARAMS((void));
unsigned int LOG_match_name _PARAMS((char *));

/* query_man.c */
int QM_query_manager _PARAMS((event_t *));
int QM_free_qlist _PARAMS((qlist_t *));
int QM_send_bulk_begin _PARAMS((int));
int QM_send_bulk_err _PARAMS((int));
int QM_send_bulk_end _PARAMS((int));
int QM_send_bulk_fd _PARAMS((fd_t, FILE *, reg_t *));
int QM_user_object _PARAMS((int, fd_t, int, char **));
int QM_user_done _PARAMS((int, int));
int QM_user_warning _PARAMS((int, char *));
int QM_add_attribute _PARAMS((char *));
int QM_free_attributes _PARAMS((void));


/* brkutil.c */
int UTIL_Make_Lower _PARAMS((char *));
char *UTIL_make_admin_filename _PARAMS((char *));
time_t UTIL_Get_Time _PARAMS((void));
int do_IND_Index_Start _PARAMS((void));		/* indexer */
int do_IND_Index_Flush _PARAMS((void));
int do_IND_New_Object _PARAMS((reg_t *));
int do_IND_Destroy_Obj _PARAMS((reg_t *));
int do_IND_Index_Full _PARAMS((void));
int do_IND_Index_Incremental _PARAMS((void));
int do_IND_initialize _PARAMS((void));
int do_IND_config _PARAMS((char *, char *));
int do_IND_do_query _PARAMS((qlist_t *, int, int, time_t));
void do_IND_Init_Flags _PARAMS((void));
void do_IND_Set_Flags _PARAMS((char *, char *));

/* event.c */
int EV_Init _PARAMS((void));
int EV_add_sevent _PARAMS((int));
int EV_add_aevent _PARAMS((int));
int EV_enq _PARAMS((event_t *, int));
int EV_Do_Event _PARAMS((void));
event_t *EV_deq _PARAMS((void));

/* admin.c */
int AD_parse_templatefile _PARAMS((char *));
int AD_add_object _PARAMS((char *));
int AD_remove_name _PARAMS((char *));
int AD_log_on _PARAMS((char *));
int AD_log_off _PARAMS((char *));
int AD_restart _PARAMS((void));
int AD_do_stats _PARAMS((void));
void AD_run_admin_process _PARAMS((void));

/* stor_reg.c */
int init_registry_file _PARAMS((void));
int set_registry_file _PARAMS((char *));
void finish_registry_file _PARAMS((void));
int read_record _PARAMS((reg_t *));
int write_record _PARAMS((reg_t *));
int delete_record _PARAMS((num32));
REGISTRY_HEADER *read_header _PARAMS((void));
int write_header _PARAMS((REGISTRY_HEADER *rhdr));
int get_record _PARAMS((reg_t *));
int append_new_record _PARAMS((reg_t *));
int remove_record _PARAMS((reg_t *));
int replace_record _PARAMS((reg_t *));
int read_record_hdr _PARAMS((RECORD_HEADER *));
int write_record_hdr _PARAMS((RECORD_HEADER *));
off_t set_registry_mark _PARAMS((void));
off_t restore_registry_mark _PARAMS((void));

/* select_loop.c */
int select_loop _PARAMS((int, int, int));

/* main.c */
int verify_exe _PARAMS((char *));
int verify_path _PARAMS((char *));

#endif