sers.c

网页抓取程序

/* Sers v0.1
 *   Coded by Shen139
 *
 *  Arguments:
 *  $ ./sers <port> <lower limit> <upper limit>
 *
 *  Compile with:
 *   $ gcc sers.c -o sers `mysql_config --cflags --libs` -lpthread -ldl -rdynamic -Wall -O2
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <stdarg.h>

/*
 * SERS_MODE == 1 = use mysql
 * SERS_MODE == 0 = use ooi
 */
#define SERS_MODE	1


#ifdef WIN32
 #include <windows.h>
 #include "snprintf.c"
 #pragma comment(lib,"WS2_32.lib")
#else /*linux*/

 #define SOCKET             int
 #define SOCKADDR_IN        struct sockaddr_in
 #define LPSOCKADDR         struct sockaddr*
 #define SOCKET_ERROR       -1
 #define DWORD	            long long int
 #define LPVOID	            void*
 #define HANDLE	            pthread_t

  #define _MULTI_THREADED
  #include <pthread.h>
  #include <sched.h>
  #include <sys/time.h>
  #include <unistd.h>
  #include <mysql/mysql.h>
  #include <pthread.h>
  #include <sys/types.h>
  #include <netinet/in.h>
  #include <netdb.h>
  #include <dlfcn.h>
  #include <string.h>
  #include <sys/socket.h>
  #include <arpa/inet.h>
#endif

#include "mysql_vars.h"

#include "functions.h"

#include "sql.h"
#include "initz.h"
#include "myCStr.h"
#include "list.h"
#include "search.h"
#include "parser.h"
#include "server.h"
#include "parseConf.h"
#include "subnodes.h"
#include "rankList.h"
#include "sgnl.h"


int main(int argc, char* argv[])
{
OOI_NODE* lexicon;
RANKS* rankList;

	/* legge il conf */
	printf("Reading configuration file...");
    if( GetSubNodes() == 0 )
	{
		printf("Error\n\n");
		return 0;
	}
	printf("OK\n");

	if( SERS_MODE == 0 )
	{
		lexicon = InitLexicon();
		/* genera il lexicon */
		if( InitzSersOoi(argc, argv, &lexicon, &rankList) == 0 )
			return 0;

		/* setto la global */
		glLexicon = lexicon;
		glRanks   = rankList;

	}
	else
	{
		/* si connette al mysql server */
		if( InitzSersMysql() == 0 )
			return 0;
	}

/* si connette a tutti i suoi sotto-nodi */
    printf("Connecting to sub-nodes...\n");
    ConnectToAllSubNodes();

    Sleep(100);

/* crea il thread che legger