mod_pdf.c

在线网络蜘蛛开源代码

/* OpenWebSpider
 *
 *  Author:     Stefano Alimonti aka Shen139
 *  Mail:       shen139 [at] openwebspider (dot) org
 *
 *  Compile with
 *  + Linux:  
 *          $ gcc -g -c mod_pdf.c
 *          $ gcc -g -shared -W1,-soname,mod_pdf.so.0 -o mod_pdf.so mod_pdf.o -lc
 *
 *  + Windows: Microsoft Visual C++ 2005
 *
 *
 * Known problem:
 *
bash-3.1$ ./openwebspider -i http://www.example.com/ -s -f modules/mod_pdf.so -X pdf
OpenWebSpider(v0.6.1)
  Coded by Shen139
   shen139(at)eviltime(dot)com


Trying to open module: modules/mod_pdf.so...modules/mod_pdf.so: cannot restore segment prot after reloc: Permission denied
 *
 * Please disable SELinux to solve this problem
 *
 *
 * This file is part of OpenWebSpider
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../modHeader.h"

char pdftotextPath[500];

int UnToken(char* str,char* Tokens,char* out,int len)
{
int c,i,x=0,tokenfound,y;

y=MIN(len,(signed)strlen(str));

	for(c=0;c<y;c++)
	{
		tokenfound=0;
		for(i=0;i<(signed)strlen(Tokens);i++)
			if(str[c]==Tokens[i])
				tokenfound=1;
		if(tokenfound==0)
			out[x++]=str[c];
	}
	out[x]=0;
return 1;
}

/* modFilter should return 1 if the current page must be indexed 0 if discarded*/
#ifdef WIN32
extern __declspec(dllexport)
#endif
int modFilter (struct functArg* arg)
{
FILE* pF;
char* command;
/*
this example function checks if the text of the current page has the string "xxx" if yes tells to the webspider to don't index the page otherwise index the page
*/
	if(arg)
        /* use this module only for custom extensions */
        if(arg->hostInfo->type == 4)
        {
            
            if(pdftotextPath[0]==0)
                return 0;

            unlink("test.pdf");
            unlink("test.txt");

            pF = fopen("test.pdf","wb");

            if(pF==NULL)
                return 0;

            fwrite(arg->html,arg->htmlLength,1,pF);
            fclose(pF);

            command = malloc(strlen(pdftotextPath) + 50);
            sprintf(command,"%s test.pdf test.txt",pdftotextPath);
            system(command);

            pF = fopen("test.txt","r");

            if(pF==NULL)
                return 0;

            fread(arg->text,100000,1,pF);
            fclose(pF);
            arg->textLength = strlen(arg->text);

            unlink("test.pdf");
            unlink("test.txt");

            return 1;
        }
        else
			return 1;

return 0;
}

#ifdef WIN32
extern __declspec(dllexport)
#endif
int modInitFilter (char* hostname, char* error)
{

FILE* pF;
char sLine[500];

    pdftotextPath[0]=0;

	pF=fopen("mod_pdf.conf","r");
	if(pF==NULL)
	{
		strcpy(error,"File not found(in the current directory): mod_pdf.conf");
		return 0;
	}

	while(!feof(pF))
	{
		memset(sLine,0,sizeof(sLine));

		fgets(sLine,499,pF);

		if(sLine[0]=='#' || sLine[0]=='\r' || sLine[0]=='\n' || sLine[0]==0)
            continue;
        else
        {
            UnToken(sLine,"\r\n",pdftotextPath,499);
            break;
        }
    }

	fclose(pF);

return 1;

}