listcontainer.cpp

一个UNIX/LINUX下的基于内容的过滤服务器源代码

//Please refer to http://dansguardian.org/?page=copyright
//for the license for this code.
//Written by Daniel Barron (daniel@//jadeb/.com).
//For support go to http://groups.yahoo.com/group/dansguardian

//  This program is free software; you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation; either version 2 of the License, or
//  (at your option) any later version.
//
//  This program is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.
//
//  You should have received a copy of the GNU General Public License
//  along with this program; if not, write to the Free Software
//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#include <syslog.h>
#include <algorithm>
#include "ListContainer.hpp"
#include "OptionContainer.hpp"
#include "RegExp.hpp"
#include <cstdlib>
#include <cstdio>
#include <unistd.h>
#include "String.hpp"
#include <iostream>
#include <fstream>
#include <sys/stat.h>
#include <sys/time.h>

extern bool isDaemonised;
extern OptionContainer o;

ListContainer::ListContainer()
:refcount(1),parent(false),filedate(0),used(false),bannedpfiledate(0),exceptionpfiledate(0),weightedpfiledate(0),sourceisexception(false),sourcestartswith(false),sourcefilters(0),data(new char[0]),graphdata(new int[0]),graphitems(0),data_length(0),data_memory(0),items(0),isSW(false),issorted(false),graphused(false),force_quick_search(0) {}



ListContainer::~ListContainer() {
    delete[] data;
    if (graphused) {
        delete[] graphdata;
    }
    for(unsigned int i = 0; i < morelists.size(); i++) {
        o.lm.deRefList(morelists[i]);
    }
}


void ListContainer::reset() {
    delete[] data;
    delete[] graphdata;
    for(unsigned int i = 0; i < morelists.size(); i++) {
        o.lm.deRefList(morelists[i]);
    }
    data = new char[0];
    graphdata = new int[0];
    graphitems = 0;
    data_length = 0;
    data_memory = 0;
    items = 0;
    isSW = false;
    issorted = false;
    graphused = false;
    force_quick_search = 0;
    combilist.clear();
    slowgraph.clear();
    list.clear();
    weight.clear();
    itemtype.clear();
    morelists.clear();
    used = false;
    parent = false;
    bannedpfile = "";
    exceptionpfile = "";
    weightedpfile = "";
    bannedpfiledate = 0;
    exceptionpfiledate = 0;
    weightedpfiledate = 0;
}

// delete the memory block when the class is destryed


bool ListContainer::previousUseItem(const char* filename, bool startswith, int filters) {
    String f = filename;
    if (f == sourcefile && startswith == sourcestartswith && filters == sourcefilters) {
        return true;
    }
    return false;
}



bool ListContainer::readPhraseList(const char* filename, bool isexception) {
    sourcefile = filename;
    sourceisexception = isexception;
    std::string linebuffer;  // a string line buffer ;)
    String temp; // a String for temporary manipulation
    String line;
    int len = getFileLength(filename);
    if (len < 0) {
        if (!isDaemonised) {
            std::cerr << "Error reading file (does it exist?): " << filename << std::endl;
        }
        syslog(LOG_ERR, "%s","Error reading file (does it exist?): ");
        syslog(LOG_ERR, "%s",filename);
        return false;
    }
    if (len < 2) {
        return true; // its blank - perhaps due to webmin editing
                    // just return
    }
    filedate = getFileDate(filename);
    increaseMemoryBy(len + 1);  // Allocate some memory to hold file
    ifstream listfile(filename, ios::in);  // open the file for reading
    if (!listfile.good()) {
        if (!isDaemonised) {
            std::cerr << "Error opening file (does it exist?): " << filename << std::endl;
        }
        syslog(LOG_ERR, "%s","Error opening file (does it exist?): ");
        syslog(LOG_ERR, "%s",filename);
        return false;
    }
    while (!listfile.eof()) {  // keep going until end of file
        getline(listfile, linebuffer);  // grab a line
        if (linebuffer.length() != 0) {  // sanity checkin
            line = linebuffer.c_str();
            line.removeWhiteSpace();
            line.toLower();  // tidy up
            if (line.startsWith("<")) {
                readPhraseListHelper(line, isexception);
            }
            else if (line.startsWith(".")) {
                temp = line.after(".include<").before(">");
                if (temp.length() > 0) {
                    if (!readPhraseList(temp.toCharArray(), isexception)) {
                        listfile.close();
                        return false;
                    }
                }
            }
        }
    }
    listfile.close();
    return true;  // sucessful read
}


void ListContainer::readPhraseListHelper(String line, bool isexception) {
    int weighting = line.after("><").before(">").toInteger();
                                                  // defaults to 0
    int type;
    if (weighting != 0) {
        type = 1;
        line = line.before("><") + ">";
    }
    else {
        if (isexception) {
            type = -1;
        }
        else {
            type = 0;
        }
    }

    if (line.after(">,").length() > 2) {
        while (line.length() > 2) {
            line = line.after("<");
            readPhraseListHelper2(line.before(">"), type + 11, weighting);
            line = line.after(">,");
        }
        readPhraseListHelper2("", type + 21, weighting);
    }
    else {
        line = line.after("<").before(">");
        readPhraseListHelper2(line, type, weighting);
    }
}


void ListContainer::readPhraseListHelper2(String phrase, int type, int weighting) {
    // -1=exception
    // 0=banned
    // 1=weighted
    // 10 = combination exception
    // 11 = combination banned
    // 12 = combination weighted
    // 20,21,22 = end of combi marker

    if (type > 19) {
        combilist.push_back(-2);  // mark an end of a combi
        combilist.push_back(type - 21);  // store the combi type
        combilist.push_back(weighting);  // store the combi weight

        return;
    }

    if (phrase.length() > 127) {
        if (!isDaemonised) {
            std::cerr << "Phrase length too long, truncating: " << phrase << std::endl;
        }
        syslog(LOG_ERR, "%s","Phrase length too long, truncating:");
        syslog(LOG_ERR, "%s",phrase.toCharArray());
        phrase = phrase.subString(0, 127);
    }

    if (phrase.length() < 1) {  // its too small to use
        return;
    }

    if (type < 10) {
        if (!addToItemListPhrase(phrase.toCharArray(), phrase.length(), type, weighting, false)) {
            if (!isDaemonised) {
                std::cerr << "Duplicate phrase, dropping: " << phrase << std::endl;
            }
            syslog(LOG_ERR, "%s","Duplicate phrase, dropping:");
            syslog(LOG_ERR, "%s",phrase.toCharArray());
        }
        return;
    }
    // must be a combi or end marker if got here

    // must be a combi if got here

    addToItemListPhrase(phrase.toCharArray(), phrase.length(), type, weighting, true);

}


bool ListContainer::addToItemListPhrase(char* s, int len, int type, int weighting, bool combi) {
    int i;
    list.push_back(data_length);
    for(i = 0; i < len; i++) {
        data[data_length + i] = s[i];
    }
    data[data_length + len] = 0;
    data_length += len + 1;
    if (combi) {
        combilist.push_back(items);
    }
    items++;
    weight.push_back(weighting);
    itemtype.push_back(type);
    return true;
}


bool ListContainer::readItemList(const char* filename, bool startswith, int filters) {
    sourcefile = filename;
    sourcestartswith = startswith;
    sourcefilters = filters;
    std::string linebuffer;
    RegExp re;
    re.comp("^.*\\:[0-9]+\\/.*");
    #ifdef DGDEBUG
        std::cout << filename << std::endl;
    #endif
    filedate = getFileDate(filename);
    if (isCacheFileNewer(filename)) {  // see if cached .process file
        linebuffer = filename; //         is available and up to date
        linebuffer += ".processed";

        if (getFileLength(linebuffer.c_str()) >= 4000) {
                                      // don't bother with small files
            if (!readProcessedItemList(linebuffer.c_str(), startswith, filters)) {  // read cached
                return false;
            }
            issorted = true;  // don't bother sorting cached file
            return true;

        }
    }
    int len = getFileLength(filename);
    if (len < 0) {
        if (!isDaemonised) {
            std::cerr << "Error reading file: " << filename << std::endl;
        }
        syslog(LOG_ERR, "%s","Error reading file:");
        syslog(LOG_ERR, "%s",filename);
        return false;
    }
    if (len < 2) {
        return true;  // its blank - perhaps due to webmin editing
                    // just return
    }
    increaseMemoryBy(len + 1);  // Allocate some memory to hold file
    // The plus one is to cope with files not ending in a new line
    ifstream listfile(filename, ios::in);
    if (!listfile.good()) {
        if (!isDaemonised) {
            std::cerr << "Error opening :"<< filename << std::endl;
        }
        syslog(LOG_ERR, "%s","Error opening :");
        syslog(LOG_ERR, "%s", filename);
        return false;
    }
    String temp, inc, hostname, url;
    while (!listfile.eof()) {
        getline(listfile, linebuffer);
        if (linebuffer.length() < 2) continue;  // its jibberish

        if (linebuffer[0] == '#') continue;  // its a comment
        temp = (char*)linebuffer.c_str();
        if (temp.contains("#")) {
            temp = temp.before("#");  // tidy up
        }
        temp.removeWhiteSpace();  // tidy up and make it handle CRLF files
        if (temp.startsWith(".Include<")) {  // see if we have another list
            inc = temp.after(".Include<");  // to include
            inc = inc.before(">");
            if (!readAnotherItemList(inc.toCharArray(), startswith, filters)) {  // read it
                listfile.close();
                return false;
            }
            continue;
        }
        if (temp.endsWith("/")) {
            temp.chop();  // tidy up
        }
        if (temp.startsWith("ftp://")) {
            temp = temp.after("ftp://");  // tidy up
        }
        if (filters == 1) {  // remove port addresses
            if (temp.contains(":")) {  // quicker than full regexp
                if (re.match(temp.toCharArray())) {
                    hostname = temp.before(":");
                    url = temp.after("/");
                    temp = hostname + "/" + url;
                }
            }
        }
        temp.toLower();  // tidy up
        addToItemList(temp.toCharArray(), temp.length());  // add to unsorted
                                                           // list
    }
    listfile.close();
    return true;  // sucessful read
}


bool ListContainer::readAnotherItemList(const char* filename, bool startswith, int filters) {

    int result = o.lm.newItemList(filename, startswith, filters, false);
    if (result < 0) {
        if (!isDaemonised) {
            std::cerr << "Error opening file:" << filename << std::endl;
        }
        syslog(LOG_ERR, "%s","Error opening file:");
        syslog(LOG_ERR, "%s",filename);
        return false;
    }
    morelists.push_back((unsigned)result);
    return true;
}


bool ListContainer::inList(char* string) {
    if (findInList(string) != NULL) {
        return true;
    }
    return false;
}



bool ListContainer::inListEndsWith(char* string) {
    if (items > 0) {
        if (searchREW(0, items-1, string) >= 0) {
            return true;
        }
    }
    bool rc;
    for(unsigned int i = 0; i < morelists.size(); i++) {
        rc = (*o.lm.l[morelists[i]]).inListEndsWith(string);
        if (rc) {
            return true;
        }
    }
    return false;
}



bool ListContainer::inListStartsWith(char* string) {
    if (items > 0) {
        if (searchRSW(0, items-1, string) >= 0) {
            return true;
        }
    }
    bool rc;
    for(unsigned int i = 0; i < morelists.size(); i++) {
        rc = (*o.lm.l[morelists[i]]).inListStartsWith(string);
        if (rc) {
            return true;
        }
    }
    return false;
}



char* ListContainer::findInList(char* string) {

    if (items > 0) {
        int r;
        if (isSW) {
            r = searchRSWF(0, items-1, string);
        }
        else {
            r = searchREWF(0, items-1, string);
        }
        if (r >= 0) {
            return (data + list[r]);
        }
    }
    char *rc;
    for(unsigned int i = 0; i < morelists.size(); i++) {
        rc = (*o.lm.l[morelists[i]]).findInList(string);
        if (rc != NULL) {
            return rc;
        }
    }
    return NULL;
}