listcontainer.cpp

一个UNIX/LINUX下的基于内容的过滤服务器源代码

                    weight[px] = weight[item];
                }
            }
        }
    }
    if (!found) {
        i = graphitems;
        graphitems++;
        numlinks = graphdata[inx * 64 + 2];
        graphdata[inx * 64 + 2] = numlinks + 1;
        graphdata[inx * 64 + 4 + numlinks] = i;
        graphdata[i * 64] = p;
        graphdata[i * 64 + 3] = item;
        s.lop();
        while(s.length() > 0) {
            numlinks = graphdata[i * 64 + 2];
            graphdata[i * 64 + 2] = numlinks + 1;
            graphdata[i * 64 + 4 + numlinks] = i + 1;
            i++;
            graphitems++;
            p = s.charAt(0);
            graphdata[i * 64] = p;
            graphdata[i * 64 + 3] = item;
            s.lop();
        }
        graphdata[i * 64 + 1] = 1;
    }
}



// quicksort with 3 way partitioning sorted by the end of the line
void ListContainer::quicksortEW(int l, int r) {
    if (r <= l) return;
    unsigned int e;
    int k;
    char* v = data + list[r];
    int i = l-1, j = r, p = i, q = r;
    for (;;) {
        while (greaterThanEWF(data + list[++i], v) > 0);
        while (greaterThanEWF(v, data + list[--j]) > 0) {
            if (j == l) break;
        }
        if (i >= j) break;
        e = list[i]; list[i] = list[j]; list[j] = e;
        if (greaterThanEWF(v, data + list[i]) == 0) {
            p++;
            e = list[p]; list[p] = list[i]; list[i] = e;
        }
        if (greaterThanEWF(v, data + list[j]) == 0) {
            q--;
            e = list[q]; list[q] = list[j]; list[j] = e;
        }
    }
    e = list[i]; list[i] = list[r]; list[r] = e;
    j = i - 1;
    i++;
    for (k = l; k <= p; k++, j--) {
        e = list[k]; list[k] = list[j]; list[j] = e;
    }
    for (k = r-1; k >= q; k--, i++) {
        e = list[k]; list[k] = list[i]; list[i] = e;
    }
    quicksortEW(l, j);
    quicksortEW(i, r);
}

// quicksort with 3 way partitioning sorted by the start of the line
void ListContainer::quicksortSW(int l, int r) {
    if (r <= l) return;
    unsigned int e;
    int k;
    char* v = data + list[r];
    int i = l-1, j = r, p = i, q = r;
    for (;;) {
        while (greaterThanSWF(data + list[++i], v) > 0);
        while (greaterThanSWF(v, data + list[--j]) > 0) {
            if (j == l) break;
        }
        if (i >= j) break;
        e = list[i]; list[i] = list[j]; list[j] = e;
        if (greaterThanSWF(v, data + list[i]) == 0) {
            p++;
            e = list[p]; list[p] = list[i]; list[i] = e;
        }
        if (greaterThanSWF(v, data + list[j]) == 0) {
            q--;
            e = list[q]; list[q] = list[j]; list[j] = e;
        }
    }
    e = list[i]; list[i] = list[r]; list[r] = e;
    j = i - 1;
    i++;
    for (k = l; k <= p; k++, j--) {
        e = list[k]; list[k] = list[j]; list[j] = e;
    }
    for (k = r-1; k >= q; k--, i++) {
        e = list[k]; list[k] = list[i]; list[i] = e;
    }
    quicksortSW(l, j);
    quicksortSW(i, r);
}

bool ListContainer::readProcessedItemList(const char* filename, bool startswith, int filters) {
    #ifdef DGDEBUG
        std::cout << "reading processed file:" << filename << std::endl;
    #endif
    int len = getFileLength(filename);
    int slen, i;
    if (len < 0) {
        if (!isDaemonised) {
            std::cerr << "Error reading file: " << filename << std::endl;
        }
        syslog(LOG_ERR, "%s","Error reading file:");
        syslog(LOG_ERR, "%s", filename);
        return false;
    }
    if (len < 5) {
        if (!isDaemonised) {
            std::cerr << "File too small (less than 5 bytes - is it corrupt?): " << filename << std::endl;
        }
        syslog(LOG_ERR, "%s","File too small (less than 5 bytes - is it corrupt?):");
        syslog(LOG_ERR, "%s",filename);
        return false;
    }
    increaseMemoryBy(len + 2);
    ifstream listfile(filename, ios::in);
    if (!listfile.good()) {
        if (!isDaemonised) {
            std::cerr << "Error opening: "<< filename << std::endl;
        }
        syslog(LOG_ERR, "%s","Error opening:");
        syslog(LOG_ERR, "%s",filename);
        return false;
    }
    std::string linebuffer;
    String temp, inc;
    while (!listfile.eof()) {
        getline(listfile, linebuffer);
        if (linebuffer[0] == '.') {
            temp = linebuffer.c_str();
            if (temp.startsWith(".Include<")) {  // see if we have another list
                inc = temp.after(".Include<").before(">");  // to include
                if (!readAnotherItemList(inc.toCharArray(), startswith, filters)) {  // read it
                    listfile.close();
                    return false;
                }
                continue;
            }
        }
        slen = linebuffer.length();
        if (slen < 3) continue;
        i = slen - 1;
        list.push_back(data_length);
        for(;i >= 0; i--) {
            data[data_length + i] = linebuffer[i];
        }
        data[data_length + slen] = 0;
        data_length += slen + 1;
        items++;
    }
    listfile.close();
    return true;
}


void ListContainer::addToItemList(char* s, int len) {
    int i;
    list.push_back(data_length);
    for(i = 0; i < len; i++) {
        data[data_length + i] = s[i];
    }
    data[data_length + len] = 0;
    data_length += len + 1;
    items++;
}



int ListContainer::searchRSWF(int a, int s, const char* p) {
    if (a > s) return (-1 - a);
    int m = (a + s) / 2;
    int r = greaterThanSWF(p, data + list[m]);
    if (r == 0) return m;
    if (r == -1) return searchRSWF(m + 1, s, p);
    if (a == s) return (-1 - a);
    return searchRSWF(a, m - 1, p);
}

int ListContainer::searchRSW(int a, int s, const char* p) {
    if (a > s) return (-1 - a);
    int m = (a + s) / 2;
    int r = greaterThanSW(p, data + list[m]);
    if (r == 0) return m;
    if (r == -1) return searchRSW(m + 1, s, p);
    if (a == s) return (-1 - a);
    return searchRSW(a, m - 1, p);
}

int ListContainer::searchREWF(int a, int s, const char* p) {
    if (a > s) return (-1 - a);
    int m = (a + s) / 2;
    int r = greaterThanEWF(p, data + list[m]);
    if (r == 0) return m;
    if (r == -1) return searchREWF(m + 1, s, p);
    if (a == s) return (-1 - a);
    return searchREWF(a, m - 1, p);
}


int ListContainer::searchREW(int a, int s, const char* p) {
    if (a > s) return (-1 - a);
    int m = (a + s) / 2;
    int r = greaterThanEW(p, data + list[m]);
    if (r == 0) return m;
    if (r == -1) return searchREW(m + 1, s, p);
    if (a == s) return (-1 - a);
    return searchREW(a, m - 1, p);
}


int ListContainer::greaterThanEWF(const char* a, const char* b) {
    int alen = strlen(a);
    int blen = strlen(b);
    int maxlen = alen < blen ? alen : blen;
    char* apos = (char*)a + alen - 1;
    char* bpos = (char*)b + blen - 1;
    unsigned char achar;
    unsigned char bchar;
    while(maxlen > 0) {
        achar = apos[0];
        bchar = bpos[0];
        if (achar > bchar) {
            return 1;
        }
        if (achar < bchar) {
            return -1;
        }
        maxlen --;
        apos --;
        bpos --;
    }
    if (alen > blen) {
        return 1;
    }
    if (alen < blen) {
        return -1;
    }
    return 0; // both equal
}


int ListContainer::greaterThanSWF(const char* a, const char* b) {
    int alen = strlen(a);
    int blen = strlen(b);
    int maxlen = alen < blen ? alen : blen;
    char* apos = (char*)a;
    char* bpos = (char*)b;
    int i = 0;
    unsigned char achar;
    unsigned char bchar;
    while(i < maxlen) {
        achar = apos[0];
        bchar = bpos[0];
        if (achar > bchar) {
            return 1;
        }
        if (achar < bchar) {
            return -1;
        }
        i ++;
        apos ++;
        bpos ++;
    }
    if (alen > blen) {
        return 1;
    }
    if (alen < blen) {
        return -1;
    }
    return 0; // both equal
}

int ListContainer::greaterThanSW(const char* a, const char* b) {
    int alen = strlen(a);
    int blen = strlen(b);
    int maxlen = alen < blen ? alen : blen;
    char* apos = (char*)a;
    char* bpos = (char*)b;
    int i = 0;  // this used to be set to 1 - I don't know why
    unsigned char achar;
    unsigned char bchar;
    while(i < maxlen) {
        achar = apos[0];
        bchar = bpos[0];
        if (achar > bchar) {
            return 1;
        }
        if (achar < bchar) {
            return -1;
        }
        i ++;
        apos ++;
        bpos ++;
    }
    if (blen > alen) return -1;
    return 0; // both equal
}

int ListContainer::greaterThanEW(const char* a, const char* b) {
    int alen = strlen(a);
    int blen = strlen(b);
    int maxlen = alen < blen ? alen : blen;
    char* apos = (char*)a + alen - 1;
    char* bpos = (char*)b + blen - 1;
    unsigned char achar;
    unsigned char bchar;
    while(maxlen > 0) {
        achar = apos[0];
        bchar = bpos[0];
        if (achar > bchar) {
            return 1;
        }
        if (achar < bchar) {
            return -1;
        }
        maxlen --;
        apos --;
        bpos --;
    }
    if (blen > alen) return -1;
    return 0; // both equal
}

bool ListContainer::isCacheFileNewer(const char* filename) {
    int len = getFileLength(filename);
    if (len < 0) {
        if (!isDaemonised) {
            std::cerr << "Error reading file: " << filename << std::endl;
        }
        syslog(LOG_ERR, "%s", "Error reading file:");
        syslog(LOG_ERR, "%s", filename);
        return false;
    }
    int bannedlistdate = getFileDate(filename);
    std::string linebuffer = filename;
    linebuffer += ".processed";
    int cachedate = getFileDate(linebuffer.c_str());
    if (cachedate < bannedlistdate) {
        return false;  // cache file is older than list file
    }
    return true;
}

void ListContainer::increaseMemoryBy(int bytes) {
    if (data_memory > 0) {
        char* temp = new char[data_memory + bytes];  // replacement store
        memcpy(temp, data, data_length);
        delete[] data;
        data = temp;
        data_memory = data_memory + bytes;
    }
    else {
        delete[] data;
        data = new char[bytes];
        data_memory = bytes;
    }
}



std::string ListContainer::toLower(std::string s) {
    int l = s.length();
    for(int i = 0; i < l; i++) {
        if ((s[i] >= 'A') && (s[i] <= 'Z')) {
            s[i] = 'a' + s[i] - 'A';
        }
    }
    return s;
}


int ListContainer::getFileLength(const char* filename) {
    int len;
    FILE* file = NULL;
    file = fopen(filename, "r");
    if (file) {
        if (!fseek(file, 0, SEEK_END)) {
            len = ftell(file);
        }
        else {
            len = -1;
        }
        fclose(file);
        return len;
    }
    return -1;
}

int ListContainer::getFileDate(const char* filename) {
    struct stat status;
    int rc = stat(filename, &status);
    if (rc != 0) {
        return -1;
    }
    struct tm *tmnow = localtime(&status.st_mtime);

    int date = (tmnow->tm_year - 100) * 31536000;
    date += tmnow->tm_mon * 2628000;
    date += tmnow->tm_mday * 86400;
    date += tmnow->tm_hour * 3600;
    date += tmnow->tm_min * 60;
    date += tmnow->tm_sec;
    return date;  // a nice int rather than a horrid struct
}


bool ListContainer::upToDate() {
    if (getFileDate(sourcefile.toCharArray()) > filedate) {
        return false;
    }
    for(unsigned int i = 0; i < morelists.size(); i++) {
        if (!(*o.lm.l[morelists[i]]).upToDate()) {
            return false;
        }
    }
    return true;
}