file_deleter.c

利用C语言实现的人工智能系统

// Berkeley Open Infrastructure for Network Computing
// http://boinc.berkeley.edu
// Copyright (C) 2005 University of California
//
// This is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation;
// either version 2.1 of the License, or (at your option) any later version.
//
// This software is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// To view the GNU Lesser General Public License visit
// http://www.gnu.org/copyleft/lesser.html
// or write to the Free Software Foundation, Inc.,
// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA


// file_deleter: deletes files that are no longer needed.
//
// options:
//
// -d N
//      set debug output level (1/2/3)
// -preserve_result_files
//      update the DB, but don't delete output files.
//      For debugging.
// -preserve_wu_files
//      update the DB, but don't delete input files.
//      For debugging.

// enum sizes.  RESULT_PER_ENUM is three times larger on the
// assumption of 3-fold average redundancy.
// This balances the rate at which input and output files are deleted
//
#define WUS_PER_ENUM        10000
#define RESULTS_PER_ENUM    10000

// how often to retry errors
//
#define ERROR_INTERVAL      86400

#include "config.h"
#include <list>
#include <cstring>
#include <cstdlib>
#include <ctime>
#include <unistd.h>
#include <errno.h>
#include <pwd.h>
#include <sys/types.h>
#include <sys/stat.h>
#ifdef HAVE_STRINGS_H
#include <strings.h>
#endif

#include "boinc_db.h"
#include "parse.h"
#include "error_numbers.h"
#include "util.h"
#include "str_util.h"
#include "filesys.h"
#include "strings.h"

#include "sched_config.h"
#include "sched_util.h"
#include "sched_msgs.h"

using namespace std;

#define LOCKFILE "file_deleter.out"
#define PIDFILE  "file_deleter.pid"

#define RESULTS_PER_WU 1        // an estimate of redundancy 

SCHED_CONFIG config;

int id_modulus=0, id_remainder=0;
bool dont_retry_errors = false;

// Given a filename, find its full path in the upload directory hierarchy
// Return an error if file isn't there.
int get_file_path(
    const char *filename, char* upload_dir, int fanout, char* path
) {
    dir_hier_path(filename, upload_dir, fanout, path);
    if (boinc_file_exists(path)) {
        return 0;
    }
    return ERR_NOT_FOUND;
}


int wu_delete_files(WORKUNIT& wu) {
    char* p;
	char command[1000];	
	char file_name[50];
    char filename[256], pathname[256], buf[LARGE_BLOB_SIZE];
    bool no_delete=false;
    int count_deleted = 0, retval, mthd_retval = 0;

    if (strstr(wu.name, "nodelete")) return 0;

    safe_strcpy(buf, wu.xml_doc);
    
    p = strtok(buf, "\n");
    strcpy(filename, "");
    while (p) {
        if (parse_str(p, "<name>", filename, sizeof(filename))) {
        } else if (match_tag(p, "<file_info>")) {
            no_delete = false;
            strcpy(filename, "");
        } else if (match_tag(p, "<no_delete/>")) {
            no_delete = true;
        } else if (match_tag(p, "</file_info>")) {
            if (!no_delete) {
                retval = get_file_path(filename, config.download_dir, config.uldl_dir_fanout,
                    pathname
                );
                if (retval) {
                    log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,
                        "[WU#%d] get_file_path: %s: %d\n",
                        wu.id, filename, retval
                    );
                } else {
					// we rename, compress and move the file so that it can be reused for another WU
					// we always use work0 folder for this process
					strcpy(command, "/home/ovi/aisystem/work0/processor ");
					strcat(command, pathname);

					retval = system(command);
					if (retval != 0)
						log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL, "[WU#%d] couldn't process file %s\n", wu.id, filename);
					else
					{
						log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL, "[WU#%d] successful processing of file %s \n", wu.id, filename);
						count_deleted++;
                     }	
                }
            }
        }
        p = strtok(0, "\n");
    }
    log_messages.printf(SCHED_MSG_LOG::MSG_DEBUG,
        "[WU#%d] deleted %d file(s)\n", wu.id, count_deleted
    );
    return mthd_retval;
}

int result_delete_files(RESULT& result) {
    char* p;
    char filename[256], pathname[256], buf[LARGE_BLOB_SIZE];
    bool no_delete=false;
    int count_deleted = 0, retval, mthd_retval = 0;

    safe_strcpy(buf, result.xml_doc_in);
    p = strtok(buf,"\n");
    while (p) {
        if (parse_str(p, "<name>", filename, sizeof(filename))) {
        } else if (match_tag(p, "<file_info>")) {
            no_delete = false;
            strcpy(filename, "");
        } else if (match_tag(p, "<no_delete/>")) {
            no_delete = true;
        } else if (match_tag(p, "</file_info>")) {
            if (!no_delete) {
                retval = get_file_path(
                    filename, config.upload_dir, config.uldl_dir_fanout,
                    pathname
                );
                if (retval) {
                    // the fact that no result files were found is a critical
                    // error if this was a successful result, but is to be
                    // expected if the result outcome was failure, since in
                    // that case there may well be no output file produced.
                    //
                    int debug_or_crit;
                    if (RESULT_OUTCOME_SUCCESS == result.outcome) {
                        debug_or_crit=SCHED_MSG_LOG::MSG_CRITICAL;
                    } else {
                        debug_or_crit=SCHED_MSG_LOG::MSG_DEBUG;
                    }
                    log_messages.printf(debug_or_crit,
                        "[RESULT#%d] outcome=%d client_state=%d No file %s to delete\n",
                        result.id, result.outcome, result.client_state, filename
                    );
                } else {
                    retval = unlink(pathname);
                    if (retval) {
                        mthd_retval = ERR_UNLINK;
                        log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,
                            "[RESULT#%d] unlink %s returned %d %s\n",
                            result.id, pathname, retval,
                            (retval && errno)?strerror(errno):""
                        );
                    } else {
                        count_deleted++;
                        log_messages.printf(SCHED_MSG_LOG::MSG_NORMAL,
                            "[RESULT#%d] unlinked %s\n", result.id, pathname
                        );
                    }
                }
            }
        }
        p = strtok(0, "\n");
    }

    log_messages.printf(SCHED_MSG_LOG::MSG_DEBUG,
        "[RESULT#%d] deleted %d file(s)\n", result.id, count_deleted
    );
    return mthd_retval;
}

// set by corresponding command line arguments.
static bool preserve_wu_files=false;
static bool preserve_result_files=false;

// return nonzero if did anything
//
bool do_pass(bool retry_error) {
    DB_WORKUNIT wu;
    DB_RESULT result;
    bool did_something = false;
    char buf[256];
    char clause[256];
    int retval;

    check_stop_daemons();

    strcpy(clause, "");

    sprintf(buf,
        "where file_delete_state=%d %s limit %d",
        retry_error?FILE_DELETE_ERROR:FILE_DELETE_READY,
        clause, WUS_PER_ENUM
    );

    while (!wu.enumerate(buf)) {
        did_something = true;

        retval = 0;
        if (!preserve_wu_files) {
            retval = wu_delete_files(wu);
        }
        if (retval) {
            wu.file_delete_state = FILE_DELETE_ERROR;
            log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,
                "[WU#%d] update failed: %d\n", wu.id, retval
            );
        } else {
            wu.file_delete_state = FILE_DELETE_DONE;
        }
        sprintf(buf, "file_delete_state=%d", wu.file_delete_state);
        retval= wu.update_field(buf);
    }

    sprintf(buf,
        "where file_delete_state=%d %s limit %d",
        retry_error?FILE_DELETE_ERROR:FILE_DELETE_READY,
        clause, RESULTS_PER_ENUM
    );

    while (!result.enumerate(buf)) {
        did_something = true;
        retval = 0;
        if (!preserve_result_files) {
            retval = result_delete_files(result);
        }
        if (retval) {
            result.file_delete_state = FILE_DELETE_ERROR;
            log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,
                "[RESULT#%d] update failed: %d\n", result.id, retval
            );
        } else {
            result.file_delete_state = FILE_DELETE_DONE;
        }
        sprintf(buf, "file_delete_state=%d", result.file_delete_state); 
        retval= result.update_field(buf);
    } 

    return did_something;
}

struct FILE_RECORD {
     string name;
     int date_modified;
};

bool operator == (const FILE_RECORD& fr1, const FILE_RECORD& fr2) {
    return (fr1.date_modified == fr2.date_modified && fr1.name == fr2.name);
}

bool operator < (const FILE_RECORD& fr1, const FILE_RECORD& fr2) {
    if (fr1.date_modified < fr2.date_modified) return true;
    if (fr1.date_modified > fr2.date_modified) return false;
    if (fr1.name < fr2.name) return true;
    return false;
}


int main(int argc, char** argv) {
    int retval;
    bool one_pass = false;
    int i;

    check_stop_daemons();
    for (i=1; i<argc; i++) {
		if (!strcmp(argv[i], "-d")) {
            log_messages.set_debug_level(atoi(argv[++i]));
        } else {
            log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL, "Unrecognized arg: %s\n", argv[i]);
        }
    }

	// these are default settings
	one_pass = true;
	preserve_wu_files = false;			// we don't need to preserve the files because we will rotate them
	preserve_result_files = false;		// the results are stored in the database in std_out field of result table
	dont_retry_errors = true;

    retval = config.parse_file("..");
    if (retval) {
        log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,
            "Can't parse ../config.xml: %s\n", boincerror(retval)
        );
        exit(1);
    }

    log_messages.printf(SCHED_MSG_LOG::MSG_NORMAL, "Starting\n");

    retval = boinc_db.open(config.db_name, config.db_host, config.db_user, config.db_passwd);
    if (retval) {
        log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL, "can't open DB\n");
        exit(1);
    }
    retval = boinc_db.set_isolation_level(READ_UNCOMMITTED);
    if (retval) {
        log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,
            "boinc_db.set_isolation_level: %d; %s\n", retval, boinc_db.error_string()
        );
    }
    install_stop_signal_handler();

    bool retry_errors_now = !dont_retry_errors;
    double next_error_time=0;

	// do just one pass
    bool got_any = do_pass(false);
}

const char *BOINC_RCSID_bd0d4938a6 = "$Id: file_deleter.C 13804 2007-10-09 11:35:47Z fthomas $";