zebraapi.c

harvest是一个下载html网页得机器人

/* $Id: zebraapi.c,v 1.115 2003/11/28 14:47:45 adam Exp $
   Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003
   Index Data Aps

This file is part of the Zebra server.

Zebra is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 2, or (at your option) any later
version.

Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with Zebra; see the file LICENSE.zebra.  If not, write to the
Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
02111-1307, USA.
*/

#include <assert.h>
#include <stdio.h>
#ifdef WIN32
#include <io.h>
#include <process.h>
#include <direct.h>
#else
#include <unistd.h>
#endif

#include <yaz/diagbib1.h>
#include <yaz/pquery.h>
#include <yaz/sortspec.h>
#include "index.h"
#include <charmap.h>
#include "zebraapi.h"

/* simple asserts to validate the most essential input args */
#define ASSERTZH assert(zh && zh->service)
#define ASSERTZHRES assert(zh && zh->service && zh->res)
#define ASSERTZS assert(zs)

/* A simple log macro */
/* don't break with older yazen that lack log_app2 */
#ifdef LOG_APP2
#define LOG_API LOG_APP2
#else
#define LOG_API LOG_DEBUG
#endif

static Res zebra_open_res (ZebraHandle zh);
static void zebra_close_res (ZebraHandle zh);


static void zebra_chdir (ZebraService zs)
{
    const char *dir ;
    ASSERTZS;
    yaz_log(LOG_API,"zebra_chdir");
    dir = res_get (zs->global_res, "chdir");
    if (!dir)
	return;
    logf (LOG_DEBUG, "chdir %s", dir);
#ifdef WIN32
    _chdir(dir);
#else
    chdir (dir);
#endif
}

static void zebra_flush_reg (ZebraHandle zh)
{
    ASSERTZH;
    yaz_log(LOG_API,"zebra_flush_reg");
    zh->errCode=0;
    zebraExplain_flush (zh->reg->zei, zh);
    
    extract_flushWriteKeys (zh);
    zebra_index_merge (zh);
}

static struct zebra_register *zebra_register_open (ZebraService zs, 
                                                   const char *name,
                                                   int rw, int useshadow,
                                                   Res res,
                                                   const char *reg_path);
static void zebra_register_close (ZebraService zs, struct zebra_register *reg);

ZebraHandle zebra_open (ZebraService zs)
{
    ZebraHandle zh;
    const char *default_encoding;
    ASSERTZS;
    yaz_log(LOG_API,"zebra_open");

    if (!zs)
        return 0;

    zh = (ZebraHandle) xmalloc (sizeof(*zh));
    yaz_log (LOG_DEBUG, "zebra_open zs=%p returns %p", zs, zh);

    zh->service = zs;
    zh->reg = 0;          /* no register attached yet */
    zh->sets = 0;
    zh->destroyed = 0;
    zh->errCode = 0;
    zh->errString = 0;
    zh->res = 0; 

    zh->reg_name = xstrdup ("");
    zh->path_reg = 0;
    zh->num_basenames = 0;
    zh->basenames = 0;

    zh->trans_no = 0;
    zh->trans_w_no = 0;

    zh->lock_normal = 0;
    zh->lock_shadow = 0;

    zh->admin_databaseName = 0;

    zh->shadow_enable = 1;

    default_encoding = res_get_def(zs->global_res, "encoding", "ISO-8859-1");
    zh->record_encoding = xstrdup (default_encoding);

    zh->iconv_to_utf8 =
        yaz_iconv_open ("UTF-8", default_encoding);
    if (zh->iconv_to_utf8 == 0)
        yaz_log (LOG_WARN, "iconv: %s to UTF-8 unsupported",
           default_encoding);
    zh->iconv_from_utf8 =
        yaz_iconv_open (default_encoding, "UTF-8");
    if (zh->iconv_to_utf8 == 0)
        yaz_log (LOG_WARN, "iconv: UTF-8 to %s unsupported",
           default_encoding);

    zebra_mutex_cond_lock (&zs->session_lock);

    zh->next = zs->sessions;
    zs->sessions = zh;

    zebra_mutex_cond_unlock (&zs->session_lock);

    return zh;
}

ZebraService zebra_start (const char *configName)
{
    Res res;

    yaz_log(LOG_API|LOG_LOG,"zebra_start %s",configName);

    if ((res = res_open (configName, 0)))
    {
        ZebraService zh = xmalloc (sizeof(*zh));

	yaz_log (LOG_DEBUG, "Read resources `%s'", configName);
        
        zh->global_res = res;
        zh->configName = xstrdup(configName);
        zh->sessions = 0;
        
        zebra_chdir (zh);
        
        zebra_mutex_cond_init (&zh->session_lock);
        if (!res_get (zh->global_res, "passwd"))
            zh->passwd_db = NULL;
        else
        {
            zh->passwd_db = passwd_db_open ();
            if (!zh->passwd_db)
                logf (LOG_WARN|LOG_ERRNO, "passwd_db_open failed");
            else
                passwd_db_file (zh->passwd_db,
                                res_get (zh->global_res, "passwd"));
        }
        zh->path_root = res_get (zh->global_res, "root");
        return zh;
    }
    return 0;
}


void zebra_pidfname(ZebraService zs, char *path)
{
    zebra_lock_prefix (zs->global_res, path);
    strcat(path, "zebrasrv.pid");
}

static
struct zebra_register *zebra_register_open (ZebraService zs, const char *name,
                                            int rw, int useshadow, Res res,
                                            const char *reg_path)
{
    struct zebra_register *reg;
    int record_compression = REC_COMPRESS_NONE;
    char *recordCompression = 0;

    ASSERTZS;
    
    reg = xmalloc (sizeof(*reg));

    assert (name);
    reg->name = xstrdup (name);

    reg->seqno = 0;
    reg->last_val = 0;

    assert (res);

    yaz_log (LOG_LOG|LOG_API, "zebra_register_open rw = %d useshadow=%d p=%p,n=%s,rp=%s",
             rw, useshadow, reg, name, reg_path ? reg_path : "(none)");
    
    reg->dh = data1_createx (DATA1_FLAG_XML);
    if (!reg->dh)
        return 0;
    reg->bfs = bfs_create (res_get (res, "register"), reg_path);
    if (!reg->bfs)
    {
        data1_destroy(reg->dh);
        return 0;
    }
    if (useshadow)
        bf_cache (reg->bfs, res_get (res, "shadow"));
    data1_set_tabpath (reg->dh, res_get_def(res, "profilePath",
                                            DEFAULT_PROFILE_PATH));
    data1_set_tabroot (reg->dh, reg_path);
    reg->recTypes = recTypes_init (reg->dh);
    recTypes_default_handlers (reg->recTypes);

    reg->zebra_maps = zebra_maps_open (res, reg_path);
    reg->rank_classes = NULL;

    reg->key_buf = 0;

    reg->keys.buf_max = 0;
    reg->keys.buf = 0;
    reg->sortKeys.buf = 0;
    reg->sortKeys.buf_max = 0;

    reg->records = 0;
    reg->dict = 0;
    reg->sortIdx = 0;
    reg->isams = 0;
    reg->matchDict = 0;
    reg->isam = 0;
    reg->isamc = 0;
    reg->isamd = 0;
    reg->isamb = 0;
    reg->zei = 0;
    reg->matchDict = 0;
    reg->key_file_no = 0;
    
    zebraRankInstall (reg, rank1_class);
    zebraRankInstall (reg, rankzv_class);
    zebraRankInstall (reg, rankliv_class);

    recordCompression = res_get_def (res, "recordCompression", "none");
    if (!strcmp (recordCompression, "none"))
	record_compression = REC_COMPRESS_NONE;
    if (!strcmp (recordCompression, "bzip2"))
	record_compression = REC_COMPRESS_BZIP2;

    if (!(reg->records = rec_open (reg->bfs, rw, record_compression)))
    {
	logf (LOG_WARN, "rec_open");
	return 0;
    }
    if (rw)
    {
        reg->matchDict = dict_open (reg->bfs, GMATCH_DICT, 20, 1, 0);
    }
    if (!(reg->dict = dict_open (reg->bfs, FNAME_DICT, 40, rw, 0)))
    {
	logf (LOG_WARN, "dict_open");
	return 0;
    }
    if (!(reg->sortIdx = sortIdx_open (reg->bfs, rw)))
    {
	logf (LOG_WARN, "sortIdx_open");
	return 0;
    }
    if (res_get_match (res, "isam", "s", ISAM_DEFAULT))
    {
	struct ISAMS_M_s isams_m;
	if (!(reg->isams = isams_open (reg->bfs, FNAME_ISAMS, rw,
				      key_isams_m(res, &isams_m))))
	{
	    logf (LOG_WARN, "isams_open");
	    return 0;
	}
    }
    if (res_get_match (res, "isam", "i", ISAM_DEFAULT))
    {
	if (!(reg->isam = is_open (reg->bfs, FNAME_ISAM, key_compare, rw,
				  sizeof (struct it_key), res)))
	{
	    logf (LOG_WARN, "is_open");
	    return 0;
	}
    }
    if (res_get_match (res, "isam", "c", ISAM_DEFAULT))
    {
	struct ISAMC_M_s isamc_m;
	if (!(reg->isamc = isc_open (reg->bfs, FNAME_ISAMC,
				    rw, key_isamc_m(res, &isamc_m))))
	{
	    logf (LOG_WARN, "isc_open");
	    return 0;
	}
    }
    if (res_get_match (res, "isam", "d", ISAM_DEFAULT))
    {
	struct ISAMD_M_s isamd_m;
	
	if (!(reg->isamd = isamd_open (reg->bfs, FNAME_ISAMD,
				      rw, key_isamd_m(res, &isamd_m))))
	{
	    logf (LOG_WARN, "isamd_open");
	    return 0;
	}
    }
    if (res_get_match (res, "isam", "b", ISAM_DEFAULT))
    {
	struct ISAMC_M_s isamc_m;
	
	if (!(reg->isamb = isamb_open (reg->bfs, "isamb",
                                       rw, key_isamc_m(res, &isamc_m), 0)))
	{
	    logf (LOG_WARN, "isamb_open");
	    return 0;
	}
    }
    if (res_get_match (res, "isam", "bc", ISAM_DEFAULT))
    {
	struct ISAMC_M_s isamc_m;
	
	if (!(reg->isamb = isamb_open (reg->bfs, "isamb",
                                       rw, key_isamc_m(res, &isamc_m), 1)))
	{
	    logf (LOG_WARN, "isamb_open");
	    return 0;
	}
    }
    if (res_get_match (res, "isam", "null", ISAM_DEFAULT))
    {
	struct ISAMC_M_s isamc_m;
	
	if (!(reg->isamb = isamb_open (reg->bfs, "isamb",
                                       rw, key_isamc_m(res, &isamc_m), -1)))
	{
	    logf (LOG_WARN, "isamb_open");
	    return 0;
	}
    }
    reg->zei = zebraExplain_open (reg->records, reg->dh,
                                  res, rw, reg,
                                  explain_extract);
    if (!reg->zei)
    {
	logf (LOG_WARN, "Cannot obtain EXPLAIN information");
	return 0;
    }
    reg->active = 2;
    yaz_log (LOG_DEBUG, "zebra_register_open ok p=%p", reg);
    return reg;
}

int zebra_admin_shutdown (ZebraHandle zh)
{
    ASSERTZH;
    yaz_log(LOG_API,"zebra_admin_shutdown");
    zh->errCode=0;

    zebra_mutex_cond_lock (&zh->service->session_lock);
    zh->service->stop_flag = 1;
    zebra_mutex_cond_unlock (&zh->service->session_lock);
    return 0;
}

int zebra_admin_start (ZebraHandle zh)
{
    ZebraService zs;
    ASSERTZH;
    yaz_log(LOG_API,"zebra_admin_start");
    zh->errCode=0;
    zs = zh->service;
    zebra_mutex_cond_lock (&zs->session_lock);
    zebra_mutex_cond_unlock (&zs->session_lock);
    return 0;
}

static void zebra_register_close (ZebraService zs, struct zebra_register *reg)
{
    ASSERTZS;
    yaz_log(LOG_LOG|LOG_API, "zebra_register_close p=%p", reg);
    reg->stop_flag = 0;
    zebra_chdir (zs);
    if (reg->records)
    {
        zebraExplain_close (reg->zei);
        dict_close (reg->dict);
        if (reg->matchDict)
            dict_close (reg->matchDict);
	sortIdx_close (reg->sortIdx);
	if (reg->isams)
	    isams_close (reg->isams);
        if (reg->isam)
            is_close (reg->isam);
        if (reg->isamc)
            isc_close (reg->isamc);
        if (reg->isamd)
            isamd_close (reg->isamd);
        if (reg->isamb)
            isamb_close (reg->isamb);
        rec_close (&reg->records);
    }

    recTypes_destroy (reg->recTypes);
    zebra_maps_close (reg->zebra_maps);
    zebraRankDestroy (reg);
    bfs_destroy (reg->bfs);
    data1_destroy (reg->dh);

    xfree (reg->sortKeys.buf);
    xfree (reg->keys.buf);

    xfree (reg->key_buf);
    xfree (reg->name);
    xfree (reg);
}

int zebra_stop(ZebraService zs)
{
    if (!zs)
        return 0;
    yaz_log (LOG_LOG|LOG_API, "zebra_stop");

    while (zs->sessions)
    {
        zebra_close (zs->sessions);
    }
        
    zebra_mutex_cond_destroy (&zs->session_lock);

    if (zs->passwd_db)
	passwd_db_close (zs->passwd_db);

    res_close (zs->global_res);
    xfree (zs->configName);
    xfree (zs->path_root);
    xfree (zs);
    return 0;
}

int zebra_close (ZebraHandle zh)
{
    ZebraService zs;
    struct zebra_session **sp;
    int i;

    yaz_log(LOG_API,"zebra_close");
    if (!zh)
        return 0;
    ASSERTZH;
    zh->errCode=0;
    
    zs = zh->service;
    yaz_log (LOG_DEBUG, "zebra_close zh=%p", zh);
    resultSetDestroy (zh, -1, 0, 0);

    if (zh->reg)
        zebra_register_close (zh->service, zh->reg);
    zebra_close_res (zh);

    xfree (zh->record_encoding);

    for (i = 0; i < zh->num_basenames; i++)
        xfree (zh->basenames[i]);
    xfree (zh->basenames);

    if (zh->iconv_to_utf8 != 0)
        yaz_iconv_close (zh->iconv_to_utf8);
    if (zh->iconv_from_utf8 != 0)
        yaz_iconv_close (zh->iconv_from_utf8);

    xfree (zh->admin_databaseName);
    zebra_mutex_cond_lock (&zs->session_lock);
    zebra_lock_destroy (zh->lock_normal);
    zebra_lock_destroy (zh->lock_shadow);
    sp = &zs->sessions;
    while (1)
    {
	assert (*sp);
	if (*sp == zh)
	{
	    *sp = (*sp)->next;
	    break;
	}
	sp = &(*sp)->next;
    }
    zebra_mutex_cond_unlock (&zs->session_lock);
    xfree (zh->reg_name);
    zh->service=0; /* more likely to trigger an assert */
    xfree (zh);
    return 0;
}

struct map_baseinfo {
    ZebraHandle zh;
    NMEM mem;
    int num_bases;
    char **basenames;
    int new_num_bases;
    char **new_basenames;
    int new_num_max;
};

static Res zebra_open_res (ZebraHandle zh)
{
    Res res = 0;
    char fname[512];
    ASSERTZH;
    zh->errCode=0;

    if (zh->path_reg)
    {
        sprintf (fname, "%.200s/zebra.cfg", zh->path_reg);
        res = res_open (fname, zh->service->global_res);
        if (!res)
            res = zh->service->global_res;
    }
    else if (*zh->reg_name == 0)
    {
        res = zh->service->global_res;
    }
    else
    {