zebraapi.c

harvest是一个下载html网页得机器人

    else
    {
        if (action == 2 || action == 3) /* fail if delete or update */
        {
	    zebra_end_trans(zh);
            return -1;
	}
    }
    extract_rec_in_mem (zh, "grs.sgml", rec_buf, rec_len, database,
                        action == 3 ? 1 : 0 /* delete flag */,
                        0, &sysno, 1, 1, 0);
    if (action == 1)
    {
        dict_insert (zh->reg->matchDict, recid_z, sizeof(sysno), &sysno);
    }
    else if (action == 3)
    {
        dict_delete (zh->reg->matchDict, recid_z);
    }
    zebra_end_trans(zh);
    return 0;
}

int delete_w_handle(const char *info, void *handle)
{
    ZebraHandle zh = (ZebraHandle) handle;
    ISAMC_P pos;

    if (*info == sizeof(pos))
    {
	memcpy (&pos, info+1, sizeof(pos));
	isamb_unlink(zh->reg->isamb, pos);
    }
    return 0;
}

static int delete_SU_handle(void *handle, int ord)
{
    ZebraHandle zh = (ZebraHandle) handle;
    char ord_buf[20];
    int ord_len;

    ord_len = key_SU_encode (ord, ord_buf);
    ord_buf[ord_len] = '\0';

    assert (zh->reg->isamb);
    dict_delete_subtree(zh->reg->dict, ord_buf,
			zh, delete_w_handle);
    return 0;
}

int zebra_drop_database  (ZebraHandle zh, const char *database)
{
    int ret = 0;
    ASSERTZH;
    yaz_log(LOG_API,"zebra_drop_database");
    zh->errCode = 0;

    if (zebra_select_database (zh, database))
        return -1;
    if (zebra_begin_trans (zh, 1))
        return -1;
    if (zh->reg->isamb)
    {
	zebraExplain_curDatabase (zh->reg->zei, database);
	
	zebraExplain_trav_ord(zh->reg->zei, zh, delete_SU_handle);
	zebraExplain_removeDatabase(zh->reg->zei, zh);
    }
    else
    {
	yaz_log(LOG_WARN, "drop database only supported for isam:b");
	ret = -1;
    }
    zebra_end_trans (zh);
    return ret;
}

int zebra_create_database (ZebraHandle zh, const char *database)
{
    ASSERTZH;
    yaz_log(LOG_API,"zebra_create_database");
    zh->errCode=0;

    if (zebra_select_database (zh, database))
        return -1;
    if (zebra_begin_trans (zh, 1))
        return -1;

    /* announce database */
    if (zebraExplain_newDatabase (zh->reg->zei, database, 0 
                                  /* explainDatabase */))
    {
        zebra_end_trans (zh);
	zh->errCode = 224;
	zh->errString = "database already exist";
	return -1;
    }
    zebra_end_trans (zh);
    return 0;
}

int zebra_string_norm (ZebraHandle zh, unsigned reg_id,
		       const char *input_str, int input_len,
		       char *output_str, int output_len)
{
    WRBUF wrbuf;
    ASSERTZH;
    yaz_log(LOG_API,"zebra_string_norm ");
    zh->errCode=0;
    if (!zh->reg->zebra_maps)
	return -1;
    wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, "",
			  input_str, input_len);
    if (!wrbuf)
	return -2;
    if (wrbuf_len(wrbuf) >= output_len)
	return -3;
    if (wrbuf_len(wrbuf))
	memcpy (output_str, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
    output_str[wrbuf_len(wrbuf)] = '\0';
    return wrbuf_len(wrbuf);
}


int zebra_set_state (ZebraHandle zh, int val, int seqno)
{
    char state_fname[256];
    char *fname;
    long p = getpid();
    FILE *f;
    ASSERTZH;
    yaz_log(LOG_API,"zebra_set_state v=%d seq=%d", val, seqno);
    zh->errCode=0;

    sprintf (state_fname, "state.%s.LCK", zh->reg_name);
    fname = zebra_mk_fname (res_get(zh->res, "lockDir"), state_fname);
    f = fopen (fname, "w");

    yaz_log (LOG_DEBUG, "%c %d %ld", val, seqno, p);
    fprintf (f, "%c %d %ld\n", val, seqno, p);
    fclose (f);
    xfree (fname);
    return 0;
}

int zebra_get_state (ZebraHandle zh, char *val, int *seqno)
{
    char state_fname[256];
    char *fname;
    FILE *f;

    ASSERTZH;
    yaz_log(LOG_API,"zebra_get_state ");
    zh->errCode=0;
    sprintf (state_fname, "state.%s.LCK", zh->reg_name);
    fname = zebra_mk_fname (res_get(zh->res, "lockDir"), state_fname);
    f = fopen (fname, "r");
    *val = 'o';
    *seqno = 0;

    if (f)
    {
        fscanf (f, "%c %d", val, seqno);
        fclose (f);
    }
    xfree (fname);
    return 0;
}

int zebra_begin_read (ZebraHandle zh)
{
    return zebra_begin_trans(zh, 0);
}

int zebra_end_read (ZebraHandle zh)
{
    return zebra_end_trans(zh);
}

int zebra_begin_trans (ZebraHandle zh, int rw)
{
    if (!zh->res)
    {
        zh->errCode = 2;
        zh->errString = "zebra_begin_trans: no database selected";
        return -1;
    }
    ASSERTZHRES;
    yaz_log(LOG_API,"zebra_begin_trans rw=%d",rw);
    assert (zh->res);
    if (rw)
    {
        int pass;
        int seqno = 0;
        char val = '?';
        const char *rval = 0;
        
        (zh->trans_no++);
        if (zh->trans_w_no)
            return 0;
        if (zh->trans_no != 1)
        {
            zh->errCode = 2;
            zh->errString = "zebra_begin_trans: write trans not allowed within read trans";
            return -1;
        }
        if (zh->reg)
	{
            resultSetInvalidate (zh);
            zebra_register_close (zh->service, zh->reg);
	}
        zh->trans_w_no = zh->trans_no;

        zh->errCode=0;
        
        zh->records_inserted = 0;
        zh->records_updated = 0;
        zh->records_deleted = 0;
        zh->records_processed = 0;
        
#if HAVE_SYS_TIMES_H
        times (&zh->tms1);
#endif
        /* lock */
        if (zh->shadow_enable)
            rval = res_get (zh->res, "shadow");
        
        for (pass = 0; pass < 2; pass++)
        {
            if (rval)
            {
                zebra_lock_r (zh->lock_normal);
                zebra_lock_w (zh->lock_shadow);
            }
            else
            {
                zebra_lock_w (zh->lock_normal);
                zebra_lock_w (zh->lock_shadow);
            }
            
            zebra_get_state (zh, &val, &seqno);
            if (val == 'c')
            {
                yaz_log (LOG_LOG, "previous transaction didn't finish commit");
                zebra_unlock (zh->lock_shadow);
                zebra_unlock (zh->lock_normal);
                zebra_commit (zh);
                continue;
            }
            else if (val == 'd')
            {
                if (rval)
                {
                    BFiles bfs = bfs_create (res_get (zh->res, "shadow"),
                                             zh->path_reg);
                    yaz_log (LOG_LOG, "previous transaction didn't reach commit");
                    bf_commitClean (bfs, rval);
                    bfs_destroy (bfs);
            }
                else
                {
                    yaz_log (LOG_WARN, "your previous transaction didn't finish");
                }
            }
            break;
        }
        if (pass == 2)
        {
            yaz_log (LOG_FATAL, "zebra_begin_trans couldn't finish commit");
            abort();
            return -1;
        }
        zebra_set_state (zh, 'd', seqno);
        
        zh->reg = zebra_register_open (zh->service, zh->reg_name,
                                       1, rval ? 1 : 0, zh->res,
                                       zh->path_reg);
        if (zh->reg)
            zh->reg->seqno = seqno;
        else
        {
            zebra_set_state (zh, 'o', seqno);
            
            zebra_unlock (zh->lock_shadow);
            zebra_unlock (zh->lock_normal);

            zh->trans_no--;
            zh->trans_w_no = 0;

            zh->errCode = 2;
            zh->errString = "zebra_begin_trans: cannot open register";
            yaz_log(LOG_FATAL, zh->errString);
            return -1;
        }
    }
    else
    {
        int dirty = 0;
        char val;
        int seqno;
        
        (zh->trans_no)++;
        
        if (zh->trans_no != 1)
        {
            zebra_flush_reg (zh);
            return 0;
        }
        zh->errCode=0;
#if HAVE_SYS_TIMES_H
        times (&zh->tms1);
#endif
        if (!zh->res)
        {
            (zh->trans_no)--;
            zh->errCode = 109;
            return -1;
        }
        if (!zh->lock_normal || !zh->lock_shadow)
        {
            (zh->trans_no)--;
            zh->errCode = 2;
            return -1;
        }
        zebra_get_state (zh, &val, &seqno);
        if (val == 'd')
            val = 'o';
        
        if (!zh->reg)
            dirty = 1;
        else if (seqno != zh->reg->seqno)
        {
            yaz_log (LOG_LOG, "reopen seqno cur/old %d/%d",
                     seqno, zh->reg->seqno);
            dirty = 1;
        }
        else if (zh->reg->last_val != val)
        {
            yaz_log (LOG_LOG, "reopen last cur/old %d/%d",
                     val, zh->reg->last_val);
            dirty = 1;
        }
        if (!dirty)
            return 0;
        
        if (val == 'c')
            zebra_lock_r (zh->lock_shadow);
        else
            zebra_lock_r (zh->lock_normal);
        
        if (zh->reg)
	{
            resultSetInvalidate (zh);
            zebra_register_close (zh->service, zh->reg);
	}
        zh->reg = zebra_register_open (zh->service, zh->reg_name,
                                       0, val == 'c' ? 1 : 0,
                                       zh->res, zh->path_reg);
        if (!zh->reg)
        {
            zebra_unlock (zh->lock_normal);
            zebra_unlock (zh->lock_shadow);
            zh->trans_no--;
            zh->errCode = 109;
            return -1;
        }
        zh->reg->last_val = val;
        zh->reg->seqno = seqno;
    }
    return 0;
}

int zebra_end_trans (ZebraHandle zh)
{
    ZebraTransactionStatus dummy;
    yaz_log(LOG_API,"zebra_end_trans");
    return zebra_end_transaction(zh, &dummy);
}

int zebra_end_transaction (ZebraHandle zh, ZebraTransactionStatus *status)
{
    char val;
    int seqno;
    const char *rval;

    ASSERTZH;
    yaz_log(LOG_API,"zebra_end_transaction");

    status->processed = 0;
    status->inserted  = 0;
    status->updated   = 0;
    status->deleted   = 0;
    status->utime     = 0;
    status->stime     = 0;

    if (!zh->res || !zh->reg)
    {
        zh->errCode = 2;
        zh->errString = "zebra_end_trans: no open transaction";
        return -1;
    }
    if (zh->trans_no != zh->trans_w_no)
    {
        zh->trans_no--;
        if (zh->trans_no != 0)
            return 0;

        /* release read lock */

        zebra_unlock (zh->lock_normal);
        zebra_unlock (zh->lock_shadow);
    }
    else
    {   /* release write lock */
        zh->trans_no--;
        zh->trans_w_no = 0;
        
        yaz_log (LOG_LOG, "zebra_end_trans");
        rval = res_get (zh->res, "shadow");
        
        zebraExplain_runNumberIncrement (zh->reg->zei, 1);
        
        zebra_flush_reg (zh);
        
        resultSetInvalidate (zh);

        zebra_register_close (zh->service, zh->reg);
        zh->reg = 0;
        
        yaz_log (LOG_LOG, "Records: %7d i/u/d %d/%d/%d", 
                 zh->records_processed, zh->records_inserted,
                 zh->records_updated, zh->records_deleted);
        
        status->processed = zh->records_processed;
        status->inserted = zh->records_inserted;
        status->updated = zh->records_updated;
        status->deleted = zh->records_deleted;
        
        zebra_get_state (zh, &val, &seqno);
        if (val != 'd')
        {
            BFiles bfs = bfs_create (rval, zh->path_reg);
            yaz_log (LOG_LOG, "deleting shadow stuff val=%c", val);
            bf_commitClean (bfs, rval);
            bfs_destroy (bfs);
        }
        if (!rval)
            seqno++;
        zebra_set_state (zh, 'o', seqno);
        
        zebra_unlock (zh->lock_shadow);
        zebra_unlock (zh->lock_normal);
        
    }
#if HAVE_SYS_TIMES_H
    times (&zh->tms2);
    logf (LOG_LOG, "user/system: %ld/%ld",
          (long) (zh->tms2.tms_utime - zh->tms1.tms_utime),
          (long) (zh->tms2.tms_stime - zh->tms1.tms_stime));
    
    status->utime = (long) (zh->tms2.tms_utime - zh->tms1.tms_utime);
    status->stime = (long) (zh->tms2.tms_stime - zh->tms1.tms_stime);
#endif
    return 0;
}

int zebra_repository_update (ZebraHandle zh)
{
    ASSERTZH;
    zh->errCode=0;
    logf (LOG_LOG|LOG_API, "updating %s", zh->rGroup.path);
    repositoryUpdate (zh);    
    return zh->errCode;
}

int zebra_repository_delete (ZebraHandle zh)
{
    ASSERTZH;
    zh->errCode=0;
    logf (LOG_LOG|LOG_API, "deleting %s", zh->rGroup.path);
    repositoryDelete (zh);
    return zh->errCode;
}

int zebra_repository_show (ZebraHandle zh)
{
    ASSERTZH;
    yaz_log(LOG_API,"zebra_repository_show");
    zh->errCode=0;
    repositoryShow (zh);
    return zh->errCode;
}

static int zebra_commit_ex (ZebraHandle zh, int clean_only)
{
    int seqno;
    char val;
    const char *rval;
    BFiles bfs;
    ASSERTZH;
    zh->errCode=0;

    if (!zh->res)
    {
        zh->errCode = 109;
        return -1;
    }
    rval = res_get (zh->res, "shadow");    
    if (!rval)
    {
        logf (LOG_WARN, "Cannot perform commit");
        logf (LOG_WARN, "No shadow area defined");
        return 0;
    }

    zebra_lock_w (zh->lock_normal);
    zebra_lock_r (zh->lock_shadow);

    bfs = bfs_create (res_get (zh->res, "register"), zh->path_reg);

    zebra_get_state (zh, &val, &seqno);

    if (rval && *rval)
        bf_cache (bfs, rval);
    if (bf_commitExists (bfs))
    {
        if (clean_only)
            zebra_set_state (zh, 'd', seqno);
        else
        {
            zebra_set_state (zh, 'c', seqno);
            
            logf (LOG_LOG, "commit start");
            bf_commitExec (bfs);
#ifndef WIN32
            sync ();
#endif
        }
        logf (LOG_LOG, "commit clean");
        bf_commitClean (bfs, rval);
        seqno++;