trunc.c

harvest是一个下载html网页得机器人

        ISAMD_PP *ispt;
        int i;
        struct trunc_info *ti;

        ispt = (ISAMD_PP *) xmalloc (sizeof(*ispt) * (to-from));

        ti = heap_init (to-from, sizeof(struct it_key),
                        key_compare_it);
        for (i = to-from; --i >= 0; )
        {
            logf(LOG_FATAL, "isam_d does not (currently) support truncs");
            abort();
            /*ispt[i] = isamd_pp_open (zi->reg->isamd, isam_p[from+i]); */
            if (isamd_pp_read (ispt[i], ti->tmpbuf))
                heap_insert (ti, ti->tmpbuf, i);
            else
                isamd_pp_close (ispt[i]);
        }
        while (ti->heapnum)
        {
            int n = ti->indx[ti->ptr[1]];

            rset_write (result, result_rsfd, ti->heap[ti->ptr[1]]);
            nn++;
#if 0
/* section that preserve all keys */
            heap_delete (ti);
            if (isamd_pp_read (ispt[n], ti->tmpbuf))
                heap_insert (ti, ti->tmpbuf, n);
            else
                isamd_pp_close (ispt[n]);
#else
/* section that preserve all keys with unique sysnos */
            while (1)
            {
                if (!isamd_pp_read (ispt[n], ti->tmpbuf))
                {
                    heap_delete (ti);
                    isamd_pp_close (ispt[n]);
                    break;
                }
                if ((*ti->cmp)(ti->tmpbuf, ti->heap[ti->ptr[1]]) > 1)
                {
                    heap_delete (ti);
                    heap_insert (ti, ti->tmpbuf, n);
                    break;
                }
            }
#endif
        }
        heap_close (ti);
        xfree (ispt);
    }
    else if (zi->reg->isams)
    {
        ISAMS_PP *ispt;
        int i;
        struct trunc_info *ti;
        int nn = 0;

        ispt = (ISAMS_PP *) xmalloc (sizeof(*ispt) * (to-from));

        ti = heap_init (to-from, sizeof(struct it_key),
                        key_compare_it);
        for (i = to-from; --i >= 0; )
        {
            ispt[i] = isams_pp_open (zi->reg->isams, isam_p[from+i]);
            if (isams_pp_read (ispt[i], ti->tmpbuf))
                heap_insert (ti, ti->tmpbuf, i);
            else
                isams_pp_close (ispt[i]);
        }
        while (ti->heapnum)
        {
            int n = ti->indx[ti->ptr[1]];

            rset_write (result, result_rsfd, ti->heap[ti->ptr[1]]);
            nn++;
            while (1)
            {
                if (!isams_pp_read (ispt[n], ti->tmpbuf))
                {
                    heap_delete (ti);
                    isams_pp_close (ispt[n]);
                    break;
                }
                if ((*ti->cmp)(ti->tmpbuf, ti->heap[ti->ptr[1]]) > 1)
                {
                    heap_delete (ti);
                    heap_insert (ti, ti->tmpbuf, n);
                    break;
                }
            }
        }
        heap_close (ti);
        xfree (ispt);
    }
    else if (zi->reg->isamb)
    {
        ISAMB_PP *ispt;
        int i;
        struct trunc_info *ti;

        ispt = (ISAMB_PP *) xmalloc (sizeof(*ispt) * (to-from));

        ti = heap_init (to-from, sizeof(struct it_key),
                        key_compare_it);
        for (i = to-from; --i >= 0; )
        {
            ispt[i] = isamb_pp_open (zi->reg->isamb, isam_p[from+i]);
            if (isamb_pp_read (ispt[i], ti->tmpbuf))
                heap_insert (ti, ti->tmpbuf, i);
            else
                isamb_pp_close (ispt[i]);
        }
        while (ti->heapnum)
        {
            int n = ti->indx[ti->ptr[1]];

            rset_write (result, result_rsfd, ti->heap[ti->ptr[1]]);
            nn++;

            if (preserve_position)
            {
                heap_delete (ti);
                if (isamb_pp_read (ispt[n], ti->tmpbuf))
                    heap_insert (ti, ti->tmpbuf, n);
                else
                    isamb_pp_close (ispt[n]);
            }
            else
            {
                while (1)
                {
                    if (!isamb_pp_read (ispt[n], ti->tmpbuf))
                    {
                        heap_delete (ti);
                        isamb_pp_close (ispt[n]);
                        break;
                    }
                    if ((*ti->cmp)(ti->tmpbuf, ti->heap[ti->ptr[1]]) > 1)
                    {
                        heap_delete (ti);
                        heap_insert (ti, ti->tmpbuf, n);
                        break;
                    }
                }
            }
        }
        heap_close (ti);
        xfree (ispt);
    }
    else
        logf (LOG_WARN, "Unknown isam set in rset_trunc_r");

    parms.rset_term->nn = nn;
    rset_close (result, result_rsfd);
    return result;
}

static int isams_trunc_cmp (const void *p1, const void *p2)
{
    ISAMS_P i1 = *(ISAMS_P*) p1;
    ISAMS_P i2 = *(ISAMS_P*) p2;

    return i1 - i2;
}

static int isam_trunc_cmp (const void *p1, const void *p2)
{
    ISAM_P i1 = *(ISAM_P*) p1;
    ISAM_P i2 = *(ISAM_P*) p2;
    int d;

    d = is_type (i1) - is_type (i2);
    if (d)
        return d;
    return is_block (i1) - is_block (i2);
}

static int isamc_trunc_cmp (const void *p1, const void *p2)
{
    ISAMC_P i1 = *(ISAMC_P*) p1;
    ISAMC_P i2 = *(ISAMC_P*) p2;
    int d;

    d = isc_type (i1) - isc_type (i2);
    if (d)
        return d;
    return isc_block (i1) - isc_block (i2);
}
static int isamd_trunc_cmp (const void *p1, const void *p2)
{
    ISAMD_P i1 = *(ISAMD_P*) p1;
    ISAMD_P i2 = *(ISAMD_P*) p2;
    int d;

    d = isamd_type (i1) - isamd_type (i2);
    if (d)
        return d;
    return isamd_block (i1) - isamd_block (i2);
}

RSET rset_trunc (ZebraHandle zi, ISAMS_P *isam_p, int no,
		 const char *term, int length, const char *flags,
                 int preserve_position, int term_type)
{
    logf (LOG_DEBUG, "rset_trunc no=%d", no);
    if (no < 1)
    {
	rset_null_parms parms;
	parms.rset_term = rset_term_create (term, length, flags, term_type);
	return rset_create (rset_kind_null, &parms);
    }
    if (zi->reg->isams)
    {
        if (no == 1)
        {
            rset_isams_parms parms;

            parms.pos = *isam_p;
            parms.is = zi->reg->isams;
	    parms.rset_term = rset_term_create (term, length, flags,
                                                term_type);
            return rset_create (rset_kind_isams, &parms);
        }
        qsort (isam_p, no, sizeof(*isam_p), isams_trunc_cmp);
    }
    else if (zi->reg->isam)
    {
        if (no == 1)
        {
            rset_isam_parms parms;

            parms.pos = *isam_p;
            parms.is = zi->reg->isam;
	    parms.rset_term = rset_term_create (term, length, flags,
                                                term_type);
            return rset_create (rset_kind_isam, &parms);
        }
        qsort (isam_p, no, sizeof(*isam_p), isam_trunc_cmp);
    }
    else if (zi->reg->isamc)
    {
        if (no == 1)
        {
            rset_isamc_parms parms;

            parms.key_size = sizeof(struct it_key);
            parms.cmp = key_compare_it;
            parms.pos = *isam_p;
            parms.is = zi->reg->isamc;
	    parms.rset_term = rset_term_create (term, length, flags,
                                                term_type);
            return rset_create (rset_kind_isamc, &parms);
        }
#if NEW_TRUNC
        else if (no < 10000)
        {
            rset_m_or_parms parms;

            parms.key_size = sizeof(struct it_key);
            parms.cmp = key_compare_it;
            parms.isc = zi->reg->isamc;
            parms.isam_positions = isam_p;
            parms.no_isam_positions = no;
            parms.no_save_positions = 100000;
	    parms.rset_term = rset_term_create (term, length, flags,
                                                term_type);
            return rset_create (rset_kind_m_or, &parms);
        }
#endif
        qsort (isam_p, no, sizeof(*isam_p), isamc_trunc_cmp);
    }
    else if (zi->reg->isamd)
    {
        if (no == 1)
        {
            rset_isamd_parms parms;

            logf(LOG_FATAL, "isam_d does not (currently) support truncs");
            abort();
            /* parms.pos = *isam_p; */
            parms.is = zi->reg->isamd;
	    parms.rset_term = rset_term_create (term, length, flags,
                                                term_type);
            return rset_create (rset_kind_isamd, &parms);
        }
#if NEW_TRUNC_NOT_DONE_FOR_ISAM_D
        else if (no < 10000)
        {
            rset_m_or_parms parms;

            parms.key_size = sizeof(struct it_key);
            parms.cmp = key_compare_it;
            parms.isc = 0;
            parms.isamd=zi->reg->isamd;
            parms.isam_positions = isam_p;
            parms.no_isam_positions = no;
            parms.no_save_positions = 100000;
	    parms.rset_term = rset_term_create (term, length, flags);
            return rset_create (rset_kind_m_or, &parms);
        }
#endif
        qsort (isam_p, no, sizeof(*isam_p), isamd_trunc_cmp);
    }
    else if (zi->reg->isamb)
    {
        if (no == 1)
        {
            rset_isamb_parms parms;

            parms.key_size = sizeof(struct it_key);
            parms.cmp = key_compare_it;
            parms.pos = *isam_p;
            parms.is = zi->reg->isamb;
	    parms.rset_term = rset_term_create (term, length, flags,
                                                term_type);
            return rset_create (rset_kind_isamb, &parms);
        }
        qsort (isam_p, no, sizeof(*isam_p), isamd_trunc_cmp);
    }
    else
    {
        logf (LOG_WARN, "Unknown isam set in rset_trunc");
	return rset_create (rset_kind_null, NULL);
    }
    return rset_trunc_r (zi, term, length, flags, isam_p, 0, no, 100,
                         preserve_position, term_type);
}