livcode.c

harvest是一个下载html网页得机器人

/*
  
The University of Liverpool

Modifications to Zebra 1.1 / YAZ 1.7 to enable ranking
by attribute weight.

Copyright (c) 2001-2002 The University of Liverpool.  All
rights reserved.

Licensed under the Academic Free License version 1.1.
http://opensource.org/licenses/academic.php

$Id: livcode.c,v 1.1 2003/03/26 16:41:48 adam Exp $

*/

#include <stdlib.h>
#include <stdio.h>
#ifdef WIN32
#include <process.h>
#else
#include <unistd.h>
#endif
#include <assert.h>

#include "index.h"
#include "zserver.h"

/*
** These functions/routines 
** 1. reads in and builds a linked list of rank attr/rank score pairs
** 2. expand a simple query into a paired list of complex/simple nodes.
*/

typedef struct rstype
{
    struct rstype *next_rsnode ;
    int rank ;
    int score ;
    char *rankstr ;
} rsnode, *refrsnode ;

refrsnode start_rsnode = NULL ;

/*
** Function/Routine prototypes
*/
static int        search_for_score( char *rankstr ) ;
static char      *search_for_rankstr( int rank ) ;
static int        search_for_rank( int rank ) ;
static refrsnode  set_rsnode( int rank, int score ) ;
static int        read_zrank_file(ZebraHandle zh) ;

static void       convert_simple2complex(ZebraHandle zh, Z_RPNStructure *rpnstruct ) ;
static void       walk_complex_query(ZebraHandle zh, Z_RPNStructure *rpnstruct ) ;
static Z_Complex *expand_query(ZebraHandle zh, Z_Operand *thisop ) ;
static Z_Complex *set_1complex_1operand( Z_Complex *comp,Z_Operand *simp ) ;
static Z_Complex *set_2operands( Z_Operand *sim1,Z_Operand *sim2 ) ;
static Z_Operand *set_operand( Z_Operand *thisop, int newattr ) ;
static int        check_operand_attrs( Z_Operand *thisop ) ;

/*
** search_for_score()
** given a rank-string traverse down the linked list ;
** return its score if found otherwise return -1.
*/
int search_for_score( char *rankstr )
{
    refrsnode node = start_rsnode ;
    int rank ;
  
    if ( sscanf( rankstr,"%d",&rank ) )
    {
        while ( node )
        {
            if ( node->rank == rank ) return node->score ;
            node = node->next_rsnode ;
        }
    }
    return -1 ;
}

/*
** search_for_rankstr()
** given a rank traverse down the linked list ; 
** return its string if found otherwise return NULL.
*/
char *search_for_rankstr( int rank )
{
    refrsnode node = start_rsnode ;
  
    while ( node )
    {
        if ( node->rank == rank ) return node->rankstr ;
        node = node->next_rsnode ;
    }
    return "rank" ;
}

/*
** search_for_rank()
** given a rank traverse down the linked list ;
** return 1 if found otherwise return 0.
*/
int search_for_rank( int rank )
{
    refrsnode node = start_rsnode ;

    while ( node )
    {
        if ( node->rank == rank ) return 1 ;
        node = node->next_rsnode ;
    }
    return 0 ;
}

/*
** set_rsnode()
** given a rank and a score, build the rest of the rsnode structure.
*/
refrsnode set_rsnode( int rank, int score )
{
#define BUFFMAX 128
    refrsnode node = (refrsnode)malloc( sizeof(rsnode) ) ;
    char buff[BUFFMAX] ;

    node->next_rsnode = NULL  ;
    node->rank        = rank  ;
    node->score       = score ;

    sprintf( buff,"%d",rank ) ;
    node->rankstr     = (char *)malloc( strlen(buff)+1 ) ;
    strcpy( node->rankstr, buff ) ;

    return node ;
}

/*
** read_zrank_file(zh)
** read in the rankfile and build the rank/score linked list ; 
** return 0 : can't open the zebra config. file
** return 0 : can't find the rankfile entry in the zebra config. file
** return 0 : can't open the rankfile itself
** return the number of distinct ranks read in.
*/
int read_zrank_file(ZebraHandle zh)
{
#define LINEMAX 256
    char line[ LINEMAX ] ;
    char rname[ LINEMAX ] ;
    char *lineptr ;
    FILE *ifd ;
    int rank = 0 ;
    int score = 0 ;
    int numranks = 0 ;
 
    /*
    ** open the zebra configuration file and look for the "rankfile:"
    ** entry which contains the path/name of the rankfile
    */
    
    const char *rankfile = res_get_def(zh->res, "rankfile", 0);
    const char *profilePath = res_get_def(zh->res, "profilePath",
                                          DEFAULT_PROFILE_PATH);

    if (!rankfile)
    {
        yaz_log(LOG_LOG, "rankfile entry not found in config file" ) ;
        return 0 ;
    }
    ifd = yaz_path_fopen(profilePath, rankfile, "r" ) ;
    if ( ifd )
    {
        while ( (lineptr = fgets( line,LINEMAX,ifd )) )
        {
            if ( sscanf( lineptr,"rankfile: %s", rname ) == 1 )
                rankfile  = rname ;
        }

        /*
        ** open the rankfile and read the rank/score pairs
        ** ignore 1016 
        ** ignore duplicate ranks 
        ** ignore ranks without +ve scores
        */
        if ( rankfile )
        { 
            if ( !(ifd = fopen( rankfile, "r" )) )
            {
                logf( LOG_LOG, "unable to open rankfile %s",rankfile ) ;
                return 0;
            }
     
            while ( (lineptr = fgets( line,LINEMAX,ifd )) )
            {
                sscanf( lineptr,"%d : %d", &rank,&score ) ;
                if ( ( score > 0 ) && ( rank != 1016 ) )
                {
                    refrsnode new_rsnode ;
    
                    if ( search_for_rank( rank ) == 0 )
                    {
                        new_rsnode              = set_rsnode( rank,score ) ;
                        new_rsnode->next_rsnode = start_rsnode ;
                        start_rsnode            = new_rsnode ;
                        numranks++ ;
                    }
                }
            }
        }
        else 
        {
            yaz_log(LOG_WARN|LOG_ERRNO, "unable to open config file (%s)",
                    rankfile);
        }
    }
    return numranks ;
}

/*
** set_operand() 
** build an operand "node" - hav to make a complete copy of thisop and 
** then insert newattr in the appropriate place
** 
*/
Z_Operand *set_operand( Z_Operand *thisop, int newattr )
{
    Z_Operand            *operand ;
    Z_AttributesPlusTerm *attributesplusterm ;
    Z_AttributeList      *attributelist ;
    Z_AttributeElement   *attributeelement ;
    Z_AttributeElement   *attrptr ;
    Z_AttributeElement   **attrptrptr ;
    Z_Term               *term ;
    Odr_oct              *general ;
    int i ;

    operand            = (Z_Operand *) 
                         malloc( sizeof( Z_Operand ) ) ;
    attributesplusterm = (Z_AttributesPlusTerm *)
                         malloc( sizeof( Z_AttributesPlusTerm ) ) ;
    attributelist      = (Z_AttributeList *)
                         malloc( sizeof( Z_AttributeList ) ) ;
    attributeelement   = (Z_AttributeElement *)
                         malloc( sizeof( Z_AttributeElement ) ) ;
    term               = (Z_Term *)
                         malloc( sizeof( Z_Term ) ) ;
    general            = (Odr_oct *)
                         malloc( sizeof( Odr_oct ) ) ;

    operand->which                 = Z_Operand_APT ;
    operand->u.attributesPlusTerm  = attributesplusterm ;

    attributesplusterm->attributes = attributelist ;
    attributesplusterm->term       = term ;

    attributelist->num_attributes  = thisop->u.attributesPlusTerm->
                                     attributes->num_attributes ;

    attrptr = (Z_AttributeElement *) malloc( sizeof(Z_AttributeElement) *
                                             attributelist->num_attributes ) ;
    attrptrptr = (Z_AttributeElement **) malloc( sizeof(Z_AttributeElement) *
                                           attributelist->num_attributes ) ; 

    attributelist->attributes = attrptrptr ;

    for ( i = 0 ; i < attributelist->num_attributes ; i++ )
    {
        *attrptr = *thisop->u.attributesPlusTerm->attributes->attributes[i] ;

         attrptr->attributeType  = (int *)malloc( sizeof(int *) ) ;
        *attrptr->attributeType  = *thisop->u.attributesPlusTerm->attributes->
                                    attributes[i]->attributeType;

         attrptr->value.numeric  = (int *)malloc( sizeof(int *) ) ;
        *attrptr->value.numeric  = *thisop->u.attributesPlusTerm->attributes->
                                    attributes[i]->value.numeric;

        if ( (*attrptr->attributeType == 1) && 
             (*attrptr->value.numeric == 1016) )
        {
            *attrptr->value.numeric = newattr ;
        }
        *attrptrptr++ = attrptr++ ;
    }

    term->which     = Z_Term_general ;
    term->u.general = general ;

    general->len  = thisop->u.attributesPlusTerm->term->u.general->len ;
    general->size = thisop->u.attributesPlusTerm->term->u.general->size ;
    general->buf  = malloc( general->size ) ;
    strcpy( general->buf,
            thisop->u.attributesPlusTerm->term->u.general->buf ) ;

    return operand ;
}

/*
** set_2operands()
** build a complex "node" with two (simple) operand "nodes" as branches
*/
Z_Complex *set_2operands( Z_Operand *sim1,Z_Operand *sim2 )
{
    Z_Complex *top        ;
    Z_RPNStructure *s1    ;
    Z_RPNStructure *s2    ;
    Z_Operator *roperator ;

    top       = (Z_Complex *)     malloc( sizeof( Z_Complex      ) ) ;
    s1        = (Z_RPNStructure *)malloc( sizeof( Z_RPNStructure ) ) ;
    s2        = (Z_RPNStructure *)malloc( sizeof( Z_RPNStructure ) ) ;
    roperator = (Z_Operator *)    malloc( sizeof( Z_Operator     ) ) ;

    top->roperator          = roperator     ;
    top->roperator->which   = Z_Operator_or ;
    top->roperator->u.op_or = odr_nullval() ;

    top->s1                 = s1                    ;
    top->s1->which          = Z_RPNStructure_simple ;
    top->s1->u.simple       = sim1                  ;

    top->s2                 = s2                    ;
    top->s2->which          = Z_RPNStructure_simple ;
    top->s2->u.simple       = sim2                  ;

    return top ;
}

/*
** set_1complex_1operand()
** build a complex "node" with a complex "node" branch and an 
** operand "node" branch
*/
Z_Complex *set_1complex_1operand( Z_Complex *comp,Z_Operand *simp )
{
    Z_Complex *top        ;
    Z_RPNStructure *s1    ;
    Z_RPNStructure *s2    ;
    Z_Operator *roperator ;

    top       = (Z_Complex *)     malloc( sizeof( Z_Complex      ) ) ;
    s1        = (Z_RPNStructure *)malloc( sizeof( Z_RPNStructure ) ) ;
    s2        = (Z_RPNStructure *)malloc( sizeof( Z_RPNStructure ) ) ;
    roperator = (Z_Operator *)    malloc( sizeof( Z_Operator     ) ) ;

    top->roperator          = roperator     ;
    top->roperator->which   = Z_Operator_or ;
    top->roperator->u.op_or = odr_nullval() ;

    top->s1                 = s1                     ;
    top->s1->which          = Z_RPNStructure_complex ;
    top->s1->u.complex      = comp                   ;