tagger.h

Conditional Random Fields的训练识别工具

/*
  CRF++ -- Yet Another CRF toolkit

  $Id: tagger.h 1558 2006-11-25 04:59:20Z taku $;

  Copyright(C) 2005 Taku Kudo <[e-mail]>

  This is free software with ABSOLUTELY NO WARRANTY.

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or(at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/
#ifndef _CRFPP_TAGGER_H
#define _CRFPP_TAGGER_H

#include <iostream>
#include <vector>
#include <queue>
#include "param.h"
#include "crfpp.h"
#include "scoped_ptr.h"
#include "feature_index.h"

namespace CRFPP {

  static inline double toprob(Node *n, double Z) {
    return std::exp(n->alpha + n->beta - n->cost - Z);
  }

  class TaggerImpl : public Tagger {
  private:

    struct QueueElement {
      Node *node;
      QueueElement *next;
      double fx;
      double gx;
    };

    class QueueElementComp {
    public:
      const bool operator()(QueueElement *q1,
                            QueueElement *q2)
        { return(q1->fx > q2->fx); }
    };

    enum { TEST, LEARN };
    unsigned int mode_   : 2;
    unsigned int vlevel_ : 3;
    unsigned int nbest_  : 11;
    size_t ysize_;
    double cost_;
    double Z_;
    size_t feature_id_;
    unsigned short thread_id_;
    FeatureIndex *feature_index_;
    std::vector <std::vector <const char *> > x_;
    std::vector <std::vector <Node *> > node_;
    std::vector <unsigned short int> answer_;
    std::vector <unsigned short int> result_;
    whatlog what_;
    string_buffer os_;

    scoped_ptr<std::priority_queue <QueueElement*, std::vector <QueueElement *>,
      QueueElementComp> > agenda_;
    scoped_ptr<FreeList <QueueElement> > nbest_freelist_;

    void forwardbackward();
    void viterbi();
    void buildLattice();
    bool initNbest();
    bool add2(size_t, const char **, bool);

  public:
    explicit TaggerImpl(): mode_(TEST), vlevel_(0), nbest_(0),
      ysize_(0), Z_(0), feature_id_(0),
      thread_id_(0), feature_index_(0) {};
    virtual ~TaggerImpl() { close(); };

    void   set_feature_id(size_t id) { feature_id_ = id; }
    size_t feature_id() { return feature_id_; }
    void   set_thread_id(unsigned short id) { thread_id_ = id; }
    unsigned short thread_id() { return thread_id_; }
    Node  *node(size_t i, size_t j) { return node_[i][j]; }
    void   set_node(Node *n, size_t i, size_t j) { node_[i][j] = n; }
    int    eval();
    double gradient(double *);
    double collins(double *);
    bool   shrink();
    bool   parse(std::istream &, std::ostream &);
    bool   read(std::istream &);
    bool   open(Param &);
    bool   open(FeatureIndex *);
    bool   open(const char*);
    bool   open(int, char **);
    void   close();
    bool   add(size_t, const char **);
    bool   add(const char*);
    size_t size() { return x_.size(); }
    size_t xsize() { return feature_index_->xsize(); }
    size_t dsize() { return feature_index_->size(); }
    const float *weight_vector() { return feature_index_->alpha_float(); }
    bool   empty() { return x_.empty(); }
    size_t ysize() { return ysize_; }
    double cost() { return cost_; }
    double Z() { return Z_; }
    double prob() { return std::exp(- cost_ - Z_); }
    double prob(size_t i, size_t j) { return toprob(node_[i][j], Z_); }
    double prob(size_t i) { return toprob(node_[i][result_[i]], Z_); }
    double alpha(size_t i, size_t j) { return node_[i][j]->alpha; }
    double beta(size_t i, size_t j) { return node_[i][j]->beta; }
    double emission_cost(size_t i, size_t j) { return node_[i][j]->cost; }
    double next_transition_cost(size_t i, size_t j, size_t k) {
      return node_[i][j]->rpath[k]->cost;
    }
    double prev_transition_cost(size_t i, size_t j, size_t k) {
      return node_[i][j]->lpath[k]->cost;
    }
    double best_cost(size_t i, size_t j) { return node_[i][j]->bestCost; }
    const int *emission_vector(size_t i, size_t j) { return const_cast<int *>(node_[i][j]->fvector); }
    const int* next_transition_vector(size_t i, size_t j, size_t k) {
      return node_[i][j]->rpath[k]->fvector;
    }
    const int* prev_transition_vector(size_t i, size_t j, size_t k) {
      return node_[i][j]->lpath[k]->fvector;
    }
    size_t answer(size_t i) { return answer_[i]; }
    size_t result(size_t i) { return result_[i]; }
    size_t y(size_t i)      { return result_[i]; }
    const char*  yname(size_t i)     { return feature_index_->y(i); }
    const char*  y2(size_t i)      { return yname(result_[i]); }
    const char*  x(size_t i, size_t j) { return x_[i][j]; }
    const char** x(size_t i)           { return &x_[i][0]; }
    const char* toString();
    const char* toString(char *, size_t);
    const char* parse(const char*);
    const char* parse(const char*, size_t);
    const char* parse(const char*, size_t, char*, size_t);
    bool parse();
    bool clear();
    bool next();

    const char* what() { return what_.str(); }
  };
}

#endif