amisfixformat.h

Amis - A maximum entropy estimator 一个最大熵模型统计工具

//////////////////////////////////////////////////////////////////////
///
///  Copyright (c) 2000, Yusuke Miyao
///  You may distribute under the terms of the Artistic License.
///
///  <id>$Id: AmisFixFormat.h,v 1.7 2003/05/16 05:11:05 yusuke Exp $</id>
///  <collection>Maximum Entropy Estimator</collection>
///  <name>AmisFixFormat.h</name>
///  <overview>Parser for Amis-style event files for fixed targets</overview>
///
//////////////////////////////////////////////////////////////////////

#ifndef Amis_AmisFixFormat_h_
#define Amis_AmisFixFormat_h_

#include <amis/configure.h>
#include <amis/ModelFormat.h>
#include <amis/AmisEventFormat.h>
#include <amis/ModelFix.h>
#include <amis/EventFixSpace.h>
#include <amis/Tokenizer.h>
#include <amis/Property.h>

AMIS_NAMESPACE_BEGIN

template < FeatureWeightType FW, class Name = std::string >
class AmisFixModelFormat : public ModelFormat
{
public:
  AmisFixModelFormat() : ModelFormat() {}
  /// Constructor
  virtual ~AmisFixModelFormat() {}
  /// Destructor

  std::string modelFormatName() const {
    return "AmisFixModel";
  }
  /// Name of the model format

protected:
  void readTargetDef( Tokenizer& t, ModelFix< Name >& model_fix ) {
    // target definition
    std::string str;
    while ( t.nextToken( str ) ) {
      IStringStream target_is( str );
      Name target;
      target_is >> target;
      if ( ! target_is ) {
	throw IllegalModelFormatError( "Cannot read target definition: " + str, t.lineNumber() );
      }
      model_fix.newTarget( target );
    }
  }

public:
  void inputModel( std::istream& is, ModelBase& model_base ) {
    ModelFix< Name >* model = dynamic_cast< ModelFix< Name >* >( &model_base );
    if ( model == NULL ) {
      throw IllegalModelFormatError( "AmisFixModelFormat can be used only for ModelFix< ... > class", 0 );
    }
    inputModel( is, *model );
  }
  virtual void inputModel( std::istream& is, ModelFix< Name >& model_fix ) {
    Tokenizer t( is );
    readTargetDef( t, model_fix );
    while ( ! t.endOfStream() ) {
      Name history;
      if ( ! t.nextToken( history ) ) continue;
      FeatureID hid = model_fix.newHistory( history );
      Name target;
      while ( t.nextToken( target ) ) {
	FeatureID tid = model_fix.targetID( target );
	Name feature;
	if ( ! t.nextToken( feature ) ) {
	  IllegalModelFormatError( "Cannot read feature name", t.lineNumber() );
	}
	Real weight;
	if ( ! t.nextToken( weight ) ) {
	  IllegalModelFormatError( "Cannot read feature weight", t.lineNumber() );
	}
	model_fix.newFeature( hid, tid, feature, ModelFormat::inputLambda< FW >( weight ) );
      }
    }
  }

  void outputModel( const ModelBase& model_base, std::ostream& os, int precision = DEFAULT_PRECISION ) const {
    const ModelFix< Name >* model = dynamic_cast< const ModelFix< Name >* >( &model_base );
    if ( model == NULL ) {
      throw IllegalModelFormatError( "AmisFixModelFormat can be used only for ModelFix< ... > class", 0 );
    }
    outputModel( *model, os, precision );
  }
  virtual void outputModel( const ModelFix< Name >& model_fix, std::ostream& os, int precision ) const {
    os.precision( precision );
    os.setf( std::ios::scientific );
    for( int i = 0; i < model_fix.numTargets(); ++i ) {
      os << model_fix.targetName( i ) << " ";
    }
    os << "\n";
    for ( int i = 0; i < model_fix.numHistories(); ++i ) {
      os << model_fix.historyName( i ) << " ";
      std::vector< std::pair<FeatureID, FeatureID> >::const_iterator itr;
      const std::vector< std::pair<FeatureID, FeatureID> >& fv = model_fix.getFeatures( i );
      for(itr = fv.begin(); itr != fv.end(); itr++){
	os << model_fix.targetName( itr->first ) << " ";
	os << model_fix.featureName( itr->second ) << " ";
	os << ModelFormat::outputLambda< FW >( model_fix.getLambda( itr->second ) ) << " "; // lambda
      }
      os << "\n";
    }
    os << std::flush;
  }
};

//////////////////////////////////////////////////////////////////////

template < class Feature, class Name = std::string >
class AmisFixEventFormat : public EventFormat
{
public:
  typedef typename Feature::FeatureFreq FeatureFreq;

  AmisFixEventFormat() : EventFormat() {}
  virtual ~AmisFixEventFormat() {}

  std::string eventFormatName() const {
    return "AmisFixEvent";
  }

  void inputHistoryList( Tokenizer& t, const ModelFix< Name >& model, std::vector< Feature >& hv ) const {
    std::vector< std::pair<std::string, FeatureFreq> > org_vec;
    std::pair< std::string, FeatureFreq >              tmp_pair;
		
    std::string history;
    while ( t.nextToken( history ) ) {
      FeatureFreq val = AmisEventFormat<Feature>::extractFreq( history );
      if ( val <= 0 ) {
	throw IllegalModelFormatError( "History value must be positive in Event " , t.lineNumber() );
      }
      IStringStream history_is( history );
      Name history_feat;
      history_is >> history_feat;
      FeatureID hid = model.historyID( history_feat );

      if( hid >= 0 ) {
	hv.push_back( Feature( hid, val ) );
        /*
	if( make_comb_feature ) {
	  tmp_pair.first  = history;
	  tmp_pair.second = val;
	  org_vec.push_back( tmp_pair );
	}
        */
      }
      else{
	// ignore
      }
    }

    /*		
    if( make_comb_feature ) {
#if defined(AMIS_NAME_INTEGER)
      AMIS_ABORT( "Sorry, make_comb_feature does not work with AMIS_NAME_INTEGER" );
#else 
      for( int i = 0; i < org_vec.size(); i++ ) {
	for( int j = i; j < org_vec.size(); j++ ) {
	  std::string hist = org_vec[i].first + "_" + org_vec[j].first;
	  FeatureFreq val  = org_vec[i].second * org_vec[j].second;
	  FeatureID hid = model.historyID( hist );
	  if( hid >= 0 ) {
	    hv.push_back( Feature( hid, val ) );
	  }
	}
      }
			
			
#endif
    }
    */
		
  }

  /*	
  void readTargetDef( std::istream& is, ModelFix* m ){
    // target definition
    std::string line;
    getline(is, line);
    std::istringstream str(line.c_str());
		
    while(str){
#if defined( AMIS_NAME_INTEGER )
      Name target = -1;
      str >> target;
      if( target == -1 ){ break; }
#else
      Name target = "";
      str >> target;
      if( target == "" ){ break; }
#endif
      m->newTarget( target );
    }
  }
  */

  // read one event from stream
  bool inputEvent( std::istream& is, const ModelFix< Name >& model, EventFix<Feature>& to ) const {
    Tokenizer t( is );
		
    Name target;
    if ( !t.nextToken(target) ) {
      return false;
    }
		
    FeatureID tid = model.targetID( target );
		
    FeatureListFreq freq;
    if( !t.nextToken( freq ) || freq < 0 ){
      throw IllegalEventFormatError( "AmisFixFormat: No frequency or invalid value. line=", t.lineNumber() );
    }
		
    std::vector< Feature > hv;
    inputHistoryList( t, model, hv ); // without registering
    FeatureList<Feature> hl( hv, 1 );
    to.add( tid, freq, hl, model.numTargets() );
    return true;
  }

  void inputEventSpace( std::istream& s, const ModelBase& model_base, EventSpaceBase& event_space_base ) {
    const ModelFix< Name >* model = dynamic_cast< const ModelFix< Name >* >( &model_base );
    if ( model == NULL ) {
      throw IllegalEventFormatError( "AmisFixEventFormat can be used only for ModelFix< ... > class", 0 );
    }
    EventFixSpaceBase< Feature >* event_space =
      dynamic_cast< EventFixSpaceBase< Feature >* >( &event_space_base );
    if ( event_space == NULL ) {
      throw IllegalEventFormatError( "AmisFixEventFormat can be used only for EventFixSpaceBase class", 0 );
    }
    inputEventSpace( s, *model, *event_space );
  }

  virtual void inputEventSpace( std::istream& is, const ModelFix< Name >& model, EventFixSpaceBase< Feature >& event_space ) {
    std::string line;
    getline(is, line); // ignore the first line (target definition)
    while( true ) {
      EventFix<Feature> event;
      bool suc = inputEvent( is, model, event );
      if ( !suc ) { return; }
      event_space.addEvent( event );
    }
  }
};

AMIS_NAMESPACE_END

#endif // Amis_AmisFixFormat_h_
// end of AmisFixFormat.h