amisdriver.cc

Amis - A maximum entropy estimator 一个最大熵模型统计工具

#include <amis/AmisDriver.h>
#include <amis/LogStream.h>
#include <amis/Profile.h>
#include <amis/ErrorBase.h>
#include <amis/MultiStream.h>
#include <iostream>
#ifdef AMIS_USE_TAO
#include "tao.h"
#endif

AMIS_NAMESPACE_BEGIN

#ifdef AMIS_DEBUG_LEVEL
const int AmisDriver::AMIS_DEBUG_TYPE = AMIS_DEBUG_LEVEL;
#else // AMIS_DEBUG_LEVEL
const int AmisDriver::AMIS_DEBUG_TYPE = 0;
#endif // AMIS_DEBUG_LEVEL

#ifdef AMIS_PROFILE_LEVEL
const int AmisDriver::AMIS_PROFILE_TYPE = AMIS_PROFILE_LEVEL;
#else // AMIS_PROFILE_LEVEL
const int AmisDriver::AMIS_PROFILE_TYPE = 0;
#endif // AMIS_PROFILE_LEVEL

#ifdef AMIS_FEATURE_WEIGHT_LAMBDA
const std::string AmisDriver::AMIS_FEATURE_WEIGHT_TYPE( "lambda" );
#else // AMIS_FEATURE_WEIGHT_LAMBDA
const std::string AmisDriver::AMIS_FEATURE_WEIGHT_TYPE( "alpha" );
#endif // AMIS_FEATURE_WEIGHT_LAMBDA

#ifdef AMIS_PARALLEL
const std::string AmisDriver::AMIS_PARALLEL_TYPE( "on" );
#else // AMIS_PARALLEL
const std::string AmisDriver::AMIS_PARALLEL_TYPE( "off" );
#endif // AMIS_PARALLEL

#ifdef AMIS_USE_LONG_DOUBLE
const std::string AmisDriver::AMIS_REAL_TYPE( "long double" );
#else
const std::string AmisDriver::AMIS_REAL_TYPE( "double" );
#endif

#ifdef AMIS_USE_TAO
const std::string AmisDriver::AMIS_USE_TAO_TYPE( "on" );
#else
const std::string AmisDriver::AMIS_USE_TAO_TYPE( "off" );
#endif

//////////////////////////////////////////////////////////////////////

int AmisDriver::run( int argc, const char** argv ) {
  ProfTimer total_timer;
  total_timer.startProf();

  // Read options
  try {
    AMIS_DEBUG_MESSAGE( 1, "Input a property file... " );
    importProperty( argc, argv );
  } catch ( ShowHelpMessage help ) {
    std::cerr << amisName();
    if ( help.isHeavy() ) {
      std::cerr << property.getLongHelp();
    } else {
      std::cerr << property.getShortHelp();
    }
    return 0;
  } catch ( ShowVersion version ) {
    std::cerr << amisName();
    return 0;
  } catch ( ShowOptionStatus option ) {
    std::cerr << amisName();
    std::cerr << property.getSpecificationStatus();
    return 0;
  } catch ( ErrorBase err ) {
    std::cerr << amisName();
    std::cerr << err.message() << std::endl;
    std::cerr << "use -h or --help to see the usage" << std::endl;
    std::cerr << property.getShortHelp();
    return 1;
  }

  // Initialization
  initialize();

  // Print startup messages
  showStartup();

  // Main program
  try {
    // Construct main objects
    AMIS_DEBUG_MESSAGE( 1, "Construct main objects...\n" );
    ModelPtr model = newModel();
    EventSpacePtr event_space = newEventSpace();
    ModelFormatPtr model_format = newModelFormat();
    EventFormatPtr event_format = newEventFormat();
    EstimatorPtr estimator = newEstimator();

    AMIS_PROF_MESSAGE( "  Model:      " << model->modelName() << '\n' );
    AMIS_PROF_MESSAGE( "  EventSpace: " << event_space->eventSpaceName() << '\n' );
    AMIS_PROF_MESSAGE( "  Estimator:  " << estimator->estimatorName() << '\n' );
    AMIS_DEBUG_MESSAGE( 1, "done.\n" );

    // Import model & event files
    AMIS_DEBUG_MESSAGE( 1, "Initialize a model...\n" );
    inputModel( model_format, model );
    AMIS_PROF_MESSAGE( "  # features = " << model->numFeatures() << '\n' );
    AMIS_DEBUG_MESSAGE( 1, "done.\n" );
#if AMIS_DEBUG_MODE( 5 )
    model->debugInfo( std::cerr );
#endif // DEBUG_MODE

    AMIS_DEBUG_MESSAGE( 1, "Initialize events...\n" );
    inputEventSpace( event_format, model, event_space );
    AMIS_PROF_MESSAGE( "  # events = " << event_space->numEvents() );
    AMIS_PROF_MESSAGE( "  # feature lists = " << event_space->numFeatureLists() << '\n' );
    AMIS_PROF_MESSAGE( "  event count = " << event_space->sumEventCount() << '\n' );
    AMIS_DEBUG_MESSAGE( 1, "done.\n" );

    // Run the estimator
    ProfTimer estimation_timer;
    AMIS_DEBUG_MESSAGE( 1, "Start estimation.\n" );
    estimation_timer.startProf();
    estimation( model, event_space, estimator );
    estimation_timer.stopProf();
    AMIS_DEBUG_MESSAGE( 1, "Done.\n" );

    // Output the model
    AMIS_DEBUG_MESSAGE( 1, "Output a model... " );
    outputModel( model_format, model );
    AMIS_DEBUG_MESSAGE( 1, "done.\n" );
    //AMIS_PROF_MESSAGE( "# of nonzero lambdas = " << model->numNonzeroLambdas() << " (out of " << model->numFeatures() << ")" << '\n' );

    total_timer.stopProf();

    // Output statistics if required
    if ( property.getModelStat() ) {
      ModelStatPtr model_stat = property.newModelStat();
      model_stat->setModelStatistics( estimator->getEmpiricalExpectation() );
      outputStatistics( model, model_stat, total_timer, estimation_timer );
    }

    // End...
    finalize();
    showEnding();
  } catch ( ErrorBase err ) {
    AMIS_PROF_MESSAGE( err.message() << '\n' );
    return 1;
  }

  return 0;
}

//////////////////////////////////////////////////////////////////////

std::string AmisDriver::amisName() const {
  OStringStream os;
  os << AMIS_PACKAGE << " " << AMIS_VERSION << " - A maximum entropy estimator by Yusuke Miyao\n";
  os << "[Debug=" << AMIS_DEBUG_TYPE << "] [Profile=" << AMIS_PROFILE_TYPE << "] [Weight=" << AMIS_FEATURE_WEIGHT_TYPE << "] [Parallel=" << AMIS_PARALLEL_TYPE << "] [TAO=" << AMIS_USE_TAO_TYPE << "]" << '\n';
  return os.str();
}

//////////////////////////////////////////////////////////////////////

void AmisDriver::importProperty( int argc, const char** argv ) {
  // Read options
  property.parseArguments( argc, argv );
  if ( property.isHelp() || property.isHeavyHelp() ) {
    throw ShowHelpMessage( "Show help message", property.isHeavyHelp() );
  }
  if ( property.isShowVersion() ) {
    throw ShowVersion( "Show version" );
  }
  property.importProperty();
  property.parseArguments( argc, argv );

  if ( property.isHelp() ) {
    throw ShowHelpMessage( "Show help message" );
  }
  if ( property.isShowOptionStatus() ) {
    throw ShowOptionStatus( "Show option status" );
  }
}

//////////////////////////////////////////////////////////////////////

void AmisDriver::initialize() {
  // Set up logging
  LogStream::DefaultLogStream.setStream( property.getLogFile() );
  // TAO initialization
#ifdef AMIS_USE_TAO
  int tao_info;
  tao_info = PetscInitialize( &argc, &(char**)(argv), TAO_NULL, TAO_NULL );
  CHKERRQ( tao_info );
  tao_info = TaoInitialize( &argc, &(char**)(argv), TAO_NULL, TAO_NULL );
  CHKERRQ( tao_info );
#endif // AMIS_USE_TAO
}

//////////////////////////////////////////////////////////////////////

void AmisDriver::showStartup() {
  AMIS_PROF_MESSAGE( amisName() );
  AMIS_PROF_MESSAGE( "[Real=" << AMIS_REAL_TYPE << "] (" << sizeof( Real ) << " bytes)" << '\n' ); 
  AMIS_PROF_MESSAGE( "[REAL_MAX=" << REAL_MAX << "]" << '\n' );
  AMIS_PROF_MESSAGE( "[REAL_MIN=" << REAL_MIN << "]" << '\n' );
  AMIS_PROF_MESSAGE( "[REAL_EPSILON=" << REAL_EPSILON << "]" << '\n' );
  AMIS_PROF_MESSAGE( "[REAL_INF=" << REAL_INF << "]" << '\n' );
  AMIS_PROF_MESSAGE( "[(REAL_INF > 0.0)=" << (REAL_INF > 0.0) << "]\n" );
  AMIS_PROF_MESSAGE( "[exp(-REAL_INF) =" << exp( -REAL_INF ) << "]\n" );
  AMIS_PROF_MESSAGE( "  Model files:" );
  std::vector<std::string> model_list = property.getModelFileList();
  std::vector<std::string> event_list = property.getEventFileList();
  for ( std::vector< std::string >::const_iterator m = model_list.begin(); m != model_list.end(); ++m ) {
    AMIS_PROF_MESSAGE( " " << *m << "\n" );
  }
  AMIS_PROF_MESSAGE( "  Event files:" );
  for ( std::vector< std::string >::const_iterator e = event_list.begin(); e != event_list.end(); ++e ) {
    AMIS_PROF_MESSAGE( " " << *e << "\n" );
  }
  AMIS_PROF_MESSAGE( "  Output file: " << property.getOutputFile() << '\n' );
  AMIS_DEBUG_MESSAGE( 1, "done.\n" );
}

void AmisDriver::inputModel( const ModelFormatPtr& format, ModelPtr& model ) {
  MultiIFStream model_file( property.getModelFileList() );
  if ( ! model_file ) throw( ErrorBase( "Cannot open a model file stream" ) );
  format->inputModel( model_file, *model );
}

void AmisDriver::inputEventSpace( const EventFormatPtr& format, const ModelPtr& model, EventSpacePtr& event_space ) {
  MultiIFStream event_file( property.getEventFileList() );
  if ( ! event_file ) throw( ErrorBase( "Cannot open an event file stream" ) );
  format->inputEventSpace( event_file, *model, *event_space );
}

void AmisDriver::estimation( ModelPtr& model, EventSpacePtr& event_space, EstimatorPtr& estimator ) {
  estimator->setModel( *model );
  estimator->setEventSpace( *event_space );
  estimator->estimate( property.getNumIterations(),
                       property.getPrecision(),
                       property.getReportInterval() );
}

void AmisDriver::outputModel( const ModelFormatPtr& format, const ModelPtr& model ) {
  std::ofstream output_file( property.getOutputFile().c_str() );
  if ( ! output_file ) throw( ErrorBase( "Cannot open an output file: " + property.getOutputFile() ) );
  format->outputModel( *model, output_file, property.getPrecision() );
}

void AmisDriver::outputStatistics( const ModelPtr& model, const ModelStatPtr& model_stat,
				   const ProfTimer& total_timer, const ProfTimer& estimation_timer ) {
  // not implemented yet
//   AMIS_DEBUG_MESSAGE( 1, "Ouput a mode statistics..." );
  std::ofstream os( property.getModelStatFile().c_str() );
  os << "AmisDriver::outputStatistics() is not implemented yet" << std::endl;
/*
  os << "####Model####" << std::endl;
  os << "Model:NUM_FEATURES " << numFeatures() << std::endl;
  os << "Model:NUM_FREEZED_FEATURES " << freezed_num << std::endl;
  os << "Model:NUM_NONZERO_LAMBDAS " << numNonzeroLambdas() << std::endl;
  os << "Model:LOG_LIKELIHOOD " << log_likelihood << std::endl;
  os << "Model:LOG_LIKELIHOOD_NORM " << log_likelihood_norm  << std::endl;
  os << "Model:ESTIMATION_REAL_SEC " << estimation_real_sec << std::endl;
  os << "Model:ESTIMATION_USER_SEC " << estimation_user_sec << std::endl;
  os << "Model:ESTIMATION_SYS_SEC " << estimation_sys_sec << std::endl;
  os << "Model:TOTAL_REAL_SEC " << total_real_sec << std::endl;
  os << "Model:TOTAL_USER_SEC " << total_user_sec << std::endl;
  os << "Model:TOTAL_SYS_SEC " << total_sys_sec << std::endl;
  os << "Model:NUM_EVENTS "  << getNumEvents() << std::endl;
			
  Real avg_bayes_raw_sigma = accumulate( bayes_raw_sigmas.begin(), bayes_raw_sigmas.end(), 0.0 ) / (Real)bayes_raw_sigmas.size();
  Real avg_bayes_sigma = accumulate( bayes_sigmas.begin(), bayes_sigmas.end(), 0.0 ) / (Real)bayes_sigmas.size();
  Real max_bayes_raw_sigma = *max_element( bayes_raw_sigmas.begin(), bayes_raw_sigmas.end() );
  Real min_bayes_raw_sigma = *min_element( bayes_raw_sigmas.begin(), bayes_raw_sigmas.end() );
  Real max_bayes_sigma = *max_element( bayes_sigmas.begin(), bayes_sigmas.end() );
  Real min_bayes_sigma = *min_element( bayes_sigmas.begin(), bayes_sigmas.end() );
			
  os << "Model:AVG_BAYES_RAW_SIGMA " << avg_bayes_raw_sigma << std::endl;
  os << "Model:AVG_BAYES_SIGMA " << avg_bayes_sigma << std::endl;
  os << "Model:MAX_BAYES_RAW_SIGMA " << max_bayes_raw_sigma << std::endl;
  os << "Model:MIN_BAYES_RAW_SIGMA " << min_bayes_raw_sigma << std::endl;
  os << "Model:MAX_BAYES_SIGMA " << max_bayes_sigma << std::endl;
  os << "Model:MIN_BAYES_SIGMA " << min_bayes_sigma << std::endl;
			
  Real avg_bayes_hist_raw_sigma = accumulate( bayes_hist_raw_sigmas.begin(), bayes_hist_raw_sigmas.end(), 0.0 ) / (Real)bayes_hist_raw_sigmas.size();
  Real avg_bayes_hist_sigma = accumulate( bayes_hist_sigmas.begin(), bayes_hist_sigmas.end(), 0.0 ) / (Real)bayes_hist_sigmas.size();
  Real max_bayes_hist_raw_sigma = *max_element( bayes_hist_raw_sigmas.begin(), bayes_hist_raw_sigmas.end() );
  Real min_bayes_hist_raw_sigma = *min_element( bayes_hist_raw_sigmas.begin(), bayes_hist_raw_sigmas.end() );
  Real max_bayes_hist_sigma = *max_element( bayes_hist_sigmas.begin(), bayes_hist_sigmas.end() );
  Real min_bayes_hist_sigma = *min_element( bayes_hist_sigmas.begin(), bayes_hist_sigmas.end() );
			
  os << "Model:AVG_BAYES_HIST_RAW_SIGMA " << avg_bayes_hist_raw_sigma << std::endl;
  os << "Model:AVG_BAYES_HIST_SIGMA " << avg_bayes_hist_sigma << std::endl;
  os << "Model:MAX_BAYES_HIST_RAW_SIGMA " << max_bayes_hist_raw_sigma << std::endl;
  os << "Model:MIN_BAYES_HIST_RAW_SIGMA " << min_bayes_hist_raw_sigma << std::endl;
  os << "Model:MAX_BAYES_HIST_SIGMA " << max_bayes_hist_sigma << std::endl;
  os << "Model:MIN_BAYES_HIST_SIGMA " << min_bayes_hist_sigma << std::endl;

  if ( heavy ) {
    os << "Model:FEATURE_INFO " << std::endl; 
    for ( int i = 0; i < numFeatures(); ++i ) {
      os << featureName( i ) << ' ' << "count=" << feature_count[ i ] << ' ' << "emp=" << emps[ i ] << ' ' << "bayes_raw_sigma=" << bayes_raw_sigmas[i] << ' ' << "bayes_sigma=" << bayes_sigmas[ i ] << std::endl;
    }
    os << std::endl;
				
    os << "Model:FEATURE_SORTED_BY_LAMBDA_ABSOLUTE_VALUE " << std::endl;
				
    std::vector< std::pair<FeatureID, Real> > sort_vec;
    for ( int i = 0; i < numFeatures(); i++ ) {
      sort_vec.push_back( std::pair<FeatureID, Real>( i, fabs( (*this)[ i ] ) ) );
    }
				
    sort( sort_vec.begin(), sort_vec.end(), IndexValuePairCmpByValue<FeatureID, Real>() );
				
    // into descreasing order
    reverse( sort_vec.begin(), sort_vec.end() );
				
    for ( int i = 0; i < numFeatures(); i++ ) {
      FeatureID fid = sort_vec[i].first;
      os << featureName( fid ) << " " << (*this)[ fid ] << std::endl;
    }
    os << std::endl;
  }
*/
}

void AmisDriver::finalize() {
  /// TAO Finalize
#ifdef AMIS_USE_TAO
  tao_info = TaoFinalize();
  CHKERRQ( tao_info );
  tao_info = PetscFinalize();
  CHKERRQ( tao_info );
#endif // AMIS_USE_TAO
}

void AmisDriver::showEnding() {
  AMIS_PROF_MESSAGE( Profile::profileResult() );
}

AMIS_NAMESPACE_END

// end of AmisDriver.cc