mpiexec.c

刚才是说明 现在是安装程序在 LINUX环境下进行编程的MPICH安装文件

/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*  $Id: mpiexec.c,v 1.12 2002/11/15 22:04:21 gropp Exp $
 *
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

/* quick hack forking mpiexec process manager and implementation of PMI */
/*
   1. parse mpiexec args.  See p 353 of vol 1 of MPI std.
   2. initialize data structures:
        fdtable has one entry for each forked process, containing, among other things, 
          an fd for communicating with that process
        kvs table has one entry for each keyval space.  A keyval space contains
	  key=value pairs for put and get.
        first group id assigned will be 0
   3. fork-exec executables, passing rank, pmi listening port in env, other args
   4. enter select
   5. handle requests from clients
   6. collect finalize messages, exit when all in or pmi_abort
*/
  
/*
  The todo list:
    Add debug argument to mpiexec and use it to control PMIU_printf
    Implement PMI_Spawn_multiple
    Set up SIGCHLD handler for unexpected client deaths
        (This is partially done, but needs to be integrated with the
	handling of process exit.  At least you get one message now)
    Handle mpiexec -path argument correctly
*/

#include "forkerconf.h"

#include <stdio.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <stdlib.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <errno.h>
#ifdef HAVE_SIGNAL_H
#include <signal.h>
#endif
#include "simple_pmiutil.h"

void mpiexec_usage( const char * );
int  GetIntValue( const char [], int );
void InitTimeout( int );
int  GetRemainingTime( void );
int  allocate_fdentry( void );
int  handle_input_fd( int );
#ifdef USE_SIGCHLD_HANDLER
void setup_sigchild( void );
#endif
void KillChildren( void );
void PrintExitStatus( void );
int GetExitStatus( void );
void forkProcesses( int, char [], char *[], char *[], char [], int, char [] );
void SignalAllProcesses( int , const char [] );
/* int  waitOnProcess( int, int, exit_state_t ); - below for exit_state_t def */

/* Routines for tracing */
int  InitHandleStopped( void );
void CheckIfTraced( void );
void CheckForStopped( const char [] );
void SetCommandOnStopped( const char [] );
void SetDefaultCommandOnStopped( void );
void KillTracedProcesses( void );

/* Functions that handle PMI requests */
int  fPMI_Init( int, char [] );
int  fPMI_Allocate_kvs( int *, char [] );
int  fPMI_Allocate_kvs_group( void );
void fPMI_Handle_barrier( int );
void fPMI_Handle_create_kvs( int );
void fPMI_Handle_destroy_kvs( int );
void fPMI_Handle_put( int );
void fPMI_Handle_get( int );
void fPMI_Handle_get_my_kvsname( int );
void fPMI_Handle_get_maxes( int );
void fPMI_Handle_getbyidx( int );

/* array sizes */
#define MAXNAMELEN  256		/* max length of vairous names */
#ifndef MAXPATHLEN
#define MAXPATHLEN 2048		/* max length of PATH */
#endif
#define MAXPROCS     64		/* max number of processes to manage */
#define MAXFDENTRIES 75		/* max number of clients (procs), plus a few more */

/* handlers */
#define NOTSET        0         /* no handler set on fd */
#define CLIENT        1         /* handler for pipe to client */

/* client states, maybe not all used (yet) */
typedef enum { UNINITIALIZED=-1, UNKNOWN, ALIVE, COMMUNICATING, FINALIZED, EXITING, GONE } 
    client_state_t;

/* Record the return value from each process */
typedef enum { NORMAL,     /* Normal exit (possibly with nonzero status) */
	       SIGNALLED,  /* Process died on an uncaught signal (e.g., 
			      SIGSEGV) */
	       NOFINALIZE, /* Process exited without calling finalize */
	       KILLED      /* Process was killed by mpiexec */ 
             } exit_state_t;
struct pidstat { 
    int          rc;
    int          sig;
    exit_state_t exit_state;
};
int  waitOnProcess( int, int, exit_state_t );

/*
 * Each in-use fd gets an fdentry.  Because of MPI_Comm_spawn, there isn't
 * necessarily a 1-1 correspondence between these entries and the processes
 * (though there is a 1-1 for the initial set of processes, i.e., those in 
 * MPI_COMM_WORLD).
 */
struct fdentry {
    int  active;		/* whether this entry is filled */
    int  fd;			/* fd assigned by system when opened */
    int  pid;			/* pid of client, if applicable, else -1 */
    int  group;			/* group that client belongs to, if applicable, else -1 */
    int  rank;			/* rank of client, if applicable, else -1 */
    client_state_t state;	/* state of client, if applicable, else -1 */
    int  read;			/* whether this fd should be selected for reading */
    int  write;			/* whether this fd should be selected for writing */
    FILE *file;			/* file from fdopen, if present, for using fgets */
    int  handler;		/* function to call to handle input/output when selected */
    char name[MAXNAMELEN];	/* name of fd, for debugging */
    char kvsname[MAXNAMELEN];	/* name of KVS associated with this process, if any */
};
struct fdentry fdtable[MAXFDENTRIES];
int maxfdentryInUse = -1;       /* Index of the last fdentry in use */

/* Because we may reuse fdentries, we will eventually need to keep
   the results of all process creation in a separate array.  Since this
   is just a stack of results, we don't need a special data structure 
   for it.  We save the exitstatus of each process */
struct pidstat exitstatus[MAXFDENTRIES];
/* How many processes have exited (including the number aborted) */
int num_exited = 0;
/* How many processes have exited with a nonzero code or a signal */
int num_aborted = 0;

/* This is the name of the kvs associated with 
   the initial group of processes */
char initial_kvsname[MAXNAMELEN];

/*
 * Global variables and default values
 */
int numprocs = 1; /* Default if no -n argument provided */
int debug = 0;

/* Set killOnAbort to 1 to kill all children when one dies with a nonzero
   return code or a signal; 0 to continue */
int killOnAbort = 1;
/* This flag lets us know when we're terminating the children */
int inKillChildren = 0;


#define MAX_CLIENT_ARG 50
#define MAX_CLIENT_ENV 200
/* Note that envp is common but not standard */
int main( int argc, char *argv[], char *envp[] )
{
    int  n;
    char softness[MAXNAMELEN];
    char hostname[MAXNAMELEN];
    char archname[MAXNAMELEN];
    char wdirname[MAXNAMELEN];
    char pathname[MAXPATHLEN];
    char filename[MAXNAMELEN];
    char execname[MAXNAMELEN];
    char *client_arg[MAX_CLIENT_ARG];
    int  i, j, rc, num_fds;
#ifdef USE_SIGCHLD_HANDLER
    int  catch_sigchild = 0;
#endif
    int  groupid;
    int  done;
    int  timeout_seconds;
    int  poll_deltat = 0;
    struct timeval tv;
    fd_set readfds, writefds;

    strncpy( PMIU_print_id, "mpiexec", PMIU_IDSIZE );

/**************************** argument processing **********************************/

    /* set defaults for arguments.  Most of these can also be set
       with environment variables */
    n = 1;			  /* number of processes to start */
    /* Simple test for debugging */
    if (getenv( "MPIEXEC_DEBUG" )) debug = 1;
    timeout_seconds = GetIntValue( "MPIEXEC_TIMEOUT", 60 ) * 60;
    if (debug) printf( "timeout_seconds = %d\n", timeout_seconds );

    /* Defaults for handling stopped processes */
    SetDefaultCommandOnStopped();

    if ( !getcwd( wdirname, MAXNAMELEN ) ) {
	mpiexec_usage( "current working directory name too long\n" );
	/* mpiexec_usage exits */
    }

    strncpy( pathname, getenv( "PATH" ), MAXPATHLEN );
    strncpy( execname, "a.out", 6 );
    /* others default to empty strings */
    softness[0] = hostname[0] = archname[0] = filename[0] = '\0';
    
    /* process argments */
    if ( argc == 1 )
	mpiexec_usage( NULL );	/* note: mpiexec_usage exits */

    /* process std args before execname: -n, -soft, -host, -arch, -wdir, -path, -file */
    for ( i = 1; i < argc && argv[i][0] == '-'; i+=2 ) {
	if ( strncmp( argv[i], "-n",  strlen( argv[i] ) ) == 0 ||
	     strncmp( argv[i], "-np", strlen( argv[i] ) ) == 0 ) /* undoc'd option */
	    if ( i+1 < argc ) {
		n = atoi( argv[i+1] );
		if ( 0 < n && n <= MAXPROCS )
		    numprocs = n;
		else {
		    fprintf( stderr, "invalid number of processes, max is %d\n",MAXPROCS );
		    exit( -1 );
		}
	    }
	    else
		mpiexec_usage( "Missing argument to -n\n" );
	else if ( strncmp( argv[i], "-soft", 6 ) == 0 )
	    if ( i+1 < argc )
		strncpy( softness, argv[i+1], MAXNAMELEN );
	    else
		mpiexec_usage( "Missing argument to -soft\n" );
	else if ( strncmp( argv[i], "-host", 6 ) == 0 )
	    if ( i+1 < argc )
		strncpy( hostname, argv[i+1], MAXNAMELEN );
	    else
		mpiexec_usage( "Missing argument to -host\n" );		    
	else if ( strncmp( argv[i], "-arch", 6 ) == 0 )
	    if ( i+1 < argc )
		strncpy( archname, argv[i+1], MAXNAMELEN );
	    else
		mpiexec_usage( "Missing argument to -arch\n" );		    
	else if ( strncmp( argv[i], "-wdir", 6 ) == 0 )
	    if ( i+1 < argc )
		strncpy( wdirname, argv[i+1], MAXNAMELEN );
	    else
		mpiexec_usage( "Missing argument to -wdir\n" );		    
	else if ( strncmp( argv[i], "-path", 6 ) == 0 )
	    if ( i+1 < argc )
		strncpy( pathname, argv[i+1], MAXPATHLEN );
	    else
		mpiexec_usage( "Missing argument to -path\n" );		    
	else if ( strncmp( argv[i], "-file", 6 ) == 0 ) {
	    if ( i+1 < argc )
		strncpy( filename, argv[i+1], MAXNAMELEN );
	    else
		mpiexec_usage( "Missing argument to -file\n" );
	}
	/* Here begin the options that are not part of the MPI 
	   recommendations */
	else if ( strncmp( argv[i], "-maxtime", 8 ) == 0 ) {
	    if ( i+1 < argc ) 
		timeout_seconds = atoi( argv[i+1] );
	    else
		mpiexec_usage( "Missing argument to -maxtime\n" );
	}
	else if ( strncmp( argv[i], "-onsig", 6 ) == 0) {
	    if ( i + 1 < argc )
		SetCommandOnStopped( argv[i+1] );
	    else
		mpiexec_usage( "Missing argument to -onsig\n" );
	}
	else {
	    fprintf( stderr, "invalid mpiexec argument %s\n", argv[i] );
	    mpiexec_usage( NULL );
	}
    }

    if ( i < argc )
	strncpy( execname, argv[i], strlen( argv[i] ) + 1 );
    else {
	mpiexec_usage( "no program specified\n" );
    }

    /*
    fprintf( stdout, "arguments: n = %d, softness = %s, hostname = %s, "
	     "archname = %s, wdirname = %s, pathname = %s, filename = %s, execname = %s\n",
	     n, softness, hostname, archname, wdirname, pathname, filename, execname );
    */

    /* Start the timeout period */
    InitTimeout( timeout_seconds );

    poll_deltat = InitHandleStopped();

    /* now process application program arguments */
    for ( j = 0; i < argc && j < MAX_CLIENT_ARG; i++, j++ )
	client_arg[j] = argv[i]; /* client_arg[0] is execname */
    if (j == MAX_CLIENT_ARG) {
	
    }
    client_arg[j] = NULL;	 /* null out arg pointer after last arg */

/******************************* other initialization *****************************/

    for ( i = 0; i < MAXFDENTRIES; i++ )
	fdtable[i].active = 0;
    groupid = fPMI_Init( numprocs, initial_kvsname );

/***************************  fork client processes **********************************/

    /* Catch process exit unless specifically requested otherwise */
#ifdef USE_SIGCHLD_HANDLER
    if (catch_sigchild) 
	setup_sigchild( );
#endif

    forkProcesses( numprocs, execname, client_arg, envp, wdirname, 
		   groupid, initial_kvsname );
    
    /* Allocated a new group (this should be deferred until we decide to
       fork more processes) */ 
    groupid = fPMI_Allocate_kvs_group();

/************************ enter select loop to process pmi requests *****************/
    
    done = 0;
    while ( !done ) {
        FD_ZERO( &readfds );
        FD_ZERO( &writefds );

	/* Start with num_fds as the maximum fd seen */
	num_fds = -1;
	for ( i = 0; i < MAXFDENTRIES; i++ )
	    if ( fdtable[i].active && fdtable[i].read ) {
		FD_SET( fdtable[i].fd, &readfds );
		if (fdtable[i].fd > num_fds) 
		    num_fds = fdtable[i].fd;
	    }
	    
	/* Now make this the number of fds */
        num_fds++;

	/* Set a timeout on the select.  This allows us to set limits
	   on how long a parallel job can run, and helps catch
	   deadlocked programs */
        tv.tv_sec  = GetRemainingTime();
	/* We may also want to check for special events outside of
	   select.  Currently, this means ptrace (requires calling
	   wait).  */
	if (poll_deltat > 0) {
	    if (poll_deltat < tv.tv_sec) tv.tv_sec = poll_deltat;
	}
        tv.tv_usec = 0;

	if (num_fds == 0) {
	    /* Special case: everyone has exited! */
	    break;
	}
	/* If we are aborting due to child failure, we may want to
	   abort this select */
        rc = select( num_fds, &readfds, &writefds, NULL, &tv );

        if ( rc == 0 ) {
	    /* We timed out of the select.  This could be
	       either total time expired or a poll timeout for ptrace */
	    if (GetRemainingTime() <= 0) {
		/* Time to abort all proceses */
		fprintf( stderr, "Timeout of %d minutes expired; job aborted\n",
			 timeout_seconds / 60 );
		KillChildren( );
		done = 1;
	    }
	    else {
		/* Check for any stopped processes. */
		CheckForStopped( execname );
	    }
        } 
        if ( ( rc == -1 ) && ( errno == EINTR ) ) {
	    /* FIXME: this could happen on a SIGCHLD */
            if (debug) {
		printf( "select interrupted; continuing\n" );
		fflush( stdout );
	    }
            continue;
        }
        if ( rc < 0 ) {
	    perror( "select failed:" );
            fprintf( stderr, "mpiexec main loop: select failed\n" );
	    KillChildren();
	    exit( -1 );
        }

	/* Check only the active fds */
	for ( i = 0; i < num_fds; i++ )
            if ( fdtable[i].active ) 
                if ( FD_ISSET( fdtable[i].fd, &readfds ) )
                    done |= handle_input_fd( i );
    }
    /* On exit from all processes, determine the return code and print
       any requested information about process exit status */
    PrintExitStatus();
    rc = GetExitStatus( );
    return( rc );
}

void mpiexec_usage( const char *msg )
{
    if (msg)
	fputs( msg, stderr );
    fprintf( stderr, 
     "Usage: mpiexec -n <numprocs> -soft <softness> -host <hostname> \\\n" );
    fprintf( stderr, 
     "               -wdir <working directory> -path <search path> \\\n" );
    fprintf( stderr, "               -file <filename> execname <args>\n" );
    exit( -1 );
}

/* 
 * Try to get an integer value from the enviroment.  Return the default
 * if the value is not available or invalid
 */
int GetIntValue( const char name[], int default_val )
{
    const char *env_val;
    int  val = default_val;

    env_val = getenv( name );
    if (env_val) {
#ifdef HAVE_STRTOL
	char *invalid_char; /* Used to detect invalid input */
	val = (int) strtol( env_val, &invalid_char, 0 );
	if (*invalid_char != '\0') val = default_val;
#else
	val = atoi( env_val );
#endif
    }
    return val;
}

/*
 * Provide a simple timeout capability.  Initialize the time with 
 * InitTimeout.  Call GetRemainingTime to get the time in seconds left.
 */
int end_time = -1;  /* Time of timeout in seconds */
void InitTimeout( int seconds )
{
#ifdef HAVE_TIME
    time_t t;
    t = time( NULL );
    end_time = seconds + t;
#elif defined(HAVE_GETTIMEOFDAY)
    struct timeval tp;
    gettimeofday( &tp, NULL );
    end_time = seconds + tp.tv_sec;
#else
#   error 'No timer available'
#endif
}

/* Return remaining time in seconds */
int GetRemainingTime( void )
{
    int time_left;
#ifdef HAVE_TIME
    time_t t;
    t = time( NULL );
    time_left = end_time - t;
#elif defined(HAVE_GETTIMEOFDAY)
    struct timeval tp;
    gettimeofday( &tp, NULL );
    time_left = end_time - tp.tv_sec;
#else
#   error 'No timer available'
#endif
    if (time_left < 0) time_left = 0;
    return time_left;
}

/* 
 * Allocate a new fdentry, using the active field to identify 
 * candidates.
 */
int allocate_fdentry( void )
{
    int i;

    for ( i = 0; i < MAXFDENTRIES; i++ )
	if ( fdtable[i].active == 0 )
	    break;
    if ( i >= MAXFDENTRIES ) {
	printf( "too many fd's\n" );
        exit( -1 );
    }
    fdtable[i].active	= 1;
    fdtable[i].fd	= -1;
    fdtable[i].pid      = -1;
    fdtable[i].group    = -1;
    fdtable[i].rank     = -1;
    fdtable[i].state    = UNINITIALIZED;