fmgm.c

基于linux环境的ns2多机并行仿真补丁

/*---------------------------------------------------------------------------*/
/* Implementation of FM-like interface to Myrinet GM 1.2 calls.              */
/* Author(s): Kalyan Perumalla <http://www.cc.gatech.edu/~kalyan> 14Aug2001  */
/* $Revision: 1.3 $ $Name: v26apr05 $ $Date: 2005/02/03 14:56:23 $ */
/*---------------------------------------------------------------------------*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "mycompat.h"
#include "fmgm.h"
#if GM_AVAILABLE
    #include <gm.h>
#else
    #define GM_FAILURE 0
    #define GM_SUCCESS 1
    #define GM_LOW_PRIORITY 1
    #define GM_RECV_EVENT 1
    #define GM_MAX_HOST_NAME_LEN 1000
    typedef int gm_status_t;
    struct gm_port { int dummy; };
    struct gm_recv { int type, length; void *buffer; };
    union gm_recv_event { struct gm_recv recv; };
    #define gm_perror(m,s)          printf("%s status=%d\n",m,s)
    #define gm_init()                         GM_FAILURE
    #define gm_open(a1,a2,a3,a4,a5)           GM_FAILURE
    #define gm_get_host_name(a1,a2)           GM_FAILURE
    #define gm_get_node_id(a1,a2)             GM_FAILURE
    #define gm_max_node_id_inuse(a1,a2)       GM_FAILURE
    #define gm_node_id_to_host_name(a1,a2)    ""
    #define gm_min_size_for_length(a1)        0
    #define gm_set_acceptable_sizes(a1,a2,a3) GM_FAILURE
    #define gm_num_send_tokens(a1)            0
    #define gm_num_receive_tokens(a1)         0
    #define gm_dma_malloc(a1,a2)              0
    #define gm_provide_receive_buffer_with_tag(a1,a2,a3,a4,a5)
    #define gm_close(a1)
    #define gm_finalize()
    #define gm_send_with_callback(a1,a2,a3,a4,a5,a6,a7,a8,a9)  GM_FAILURE
    #define gm_receive_pending(a1)            0
    #define gm_receive(a1)                    0
    #define gm_unknown(a1,a2)                 GM_FAILURE
    #define gm_ntohc(a1)                      a1
    #define gm_ntohl(a1)                      a1
    #define gm_ntohp(a1)                      a1
#endif

/*---------------------------------------------------------------------------*/
static int gmfmdbg = 1;

/*---------------------------------------------------------------------------*/
typedef struct
{
    int length;    /*Actual #bytes in this piece*/
    int offset;    /*Into data byte array; pointer alignment accounted for*/
} GMMsgPieceSpec;

/*---------------------------------------------------------------------------*/
typedef struct
{
    int npieces;                       /*Actual #pieces in this message*/
    GMMsgPieceSpec pspec[GMMAXPIECES]; /*Pieces pointing into data array*/
    char pdata[GMMAXDATALEN];          /*Compacted/linearized data of pieces*/
} GMMsgPieces;

/*---------------------------------------------------------------------------*/
typedef struct _GMMsg
{
    int internal; /*Is this a GMFM-specific control/internal message?*/
    int src_id;
    int dest_id;
    int handler;
    int src_pe;
    int dest_pe;
    struct _GMMsg *next;
    GMMsgPieces pieces; /*This should always be at the end of the message*/
} GMMsg;

/*---------------------------------------------------------------------------*/
typedef char GMHostName[GM_MAX_HOST_NAME_LEN];

/*---------------------------------------------------------------------------*/
typedef struct
{
    unsigned int inodeid;
    int iportnum;
    GMHostName hostname;
} GMPeer;

/*---------------------------------------------------------------------------*/
typedef struct
{
    int unitnum, portnum;
    char *portname;
    unsigned int inodeid, maxinodeid;
    GMHostName localhostname;
    GMHostName ihostname[GMMAXPE];
} GMBoardInfo;

/*---------------------------------------------------------------------------*/
typedef struct
{
    int nrecd;
} GMBarrierInfo;

/*---------------------------------------------------------------------------*/
typedef struct
{
    int nodeid, numnodes;
    struct gm_port *port;
    GMBoardInfo gmb;
    struct{ unsigned int nst, nrt; } orig, curr;
    unsigned int maxmsglen, maxmsgszlog;
    GMPeer peers[GMMAXPE];
    GMMsg *sendbufs;
    GMBarrierInfo bar;
} GMState;

/*---------------------------------------------------------------------------*/
int GM_nodeid;
int GM_numnodes;

/*---------------------------------------------------------------------------*/
static GMState gmstate, *gm = &gmstate;
static GMCallback *fmcb = 0, *fmprivcb = 0;

/*---------------------------------------------------------------------------*/
static void print_gmmsg( GMMsg *msg )
{
    MYASSERT( msg, ("!") );
    printf("internal=%d\n",msg->internal);fflush(stdout);
    printf("src_id=%d\n",msg->src_id);fflush(stdout);
    printf("dest_id=%d\n",msg->dest_id);fflush(stdout);
    printf("handler=%d\n",msg->handler);fflush(stdout);
}

/*---------------------------------------------------------------------------*/
int GM_internal_callback(int handler, GM_stream *stream,
    int src_pe, int src_id, int dest_id)
{
    gm->bar.nrecd++;

    return 1;
}

/*---------------------------------------------------------------------------*/
GM_stream *GM_begin_internal_message( int recipient, int length, int handler,
    int src_id, int dest_id )
{
    GM_stream *stream = 0;

    stream = GM_begin_message( recipient, length, handler, src_id, dest_id );

    {
    GMMsg *msg = 0;
    msg = (GMMsg *)stream;
    msg->internal = 1;
    }

    return stream;
}

/*---------------------------------------------------------------------------*/
void GM_barrier( void )
{
if(gmfmdbg>=1){printf("GM_nodeid=%d starting barrier\n",GM_nodeid);fflush(stdout);}
    /*Every one sends to node 0, and node 0 responds back to everyone*/
    if( GM_nodeid == 0 )
    {
	int i = 0;

if(gmfmdbg>=1){printf("GM_nodeid=%d waiting for %d msgs\n",GM_nodeid, GM_numnodes-1);fflush(stdout);}

        /*Wait for everyone*/
	while( gm->bar.nrecd < GM_numnodes-1 ) { GM_extract(1); }

if(gmfmdbg>=1){printf("GM_nodeid=%d recd all %d msgs\n",GM_nodeid, GM_numnodes-1);fflush(stdout);}

        gm->bar.nrecd -= (GM_numnodes-1);

if(gmfmdbg>=1){printf("GM_nodeid=%d replying to 1..%d nodes\n",GM_nodeid, GM_numnodes-1);fflush(stdout);}

	/*Reply to everyone*/
	for( i = 1; i < GM_numnodes; i++ )
	{
	    GM_stream *s = GM_begin_internal_message( i, 0, 0, 0, i );
	    GM_end_message( s );
	}
    }
    else
    {
	GM_stream *s = GM_begin_internal_message( 0, 0, 0, GM_nodeid, 0 );
	GM_end_message( s );

if(gmfmdbg>=1){printf("GM_nodeid=%d sent msg to node 0\n",GM_nodeid);fflush(stdout);}

        while( gm->bar.nrecd <= 0 ) { GM_extract(1); }

if(gmfmdbg>=1){printf("GM_nodeid=%d recd msg from node 0\n",GM_nodeid);fflush(stdout);}

        gm->bar.nrecd--;
    }
if(gmfmdbg>=1){printf("GM_nodeid=%d done barrier\n",GM_nodeid);fflush(stdout);}
}

/*---------------------------------------------------------------------------*/
void GM_initialize( int nodeid, int numnodes, char *hosts[], GMCallback *cb )
{
    char *progname = "GMFM";
    int i = 0;
    gm_status_t status;
    unsigned int maxportnum = 16 /*XXX: KALYAN 128*/;
    int bound = 0;
    char *estr = getenv("FMGM_DEBUG");

    gmfmdbg = (estr ? atoi(estr) : 0);

    gm->nodeid = GM_nodeid = nodeid;
    gm->numnodes = GM_numnodes = numnodes;
if(gmfmdbg>=0){printf("GM_nodeid=%d, GM_numnodes=%d\n",GM_nodeid,GM_numnodes);fflush(stdout);}

    MYASSERT( cb, ("Message callback required") );
    fmcb = cb;
    fmprivcb = GM_internal_callback;

    for( i = 0; i < gm->numnodes; i++ )
    {
	GMPeer *peer = &gm->peers[i];
	peer->inodeid = -1;
	peer->iportnum = -1;
	strcpy( peer->hostname, hosts[i] );
    }

#define ON_ERR(_act) MYASSERT(status == GM_SUCCESS, _act ; gm_perror("",status))

    status = gm_init();
    ON_ERR( ("%d",status) );
if(gmfmdbg>=1){printf("gm_init() done\n");fflush(stdout);}

    gm->gmb.unitnum = 0; gm->gmb.portname = progname; gm->gmb.portnum = 2;
    do
    {
if(gmfmdbg>=0){printf("Binding to port %d...",gm->gmb.portnum);fflush(stdout);}

        status = gm_open( &gm->port, gm->gmb.unitnum, gm->gmb.portnum,
	                  gm->gmb.portname, GM_API_VERSION_1_2 );
        if( status == GM_SUCCESS )
	{
if(gmfmdbg>=0){printf("Succeeded\n");fflush(stdout);}
	    bound = 1;
	}
	else
	{
if(gmfmdbg>=0){printf("Failed\n");fflush(stdout);}
            gm_perror("gm_open", status);
	    gm->gmb.portnum++;
	}
    }while(!bound && gm->gmb.portnum < maxportnum);

    ON_ERR( ("Couldn't bind to any port upto port %d",gm->gmb.portnum) );
if(gmfmdbg>=0){printf("GM: Bound to local port %u\n",gm->gmb.portnum);fflush(stdout);}

    status = gm_get_host_name( gm->port, gm->gmb.localhostname );
    ON_ERR( ("Couldn't get local host name") );
if(gmfmdbg>=0){printf("GM: Local host name \"%s\"\n",gm->gmb.localhostname);fflush(stdout);}

    status = gm_get_node_id( gm->port, &gm->gmb.inodeid );
    ON_ERR( ("Couldn't get local node id") );
if(gmfmdbg>=0){printf("GM: Local node id %u\n",gm->gmb.inodeid);fflush(stdout);}

    status = gm_max_node_id_inuse( gm->port, &gm->gmb.maxinodeid );
    ON_ERR( ("Couldn't get max node id in use") );
if(gmfmdbg>=0){printf("GM: Max node id in use %u\n",gm->gmb.maxinodeid);fflush(stdout);}

    for( i = 0; i <= gm->gmb.maxinodeid; i++ )
    {
        char *nm = gm_node_id_to_host_name( gm->port, i );
	strcpy( gm->gmb.ihostname[i], nm?nm:"" );
    }

if(gmfmdbg>=1){printf("Node IDs:\n----------\n");for(i=0;i<=gm->gmb.maxinodeid;i++){printf("[%u] %-12.12s%s",i,gm->gmb.ihostname[i],(i+1)%3==0?"\n":"\t");}printf("\n----------\n");fflush(stdout);}

    for( i = 0; i < gm->numnodes; i++ )
    {
	GMPeer *peer = &gm->peers[i];
	int j = 0, inodeid = -1;
	for( j = 0; j <= gm->gmb.maxinodeid; j++ )
	{
	    if( !strcmp( peer->hostname, gm->gmb.ihostname[j] ) )