p4_sock_util.c

MPICH是MPI的重要研究,提供了一系列的接口函数,为并行计算的实现提供了编程环境.

#include "p4.h"
#include "p4_sys.h"
/* p4_net_utils.h generally would suffice here */

#ifdef SCYLD_BEOWULF
#include <sys/bproc.h>
#endif

/* Type for get/setsockopt calls */
#ifdef USE_SOCKLEN_T
typedef socklen_t p4_sockopt_len_t;
#elif defined(USE_SIZE_T_FOR_SOCKLEN_T)
typedef size_t p4_sockopt_len_t;
#else
typedef int p4_sockopt_len_t;
#endif

/*  removed 11/27/94 by RL.  Causes problems in FreeBSD and is not used.
extern int errno;
extern char *sys_errlist[];
*/

/* getenv is part of stdlib.h */
#ifndef HAVE_STDLIB_H
extern char *getenv();
#endif

#ifdef HAVE_ARPA_INET_H
/* prototype for inet_ntoa() */
#include <arpa/inet.h>
#endif

#if defined(HAVE_WRITEV) && defined(HAVE_SYS_UIO_H)
#include <sys/uio.h>
#endif
/*
 *    Utility routines for socket hacking in p4:
 *        P4VOID p4_socket_control( argstr ) 
 *        P4VOID net_set_sockbuf_size(size, skt)
 *        P4VOID net_setup_listener(backlog, port, skt)
 *        P4VOID net_setup_anon_listener(backlog, port, skt)
 *        int net_accept(skt)
 *        int net_conn_to_listener(hostname, port, num_tries)
 *        int net_recv(fd, buf, size)
 *        P4VOID net_send(fd, buf, size)
 *        get_inet_addr(addr)
 *        get_inet_addr_str(str)
 *        dump_sockaddr(who, sa)
 *        dump_sockinfo(msg, fd)
 */

/* Forward references */
static void get_sock_info_by_hostname ( char *, struct sockaddr_in ** );

/*
 * Socket control - allows various socket parameters to be set through
 * the command line.  The format is
 * -p4sctrl bufsize=n:winsize=n:netsendw=y/n:stat=y/n:netrecvw=y/n:writev=y/n
 *
 * For example
 *   -p4sctrl bufsize=64:netsendw=y
 * selects 64 k socket buffers and uses the alternate net_send routine
 * 
 * bufsize is in k
 * netsendw is either y or n 
 *
 */

#define COLLECT_PERF_STAT
#ifdef COLLECT_PERF_STAT
static int n_send_w_calls = 0;
static int n_send_eagain = 0;
static int n_send_max = -1;
static int n_send_looped = 0;
static int n_send_loopcnt = 0;
static int n_writev_first = 0;

static int n_recv_calls = 0;
static int n_recv_eagain = 0;
static int n_recv_select = 0;
static int n_recv_max = 0;
static int n_recv_maxloop = 0;
#define COLLECT_STAT(a) a
#else
#define COLLECT_STAT(a)
#endif

/* Local variable controling socket behavior */
/*
 * After some testing, the following defaults seem appropriate:
 *    net_send_w   Yes
 *    net_recv_w   Yes
 *    writev       Yes
 *    readb        No
 * We may want to encourage a socket buffer size of 32k or 64k
 */
/* SOCK_BUF_SIZE is defined in p4_sock_util.h */
static int p4_default_sock_buf_size = SOCK_BUFF_SIZE;  
/* use_net_send_w selects a form of netsend that uses a blocking (waiting)
   select when writes fail (because the socket buffer is full) */
static int p4_use_net_send_w = 1;

/* net_recv_w is a special test in the net_recv code that allows the 
   net_recv to use select to wait for an incoming message */
static int p4_use_net_recv_w = 1;

/* P4_WINSHIFT can also override this */
static int p4_default_win_shft = 0;

/* Whether to output statistics on the performance of net_send_w */
static int p4_output_socket_stat = 0;

static int p4_use_writev = 1;

/* switch the fd to blocking mode for the duration of a net_recv .  
   Requires netrecvw be false */
static int p4_use_readb = 0;

/* in_str is foo=value; find value and copy to out_str */
static void p4_copy_parm( char *in_str, char *out_str, int out_size )
{
    while (*in_str && *in_str != '=') in_str++;
    if (*in_str != '=') { out_str[0] = 0; return; }
    in_str++;
    while (*in_str && *in_str != ':' && out_size-- > 0) {
	*out_str++ = *in_str++;
    }
    *out_str = 0;
}

void p4_socket_control( char *argstr )
{
    char *p;
    char digits[10], value[10];
    char *endptr;
    int  val;

    if (!*argstr) return;

    p = argstr;
    while (p) {
	if (strncmp("bufsize=",p,8) == 0) {
	    /* P4_SOCKBUFSIZE */
	    p4_copy_parm( p, digits, 10 );
	    val = strtol( digits, &endptr, 10 ) * 1024;
	    if (endptr && *endptr == '\0') 
		p4_default_sock_buf_size = val; 
	    p4_dprintfl( 5, "default sockbuf size is %d\n", 
			 p4_default_sock_buf_size );
	}
	else if (strncmp( "winsize=",p,8) == 0) {
	    /* P4_WINSHIFT */
	    p4_copy_parm( p, digits, 10 );
	    val = strtol( digits, &endptr, 10 ) * 1024;
	    if (endptr && *endptr == '\0') 
		p4_default_win_shft = val;
	    p4_dprintfl( 5, "default win shift size is %d\n",
			 p4_default_win_shft );
	}
	else if (strncmp( "netsendw=",p,9) == 0) {
	    p4_copy_parm( p, value, 2 );
	    p4_use_net_send_w = (value[0] == 'y');
	    p4_dprintfl( 5, "Using net_send_w = %d\n", p4_use_net_send_w );
	}
	else if (strncmp( "netrecvw=",p,9) == 0) {
	    p4_copy_parm( p, value, 2 );
	    p4_use_net_recv_w = (value[0] == 'y');
	    p4_dprintfl( 5, "Using net_recv_w = %d\n", p4_use_net_recv_w );
	}
	else if (strncmp( "stat=",p,5) == 0) {
	    p4_copy_parm( p, value, 2 );
	    p4_output_socket_stat = (value[0] == 'y');
	    p4_dprintfl( 5, "Socket stat = %d\n", p4_output_socket_stat );
	}
	else if (strncmp( "writev=",p,7) == 0) {
	    p4_copy_parm( p, value, 2 );
	    p4_use_writev = (value[0] == 'y');
	    p4_dprintfl( 5, "Writev = %d\n", p4_use_writev );
	}
	else if (strncmp( "readb=",p,6) == 0) {
	    p4_copy_parm( p, value, 2 );
	    p4_use_readb = (value[0] == 'y');
	    p4_dprintfl( 5, "Read with blocking = %d\n", p4_use_readb );
	}
	p = strchr( p, ':' );
	if (p && *p) p++;
    }
}

/*
 *    Setup a listener:
 *        get a socket
 *        get a port
 *        listen on the port
 *
 *    Note that this does NOT actually start a listener process, but
 *    merely does the listen command.  It might be executed by a
 *    listener process, but we commonly use it prior to actually
 *    forking off the listener.
 */

/*
   Still needed:
   The prototypes for getsockopt, accept, etc pass the address of an
   integer of some kind to hold a length or other output value.
   Unfortunately, there is no standardization for this.  
   AIX: size_t
   Solaris, LINUX: socklen_t
   IRIX, SunOS: int
 */

/* 
 * If size is -1, get the size from either the environment (P4_SOCKBUFSIZE) or
 * the default (which may have been set through the command line)
 */
P4VOID net_set_sockbuf_size(int size, int skt)	/* 7/12/95, [e-mail] */
{
    int rc;
    char *env_value;
    int rsz,ssz;
    p4_sockopt_len_t dummy;
#ifdef TCP_WINSHIFT
    int shft = 0; /* Window shift; helpful on CRAY */
#endif
    /*
     * Need big honking socket buffers for fast honking networks.  It
     * would be nice if these would "autotune" for the underlying network,
     * but until then, we'll let the user specify socket send/recv buffer
     * sizes with P4_SOCKBUFSIZE.
     *
     */

#ifdef CAN_DO_SETSOCKOPT
    /* For the environment variables to work, the user really needs to
       set them in their .cshrc file (otherwise, the spawned processes
       may not get the correct values).

       Rumor has it that 0x40000 is a good size for AIX 4.x
     */
    /* 
     * Take the size either from the environment variable or from the
     * default set in p4_sock_util.h .
     */
    if (size <= 0)
    {
	    env_value = getenv("P4_SOCKBUFSIZE");
	    if (env_value) 
		size = atoi(env_value);
	    else 
		size = p4_default_sock_buf_size;
#ifdef TCP_WINSHIFT
	    shft = p4_default_win_shft;
            env_value = getenv("P4_WINSHIFT");
            if (env_value) shft = atoi(env_value);
#endif
    }

    if (size > 0)
    {
	    	/* Set Send & Receive Socket Buffers */

	    SYSCALL_P4(rc, setsockopt(skt,SOL_SOCKET,SO_SNDBUF,(char *)&size,sizeof(size)));
	    /* These should only generate informational messages ..., 
	       particularly for something like ENOBUFS */
	    if (rc < 0) {
		perror( "Set SO_SNDBUF" );
		p4_error("net_set_sockbuf_size socket", skt);
		}
	    SYSCALL_P4(rc, setsockopt(skt,SOL_SOCKET,SO_RCVBUF,(char *)&size,sizeof(size)));
	    if (rc < 0) {
		perror( "Set SO_RCVBUF" );
		p4_error("net_set_sockbuf_size socket", skt);
		}

	    	/* Fetch Back the Newly Set Sizes */

            dummy = sizeof(ssz);
            rc = getsockopt(skt,SOL_SOCKET,SO_SNDBUF,(char *)&ssz,&dummy);
            dummy = sizeof(rsz);
            rc = getsockopt(skt,SOL_SOCKET,SO_RCVBUF,(char *)&rsz,&dummy);

	    p4_dprintfl(80,
			"net_set_sockbuf_size: skt %d, new sizes = [%d,%d]\n",
			skt,ssz,rsz);
    }

#ifdef TCP_WINSHIFT
    /* 
       This code came from Dan Anderson ([e-mail]) for the
       CRAYs.  This is for systems that don't handle buffer sizes greater
       than 16 bits by default. An alternate mechanism is to do something
       like this:

	winshift = 0;
	bufsiz = SOCK_BUFF_SIZE; (use the actual size)
	while (bufsiz > 0XFFFF) {
		bufsiz >>= 1;
		++winshift;
	}

     */
    if ( shft > 0)
    {
	int wsarray[3];

	/* Set socket WINSHIFT */
        dummy = sizeof(wsarray);
        getsockopt(skt,IPPROTO_TCP,TCP_WINSHIFT,&wsarray,&dummy);
        if(wsarray[1] != shft){
	    
	    dummy = sizeof(shft);

            SYSCALL_P4(rc, setsockopt(skt,IPPROTO_TCP,TCP_WINSHIFT,&shft,dummy));
            if (rc < 0) {
		char hostname[MAXHOSTNAMELEN];
		gethostname_p4(hostname,255);
			 fprintf(stdout,
                        "ERROR_WINSHIFT in %s rc=%d, shft=%d, size_shft=%d \n",
                        hostname, rc,shft,(int)dummy);
                         p4_error("net_set_WINSHIFT socket", skt);}

                /* Fetch Back the Newly Set Sizes */

            dummy = sizeof(wsarray);
            rc = getsockopt(skt,IPPROTO_TCP,TCP_WINSHIFT,&wsarray,&dummy);

            p4_dprintfl(80,
                "net_set_sockbuf_WINSHIFT: skt %d, new values = [%x,%d,%d]\n",
                        skt,wsarray[0],wsarray[1],wsarray[2]);
        }
      }

#endif

#ifdef TCP_FASTACK
    { int arg;
    /*
      Some SGI systems will delay acks unless this field is set (even with
      TCP_NODELAY set!).  Without this, occassional 5 second (!) delays
      are introduced.
     */
    arg = 1;
    SYSCALL_P4(rc,setsockopt(skt,IPPROTO_TCP,TCP_FASTACK,&arg,sizeof(arg)));
    }
#endif

#endif
}

P4VOID net_setup_listener(int backlog, int port, int *skt)
{
    struct sockaddr_in s_in;
    int rc, optval = P4_TRUE;

    SYSCALL_P4(*skt, socket(AF_INET, SOCK_STREAM, 0));
    if (*skt < 0)
	p4_error("net_setup_listener socket", *skt);

#ifdef CAN_DO_SETSOCKOPT
    net_set_sockbuf_size(-1,*skt);     /* 7/12/95, [e-mail] */
    SYSCALL_P4(rc,setsockopt(*skt, IPPROTO_TCP, TCP_NODELAY, (char *) &optval, sizeof(optval)));

    if (p4_debug_level > 79)
	p4_print_sock_params( *skt );
#endif

    s_in.sin_family	= AF_INET;
    s_in.sin_addr.s_addr	= INADDR_ANY;
    s_in.sin_port	= htons(port);

    SYSCALL_P4(rc, bind(*skt, (struct sockaddr *) & s_in, sizeof(s_in)));
    if (rc < 0)
	p4_error("net_setup_listener bind", -1);

    SYSCALL_P4(rc, listen(*skt, backlog));
    if (rc < 0)
	p4_error("net_setup_listener listen", -1);
}

/* This sets up the sockets but not the listener process */
P4VOID net_setup_anon_listener(int backlog, int *port, int *skt)
{
    int rc;
    p4_sockopt_len_t sinlen;
    struct sockaddr_in s_in;
    int optval = P4_TRUE;

    SYSCALL_P4(*skt, socket(AF_INET, SOCK_STREAM, 0));
    if (*skt < 0)
	p4_error("net_setup_anon_listener socket", *skt);

#ifdef CAN_DO_SETSOCKOPT
    net_set_sockbuf_size(-1,*skt);	/* 7/12/95, [e-mail] */
    SYSCALL_P4(rc, setsockopt(*skt, IPPROTO_TCP, TCP_NODELAY, (char *) &optval, sizeof(optval)));

    if (p4_debug_level > 79)
	p4_print_sock_params( *skt );
#endif

    s_in.sin_family = AF_INET;
    s_in.sin_addr.s_addr = INADDR_ANY;
    s_in.sin_port = htons(0);

    sinlen = sizeof(s_in);

    SYSCALL_P4(rc, bind(*skt, (struct sockaddr *) & s_in, sizeof(s_in)));
    if (rc < 0)
	p4_error("net_setup_anon_listener bind", -1);

    SYSCALL_P4(rc, listen(*skt, backlog));