adapt.c

fortran并行计算包

	    if ((args12.rbuff = (char *)malloc(args12.bufflen + bufalign)) == (char *)NULL)
	    {
		fprintf(stdout,"Couldn't allocate memory\n");
		fflush(stdout);
		break;
	    }

	    /* save the original pointers in case alignment moves them */
	    memtmp = args12.sbuff;
	    memtmp1 = args12.rbuff;
	    
	    /* Possibly align the data buffer */
	    if (!bNoCache)
	    {
		if (bufalign != 0)
		{
		    args12.sbuff += (bufalign - (POINTER_TO_INT(args12.sbuff) % bufalign) + bufoffset) % bufalign;
		    /* args12.rbuff += (bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign; */
		}
	    }
	    args12.rbuff += (bufalign - (POINTER_TO_INT(args12.rbuff) % bufalign) + bufoffset) % bufalign;
	    
	    if (args12.tr && printopt)
	    {
		printf("%3d: %9d bytes %4d times --> ", n, args12.bufflen, nrepeat12);
		fflush(stdout);
	    }
	    
	    /* Finally, we get to transmit or receive and time */
	    if (args12.tr)
	    {
		bwdata12[n].t = LONGTIME;
		t1 = 0;
		for (i = 0; i < TRIALS; i++)
		{
		    if (bNoCache)
		    {
			if (bufalign != 0)
			{
			    args12.sbuff = memtmp + ((bufalign - (POINTER_TO_INT(args12.sbuff) % bufalign) + bufoffset) % bufalign);
			    /* args12.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign); */
			}
			else
			{
			    args12.sbuff = memtmp;
			    /* args12.rbuff = memtmp1; */
			}
		    }
		    
		    Sync(&args12);
		    t0 = MPI_Wtime();
		    for (j = 0; j < nrepeat12; j++)
		    {
			MPI_Send(args12.sbuff,  args12.bufflen, MPI_BYTE,  args12.nbor, MSG_TAG_12, MPI_COMM_WORLD);
			MPI_Recv(args12.rbuff,  args12.bufflen, MPI_BYTE,  args12.nbor, MSG_TAG_12, MPI_COMM_WORLD, &status);
			if (bNoCache)
			{
			    args12.sbuff += args12.bufflen;
			    /* args12.rbuff += args12.bufflen; */
			}
		    }
		    t = (MPI_Wtime() - t0)/(2 * nrepeat12);

		    t1 += t;
		    bwdata12[n].t = MIN(bwdata12[n].t, t);
		}
		SendTime(&args12, &bwdata12[n].t);
	    }
	    else
	    {
		bwdata12[n].t = LONGTIME;
		t1 = 0;
		for (i = 0; i < TRIALS; i++)
		{
		    if (bNoCache)
		    {
			if (bufalign != 0)
			{
			    args12.sbuff = memtmp + ((bufalign - (POINTER_TO_INT(args12.sbuff) % bufalign) + bufoffset) % bufalign);
			    /* args12.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign); */
			}
			else
			{
			    args12.sbuff = memtmp;
			    /* args12.rbuff = memtmp1; */
			}
		    }
		    
		    Sync(&args12);
		    t0 = MPI_Wtime();
		    for (j = 0; j < nrepeat12; j++)
		    {
			MPI_Recv(args12.rbuff,  args12.bufflen, MPI_BYTE,  args12.nbor, MSG_TAG_12, MPI_COMM_WORLD, &status);
			MPI_Send(args12.sbuff,  args12.bufflen, MPI_BYTE,  args12.nbor, MSG_TAG_12, MPI_COMM_WORLD);
			if (bNoCache)
			{
			    args12.sbuff += args12.bufflen;
			    /* args12.rbuff += args12.bufflen; */
			}
		    }
		    t = (MPI_Wtime() - t0)/(2 * nrepeat12);
		}
		RecvTime(&args12, &bwdata12[n].t);
	    }
	    tlast12 = bwdata12[n].t;
	    bwdata12[n].bits = args12.bufflen * CHARSIZE;
	    bwdata12[n].bps = bwdata12[n].bits / (bwdata12[n].t * 1024 * 1024);
	    bwdata12[n].repeat = nrepeat12;

	    if (args12.tr)
	    {
		if (bSavePert)
		{
		    if (g_nIproc == 0)
		    {
			if (bUseMegaBytes)
			    fprintf(out,"%f\t%0.9f\t", bwdata12[n].bps / 8, bwdata12[n].t);
			else
			    fprintf(out,"%f\t%0.9f\t", bwdata12[n].bps, bwdata12[n].t);
			fflush(out);
		    }
		    else
		    {
			MPI_Send(&bwdata12[n].bps, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
			MPI_Send(&bwdata12[n].t, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
		    }
		}
	    }
	    
	    free(memtmp);
	    free(memtmp1);
	    
	    if (args12.tr && printopt)
	    {
		if (bUseMegaBytes)
		    printf(" %6.2f MBps in %0.9f sec\n", bwdata12[n].bps / 8, tlast12);
		else
		    printf(" %6.2f Mbps in %0.9f sec\n", bwdata12[n].bps, tlast12);
		fflush(stdout);
	    }

skip_12_trial:
	    if (g_proc_loc == LEFT_PROCESS && g_nIproc == 0 && bSavePert)
	    {
		MPI_Recv(&tdouble, 1, MPI_DOUBLE, g_middle_rank, 1, MPI_COMM_WORLD, &status);
		if (bUseMegaBytes)
		    tdouble = tdouble / 8.0;
		fprintf(out, "%f\t", tdouble);
		MPI_Recv(&tdouble, 1, MPI_DOUBLE, g_middle_rank, 1, MPI_COMM_WORLD, &status);
		fprintf(out, "%0.9f\t", tdouble);
		fflush(out);
	    }


#ifdef CREATE_DIFFERENCE_CURVES
	    /*****************************************************/
	    /*         Run a trial between rank 0, 1 and 2       */
	    /*****************************************************/

	    MPI_Barrier(MPI_COMM_WORLD);


	    /* Calculate howmany times to repeat the experiment. */
	    if (g_nIproc == 0)
	    {
		if (args012.bufflen == 0)
		    nrepeat012 = g_latency012_reps;
		else
		    nrepeat012 = (int)(MAX((RUNTM / ((double)args012.bufflen /
			           (args012.bufflen - inc + 1.0) * tlast012)), TRIALS));
		MPI_Bcast(&nrepeat012, 1, MPI_INT, 0, MPI_COMM_WORLD);
	    }
	    else
	    {
		MPI_Bcast(&nrepeat012, 1, MPI_INT, 0, MPI_COMM_WORLD);
	    }

	    /* Allocate the buffer */
	    args012.bufflen = len + pert;
	    /* printf("allocating %d bytes\n", args12.bufflen * nrepeat012 + bufalign); */
	    if (bNoCache)
	    {
		if ((args012.sbuff = (char *)malloc(args012.bufflen * nrepeat012 + bufalign)) == (char *)NULL)
		{
		    fprintf(stdout,"Couldn't allocate memory\n");
		    fflush(stdout);
		    break;
		}
	    }
	    else
	    {
		if ((args012.sbuff = (char *)malloc(args012.bufflen + bufalign)) == (char *)NULL)
		{
		    fprintf(stdout,"Couldn't allocate memory\n");
		    fflush(stdout);
		    break;
		}
	    }
	    /* if ((args012.rbuff = (char *)malloc(args012.bufflen * nrepeat012 + bufalign)) == (char *)NULL) */
	    if ((args012.rbuff = (char *)malloc(args012.bufflen + bufalign)) == (char *)NULL)
	    {
		fprintf(stdout,"Couldn't allocate memory\n");
		fflush(stdout);
		break;
	    }

	    /* save the original pointers in case alignment moves them */
	    memtmp = args012.sbuff;
	    memtmp1 = args012.rbuff;
	    
	    /* Possibly align the data buffer */
	    if (!bNoCache)
	    {
		if (bufalign != 0)
		{
		    args012.sbuff += (bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) + bufoffset) % bufalign;
		    /* args12.rbuff += (bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign; */
		}
	    }
	    args012.rbuff += (bufalign - (POINTER_TO_INT(args012.rbuff) % bufalign) + bufoffset) % bufalign;
	    
	    if (g_nIproc == 0 && printopt)
	    {
		printf("%3d: %9d bytes %4d times --> ", n, args012.bufflen, nrepeat012);
		fflush(stdout);
	    }

	    for (itrial=0, ii=1; ii <= nrepeat012 && itrial < ntrials; ii <<= 1, itrial++)
	    {
		/* Finally, we get to transmit or receive and time */
		switch (g_proc_loc)
		{
		case LEFT_PROCESS:
		    bwdata012[n].t = LONGTIME;
		    t1 = 0;
		    for (i = 0; i < TRIALS; i++)
		    {
			if (bNoCache)
			{
			    if (bufalign != 0)
			    {
				args012.sbuff = memtmp + ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) + bufoffset) % bufalign);
				/* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
			    }
			    else
			    {
				args012.sbuff = memtmp;
				/* args012.rbuff = memtmp1; */
			    }
			}

			Sync012(&args012);
			t0 = MPI_Wtime();
			for (j = 0; j < nrepeat012; j++)
			{
			    MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD);
			    MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD, &status);
			    if (bNoCache)
			    {
				args012.sbuff += args012.bufflen;
				/* args012.rbuff += args012.bufflen; */
			    }
			}
			t = (MPI_Wtime() - t0)/(2 * nrepeat012);

			t1 += t;
			bwdata012[n].t = MIN(bwdata012[n].t, t);
		    }
		    MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
		    break;
		case MIDDLE_PROCESS:
		    bwdata012[n].t = LONGTIME;
		    t1 = 0;
		    for (i = 0; i < TRIALS; i++)
		    {
			if (bNoCache)
			{
			    if (bufalign != 0)
			    {
				args012.sbuff = memtmp + ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) + bufoffset) % bufalign);
				/* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
			    }
			    else
			    {
				args012.sbuff = memtmp;
				/* args012.rbuff = memtmp1; */
			    }
			}

			Sync012(&args012);
			t0 = MPI_Wtime();

			/******* use the ii variable here !!! ******/

			for (j = 0; j <= nrepeat012-ii; j+=ii)
			{
			    for (k=0; k<ii; k++)
			    {
				MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor2, MSG_TAG_012, MPI_COMM_WORLD);
				MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor2, MSG_TAG_012, MPI_COMM_WORLD, &status);
			    }
			    /* do the left process second because it does the timing and needs to include time to send to the right process. */
			    for (k=0; k<ii; k++)
			    {
				MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD, &status);
				MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD);
			    }
			    if (bNoCache)
			    {
				args012.sbuff += args012.bufflen;
				/* args012.rbuff += args012.bufflen; */
			    }
			}
			j = nrepeat012 % ii;
			for (k=0; k < j; k++)
			{
			    MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor2, MSG_TAG_012, MPI_COMM_WORLD);
			    MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor2, MSG_TAG_012, MPI_COMM_WORLD, &status);
			}
			/* do the left process second because it does the timing and needs to include time to send to the right process. */
			for (k=0; k < j; k++)
			{
			    MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD, &status);
			    MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD);
			}
			t = (MPI_Wtime() - t0)/(2 * nrepeat012);
		    }
		    MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
		    break;
		case RIGHT_PROCESS:
		    bwdata012[n].t = LONGTIME;
		    t1 = 0;
		    for (i = 0; i < TRIALS; i++)
		    {
			if (bNoCache)
			{
			    if (bufalign != 0)
			    {
				args012.sbuff = memtmp + ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) + bufoffset) % bufalign);
				/* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
			    }
			    else
			    {
				args012.sbuff = memtmp;
				/* args012.rbuff = memtmp1; */
			    }
			}

			Sync012(&args012);
			t0 = MPI_Wtime();
			for (j = 0; j < nrepeat012; j++)
			{
			    MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD, &status);
			    MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD);
			    if (bNoCache)
			    {
				args012.sbuff += args012.bufflen;
				/* args012.rbuff += args012.bufflen; */
			    }
			}
			t = (MPI_Wtime() - t0)/(2 * nrepeat012);
		    }
		    MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
		    break;
		}
		tlast012 = bwdata012[n].t;
		bwdata012[n].bits = args012.bufflen * CHARSIZE;
		bwdata012[n].bps = bwdata012[n].bits / (bwdata012[n].t * 1024 * 1024);
		bwdata012[n].repeat = nrepeat012;
		if (itrial < ntrials)
		{
		    dtrials[itrial] = MIN(dtrials[itrial], bwdata012[n].t);
		}

		if (g_nIproc == 0)
		{
		    if (bSavePert)
		    {
			fprintf(out, "\t%0.9f", bwdata012[n].t);
			fflush(out);
		    }
		    if (printopt)
		    {
			printf(" %0.9f", tlast012);
			fflush(stdout);
		    }
		}
	    }
	    if (g_nIproc == 0)
	    {
		if (bSavePert)
		{
		    fprintf(out, "\n");
		    fflush(out);
		}
		if (printopt)
		{
		    printf("\n");
		    fflush(stdout);
		}
	    }
	    
	    free(memtmp);
	    free(memtmp1);
#endif

#ifdef CREATE_SINGLE_CURVE
	    /*****************************************************/
	    /*         Run a trial between rank 0, 1 and 2       */
	    /*****************************************************/

	    MPI_Barrier(MPI_COMM_WORLD);


	    /* Calculate howmany times to repeat the experiment. */
	    if (g_nIproc == 0)
	    {
		if (args012.bufflen == 0)
		    nrepeat012 = g_latency012_reps;
		else
		    nrepeat012 = (int)(MAX((RUNTM / ((double)args012.bufflen /
			           (args012.bufflen - inc + 1.0) * tlast012)), TRIALS));
		MPI_Bcast(&nrepeat012, 1, MPI_INT, 0, MPI_COMM_WORLD);
	    }
	    else
	    {
		MPI_Bcast(&nrepeat012, 1, MPI_INT, 0, MPI_COMM_WORLD);
	    }

	    /* Allocate the buffer */
	    args012.bufflen = len + pert;
	    /* printf("allocating %d bytes\n", args12.bufflen * nrepeat012 + bufalign); */
	    if (bNoCache)
	    {
		if ((args012.sbuff = (char *)malloc(args012.bufflen * nrepeat012 + bufalign)) == (char *)NULL)
		{
		    fprintf(stdout,"Couldn't allocate memory\n");
		    fflush(stdout);
		    break;
		}
	    }
	    else
	    {
		if ((args012.sbuff = (char *)malloc(args012.bufflen + bufalign)) == (char *)NULL)
		{
		    fprintf(stdout,"Couldn't allocate memory\n");
		    fflush(stdout);
		    break;
		}
	    }
	    /* if ((args012.rbuff = (char *)malloc(args012.bufflen * nrepeat012 + bufalign)) == (char *)NULL) */
	    if ((args012.rbuff = (char *)malloc(args012.bufflen + bufalign)) == (char *)NULL)
	    {
		fprintf(stdout,"Couldn't allocate memory\n");
		fflush(stdout);
		break;
	    }

	    /* save the original pointers in case alignment moves them */
	    memtmp = args012.sbuff;
	    memtmp1 = args012.rbuff;
	    
	    /* Possibly align the data buffer */
	    if (!bNoCache)
	    {
		if (bufalign != 0)
		{
		    args012.sbuff += (bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) + bufoffset) % bufalign;
		    /* args12.rbuff += (bufalign - ((MPI_Aint)args12.rbuff % bufalign) + bufoffset) % bufalign; */
		}
	    }
	    args012.rbuff += (bufalign - (POINTER_TO_INT(args012.rbuff) % bufalign) + bufoffset) % bufalign;
	    
	    if (g_nIproc == 0 && printopt)
	    {
		printf("%3d: %9d bytes %4d times --> ", n, args012.bufflen, nrepeat012);
		fflush(stdout);
	    }
	    
	    /* Finally, we get to transmit or receive and time */
	    switch (g_proc_loc)
	    {
	    case LEFT_PROCESS:
		bwdata012[n].t = LONGTIME;
		t1 = 0;
		for (i = 0; i < TRIALS; i++)
		{
		    if (bNoCache)
		    {
			if (bufalign != 0)
			{
			    args012.sbuff = memtmp + ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) + bufoffset) % bufalign);
			    /* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
			}
			else
			{
			    args012.sbuff = memtmp;
			    /* args012.rbuff = memtmp1; */
			}
		    }
		    
		    Sync012(&args012);
		    t0 = MPI_Wtime();
		    for (j = 0; j < nrepeat012; j++)
		    {
			MPI_Send(args012.sbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD);
			MPI_Recv(args012.rbuff, args012.bufflen, MPI_BYTE, args012.nbor, MSG_TAG_012, MPI_COMM_WORLD, &status);
			if (bNoCache)
			{
			    args012.sbuff += args012.bufflen;
			    /* args012.rbuff += args012.bufflen; */
			}
		    }
		    t = (MPI_Wtime() - t0)/(2 * nrepeat012);

		    t1 += t;
		    bwdata012[n].t = MIN(bwdata012[n].t, t);
		}
		MPI_Bcast(&bwdata012[n].t, 1, MPI_DOUBLE, g_left_rank, MPI_COMM_WORLD);
		break;
	    case MIDDLE_PROCESS:
		bwdata012[n].t = LONGTIME;
		t1 = 0;
		for (i = 0; i < TRIALS; i++)
		{
		    if (bNoCache)
		    {
			if (bufalign != 0)
			{
			    args012.sbuff = memtmp + ((bufalign - (POINTER_TO_INT(args012.sbuff) % bufalign) + bufoffset) % bufalign);
			    /* args012.rbuff = memtmp1 + ((bufalign - ((MPI_Aint)args012.rbuff % bufalign) + bufoffset) % bufalign); */
			}
			else
			{