parallel_grouplock.c

lustre 1.6.5 source code

}

/*
 * process1 attempts CW(gid=1) -- granted
 * process2 attempts PR -- blocked
 * process3 attempts CW(gid=1) -> should be granted
 * process3 releases CW(gid=1)
 *   process2 should remain blocked
 * process1 releases CW(gid=1)
 *   process2's PR should be granted
 *
 * This is a lot like test1.
 */
void grouplock_test3(char *filename, int fd, char *errmsg)
{
        int rc, count, gid = 1;
        char buf[LPGL_FILEN];
        char zeros[LPGL_FILEN];
        MPI_Request req1, req2;
        int temp1, temp2;

        if (rank == 0) {
                if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                }
        }

        MPI_Barrier(MPI_COMM_WORLD);

        if (rank == 1) {
                memset(zeros, 0x0, sizeof(zeros));
                lseek(fd, 0, SEEK_SET);

                MPI_Send(&gid, 1, MPI_INT, 2, 1, MPI_COMM_WORLD);
                count = read(fd, buf, sizeof(buf));
                if (count != sizeof(buf)) {
                        if (count > 0)
                                dump_diff(zeros, buf, count, 0);
                        sprintf(errmsg, "read of file %s return %d",
                                filename, count);
                        FAIL(errmsg);
                }
                MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
        }

        if (rank == 2) {
                int temp;

                /* Wait for reading task to progress, this is probably somewhat
                   racey, though, may be adding usleep here would make things
                   better here. */
                usleep(100);
                MPI_Recv(&temp, 1, MPI_INT, 1, 1, MPI_COMM_WORLD,
                         MPI_STATUS_IGNORE);
                if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                }
                MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
        }

        if (rank == 0) {
                int iter = MAX_WAITING_TIME;
                int flag1, flag2;
                
                /* reading task will tell us when it completes */
                MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1);
                /* 2nd locking task will tell us when it completes */
                MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2);

                do {
                        iter--;
                        if (!iter) {
                                FAIL("2nd locking task is not progressing\n");
                        }
                        usleep(100);
                        MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
                        MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
                        if (flag1) {
                                FAIL("PR task progressed even though GROUP lock"
                                     " is held\n");
                        }
                } while (!flag2);
        }

        /* Now we need to release the lock */

        if (rank == 2) {
                if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                }
        }

        if (rank == 0) {
                int iter = MAX_WAITING_TIME;
                int flag1;
                
                do {
                        iter--;
                        usleep(100);
                        MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
                } while (!flag1 && iter);
                if (iter) {
                        FAIL("reading task is progressing even "
                             "though GROUP lock was not fully released\n");
                }

                iter = MAX_WAITING_TIME;

                if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                }
                
                do {
                        iter--;
                        if (!iter) {
                                FAIL("reading task is not progressing even "
                                     "though GROUP lock was released\n");
                                break;
                        }
                        usleep(100);
                        MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
                } while (!flag1);
        }

        MPI_Barrier(MPI_COMM_WORLD);

}

/* 
 * process1 attempts CW(gid=1) -- granted
 * process2 attempts PR on non-blocking fd -> should return -EWOULDBLOCK
 * process3 attempts CW(gid=2) on non-blocking fd -> should return -EWOULDBLOCK
 */
void grouplock_test4(char *filename, int fd, char *errmsg)
{
        int rc, count, gid = 1;
        char buf[LPGL_FILEN];
        char zeros[LPGL_FILEN];

        if (rank == 0) {
                if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                }
        }

        MPI_Barrier(MPI_COMM_WORLD);

        if (rank == 1) {
                memset(zeros, 0x0, sizeof(zeros));
                lseek(fd, 0, SEEK_SET);

                count = read(fd, buf, sizeof(buf));
                if (count != sizeof(buf)) {
                        if (count == -1 && errno == EWOULDBLOCK) {
                                MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
                                return;
                        }

                        if (count > 0)
                                dump_diff(zeros, buf, count, 0);
                        sprintf(errmsg, "read of file %s return %d",
                                filename, count);
                        FAIL(errmsg);
                } else {
                        FAIL("PR lock succeed while incompatible "
                             "GROUP LOCK (gid=1) is still held\n");
                }
        }

        if (rank == 2) {
                gid = 2;
                if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
                        if (errno == EWOULDBLOCK) {
                                MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
                                return;
                        }

                        sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                } else {
                        FAIL("GROUP_LOCK (gid=2) succeed while incompatible "
                             "GROUP LOCK (gid=1) is still held\n");
                }
        }


        if ( rank == 0) {
                int iter = MAX_WAITING_TIME;
                int flag1, flag2;
                MPI_Request req1, req2;
                int temp1, temp2;
                
                /* reading task will tell us when it completes */
                MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1);
                /* 2nd locking task will tell us when it completes */
                MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2);

                do {
                        iter--;
                        if (!iter) {
                                FAIL("non-blocking tasks are not progressing\n");
                        }
                        usleep(100);
                        MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
                        MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
                } while (!(flag2 && flag1));

                if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s", filename);
                        FAIL(errmsg);
                }
        }
}

/*
 * process1 attempts CW(gid=1) -- granted
 * process2 attempts CW(gid=2) -- blocked
 * process3 attempts CW(gid=2) -- blocked
 * process1 releases CW(gid=1)
 *   process2's CW(gid=2) should be granted
 *   process3's CW(gid=2) should be granted
 *
 * This is pretty much like test 3
 */
void grouplock_test5(char *filename, int fd, char *errmsg)
{
        int rc, count, gid = 1;
        char buf[LPGL_FILEN];
        char zeros[LPGL_FILEN];
        MPI_Request req1, req2;
        int temp1, temp2;

        if (rank == 0) {
                if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                }
        }

        MPI_Barrier(MPI_COMM_WORLD);

        if (rank == 2 || rank == 1) {
                gid = 2;
                if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                }
                MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
        }

        if (rank == 0) {
                int iter = MAX_WAITING_TIME;
                int flag1, flag2;
                
                /* 3rd locking task will tell us when it completes */
                MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1);
                /* 2nd locking task will tell us when it completes */
                MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2);

                do {
                        iter--;
                        usleep(100);
                        MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
                        MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
                } while (!flag2 && !flag1 && iter);
                if (iter) {
                        FAIL("incomptible locking tasks are progressing\n");
                }
        }

        /* Now we need to release the lock */

        if (rank == 0) {
                int iter = MAX_WAITING_TIME;
                int flag1, flag2;
                if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
                        sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
                                filename, rc);
                        FAIL(errmsg);
                }
                
                do {
                        iter--;
                        if (!iter) {
                                FAIL("locking tasks are not progressing even "
                                     "though incompatible lock released\n");