gnbdd.c

openGFS , a kind of file system.

/*
 *
 *    Copyright 1999 Regents of the University of Minnesota
 *    Portions Copyright 1999-2001 Sistina Software, Inc.
 *    Portions Copyright 2001 The OpenGFS Project
 *
 *    This is free software released under the GNU General Public License.
 *    There is no warranty for this software.  See the file COPYING for
 *    details.
 *
 *    See the file AUTHORS for a list of contributors.
 * 
 */

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <netinet/in.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <signal.h>
#include <sys/mount.h>
#include <sys/mman.h>
#include <errno.h>

#include "trans.h"


/* placeholder */
#define printff(fmt,args...)

/* some private data structures. */
typedef struct {
	int fd;
	pthread_mutex_t mux;
	pthread_mutex_t sender;
	/* other info? */
} clients_t;

/* Table of clients that are logged in */
clients_t *Clients = NULL;
int client_cnt = 0;

/* This is so there will only ever be one thread looking at accepts. */
pthread_mutex_t AcceptMux;

/* This global keeps the server running. */
static int Server_Runs = 1;

/* This file descriptor is the data we're serving. */
static int data_fd = -1;
static uint64_t datasize = 0;	/*bytes */
static char *datamap = NULL;
static int readonly = 0;

/* ripped out of nbd code base*/
uint64_t ntohll(uint64_t a)
{
#if (BYTE_ORDER == LITTLE_ENDIAN)
	uint32_t lo, hi;

	lo = ntohl(a & 0xffffffff);
	hi = ntohl(a >> 32U);

	return (uint64_t)lo << 32U | hi;
#else
	return a;
#endif
}

#define htonll ntohll

/******************************************************************************
 * dump_buffer()
 * prints out the contents of a buffer for inspection.  ignores formats.
 *
 * Need to get all zero lines absorbed.
 */
void
dump_buffer(void *buf, int len)
{
	int i;
	uint8_t *c = buf;

	for (i = 0; i < len; i++) {
		if (i % 16 == 0)
			printf("\n%4u: ", i);
		printf("%2x ", c[i]);
	}
	printf("\n");
}

/******************************************************************************
 * heh, simple recv/send wrappers to get all of the beffer sent over the
 * net.  Had forgotten I needed this....
 */
int
cpt_recv(int s, void *buf, size_t len, int flags)
{
	int res;
	do {
		res = recv(s, buf, len, flags);
		if (res < 0)
			return res;
		len -= res;
		buf += res;
	} while (len > 0);
	return res;
}

int
cpt_send(int s, void *buf, size_t len, int flags)
{
	int res;
	do {
		res = send(s, buf, len, flags);
		if (res < 0)
			return res;
		len -= res;
		buf += res;
	} while (len > 0);
	return res;
}

/******************************************************************************
 * client_alloc()
 * expands the size of the clients table.
 */
#define NALLOC 10
void
client_alloc(void)
{

	/* FIXME: There is NO upperbound on the number of active connections!  */
	int i;
	if (Clients == NULL)
		Clients = malloc(NALLOC * sizeof(clients_t));
	else
		Clients =
		    realloc(Clients, (client_cnt + NALLOC) * sizeof(clients_t));
	if (Clients == NULL) {
		perror("Can't alloc memory for client array");
		exit(1);
	}
	for (i = client_cnt; i < client_cnt + NALLOC; i++) {
		Clients[i].fd = -1;
		pthread_mutex_init(&Clients[i].mux, NULL);
		pthread_mutex_init(&Clients[i].sender, NULL);
	}
	client_cnt += NALLOC;
	printf("realloced clients\n");
}

/******************************************************************************
 * client_add()
 */
int
client_add(int fd)
{
	int i;

	if (Clients == NULL)
		client_alloc();
      CAagain:
	for (i = 0; i < client_cnt; i++) {
		if (Clients[i].fd == -1) {	/* FIXME: RACE */
			Clients[i].fd = fd;
			pthread_mutex_init(&Clients[i].mux, NULL);
			pthread_mutex_init(&Clients[i].sender, NULL);
			return i;
		}
	}

	/* client array must be full, expand it. */
	client_alloc();
	goto CAagain;		/* now go add it. */
}

/******************************************************************************
 * client_del()
 * resetting the mutexes here may not be a good idea. do some checking.
 */
void
client_del(int fd)
{
	int i;
	for (i = 0; i < client_cnt; i++) {
		if (Clients[i].fd == fd) {
			Clients[i].fd = -1;
			pthread_mutex_destroy(&Clients[i].mux);
			pthread_mutex_destroy(&Clients[i].sender);
			return;
		}
	}
	/* pop an error? */
}

/******************************************************************************
 * handleRequest()
 *
 * See if we can't get mmapping done instead of lseek&(read|write)
 */
int
handleRequest(clients_t * Client, theader_t * ioHdr)
{
	unsigned long len;
	unsigned long long offset;
	int n;
	int clifd = Client->fd;
	pthread_mutex_t *sender = &Client->sender;
	char buf[20480];

	len = ntohl(ioHdr->len);
	offset = ntohll(ioHdr->offset);

#if 0
	printf("ioHdr: type=0x%x, key=0x%" FMT_64 "x, len=0x%lx, offset=0x%"
	       FMT_64 "x\n", ioHdr->type, ioHdr->key, len, offset);
#endif

	if (len > 10240) {
		perror("Request too large");
		return -1;
	}
	if (len + offset > datasize) {
		perror("Out of bounds.");
		return -1;
	}

	switch (ioHdr->type) {
	case tioReadReq:
		printff("Doing a Read\n");
		/* Need to get len bytes from offset out of data_fd */
		if (datamap) {
			ioHdr->type = tioReadRpl;

			pthread_mutex_lock(sender);
			n = cpt_send(clifd, ioHdr, theader_len, 0);
			n = cpt_send(clifd, (datamap + offset), len, 0);
			pthread_mutex_unlock(sender);
		} else {
			if (lseek(data_fd, offset, SEEK_SET) < 0)
				perror("Read: llseek");
			printff("Done seek\n");
			if (read(data_fd, buf, len) < 0)
				perror("Read: read");
			printff("Done read\n");

			ioHdr->type = tioReadRpl;

			pthread_mutex_lock(sender);
			n = cpt_send(clifd, ioHdr, theader_len, 0);
			n = cpt_send(clifd, buf, len, 0);
			pthread_mutex_unlock(sender);
		}
		printff("Done send\n");
		break;
	case tioWriteReq:
		printff("Doing a Write\n");
		/* need a readonly check.... */
		/* we have len bytes that we want to write at offset in data_fd */
		if (datamap) {
			n = cpt_recv(clifd, (datamap + offset), len, 0);
		} else {
			n = cpt_recv(clifd, buf, len, 0);
			if (n == 0)
				return -1;
			if (lseek(data_fd, offset, SEEK_SET) < 0)
				perror("Write: lseek");
			if (write(data_fd, buf, len) < 0)
				perror("Write: write");
		}

		ioHdr->type = tioWriteRpl;
		ioHdr->len = 0;
		ioHdr->err = 0;
		ioHdr->offset = 0;
		/*expects an ack.. */
		pthread_mutex_lock(sender);
		n = cpt_send(clifd, ioHdr, theader_len, 0);
		pthread_mutex_unlock(sender);
		printff("Done send\n");
		break;
	case tioShutdown:
		/* Gotta close the connection, after replying. */
		/* ioHdr should be exactly what we waht to send back, so we'll
		 * just dump it right back.
		 */
		pthread_mutex_lock(sender);
		n = cpt_send(clifd, &ioHdr, theader_len, 0);
		pthread_mutex_unlock(sender);
		/* now close shop. */
		return -1;
		break;
	default:
		printf("Unrecognised header.\n");
		printf("ioHdr: type=0x%x, key=0x%llx,"
		       "len=0x%lx, offset=0x%llx\n", ioHdr->type,
		       ioHdr->key, len, offset);
		break;
	}
	return 0;
}

/******************************************************************************
 * setkeepalive
 * quicky wrapper for setting the keepalive bit on a socket.
 *
 * This works, BUT if you take a look at the defaults for linux:
 *   cat /proc/sys/net/ipv4/tcp_keepalive_probes
 *   cat /proc/sys/net/ipv4/tcp_keepalive_time
 * you look at those. 9 probes, and 7200 somthings. I think seconds. That's
 * damn long. I reset the timeout on the server machine to 60, and it
 * actually works. if you wait for it.
 *   echo "60" > /proc/sys/net/ipv4/tcp_keepalive_time
 * So I'll leave this here, but we may need a faster method of detecting
 * client death.
 */
int
setkeepalive(int sk)
{
	int iopt = 1;