gdk_storage.c

这个是内存数据库中的一个管理工具

#line 45 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
#include "monetdb_config.h"
#include "gdk.h"
#include <stdlib.h>
#include "gdk_storage.h"



#line 52 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
void
GDKfilepath(str path, str dir, str name, str ext)
{
	*path = 0;
	if (dir && *dir && *name != DIR_SEP) {
		strcpy(path, dir);
		path += strlen(dir);
		if (path[-1] != DIR_SEP) {
			*path++ = DIR_SEP;
			*path = 0;
		}
	}
	strcpy(path, name);
	if (ext) {
		path += strlen(name);
		*path++ = '.';
		strcpy(path, ext);
	}
}

int
GDKcreatedir(char *dir)
{
	char path[PATHLENGTH];
	char *r;
	int ret = FALSE;

	strcpy(path, dir);
	r = strrchr(path, DIR_SEP);
	IODEBUG THRprintf(GDKout, "#GDKcreatedir(%s)\n", path);

	if (r) {
		DIR *dirp;

		*r = 0;
		dirp = opendir(path);
		if (dirp) {
			closedir(dirp);
		} else {
			GDKcreatedir(path);
			ret = mkdir(path, 0755);
			IODEBUG THRprintf(GDKout, "#mkdir %s = %d\n", path, ret);
		}
		*r = DIR_SEP;
	}
	return !ret;
}

int
GDKremovedir(str dirname)
{
	DIR *dirp = opendir(dirname);
	char path[PATHLENGTH];
	struct dirent *dent;
	int ret;

	IODEBUG THRprintf(GDKout, "#GDKremovedir(%s)\n", dirname);

	if (dirp == NULL)
		return 0;
	while ((dent = readdir(dirp)) != NULL) {
		if ((dent->d_name[0] == '.') && ((dent->d_name[1] == 0) || (dent->d_name[1] == '.' && dent->d_name[2] == 0))) {
			continue;
		}
		GDKfilepath(path, dirname, dent->d_name, NULL);
		ret = unlink(path);
		IODEBUG THRprintf(GDKout, "#unlink %s = %d\n", path, ret);
	}
	closedir(dirp);
	ret = rmdir(dirname);
	if (ret < 0) {
		GDKsyserror("GDKremovedir: rmdir(%s) failed.\n", DELDIR);
	}
	IODEBUG THRprintf(GDKout, "#rmdir %s = %d\n", dirname, ret);

	return ret;
}

#define _FUNBUF         0x040000
#define _FWRTHR         0x080000
#define _FRDSEQ         0x100000

int
GDKfdlocate(str nme, str mode, str extension)
{
	char buf[PATHLENGTH], *path = buf;
	int fd, flags = 0;

	if ((nme == NULL) || (*nme == 0)) {
		return 0;
	}
	GDKfilepath(path, BATDIR, nme, extension);

	if (*mode == 'm') {	/* file open for mmap? */
		mode++;
#ifdef _CYGNUS_H_
	} else {
		flags = _FRDSEQ;	/* WIN32 CreateFile(FILE_FLAG_SEQUENTIAL_SCAN) */
#endif
	}

	if (strchr(mode, 'w')) {
		flags |= O_WRONLY | O_CREAT;
	} else if (!strchr(mode, '+')) {
		flags |= O_RDONLY;
	} else {
		flags |= O_RDWR;
	}
#ifdef WIN32
	flags |= strchr(mode, 'b') ? O_BINARY : O_TEXT;
#endif
	fd = open(path, flags, 0644);
	if (fd < 0 && *mode == 'w') {
		/* try to create the directory, if that was the problem */
		char tmp[PATHLENGTH];

		strcpy(tmp, buf);
		if (GDKcreatedir(tmp)) {
			fd = open(path, flags, 0644);
		}
	}
	return fd;
}

FILE *
GDKfilelocate(str nme, str mode, str extension)
{
	int fd = GDKfdlocate(nme, mode, extension);

	if (*mode == 'm')
		mode++;
	return (fd < 0) ? NULL : fdopen(fd, mode);
}




#line 190 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
static int
delete_file(str path)
{
	int err = unlink(path);

	if (err)
		GDKsyserror("GDKunlink(%s)\n", path);
	IODEBUG THRprintf(GDKout, "#unlink %s = %d\n", path, err);

	return err;
}

int
GDKunlink(str dir, str nme, str ext)
{
	if (nme && *nme) {
		char path[PATHLENGTH];
		struct stat st;

		GDKfilepath(path, dir, nme, ext);
		if (stat(path, &st) == 0)
			return delete_file(path);
	}
	return -1;
}



#line 219 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
int
GDKmove(str dir1, str nme1, str ext1, str dir2, str nme2, str ext2)
{
	char path1[PATHLENGTH];
	char path2[PATHLENGTH];
	int ret, t0 = 0;

	IODEBUG t0 = GDKms();

	if ((nme1 == NULL) || (*nme1 == 0)) {
		return -1;
	}
	GDKfilepath(path1, dir1, nme1, ext1);
	GDKfilepath(path2, dir2, nme2, ext2);
	ret = rename(path1, path2);

	IODEBUG THRprintf(GDKout, "#move %s %s = %d (%dms)\n", path1, path2, ret, GDKms() - t0);

	return ret;
}



#line 241 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"


#line 251 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
int
GDKsave(str nme, str ext, void *buf, size_t size, int mode)
{
	int fd = -1, err = 0;
	FILE *fp = NULL;

	IODEBUG THRprintf(GDKout, "#GDKsave: name=%s, ext=%s, mode %d\n", nme, ext ? ext : "", mode);

	if (mode == STORE_MMAP) {


#line 272 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
		if (size) 
			err = MT_msync(buf, size, MMAP_SYNC);
		if (err)
			GDKsyserror("GDKsave: error on: name=%s, ext=%s, mode=%d\n", nme, ext ? ext : "", mode);
		IODEBUG THRprintf(GDKout, "#MT_msync(buf " PTRFMT ", size " SZFMT ", MMAP_SYNC) = %d\n", PTRFMTCAST buf, size, err);
	} else {
		if ((fd = GDKfdlocate(nme, "wb", ext)) >= 0) {
			/* write() on 64-bits Redhat for IA64 returns 32-bits signed result (= OS BUG)!
			   write() on Windows only takes int as size */
			while (size > 0) {
				/* circumvent problems by writing huge buffers in chunks <= 1GB */
				ssize_t ret = write(fd, buf, (unsigned) MIN(1 << 30, size));

				if (ret < 0) {
					err = -1;
					GDKsyserror("GDKsave: error " SSZFMT " on: name=%s, ext=%s, mode=%d\n", ret, nme, ext ? ext : "", mode);
					break;
				}
				size -= ret;
				buf = (void *) ((char *) buf + ret);
				IODEBUG THRprintf(GDKout, "#write(fd %d, buf " PTRFMT ", size %u) = " SSZFMT "\n", fd, PTRFMTCAST buf, (unsigned) MIN(1 << 30, size), ret);
			}
		} else {
			err = -1;
		}
	}
	if (fp) {
		err |= fclose(fp);
	} else if (fd >= 0) {
		err |= close(fd);
		if (err && GDKunlink(BATDIR, nme, ext)) {
			/* do not tolerate corrupt heap images (BBPrecover on restart will kill them) */
			GDKfatal("GDKsave: could not open: name=%s, ext=%s, mode %d\n", nme, ext ? ext : "", mode);
		}
	} else if (mode != STORE_MMAP) {
		GDKerror("GDKsave failed: name=%s, ext=%s, mode %d\n", nme, ext ? ext : "", mode);
	}
	return err;
}



#line 317 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
char *
GDKload(str nme, str ext, size_t size, size_t maxsize, int mode)
{
	char *ret = NULL;

	IODEBUG {
		THRprintf(GDKout, "#GDKload: name=%s, ext=%s, mode %d\n", nme, ext ? ext : "", mode);
	}
	if (mode == STORE_MEM) {
		int fd = GDKfdlocate(nme, "rb", ext);
		if (fd >= 0) {
			char *dst = ret = (char *) GDKmalloc(maxsize);
			ssize_t n_expected, n = 0;

			if (ret) {
				/* read in chunks, some OSs do not give you all at once
				   and Windows only accepts int */
				for (n_expected = size; n_expected > 0; n_expected -= n) {
					n = read(fd, dst, (unsigned) MIN(1 << 30, n_expected));
					IODEBUG THRprintf(GDKout, "#read(dst " PTRFMT ", n_expected " SSZFMT ", fd %d) = " SSZFMT "\n", PTRFMTCAST(void *)dst, n_expected, fd, n);

					if (n <= 0)
						break;
					dst += n;
				}
				if (n_expected > 0) { 
					GDKfree(ret);
					GDKsyserror("GDKload: cannot read: name=%s, ext=%s, " SZFMT " bytes missing.\n", nme, ext?ext:"", (size_t) n_expected);
					ret = NULL; 
				}
			}
			close(fd);
		} else {
			GDKsyserror("GDKload: cannot open: name=%s, ext=%s\n", nme, ext?ext:"");
                }
	} else {
		char path[PATHLENGTH];
		struct stat st;
		FILE *fp;

		GDKfilepath(path, BATDIR, nme, ext);
		if (stat(path, &st) >= 0 &&
                    (maxsize < (size_t) st.st_size || 
		    /* mmap storage is auto-extended here */
		     ((fp = fopen(path, "rb+")) != NULL &&
#ifdef _WIN64
		      _fseeki64(fp, (ssize_t) maxsize, SEEK_SET) >= 0 &&
#else
		      fseek(fp, (long) maxsize, SEEK_SET) >= 0 &&
#endif
		      fputc('\n', fp) >= 0 &&
		      fflush(fp) >= 0 &&
		      fclose(fp) >= 0)))
		{
			int mod = MMAP_READ | MMAP_WRITE | MMAP_SEQUENTIAL | MMAP_SYNC;
			if (mode == STORE_PRIV) mod |= MMAP_COPY;
			ret = (char *) GDKmmap(path, mod, (off_t) 0, maxsize);
			if (ret == (char *) -1L) {
				GDKsyserror("GDKload: cannot mmap(): name=%s, ext=%s\n", nme, ext?ext:"");
				ret = NULL;
			}
			IODEBUG THRprintf(GDKout, "#mmap(NULL, 0, maxsize " SZFMT ", mod %d, path %s, 0) = " PTRFMT "\n", maxsize, mod, path, PTRFMTCAST(void *)ret);
		}
	}
	if (ret == NULL)
		GDKerror("GDKload failed: name=%s, ext=%s\n", nme, ext ? ext : "");
	return ret;
}



#line 387 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"


#line 404 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
static BAT *
DESCload(int i)
{
	str s, nme = BBP_physical(i);
	BATstore *bs;
	BAT *b = NULL;
	int ht, tt; 

	IODEBUG {
		THRprintf(GDKout, "#DESCload %s\n", nme);
	}
	bs = BBPgetdesc(i);

	if (bs == NULL)
		return 0;
	b = &bs->B;

	ht = b->htype;
	tt = b->ttype;
	if ((ht < 0 && (ht = ATOMindex(s = ATOMunknown_name(ht))) < 0 ) ||
	    (tt < 0 && (tt = ATOMindex(s = ATOMunknown_name(tt))) < 0 )) {
		GDKerror("DESCload: atom '%s' unknown, in BAT '%s'.\n", s, nme);
		return NULL;
	}
	b->htype = ht;
	b->ttype = tt;
	b->hhash = b->thash = NULL;
	b->void_cnt = 0;
	/* mil shouldn't mess with just loaded bats */
	if (b->batStamp > 0)
		b->batStamp = -b->batStamp;

	/* reconstruct mode from BBP status (BATmode doesn't flush descriptor, so loaded mode may be stale) */
	b->batPersistence = (BBP_status(b->batCacheid) & BBPTMP) ? TRANSIENT : (BBP_status(b->batCacheid) & (BBPNEW | BBPPERSISTENT)) ? PERSISTENT : SESSION;
	b->batCopiedtodisk = 1;
	b->batSharecnt = 0;
	DESCclean(b);
	BATcheckmodes(b, BBP_status(i) & BBPEXISTING, FALSE);
	return b;
}

void
DESCsetmodes(BAT *b, BAT *img)
{
	/* propagate the proposed modes to the separate img of the bat (so concurrent execution on this bat is not affected) */
	if (b->batMapbuns != STORE_UNSET) {
		img->batBuns->storage = b->batMapbuns;
	}
	if (img->hheap && b->batMaphheap != STORE_UNSET) {
		/* elimdoubles string heaps (small, double-eliminated) cannot be memory mapped */
		img->hheap->storage = GDK_ELIMDOUBLES(img->hheap)?STORE_MEM:b->batMaphheap;
	}
	if (img->theap && b->batMaptheap != STORE_UNSET) {
		/* elimdoubles string heaps (small, double-eliminated) cannot be memory mapped */
		img->theap->storage = GDK_ELIMDOUBLES(img->theap)?STORE_MEM:b->batMaptheap;
	}

	/* change modes in img into legal modes, and reset b's mapdity if something had to be corrected */
	b->batMapdirty |= BATcheckmodes(img, BBP_status(b->batCacheid) & BBPEXISTING, FALSE);
}

void
DESCclean(BAT *b)
{
	b->batDirtyflushed = DELTAdirty(b) ? TRUE : FALSE;
	b->batDirty = 0;
	b->batDirtydesc = 0;
	b->batDirtybuns = 0;
	b->hheapdirty = 0;
	b->theapdirty = 0;
}


BAT *
BATsave(BAT *bd)
{
	int err = 0;
	char *nme;
	BATstore bs;
	BAT *b = bd;

	BATcheck(b, "BATsave");

	/* views cannot be saved, but make an exception for force-remapped views */
	if (VIEWparent(b) && !(b->batBuns->copied && b->batBuns->storage == STORE_MMAP)) {
		GDKerror("BATsave: %s is a view on %s; cannot be saved\n", BATgetId(b), BBPname(VIEWparent(b)));
		return NULL;
	}
	if (!BATdirty(b)) {
		return b;
	}
	if (b->batCacheid < 0) {
		b = BATmirror(b);
	}
	if (!DELTAdirty(b))
		ALIGNcommit(b);
	if (!b->halign)
		b->halign = OIDnew(1);
	if (!b->talign)
		b->talign = OIDnew(1);

	/* copy the descriptor to a local variable in order to let our messing in
	 * the BAT descriptor not affect other threads that only read it. */
	bs = *(BATstore *) b;
	b = &bs.B;
	b->U = &bs.U;
	b->P = &bs.P;
	b->H = &bs.H;
	b->T = &bs.T;

	b->batBuns = &b->U->buns;

	if (b->hheap) {
		b->hheap = (Heap*)GDKmalloc(sizeof(Heap));
		*b->hheap = *bd->hheap; 
	}
	if (b->theap) {
		b->theap = (Heap*)GDKmalloc(sizeof(Heap));
		*b->theap = *bd->theap; 
	}

	/* start saving data */
	nme = BBP_physical(b->batCacheid);
	if ((b->batCopiedtodisk == 0 || b->batDirty || b->batDirtybuns) &&
	    (b->htype != TYPE_void || b->ttype != TYPE_void)) {
		if (err == 0)
			err = HEAPsave(b->batBuns, nme, "buns");
	}
	if (b->hheap && (b->batCopiedtodisk == 0 || b->batDirty || b->hheapdirty))
		if (b->htype && b->hvarsized) {
			if (err == 0)
				err = HEAPsave(b->hheap, nme, "hheap");
		}
	if (b->theap && (b->batCopiedtodisk == 0 || b->batDirty || b->theapdirty))
		if (b->ttype && b->tvarsized) {
			if (err == 0) 
				err = HEAPsave(b->theap, nme, "theap");
		}
	/* correct heap modes such that they are ACID respecting */
	DESCsetmodes(bd, b);

	if (err == 0)
		if (b->batCopiedtodisk == 0 || b->batDirty || b->batDirtydesc) {
			DESCclean(b);
			b->batCopiedtodisk = 1;
			err = BATsavedesc(b, nme);
			IODEBUG THRprintf(GDKout, "#BATsavedesc(%s,free=" SZFMT ") = %d\n", nme, b->batBuns->free, err);
		}

	if (b->hheap) GDKfree(b->hheap);
	if (b->theap) GDKfree(b->theap);

	if (err == 0) {
		bd->batCopiedtodisk = 1;
		DESCclean(bd);
		return bd;
	}
	return NULL;
}




#line 568 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_storage.mx"
BAT *
BATload_intern(bat i)
{
	bat bid = ABS(i);
	str nme = BBP_physical(bid);