gdk_heap.c

这个是内存数据库中的一个管理工具

#line 67 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
#include "monetdb_config.h"
#include "gdk.h"

static char *
decompose_filename(str nme)
{
	char *ext, *priv;

	ext = strchr(nme, '.');	/* extract base and ext from heap file name */
	if (ext) {
		*ext++ = 0;
		priv = strchr(ext, '.');
		if (priv)
			*priv = 0;
	}
	return ext;
}



#line 86 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 99 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
int
HEAPalloc(Heap *h, size_t nitems, size_t itemsize)
{
	char nme[PATHLENGTH], *ext = NULL;

	if (h->filename) {
		strcpy(nme, h->filename);
		ext = decompose_filename(nme);
	}
	h->base = NULL;
	h->maxsize = h->size = 1;
	h->copied = 0;
	if (itemsize)
		h->maxsize = h->size = MAX(1, nitems) * itemsize;
	h->free = 0;

	/* check for overflow */
	if (itemsize && nitems > (h->size/itemsize)) 
		return -1;

	/* when using anonymous vm we malloc we need 64K chunks, also we
	 * 20% extra malloc */
	if (h->size > GDK_mem_bigsize) {
		h->maxsize = (size_t) ((double) h->maxsize * BATMARGIN) - 1;
		h->maxsize = (1 + (h->maxsize >> 16)) << 16;
	}
	if (h->filename == NULL || (h->size < GDK_vm_minsize)) {
		h->storage = STORE_MEM;
		h->base = (char *) GDKmallocmax(h->size, &h->maxsize, 0);
		if (h->base == 0) 
			GDKerror("HEAPalloc: Insufficient space for HEAP.");
	}
	if (h->filename && h->base == NULL) {
		char privext[PATHLENGTH], *of = h->filename;
		FILE *fp;

		h->filename = NULL;
		sprintf(privext, "%s.priv", ext);
		fp = GDKfilelocate(nme, "wb", privext);
		if (fp != NULL) {
			fclose(fp);
			/* a non-persistent heap: we create a .priv but *not* MMAP_PRIV !!! */
			h->storage = STORE_MMAP;
			HEAPload(h, nme, ext, FALSE);
		}
		GDKfree(of);
	}
	h->newstorage = h->storage;
	return (h->base == NULL) ? -1 : 0;
}



#line 151 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 166 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
int
HEAPextend(Heap *h, size_t size)
{
	char nme[PATHLENGTH], *ext = NULL;

	if (h->filename) {
		strcpy(nme, h->filename);
		ext = decompose_filename(nme);
	}
	if (size <= h->size)
		return 0;

	if (h->storage & STORE_MMAP) {
		/* memory mapped files extend: save and remap */
		if (HEAPsave(h, nme, ext) < 0)
			return -1;
		HEAPfree(h);
		h->maxsize = h->size = size;
		if (HEAPload(h, nme, ext, FALSE) >= 0)
			return 0;
	} else {
		/* extend a malloced heap, possibly switching over to file-mapped storage */
		Heap bak = *h;
		int can_mmap = (h->filename && size >= GDK_mem_bigsize);
		int must_mmap = can_mmap && (size >= GDK_vm_minsize || (h->newstorage & STORE_MMAP));
		h->size = size;

		if (can_mmap) {
			/* in anonymous vm, if have to realloc anyway, we reserve some extra space */
			if (size > h->maxsize) {
				h->maxsize = (size_t) ((double) size * BATMARGIN);
			}
			/* when using anonymous vm we malloc we need 64K chunks */
			h->maxsize = (1 + ((h->maxsize-1) >> 16)) << 16;
		} else {
			h->maxsize = size; /* for normal GDKmalloc, maxsize = size */
		}

		/* try GDKrealloc if the heap size stays within reasonable limits */
		if (!must_mmap) {
			h->storage = STORE_MEM;
			h->base = (char *) GDKreallocmax(h->base, size, &h->maxsize, 0);
			if (h->base)
				return 0;
		}
		/* too big: convert it to a disk-based temporary heap */
		if (can_mmap) {
			char privext[PATHLENGTH], *of = h->filename;
			FILE *fp;

			h->filename = NULL;
			sprintf(privext, "%s.priv", ext);
			fp = GDKfilelocate(nme, "wb", privext);
			if (fp != NULL) {
				fclose(fp);
				/* a non-persistent heap: we create a .priv but *not* MMAP_PRIV !!! */
				h->storage = STORE_MMAP;
				if (HEAPload(h, nme, ext, FALSE) >= 0) {
					memcpy(h->base, bak.base, bak.free);
					HEAPfree(&bak);
					return 0;
				}
			}
			GDKfree(of);
		}
		*h = bak;
	}
	return -1;
}



#line 237 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 242 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
int
HEAPcopy(Heap *dst, Heap *src)
{
	if (HEAPalloc(dst, src->size, 1) == 0) {
		dst->free = src->free;
		memcpy(dst->base, src->base, src->free);
		return 0;
	}
	return -1;
}



#line 254 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 259 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
int
HEAPfree(Heap *h)
{
	if (h->base) {
		if (h->storage == STORE_MEM) {	/* plain memory */
			GDKfree(h->base);
		} else {	/* mapped file, or STORE_PRIV */
			int ret = GDKmunmap(h->base, h->maxsize);

			if (ret < 0) {
				GDKsyserror("HEAPfree: %s was not mapped\n", h->filename);
				assert(0);
			}
			IODEBUG THRprintf(GDKout, "#munmap(base=" PTRFMT ", size=" SZFMT ") = %d\n", PTRFMTCAST(void *)h->base, h->maxsize, ret);
		}
		h->base = NULL;
	}
	if (h->filename) {
		GDKfree(h->filename);
		h->filename = NULL;
	}
	return 0;
}



#line 284 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 297 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
int
HEAPload(Heap *h, str nme, str ext, int trunc)
{
	FILE *fp = (FILE *) GDKfilelocate(nme, "mrb", ext);
	int ret = 0, desc_status = 0;
	size_t truncsize = (1 + (((size_t) (h->free * 1.05)) >> REMAP_PAGE_MAXBITS)) << REMAP_PAGE_MAXBITS;
	size_t minsize = (1 + ((h->size - 1) >> REMAP_PAGE_MAXBITS)) << REMAP_PAGE_MAXBITS;
	int priv_storage = (h->storage == STORE_PRIV);
	char priv[80]={0};

	h->maxsize = h->size;
	h->filename = NULL;

	/* round up mmap heap sizes to REMAP_PAGE_MAXSIZE (usually 512KB) segments */
	if ((h->storage & STORE_MMAP) && (minsize != h->size)) {
		h->size = minsize;
		h->maxsize = MAX(minsize, h->maxsize);
	}
	IODEBUG {
		THRprintf(GDKout, "#HEAPload(%s.%s,storage=%d,free=" SZFMT ",size=" SZFMT ")\n", nme, ext, h->storage, h->free, h->size);
	}
	/* On some OSs (WIN32,Solaris), it is prohibited to write to a file that is open 
	 * in MAP_PRIVATE (FILE_MAP_COPY) solution: read from a file renamed to .ext.priv 
	 * .ex.priv files are now also used with STORE_MMAP mode on huge uncommitted BATs.
	 */
	strcpy(priv, ext);
	strcat(priv, ".priv");
	if (fp) {
		ret = fclose(fp);
	} else {
		priv_storage = TRUE;	/* file may be in X.priv file iso X */
	}
	if (priv_storage) {
		long_str path;

		GDKfilepath(path, BATDIR, nme, priv);
		if (fp == NULL) {
			struct stat st;

			ret = stat(path, &st);
		} else {
			/* silently remove any previous .ext.priv file */
			(void) unlink(path);
			ret = GDKmove(BATDIR, nme, ext, BATDIR, nme, priv);
		}
		ext = priv;
	}
	if (ret) {
		return -1;	/* file could not be located */
	}
	/* legacy 2: some old repositories have wasted space => truncate during load */
	if (trunc && truncsize < h->size) {
		fp = (FILE *) GDKfilelocate(nme, "mrb+", ext);
		if (fp) {
			ret = ftruncate(fileno(fp), (off_t) truncsize);
			IODEBUG THRprintf(GDKout, "#ftruncate(file=%s.%s, size=" SZFMT ") = %d\n", nme, ext, truncsize, ret);

			fclose(fp);
			if (ret == 0) {
				h->size = h->maxsize = truncsize;
				desc_status = 1;
			}
		}
	}
	h->base = (char *) GDKload(nme, ext, h->free, h->size, h->storage);
	if (h->base == NULL) {
		return -1;	/* file could  not be read satisfactorily */
	}
	h->newstorage = h->storage;
	h->filename = (char *) GDKmalloc(strlen(nme) + strlen(ext) + 2);
	sprintf(h->filename, "%s.%s", nme, ext);

	return desc_status;	/* ok */
}



#line 373 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 384 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 385 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
int HEAPsave(Heap* h, str nme, str ext) {
	int store = h->storage;
	char *p = h->filename;

	if (h->base == NULL) {
		return -1;
	} 
	if (p == NULL || store == STORE_PRIV) {
		/* anonymous or private VM is saved as if it were malloced */
		store = STORE_MEM; 
	}
	IODEBUG {
		THRprintf(GDKout, "#HEAPsave(%s.%s,storage=%d,free=" SZFMT ",size=" SZFMT ")\n", 
			nme, ext, h->storage, h->free, h->size);
	}
	return GDKsave(nme, ext, h->base, h->free, store);
}


#line 403 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 407 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
int
HEAPdelete(Heap *h, str o, str ext)
{
	char ext2[64];

	if (h->size <= 0) {
                return 0;
	}
	if (h->base) {
		if (h->copied == 0 && h->storage & STORE_MMAP) {
			/* truncate file, so OS does not try to flush dirty data in mmap to file that is deleted anyway */
			int ret, fd = GDKfdlocate(h->filename, "rb+", NULL);
			IODEBUG THRprintf(GDKout, "#HEAPdelete(%s) GDKfdlocate(\"rb+\") = %d\n", h->filename, fd);

			ret = ftruncate(fd, LL_CONSTANT(0));
			IODEBUG THRprintf(GDKout, "#HEAPdelete(%s) ftruncate(%d,0LL) = %d\n", h->filename, fd, ret);

			ret = close(fd);
			IODEBUG THRprintf(GDKout, "#HEAPdelete(%s) close(%d) = %d\n", h->filename, fd, ret);
		}
		HEAPfree(h);
	}
	if (h->copied) {
            return 0;
        }
	strcpy(ext2, ext);
	strcat(ext2, ".priv");
	return (GDKunlink(BATDIR, o, ext) == 0) | (GDKunlink(BATDIR, o, ext2) == 0)?0:-1;
}

int HEAPwarm(Heap *h) 
{
	int bogus_result = 0;
	if (h->storage & STORE_MMAP) {
		/* touch the heap sequentially */
                int *cur = (int*) h->base;
                int *lim = (int*) (h->base + h->free) - 4096;
		for(; cur<lim; cur+=4096) /* try to schedule 4 parallel memory accesses */
			bogus_result += cur[0] + cur[1024] + cur[2048] + cur[3072];
	}
	return bogus_result;
}




#line 452 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 456 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
size_t
HEAPvmsize(Heap *h)
{
	if (h && h->free)
		return h->maxsize;
	return 0;
}



#line 465 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"


#line 470 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
size_t
HEAPmemsize(Heap *h)
{
	if (h && h->free && h->storage != STORE_MMAP)
		return h->size;
	return 0;
}



#line 479 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"



#line 498 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
/* #define DEBUG */
/* #define TRACE */

#define HEAPVERSION	20030408

typedef struct heapheader {
	size_t head;		/* index to first free block            */
	int alignment;		/* alignment of objects on heap         */
	size_t firstblock;	/* first block in heap                  */
	int version;
	int (*sizefcn) (ptr);	/* ADT function to ask length           */
} HEADER32;

typedef struct {
	int version;
	int alignment;
	size_t head;
	size_t firstblock;
	int (*sizefcn) (ptr);
} HEADER64;

#if SIZEOF_SIZE_T==8
typedef HEADER64 HEADER;
typedef HEADER32 HEADER_OTHER;
#else
typedef HEADER32 HEADER;
typedef HEADER64 HEADER_OTHER;
#endif
typedef struct hfblock {
	size_t size;		/* Size of this block in freelist        */
	size_t next;		/* index of next block                   */
} CHUNK;



#line 532 "/export/scratch0/monet/monet.GNU.64.64.d.14791/MonetDB/src/gdk/gdk_heap.mx"
#define roundup_8(x)	(((x)+7)&~7)
#define roundup_4(x)	(((x)+3)&~3)
#define blocksize(h,p)	((p)->size)

static INLINE size_t
roundup_num(size_t number, size_t alignment)
{
	size_t rval = number + alignment - 1;

	rval -= (rval % alignment);
	return (rval);
}

size_t
HEAP_private(Heap *h)
{
	(void) h;
	return roundup_8(sizeof(HEADER));
}

#ifdef TRACE
static void
HEAP_printstatus(Heap *heap)
{
	HEADER *hheader = HEAP_index(heap, 0, HEADER);
	size_t block, cur_free = hheader->head;
	CHUNK *blockp;

	THRprintf(GDKout, "#HEAP has head " SZFMT " and alignment %d and size " SZFMT "\n", hheader->head, hheader->alignment, heap->free);

	/*
	   // Walk the blocklist;
	 */
	block = hheader->firstblock;

	while (block < heap->free) {
		blockp = HEAP_index(heap, block, CHUNK);

		if (block == cur_free) {
			THRprintf(GDKout, "#   free block at " PTRFMT " has size " SZFMT " and next " SZFMT "\n", PTRFMTCAST(void *)block, blockp->size, blockp->next);

			cur_free = blockp->next;
			block += blockp->size;
		} else {
			size_t size = blocksize(hheader, blockp);

			THRprintf(GDKout, "#   block at " SZFMT " with size " SZFMT "\n", block, size);