gdk_utils.mx

这个是内存数据库中的一个管理工具

#endif
		if (_memdelta > GDK_mallocedbytes_estimate) {
			/* clearly, the stats are off: it should never become less-than-zero */
			GDKmem_heapcheck(GDKms());
		} else {
			GDK_mallocedbytes_estimate -= _memdelta;
		}
#ifdef GDK_MEM_KEEPHISTO
		{
			int _idx;
			GDKmallidx(_idx, _memdelta);
			GDK_nmallocs[_idx]--;
		}
#endif
	}

@- meminc(size_t memdelta, size_t vmdelta, str fcn)
@= meminc
	{
		size_t _memdelta = (size_t) @1;
		size_t _vmdelta = (size_t) SEG_SIZE(@2,MT_VMUNITLOG);

		gdk_set_lock(GDKthreadLock, @3);
		GDK_mem_cursize += _memdelta;
#ifdef GDK_VM_KEEPHISTO
		{
			int _idx;
			GDKmallidx(_idx, _vmdelta);
			GDK_vm_nallocs[_idx]++;
		}
#endif
		GDK_vm_cursize += _vmdelta;
		gdk_unset_lock(GDKthreadLock, @3);
	}

@- memdec(size_t memdelta, size_t vmdelta, str nme)
@= memdec
	{
		size_t _memdelta = (size_t) @1;
		size_t _vmdelta = (size_t) SEG_SIZE(@2,MT_VMUNITLOG);

		gdk_set_lock(GDKthreadLock, @3);
		GDK_mem_cursize -= _memdelta;
#ifdef GDK_VM_KEEPHISTO
		{
			int _idx;
			GDKmallidx(_idx, _vmdelta);
			GDK_vm_nallocs[_idx]--;
		}
#endif
		GDK_vm_cursize -= _vmdelta;
		gdk_unset_lock(GDKthreadLock, @3);
	}
@c
static void
GDKmemdump(void)
{
	struct mallinfo m = MT_mallinfo();

	THRprintf(GDKout, "\n#mallinfo.arena = " SSZFMT "\n", (ssize_t) m.arena);
	THRprintf(GDKout, "#mallinfo.ordblks = " SSZFMT "\n", (ssize_t) m.ordblks);
	THRprintf(GDKout, "#mallinfo.smblks = " SSZFMT "\n", (ssize_t) m.smblks);
	THRprintf(GDKout, "#mallinfo.hblkhd = " SSZFMT "\n", (ssize_t) m.hblkhd);
	THRprintf(GDKout, "#mallinfo.hblks = " SSZFMT "\n", (ssize_t) m.hblks);
	THRprintf(GDKout, "#mallinfo.usmblks = " SSZFMT "\n", (ssize_t) m.usmblks);
	THRprintf(GDKout, "#mallinfo.fsmblks = " SSZFMT "\n", (ssize_t) m.fsmblks);
	THRprintf(GDKout, "#mallinfo.uordblks = " SSZFMT "\n", (ssize_t) m.uordblks);
	THRprintf(GDKout, "#mallinfo.fordblks = " SSZFMT "\n", (ssize_t) m.fordblks);
#ifdef GDK_MEM_KEEPHISTO
	{
		int i;

		THRprintf(GDKout, "#memory histogram\n");
		for (i = 3; i < GDK_HISTO_MAX_BIT - 1; i++) {
			size_t j = 1 << i;

			THRprintf(GDKout, "# " SZFMT " " SZFMT "\n", j, GDK_nmallocs[i]);
		}
	}
#endif
#ifdef GDK_VM_KEEPHISTO
	{
		int i;

		THRprintf(GDKout, "\n#virtual memory histogram\n");
		for (i = 12; i < GDK_HISTO_MAX_BIT - 1; i++) {
			size_t j = 1 << i;

			THRprintf(GDKout, "# " SZFMT " " SZFMT "\n", j, GDK_vm_nallocs[i]);
		}
	}
#endif
}


static void
GDKmemchk(int memchk, int vmchk)
{
	size_t memtarget = GDKmem_inuse();
	size_t vmtarget = GDKvm_cursize();

	MEMDEBUG {
		/* Protect from being called recursivly because THRprintf
		   allocates memory */
		static int printing[THREADS];
		int tid = THRgettid();

		if (!printing[tid - 1]) {
			printing[tid - 1] = TRUE;
			THRprintf(GDKout, "#GDKmemchk (memcur=" SZFMT ",memmax=" SZFMT ") (vmcur=" SZFMT ",vmmax=" SZFMT ")\n", memtarget, GDK_mem_maxsize, GDKvm_cursize(), GDK_vm_maxsize);
			printing[tid - 1] = FALSE;
		}
	}
	memtarget = (memchk && memtarget > GDK_mem_maxsize) ? memtarget - GDK_mem_maxsize : 0;
	vmtarget = (vmchk && vmtarget > GDK_vm_maxsize) ? vmtarget - GDK_vm_maxsize : 0;
	if (memtarget > 0 || vmtarget > 0) {
		if (memtarget > 0) {
			int t = GDKms();

			/* check max every 10 secs, and only if its incorrectness bothers us (i.e. causes BBPtrim) */
			if ((t - GDK_heapcheck_last) > 10000) {
				GDKmem_heapcheck(t);	/* correct thread-unsafe estimate */
			}
		}
		BBPtrim(memtarget, vmtarget);
		GDK_mem_allocs = GDK_vm_allocs = 0;
	} else {
		if (memchk)
			GDK_mem_allocs = 0;
		if (vmchk)
			GDK_vm_allocs = 0;
	}
}

@}

@+ Stack tracing

Very simple malloc tracing implementation that allows to retain on which stack frame
a malloc was done. This is code originally developed by DD, and
is used only  by them.
Two bats are used: stackBat and memBat.
StackBat contains a DAG of all methods used for memory allocation.
MemBat contains a mapping from allocated memory addresses to an index
in the stackBat which indicates the place the memory is allocated.
@{
@c
#ifdef GDK_MEM_TRACE

static BAT *stackBat = NULL;
static BAT *memBat = NULL;
typedef ptr mem_t;

char *
stack_test(int i, int j)
{
	if (--i == 0)
		return (char *) GDKmalloc(j);

	return stack_test(i, j);
}


void
mtrace_test()
{
	char *t = stack_test(1, 1);

	GDKfree(t);
	GDKfree(t);		/* double delete */
	t = stack_test(2, 2);
	t = GDKrealloc(t, 20);	/* leaked */
}


static int
GDKmtrace(bit *enable)
{
	if (*enable) {
		if (memBat == NULL) {
			memBat = BATnew(TYPE_ptr, TYPE_int, 16384);
			if (memBat == NULL)
				return GDK_FAIL;
			stackBat = BATnew(TYPE_ptr, TYPE_ptr, 16384);
			if (stackBat == NULL)
				return GDK_FAIL;
			BBPrename(memBat->batCacheid, "mem_trace");
			BBPrename(stackBat->batCacheid, "mem_stack");
			memBat->hsorted = memBat->tsorted = 0;
			BATkey(memBat, TRUE);
			stackBat->hsorted = stackBat->tsorted = 0;
			BATset(stackBat, TRUE);
		}
		mtrace++;
	} else if (mtrace > 0)
		mtrace--;

	memBat->batDirty = TRUE;
	stackBat->batDirty = TRUE;

	if (*enable == bit_nil) {
		printf("testing mtrace:\n");
		mtrace_test();
		--mtrace;
	}

	return GDK_SUCCEED;
}


BUN
add_dag(BAT *b, BUN p, mem_t current, mem_t parent)
{
	const static int bunsize = sizeof(mem_t) + sizeof(mem_t);
	BUN last = BUNlast(b);

	while (p != last) {
		if ((*(mem_t *) BUNhloc(b, p)) == current && (*(mem_t *) BUNtloc(b, p)) == parent)
			return p;
		p += bunsize;
	}

	if (b->batBuns->free + bunsize > b->batBuns->size) {
		int tmp = mtrace;

		mtrace = 0;
		if (BATextend(b, BATgrows(b)) == NULL)
			return BUNlast(b);
		mtrace = tmp;
		last = BUNlast(b);
	}
	bunfastins_nocheck(b, last, &current, &parent, bunsize);
	return last;
}


#define MAX_STACK_ADDR 16
#define STACK_LIST(STACK_ENTRY)						\
	STACK_ENTRY(1) STACK_ENTRY(2) STACK_ENTRY(3)			\
	STACK_ENTRY(4) STACK_ENTRY(5) STACK_ENTRY(6) STACK_ENTRY(7)	\
	STACK_ENTRY(8) STACK_ENTRY(9) STACK_ENTRY(10) STACK_ENTRY(11)	\
	STACK_ENTRY(12) STACK_ENTRY(13) STACK_ENTRY(14) STACK_ENTRY(15)

#define STACK_READ(X)						\
	if (continue_stack_trace &&				\
	    ((mem_t)__builtin_frame_address((X)) != 0L) &&	\
	    ((X) < MAX_STACK_ADDR)) {				\
		parent = current;				\
		current = (mem_t)__builtin_return_address((X)); \
		if(parent)					\
			p = add_dag(b, p, current, parent);	\
	} else if (continue_stack_trace) {			\
		continue_stack_trace = FALSE;			\
	}

int
add_stack(BAT *b)
{
	mem_t current = 0, parent = 0;
	bit continue_stack_trace = TRUE;
	BUN p = BUNfirst(b);

	STACK_LIST(STACK_READ);
	return BUNindex(b, p);
}

#define __USE_GNU 1
#include <dlfcn.h>
void
print_address(ptr address)
{
	Dl_info dlip;
	char *filename;

	dladdr(address, &dlip);

	filename = 0;		/* strrchr(dlip.dli_fname,'/'); */
	printf(PTRFMT "\t%s\t%s\n", PTRFMTCAST address, dlip.dli_sname, filename ? filename + 1 : dlip.dli_fname);
}

static int
GDKmprint(int *pidx)
{
	int idx = *pidx;
	BUN p = BUNptr(stackBat, idx), first = BUNfirst(stackBat);
	const static int bunsize = sizeof(mem_t) + sizeof(mem_t);
	mem_t *a;

	if (p == NULL)
		printf("No stack for index %d\n", idx);

	a = (mem_t *) BUNhloc(stackBat, p);
	print_address(*a);
	a = (mem_t *) BUNtloc(stackBat, p);
	print_address(*a);

	while (p > first) {
		p -= bunsize;
		if (*(mem_t *) BUNhloc(stackBat, p) == *a) {
			a = (mem_t *) BUNtloc(stackBat, p);
			print_address(*a);
		}
	}

	return GDK_SUCCEED;
}


void
add_mem(BAT *b, mem_t mem, int idx)
{
	REGISTER BUN last;
	const static int bunsize = sizeof(mem_t) + sizeof(int);

	if (b->batBuns->free + bunsize > b->batBuns->size) {
		int tmp = mtrace;

		mtrace = 0;
		if (BATextend(b, BATgrows(b)))
			return;
		mtrace = tmp;
	}
	last = BUNlast(b);
	bunfastins_nocheck(b, last, &mem, &idx, bunsize);
}


void
del_mem(BAT *b, mem_t mem)
{
	REGISTER BUN first = BUNfirst(b), p = BUNlast(b);
	const static int bunsize = sizeof(mem_t) + sizeof(int);

	for (p -= bunsize; p >= first; p -= bunsize) {
		if (*(mem_t *) BUNhloc(b, p) == mem) {
			int idx = -*(int *) BUNtloc(b, p);

			if (idx > 0) {
				printf("Double deletion of memory " PTRFMT ", " "size: " SSZFMT ", stack:%d\n", PTRFMTCAST(void *)mem, ((ssize_t *) mem)[-1], idx);

				if (mtrace > 1)
					GDKmprint(&idx);
			} else {
				BUNdelete(b, p);
				/* BUNinplace(b,p,&mem,&idx,0); */
			}
			return;
		}
	}
	printf("Free of unknown memory " PTRFMT ", size: " SSZFMT "\n", PTRFMTCAST(void *) mem, ((ssize_t *) mem)[-1]);
}
#endif
@}

@+ Malloc
Malloc normally maps through directly to the OS provided malloc/free/realloc
calls. Where possible, we want to use the -lmalloc library on Unix systems,
because it allows to influence the memory allocation strategy. This can prevent
fragmentation and greatly help enhance performance.

The "added-value" of the GDKmalloc/GDKfree/GDKrealloc over the standard OS
primitives is that the GDK versions try to do recovery from failure to malloc by
initiating a BBPtrim. Also, big requests are redirected to anonymous virtual
memory. Finally, additional information on block sizes is kept (helping efficient
reallocations) as well as some debugging that guards against duplicate frees.

A number of different strategies are available using different switches, however:
@table @samp

@item zero sized blocks
Normally, GDK gives fatal errors on illegal block sizes.
This can be overridden with  GDK_MEM_NULLALLOWED.

@item resource tracking
Many malloc interfaces lack a routine that tells the size of a block
by the pointer. We need this information for correct malloc statistics.

@item outstanding block histograms
In order to solve the problem, we allocate extra memory in front of the
returned block. With the resource tracking in place, we keep a total of
allocated bytes.  Also, if GDK_MEM_KEEPHISTO is defined, we keep a histogram
of the outstanding blocks on the log2 of the block size (similarly for virtual.
memory blocks; define GDK_VM_KEEPHISTO).

@item ensuring 8-byte alignment
While doing the resource tracking, we can in one go solve the problem
of 32-bit systems that return non-8 bytes aligned pointers. This is
encoded by storing a size + 1 (we always normalize sizes to multiples of
8, so this is easily recognized). This check only done if
GDK_MEM_MISALIGN is @strong{not} enabled.

@item redirection to anonymous VM
Sometimes, fragmentation problems arise on mallocs that are not tunable.
In that case, it makes sense to redirect large block requests (taken to be
larger than GDK_mem_bigsize) to anonymous virtual memory. The seamless
implementation of this stores the @strong{negative} block size in front of the
pointer (as well as the VM maxsize), so the malloc primitives can recognize
these redirected blocks.
@end table

64-bits update: POSIX mallinfo is severely broken, as it uses int-s for memory sizes!!
This causes corruption of mallinfo stats. As we depend on those, we should keep the
malloc arena small. Thus, VM redirection is now quickly applied: for all mallocs > 1MB.
@{
@c
static void
GDKmemfail(str s, size_t len, size_t memtarget, size_t vmtarget)
{
	int bak = GDKdebug;

	/* bumped your nose against the wall; try to prevent repetition by adjusting maxsizes */
	if (memtarget < 0.3 * GDKmem_inuse()) {
		size_t newmax = (size_t) (0.7 * (double) GDKmem_inuse());

		if (newmax < GDK_mem_maxsize)
			GDK_mem_maxsize = newmax;
	}
	if (vmtarget < 0.3 * GDKvm_cursize()) {
		size_t newmax = (size_t) (0.7 * (double) GDKvm_cursize());

		if (newmax < GDK_vm_maxsize)
			GDK_vm_maxsize = newmax;
	}

	gdk_set_lock(GDKthreadLock, "GDKmemfail");
	THRprintf(GDKout, "#%s(" SZFMT ") fail => BBPtrim(enter) usage[mem=" SZFMT ",vm=" SZFMT "]\n", s, len, GDKmem_inuse(), GDKvm_cursize());
	GDKmemdump();
	GDKdebug |= 4;
	gdk_unset_lock(GDKthreadLock, "GDKmemfail");

	BBPtrim(memtarget, vmtarget);

	gdk_set_lock(GDKthreadLock, "GDKmemfail");
	GDKdebug = MIN(GDKdebug, bak);
	THRprintf(GDKout, "#%s(" SZFMT ") fail => BBPtrim(ready) usage[mem=" SZFMT ",vm=" SZFMT "]\n", s, len, GDKmem_inuse(), GDKvm_cursize());
	GDKmemdump();
	gdk_unset_lock(GDKthreadLock, "GDKmemfail");
}

/* the blocksize is stored in the ssize_t before it. Negative size <=> VM memory */
#define GDK_MEM_BLKSIZE(p) ((ssize_t*) (p))[-1]

#ifndef GDK_MEM_MISALIGN

/* allocate 8 bytes extra (so it stays 8-bytes aligned) and put realsize in front */
#define GDKmalloc_prefixsize(s,size) {					\
	s = (ssize_t *) malloc(size + 8);				\
	if (s != NULL) {						\
		assert((((size_t) s)&7) == 0); /* no MISALIGN */	\
		s = (ssize_t*) ((char*) s + 8);				\
		s[-1] = (ssize_t) (size + 8);				\
	}								\
}

#else
/* work around old stupid libc mallocs that give 4-byte aligned pointers */
#define GDKmalloc_prefixsize(s,size) {					\
	s = (ssize_t *) malloc(size+8);					\
	if (((size_t) s) & 4) { /* misaligned */			\
		assert(sizeof(size_t) == 4); /* not on 64-bits */	\
		s = (ssize_t*) ((char*) s + 4);				\
		s[-1] = (ssize_t) (size + 9);  /* 1-bit is a marker */	\
	} else if (s != NULL) {						\
		s = (ssize_t*) ((char*) s + 8);				\
		s[-1] = (ssize_t) (size + 8);				\
	}								\
}
#endif


void *
GDKmallocmax(size_t size, size_t * maxsize, int emergency)
{
	ssize_t *s = NULL;

	if (size == 0) {
#ifdef GDK_MEM_NULLALLOWED
		return NULL;
#else
		GDKfatal("GDKmallocmax: called with size " SZFMT "", size);
#endif
	}
	size = (size + 3) & ~3;	/* round up to a multiple of four */
	if (size > GDK_mem_bigsize) {
		size_t newsize = size + sizeof(size_t) + sizeof(size_t);
		size_t newmax = MAX(*maxsize, newsize);

		s = (ssize_t *) GDKvmalloc(newsize, &newmax, emergency);
		if (s == 0 && emergency == 0)
			return s;
		MT_alloc_register(s, *maxsize, 'S');
		s += 2;
		s[-2] = (ssize_t) newmax;
		s[-1] = -((ssize_t) newsize);
		*maxsize = newmax - (sizeof(size_t) + sizeof(size_t));
		return (void *) s;
	}
	CHKMEM(size, 0);
	GDKmalloc_prefixsize(s, size);
	if (s == NULL) {
		GDKmemfail("GDKmalloc", size, BBPTRIM_ALL, 0);
		GDKmalloc_prefixsize(s, size);
		if (s == NULL) {
			if (emergency)
				return NULL;
			MT_alloc_print();
			GDKfatal("GDKmallocmax: failed for %u bytes", size);
		} else {
			THRprintf(GDKout, "#GDKmallocmax: recovery ok. Continuing..\n");
		}
	}
	*maxsize = size;
	@:heapinc(size+8,s)@
	return (void *) s;
}

void *
GDKmalloc(size_t size)
{
	return GDKmallocmax(size, &size, 1);
}

void *
GDKzalloc(size_t size)
{
	size_t maxsize;
	void *p = GDKmallocmax(size, &maxsize, 1);
	if (p)
		memset(p,0,size);
	return p;
}

void
GDKfree(void *blk)
{
	ssize_t size = 0, *s = (ssize_t *) blk;

	if (s == NULL)
		return;

	size = GDK_MEM_BLKSIZE(s);

	/* check against duplicate free */
	assert((size & 2) == 0);