gdk_utils.mx

这个是内存数据库中的一个管理工具

@' The contents of this file are subject to the MonetDB Public License
@' Version 1.1 (the "License"); you may not use this file except in
@' compliance with the License. You may obtain a copy of the License at
@' http://monetdb.cwi.nl/Legal/MonetDBLicense-1.1.html
@'
@' Software distributed under the License is distributed on an "AS IS"
@' basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
@' License for the specific language governing rights and limitations
@' under the License.
@'
@' The Original Code is the MonetDB Database System.
@'
@' The Initial Developer of the Original Code is CWI.
@' Portions created by CWI are Copyright (C) 1997-2007 CWI.
@' All Rights Reserved.

@f gdk_utils
@a M. L. Kersten, P. Boncz

@* Utilities
The utility section contains functions to initialize the Monet database
system, memory allocation details, and a basic system logging scheme.
@-
@{
@h
#ifndef _GDK_UTILS_H_
#define _GDK_UTILS_H_

gdk_export void GDKlog(const char *format, ...);
gdk_export void GDKlockHome(void);
gdk_export void GDKunlockHome(void);
gdk_export int GDKgetHome(void);

gdk_export lng GDKusec(void);
gdk_export int GDKms(void);

gdk_export int gdk_alloc_map;
gdk_export BAT *GDKenv;

@c
#include "monetdb_config.h"

#ifndef GDK_NOLINK
#include "gdk.h"

#else /* GDK_NOLINK */

#undef GDKmalloc
#undef GDKrealloc
#undef GDKfree

#define GDKmalloc(size)	malloc(size)
#define GDKrealloc(buf,size)	realloc(buf,size)
#define GDKfree(buf)	free(buf)
#endif

int gdk_alloc_map = 1;
char GDKdbfarmStr[PATHLENGTH] = { "dbfarm" };
char GDKdbnameStr[PATHLENGTH] = { 0 };

BAT *GDKenv = NULL;

#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif

#ifdef HAVE_FTIME
#include <sys/timeb.h>
#endif

#if TIME_WITH_SYS_TIME
# include <sys/time.h>
# include <time.h>
#else
# if HAVE_SYS_TIME_H
#  include <sys/time.h>
# else
#  include <time.h>
# endif
#endif

#ifdef HAVE_PWD_H
# include <pwd.h>
#endif

#ifdef NATIVE_WIN32
#define chdir _chdir
#endif
@}
@+ Monet configuration file
Parse a possible MonetDB.conf file (either in the default location
<prefix>/etc/MonetDB.conf, or as specified by either an environment variable
$MONETDB_CONFIG, or by command line option -c/--config)
to extract pre-settings of system variables.
Un-recognized parameters are simply skipped, because they  may be
picked up by other components of the system.
The consequence is that making a typing error in the configuration file
may be unnoticed for a long time.
Syntax errors are immediately flagged, though.

Since the GDK kernel moves into the database directory, we need
to keep the absolute path to the MonetDB.conf file for top-levels
to access its information.
@{
@c

static int
GDKenvironment(str dbname, str dbfarm)
{
	if (dbname == 0) {
		fprintf(stdout, "!GDKenvironment: database name missing.\n");
		return 0;
	}
	if (dbfarm == 0) {
		fprintf(stdout, "!GDKenvironment: dbfarm missing.\n");
		return 0;
	}

	if (!MT_path_absolute(dbfarm)) {
		fprintf(stdout, "!GDKenvironment: wrong directory %s.\n", dbfarm);
		return 0;
	}

	strncpy(GDKdbnameStr, dbname, PATHLENGTH);
	strncpy(GDKdbfarmStr, dbfarm, PATHLENGTH);
	return 1;
}

@h
gdk_export char *GDKgetenv(const char *name);

@c
char *
GDKgetenv(const char *name)
{
	BUN b = BUNfnd(GDKenv, (ptr) name);

	if (b)
		return BUNtail(GDKenv, b);
	return NULL;
}

@h
gdk_export int GDKgetenv_isyes(const char *name);

@c
int
GDKgetenv_isyes(const char *name)
{
	char *val = GDKgetenv(name);

	if (val && strcasecmp(val, "yes") == 0) {
		return 1;
	}
	return 0;
}

@h
gdk_export void GDKsetenv(str name, str value);

@c
void
GDKsetenv(str name, str value)
{
	BUNins(GDKenv, name, value, FALSE);
	BATfakeCommit(GDKenv);
}

@}

@+ System logging
Per database a log file can be maintained for collection
of system management information. Its contents is driven
by the upper layers, which encode information such as
who logged on and how long the session went on.
The lower layers merely store error information on the file.
It should not be used for crash recovery, because this should be
dealt with on a per client basis.

@-
A system log can be maintained in the database to keep track
of session and crash information. It should regularly be
refreshed to avoid disk overflow.
@{
@c
#define GDKLOCK	".gdk_lock"

static FILE *GDKlockFile = 0;

#define GDKLOGOFF	"LOGOFF"
#define GDKFOUNDDEAD	"FOUND	DEAD"
#define GDKLOGON	"LOGON"
#define GDKCRASH	"CRASH"

@-
Single-lined comments can now be logged safely, together with process, thread
and user ID, and the current time.
@c
/* VARARGS */
void
GDKlog(const char *format, ...)
{
	va_list ap;
	char *p = 0, buf[1024];
	int mustopen = GDKgetHome();
	time_t tm = time(0);

	if (!MT_initialized()) 
		return;

	va_start(ap, format);
	vsprintf(buf, format, ap);
	va_end(ap);

	/* remove forbidden characters from message */
	for (p = buf; (p = strchr(p, '\n')) != NULL; *p = ' ')
		;
	for (p = buf; (p = strchr(p, '\@')) != NULL; *p = ' ')
		;

	fseek(GDKlockFile, 0, SEEK_END);
#ifndef HAVE_GETUID
#define getuid() 0
#endif
	fprintf(GDKlockFile, "USR=%d PID=%d TIME=%s \@ %s", 
		(int) getuid(), (int) getpid(), ctime(&tm),buf);
	fflush(GDKlockFile);

	if (mustopen)
		GDKunlockHome();
}

@}
@+ Interrupt handling
The current version simply catches signals and prints a warning.
It should be extended to cope with the specifics of the interrupt
received.
@{
@c
#ifndef NATIVE_WIN32
#include <signal.h>
#if 0 /* these are unused */
static RETSIGTYPE
BATSIGignore(int nr)
{
	GDKsyserror("! ERROR signal %d caught by thread %lx\n", nr, (size_t) MT_getpid());
}

static RETSIGTYPE
BATSIGabort(int nr)
{
	GDKfatal("BATSIGabort: signal %d caught by thread %lx\n", nr, (size_t) MT_getpid());
}
#endif

static RETSIGTYPE
BATSIGinterrupt(int nr)
{
	GDKexit(nr);
}

#ifdef SIGCHLD
static RETSIGTYPE
BATSIGchild(int nr)
{
	int status;

	(void) nr;

	while (waitpid(-1, &status, WNOHANG) > 0)
		;
	PARDEBUG THRprintf(GDKerr, "CHILD EXITED \n");

	(void) signal(SIGPIPE, BATSIGchild);
}
#endif


static RETSIGTYPE
BATSIGcrash(int nr)
{
	static int crash = 0;

	if (!crash++) {
		GDKerror("BATSIGcrash: Mserver internal error (%s), please restart.\n" "(One potential cause could be that your disk might be full...)\n",
#ifdef SIGBUS
			 nr == SIGBUS ? "Bus error" :
#endif
			 (nr == SIGSEGV ? "Segmentation fault" : "unknown"));
		GDKlog(GDKCRASH);
		MT_global_exit(1);
	}
}

static int
BATSIGinit(void)
{
#ifdef SIGBUS
	(void) signal(SIGBUS, BATSIGcrash);
#endif
	(void) signal(SIGSEGV, BATSIGcrash);
#ifdef SIGCHLD
	(void) signal(SIGCHLD, BATSIGchild);
#endif
/* HACK to pacify compiler */
#if (defined(__INTEL_COMPILER) && (SIZEOF_VOID_P > SIZEOF_INT))
#undef  SIG_IGN			/*((__sighandler_t)1 ) */
#define SIG_IGN   ((__sighandler_t)1L)
#endif

#ifdef SIGPIPE
	(void) signal(SIGPIPE, SIG_IGN);
#endif
#ifdef __SIGRTMIN
	(void) signal(__SIGRTMIN + 1, SIG_IGN);
#endif
#ifdef SIGHUP
	(void) signal(SIGHUP, MT_global_exit);
#endif
#ifdef SIGINT
	(void) signal(SIGINT, BATSIGinterrupt);
#endif
#ifdef SIGTERM
	(void) signal(SIGTERM, BATSIGinterrupt);
#endif
	return 0;
}
#endif	/* NATIVE_WIN32 */

@}
@+ Memory management
Memory management in GDK mostly relies on the facilities offered by the
underlying OS.  The below routines monitor the available memory resources
which consist of physical swap space and logical vm space.
There are three kinds of memory, that affect these two resources in different ways:
@table @samp

@item memory mapping
 which ask for a logical region of virtual memory space.
In principal, no physical memory is needed to keep the system afloat here,
as the memory mapped file is swapped onto a disk object that already exists.

Actually, there are two kings of memory mapping used in GDK, namely
read-only direct mapped and writable copy-on write. For the dirty
pages, the latter actually also consumes physical memory resources,
but that is ignored here for simplicity.

@item anonymous virtual memory
This is virtual memory that is mapped on the swap file. Hence, this consumes
both logical VM space resources and physical memory space.

@item malloced memory
comes from the heap and directly consumes physical memory resources.
@end table

We check the resource consumption with preset target values, and if these
are exceeded, the routine BBPtrim is called that will unload the
least recently used BATs in order to decrease memory usage.

The malloc routine checks the memory consumption every 1000 calls,
or for calls larger that 50000 bytes. Consequently, at least every
50MB increase, alloc memory is checked. The VM calls always check
the memory consumption.
@{
@h
/* default setting to administer everything */
#define GDK_MEM_NULLALLOWED

#if SIZEOF_VOID_P==8
#define GDK_VM_MAXSIZE	LL_CONSTANT(137438953472)	/* :-) a 64-bit OS: 128 GB */
#elif defined(WIN32)
#define GDK_VM_MAXSIZE	LL_CONSTANT(536870912)	/* :-( New Technology DOS extender: 512MB */
#else
#define GDK_VM_MAXSIZE	LL_CONSTANT(1610612736)	/* :-| a 32-bit OS: 1.5GB */
#endif
@c
size_t GDK_mem_maxsize = GDK_VM_MAXSIZE;
size_t GDK_mem_bigsize = 1<<20;
size_t GDK_vm_minsize = GDK_VM_MAXSIZE;
size_t GDK_vm_maxsize = GDK_VM_MAXSIZE;

int GDK_vm_allocs = 0;
int GDK_mem_allocs = 0;

/* at least each 50M of memory increase, BBPtrim is run */
#define CHKMEM(meminc, vminc) do {					\
	int memchk = (meminc>0 && (++GDK_mem_allocs>=1000 || meminc>LL_CONSTANT(50000))); \
	int vmchk = (vminc>0 && (++GDK_vm_allocs>=10 || vminc>LL_CONSTANT(5000000))); \
	if (memchk || vmchk) GDKmemchk(memchk, vmchk);			\
} while (0)

#define SEG_SIZE(x,y)   ((x)+(((x)&((1<<(y))-1))?(1<<(y))-((x)&((1<<(y))-1)):0))
#define MAX_BIT         ((int) (sizeof(ssize_t)<<3))

/* histogram update macro */
#define GDKmallidx(idx,size)				\
	{						\
		int _mask;				\
		if (size < 128) {			\
			_mask = (1<<6);			\
			idx = 7;			\
		} else {				\
			_mask = (1<<(MAX_BIT-1));	\
			idx = MAX_BIT;			\
		}					\
		while(idx-- > 4) {			\
			if (_mask&size) break;		\
			_mask >>=1;			\
		}					\
	}

volatile size_t GDK_mallocedbytes_estimate = 0;
static ssize_t GDK_mem_cursize = 0;
static ssize_t GDK_vm_cursize = 0;

#ifdef GDK_MEM_TRACE
static int mtrace = 0;
#endif

size_t
GDKvm_heapsize(void)
{
#ifdef _CYGNUS_H_
	return ((size_t) (96 << 20));
#else
	size_t ret = GDKmem_heapsize();

#ifdef __linux__
	/* on linux, malloc may also use mmapped space, so the bytes-in-malloc may be much bigger than the sbrk() region */
	ret = MAX(GDK_mallocedbytes_estimate, ret);
#endif
	return ret;
#endif
}

size_t
GDKmem_heapsize(void)
{
	size_t heapsize = (MT_heapcur() - MT_heapbase);

	return (size_t) SEG_SIZE(heapsize, MT_VMUNITLOG);
}

size_t
GDKmem_inuse(void)
{
	/* RAM/swapmem that Monet is really using now */
	ssize_t mem_cursize = GDK_mem_cursize;
	size_t mem_mallocedbytes_estimate = GDK_mallocedbytes_estimate;

	if (mem_cursize < 0)
		mem_cursize = GDK_mem_cursize = 0;

	return mem_cursize + mem_mallocedbytes_estimate;
}

size_t
GDKmem_cursize(void)
{
	/* RAM/swapmem that Monet has claimed from OS */
	ssize_t mem_cursize = GDK_mem_cursize;

	if (mem_cursize < 0)
		mem_cursize = GDK_mem_cursize = 0;

	return mem_cursize + GDKmem_heapsize();
}

size_t
GDKvm_cursize(void)
{
	/* current Monet VM address space usage */
	ssize_t vm_cursize = GDK_vm_cursize;

	if (vm_cursize < 0)
		vm_cursize = GDK_vm_cursize = 0;

	return vm_cursize + GDKvm_heapsize();
}

#ifdef GDK_VM_KEEPHISTO
volatile ssize_t GDK_vm_nallocs[MAX_BIT] = { 0 };
#endif
#ifdef GDK_MEM_KEEPHISTO
volatile ssize_t GDK_nmallocs[MAX_BIT] = { 0 };
#endif

size_t
GDKmem_heapinuse(void)
{
	return GDK_mallocedbytes_estimate;
}

volatile int GDK_heapcheck_last = 0;
int mallinfo_ok = 1;

static INLINE void
GDKmem_heapcheck(int t)
{
	/* correct heap estimate with the real thing */
	if (mallinfo_ok) {
		struct mallinfo m = MT_mallinfo();
#if ((SIZEOF_VOID_P==8) && defined(HAVE_SIGNED_MALLINFO))
		if (m.usmblks < 0 || m.uordblks < 0 || m.hblkhd < 0) 
			mallinfo_ok = 0; /* incredible POSIX incompetence!! non-64-bit safe mallinfo */
		else 
#endif 
			GDK_mallocedbytes_estimate = (size_t) (m.usmblks + m.uordblks + m.hblkhd);
	}
	GDK_heapcheck_last = t;
}

@- heapinc(size_t _memdelta, void* _blk)
@= heapinc
	{
		size_t _memdelta = (size_t) @1;
#ifdef GDK_MEM_TRACE
		void*  _blk = (void*) @2;

		if (mtrace) {
			int _idx = add_stack(stackBat);
			add_mem(memBat,_blk,_idx);
		}
#endif
		GDK_mallocedbytes_estimate += _memdelta;
#ifdef GDK_MEM_KEEPHISTO
		{
			int _idx;
			GDKmallidx(_idx, _memdelta);
			GDK_nmallocs[_idx]++;
		}
#endif
	}

@- heapdec(size_t memdelta, void* _blk)
@= heapdec
	{
		size_t _memdelta = (size_t) @1;
#ifdef GDK_MEM_TRACE
		void*  _blk = (void*) @2;

		if (mtrace) {
			del_mem(memBat,_blk);
		}