statistics.mx

一个内存数据库的源代码这是服务器端还有客户端

@' The contents of this file are subject to the MonetDB Public License
@' Version 1.1 (the "License"); you may not use this file except in
@' compliance with the License. You may obtain a copy of the License at
@' http://monetdb.cwi.nl/Legal/MonetDBLicense-1.1.html
@'
@' Software distributed under the License is distributed on an "AS IS"
@' basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
@' License for the specific language governing rights and limitations
@' under the License.
@'
@' The Original Code is the MonetDB Database System.
@'
@' The Initial Developer of the Original Code is CWI.
@' Portions created by CWI are Copyright (C) 1997-2007 CWI.
@' All Rights Reserved.

@f statistics
@a M.L. Kersten
@+ Statistics box.
Most optimizers need easy access to key information 
for proper plan generation. Amongst others, this
volatile information consists of the tuple count, size,
min- and max-value, the null-density, and 
a histogram of the value distribution.

The statistics are management by a Box, which gives 
a controlled environment to manage a collection of BATs
and system variables.

BATs have to be deposit into the  statistics box 
separately, because the costs attached maintaining them are high.
The consistency of the statistics box is partly
the responsibility of the upper layers. There is
no automatic triggering when the BATs referenced
are heavily modified or are being destroyed. 
They disappear from the statistics box the first time 
an invalid access is attempted or during system reboot.

The staleness of the information can be controlled in several
ways. The easiest, and most expensive, is to assure that
the statistics are updated when you start the server.
Alternative, you can set a expiration interval, which will
update the information only when it is considered expired.
This test will be triggered either at server restart or
your explicit call to update the statistics tables.
The statistics table is commited each time you change it.

A forced update can be called upon when the front-end
expects the situation to be changed drastically.

The statistics table is mostly used internally, but
once in a while you need a dump for closed inspection.
in your MAL program for inspection. Just
use the BBP bind operation to locate them in the buffer pool.
@mal
module statistics;

pattern open():void
address STATopen
comment "Locate and open the statistics box";
pattern close():void
address STATclose
comment "Close the statistics box ";
pattern destroy():void
address STATdestroy
comment "Destroy the statistics box";
pattern take(name:any_1):any_2
address STATtake
comment "Take a variable out of the statistics box";
command deposit(name:str) :void
address STATdepositStr
comment "Enter a new BAT into the statistics box";
command deposit(name:bat[:any_1,:any_2]) :void
address STATdeposit
comment "Enter a new BAT into the statistics box";

pattern releaseAll():void
address STATreleaseAll
comment "Release all variables in the box";
pattern release(name:str) :void
address STATreleaseStr
comment "Release a single BAT from the  box";
command release(name:bat[:any_1,:any_2]):void
address STATrelease
comment "Release a single BAT from the  box";
pattern toString(name:any_1):str
address STATtoString
comment "Get the string representation of an element in the box";
pattern discard(name:str) :void
address STATdiscard
comment "Release a BAT by name from the box";
command discard(name:bat[:any_1,:any_2]) :void
address STATdiscard2
comment "Release a BAT variable from the box";
pattern newIterator()(:lng,:str)
address STATnewIterator
comment "Locate next element in the box";
pattern hasMoreElements()(:lng,:str)
address STAThasMoreElements
comment "Locate next element in the box";

command update()
address STATupdate
comment "Check for stale information";

command forceUpdate()
address STATforceUpdateAll
comment "Bring all information up to date";

command forceUpdate(bnme:str)
address STATforceUpdate
comment "Bring the statistics up to date for one BAT";

command prelude() :void
address STATprelude
comment "Initialize the statistics package";

command epilogue() :void
address STATepilogue
comment "Release the resources of the statistics package";

command dump() :void
address STATdump
comment "Display the statistics table";

command getObjects():bat[:int,:str]
address STATgetObjects
comment "Return a table with BAT names managed";

command getHotset():bat[:int,:str]
address STATgetHotset
comment "Return a table with BAT names that have been touched
since the start of the session";

command getCount(nme:str):lng
address STATgetCount
comment "Return latest stored count information";

command getSize(nme:str):lng
address STATgetSize
comment "Return latest stored count information";

command getMin(nme:str):lng
address STATgetMin
comment "Return latest stored minimum information";

command getMax(nme:str):lng
address STATgetMax
comment "Return latest stored maximum information";

command getHistogram(nme:str):bat[:any_1,:any_2]
address STATgetHistogram
comment "Return the latest histogram");
@{
@- Implementation

The Box can optimize its behaviour by e.g. keeping 
often used histograms locked into memory or propagate
updates quickly. This is only applicable for objects
taken from the box and until it is released.

The implementation is based on the assumption that in
most cases front-ends deal with BAT names as opposed to
the internal index.

Furthermore, the code is split in two sections. The first
contains the primitves as they will also be used in
linked-in code. The second part provides a MAL interface.
It is assumed that any direct call obeys the semantics of
this interface.
@include prelude.mx
@h
#ifndef _STATISTICS_DEF
#define _STATISTICS_DEF

/* #define DEBUG_STATISTICS */

#include <mal.h>
#include <mal_client.h>
#include <mal_interpreter.h>

#ifdef WIN32
#ifndef LIBSTATISTICS
#define s_export extern __declspec(dllimport)
#else
#define s_export extern __declspec(dllexport)
#endif
#else
#define s_export extern
#endif

#endif /* _STATISTICS_DEF */
@-
@c
#include "mal_config.h"
#include "statistics.h"
#include "algebra.h"
s_export str STATforceUpdateAll(int *ret);
s_export str STATdrop(str nme);
s_export str STATforceUpdateAll(int *ret);
s_export str STATenroll(int *ret, str *nme);
s_export str STATenrollHistogram(int *ret, str *nme);
s_export str STATupdateAll(int *ret, int forced);
s_export str STATupdate(int *ret);
s_export str STATforceUpdate(int *ret, str *nme);
s_export str STATdump(int *ret);
s_export str STATprelude(int *ret);
s_export str STATepilogue(int *ret);
s_export str STATopen(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATclose(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATdestroy(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATdepositStr(int *ret, str *nme);
s_export str STATdeposit(int *ret, int *bid);
s_export str STATtake(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATrelease(int *ret, int *bid);
s_export str STATreleaseStr(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATreleaseAll(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATdiscard(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATdiscard2(int *ret, int *bid);
s_export str STATtoString(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATnewIterator(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STAThasMoreElements(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
s_export str STATgetHotset(int *bid);
s_export str STATgetObjects(int *bid);
s_export str STATgetHistogram(int *ret, str *nme);


BAT *STAT_id_inuse;		/* BATs information taken from the box */
BAT *STAT_id_nme;		/* mapping from BBP index */
BAT *STAT_id_expire;
BAT *STAT_id_stamp;		/* BAT last time stamp */
BAT *STAT_id_count;
BAT *STAT_id_size;
BAT *STAT_id_min_lng;
BAT *STAT_id_max_lng;
BAT *STAT_id_histogram;

@-
The statistics are currently limited to the server session.
Upon need we can turn it into a persistent mode.
@c
static int statisticsMode= TRANSIENT;

static BAT *
STAT_create(str hnme, str tnme, int ht, int tt)
{
	BAT *b;
	char buf[128];

	snprintf(buf, 128, "stat_%s_%s", hnme, tnme);
	b = BATdescriptor(BBPindex(buf));
	if (b)
		return b;

	b = BATnew(ht, tt, 256);
	if (b == NULL)
		GDKfatal("STAT_create: could not allocate.");

	BATkey(b, TRUE);
	BBPrename(b->batCacheid, buf);
	BATmode(b, statisticsMode);
#ifdef DEBUG_STATISTICS
	printf("created %s\n", buf);
#endif
	return b;
}

static void
STATcommit()
{
	BAT *b = BATnew(TYPE_void, TYPE_str, 10);
	BATseqbase(b,0);
	BUNappend(b, BBPname(STAT_id_inuse->batCacheid), FALSE);
	BUNappend(b, BBPname(STAT_id_nme->batCacheid), FALSE);
	BUNappend(b, BBPname(STAT_id_expire->batCacheid), FALSE);
	BUNappend(b, BBPname(STAT_id_stamp->batCacheid), FALSE);
	BUNappend(b, BBPname(STAT_id_count->batCacheid), FALSE);
	BUNappend(b, BBPname(STAT_id_size->batCacheid), FALSE);
	BUNappend(b, BBPname(STAT_id_min_lng->batCacheid), FALSE);
	BUNappend(b, BBPname(STAT_id_max_lng->batCacheid), FALSE);
	BUNappend(b, BBPname(STAT_id_histogram->batCacheid), FALSE);
	TMsubcommit(b);
	BBPreclaim(b);
}

static void
STATinit()
{
	if( STAT_id_inuse) 
		return;
	mal_set_lock(mal_contextLock, "statistics");
	STAT_id_inuse = STAT_create("id", "inuse", TYPE_int, TYPE_int);
	STAT_id_nme = STAT_create("id", "nme", TYPE_int, TYPE_str);
	STAT_id_expire = STAT_create("id", "expire", TYPE_int, TYPE_int);
	STAT_id_stamp = STAT_create("id", "stamp", TYPE_int, TYPE_int);
	STAT_id_count = STAT_create("id", "count", TYPE_int, TYPE_lng);
	STAT_id_size = STAT_create("id", "size", TYPE_int, TYPE_lng);
	STAT_id_min_lng = STAT_create("id", "min_lng", TYPE_int, TYPE_lng);
	STAT_id_max_lng = STAT_create("id", "max_lng", TYPE_int, TYPE_lng);
	STAT_id_histogram = STAT_create("id", "histogram", TYPE_int, TYPE_str);
	STATcommit();
	mal_unset_lock(mal_contextLock, "statistics");
}

static void
STATexit()
{
	if(STAT_id_inuse ==0) return;
	mal_set_lock(mal_contextLock, "statistics");

	BBPreclaim(STAT_id_inuse);
	BBPreclaim(STAT_id_nme);
	BBPreclaim(STAT_id_expire);
	BBPreclaim(STAT_id_stamp);
	BBPreclaim(STAT_id_count);
	BBPreclaim(STAT_id_size);
	BBPreclaim(STAT_id_min_lng);
	BBPreclaim(STAT_id_max_lng);
	BBPreclaim(STAT_id_histogram);

	STAT_id_inuse= NULL;
	STAT_id_nme= NULL;
	STAT_id_expire= NULL;
	STAT_id_stamp= NULL;
	STAT_id_count= NULL;
	STAT_id_size= NULL;
	STAT_id_min_lng= NULL;
	STAT_id_max_lng= NULL;
	STAT_id_histogram= NULL;

	TMcommit();
	mal_unset_lock(mal_contextLock, "statistics");
}

str
STATdrop(str nme)
{
	ptr p;
	int idx;

	if(STAT_id_inuse ==0) 
		throw(MAL, "statistics.drop","Statistics not initialized");
	p = BUNfnd(BATmirror(STAT_id_nme), nme);
	if (p == 0)
		throw(MAL, "statistics.drop", "BAT not enrolled");
	idx = *(int *) BUNhead(STAT_id_nme,p);
	BUNdelHead(STAT_id_nme, &idx, FALSE);
	BUNdelHead(STAT_id_expire, &idx, FALSE);
	BUNdelHead(STAT_id_stamp, &idx, FALSE);
	BUNdelHead(STAT_id_count, &idx, FALSE);
	BUNdelHead(STAT_id_size, &idx, FALSE);
	BUNdelHead(STAT_id_min_lng, &idx, FALSE);
	BUNdelHead(STAT_id_max_lng, &idx, FALSE);
	BUNdelHead(STAT_id_histogram, &idx, FALSE);
	BUNdelHead(STAT_id_inuse, &idx, FALSE);
	STATcommit();
	return MAL_SUCCEED;
}

str
STATenroll(int *ret, str *nme)
{
	return STATforceUpdate(ret, nme);
}

str
STATenrollHistogram(int *ret, str *nme)
{
	(void) ret;
	(void) nme;
	return MAL_SUCCEED;
}

@-
An update on all BATs in use can be requested.
The amount of work is somewhat limited by ensuring
that the underlying store has been changed 
@c
str
STATupdateAll(int *ret, int forced)
{
	BAT *b;
	BUN p, q;
	str name;
	int i;

	if (STAT_id_nme)
		BATloop(STAT_id_nme, p, q) {
		name = (str) BUNtail(STAT_id_nme, p);
#ifdef DEBUG_STATISTICS
		stream_printf(GDKout, "update statistics of %d\n", name);
#endif
		i = BBPindex(name);
		if (i == 0)
			continue;
		if (forced == FALSE && BUNfnd(STAT_id_inuse, &i) == 0)
			continue;
		b = BATdescriptor(i);
		if (b == 0) {
			/* BAT disappeared */
			STATdiscard2(ret, &i);
			continue;
		}
		/* check the modification time with histogram */
		/* if BBPolder(i,j) ) */  {
			STATforceUpdate(ret, &name);
		}
		}

	return MAL_SUCCEED;
}

str
STATupdate(int *ret)
{
	return STATupdateAll(ret, FALSE);
}

str
STATforceUpdateAll(int *ret)
{
	return STATupdateAll(ret, TRUE);
}

@-
Here the real work is done. This should be refined to
accomodate different base types .
@c
str
STATforceUpdate(int *ret, str *nme)
{
	BAT *b, *h;
	char buf[PATHLENGTH];
	ptr p;