chopper.mx

一个内存数据库的源代码这是服务器端还有客户端

@' The contents of this file are subject to the MonetDB Public License
@' Version 1.1 (the "License"); you may not use this file except in
@' compliance with the License. You may obtain a copy of the License at
@' http://monetdb.cwi.nl/Legal/MonetDBLicense-1.1.html
@'
@' Software distributed under the License is distributed on an "AS IS"
@' basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
@' License for the specific language governing rights and limitations
@' under the License.
@'
@' The Original Code is the MonetDB Database System.
@'
@' The Initial Developer of the Original Code is CWI.
@' Portions created by CWI are Copyright (C) 1997-2007 CWI.
@' All Rights Reserved.

@f chopper
@v 2.0
@a M.L.Kersten
@+ BAT Iterators
Many low level algorithms rely on an iterator to break a
collection into smaller pieces. Each piece is subsequently processed
by a block.

For very large BATs it may make sense to break it into chunks
and process them separately to solve a query. An iterator pair is
provided to chop a BAT into fixed size elements.
Each chunk is made available as a BATview.
It provides read-only access to an underlying BAT. Adjusting the bounds
is cheap, once the BATview descriptor has been constructed.

The smallest granularity is a single BUN, which can be used
to realize an iterator over the individual BAT elements.
For larger sized chunks, the operators return a BATview. 

All iterators require storage space to administer the
location of the next element. The BAT iterator module uses a simple
lng variable, which also acts as a cursor for barrier statements.

The larger chunks produced are currently static, i.e.
their size is a parameter of the call. Dynamic chunk sizes
are interesting for time-series query processing. (See another module)

@mal

command bat.newIterator(b:bat[:any_1,:any_2], size:lng)
	(:lng,:bat[:any_1,:any_2])
address CHPnewChunkIterator
comment "Create an iterator with fixed granule size.
	  The result is a view.";

command bat.hasMoreElements(b:bat[:any_1,:any_2], size:lng)
	(:lng, :bat[:any_1,:any_2])
address CHPhasMoreChunks
comment "Produce the next chunk for processing.";

pattern bat.newIterator(b:bat[:any_1,:any_2]) (:lng, h:any_1, t:any_2)
address CHPbunIterator;
pattern bat.newIterator(b:bat[:oid,:any_2]) (:lng, h:oid, t:any_2)
address CHPbunIterator
comment "Process the buns one by one extracted from a void table.";

pattern bat.hasMoreElements(b:bat[:any_1,:any_2]) (:lng, h:any_1, t:any_2)
address CHPbunHasMoreElements;
pattern bat.hasMoreElements(b:bat[:oid,:any_2]) (:lng, h:oid, t:any_2)
address CHPbunHasMoreElements
comment "Produce the next bun for processing.";

@-
The head and tail values can also be extracted using the cursor.
It point to the first bun in the chunk under consideration.
It is often more effective due to use the iterator with automatic
extraction of head and tail value; the overhead involved is much less.
@mal
pattern bat.getHead(b:bat[:any_1,:any],i:lng):any_1 
address CHPgetHead
comment "return the BUN head value using the cursor.";

pattern bat.getTail(b:bat[:any_2,:any_1],i:lng):any_1 
address CHPgetTail
comment "return the BUN tail value using the cursor.";
@-
@{

@include prelude.mx
@+ BAT Iterator Implementation
@c
#include "mal_config.h"
#include "mal.h"
#include "mal_interpreter.h"

#ifdef WIN32
#ifndef LIBCHOPPER
#define chopper_export extern __declspec(dllimport)
#else
#define chopper_export extern __declspec(dllexport)
#endif
#else
#define chopper_export extern
#endif

chopper_export str CHPnewChunkIterator(lng *res, int *vid, int *bid, lng *granule);
chopper_export str CHPhasMoreChunks(lng *res, int *vid, int *bid, lng *granule);
chopper_export str CHPbunIterator(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
chopper_export str CHPbunHasMoreElements(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
chopper_export str CHPgetHead(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
chopper_export str CHPgetTail(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);

@-
We start with the large chunk iterator.
The definition of the control statements require the same
control variables, which means that the BATview is accessible
to determine how far to advance when the next chunk is retrieved.
The number of elements in the chunk is limited by the granule
size.
@c
str
CHPnewChunkIterator(lng *res, int *vid, int *bid, lng *granule)
{
	BAT *b, *view;
	size_t cnt, first;

	if ((b = BATdescriptor(*bid)) == NULL) {
		throw(MAL, "chop.newChunkIterator", "Cannot access descriptor");
	}
	cnt = BATcount(b);
	first = BUNindex(b, BUNfirst(b));
	view = VIEWcreate_(b, TRUE);

	/*  printf("set bat chunk bound to %d %d - %d\n",
	 *granule, first, MIN(cnt,(size_t) *granule)); */
	VIEWbounds(view, (size_t) first, first + MIN(cnt, (size_t) * granule) - 1);
	BATseqbase(view, first - 1);
	*vid = view->batCacheid;
	BBPkeepref(view->batCacheid); 
	BBPunfix(b->batCacheid);
	*res = first;
	return MAL_SUCCEED;
}

@-
The hasMoreElements version advances the chunk reader,
which also means that the view descriptor is already available.
The granule size may differ in each call.
@c
str
CHPhasMoreChunks(lng *res, int *vid, int *bid, lng *granule)
{
	BAT *b, *view;
	size_t i;

	if ((b = BATdescriptor(*bid)) == NULL) {
		throw(MAL, "chop.newChunkMoreElements", "Cannot access descriptor");
	}
	if ((view = BATdescriptor(*vid)) == NULL) {
		BBPunfix(b->batCacheid);
		throw(MAL, "chop.newChunkMoreElements", "Cannot access 'vid' descriptor");
	}
	i = (size_t) (*res + BATcount(view));
	if (i >= BUNindex(b, BUNlast(b))) {
		*res = -1;
		BBPunfix(b->batCacheid);
		BBPunfix(view->batCacheid);
		return MAL_SUCCEED;
	}
	/* printf("set bat chunk bound to %d - %d \n",
	   i, i+(size_t) *granule-1); */
	VIEWbounds(view, i, i + (size_t) * granule - 1);
	BATseqbase(view, i - 1);
	BBPunfix(b->batCacheid);
	BBPkeepref(view->batCacheid);
	*res = i;
	return MAL_SUCCEED;
}

@-
The BUN- and BAT-stream manipulate a long handle, i.e.
the destination variable. It assumes it has been set to
zero as part of runtime stack initialization. Subsequently,
it fetches a bun and returns the increment to the control
variable. If it returns zero the control variable has been reset
to zero and end of stream has been reached.
@c
str
CHPbunIterator(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
	BAT *b;
	lng *cursor;
	int *bid;
	lng yy;
	BUN p;
	ValPtr head, tail;
	oid o;

	cursor = (lng *) getArgReference(stk, pci, 0);
	head = &stk->stk[pci->argv[1]];
	tail = &stk->stk[pci->argv[2]];
	bid = (int *) getArgReference(stk, pci, 3);

	if ((b = BATdescriptor(*bid)) == NULL) {
		throw(MAL, "chop.newChunkMoreElements", "Cannot access 'bid' descriptor");
	}

	if (BATcount(b) == 0) {
		*cursor = -1;
		BBPunfix(b->batCacheid);
		return MAL_SUCCEED;
	}
	*cursor = BUNindex(b, BUNfirst(b));

	/* get head = ... tail = ... */
	yy = *cursor;
	p = BUNptr(b, yy);
	if( b->htype == TYPE_void){
		o= b->hseqbase;
		VALinit(head, TYPE_oid, &o);
	} else
		VALinit(head, getArgType(mb, pci, 1), BUNhead(b, p));
	VALinit(tail, getArgType(mb, pci, 2), BUNtail(b, p));
	BBPunfix(b->batCacheid);
	return MAL_SUCCEED;
}

str
CHPbunHasMoreElements(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
	BAT *b;
	lng *cursor;
	lng yy;
	bat *bid;
	BUN p;
	ValPtr head, tail;
	oid o;

	cursor = (lng *) getArgReference(stk, pci, 0);
	head = &stk->stk[pci->argv[1]];
	tail = &stk->stk[pci->argv[2]];
	bid = (bat *) getArgReference(stk, pci, 3);

	if ((b = BATdescriptor(*bid)) == NULL) {
		throw(MAL, "chop.newChunkMoreElements", "Cannot access 'bid' descriptor");
	}

	/* get head = ... tail = ... */
	yy = *cursor + 1;
	*cursor = yy;

	p = BUNptr(b, yy);
	if (p >= BUNlast(b)) {
		*cursor = -1;
		BBPunfix(b->batCacheid);
		return MAL_SUCCEED;
	}
	if( b->htype == TYPE_void){
		o= yy - BUNindex(b,BUNfirst(b))+  b->hseqbase;
		VALinit(head, TYPE_oid, &o);
	} else {
		assert(b->htype == getArgType(mb,pci,1));
		VALinit(head, getArgType(mb, pci, 1), BUNhead(b, p));
	}
	assert(b->ttype == getArgType(mb,pci,2));
	VALinit(tail, getArgType(mb, pci, 2), BUNtail(b, p));
	BBPunfix(b->batCacheid);
	return MAL_SUCCEED;
}

@-

@-
The fetch operations are all pretty straight forward, provided
you know the underlying type. Often it is cheaper to use
the extended BAT iterator, because then it can re-use the
BAT descriptor.
@c
str
CHPgetHead(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
	BAT *b;
	lng *cursor;
	int *bid;
	int limit;
	BUN p;
	ValPtr head;
	oid o;

	cursor = (lng *) getArgReference(stk, pci, 2);
	bid = (int *) getArgReference(stk, pci, 1);
	head = &stk->stk[pci->argv[0]];

	if ((b = BATdescriptor(*bid)) == NULL) {
		throw(MAL, "chop.getHead", "Cannot access 'bid' descriptor");
	}
	limit = BUNindex(b, BUNlast(b));
	if (*cursor < 0 || *cursor >= limit) {
		BBPunfix(b->batCacheid);
		throw(MAL, "mal.getHead", "range error");
	}

	/* get head = ... */
	p = BUNptr(b, *cursor);
	if( getArgType(mb,pci,3) == TYPE_void){
		o= BUNindex(b,p)+  b->hseqbase;
		VALinit(head, TYPE_oid, &o);
	} else
		VALinit(head, getArgType(mb, pci, 3), BUNhead(b, p));
	BBPunfix(b->batCacheid);
	return MAL_SUCCEED;
}

str
CHPgetTail(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
	BAT *b;
	lng *cursor;
	int *bid;
	int limit;
	BUN p;
	ValPtr tail;

	cursor = (lng *) getArgReference(stk, pci, 2);
	bid = (int *) getArgReference(stk, pci, 1);
	tail = &stk->stk[pci->argv[0]];

	if ((b = BATdescriptor(*bid)) == NULL) {
		throw(MAL, "chop.getTail", "Cannot access 'bid' descriptor");
	}
	limit = BUNindex(b, BUNlast(b));
	if (*cursor < 0 || *cursor >= limit) {
		BBPunfix(b->batCacheid);
		throw(OUTOFBNDS, "mal.getTail", "range error: %d", *cursor);
	}

	/* get tail = ... */
	p = BUNptr(b, *cursor);
	VALinit(tail, getArgType(mb, pci, 3), BUNtail(b, p));
	BBPunfix(b->batCacheid);
	return MAL_SUCCEED;
}
@-
@}