mfilter.py

这个是内存数据库中的一个管理工具

#!/usr/bin/python

# The contents of this file are subject to the MonetDB Public License
# Version 1.1 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
# http://monetdb.cwi.nl/Legal/MonetDBLicense-1.1.html
#
# Software distributed under the License is distributed on an "AS IS"
# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
# License for the specific language governing rights and limitations
# under the License.
#
# The Original Code is the MonetDB Database System.
#
# The Initial Developer of the Original Code is CWI.
# Portions created by CWI are Copyright (C) 1997-2007 CWI.
# All Rights Reserved.

import sys
import os
import getopt
import re
import fileinput
import string
import time

#############################################################################
#       FUNCTIONS

def Usage(THISFILE) :
    print """

Usage:  %s [-I<exp>] <files>

-I<exp> : ignore lines matching <exp> during first count (optional, default: -I'^#`)
<files> : list of files to be processed

""" % THISFILE
### Usage(THISFILE) #

def warn(THISFILE,TEXT) :
    sys.stderr.write("%s warning: %s\n" % (THISFILE,TEXT))
### warn(THISFILE,TEXT) #

def wlen(str) :
    return len(string.join(string.split(str)))
### wlen(str) #

test = (
        # potential differences, which we want to ignore
        re.compile('(?:'+')|(?:'.join([
## sql5
                      r"^printf\(\"\\nReady.\\n\"\);",
                      r"^quit\(\);",
##                    r"^module\(mprof\);$",
##                    r'^pm(?:[BE]\("(?:global_.+|[a-zA-Z0-9_]+[0-9])"\)|Summary\(\));$',
##                    r'^printf\("#~BeginProfilingOutput~#\\n"\);$',
##                    r"^Mdestroydb: processing database ..*. in ",
##                    r"^Mcreatedb: processing database ..*. in ",
##                    r"^Mcreatedb: using default dbdir ",
##                    r"^Mcreatedb: calling .Mserver .*-db .* for the first time..$",
##                    r"^Mcreatedb: database .* has been initialized succesfully!$",
                      r"^[Uu]sage: (?:/.*/)?(?:lt-)?(?:Mserver|MapiClient|msqldump)(?: \[ options \])?(?: \[ script\+ \])? *$",
                      r"^!WARNING: GDKlockHome: created directory .*$",
                      r"^!WARNING: GDKlockHome: ignoring empty or invalid .gdk_lock.$",
                      r"^!WARNING: BBPdir: initializing BBP.$",
##                    r"^connection= .*:[0-9]*$",
##                    r"^Connecting to server: localhost:[0-9]*$",
##                    r"^server listening on port [0-9]*$",
##                    r"^SQL  connected to database .* using schema .*$",
                      r"^MAPI  = .*@.*:[0-9]*$",
                      r'^QUERY = execute at { "localhost:[0-9]*" }$',
                    r"^Pathfinder (?:XQuery Compiler|XML Shredder) \(.Revision: .* .\)$",
##                    r"^SQL  .* affected$",
##                    r": Symbol .(?:exit|_setjmp). causes overflow in R_X86_64_PC32 relocation$",
##                    r"^\+\=+\+$",
##                    r"^\+\-+\+$",
                # Tokens depend on flex/bison
                    r'^\[ "TOK_[A-Z]+",\t* +"int",\t* +"frozen",\t* +"constant",\t* +"[0-9]+"\t* +\]',
                # ant output we want to ignore 
                    r'^Buildfile:.*$',
                    r'^Building jar: .*$',
                    r'^Total time: [0-9]+ seconds?$',
                # remove wine errors:
		    r"^fixme:.*$",
		# ignore big- vs. little-endian byte-orders in test tests/BugDay_2005-10-06_4.8/Tests/print_CTgroup.SF-1315791.*:
		    r"^\[ .?\\000\\000\\000.?\\000\\000\\000.? \]$",
                # ignore compile date/time in helps messages:
                    r"^COMPILED:  by adm on .*$",
                # `fuser` (in MkillUsers) triggers these errors on Gentoo with "hardened" kernel
                    r'^! Cannot open /proc/net/unix: Permission denied$',
                    r'^! Cannot stat file /proc/[0-9]+/fd/[0-9]+: (?:No such file or directory|Permission denied)$',
                # Pathfinder: extra error messages that occur only with assertions enabled
                    r'^[ \t]*!# halted in .*, line [0-9]+$',
                # suspicious output/messages/errors? most probably triggered by MkillUsers, but apparently not causing any harm:
                    r'^! /.*/bin/[Mm](?:(?:server5?|apiClient|timeout|diff):e+|log:)$',
                 ])+')',  re.MULTILINE),
        # differing error messages when loading non-existing modules in dynamic and static Mserver
        re.compile('(?:'+')|(?:'.join([
                    r"^!ERROR: DL_open: Unknown error.$",
                    r"^!ERROR: DL_open: library not found \(STATIC\).$",
                 ])+')',  re.MULTILINE),
        re.compile('(?:'+')|(?:'.join([
                    r"^!ERROR: TBL_loadmod: module load .* cancelled.$",
                 ])+')',  re.MULTILINE),
        re.compile('(?:'+')|(?:'.join([
                    r"^!ERROR: module\(.*\): load error.$",
                 ])+')',  re.MULTILINE),
        # differences due to property-checking, which we want to treat as "minor differences"
        re.compile('(?:'+')|(?:'.join([
##                    r'^synced\(x,[yz]\) := (?:true|false)$',
##                #dir(): # name                 htype          ttype          count       heat        dirty                      status             kind               refcnt      lrefcnt     #
##                    r'^\[ "[A-Za-z0-9_]+",\t* +"[a-z_]+",\t* +"[a-z_]+",\t* +[0-9]+,\t* +[0-9]+,\t* +"(?:dirty|clean|diffs)",\t* +"(?:load|disk)",\t* +"(?:tran|pers)",\t* +[0-9]+,\t* +[0-9]+\t* +\]$',
##                    r'^\[ ".+",\t* +0,\t* +"(?:load|disk)",\t* +"(?:tran|pers)",\t* +0\t* +\]$',
##                # tests/from_Twente/Bugs/Arjen_01:
##                    r'^\[ "(?:(?:bat|cur)Stamp|[ht]align(?:|flushed))",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+"\t* +\]$',
##                    r'^\[ "batId",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+"\t* +\]$',
##                    r'^\[ "lastUsed",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+"\t* +\]$',
##                    r'^\[ "bat(?:Cacheid|Parentid)",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+"\t* +\]$',
##                    r'^\[ "bat(?:Share|Ref)cnt",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]",\t* +"[0-3]"\t* +\]$',
##                    r'^\[ "batDirtydesc",\t* +"dirty",\t* +"dirty",\t* +"dirty",\t* +"dirty",\t* +"dirty",\t* +"dirty",\t* +"(?:dirty|clean)",\t* +"dirty",\t* +"dirty",\t* +"dirty",\t* +"dirty"\t* +\]$',
##                    r'^\[ "batBuns\.(?:free|(?:max)?size)",\t* +"(?:20|212|40|424)",\t* +"(?:20|212)",\t* +"(?:8|56)",\t* +"(?:4|16|32|212|424)",\t* +"(?:8|56)",\t* +"(?:16|212|32|424)",\t* +"(?:3|24|48|424|848)",\t* +"(?:3|24|48|424|848)",\t* +"(?:12|212|24|424)",\t* +"(?:3|24|48|424|848)",\t* +"(?:12|212)"\t* +\]$',
##                    r'^\[ "head",\t* +"void",\t* +"void",\t* +"void",\t* +"v?oid",\t* +"void",\t* +"oid",\t* +"v?oid",\t* +"v?oid",\t* +"oid",\t* +"void",\t* +"void"\t* +\]$',
##                    r'^\[ "hdense",\t* +"1",\t* +"1",\t* +"1",\t* +"[01]",\t* +"1",\t* +"[01]",\t* +"[01]",\t* +"[01]",\t* +"[01]",\t* +"1",\t* +"1"\t* +\]$',
##                    r'^\[ "hvarsized",\t* +"1",\t* +"1",\t* +"1",\t* +"[01]",\t* +"1",\t* +"0",\t* +"[01]",\t* +"[01]",\t* +"0",\t* +"1",\t* +"1"\t* +\]$',
##                    r'^\[ "hloc",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"[04]",\t* +"0"\t* +\]$',
##                    r'^\[ "hsorted",\t* +"65",\t* +"65",\t* +"65",\t* +"65",\t* +"65",\t* +"65",\t* +"65",\t* +"65",\t* +"(?:1|65)",\t* +"65",\t* +"65"\t* +\]$',
##                    r'^\[ "hseqbase",\t* +"0@0",\t* +"0@0",\t* +"0@0",\t* +"(?:1@0|nil)",\t* +"0@0",\t* +"(?:0@0|nil)",\t* +"(?:1@0|nil)",\t* +"(?:1@0|nil)",\t* +"(?:1@0|nil)",\t* +"0@0",\t* +"0@0"\t* +\]$',
##                    r'^\[ "tail",\t* +"oid",\t* +"int",\t* +"bit",\t* +"void",\t* +"bit",\t* +"void",\t* +"v?oid",\t* +"void",\t* +"void",\t* +"v?oid",\t* +"int"\t* +\]$',
##                    r'^\[ "tdense",\t* +"1",\t* +"0",\t* +"0",\t* +"1",\t* +"0",\t* +"1",\t* +"[01]",\t* +"1",\t* +"1",\t* +"[01]",\t* +"0"\t* +\]$',
##                    r'^\[ "tvarsized",\t* +"0",\t* +"0",\t* +"0",\t* +"1",\t* +"0",\t* +"1",\t* +"[01]",\t* +"1",\t* +"1",\t* +"[01]",\t* +"0"\t* +\]$',
##                    r'^\[ "tloc",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"[04]",\t* +"[04]",\t* +"0",\t* +"0",\t* +"0"\t* +\]$',
##                    r'^\[ "tsorted",\t* +"(?:1|65)",\t* +"(?:1|65)",\t* +"(?:0|65)",\t* +"65",\t* +"0",\t* +"65",\t* +"65",\t* +"65",\t* +"65",\t* +"65",\t* +"(?:1|65)"\t* +\]$',
##                    r'^\[ "tseqbase",\t* +"0@0",\t* +"nil",\t* +"nil",\t* +"0@0",\t* +"nil",\t* +"0@0",\t* +"(?:0@0|nil)",\t* +"0@0",\t* +"0@0",\t* +"(?:1@0|nil)",\t* +"nil"\t* +\]$',
##                    r'^\[ "batCopiedtodisk",\t* +"[01]",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0",\t* +"0"\t* +\]$',
##                # tests/from_Twente/Bugs/Arjen_02:
##                    r'^\[ "(?:(?:bat|cur)Stamp|[ht]align(?:|flushed))",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+"\t* +\]$',
##                    r'^\[ "batId",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+",\t* +"tmp_[0-9]+"\t* +\]$',
##                    r'^\[ "lastUsed",\t* +"[0-9]+",\t* +"[0-9]+",\t* +"[0-9]+"\t* +\]$',
##                    r'^\[ "bat(?:Cacheid|Parentid)",\t* +"[0-9\-]+",\t* +"[0-9\-]+",\t* +"[0-9\-]+"\t* +\]$',
##                    r'^\[ "bat(?:Share|Ref)cnt",\t* +"[0-2]",\t* +"[0-2]",\t* +"[0-2]"\t* +\]$',
##                    r'^\[ "batBuns\.(?:free|(?:max)?size)",\t* +"(?:16|212)",\t* +"(?:7|56)",\t* +"(?:2|8|212|16|424)"\t* +\]$',
##                    r'^\[ "head",\t* +"void",\t* +"void",\t* +"v?oid"\t* +\]$',
##                    r'^\[ "h(?:dense|varsized)",\t* +"1",\t* +"1",\t* +"[01]"\t* +\]$',
##                    r'^\[ "hseqbase",\t* +"0@0",\t* +"0@0",\t* +"(?:2@0|nil)"\t* +\]$',
##                    r'^\[ "tsorted",\t* +"(?:1|65)",\t* +"(?:0|65)",\t* +"65"\t* +\]$',
##                    r'^\[ "batCopiedtodisk",\t* +"[01]",\t* +"0",\t* +"0"\t* +\]$',
##                # tests/from_Twente/moa/test:
##                    r'^\[ [01]@0,\t* +[10]@0\t* +\]$',
##                    r'^\[ (?:[01]@0,\t* +)?(?:40[89]@0|[89]000)\t* +\]$',
##                    r'^\[ (?:[24]@0,\t* +)?(?:40[57]@0|[57]000)\t* +\]$',
##                    r'^\[ (?:[58]@0,\t* +)?(?:40[14]@0|[14]000)\t* +\]$',
##                    r'^\[ (?:[67]@0,\t* +)?(?:40[23]@0|[23]000)\t* +\]$',
##                    r'^\[ 3@0,\t* +40[1234]@0\t* +\]$',
##                    r'^\[ 2@0,\t* +406@0\t* +\]$',
##                    r'^_MMjoin_18\.find\(7@0\)\.print;$',
##                # src/modules/plain/salgebra (on WindowsNT):
##                    r'^\[ "batId",\t* +"(?:28|29|30|31)"\t* +\]$',
##                    r'^\[ (?:28|29|30|31),\t* +"ti",\t* +2,\t* +ti\t* +\]$',
##                # src/modules/plain/mmath[2], tests/BugReports/Tests/no.125 (Linux.GNU.32):
##                    r'^\[ \-?1\.2246(?:467991473532|063538223773)e\-0?16 \]$',
##                    r'^\[ 6\.123(?:233995736766|0317691118863)e\-0?17 \]$',
##                    r'^\[ -1\.8369(?:701987210297|095307335659)e\-0?16 \]$',
##                    r'^\[ 1\.6331[28]e\+0?16 \]$',
##                    r'^\[ 5\.443(?:75|93)e\+0?15 \]$',
##                    r'^\[ 2\.302585092994045[59] \]$',
##                    r'^\[ 0\.707106781186547(?:46|57) \]$',
##                    r'^\[ 1\.047197551196597[69] \]$',
##                    r'^\[ 16331(?:778728383844|239353195370) \]$',
##                    r'^\[ 5443(?:926242794615|746451065123) \]$',
##                    r'^\[ 0\.1000000014901161[12] \]$',
##                    r'^\[ (?:0\.99999999999999989|1) \]$',
##                    r'^\[ \-?0\.523598775598298(?:93|82) \]$',
##                    r'^\[ 2\.094395102393195[73] \]$',
                # src/modules/plain/aggr[X3]:
                    r'^\[ .*(?:,\t* ? "?2\.562203[64]3e\+13"?)+\t* ? \]$',
                    r'^\[ .*(?:,\t* ? "?7\.65432(?:119|072)"?)+\t* ? \]$',
                    r'^\[ .*(?:,\t* ? "?7\.6543209876543(?:2(?:[13]2?|21)|195)"?)+\t* ? \]$',
                    r'^\[ .*(?:,\t* ? "?6\.6666666666666(?:572|67)"?)+\t* ? \]$',
                    r'^\[ .*(?:,\t* ? "?6\.6666666666666(?:288|67)"?)+\t* ? \]$',
                    r'^\[ .*(?:,\t* ? "?6\.6666666666666(?:128|67)"?)+\t* ? \]$',
                    r'^\[ .*(?:,\t* ? "?158\.765432098765(?:22|36?|44|59|64)"?)+\t* ? \]$',
##                # src/modules/plain/decimal:
##                    r'^\[ "[dn]ec_tiny",\t* +1122E\-30\t* +\]$',
##                    r'^\[ "[dn]ec_small",\t* +2244\t* +\]$',
##                    r'^\[ "[dn]ec_num",\t* +4488E15\t* +\]$',
##                    r'^\[ "[dn]ec_big",\t* +1122E40\t* +\]$',
##                    r'^\[ "[dn]ec_huge",\t* +2244E9010\t* +\]$',
##                # src/modules/plain/color:
##                    r'^\[ 0\.498039(?:18|21)6 \]$',
##                # src/monet/monet_mod_cnt-copy_C:
##                    r'^\[ "(?:mapi_params|tmp_62)",\t* *[10],\t* *[10],\t* *0\t* *\]$',
                # src/gdk/cast:
                    r'^\[ (?:\-|)0 \]$',
                # src/monet/modrefcnt1 & tests/BugDay_2005-11-09_4.9.3/double_load_double_drop.SF-1082293.milM (on Itanium)
                    r"^!ERROR: DESCload: atom 'lock' unknown, in BAT '[0-9]+'.$",
                # pathfinder/tests/XQuery/fn_aps:
                    r'^0.900000 0.500000 0.400000 (?:\-|)0.000000 0.400000 0.500000 0.900000 1 *$',
                # pathfinder/tests/XPathMark/Tests/Q1[023]:
                    r'^(?:</item>|)<item (?:id="item9"|featured="yes") (?:id="item9"|featured="yes")>$',
##                # src/mapi/monet_BATs (on Linux2.4.26.ntv_32-d):
##                    r'^\[ "monet_mod_use",\t* *(?:44|66)\t* *\]$',
##                # Intel compiler seems to do some (re-)compilation for src/mapi/examples/C/
##                    r'^IPO: using IR for .*$',
##                    r'^IPO: performing single-file optimizations$',
##                # src/tools/{client,multiClientSeq}
##                    r'^ERROR = (?:Connection terminated|Lost connection with server)$',
##                    r'^ERROR = (?:communication line broken|Connection lost)$',
                # sql/src/benchmarks/arno_flt/Tests/init:
                    r'^.*\t2.7445021260064(?:099|1)e\-06\t',
                # Broken sql error messages:
                    r'^QUERY = (?:\-\-|select|insert|INSERT) ',
##                # pathfinder: loader/usage (without BerkeleyDB):
##                    r'^     -c: compress node properties \(40% encoding speed\)$',
                # Mtest warnings:
                    r"^! Socket-Check failed for (?:SQL|MAPI)server on <[a-zA-Z0-9]*:[0-9]+> with #[0-9]+; '.*' !$",
##                    r'^! Waiting [0-9]+ seconds \([0-9]/[0-9]\) \.\.\. (?:Done\.)?$',
                    r'^! Using new (?:SQL|MAPI)PORT=[0-9]+ !$',
                    r'^! (?:SQL|MAPI)PORT was not properly released by Mserver !$',
                # ld warnings on IRIX64:
                    r"^ld64: WARNING 84 : .*/common/.libs/libmutils.so is not used for resolving any symbol.$",
                # (extra) error messages that used to occur with M4/sql but don't any more with M5/sql:
                    r'^.*!ERROR: .*operation failed.$',
                # suggested alternatives in case of XQuery type errors: