mfilter.py

这个是内存数据库中的一个管理工具

                    r'^type error:   .*$',
                # Debugging "strange" errors on Solaris x86:
                    "^!ERROR: interpret: no matching MIL operator to '\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb\xdb.*$",
                 ])+')',  re.MULTILINE)
       )

        # "normalize" differences in error messages caused by flex/bison:
        # matched groups from "norm_in" are replaced by the respective
        # element of "norm_out", unless the latter is "None" (then, the
        # respective match is kept as-is).
norm_in  = re.compile('(?:'+')|(?:'.join([
    r'^(ERROR = !| *!|)(syntax|parse|parse error: syntax)( error, )(unexpected .* on line |unexpected .* in: )?(.*)\n',				# 01: 5
    r'^(!ERROR: .*: [0-9]+ times inserted nil due to errors at tuples )([0-9]+)(\..*)\n',							# 02: 3
    r'^(ERROR =|       )( !ERROR: | !SQLException:assert:| !SQLException:sql: | !SQLException:| !MALException:)([^:]+)(: *)([^ ].*)\n',		# 03: 5
    r'^(Connection warning:.*\()(mserver_mal|mserver_sql)(\).*)\n',										# 04: 3
    r'^(ERROR: I/O warning : failed to load external entity ")(.*)(".*)\n',									# 05: 3
    r'^(ERROR: shred:mainSHREDDER: xmlCreateURLParserCtxt\(")(.*)("\) failed.*)\n',								# 06: 3
    r'^(ERROR: err:FORG0001: could not cast value)( from )?(.*)( to )(.*)(\..*)\n',								# 07: 6
    r"^(type error: can't cast type ')(.*)(' to type ')(.*)('.*)\n",										# 08: 5
    r'^(type error: no variant of function )(.*)( accepts the given argument type\(s\): )(.*)\n',						# 09: 4
    r'^(type error: illegal arguments for function )(.*)\n',											# 10: 2
    r"^(ERROR: pcre_replace: .flags. contains invalid character ')(.*)('.*)\n",									# 11: 3
    r"^(ERROR: err:XPTY0004: illegal qname ')(.*)('.*)\n",											# 12: 3
    r'^(ERROR: err:XQDY0025: attribute names are not unique )(in constructed element )?(.*)\n',							# 13: 3
    r"^(error in function application: at )(.*)(: wrong number of arguments for function `)(.*)(' \(expected )(.*)(, got )(.*)(\).*)\n",	# 14: 9
    r'^(error in function application: at )(.*)(: )(.*)( expects at least .* arguments? \(got .*\).*)\n',					# 15: 5
    r'^(ERROR: err:FOER0000)(.*)\n',														# 16: 2
    r"^(illegal reference to context node: at )(.*)(: )(``.'' is unbound|position information not available)(.*)\n",				# 17: 5
    r'^(variable\(s\) out of scope or unknown: at )(.*)\n',											# 18: 2
    r'^(bad usage of XML namespaces: at )(.*)\n',												# 19: 2
    r'^(QUERY = import rpc-module namespace xrpc = "http://www.monetdb.nl/basicfunctions" at ")(/.*)(/tests/XRpc/basic_queries/Tests/basicfunctions.xq";)\n',	# 20: 3
    r'^(.*: at )(\([0-9]+,[0-9]+\-[0-9]+,[0-9]+\))(: .*)\n',											# 90: 3
    r'^(fatal error|ERROR)(: err:FORG0005, fn:exactly-one called with a sequence containing zero or more than one item..*)()\n',		# 99: 3
])+')',  re.MULTILINE)
norm_hint = '# the original non-normalized output was: '
norm_out = (
#   None, 'syntax/parse', None, 'unexpected ... on line/in: ', None,					# 01: 5
    None, 'syntax/parse', None, '', '',									# 01: 5
    None, '...', None,											# 02: 3
    None, '', None, ': ', None,										# 03: 5
    None, 'mserver_sql', None,										# 04: 3
    None, '...', None,											# 05: 3
    None, '...', None,											# 06: 3
    None, None, '...', None, '...', None,								# 07: 6
    None, '...', None, '...', None,									# 08: 5
    None, '...', None, '...',										# 09: 4
    None, '...',											# 10: 2
    None, '...', None,											# 11: 3
    None, '...', None,											# 12: 3
    None, None, '...',											# 13: 3
    None, '...', None, '...', None, '...', None, '...', None,						# 14: 9
    None, "...: wrong number of arguments for function `...' (expected ..., got ...)", '', '', '',	# 15: 5
    None, ': ...',											# 16: 2
    None, '...', None, '...', None,									# 17: 5
    None, '...',											# 18: 2
    None, '...',											# 19: 2
    None, '$TSTSRCBASE', None,										# 20: 3
    None, '...', None,											# 90: 3
    '=ERROR', None, '\n#',										# 99: 3
##    '=ERROR', None, '''
##type error: no variant of function ... accepts the given argument type(s): ... %s
###type error: no variant of function fn:exactly-one accepts the given argument type(s): ...
##type error: maybe you meant:
##=type error:   fn:exactly-one (node*) as node %s
###type error:   fn:exactly-one (node*) as node
##type error: illegal arguments for function ... %s
###type error: illegal arguments for function fn:exactly-one
###''' % (norm_hint[:-1], norm_hint[:-1], norm_hint[:-1]),						# 99: 3
)


def mFilter (FILE, IGNORE) :
    fin = open(FILE, "rU")
    LINE = fin.readline().replace('\r','')
    while  len(LINE)  and  ( len(LINE) < 15  or  LINE[:15] not in ("stdout of test ", "stderr of test ") ):
        LINE = fin.readline().replace('\r','')
    fin.close()
    if  len(LINE) >= 15  and  LINE[:15] in ("stdout of test ", "stderr of test "):
        WHAT, TST, TSTDIR = re.search("^std(out|err) of test .(.*). in directory .(.*). itself:", LINE, re.MULTILINE).groups()
    else:
        WHAT, TST, TSTDIR = "", "", ""


    ftmp = []
    ig = n = 0
    il = iw = ic = el = ew = ec = al = aw = ac = 0
    for iline in fileinput.input(FILE):
	iline = iline.replace('\r', '')
        oline = xline = ""
        if iline == "#~BeginVariableOutput~#\n"  or  iline == "#~BeginProfilingOutput~#\n":
            ig = 1
            n = 0
        if ig  and  ( len(iline) == 0  or  iline[0] != "!"  or  iline[:9] != "ERROR = !" ):
            # ignore differences in "VariableOutput" or "ProfilingOutput"
            oline = "#~ " + iline
            n = n + 1
        elif test[0].match(iline):
            # potential differences, which we want to ignore; see above
            oline = "#~ " + iline
        elif test[4].match(iline):
            # differences due to property-checking, which we want to treat as "minor differences"; see above
##            if os.name == "nt" and iline[:5] == 'quit();':
##                oline = "#" + iline
##            else:
            oline = "=" + iline
##        elif len(ftmp) > 2              and \
##             test[3].match(iline)       and \
##             test[2].match(ftmp[-1])    and \
##             test[1].match(ftmp[-2]):
##            # differing error messages when loading non-existing modules in dynamic and static Mserver; see above
##            oline = iline
##            for i in -1, -2:
##                w = len(string.split(ftmp[i]))
##                c = wlen(ftmp[i])
##                il = il - 1
##                iw = iw - w
##                ic = ic - c
##                el = el - 1
##                ew = ew - w
##                ec = ec - c
##                ftmp[i] = "#~ " + ftmp[i]
        elif norm_in.match(iline):
            # "normalize" differences in error messages caused by flex/bison; see above
            grps_in = norm_in.match(iline).groups()
            oline = ''
            i = 0
            while i < len(grps_in):
                if grps_in[i] is not None:
                    if norm_out[i] is None:
                        oline += grps_in[i]
                    else:
                        oline += norm_out[i]
                i+=1
            oline += '\n'
            xline = norm_hint + iline
        else:
            oline = iline
        if iline == "#~EndVariableOutput~#\n":
            ig = 0
            xline = "~ " + str(n) + " ~\n"
        if iline == "#~EndProfilingOutput~#\n":
            ig = 0
        for ln in oline, xline:
            if len(ln):
                w = len(string.split(ln))
                c = wlen(ln)
                al = al + 1
                aw = aw + w
                ac = ac + c
                if ln != os.linesep  and  \
                   not re.match(IGNORE,ln, re.MULTILINE):
                    el = el + 1
                    ew = ew + w
                    ec = ec + c
                    if not re.match('^=',ln, re.MULTILINE):
                        il = il + 1
                        iw = iw + w
                        ic = ic + c
                ftmp.append(ln)

    fout = open(FILE + ".FILTERED", "w")
    if (al > 0) or (aw > 0) or (ac >0):
        fout.write("Statistics about std"+WHAT+" of test '"+TST+"` in directory '"+TSTDIR+"`:\n")
        fout.write("  %9d lines, %9d words, %9d chars   in lines not matching '^$|%s|^=`\n" % (il,iw,ic,IGNORE))
        fout.write("= %9d lines, %9d words, %9d chars   in lines not matching '^$|%s`\n"    % (el,ew,ec,IGNORE))
        fout.write("# %9d lines, %9d words, %9d chars   in all lines\n"                     % (al,aw,ac))
        fout.write("\n")
        for ln in ftmp:
            try:
                fout.write(ln)
            except IOError, (IOerrNo, IOerrStr):
                warn(THISFILE, "Writing to output file '%s' failed with #%d: '%s'." % (fout.name, IOerrNo, IOerrStr))
                if IOerrNo == 28:
                    # No space left on device
                    warn(THISFILE, "Removing input file '%s'." % FILE)
                    try:
                        os.remove(FILE)
                        fin = open(FILE,"w")
                        fin.write("%s: Removed '%s' to create space for '%s'.\n" % (THISFILE, FILE, fout.name))
                        fin.close()
                    except:
                        pass
                    try:
                        fout.write(ln)
                    except IOError, (IOerrNo, IOerrStr):
                        warn(THISFILE, "Writing to output file '%s' failed with #%d: '%s'." % (fout.name, IOerrNo, IOerrStr))
        fout.flush()
    fout.close()
    #if os.name == "nt":
        #time.sleep(1)
### mFilter (FILE, IGNORE) #

#############################################################################
#       MAIN

def main(argv) :
    THISFILE = os.path.basename(argv[0])
    try:
        opts, args = getopt.getopt(argv[1:], "?hI:", ["help"])
    except getopt.GetoptError:
        Usage(THISFILE)
        sys.exit(1)

    IGNORE = "^#"
    for o, a in opts:
        if o in ("-?", "-h", "--help"):
            Usage(THISFILE)
            sys.exit(0)
        if o == "-I":
            IGNORE = a

    for f in args:
        if os.path.isfile(f):
            mFilter(f, IGNORE)
        else:
            warn(THISFILE, "file missing: " + f)
### main(argv) #

if __name__ == "__main__":
    main(sys.argv)

#       END
#############################################################################