t.nonascii.replace

一个C语言写的快速贝叶斯垃圾邮件过滤工具

#! /bin/sh

. ${srcdir=.}/t.frame

# create a test wordlist with tokens containing non-ascii characters,
# replace the non-ascii characters to question marks,
# check the number of tokens in the wordlist
#
# test below
# remember to use ${srcdir}

# check for non-default configuration
if [ -n "$USE_UNICODE" -a "$USE_UNICODE" != "yes" ] ; then
    exit 77
fi

if [ -z "$USE_UNICODE" ] ; then
    UNICODE_NO="--unicode=no"
fi

echo  	41 A4 BA B5 B5 20 31     20  30 20  32 30 30 33 30 33 30 33 0A \
	41 C1 BA B8 B5 20 32     20  30 20  32 30 30 32 31 32 30 32 0A \
	41 BA C1 B8 B5 20 33     20  30 20  32 30 30 33 30 33 30 31 0A \
  	42 A4 BA B8 B5 B5 20 31  20  30 20  32 30 30 33 30 33 30 33 0A \
	42 C1 BA B8 B5 B5 20 32  20  30 20  32 30 30 32 31 32 30 32 0A \
	42 BA C1 B8 B5 B5 20 33  20  30 20  32 30 30 33 30 33 30 31 0A \
	42 C1 BA B5 B8 B5 20 34  20  30 20  32 30 30 33 30 33 30 34 0A \
| ./dehex >${TMPDIR}/input

$BOGOUTIL -C -l ${WORDLIST} < ${TMPDIR}/input
$BOGOUTIL -C -d ${WORDLIST} | egrep -v "^(\.MSG_COUNT|\.ENCODING)" > ${TMPDIR}/output.1
$BOGOUTIL -C -n -m ${WORDLIST} $UNICODE_NO
$BOGOUTIL -C -d ${WORDLIST} | egrep -v "^(\.MSG_COUNT|\.ENCODING)" > ${TMPDIR}/output.2

LEN1=`wc -l ${TMPDIR}/output.1 | $AWK '{print $1}'`
LEN2=`wc -l ${TMPDIR}/output.2 | $AWK '{print $1}'`

TOKDAT1=`head -1 ${TMPDIR}/output.2 | $AWK '{print $2 "." $4 }'`
TOKDAT2=`tail -1 ${TMPDIR}/output.2 | $AWK '{print $2 "." $4 }'`

RESULT=`printf "%d.%d %s %s" $LEN1 $LEN2 $TOKDAT1 $TOKDAT2`

WANT="7.2 6.20030303 10.20030304"

if [ "$RESULT" != "$WANT" ] || [ $verbose -gt 0 ] ; then
    echo want: \"$WANT\", have: \"$RESULT\"
fi

test "$RESULT" = "$WANT"