make-batch-counts

这是一款很好用的工具包

#!/bin/sh
#
# make-batch-counts --
#	generate n-gram counts in batches
#
# A list of data files is partitioned into batches, results from each of
# which are deposited in a separate ngram-count file.
#
# usage: make-batch-count file-list [batch-size [filter \
#		[countdir [options]]]]
#
# file-list	is a file containing a list of data files
#		(lines starting with # are ignored)
# batch-size	is the number of input files per batch
# filter	is preprocessor filter to condition the data 
# countdir	is the directory where count files are deposited
# options	are arguments passed on to ngram-count
#
# $Header: /home/srilm/devel/utils/src/RCS/make-batch-counts,v 1.3 2004/03/01 05:45:57 stolcke Exp $
#

if [ $# -lt 1 ]; then
	echo "usage: $0 file-list [batch-size [filter [countdir [options]]]]" >&2
	exit 2
fi

filelist=$1
batchsize=${2-10}
filter=${3-/bin/cat}
countdir=${4-./counts}

case $# in
1)	shift;;
2)	shift; shift;;
3)	shift; shift; shift;;
4)	shift; shift; shift; shift;;
esac

options="$@"

what=`basename $filelist .files`
statsfile=$countdir/$what.stats
infiles=$countdir/$what.files

set -e

if [ ! -d $countdir ]; then
	mkdir $countdir
fi

grep -v '^#' $filelist | xargs -l$batchsize echo > $infiles

trap 'rm -f $newfile; exit 1' 1 2 15

fileno=1
> $statsfile
while read datafiles; do
	newfile=$countdir/$what-$fileno.ngrams.gz

	echo "counting in $newfile sources $datafiles" >&2
	
	$filter $datafiles | \
	ngram-count -text - \
		-tag $newfile \
		-sort \
		-write-order 0 \
		-write $newfile \
		$options \
		2>> $statsfile

	fileno=`expr $fileno + 1`
done < $infiles