merge-batch-counts

这是一款很好用的工具包

#!/bin/sh
#
# merge-batch-counts --
#	combine batch count files into a single count file
#
# $Header: /home/srilm/devel/utils/src/RCS/merge-batch-counts,v 1.3 1997/11/16 04:01:40 stolcke Exp $
# 

if [ $# -lt 1 ]; then
	echo "usage: $0 countdir [file-list | start-iter]" >&2
	exit 2
fi

countdir=${1-./counts}
filelist=$2
iter=0

mergedir=$countdir

merger=ngram-merge

newfilefile=/tmp/newfiles$$

set -e

# make sure partially generated files are deleted
trap 'rm -f $newfile; exit 1' 1 2 15

case X$filelist in
X[0-9]*)
	# restart a previous run
	what=merge
	iter=`expr $filelist + 1`
	infiles=$mergedir/$what-iter$iter.files
	find $countdir/. -name $what-iter$filelist-\*.ngrams.gz -print | \
		sort | xargs -l2 /bin/echo > $infiles
	;;
X)
	what=merge
	infiles=$mergedir/$what-iter$iter.files
	find $countdir/. -name \*.ngrams.gz -print | sort | \
		xargs -l2 /bin/echo > $infiles
	;;
X*)
	what=`basename $filelist .files`
	infiles=$mergedir/$what-iter$iter.files
	cat $filelist > $infiles
	;;
esac

numfiles=`wc -w < $infiles`

while [ $numfiles -gt 1 ]; do
	echo "ITERATION $iter, $numfiles files" >&2
	fileno=1
	> $newfilefile
	while read file1 file2; do
		newfile=$mergedir/$what-iter$iter-$fileno.ngrams.gz
		echo $newfile >>$newfilefile

		if [ -f $newfile ]; then
			echo "retaining old $newfile" >&2
		elif [ -z "$file2" ]; then
			echo "linking $file1 to $newfile" >&2
			rm -f $newfile
			ln $file1 $newfile
		else 
			echo "merging $file1 $file2 into $newfile" >&2
			$merger -write $newfile $file1 $file2
			newfile=
		fi
		fileno=`expr $fileno + 1`
	done < $infiles

	xargs rm -f < $infiles 

	iter=`expr $iter + 1`
	infiles=$mergedir/$what-iter$iter.files
	cat $newfilefile | xargs -l2 /bin/echo > $infiles
	numfiles=`wc -w < $infiles`
done
rm -f $newfilefile

echo "final counts in `cat $infiles`" >&2