getwww

来自「blog generator是一个基于Catty 2引擎的工具」· 代码 · 共 47 行

#!/bin/sh

if [ "$1" = "" ]; then
  echo "Incorrect usage." 1>&2
  exit 1
fi

touch data/visited

GOTIT=`grep -F "[$1]" data/visited`

if [ ! "$GOTIT" = "" ]; then
  echo "[-] Already got $1." 1>&2
  exit 0
fi


echo "[+] Trying to read $1..." 1>&2

echo "[$1]" >>data/visited

(sleep 40;killall -9 lynx ) &>/dev/null &

lynx -connect_timeout=30 -source "$1" >.tmpwww 2>/dev/null

killall -9 sleep &>/dev/null

if [ ! -s .tmpwww ]; then
  rm -f .tmpwww
  exit 0
fi


#cat .tmpwww >FIKU
./htmlparser <.tmpwww >.tmpwww2
#exit 1
rm -f .tmpwww

# Some minimal filtering
grep -E '{10,}[a-z0-9 ]' .tmpwww2 |grep -E '{4,}[A-z]' | sed 's/  / /g;s/  / /g;s/  / /g;s/  / /g;s/  / /g;s/  / /g;s/  / /g;s/  / /g;s/  / /g;s/  / /g;s/  / /g;s/  / /g;s/^ //g;s/ &//g' >.tmpwww
rm -f .tmpwww2

grep -E '^{2,}[a-z].*+[ ].*{2,}[a-z]$' .tmpwww >.tmpwww2
rm -f .tmpwww
cat .tmpwww2 | grep -vE ' a$| the$| from$| to$| of$| and$ | or$| mr$| at$' | awk -F"^^OINK^^"  '{print $0 " "}'
rm -f .tmpwww2