bbscatcher.java

用java实现的一个bbs的portal

/**
 * Created by IntelliJ IDEA.
 * User: Administrator
 * Date: Sep 19, 2003
 * Time: 8:59:29 PM
 * To change this template use Options | File Templates.
 */
package Kernel;

import BBSSpider.BBSSpider;

import java.util.Date;
import java.util.Properties;
import java.util.Enumeration;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.text.SimpleDateFormat;
import java.text.ParseException;

import Classification.ArticleBatch;

public class BBSCatcher {
    static Properties BBSSpiderProps = new Properties();
    static SimpleDateFormat dateFormat =  new SimpleDateFormat("yyyy-MM-dd HH-mm-ss");

    public synchronized static Date loadBBSDate(String bbsname,String boardname){
        String propertiesFileName=ArticleBatch.PropertiesPath+bbsname+".spider.properties";
        FileInputStream is = null;
        try {
            is = new FileInputStream(propertiesFileName);
        } catch (FileNotFoundException e) {
            e.printStackTrace();  //To change body of catch statement use Options | File Templates.
        }
        try {
            BBSSpiderProps.load(is);
        }
        catch (Exception e) {
            System.err.println(e.toString());
            return null;
        }
        return String2Date(BBSSpiderProps.getProperty(bbsname+"_"+boardname));
    }

    public synchronized static boolean storeBBSDate(String bbsname,String boardname,Date newDate){
        String propertiesFileName=ArticleBatch.PropertiesPath+bbsname+".spider.properties";
        String dateStr = dateFormat.format(newDate);
        BBSSpiderProps.setProperty(bbsname+"_"+boardname,dateStr);
        try {
            FileWriter os  = new FileWriter(propertiesFileName);
            for (Enumeration e = BBSSpiderProps.keys(); e.hasMoreElements();) {
                String keys = (String)e.nextElement();
                dateStr = BBSSpiderProps.getProperty(keys);
                os.write(keys+":"+dateStr+"\n");
            }
            os.close();
            return true;
        } catch (Exception e) {
            e.printStackTrace();  //To change body of catch statement use Options | File Templates.
        }
        return false;
    }

    private static Date String2Date(String dateStr){
        try {
            return dateFormat.parse(dateStr);
        } catch (ParseException e) {
            System.out.println("can't convert the date String:"+dateStr+" to Date");
        }
        return null;
    }

    public void doCatch(String bbsname,String boardname,int threadnum){
        comTool.initLog();
        while(true)
        {
            Date tempArticleDate = null;
            tempArticleDate = loadBBSDate(bbsname,boardname);
            System.out.println("load tempArticleDate="+tempArticleDate);
            if (tempArticleDate != null){
            comTool.SpiderLog.info("Retrieve from "+boardname+" of "+bbsname+" Begin !");
            tempArticleDate = BBSSpider.doGet(tempArticleDate,bbsname,boardname,threadnum);
            if (tempArticleDate != null){
                comTool.SpiderLog.info("Retrieve from "+boardname+" of "+bbsname+" End ! "+tempArticleDate);
                storeBBSDate(bbsname,boardname,tempArticleDate);
                 //System.out.println("StoreBBSDate Successful!");
                 //System.out.println("store tempArticleDate="+tempArticleDate);
            }else{
                comTool.SpiderLog.error("Retrieve from "+boardname+" of "+bbsname+" End And Can't get return date !");
            }
            try{
            Thread.sleep(60000); //60 sec

            }catch(Exception ex){
            ex.printStackTrace();
        }
        }
            else{
            comTool.SpiderLog.error("Get the Date of "+bbsname+" "+boardname+" from file error !");
        }
        }
    }

    public static void main(String[] args) {
        System.out.println(BBSCatcher.storeBBSDate("bbs.pku.edu.cn","secondhand",new Date()));
        System.out.println(BBSCatcher.storeBBSDate("bbs.pku.edu.cn","house",new Date()));
    }
}