dbmanager.java

用来为垂直搜索引擎抓取数据的采集系统

/*
 * *****************************************************
 * Copyright (c) 2005 IIM Lab. All  Rights Reserved.
 * Created by xuehao at 2005-10-12
 * Contact: [e-mail]
 * *****************************************************
 */
package org.indigo.db;


import java.sql.Date;

import org.indigo.db.fieldchecker.UrlCheck;
import org.indigo.db.fieldfilter.*;
import org.indigo.filters.FiltEnter;
import org.indigo.filters.FiltFirstColon;
import org.indigo.filters.FiltFrontBack;
import org.indigo.filters.FiltGtLt;
import org.indigo.filters.FiltSpace;
import org.indigo.filters.FiltUntil;
import org.indigo.filters.Filter;
import org.indigo.log.FileLoggerError;
import org.indigo.util.DBConfig;
/**
 * 此类主要完成把采集到的已经封装在对象中的数据
 * 利用DBConnection或者DBOracleConnection提供的对数据库操作的方法
 * 完成数据进库工作。
 * @author wbz
 *
 */
public class DBManager
{
    private static DBManager itsDB = null;
    private static Drop2DBGate itsGate = null;
    private static boolean oracleFlag = false;
    private static final String PRICE_TABLE = "agriprice_caiji";
    private static final String SUPDEM_TABLE = "gqxx_caiji";
    private static final String NEWS_TABLE = "news_info_caiji";
    private static final String TECH_TABLE = "bus_info_caiji";
    private static final String MARKET_TABLE = "message_info_caiji";
    private static final String JOB_TABLE="Job_caiji";
    private static final String QUESTION_TABLE="question_caiji";
    private static final String COMPANY_TABLE="CORPUSER_INFO";
    private static int CONTENT_SIZE=5500;

    private Filter [] filters = null;
    /**
     * 构造函数，完成过滤类的实例化。
     *
     */
    private DBManager()
    {

        filters = new Filter[6];
        filters[0] = new FiltUntil( ">" );
        filters[1] = new FiltFrontBack();
        filters[2] = new FiltSpace();
        filters[3] = new FiltEnter();
        filters[4] = new FiltFirstColon();
        filters[5] = new FiltGtLt();
        
        String str=null;
        str = DBConfig.getInstance().getProperty( "db" ).trim();
        oracleFlag = str.equalsIgnoreCase( "oracle" );
    }
    public static DBManager getInstance()
    {
        if( itsDB==null )
            itsDB = new DBManager();
        return itsDB;
    }
    public Drop2DBGate getImpl()
    {
        if (itsGate == null)
        {
//            itsGate = new FakeDBGate();
            itsGate = new RealDBGate();
        }
        return itsGate;

    }
    private String filt( String str )
    {
      
        int tmpsz,j;
        tmpsz = filters.length;
        for( j=0; j<tmpsz; j++ )
            str = filters[j].filt( str );
        return str;
        
    }
    /**
     * 完成新闻的插入工作。
     * @param nInfo
     */
    public void insertNewsInfo2DB( NewsInfo nInfo )
    {
    	/**
    	 * 从对象中取出数据，
    	 */
        String title=null,content=null,url=null;
        int typeid=-1;
        title = nInfo.getTitle();
        content = nInfo.getContent();
        typeid = nInfo.getTypeId();
        url=nInfo.getItsUrl();
        Date newsTime = nInfo.getNewsTime();
        
        content = NewsContentFilter.getInstance().filter( content );

        title = NewsStringFilter.getInstance().filter( title );
        content = NewsStringFilter.getInstance().filter( content );
        
        title = title.replace( '\'', '"' );
        title = filt( title );
        content = content.replace( '\'', '"' );
        
        String sql=null;
        if( !oracleFlag )
        {
	        sql  = "insert into " + NEWS_TABLE + "(";
	        sql += "news_title, news_content, newstype_id, news_time )";
	        sql += "values( '" + title + "', ";
	        sql += "'" + content + "', ";
	        sql += typeid + ", ";
	        sql += "'" + newsTime + "')";
//	        System.out.println( sql );
	        DBConnection.getInstance().executeInsert( sql );
        }else
        {
	        sql  = "insert into " + NEWS_TABLE + "(";
	        sql += "news_title, news_content, newstype_id, news_time,HTMURL )";
	        sql += "values( '" + title + "', ";
	        sql += "?, ";
	        sql += typeid + ", ";
	        sql += "to_date('"+newsTime+"','yyyy-mm-dd'),'"+url+"')";
//	        System.out.println( sql );
	        DBOracleConnection.getInstance().executeInsert( sql, content );
	        
        }
    }
    /**
     * 完成市场信息的插入工作，
     * @param nInfo
     */
    public void insertMarketNewsInfo2DB( NewsInfo nInfo )
    {
        String title=null,content=null,itsUrl=null;
        int typeid=-1;
        title = nInfo.getTitle();
        content = nInfo.getContent();
        typeid = nInfo.getTypeId();
        Date newsTime = nInfo.getNewsTime();
        itsUrl=nInfo.getItsUrl();
        content = NewsContentFilter.getInstance().filter( content );

        title = NewsStringFilter.getInstance().filter( title );
        title = filt( title );
        content = NewsStringFilter.getInstance().filter( content );
        
        if( title==null || content==null  )
        	return ;
        title = title.trim();
        content = content.trim();
        if( title.equals("") || content.equals("") )
        	return ;
        
        title = title.replace( '\'', '"' );
        title = filt( title );
        content = content.replace( '\'', '"' );
        
        String sql=null;
        
        if( !oracleFlag )
        {
            sql  = "insert into " + MARKET_TABLE + "(";
	        sql += "message_title, message_content, messagetype_id, message_time )";
	        sql += "values( '" + title + "', ";
	        sql += "'" + content + "', ";
	        sql += typeid + ", ";
	        sql += "'" + newsTime + "')";
	//        System.out.println( sql );
	        DBConnection.getInstance().executeInsert( sql );
        }else
        {
            sql  = "insert into " + MARKET_TABLE + "(";
	        sql += "message_title, message_content, messagetype_id, message_time,page_url )";
	        sql += "values( '" + title + "', ";
	        sql += "?, ";
	        sql += typeid + ", ";
	        sql += "to_date('"+newsTime+"','yyyy-mm-dd'),'"+itsUrl+"')";
	        
	        DBOracleConnection.getInstance().executeInsert( sql, content );
        }
        
    }
    /**
     * 完成技术的插入工作。
     * @param itsTechInfo
     */
    public void insertTechInfo2DB(TechInfo itsTechInfo)
    {
        String title=null,content=null,typeid=null,publisher=null,itsUrl=null;
        int auditing=1;
        Date techTime = itsTechInfo.getTechTime();
        
        title = itsTechInfo.getTitle();
        content = itsTechInfo.getContent();
        typeid = itsTechInfo.getTypeId();
        publisher = itsTechInfo.getPublisher();
        auditing = itsTechInfo.getAuditing();
        itsUrl=itsTechInfo.getItsUrl();
        content = NewsContentFilter.getInstance().filter( content );
        
        title = NewsStringFilter.getInstance().filter( title );
        title = filt( title );
        content = NewsStringFilter.getInstance().filter( content );
        
        if( title==null || content==null  )
        	return ;
        title = title.trim();
        content = content.trim();
        if( title.equals("") || content.equals("") )
        	return ;
        
        title = title.replace( '\'', '"' );
        title = filt( title );
        content = content.replace( '\'', '"' );
        
        String sql=null;
        
        if( !oracleFlag )
        {
	        sql  = "insert into " + TECH_TABLE + "(";
	        sql += "bus_title, bus_content, busbigtype_id, bus_time1, bus_auditing, bus_publisher ) ";
	        sql += "values( '" + title + "', ";
	        sql += "'" + content + "', ";
	        sql += "'" + typeid + "', ";
	        sql += "'" + techTime + "', ";
	        sql += auditing + ", ";
	        sql += "'" + publisher + "' )";
	//        System.out.println( sql );
	        DBConnection.getInstance().executeInsert( sql );
        }else
        {
	        sql  = "insert into " + TECH_TABLE + "(";
	        sql += "bus_title, bus_content, busbigtype_id, bus_time1, bus_auditing, bus_publisher,PAGE_URL ) ";
	        sql += "values( '" + title + "', ";
	        sql += "?, ";
	        sql += "'" + typeid + "', ";
	        sql += "to_date('"+techTime+"','yyyy-mm-dd'), ";
	        sql += auditing + ", ";
	        sql += "'" + publisher + "','"+itsUrl+"' )";
	        
	        DBOracleConnection.getInstance().executeInsert( sql, content );
        }
    }    
    /**
     * 完成价格的插入工作。
     * @param pInfo
     */
    public void inserPriceInfo2DB( PriceInfo pInfo )
    {
        String agriclass, agricategory, agrimarket;
        String agriunit,agrisite;
        double agriprice;
        java.sql.Date   agridate;
        String other1, other2;
        
        agriclass = pInfo.getAgriClass();
        agricategory = pInfo.getAgriCategory();
        agrimarket = pInfo.getAgriMarket();
        agriunit = pInfo.getAgriUnit();
        agrisite = pInfo.getAgriSite();
        agriprice = pInfo.getAgriPrice();
        agridate = pInfo.getAgriDate();
      
        java.sql.Date collectedtime = pInfo.getCollectedTime();

        other1 = pInfo.getOther1();
        other2 = pInfo.getOther2();
        
        /*
         	Filt the fields
         */
        agrisite = SiteFilter.getInstance().filter( agrisite );
        
        agriclass = ColonFilter.getInstance().filter( agriclass );
        agricategory = ColonFilter.getInstance().filter( agricategory );
        agrimarket = ColonFilter.getInstance().filter( agrimarket );
        agriunit = ColonFilter.getInstance().filter( agriunit );
//        agrisite = ColonFilter.getInstance().filter( agrisite );
        other1 = ColonFilter.getInstance().filter( other1 );
        other2 = ColonFilter.getInstance().filter( other2 );

        if( !UrlCheck.getInstance().check( agrisite ) )
        {
            String str = "Ignore an item, mistake happen while insert the item into agriprice_tbl.";
            FileLoggerError.getInstance().warning( str );
            return;
        }
        
        String sql=null;
        
        if( !oracleFlag )
        {
	        sql  = "insert into " + PRICE_TABLE + "(";
	        sql += "agriclass, agricategory, agrimarket,agriunit,agrisite,agriprice,agridate, collectedtime, other1, other2) ";
	        sql += "values( '" + agriclass + "', ";
	        sql += "'" + agricategory + "', ";
	        sql += "'" + agrimarket + "', ";
	        sql += "'" + agriunit + "', ";
	        sql += "'" + agrisite + "', ";
	        sql += "" + agriprice + ", ";
	        sql += "'" + agridate + "', ";
	        sql += "'" + collectedtime + "', ";
	        sql += "'" + other1 + "', ";
	        sql += "'" + other2 + "' ) ";
	//        System.out.println( sql );
	        
	        DBConnection.getInstance().executeInsert( sql );
        }else
        {
            //String pricetable = "agriprice_split";
            if(agridate==null)
        	{
            	agridate=collectedtime;
        	}
	        sql  = "insert into " + PRICE_TABLE + "(";
	        sql += "agriclass, agricategory, agrimarket,agriunit,agrisite,agriprice,agridate, collectedtime, other1, other2) ";
	        sql += "values( '" + agriclass + "', ";
	        sql += "'" + agricategory + "', ";
	        sql += "'" + agrimarket + "', ";
	        sql += "'" + agriunit + "', ";
	        sql += "'" + agrisite + "', ";
	        sql += "" + agriprice + ", ";
	        sql += "to_date('"+agridate+"','yyyy-mm-dd'), ";
	        sql += "to_date('"+collectedtime+"','yyyy-mm-dd'), ";
	        sql += "'" + other1 + "', ";
	        sql += "'" + other2 + "' ) ";
//	        System.out.println( sql );
	        
    		DBOracleConnection.getInstance().executeInsert( sql );
        }
    }
    /**
     * 完成供求信息的插入工作。
     * @param sdInfo
     */
    public void inserSplDemInfo2DB( SplDemInfo sdInfo )
    {
        String page_url="", info_title="", info_type="";
        String linkman="", contact_means="", email="";
        String info_content="";
        String other1="", other2="";
        java.sql.Date reg_time=null;
        
        page_url = sdInfo.getPageUrl();
        
        info_title = sdInfo.getInfoTitle();
        if( info_title==null || info_title.equalsIgnoreCase("") )
        	return ;
        if( info_title.length()>40 )
        	info_title = info_title.substring( 0, 39 );
        
        info_type = sdInfo.getInfoType();
        if( info_type.equals("未知") || info_type.equals("") )
        {
            int i,j;
            i = info_title.indexOf( "供" );
            if( i==-1 )
                i = info_title.indexOf( "售" );
            if( i==-1 )
                i = info_title.indexOf( "销" );
            
            j = info_title.indexOf( "求" );
            if( j==-1 )
                j = info_title.indexOf( "购" );
            if( j==-1 )
                j = info_title.indexOf( "收" );