pgdblogger.java

利用广度优先遍历搜索一定范围内的所有网页,可用于建立搜索引擎和查找网络错误.

/**
 * PgDBLogger
 * 
 * Prerequisite: PsqlPool must be initialized.
 * 
 */
package log;


import java.sql.*;

import WebCrawler.*;
import io.MyPrintStream;
import db.PsqlPool;

public class PgDBLogger {
    //private int contents;
    private boolean available = false;

    /**
     * Consume <code>LogItem</code>s as long as there exist any.
     * 1. Get a <code>link</code> item is <code>visited</code> is NULL.
     * 2. If successful, set the <code>visited</code> of <code>link</code> to the current timestamp.
     * 
     * @return
     * @throws InterruptedException
     */
    public synchronized LogItem get()
    throws InterruptedException {
    	while( available == false ) {
            // wait for Producer to put value
   	        wait();
    	} // while
    	
    	//available = false;
    	
    	LogItem item = null;
    	String expr = "SELECT source,target FROM link WHERE visited IS NULL"; 
    	try {
    		if( _DEBUG_PGDBLOGGER )
    			MyPrintStream.out.println("[PgDBLogger.get] PsqlPool size before: "+PsqlPool.size());
    		Connection connection = PsqlPool.getConnection();
    		
    		// Get a <code>link</code> item is <code>visited</code> is NULL.
    		ResultSet rs = connection.createStatement().executeQuery( expr );
			rs.next();
			item = new LogItem( new Link( null, rs.getString(1), true, MyPrintStream.out ), new Link( null, rs.getString(2), true, MyPrintStream.out ), null );
    		if( _DEBUG_PGDBLOGGER )
    			MyPrintStream.out.println("[PgDBLogger.get] Found: "+item.getSource()+", "+item.getTarget());
			
			// Set the <code>visited</code> of <code>link</code> to the current timestamp.
			int status = connection.createStatement().executeUpdate( item.updateString() );
			if( status<1 ) {
				MyPrintStream.out.println("[PgDBLogger.get] update failed: "+item.updateString());
			}
			
			// Check if this item was the last item in the database.
			if( rs.isLast() ) {
				MyPrintStream.out.println("[PgDBLogger.get] no more item to get.");
				available = false;
			}
			else {
				MyPrintStream.out.println("[PgDBLogger.get] more items to get.");
				available = true;
			}
			
			PsqlPool.returnConnection(connection);
    		if( _DEBUG_PGDBLOGGER )
    			MyPrintStream.out.println("[PgDBLogger.get] PsqlPool size after: "+PsqlPool.size());
		} catch (SQLException e) {
			//MyPrintStream.out.println("[PgDBLogger.get] failed for: "+expr);
			MyPrintStream.out.println("[PgDBLogger.get] failed: "+e.getMessage());
			//e.printStackTrace();
		}
    	// notify Producer that value has been retrieved
    	notify();
    	return item;
	}
    /**
     * Produce <code>LogItem</item> as many as possible and store them into the database
     * without checking if previously produced items have been depleted.
     * 
     * @param item
     * @throws InterruptedException
     */
	public synchronized void put( LogItem item )
	throws InterruptedException {
    	//while( available == true ) {
        //   	// wait for Consumer to get value
        //   	wait();
    	//}
		
    	//contents = value;
		
    	Connection connection = null; 
		try {
    		if( _DEBUG_PGDBLOGGER )
    			MyPrintStream.out.println("[PgDBLogger.put] PsqlPool size before: "+PsqlPool.size());
    		connection = PsqlPool.getConnection();
    		
			//int rowcount = statement.executeUpdate("INSERT INTO link VALUES('"+source.getURL()+"','"+target.getURL()+"', null)");
			int rowcount = connection.createStatement().executeUpdate(item.insertString());
			if( rowcount<1 ) {
				//MyPrintStream.out.println("[PgDBLogger.put] error: ("+item.getSource()+","+item.getTarget()+")");
				MyPrintStream.out.println("[PgDBLogger.put] error: "+item.insertString());
			}
    		if( _DEBUG_PGDBLOGGER )
    			MyPrintStream.out.println("[PgDBLogger.put] logging done: "+item.toString());
    		PsqlPool.returnConnection( connection );
    		if( _DEBUG_PGDBLOGGER )
    			MyPrintStream.out.println("[PgDBLogger.put] PsqlPool size after: "+PsqlPool.size());

			available = true;
		} catch (SQLException e) {
			MyPrintStream.out.println("[PgDBLogger.put] "+e.getMessage()+" by "+item.insertString());
			//e.printStackTrace( MyPrintStream.out );
		}
		if( null!=connection )
			PsqlPool.returnConnection( connection );
		
    	// notify Consumer that value has been set
    	notify();
	}
	public boolean isAvailable() {
		return available;
	}
	public void setAvailable( boolean val ) {
		available = val;
	}
	public static void main( String[] args ) {
		String dbHostName = "jdbc:postgresql://localhost:5432/crawler";
		String dbUsername = "postgres";
		String dbPassword = "postgres";

		try {
			PsqlPool.init(dbHostName, dbUsername, dbPassword);
			@SuppressWarnings("unused")
			Connection conn = PsqlPool.getConnection();
		} catch( SQLException e ) {
			e.printStackTrace();
		}
	}
	private boolean _DEBUG_PGDBLOGGER = false;
	
}