spider.txt

用java实现一个简单的spider程序.

import java.util.Vector;



/**

*线程池的方法使得在线程密集的情况下原有线程得到重用

*从而提高性能

*@since 0.6

*/

public class ThreadPool implements Runnable{



//ThreadPool里最大线程数、最少线程数、释放延时的确省值

public final static int DEFAULT_MIN_SIZE = 0;

public final static int DEFAULT_MAX_SIZE = Integer.MAX_VALUE;

public final static long DEFAULT_RELEASE_DELAY = 10 * 1000;

//用户定制的threadpool属性值

protected int minSize;

protected int maxSize;

protected long releaseDelay;

//ThreadPool里现有的线程数

protected int currentSize;

//ThradPool里还可以提供的线程数

protected int availableThreads;

//任务列表(实质意义上的线程池)

protected Vector taskList;



/**

   *用户定制的ThradPool

   *@param minSize 线程池最少线程数

   *@param maxSize 线程池最多线程数

   *@param releaseDelay 线程释放延时

   */

public ThreadPool(int minSize,int maxSize,long releaseDelay) {

    this.minSize = minSize;

    this.maxSize = maxSize;

    this.releaseDelay = releaseDelay;

    taskList = new Vector(100);

    availableThreads = 0;

}

/**

   *缺省的ThreadPool

   */

public ThreadPool(){

    this(DEFAULT_MIN_SIZE,DEFAULT_MIN_SIZE,DEFAULT_RELEASE_DELAY);

}

/**

   *设置线程池中最少线程数

   *@param minSize 线程池最少线程数

   */

public synchronized void setMinSize(int minSize){

    this.minSize = minSize;

}

/**

   *读取线程池最少线程数这一属性

   */

public synchronized int getMinSize(){

    return minSize;

}

/**

   *设置线程池中最大线程数

   *@param maxSize 线程池最大线程数

   */

public synchronized void setMaxSize(int maxSize){

    this.maxSize = maxSize;

}

/**

   *读取线程池最大线程数这一属性

   */

public synchronized int getMaxSize(){

    return maxSize;

}

/**

   *设置线程池中线程释放延时

   *@param releaseDelay 线程释放延时

   */

public synchronized void setReleaseDelay(long releaseDelay){

    this.releaseDelay = releaseDelay;

}

/**

   *读取线程释放延时这一属性

   */

public synchronized long getReleaseDelay(){

    return releaseDelay;

}

/**

   *往ThreadPool里添加新的任务

   *@param runnable 新的任务

   */

public synchronized void addTask(Runnable runnable){



    taskList.addElement(runnable);

    if(availableThreads > 0){

       this.notify();

    }

    else{

      if(currentSize < maxSize){

        Thread t = new Thread(this);

        currentSize++;

        t.start();

      }

    }

}



public void run(){

    Runnable task;

    while(true){

      synchronized(this){

        if(currentSize > maxSize){

          currentSize--;

          break;

        }

        task = getNextTask();

        if(task == null ){

          try{

            availableThreads++;

            wait(releaseDelay);

            availableThreads--;

          }catch(InterruptedException ie){

            // do something you wanna

          }

          task = getNextTask();

          if(task == null){

            if(currentSize < minSize)

              continue;

            currentSize--;

            break;

          }

        }

      }

      try{

        task.run();

      }catch(Exception e){

        System.err.println("Uncaught exception");

        e.printStackTrace(System.err);

      }

    }

}

/**

   *从任务列表中获取下一个任务

   *

   */

protected synchronized Runnable getNextTask(){

    Runnable task = null;

    if(taskList.size() > 0){

      task = (Runnable)(taskList.elementAt(0));

      taskList.removeElementAt(0);

    }

    return task;

}

/**

   *返回ThreadPool的信息

   */

public String toString(){

    StringBuffer sb = new StringBuffer();

    sb.append("DEFAULT_MIN_SIZE : " + DEFAULT_MIN_SIZE + "\n");

    sb.append("DEFAULT_MAX_SIZE : " + DEFAULT_MAX_SIZE + "\n");

    sb.append("DEFAULT_RELEASE_DELAY : " + DEFAULT_RELEASE_DELAY + "\n" );

    sb.append("the information about your's construct ThreadPool below : \n");

    sb.append("minSize \t maxSize \t releaseDelay \n");

    sb.append(minSize + "\t" + maxSize +"\t" + releaseDelay );



    return sb.toString();

}



}