hostnamequeueassignmentpolicy.java

最强的爬虫工程

/* HostnameQueueAssignmentPolicy
*
* $Id: HostnameQueueAssignmentPolicy.java,v 1.9 2005/09/21 23:00:47 gojomo Exp $
*
* Created on Oct 5, 2004
*
* Copyright (C) 2004 Internet Archive.
*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Heritrix is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* any later version.
*
* Heritrix is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU Lesser Public License for more details.
*
* You should have received a copy of the GNU Lesser Public License
* along with Heritrix; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/ 
package org.archive.crawler.frontier;

import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.commons.httpclient.URIException;
import org.archive.crawler.datamodel.CandidateURI;
import org.archive.crawler.framework.CrawlController;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;

/**
 * QueueAssignmentPolicy based on the hostname:port evident in the given
 * CrawlURI.
 * 
 * @author gojomo
 */
public class HostnameQueueAssignmentPolicy extends QueueAssignmentPolicy {
    private static final Logger logger = Logger
        .getLogger(HostnameQueueAssignmentPolicy.class.getName());
    /**
     * When neat host-based class-key fails us
     */
    /**
     * When neat host-based class-key fails us
     */
    private static String DEFAULT_CLASS_KEY = "default...";

    private static final String DNS = "dns";
    
    private static int num = 0;

    public String getClassKey(CrawlController controller, CandidateURI cauri) {
        String uri = cauri.getUURI().toString();
        long hash = ELFHash(uri);
//        System.out.println(hash);
        String a = Long.toString(hash % 100);
//        System.out.println(a);
        return a;
    }

    public long ELFHash(String str)
    {
       long hash = 0;
       long x    = 0;

       for(int i = 0; i < str.length(); i++)
       {
          hash = (hash << 4) + str.charAt(i);

          if((x = hash & 0xF0000000L) != 0)
          {
             hash ^= (x >> 24);
             hash &= ~x;
          }
       }

       return (hash & 0x7FFFFFFF);
    }
//    public String getClassKey(CrawlController controller, CandidateURI cauri) {
//    	
//    	num++;
//    	
//    	if (num >= 100) {
//    		num = 1;
//    		
//    	}
//    	return num+"";
//    }

}