generateterms.java

利用多线程从搜索引擎下载网页并提取数据到数据库。

//import java.sql.Connection;
import com.mysql.jdbc.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;

public class GenerateTerms {

	// String s="http://www.linkedin.com/pub/0/111";
	static String s = "http://www.linkedin.com/pub/";
	static int count;
	static Connection cnn;

	/**
	 * @throws SQLException 
	 * @throws SQLException
	 */
	public static void main(String[] args) throws SQLException {

		//bugs with this program. it generates something like http://..../_/00_,0__...000,001, 012
		//generate();
	}

	private static void generate() throws SQLException {
		int i, j, k, l;

		StringBuilder sb;
		cnn = DataAccess.getNewConnection();
		try {
			cnn.setAutoCommit(false);
		} catch (SQLException e) {
			e.printStackTrace();
		}
		for (i = 0; i <= 6; i++) {
			for (j = 0; j <= 0xE; j++)
				for (k = 0; k <= 0xE; k++)
					for (l = 0; l <= 0xE; l++) {
						sb = new StringBuilder();
						sb.append(s);
						sb.append(String.valueOf(i));
						sb.append("/");
						sb.append(Integer.toHexString(j));
						sb.append(Integer.toHexString(k));
						sb.append(Integer.toHexString(l));
						String s = sb.toString();
						insert(s);
					}
		}
	}

	private static void insert(String s) {
		PreparedStatement ps;
		try {
			ps = cnn
					.prepareStatement("insert into search_terms(term) select ?");
			ps.setString(1, s);
			ps.execute();
		} catch (SQLException e) {
			e.printStackTrace();
		}

		count++;
		// if (count == 100)
		{
			try {
				cnn.commit();
				System.out.println("1 rows committed!");
			} catch (SQLException e) {
				try {
					cnn.rollback();
				} catch (SQLException e1) {
					e1.printStackTrace();
				}
				e.printStackTrace();
			}
			try {
				if (!cnn.isClosed())
					cnn.close();
				cnn = DataAccess.getNewConnection();
				cnn.setAutoCommit(false);
				count = 0;
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}

	}

}