updatedb.java

Light in the box 抓取程序。 使用HttpClient

package com.blogool.export;

import java.io.*;
import java.sql.*;
import java.util.*;

import com.blogool.crawl.*;
import com.blogool.crawl.lib.*;

public class UpdateDB {
	public static void main(String[] args) throws Exception {
		Cat root = Util.loadCat(new File("d:/libox1/cats4.xml"));
		Connection conn = DBUtil.getConnection();
		update(conn, root);
	}

	public static void update(Connection conn, Cat root) {
		for (int i = 0; i < root.getCats().size(); i++) {
			Cat c = root.getCats().get(i);
			for (int j = 0; j < c.getCats().size(); j++) {
				Cat cat = c.getCats().get(j);
				List<Item> list = cat.getItems();
				if (list != null) {
					for (int k = 0; k < list.size(); k++) {
						Item item = list.get(k);
						updateItem(conn, item);
					}
				}
			}
		}
	}

	public static void updateItem(Connection conn, Item item) {
		// 更新item的imageUrl
		/*
		if (item.getImageUrls() != null) {
			String sql = null;
			if (item.getImageUrls().length == 1) {
				sql = "update product set pic = ? where id = ?";
				DBUtil.update(conn, sql, new Object[] {
						getImageUrl(item.getImageUrls()[0]), item.getId() });
			} else if (item.getImageUrls().length == 2) {
				sql = "update product set pic = ?, pic1 = ? where id = ?";
				DBUtil.update(conn, sql, new Object[] {
						getImageUrl(item.getImageUrls()[0]),
						getImageUrl(item.getImageUrls()[1]), item.getId() });

			} else if (item.getImageUrls().length == 3) {
				sql = "update product set pic=?, pic1=?, pic2=? where id=?";
				DBUtil.update(conn, sql, new Object[] {
						getImageUrl(item.getImageUrls()[0]),
						getImageUrl(item.getImageUrls()[1]),
						getImageUrl(item.getImageUrls()[2]),
						item.getId() });
			}
		}*/
		//更新desc,notice,packageinfo,spec
		/*
		String sqlDesc = "update product set content=?, memo = ?, packageinfo = ?, spec = ? where id = ?";
		String content = null, notice = null, packageinfo = null, spec = null;
		content = modifyText(item.getDescription());
		notice = modifyText(item.getNotice());
		packageinfo = modifyText(item.getPackageinfo());
		spec = modifyText(item.getSpecfication());
		
		//对以上字段重新进行编制
		DBUtil.update(conn, sqlDesc, new Object[] {content, notice, packageinfo, spec, item.getId()});
		*/
		String weightSql = "update product set weight = ? where id = ?";
		DBUtil.update(conn, weightSql, new Object[] {new Float(item.getWeight()), item.getId()});
	}
	
	public static String modifyText(String text) {
		if (text == null) return null;
		text = text.replaceAll("&nbsp;", " ");
		text = text.replaceAll("</?.+?>", "");
		while (text.startsWith("\r\n")) text = text.substring(2);
		while (text.startsWith("\n")) text = text.substring(1);
		while (text.startsWith(" ")) text = text.substring(1);
		
		if (text.startsWith("Description")) text = text.substring("Description".length());
		else if (text.startsWith("Notice")) text = text.substring("Notice".length());
		else if (text.startsWith("NOTICE")) text = text.substring("NOTICE".length());
		else if (text.startsWith("Package include")) text = text.substring("Package include".length());
		else if (text.startsWith("Specification")) text = text.substring("Specification".length());
		while (text.startsWith(" ")) text = text.substring(1);
		if (text.startsWith(":")) text = text.substring(1);
		
		while (text.startsWith("\r\n")) text = text.substring(2);
		while (text.startsWith("\n")) text = text.substring(1);
		while (text.startsWith(" ")) text = text.substring(1);
		System.out.println(text);
		return text;
	}

	public static String getImageUrl(String imageUrl) {
		if (imageUrl == null)
			return null;

		String name = Util.getImageFileName(imageUrl);
		String md5 = Util.md5Encoding(name);
		String folder = md5.substring(0, 2);
		String result = folder + "/" + name;
		return result;
	}
}