channelparser.java

模拟的土豆网视频网站

package cn.myvideosite.exe.parser;

import java.util.ArrayList;
import java.util.List;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

import cn.myvideosite.commons.Constant;
import cn.myvideosite.data.model.bean.ChannelClass;
import cn.myvideosite.data.model.services.ChannelService;
import cn.myvideosite.util.HttpUtil;


public class ChannelParser {
	/**
	 * 定义<div id="ListTags">标签过滤器常量 抓取56频道首页的所有频道名html代码
	 * 根据div
	 */
	private static final NodeFilter FILTER_DIV_LISTTAGS=
		    new AndFilter(new TagNameFilter("div"),new HasAttributeFilter("id","ListTags"));
	/**
	 * <div class="Tag2">
	 * 
	 */
	private static final NodeFilter FILTER_DIV_TAG2=
	    new AndFilter(new TagNameFilter("div"),new HasAttributeFilter("class","Tag2"));
	/**
	 * A标签
	 */
	//private static final NodeFilter FILTER_DIV_LI=new TagNameFilter("li");
      /**
       * <a></a>抓取连接 A标签
       */
	private static final NodeFilter FILTER_DIV_A=new TagNameFilter("a");
	
	public static ChannelClass parser(String url){
		String page=HttpUtil.request(url, Constant.CHARSET_GB2312);
	    Parser pageParser=Parser.createParser(page,Constant.CHARSET_GB2312);
	    ChannelClass channelclass=null;
	    try {
			NodeList nl=pageParser.parse(FILTER_DIV_LISTTAGS);
			pageParser=Parser.createParser(nl.toHtml(),  Constant.CHARSET_GB2312);
			NodeList tag2NL=pageParser.parse(FILTER_DIV_TAG2);
			if(tag2NL!=null && tag2NL.size()>0){
				for(int i=0;i<tag2NL.size();i++){
					Node tag2Node=tag2NL.elementAt(i);
                    
					pageParser=Parser.createParser(tag2Node.toHtml(), Constant.CHARSET_GB2312);
					NodeList aNL=pageParser.parse(FILTER_DIV_A);
					if(aNL!=null && aNL.size()>0){
						List<ChannelClass> list =new ArrayList<ChannelClass>(aNL.size());
						for(int ia=0;ia<aNL.size();ia++){
							LinkTag link=(LinkTag)aNL.elementAt(ia);
							System.out.println(link.getLinkText());							
								channelclass=new ChannelClass();
								channelclass.setChannelName(link.getLinkText());
								list.add(channelclass);							
						}
						ChannelService.save(list);
					}
				}					
			}						
		} catch (ParserException e) {			
			e.printStackTrace();
		}
		return channelclass;
	}
	public static void main(String[] args){
		parser("http://www.56.com/w/show_channel.phtml");
	}
}