savepage.java

针对音乐论坛的爬虫程序 给出地址匹配特征

package savePage;


import java.net.*;
import java.io.*;
public class SavePage {
	
	//获取主题正文页面 
	public void getTopicContent(String strUrl,String dirname,String filename)
	 // 一个public方法，返回字符串，错误则返回"error open url"
	 {
	  try{
		   
		   File dir =new File("D:\\Test\\"+dirname);
		   dir.mkdirs();
		   
		   URL url=new URL(strUrl);
		   BufferedReader br=new BufferedReader(new InputStreamReader(url.openStream()));
		   String s="";
		   StringBuffer sb=new StringBuffer("");
		   while((s=br.readLine())!=null)
		   {     
		    sb.append(s+"\r\n");    
		   }
		   //文件写入
		   File file1=new File(dir+"\\"+filename+".html");       
		   FileOutputStream fileoutstream = new FileOutputStream(file1, true);
		   Writer writer = new OutputStreamWriter(fileoutstream, "gbk");
		   writer.write(sb.toString());
		   writer.close();
		   fileoutstream.close();
		   br.close();
		   System.out.println(strUrl+"已下载");
		  }
		  catch(Exception e){
		   System.out.println("error open url" + strUrl);
		  }  
	  }
	 
	 //获取主题列表页面
	public void getBlockContent(String strUrl,String dirname,String filename)
	 // 一个public方法，返回字符串，错误则返回"error open url"
	 {
	  try{
		  File dir =new File("D:\\Test\\TopicList"+dirname);
		   dir.mkdirs();
		   
		   
		   URL url=new URL(strUrl);
		   BufferedReader br=new BufferedReader(new InputStreamReader(url.openStream()));
		   String s="";
		   StringBuffer sb=new StringBuffer("");
		   while((s=br.readLine())!=null)
		   {     
		    sb.append(s+"\r\n");    
		   }
		   //文件写入时存在问题，待解决。。。
		   File file1=new File(dir+"\\"+filename+".html");       
		   FileOutputStream fileoutstream = new FileOutputStream(file1, true);
		   Writer writer = new OutputStreamWriter(fileoutstream, "gbk");
		   writer.write(sb.toString());
		   writer.close();
		   fileoutstream.close();
		   br.close();
		   System.out.println(strUrl+"已下载");
		  }
		  catch(Exception e){
		   System.out.println("error open url" + strUrl);
		  }  
	  }
}