writer.java

使用Eclipse编写的java的网络图片爬虫

		int nstartpos = 0;
		int nstoppos = 0;
		int pos = 0;
		int i = 0;
		boolean bSaveToFile;
		int nCount = 0;
		int nCount2 = 0;

		if (strHtml == null || strHtml.isEmpty())
			return;
		try
		{
			strHtml = strHtml.toLowerCase();

			while (pos != -1)
			{
				pos = strHtml.indexOf(strBZ, nstoppos);
				if (-1 == pos)
					break;
				nCount++;
				char c = strHtml.charAt(pos + strBZ.length());
				if ('"' == c || '\'' == c)
					nstartpos = pos + strBZ.length() + 1;
				else
					nstartpos = pos + strBZ.length();

				for (i = nstartpos; i < strHtml.length(); i++)
				{
					if ((strHtml.charAt(i) == '"') || (strHtml.charAt(i) == '\'') || (strHtml.charAt(i) == '>') || (strHtml.charAt(i) == 0x20) || (strHtml.charAt(i) == 0x0d) || (strHtml.charAt(i) == 0x0a) || (strHtml.charAt(i) == ',') || (strHtml.charAt(i) == ';'))
					{
						break;
					}
				}
				nstoppos = i;

				strmylinktext = strHtml.substring(nstartpos, i);
				strmylinktext = completeURL(strmylinktext);
				//----------------------------------------------
				gather g = new gather();
				g.strurl = strmylinktext;
				if (strmylinktext.endsWith(".jpg") || strmylinktext.endsWith(".bmp") || strmylinktext.endsWith(".jpeg"))
				{
					if (!controler.IMGList.contains(strmylinktext))
					{
						controler.IMGList.add(strmylinktext);
						controler.MissionList.add(strmylinktext);
					}
				}
			}
		}
		catch (Exception e)
		{
			System.out.println("Error:" + e.getMessage());
		}
	}
	
	public synchronized void writeStringToFile(String str)
	{//测试用：
		try
		{
			str+="\r\n";
			new File("d:\\temp\\").mkdirs();
			String strfile = "d:\\temp\\由href解析出的URL.txt";
			FileOutputStream fos = new FileOutputStream(strfile,true);
			fos.write(str.getBytes());
			fos.close();
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
	}
	public void getOtherUrlLink(String strHtml, String strcharset) throws Exception
	{
		String line = "";
		String strBZ = "href=";
		String strmylinktext = "";

		int nstartpos = 0;
		int nstoppos = 0;
		int pos = 0;
		int i = 0;
		boolean bSaveToFile;
		char c_start=0;//记录href= 后的起始字符，即引号；
		boolean b_havestartchar;//href= 后是否有起始字符，即引号；

		if (strHtml == null || strHtml.isEmpty())
			return;
		try
		{
			strHtml = strHtml.toLowerCase();

			while (pos != -1)
			{
				pos = strHtml.indexOf(strBZ, nstoppos);
				if (-1 == pos)
					break;
				char c = strHtml.charAt(pos + strBZ.length());
				if ('\\' == c)
					continue;
				if ('"' == c || '\'' == c)
				{
					c_start=c;//href后的链接开始字符，如果是引号，那么结束字符也是它。
					b_havestartchar=true;
					nstartpos = pos + strBZ.length() + 1;
				}
				else
				{
					b_havestartchar=false;
					nstartpos = pos + strBZ.length();
				}

				for (i = nstartpos; i < strHtml.length(); i++)
				{
					if(b_havestartchar&&(strHtml.charAt(i) == c_start))
					{//有起始字符时：遇到起始字符,则结束
						break;
					}
					else if ((strHtml.charAt(i) == '>') || (strHtml.charAt(i) == 0x20) || (strHtml.charAt(i) == 0x0d) || (strHtml.charAt(i) == 0x0a) || (strHtml.charAt(i) == ',') || (strHtml.charAt(i) == ';'))
					{//没有起始字符时：遇到 >或者空格或者回车换行，则就是结束
						break;
					}
				}
				nstoppos = i;

				strmylinktext = strHtml.substring(nstartpos, i);
				//---------------------------------------------------------
				//测试：
//				writeStringToFile(strmylinktext);
				//---------------------------------------------------------
				if (strmylinktext.equals("../") || strmylinktext.endsWith(".css"))
					continue;
				strmylinktext = completeURL(strmylinktext);
				//----------------------------------------------
				if (strmylinktext.endsWith(".jpg") || strmylinktext.endsWith(".bmp") || strmylinktext.endsWith(".jpeg"))
				{
					if (!controler.IMGList.contains(strmylinktext))
					{
						controler.IMGList.add(strmylinktext);
						controler.MissionList.add(strmylinktext);
					}
				}
				else if (strmylinktext.startsWith(urlPath[0]))/*只处理用户输入的网站的链接和子链接，不向外扩展*/
				{
					if ((!controler.URLList.contains(strmylinktext)))
					{
						controler.URLList.add(strmylinktext);
						controler.MissionList.add(strmylinktext);
					}
				}
			}
		}
		catch (Exception e)
		{
			System.out.println("Error:" + e.getMessage());
		}
	}

	/**
	 * 功 能：判断字符串去掉左右空格后是否为空
	 */
	public static boolean isTrimEmpty(String astr)
	{
		if ((null == astr) || (astr.length() == 0))
		{
			return true;
		}
		if (isBlank(astr.trim()))
		{
			return true;
		}
		return false;
	}

	/**
	 * 功 能：判断字符串是否为空:null或者长度为0.
	 */
	public static boolean isBlank(String astr)
	{
		if ((null == astr) || (astr.length() == 0))
		{
			return true;
		}
		else
		{
			return false;
		}
	}

	/**
	 * 功 能：判断输入的字符串是否全部是数字 输入参数：String 返 回 值：true：全数字；false：不全为数字
	 */
	public boolean isNumeral(String str)
	{
		if ((null == str) || (str.length() == 0))
		{
			return true;
		}
		str = filtedString(str);
		String strSYMBOL = "0123456789";
		for (int i = 0; i < strSYMBOL.length(); i++)
		{
			String oldC = strSYMBOL.substring(i, i + 1);
			str = str.replaceAll(oldC, "");// 将要过滤的字符删除
		}
		if (str.isEmpty())// 去掉符号和数字后如果为空：则可判断只有数字：
			return true;
		return false;
	}

	/**
	 * 功 能：过滤掉：可显示的、非文字的字符 作 者：
	 */
	private String filtedString(String instr)
	{
		if ((null == instr) || (instr.length() == 0))
			return "";
		String str = "";
		str = instr;
		String strSYMBOL = "`-=[]\\;',./~!@#$%^&*()_+{}|:\"<>?\r\n\t ";// 最后一个是空格
		str = str.replace("&nbsp", "");// 过滤掉：&nbsp
		for (int i = 0; i < strSYMBOL.length(); i++)
		{
			String oldC = strSYMBOL.substring(i, i + 1);
			str = str.replace(oldC, "");// 将要过滤的字符删除
		}
		return str;
	}

	public String completeURL(String strNewUrl)
	{
		if (strNewUrl == null || strNewUrl.length() == 0)
			return "";
		if ((strNewUrl.startsWith("http://"))||(strNewUrl.startsWith("#")) || (nUrlPath <= 0))
			return strNewUrl;
		
		if(strNewUrl.startsWith("javascript:mm_openbrwindow("))
		{//增加功能：解析javascript:mm_openbrwindow语句中的URL：(2007-11-13)
			int nstartpos="javascript:mm_openbrwindow(".length();
			char c=strNewUrl.charAt(nstartpos);
			if ('"' == c || '\'' == c)
			{
				strNewUrl=strNewUrl.substring(nstartpos+1,strNewUrl.length()-2);
			}
			else
			{
				strNewUrl=strNewUrl.substring(nstartpos,strNewUrl.length()-1);
			}
		}
		String strcompleteurl = "";
		String strParentURL = "";

		try
		{
			if (strNewUrl.startsWith("/"))
			{// 一、相对链接是以"/"开头，则直接加上URL的根目录：
				strParentURL = urlPath[0];
				strNewUrl = strNewUrl.substring(1);
			}
			else if (strNewUrl.startsWith("./"))
			{// 二、相对链接是以"../"开头，则回退到上一级目录：
				int nNeedUrlPath = nUrlPath;
				while (strNewUrl.startsWith("./"))
				{
					nNeedUrlPath--;
					strNewUrl = strNewUrl.substring(2);
				}
				if (nNeedUrlPath <= 0)
					strParentURL = urlPath[0];
				else
				{
					for (int t = 0; t < nNeedUrlPath; t++)
					{
						strParentURL += urlPath[t];
					}
				}
			}
			else if (strNewUrl.startsWith("../"))
			{// 同上：相对链接是以"../"开头，则回退到上一级目录：
				int nNeedUrlPath = nUrlPath;
				while (strNewUrl.startsWith("../"))
				{
					nNeedUrlPath--;
					strNewUrl = strNewUrl.substring(3);
				}
				if (nNeedUrlPath <= 0)
					strParentURL = urlPath[0];
				else
				{
					for (int t = 0; t < nNeedUrlPath; t++)
					{
						strParentURL += urlPath[t];
					}
				}
			}
			else
			{// 三、没有斜杠开头，则直接加上当前的目录：
				for (int t = 0; t < nUrlPath; t++)
				{
					strParentURL += urlPath[t];
				}
			}
			strcompleteurl = strParentURL + strNewUrl;
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}

		return strcompleteurl;
	}

}