logmonitor.java

Java下的Apache日志分析器

package com.blogool.util;

import java.io.*;

import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.text.SimpleDateFormat;
import java.util.*;

import com.blogool.ip.*;

public class LogMonitor {
	private static String sQuitFilename = "/BLOGOOL/bloglink/monitorfile.app.quit";

	private static HashMap blogVisitorInfo = new HashMap();

	private static ArrayList blogList = new ArrayList();

	public static void main(String[] args) {

		int action = 0;

		String srcFile = "/usr/share/httpd-2.2.4/logs/access_log";
		String destFile = "/BLOGOOL/accesslog_ip";

		long seekStartPos = 0; // defaultly startPos is 0
		long offSize = 0; // defaultly off from tail is 0

		// Date yesterday = new Date();
		String sDestDate = getYesterday() + ":00:00:00"; // defaultly
															// yesterday

		for (int i = 0; i < args.length; i++) { // parse command line
			if (args[i] == null) {
				continue;
			} else if (args[i].equals("-start")) { // found -threads option
				seekStartPos = Long.parseLong(args[++i]);
			} else if (args[i].equals("-srcf")) {
				srcFile = args[++i];
			} else if (args[i].equals("-destf")) {
				destFile = args[++i];
			} else if (args[i].equals("-analyseByPos")) {
				action = 1;
			} else if (args[i].equals("-checkDate")) {
				action = 2;
			} else if (args[i].equals("-off")) {
				offSize = Long.parseLong(args[++i]);
			} else if (args[i].equals("-analyseByDate")) {
				action = 3;
			} else if (args[i].equals("-destDate")) {
				sDestDate = args[++i];
			}
		}

		switch (action) {
		case 0: {
			System.out
					.println("No Action! 1--analyseByPos; 2--checkDate; 3--analyseByDate");
			System.out
					.println("usage:  -analyseByPos [-srcf srcFilename] [-destf destFilename] [-start seekStartPos]");
			System.out
					.println("usage:  -checkDate [-off offSize] [-srcf srcFilename]");
			System.out
					.println("usage:  -analyseByDate [-destDate 31/Aug/2007:00:00:00] [-srcf srcFilename] [-destf destFilename]");
			break;
		}
		case 1: {
			analyseByPos(seekStartPos, 0, srcFile);
			break;
		}
		case 2: {
			checkDate(offSize, srcFile);
			break;
		}
		case 3: {
			long lDate = transferDate(sDestDate);
			if (lDate == 0) {
				System.out.println("Wrong Date: " + sDestDate
						+ " and yesterday will be analyzed");
				sDestDate = getYesterday() + ":00:00:00";
				lDate = transferDate(sDestDate);
			}
			System.out.println("analyseByDate:" + sDestDate);
			analyseByDate(lDate, srcFile, destFile);

			break;
		}
		}// end switch

	}

	private static void analyseByDate(long lDate, String srcFile,
			String destFile) {
		long lSeekStart = getSeekPosByAssignedDate(lDate, srcFile);
		long lSeekEnd = getSeekPosByAssignedDate(lDate + 3600 * 24 * 1000 + 5
				* 60 * 1000, srcFile);
		System.out.println("Begin to seek from " + lSeekStart + " to "
				+ lSeekEnd);
		analyseByPos(lSeekStart, lSeekEnd, srcFile);

		System.out.println("analyseByDate end, begin to save file");
		long start = System.currentTimeMillis();
		// write object file
		try {
			File logPath = new File(destFile + "/" + getDateFolderName(lDate));
			if (!logPath.exists()) {
				logPath.mkdir();
			}

			for (int k = 0, i = 0; i < blogList.size(); i++) {
				String blogId = (String) blogList.get(i);

				/*
				 * write to Object FileOutputStream fileOutput = new
				 * FileOutputStream(logPath + "/" + blogId); ObjectOutputStream
				 * objOutput = new ObjectOutputStream(fileOutput); ArrayList
				 * iplist = (ArrayList)blogVisitorInfo.get(blogId);
				 * objOutput.writeObject(iplist); fileOutput.close();
				 */
				/*
				 * write to XML
				 */
				FileWriter fw = new FileWriter(new java.io.File(logPath + "/"
						+ blogId + ".xml"));
				BufferedWriter bw = new BufferedWriter(fw);
				bw
						.write("<?xml version=\"1.0\" encoding=\"utf-8\"?><alldata>\n");

				ArrayList iplist = (ArrayList) blogVisitorInfo.get(blogId);
				for (int j = 0; j < iplist.size(); j++, k++) {
					DetailVisitorInfo info = (DetailVisitorInfo) iplist.get(j);
					bw.write("<data ip=\"" + info.IP + "\" time=\"" + info.time
							+ "\" addr=\"" + info.addr + "\" />\n");

					if (k % 10000 == 0) {
						long end = System.currentTimeMillis();
						System.out.println("Write Object Spend: "
								+ (double) (end - start) / 1000
								+ "s, begin to sleep 5s");
						start = end;
						Thread.sleep(5000);
					}
				}
				bw.write("</alldata>");
				bw.close();
			}
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	}

	private static long getSeekPosByAssignedDate(long lDate, String srcFile) {
		try {
			RandomAccessFile rafi = new RandomAccessFile(srcFile, "r");

			long endPos = rafi.length();
			long startPos = 0;
			long lSeek = 0; // 相当于指针的东西
			long lCount = 0; // 寻找次数

			// check whether the date is after the first day in apache log
			String sDate = getDateBySeekPos(rafi, lSeek);
			long lTmpDate = transferDate(sDate);
			if (lTmpDate > lDate) {
				System.out.println("the date is before the first day(" + sDate
						+ ")in apache log");
				rafi.close();
				return 0;
			}

			System.out.println("File Length:" + endPos);

			while (true) {
				lCount++;
				if (lCount == 20) {
					System.out.println("Seek 20 times, sleep 10s");
					Thread.sleep(10000);
				} else if (lCount == 40) {
					System.out.println("Seek 40 times, break");
					break;
				}

				lSeek = (startPos + endPos) / 2;
				// rafi.seek(lSeek);
				sDate = getDateBySeekPos(rafi, lSeek);
				lTmpDate = transferDate(sDate);
				if (lTmpDate == 0) {
					System.out.println("Date err:" + sDate);
					break;
				}
				if ((lDate - lTmpDate) > 300000) // 5mins
				{
					startPos = lSeek;
				} else if (lTmpDate > lDate) {
					endPos = lSeek;
				} else {
					System.out.println("Count:" + lCount + " Date:" + sDate
							+ " Seek Pos:" + lSeek);
					break;
				}
			}

			rafi.close();
			return lSeek;
		} catch (Exception e) {
			System.out.println(e);
		}
		return 0;
	}

	private static String getDateBySeekPos(RandomAccessFile rafi, long seekPos) {
		byte buf[] = new byte[1 << 10]; // read 1k
		int readcount;
		StringBuffer line = new StringBuffer();
		try {
			rafi.seek(seekPos);
			if ((readcount = rafi.read(buf)) != -1) {
				for (int i = 0; i < readcount; i++) {
					byte b = buf[i];
					line.append((char) b);
					if (b == '\n') {
						String requiredLine = getTimeInfo(line.toString());
						if (!requiredLine.equals("0")) {
							System.out.println("Time:" + requiredLine + " at "
									+ seekPos);
							return requiredLine;
						}
						line.delete(0, line.length());
					}
				}
			} else {
				System.out.println("No date to check in the 1k buff!!");
			}
		} catch (Exception e) {
			System.out.println(e);
		}

		return "0";
	}

	private static void checkDate(long offSize, String srcFile) {
		try {
			RandomAccessFile rafi = new RandomAccessFile(srcFile, "r");

			byte buf[] = new byte[1 << 10]; // read 1k
			int readcount;
			StringBuffer line = new StringBuffer();

			long endPos = rafi.length();
			long seekPos = endPos - offSize;
			System.out.println("File Length:" + endPos + " and seekPos:"
					+ seekPos);
			rafi.seek(seekPos);
			if ((readcount = rafi.read(buf)) != -1) {
				for (int i = 0; i < readcount; i++) {
					byte b = buf[i];
					line.append((char) b);
					if (b == '\n') {
						String requiredLine = getTimeInfo(line.toString());
						if (!requiredLine.equals("0")) {
							System.out.println("Time:" + requiredLine + " at "
									+ seekPos);
							break;
						}
						line.delete(0, line.length());
					}
				}
			} else {
				System.out.println("No date to check");
			}

			rafi.close();
		} catch (Exception e) {
			System.out.println(e);
		}
	}

	private static void analyseByPos(long seekStartPos, long seekEndPos,
			String srcFile) {
		try {
			RandomAccessFile rafi = new RandomAccessFile(srcFile, "r");
			// RandomAccessFile rafo = new RandomAccessFile(destFile, "rw");

			byte buf[] = new byte[1 << 19]; // 0.5M
			int readcount;
			// long seekEndPos = 0;
			StringBuffer line = new StringBuffer();
			File f = new File(sQuitFilename);

			long start = System.currentTimeMillis();
			// int j = 0;
			rafi.seek(seekStartPos);

			// rafi.getFilePointer() <= seekEndPos means it hasn't gone to the
			// destination position
			// seekEndPos ==0 means no seekEndPos, it will go go and go...
			while (rafi.getFilePointer() <= seekEndPos || seekEndPos == 0) {
				while ((readcount = rafi.read(buf)) == -1) {
					System.out.println("No new data, begin to sleep 10s");
					Thread.sleep(10000);
					if (f.exists()) {
						// seekEndPos = rafi.getFilePointer();
						break;
					}
				}

				for (int i = 0; i < readcount; i++) {
					byte b = buf[i];
					line.append((char) b);
					if (b == '\n') {
						DetailVisitorInfo info = new DetailVisitorInfo();
						String blogId = info.getAllInfo(line.toString());
						line.delete(0, line.length());
						if (blogId == null) {
							continue;
						}
						if (blogVisitorInfo.containsKey(blogId)) {
							ArrayList iplist = (ArrayList) blogVisitorInfo
									.get(blogId);
							iplist.add(info);
						} else {
							ArrayList iplist = new ArrayList();
							iplist.add(info);
							blogVisitorInfo.put(blogId, iplist);
							blogList.add(blogId);
						}

						// rafo.write(requiredLine.getBytes());
						// line.delete(0, line.length());
					}
				}
				// if(++j % 512 == 0)
				{
					// Sleep 10s for per M byte
					// j = 0;
					long end = System.currentTimeMillis();
					System.out.println("0.5M Spend: " + (double) (end - start)
							/ 1000 + "s");
					start = end;
					// System.out.println("finished 0.5M, begin to sleep 5s");
					Thread.sleep(5000);
				}

				if (f.exists()) {
					// seekEndPos = rafi.getFilePointer();
					System.out.println("seekEndPos : " + rafi.getFilePointer());
					break;
				}
			}
			rafi.close();
			// rafo.close();

			// System.out.println("Spend:
			// "+(double)(System.currentTimeMillis()-start) / 1000 + "s");
		} catch (Exception e) {
			System.out.println(e);
		}
	}

	private static Pattern timePattern = Pattern
			.compile(".*\\[(.*)\\s\\+0800\\]");

	private static String getTimeInfo(String line) {
		String sResult = "0";
		Matcher matcher = timePattern.matcher(line);
		boolean bMatch = matcher.find();
		if (bMatch) {
			sResult = matcher.group(1);
		}

		return sResult;

	}

	private static long transferDate(String sDate) {
		// "29/Jul/2007:20:44:08"
		SimpleDateFormat DateFormat = new SimpleDateFormat(
				"dd/MMM/yyyy:HH:mm:ss", Locale.US);
		try {
			Date date = DateFormat.parse(sDate);
			return date.getTime();
		} catch (Exception ex) {
			System.out.println(ex.getMessage());
		}

		return 0;
	}

	private static String getDateFolderName(long lDate) {
		SimpleDateFormat destFormat = new SimpleDateFormat("yyyyMMdd");
		try {
			return destFormat.format(lDate);
		} catch (Exception ex) {
			System.out.println(ex.getMessage());
		}

		return "errDate";
	}

	private static String getYesterday() {
		SimpleDateFormat df = new SimpleDateFormat("dd/MMM/yyyy", Locale.US);

		Calendar c = Calendar.getInstance(); // 当时的日期和时间
		int d = c.get(Calendar.DAY_OF_MONTH);
		--d;
		c.set(Calendar.DAY_OF_MONTH, d);

		return df.format(c.getTime());
	}

}

// implements Serializable
class DetailVisitorInfo {
	// 125.118.125.155 - - [27/Jul/2007:00:00:01 +0800] "GET
	// /face/blogChain/getServices.dhtml?id=75512cba-c17c-448d-bef9-c3b50a65d557&noCache=0.209011285100132
	// HTTP/1.1" 200 779
	// private static Pattern pattern = Pattern.compile("<a
	// href=\\\"(.*)\\\".*title=\\\"(.*)\\\".*img src=\\\"(.*)\\\".*border");
	private static Pattern pattern = Pattern
			.compile("(\\d+\\.\\d+\\.\\d+\\.\\d+).*\\[(.*)\\s\\+0800\\].*getServices.*\\?id=([0-9a-z\\-]+)\\&");

	public String IP = "";

	public String time = "";

	public String addr = "";

	private IPSeeker ipseeker = IPSeeker.getInstance();

	public String getAllInfo(String line) {
		Matcher matcher = pattern.matcher(line);
		boolean bMatch = matcher.find();
		if (bMatch) {
			IP = matcher.group(1);
			time = matcher.group(2);
			addr = ipseeker.getAddress_faster(IP).replaceAll("&", "&amp;");
			return matcher.group(3);
		} else {
			return null;
		}
	}
}