discoverythread.java

监控大型网络的软件。能够自动发现拓扑结构

						boolean isGw = false;
						if (foundArp) {
							String[] newHost = new String[1];
							isGw = checkRouter(host, newHost);
							if (isGw) host = newHost[0];
						}
						Log.d("RUN", "Found new netbox: " + host + " (cs " + cs + "), gw: " + isGw);
						insertNetbox(host, cs, isGw);
						if (isGw && ipScanMode) {
							Log.d("RUN", "Disabled IP scanning mode");
							ipScanMode = false;
						}
					}
				} else {
					//Log.d("RUN", "Host " + host + " does not have an open SNMP port");
				}

				if (ip.getMac() != null) {
					if (ip.getAttempts() >= DiscoveryThread.maxMacChecks) {
						accessible = true; // Give up
						Log.d("RUN", "Giving up on host: " + ip);
					}
					updateMacDb(ip, accessible);

					if (!accessible) {
						// Insert into queue
						long nextr = ip.getNextRun() - System.currentTimeMillis();
						Log.d("RUN", "Schedule host " + ip + " to be retested in: " + nextr + " ms (" + (nextr/1000) + " s)");
						if (nextr < 0) {
							Log.d("RUN", "Neg " + nextr + ", " + ip + ", " + System.currentTimeMillis() + ", " + ip.getTime() + ", " + ip.getAttempts());
							System.err.println("Neg " + nextr + ", " + ip + ", " + System.currentTimeMillis() + ", " + ip.getTime() + ", " + ip.getAttempts());
						}
						addToRunQ(ip);
					}
				}


				Log.setNetbox(null);

				// Try to get a new IP to process
				Object o = removeRunQHead();
				if (o instanceof IP) {
					ip = (IP)o;
					//Log.d("RUN", "Got new IP: " + ip);
				} else {
					// We didn't get a netbox; exit the thread
					//Log.d("RUN", "No new IP available: " + o);
					break;
				}
			}

		} catch (Throwable e) {
			Log.e("RUN", "Caught exception, should not happen: " + e.getMessage());
			e.printStackTrace(System.err);
		} finally {
			Log.d("RUN", "Thread idle, exiting...");
			if (sSnmp != null) sSnmp.destroy();
			sSnmp = null;
			System.gc();
			Log.freeThread();
			threadIdle(tid);
		}
	}

	private String[] getNetboxInfo() {
		sSnmp.setSocketTimeout(500);

		List l;
		String sysname = "(null)";
		try {
			l = sSnmp.getAll("1.3.6.1.2.1.1.5.0", true, false);
			if (l != null && !l.isEmpty()) sysname = ((String[])l.get(0))[1];
		} catch (TimeoutException te) { }

		String sysobjectid = "(null)";
		try {
			l = sSnmp.getAll("1.3.6.1.2.1.1.2.0", false, false);
			if (l != null && !l.isEmpty()) sysobjectid = ((String[])l.get(0))[1];
		} catch (TimeoutException te) { }

		String sysdescr = "(null)";
		try {
			l = sSnmp.getAll("1.3.6.1.2.1.1.1.0", true, false);
			if (l != null && !l.isEmpty()) sysdescr = ((String[])l.get(0))[1];
			sysdescr = sysdescr.replace('\r', ' ');
			sysdescr = sysdescr.replace('\n', ',');
		} catch (TimeoutException te) { }

		return new String[] { sysname, sysobjectid, sysdescr };
	}

	private void insertNetbox(String host, String cs, boolean isGw) {
		String[] sysinfo = null;
		if (!netboxesInAllFile.contains(host) || !netboxesInNewFile.contains(host)) {
			sysinfo = getNetboxInfo();
		}

		sSnmp.setHost(host);

		synchronized (insertNetboxLock) {
			if (!netboxesInAllFile.contains(host)) {
				String dns = reverseDNS(host);
				writeBulkFile(allNetboxesBulkFile, host, isGw ? "GW" : "SW", cs, dns, sysinfo[0], sysinfo[1], sysinfo[2]);
				netboxesInAllFile.add(host);
			}

			try {
				// Check duplicate
				ResultSet rs = Database.query("SELECT netboxid FROM netbox WHERE ip='"+host+"'");
				if (rs.next()) {
					Log.d("INSERT_NETBOX", "Netbox already exists: " + host);
					return;
				}

				if (!netboxesInNewFile.contains(host)) {
					String dns = reverseDNS(host);
					writeBulkFile(newNetboxesBulkFile, host, isGw ? "GW" : "SW", cs, dns, sysinfo[0], sysinfo[1], sysinfo[2]);
					netboxesInNewFile.add(host);
				}
				incNumFound();

				if (!autoInsertNetbox) return;

				// Fetch sysLocation
				sSnmp.setSocketTimeout(500);
				String oid = getOid("sysLocation");
				List l = sSnmp.getAll(oid, true, true);
				if (l != null && !l.isEmpty()) {
					String[] s = (String[]) l.get(0);
					String location = s[1];
					String org = location;

					location = DEFAULT_LOCATIONID;
					org = DEFAULT_ORGID;
					// Look for syntax: snmp-server location <roomid> : <descr> : <utm>

					{
						rs = Database.query("SELECT roomid FROM room WHERE roomid='"+location+"'");
						if (!rs.next()) {
							Database.update("INSERT INTO room (roomid) VALUES ('"+location+"')");
						}
					}

					{
						rs = Database.query("SELECT orgid FROM org WHERE orgid='"+org+"'");
						if (!rs.next()) {
							Database.update("INSERT INTO org (orgid) VALUES ('"+org+"')");
						}
					}

					// Insert device
					String deviceid = "";
					{
						String[] ins = new String[] {
							"deviceid", ""
						};
						deviceid = Database.insert("device", ins, null);
					}
					String[] ins = new String[] {
						"ip", host,
						"roomid", location,
						"deviceid", deviceid,
						"catid", isGw ? "GW" : "SW",
						"orgid", org,
						"ro", cs,
					};
					Database.insert("netbox", ins);
					Log.d("INSERT_NETBOX", "Inserted new netbox: " + host);
				}
			} catch (Exception e) {
				e.printStackTrace(System.err);
			}
		}
	}

	private void updateMacDb(IP ip, boolean accessible) {
		synchronized (updateMacLock) {
			try {
				String mac = ip.getMac();
				ResultSet rs = Database.query("SELECT attempts FROM autodisc_mac_scanned WHERE mac='"+mac+"'");
				if (rs.next()) {
					String val = "null";
					if (!accessible) {
						ip.setAttempts(rs.getInt("attempts"));
						ip.incAttempts();
						val = "" + ip.getAttempts();
					}
					Database.update("UPDATE autodisc_mac_scanned SET attempts="+val+" WHERE mac='"+mac+"'");
				} else {
					String[] ins = new String[] {
						"mac", mac,
						"time", ""+ip.getTime(),
						"attempts", accessible ? "null" : "1",
						"found_snmp", accessible ? "true" : "false",
					};
					Database.insert("autodisc_mac_scanned", ins);
					ip.setAttempts(1);
				}
			} catch (SQLException e) {
				e.printStackTrace(System.err);
			}
		}
	}

	private boolean collectARP(String cs) {
		// Check if we really should collect arp
		try {
			/*
			ResultSet rs = Database.query("SELECT arpid FROM arp JOIN netbox USING(netboxid) WHERE arp.end_time='infinity' AND netbox.ip='"+ip.getIp()+"' LIMIT 1");
			if (rs.next()) {
				return true;
			}
			*/

			// Ok, try to collect ARP
			sSnmp.setSocketTimeout(500);
			String oid = getOid("ipNetToMediaPhysAddress");
			List l = sSnmp.getAll(oid, false, true);
			int newIps = 0;
			if (l != null && !l.isEmpty()) {
				for (Iterator it = l.iterator(); it.hasNext();) {
					String[] s = (String[])it.next();
					String host = s[0].substring(s[0].indexOf(".")+1, s[0].length());
					if (isLoopback(host)) continue;
					String mac = util.remove(s[1], ":").toLowerCase();
					if (seenMacs.contains(mac)) continue; // No need to check macs more than once
					IP newip = new IP(host, mac);
					addToRunQ(newip);
					newIps++;
				}
				Log.d("COLLECT_ARP", "Found " + newIps + " new IPs from ARP (" + l.size() + " fetched)");
				return true;
			}

		} catch (TimeoutException te) {
			// Do nothing, this is normal
		} catch (Exception e) {
			e.printStackTrace(System.err);
		}
		return false;
	}

	private boolean checkRouter(String host, String[] newHost) {
		try {
			List gwipList = sSnmp.getAll(getOid("ipAdEntIfIndex"), false, true);
			Map ifdescrMap = sSnmp.getAllMap(getOid("ifDescr"), true);

			if (gwipList != null && ifdescrMap != null) {
				String loopback = null;
				long lowgwip = Long.MAX_VALUE;
				for (Iterator it = gwipList.iterator(); it.hasNext();) {
					String[] s = (String[])it.next();
					String gwip = s[0];
					String ifindex = s[1];
					String ifdescr = (String) ifdescrMap.get(ifindex);

					gwip = validateIp(gwip);
					if (gwip == null) continue;

					if (ifdescr == null) continue;
					if (isLoopback(gwip)) continue;

					if (runningSNMP(gwip)) {
						try {
							sSnmp.setHost(gwip);
							sSnmp.setSocketTimeout(200);
							List newGwipList = sSnmp.getAll(getOid("ipAdEntIfIndex"), false, true);
							if (!isListEqual(gwipList, newGwipList)) continue;
						} catch (TimeoutException te) {
							continue;
						} finally {
							sSnmp.setHost(host);
							sSnmp.setSocketTimeout(500);
						}
					}

					if (ifdescr.toLowerCase().indexOf("loopback") >= 0) {
						// This is the IP we want
						loopback = gwip;
						break;
					}
					if (host.equals(gwip)) continue;

					long num = ipToNum(gwip);
					if (num < lowgwip) {
						loopback = gwip;
					}
				}
				if (loopback != null) {
					newHost[0] = loopback;
					return true;
				}

			}
		} catch (TimeoutException te) {
		} catch (Exception e) {
			e.printStackTrace(System.err);
		}
		return false;
	}

	private static String validateIp(String ip) {
		String[] s = ip.split("\\.");
		if (s.length < 4) return null;
		ip = "";
		for (int i=0; i < 4; i++) {
			try {
				int n = Integer.parseInt(s[i]);
				if (n < 0 || n > 255) return null;
				ip += n + ".";
			} catch (NumberFormatException e) {
			}
		}
		return ip.substring(0, ip.length()-1);
	}

	private static boolean isListEqual(List l1, List l2) {
		if (l1 == null || l2 == null) return false;
		if (l1.size() != l2.size()) return false;
		for (Iterator it = l1.iterator(), it2 = l2.iterator(); it.hasNext();) {
			String[] s1 = (String[]) it.next();
			String[] s2 = (String[]) it2.next();
			if (!s1[0].equals(s2[0]) || !s1[1].equals(s2[1])) return false;
		}
		return true;
	}

	private static boolean isLoopback(String ip) {
		if (ip.equals("0.0.0.0") || ip.startsWith("127")) return true;
		return false;
	}

	private String getOid(String oidkey) {
		try {
			ResultSet rs = Database.query("SELECT snmpoid FROM snmpoid WHERE oidkey='" + oidkey + "'");
			if (rs.next()) {
				return rs.getString("snmpoid");
			}
		} catch (SQLException e) {
			e.printStackTrace(System.err);
		}
		return null;
	}


	private static class CheckRunQTask extends TimerTask {
		public void run() {
			Log.setDefaultSubsystem("CHECK_RUN_Q");
			checkRunQ();
		}
	}

	private static class UpdateFromARPTask extends TimerTask {
		public void run() {
			updateFromARP();
		}
	}

	private static class IP {
		public String ip;
		public String mac;
		public long baseTime;
		public int attempts;

		public IP(String _ip, String _mac) {
			this(_ip, _mac, System.currentTimeMillis(), 0);
		}

		public IP(String _ip, String _mac, long _baseTime, int _attempts) {
			ip = _ip;
			mac = _mac;
			baseTime = _baseTime;
			attempts = _attempts;
		}

		public String getIp() {
			return ip;
		}

		public String getMac() {
			return mac;
		}

		public long getTime() {
			return baseTime;
		}

		public void setAttempts(int _attempts) {
			attempts = _attempts;
		}

		public void incAttempts() {
			attempts++;
		}

		public int getAttempts() {
			return attempts;
		}

		public long getNextRun() {
			long next = baseTime;
			if (attempts-1 >= 0) {
				next += (long) (Math.pow(DiscoveryThread.baseCheckInterval, Math.pow(DiscoveryThread.checkMultiplier, attempts-1)) * 1000);
			}
			return next;
		}

		public String toString() {
			return ip;
		}
	}

	private static String format(long i, int n)
	{
		DecimalFormat nf = new DecimalFormat("#");
		nf.setMinimumIntegerDigits(n);
		return nf.format(i);
	}

	// DNS Check code
	private static String[] hostBinCandidates = {
		"/bin/host",
		"/sbin/host",
		"/usr/bin/host",
		"/usr/sbin/host",
		"/usr/local/bin/host",
		"/usr/local/sbin/host",
	};
	private static File hostBin;
	private static Map dnsMap = Collections.synchronizedMap(new HashMap());

	private String reverseDNS(String ip) {
		if (dnsMap.containsKey(ip)) {
			return (String) dnsMap.get(ip);
		}

		// Do DNS lookup
		InetAddress ia = null;
		try {
			ia = InetAddress.getByName(ip);
		} catch (UnknownHostException e) {
			// Cannot happen as we always supply an IP address
		}
		String dnsName = ia.getCanonicalHostName();
		if (dnsName.equals(ip)) {
			dnsName = doHostReverseDNS(ip);
			if (dnsName.equals(ip)) {
				// DNS lookup failed
			}
		}
		dnsMap.put(ip, dnsName);
		return dnsName;
	}

	private String doHostReverseDNS(String ip) {
		try {
			if (hostBin == null) {
				for (int i=0; i < hostBinCandidates.length; i++) {
					File f = new File(hostBinCandidates[i]);
					if (f.exists() && f.isFile()) {
						hostBin = f;
						break;
					}
				}
			}

			String[] hostCmd = {
				hostBin.getAbsolutePath(),
				ip
			};

			Runtime rt = Runtime.getRuntime();
			Process p = rt.exec(hostCmd);
			BufferedInputStream in = new BufferedInputStream(p.getInputStream());
			try {
				p.waitFor();
			} catch (InterruptedException e) {
				System.err.println("InterruptedException: " + e);
				e.printStackTrace(System.err);
				return ip;
			}

			byte[] b = new byte[1024];
			in.read(b, 0, 1024);
			String s = new String(b).trim();

			// Check if found
			if (s.indexOf("not found") >= 0) return ip;

			// Extract DNS name
			String pat = ".*domain name pointer +(\\S{3,})";

			if (s.matches(pat)) {
				Matcher m = Pattern.compile(pat).matcher(s);
				m.matches();
				String host = m.group(1);
				if (host.endsWith(".")) host = host.substring(0, host.length()-1);
				return host;
			}

		} catch (IOException e) {
			System.err.println("IOException: " + e);
			e.printStackTrace(System.err);
		}
		return ip;
	}


}