parsev3pmdatafile.java

解析特定16进制文件

			String[] keys = recordOrientedMap.keySet().toArray(new String[0]);
			// 注意不要直接使用recordOrientedMap.keySet()来循环key
			// 因为在循环类要删除相应的key
			for (String key : keys) {
				List<GuideLineData> guidLineDataList = recordOrientedMap
						.get(key);
				// 及时清除引用.
				recordOrientedMap.remove(key);
				insert(conn, guidLineDataList);
				// 及时清除引用,让jvm尽快回收,因为已经到数据库插入的队列里了
				guidLineDataList.clear();
			}

			executeBatch();

		} catch (DatabaseException ex) {
			throw ex;
		} finally {
			recordOrientedMap.clear();
		}
	}

	private void executeBatch() throws DatabaseException {
		// 执行批量入库
		Collection<PreparedStatement> statements = preparedStatementMap
				.values();
		for (PreparedStatement tempP : statements) {
			try {
				tempP.executeBatch();
				tempP.clearBatch();
				// tempP = null;
			} catch (SQLException e) {
				log.error("close V3 PreparedStatement error", e);
			}
		}
		// reset
		batchCounter = 0;
	}

	private PreparedStatement getPreparedStatement(GuideLineData data,
			String tableName, Collection<GuideLineData> pmdata)
			throws DatabaseException {

		// 查找SQL缓存
		PreparedStatement p = null;
		String key = null;
		StringBuilder sqlStrBuilder = new StringBuilder();
		sqlStrBuilder.append(tableName);
		for (GuideLineData guideLine : pmdata) {
			sqlStrBuilder.append(guideLine.getCounterGroupId());
			sqlStrBuilder.append('_');
			sqlStrBuilder.append(guideLine.getNvs().size());
		}
		key = sqlStrBuilder.toString();
		p = preparedStatementMap.get(key);
		if (p != null) {
			return p;
		}

		// 如果缓存中有超过100个statement,就应该清空,或者怀疑该方法是不是好方法
		if (preparedStatementMap.size() > 100) {
			log.warn(this.getClass(),
					"The PreparedStatement for v3 performance data file is:"
							+ preparedStatementMap.size());
			Collection<PreparedStatement> statements = preparedStatementMap
					.values();
			for (PreparedStatement tempP : statements) {
				try {
					tempP.executeBatch();
					tempP.close();
					// tempP = null;
				} catch (SQLException e) {
					log.error("close V3 PreparedStatement error", e);
				}
			}
			preparedStatementMap.clear();
		}

		// 重新组织sql
		StringBuilder strBuilder = new StringBuilder();
		StringBuilder strBuilder2 = new StringBuilder();
		strBuilder.append("insert into ");
		strBuilder.append(tableName);
		strBuilder.append("(dn,begin_time,end_time");

		for (NameValue nv : data.getNvs()) {
			strBuilder.append("," + nv.getName());
			strBuilder2.append(",?");
		}

		strBuilder.append(") values(?,?,?");
		strBuilder.append(strBuilder2);
		strBuilder.append(")");
		String sql = strBuilder.toString();

		log.debug(this.getClass(), "Create sql:" + sql);
		p = preparedStatementMap.get(sql);
		if (p == null) {
			// log.info(getClass(),"create p counter id " +
			// counterGroupId);
			try {
				p = conn.prepareStatement(sql);
				preparedStatementMap.put(key, p);

			} catch (SQLException e) {
				throw new DatabaseException(e);
			}
		}

		return p;

	}

	/**
	 * 合并计数器组并插入数据.这样可以节约数据库存储空间
	 * 
	 * @param conn
	 * @param pmdata
	 * @throws DatabaseException
	 */
	private void insert(Connection conn, Collection<GuideLineData> pmdata)
			throws DatabaseException {

		/** 变量定义 * */
		int idx = 0;
		PreparedStatement p = null;
		String tableName = null;
		// int counterGroupId = 0;

		GuideLineData dummyData = new GuideLineData();
		dummyData.setCounterGroupId(-1);
		List<NameValue> nvs = new ArrayList<NameValue>();
		// 记录22组的值名对
		List<NameValue> nvs22Group = new ArrayList<NameValue>();
		// 合并所有计数器到一个虚拟的计数器组里
		boolean copyCommonData = false;
		for (GuideLineData smallData : pmdata) {
			if (!copyCommonData) {// 用于向虚拟计数器组设置基本参数
				dummyData
						.setBegin(smallData.getBegin() == null ? defaultBeginTime
								: smallData.getBegin());
				dummyData.setEnd(smallData.getEnd() == null ? defaultEndTime
						: smallData.getEnd());
				dummyData.setDn(smallData.getDn());
				/** 根据计数器组id得到对应的数据表名 */
				tableName = TableConfig.getInstance().getTableName(
						smallData.getCounterGroupId());
				copyCommonData = true;
			}
			nvs.addAll(smallData.getNvs());
			if (smallData.getCounterGroupId() == 22)
				nvs22Group.addAll(smallData.getNvs());
		}
		// 排序主要目的是可以每次获取相同的sql,从而通过sql获取缓存的statement
		Collections.sort(nvs);
		dummyData.setNvs(nvs);
		// 从缓存中获取statement
		p = getPreparedStatement(dummyData, tableName, pmdata);
		/** 对每个虚拟GuideLineData执行入库 * */

		/**
		 * 从groupId和TableSql映射关系表查找是否有此计数器组的sql语句 modify by liuyuan 2008-5-30
		 * 增加不同版本的RNC时，同一个计数器组上报计数器的个数不一致 CRMCN00003777
		 */

		// 因为前面三个是固定的
		idx = 4;
		try {
			// modify by liuyuan 2007-11-6 CRMCN00002983
			// 【内部需求】性能报表优化，V3的PC需要对22组计数器的值进行换算后入库,对于无效的计数器值，判断是否等于1
			p.setString(1, dummyData.getDn());
			p.setTimestamp(2, new java.sql.Timestamp(dummyData.getBegin()
					.getTime()));
			p.setTimestamp(3, new java.sql.Timestamp(dummyData.getEnd()
					.getTime()));

			for (NameValue nv : dummyData.getNvs()) {
				if (nvs22Group.contains(nv)) {
					if (nv.getTypeOfLimit() != 1) {
						p.setDouble(idx++, nv.getValue());
					}
					// 无效的计数器值设置为Null，方便统计
					else {
						p.setNull(idx++, Types.DOUBLE);
					}
				} else
					p.setDouble(idx++, nv.getValue());
			}
			// 及时清除引用,namevalue是局部变量
			dummyData.getNvs().clear();
			dummyData = null;
			for (GuideLineData smallData : pmdata) {
				smallData.getNvs().clear();
			}
			// 支持批量更新
			p.addBatch();
			batchCounter++;
			// 要及时执行batch,否则内存会溢出
			if (batchCounter >= 60) {
				executeBatch();
			}

		} catch (SQLException e) {
			Logger.getLogger(DefaultPmDataInput.class).error(
					"数据库异常 ！ " + e.getMessage(), e);
			throw new DatabaseException("数据库异常 ！ " + e.getMessage(), e);
		} catch (Exception e) {
			Logger.getLogger(DefaultPmDataInput.class).error(
					"the other error ! " + e.getMessage(), e);
			throw new DatabaseException("the other error ! " + e.getMessage(),
					e);
		} finally {

		}
	}

	/**
	 * 插入TBL_GUIDE_LINE_DATA_RECENT，以便性能服务器可以判断是否有延迟告警
	 */
	private void insertLatecyInfo(Connection conn, File file, Integer rncId) {
		String neDn = "RAN=" + rncId + ",rRnc=" + rncId;

		// 由于一个文件中，最有可能的是只含有一个时间段，
		// 因此将Map<Integer, Set<Date>>转换为Map<Date,Set<Integer>>，再入库
		Map<Date, Set<Integer>> dateToGroupMap = new HashMap<Date, Set<Integer>>();

		for (Map.Entry<Integer, Set<Date>> entry : v3CounterGroupMap.entrySet()) {
			int v3Group = entry.getKey();
			Set<Date> endTimeSet = entry.getValue();
			if (endTimeSet == null || endTimeSet.isEmpty()) {
				log.error(ParseV3PMDataFile.class,
						"V3 endTimeSet is null or empty。file name: "
								+ file.getName());
				return;
			}

			for (Date endTime : endTimeSet) {
				if (dateToGroupMap.containsKey(endTime)) {
					dateToGroupMap.get(endTime).add(v3Group);
				} else {
					Set<Integer> v3GroupSet = new HashSet<Integer>();
					v3GroupSet.add(v3Group);
					dateToGroupMap.put(endTime, v3GroupSet);
				}
			}
		}

		for (Map.Entry<Date, Set<Integer>> entry : dateToGroupMap.entrySet()) {

			Date endTime = entry.getKey();
			Set<Integer> v3GroupSet = entry.getValue();
			int[] v3GroupArray = new int[v3GroupSet.size()];
			int i = 0;
			for (Integer v3Group : v3GroupSet) {
				v3GroupArray[i] = v3Group;
				i++;
			}

			insertLatecyInfo.insert(conn, neDn, endTime, v3GroupArray);
		}

	}

	/**
	 * 将文件重新放入解析队列中处理
	 */
	private void reputParseQueue(FileInfo fileInfo, int errorCode) {
		fileInfo.setExceptionInfo(new Date(), errorCode);
		// 放入队列中准备重新解析
		ParseFileExceptionQueue.getInstance().putParseFile(fileInfo);
	}

	/**
	 * @param filein
	 * @param datain
	 * @throws IOException
	 */
	private void closeFile(FileInputStream filein, DataInputStream datain) {

		try {

			if (datain != null) {
				datain.close();
			}
			if (filein != null) {
				filein.close();
			}
		} catch (IOException ex) {
			log.error("close v3 file error after parse file", ex);
		}
	}

	/**
	 * 向以记录为导向的Map存储指标
	 * 
	 * @param data
	 */
	private void saveToRecordOrientedMap(GuideLineData data)
			throws DatabaseException {
		String tableName = TableConfig.getInstance().getTableName(
				data.getCounterGroupId());
		if (tableName == null) {
			String errorMessage = "Can't get table name from groupid:"
					+ data.getCounterGroupId()
					+ "\nPlease check \"config/pmgrouptable.xml\"";
			log.error(ParseV3PMDataFile.class, errorMessage);
			throw new DatabaseException(errorMessage);
		}
		int counterGroupNumber = TableConfig.getInstance()
				.getCounterNumByTableName(tableName);
		// 用表名,dn还有开始结束时间来生成Map的key
		StringBuilder stringBuilder = new StringBuilder();
		Date beginTime = data.getBegin() == null ? defaultBeginTime : data
				.getBegin();
		Date endTime = data.getEnd() == null ? defaultEndTime : data.getEnd();
		stringBuilder.append(tableName).append(data.getDn()).append(
				beginTime.getTime() + endTime.getTime());
		String key = stringBuilder.toString();
		List<GuideLineData> guideLineList = recordOrientedMap.get(key);
		if (guideLineList == null) {
			// 该list的记录数不会很多,不会超过10条,而且平均在5条左右,有时甚至只有1条数据
			guideLineList = new ArrayList<GuideLineData>(1);
			guideLineList.add(data);
			recordOrientedMap.put(key, guideLineList);
		} else {
			guideLineList.add(data);
		}
		// 在这里判断是不是该记录已经满了,如果满了马上入库,这样可以节约存储空间

		boolean tableChanged = false;
		String dn = data.getDn();
		String oldTableName = tableAndDnMap.get(dn);
		if (oldTableName == null) {
			oldTableName = tableName;
			tableAndDnMap.put(dn, tableName);
			tableChanged = false;
		} else {
			if (oldTableName.equals(tableName)) {
				tableChanged = false;
			} else {
				tableChanged = true;
			}
		}
		if (guideLineList.size() == counterGroupNumber || tableChanged) {
			// log.debug(this.getClass(),"该条记录已满,插入数据库,并清除缓存");
			insert(conn, guideLineList);
			// 尽快清除缓存中已经如何的guidelinedata,尽可能节约内存
			recordOrientedMap.remove(key);
			// 尽快清除该dn的table信息,尽可能节约内存
			tableAndDnMap.remove(dn);
		}
	}

	/**
	 * 用于返回是否处于忙状态
	 * 
	 * @return
	 */
	public boolean isBusy() {
		return busy;
	}

	/**
	 * 不让外部类来设置该值
	 * 
	 * @param busy
	 */
	private void setBusy(boolean busy) {
		this.busy = busy;
	}

}