datastore.java

这是外国一个开源推理机

	 * specified amount. Data with a reference count equal to <tt>0</tt> is subject
	 * to being garbage collected. This method can only be called as part of a
	 * transaction.
	 *
	 * @param id The ID of the data.
	 * @param amount The (positive or negative) amount to change the reference count.
	 * @exception IOException If an I/O error occurred.
	 * @see #startTransaction
	 * @see #commitTransaction
	 **/
	public void changeReferenceCount(int id, int amount)
		throws IOException
	{
		// Check MRU cache
		DataItem dataItem = _getCachedDataItem(id);
		if (dataItem != null) {
			dataItem.increaseCount(amount);
			return;
		}

		// Value not found in cache, check file

		long offset = _idFile.getOffset(id, true);

		if (offset != 0L) {
			byte[] combinedData = _dataFile.getData(offset, true);

			// Decrease count by one and store it in the cache
			int count = _getCountFromCombinedArray(combinedData);
			byte[] data = _getDataFromCombinedArray(combinedData);

			dataItem = new DataItem(count, id, data, offset);
			dataItem.increaseCount(amount);

			_cacheDataItem(dataItem);
		}
	}

	/**
	 * Removes all values from the DataStore. This method can only be called as
	 * part of a transaction. The DataStore will not be cleared until the
	 * transaction is committed.
	 *
	 * @exception IOException If an I/O error occurred.
	 * @see #startTransaction
	 * @see #commitTransaction
	 **/
	public void clear()
		throws IOException
	{
		synchronized (_mruCache) {
			_mruCache.clear();
		}

		_hashFile.clear();
		_idFile.clear();
		_dataFile.clear();
	}

	/**
	 * Closes the DataStore, releasing any file references, etc. In case a
	 * transaction is currently open, it will be rolled back. Once closed, the
	 * DataStore can no longer be used.
	 *
	 * @exception IOException If an I/O error occurred.
	 **/
	public void close()
		throws IOException
	{
		_hashFile.close();
		_idFile.close();
		_dataFile.close();
	}

	private int _getHash(byte[] data) {
		synchronized (_crc32) {
			_crc32.update(data);
			int crc = (int)_crc32.getValue();
			_crc32.reset();
			return crc;
		}
	}

	/**
	 * Fetches the DataItem for the specified data from the cache.
	 **/
	private DataItem _getCachedDataItem(byte[] data) {
		synchronized (_mruCache) {
			Iterator iter = _mruCache.iterator();
			while (iter.hasNext()) {
				DataItem dataItem = (DataItem)iter.next();

				if (Arrays.equals(data, dataItem.data)) {
					// Cache hit, move hit to most-recent position
					iter.remove();
					_mruCache.addFirst(dataItem);

					// Return the matching DataItem
					return dataItem;
				}
			}
		}

		return null;
	}

	/**
	 * Fetches the DataItem with the specified ID from the cache.
	 **/
	private DataItem _getCachedDataItem(int id) {
		synchronized (_mruCache) {
			Iterator iter = _mruCache.iterator();
			while (iter.hasNext()) {
				DataItem dataItem = (DataItem)iter.next();

				if (id == dataItem.id) {
					// Cache hit, move hit to most-recent position
					iter.remove();
					_mruCache.addFirst(dataItem);

					// Return the matching DataItem
					return dataItem;
				}
			}
		}

		return null;
	}

	/**
	 * Stores the supplied DataItem in the cache.
	 **/
	private void _cacheDataItem(DataItem dataItem)
		throws IOException
	{
		synchronized (_mruCache) {
			if (_mruCache.size() >= MRU_CACHE_SIZE) {
				// Cache full, remove least-recently used data item
				DataItem expDI = (DataItem)_mruCache.removeLast();
				if (expDI.isDirty) {
					// Sync to file
					_dataFile.updateData(expDI.offset, _createCombinedArray(expDI.count, expDI.id, expDI.data));
				}
			}

			// Add the supplied DataItem to the cache
			_mruCache.addFirst(dataItem);
		}
	}

/*--------------------------------------+
| Methods for combining/filtering an ID |
| and data in/from a single byte array  |
+--------------------------------------*/

	/**
	 * Creates a combined data array that contains the supplied count, ID and data.
	 **/
	private static byte[] _createCombinedArray(int count, int id, byte[] data) {
		byte[] result = new byte[8+data.length];
		ByteArrayUtil.putInt(count, result, 0);
		ByteArrayUtil.putInt(id, result, 4);
		ByteArrayUtil.put(data, result, 8);
		return result;
	}

	private static int _getCountFromCombinedArray(byte[] data) {
		return ByteArrayUtil.getInt(data, 0);
	}

	/**
	 * Gets the ID from a combined data array.
	 **/
	private static int _getIdFromCombinedArray(byte[] data) {
		return ByteArrayUtil.getInt(data, 4);
	}

	/**
	 * Gets the data from a combined data array.
	 **/
	private static byte[] _getDataFromCombinedArray(byte[] data) {
		return ByteArrayUtil.get(data, 8);
	}

/*-------------------+
| Test/debug methods |
+-------------------*/

	public static void main(String[] args)
		throws Exception
	{
		if (args.length == 2) {
			_printContents(args);
		}
		else {
			_test(args);
		}
	}

	private static void _test(String[] args)
		throws Exception
	{
		System.out.println("Running DataStore test...");
		File dataDir = new File(args[0]);
		DataStore dataStore = new DataStore(dataDir, "strings");

		System.out.println("Creating strings...");
		int stringCount = Integer.parseInt(args[1]);
		String[] strings = new String[stringCount];
		for (int i = 0; i < stringCount; i++) {
			strings[i] = String.valueOf(i+1);
		}

		System.out.println("Starting transaction...");
		long startTime = System.currentTimeMillis();
		dataStore.startTransaction();

		for (int i = 0; i < strings.length; i++) {
			String s = strings[i];
			byte[] sBytes = s.getBytes();
			int sID = dataStore.getID(sBytes, true);
			if (sID == 0) {
				sID = dataStore.storeData(sBytes);
			}
		}

		System.out.println("Commiting transaction...");
		dataStore.commitTransaction();
		long endTime = System.currentTimeMillis();
		System.out.println("Transaction finished in " + (endTime-startTime) + " ms");

		System.out.println("Fetching IDs for all strings...");
		startTime = System.currentTimeMillis();

		for (int i = 0; i < strings.length; i++) {
			String s = strings[i];
			int sID = dataStore.getID(s.getBytes());
			if (sID != i+1) {
				System.out.println("Unexpected ID for string \""+s+"\": " + sID);
			}
		}

		endTime = System.currentTimeMillis();
		System.out.println("All IDs fetched in " + (endTime-startTime) + " ms");

		System.out.println("Fetching data for all IDs...");
		startTime = System.currentTimeMillis();

		for (int i = 0; i < strings.length; i++) {
			String s = new String(dataStore.getData(i+1));
			if (!s.equals(strings[i])) {
				System.out.println("Unexpected string for ID " + (i+1) + ": \""+s+"\"");
			}
		}

		endTime = System.currentTimeMillis();
		System.out.println("All data fetched in " + (endTime-startTime) + " ms");

		System.out.println("Closing DataStore...");
		dataStore.close();
		System.out.println("Done.");
	}

	private static void _printContents(String[] args)
		throws Exception
	{
		System.out.println("Dumping DataStore contents...");
		File dataDir = new File(args[0]);
		DataStore dataStore = new DataStore(dataDir, args[1]);

		DataFile.DataIterator iter = dataStore._dataFile.iterator();
		while (iter.hasNext()) {
			byte[] combinedData = iter.next();

			int count = _getCountFromCombinedArray(combinedData);
			int id = _getIdFromCombinedArray(combinedData);
			byte[] data = _getDataFromCombinedArray(combinedData);

			System.out.println("id="+id+" count="+count+": " + ByteArrayUtil.toHexString(data));
		}
	}

	private static class DataItem {

		public int count;
		public int id;
		public byte[] data;
		public long offset;
		public boolean isDirty;

		public DataItem(int count, int id, byte[] data, long offset) {
			this.count = count;
			this.id = id;
			this.data = data;
			this.offset = offset;
			this.isDirty = false;
		}

		/**
		 * Increases the reference count for this data item with the specified
		 * amount, which can be negative to decrease the count.
		 **/
		public void increaseCount(int amount) {
			count += amount;
			isDirty = true;
		}
	}
}