bitstreamstoragemanager.java

dspace 用j2ee架构的一个数字图书馆.开源程序

		// To remain compatible with DSpace we calculate an MD5 checksum on 
		// LOCAL registered files. But for REMOTE (e.g. SRB) files we 
		// calculate an MD5 on just the fileNAME. The reasoning is that in the 
		// case of a remote file, calculating an MD5 on the file itself will
		// generate network traffic to read the file's bytes. In this case it 
		// would be better have a proxy process calculate MD5 and store it as 
		// an SRB metadata attribute so it can be retrieved simply from SRB.
		//
		// TODO set this up as a proxy server process so no net activity
		
		// FIXME this is a first class HACK! for the reasons described above
		if (file instanceof LocalFile) 
		{

			// get MD5 on the file for local file
			DigestInputStream dis = null;
			try 
			{
				dis = new DigestInputStream(FileFactory.newFileInputStream(file), 
						MessageDigest.getInstance("MD5"));
			} 
			catch (NoSuchAlgorithmException e) 
			{
				log.warn("Caught NoSuchAlgorithmException", e);
				throw new IOException("Invalid checksum algorithm");
			}
			catch (IOException e) 
			{
				log.error("File: " + file.getAbsolutePath() 
						+ " to be registered cannot be opened - is it "
						+ "really there?");
				throw e;
			}
			final int BUFFER_SIZE = 1024 * 4;
			final byte[] buffer = new byte[BUFFER_SIZE];
			while (true) 
			{
				final int count = dis.read(buffer, 0, BUFFER_SIZE);
				if (count == -1) 
				{
					break;
				}
			}
			bitstream.setColumn("checksum", Utils.toHex(dis.getMessageDigest()
					.digest()));
			dis.close();
		} 
		else if (file instanceof SRBFile)
		{
			if (!file.exists())
			{
				log.error("File: " + file.getAbsolutePath() 
						+ " is not in SRB MCAT");
				throw new IOException("File is not in SRB MCAT");
			}

			// get MD5 on just the filename (!) for SRB file
			int iLastSlash = bitstreamPath.lastIndexOf('/');
			String sFilename = bitstreamPath.substring(iLastSlash + 1);
			MessageDigest md = null;
			try 
			{
				md = MessageDigest.getInstance("MD5");
			} 
			catch (NoSuchAlgorithmException e) 
			{
				log.error("Caught NoSuchAlgorithmException", e);
				throw new IOException("Invalid checksum algorithm");
			}
			bitstream.setColumn("checksum", 
					Utils.toHex(md.digest(sFilename.getBytes())));
		}
		else
		{
			throw new IOException("Unrecognized file type - "
					+ "not local, not SRB");
		}

		bitstream.setColumn("checksum_algorithm", "MD5");
		bitstream.setColumn("size_bytes", (int) file.length());
		bitstream.setColumn("deleted", false);
		DatabaseManager.update(context, bitstream);

		int bitstream_id = bitstream.getIntColumn("bitstream_id");
		if (log.isDebugEnabled()) 
		{
			log.debug("Stored bitstream " + bitstream_id + " in file "
					+ file.getAbsolutePath());
		}
		return bitstream_id;
	}

	/**
	 * Does the internal_id column in the bitstream row indicate the bitstream
	 * is a registered file
	 *
	 * @param internalId the value of the internal_id column
	 * @return true if the bitstream is a registered file
	 */
	public static boolean isRegisteredBitstream(String internalId) {
	    if (internalId.substring(0, REGISTERED_FLAG.length())
	            .equals(REGISTERED_FLAG)) 
	    {
	        return true;
	    }
	    return false;
	}

    /**
     * Retrieve the bits for the bitstream with ID. If the bitstream does not
     * exist, or is marked deleted, returns null.
     * 
     * @param context
     *            The current context
     * @param id
     *            The ID of the bitstream to retrieve
     * @exception IOException
     *                If a problem occurs while retrieving the bits
     * @exception SQLException
     *                If a problem occurs accessing the RDBMS
     * 
     * @return The stream of bits, or null
     */
    public static InputStream retrieve(Context context, int id)
            throws SQLException, IOException
    {
        TableRow bitstream = DatabaseManager.find(context, "bitstream", id);

		GeneralFile file = getFile(bitstream);

		return (file != null) ? FileFactory.newFileInputStream(file) : null;
    }

    /**
     * <p>
     * Remove a bitstream from the asset store. This method does not delete any
     * bits, but simply marks the bitstreams as deleted (the context still needs
     * to be completed to finalize the transaction).
     * </p>
     * 
     * <p>
     * If the context is aborted, the bitstreams deletion status remains
     * unchanged.
     * </p>
     * 
     * @param context
     *            The current context
     * @param id
     *            The ID of the bitstream to delete
     * @exception SQLException
     *                If a problem occurs accessing the RDBMS
     */
    public static void delete(Context context, int id) throws SQLException
    {
        DatabaseManager
                .updateQuery(context,
                        "update Bitstream set deleted = '1' where bitstream_id = "
                                + id);
    }

    /**
     * Clean up the bitstream storage area. This method deletes any bitstreams
     * which are more than 1 hour old and marked deleted. The deletions cannot
     * be undone.
     * 
     * @param deleteDbRecords if true deletes the database records otherwise it
     * 	           only deletes the files and directories in the assetstore  
     * @exception IOException
     *                If a problem occurs while cleaning up
     * @exception SQLException
     *                If a problem occurs accessing the RDBMS
     */
    public static void cleanup(boolean deleteDbRecords) throws SQLException, IOException
    {
        Context context = null;
        BitstreamInfoDAO bitstreamInfoDAO = new BitstreamInfoDAO();

        try
        {
            context = new Context();

            String myQuery = "select * from Bitstream where deleted = '1'";

            List storage = DatabaseManager.query(context, "Bitstream", myQuery)
                    .toList();

            for (Iterator iterator = storage.iterator(); iterator.hasNext();)
            {
                TableRow row = (TableRow) iterator.next();
                int bid = row.getIntColumn("bitstream_id");

				GeneralFile file = getFile(row);

                // Make sure entries which do not exist are removed
                if (file == null)
                {
                    log.debug("file is null");
                    if (deleteDbRecords)
                    {
                        log.debug("deleting record");
                        bitstreamInfoDAO.deleteBitstreamInfoWithHistory(bid);
                        DatabaseManager.delete(context, "Bitstream", bid);
                    }
                    continue;
                }

                // This is a small chance that this is a file which is
                // being stored -- get it next time.
                if (isRecent(file))
                {
                	log.debug("file is recent");
                    continue;
                }

                if (deleteDbRecords)
                {
                    log.debug("deleting db record");
                    bitstreamInfoDAO.deleteBitstreamInfoWithHistory(bid);
                    DatabaseManager.delete(context, "Bitstream", bid);
                }

				if (isRegisteredBitstream(row.getStringColumn("internal_id"))) {
				    continue;			// do not delete registered bitstreams
				}

                boolean success = file.delete();

                if (log.isDebugEnabled())
                {
                    log.debug("Deleted bitstream " + bid + " (file "
                            + file.getAbsolutePath() + ") with result "
                            + success);
                }

                // if the file was deleted then
                // try deleting the parents
                // Otherwise the cleanup script is set to 
                // leave the db records then the file
                // and directories have already been deleted
                // if this is turned off then it still looks like the
                // file exists
                if( success )
                {
                    deleteParents(file);
                }
            }

            context.complete();
        }
        // Aborting will leave the DB objects around, even if the
        // bitstreams are deleted. This is OK; deleting them next
        // time around will be a no-op.
        catch (SQLException sqle)
        {
            context.abort();
            throw sqle;
        }
        catch (IOException ioe)
        {
            context.abort();
            throw ioe;
        }
    }

    ////////////////////////////////////////
    // Internal methods
    ////////////////////////////////////////

    /**
     * Return true if this file is too recent to be deleted, false otherwise.
     * 
     * @param file
     *            The file to check
     * @return True if this file is too recent to be deleted
     */
    private static boolean isRecent(GeneralFile file)
    {
        long lastmod = file.lastModified();
        long now = new java.util.Date().getTime();

        if (lastmod >= now)
        {
            return true;
        }

        // Less than one hour old
        return (now - lastmod) < (1 * 60 * 1000);
    }

    /**
     * Delete empty parent directories.
     * 
     * @param file
     *            The file with parent directories to delete
     */
    private synchronized static void deleteParents(GeneralFile file)
    {
        if (file == null )
        {
            return;
        }
 
		GeneralFile tmp = file;

        for (int i = 0; i < directoryLevels; i++)
        {

			GeneralFile directory = tmp.getParentFile();
			GeneralFile[] files = directory.listFiles();

            // Only delete empty directories
            if (files.length != 0)
            {
                break;
            }

            directory.delete();
            tmp = directory;
        }
    }

    /**
     * Return the file corresponding to a bitstream. It's safe to pass in
     * <code>null</code>.
     * 
     * @param bitstream
     *            the database table row for the bitstream. Can be
     *            <code>null</code>
     * 
     * @return The corresponding file in the file system, or <code>null</code>
     * 
     * @exception IOException
     *                If a problem occurs while determining the file
     */
    private static GeneralFile getFile(TableRow bitstream) throws IOException
    {
        // Check that bitstream is not null
        if (bitstream == null)
        {
            return null;
        }

        // Get the store to use
        int storeNumber = bitstream.getIntColumn("store_number");

        // Default to zero ('assetstore.dir') for backwards compatibility
        if (storeNumber == -1)
        {
            storeNumber = 0;
        }

		GeneralFile assetstore = assetStores[storeNumber];

		// turn the internal_id into a file path relative to the assetstore
		// directory
		String sInternalId = bitstream.getStringColumn("internal_id");

		// there are 4 cases:
		// -conventional bitstream, conventional storage
		// -conventional bitstream, srb storage
		// -registered bitstream, conventional storage
		// -registered bitstream, srb storage
		// conventional bitstream - dspace ingested, dspace random name/path
		// registered bitstream - registered to dspace, any name/path
		String sIntermediatePath = null;
		if (isRegisteredBitstream(sInternalId)) {
			sInternalId = sInternalId.substring(REGISTERED_FLAG.length());
			sIntermediatePath = "";
		} else {
			sIntermediatePath = getIntermediatePath(sInternalId);
		}

		StringBuffer bufFilename = new StringBuffer();
		if (assetstore instanceof LocalFile) {
			bufFilename.append(assetstore.getCanonicalPath());
			bufFilename.append(File.separator);
			bufFilename.append(sIntermediatePath);
			bufFilename.append(sInternalId);
			if (log.isDebugEnabled()) {
				log.debug("Local filename for " + sInternalId + " is "
						+ bufFilename.toString());
			}
			return new LocalFile(bufFilename.toString());
		}
		if (assetstore instanceof SRBFile) {
			bufFilename.append(sIntermediatePath);
			bufFilename.append(sInternalId);
			if (log.isDebugEnabled()) {
				log.debug("SRB filename for " + sInternalId + " is "
						+ ((SRBFile) assetstore).toString()
						+ bufFilename.toString());
			}
			return new SRBFile((SRBFile) assetstore, bufFilename.toString());
		}
		return null;
    }

	/**
	 * Return the intermediate path derived from the internal_id. This method
	 * splits the id into groups which become subdirectories.
	 *
	 * @param iInternalId
	 *            The internal_id
	 * @return The path based on the id without leading or trailing separators
	 */
	private static String getIntermediatePath(String iInternalId) {
		StringBuffer buf = new StringBuffer();
		for (int i = 0; i < directoryLevels; i++) {
			int digits = i * digitsPerLevel;
			if (i > 0) {
				buf.append(File.separator);
			}
			buf.append(iInternalId.substring(digits, digits
							+ digitsPerLevel));
		}
		buf.append(File.separator);
		return buf.toString();
	}

}