logtofile.java

derby database source code.good for you.

/*

   Derby - Class org.apache.derby.impl.store.raw.log.LogToFile

   Copyright 1997, 2004 The Apache Software Foundation or its licensors, as applicable.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

 */

package org.apache.derby.impl.store.raw.log;

import org.apache.derby.iapi.services.diag.Performance;

import org.apache.derby.impl.store.raw.log.CheckpointOperation;
import org.apache.derby.impl.store.raw.log.LogCounter;
import org.apache.derby.impl.store.raw.log.LogRecord;
import org.apache.derby.impl.store.raw.log.StreamLogScan;

// need this to print nested exception that corrupts the database
import org.apache.derby.iapi.services.context.ErrorStringBuilder;

import org.apache.derby.iapi.services.info.ProductGenusNames;
import org.apache.derby.iapi.services.info.ProductVersionHolder;

import org.apache.derby.iapi.reference.MessageId;
import org.apache.derby.iapi.reference.Property;
import org.apache.derby.iapi.reference.SQLState;

import org.apache.derby.iapi.services.daemon.DaemonService;
import org.apache.derby.iapi.services.daemon.Serviceable;
import org.apache.derby.iapi.services.context.ContextManager;
import org.apache.derby.iapi.services.context.ContextService;
import org.apache.derby.iapi.services.context.ShutdownException;
import org.apache.derby.iapi.services.monitor.Monitor;
import org.apache.derby.iapi.services.monitor.ModuleControl;
import org.apache.derby.iapi.services.monitor.ModuleSupportable;
import org.apache.derby.iapi.services.monitor.PersistentService;
import org.apache.derby.iapi.services.sanity.SanityManager;
import org.apache.derby.iapi.services.io.Formatable;
import org.apache.derby.iapi.services.io.TypedFormat;
import org.apache.derby.iapi.services.io.FormatIdUtil;
import org.apache.derby.iapi.services.io.StoredFormatIds;
import org.apache.derby.iapi.services.stream.HeaderPrintWriter;
import org.apache.derby.iapi.services.stream.PrintWriterGetHeader;
import org.apache.derby.iapi.services.stream.InfoStreams;
import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.services.i18n.MessageService;
import org.apache.derby.iapi.store.access.AccessFactory;
import org.apache.derby.iapi.store.access.AccessFactoryGlobals;
import org.apache.derby.iapi.store.access.TransactionController;
import org.apache.derby.iapi.store.raw.Loggable;
import org.apache.derby.iapi.store.raw.RawStoreFactory;
import org.apache.derby.iapi.store.raw.ScanHandle;
import org.apache.derby.iapi.store.raw.log.LogFactory;
import org.apache.derby.iapi.store.raw.log.Logger;
import org.apache.derby.iapi.store.raw.log.LogInstant;
import org.apache.derby.iapi.store.raw.log.LogScan;
import org.apache.derby.iapi.store.raw.Transaction;
import org.apache.derby.iapi.store.raw.xact.RawTransaction;
import org.apache.derby.iapi.store.raw.xact.TransactionFactory;
import org.apache.derby.iapi.store.raw.data.DataFactory;
import org.apache.derby.iapi.services.property.PersistentSet;

import org.apache.derby.iapi.store.access.DatabaseInstant;
import org.apache.derby.catalog.UUID;
import org.apache.derby.iapi.services.uuid.UUIDFactory;
import org.apache.derby.iapi.services.property.PropertyUtil;
import org.apache.derby.iapi.reference.Attribute;
import org.apache.derby.iapi.services.io.FileUtil;
import org.apache.derby.iapi.util.ReuseFactory;

import org.apache.derby.io.StorageFactory;
import org.apache.derby.io.WritableStorageFactory;
import org.apache.derby.io.StorageFile;
import org.apache.derby.io.StorageRandomAccessFile;

import java.io.File; // Plain files are used for backups
import java.io.IOException;
import java.io.SyncFailedException;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.FileNotFoundException;

import java.net.MalformedURLException;
import java.net.URL;

import java.util.Properties;
import java.util.Vector;
import java.util.zip.CRC32;

/**

	This is an implementation of the log using a non-circular file system file.
	No support for incremental log backup or media recovery.
	Only crash recovery is supported.  
	<P>
	The 'log' is a stream of log records.  The 'log' is implemented as
	a series of numbered log files.  These numbered log files are logically
	continuous so a transaction can have log records that span multiple log files.
	A single log record cannot span more then one log file.  The log file number
	is monotonically increasing.
	<P>
	The log belongs to a log factory of a RawStore.  In the current implementation,
	each RawStore only has one log factory, so each RawStore only has one log
	(which composed of multiple log files).
	At any given time, a log factory only writes new log records to one log file,
	this log file is called the 'current log file'.
	<P>
	A log file is named log<em>logNumber</em>.dat
	<P>
	Everytime a checkpoint is taken, a new log file is created and all subsequent
	log records will go to the new log file.  After a checkpoint is taken, old
	and useless log files will be deleted.
	<P>
	RawStore exposes a checkpoint method which clients can call, or a checkpoint is
	taken automatically by the RawStore when
	<OL>
	<LI> the log file grows beyond a certain size (configurable, default 100K bytes)
	<LI> RawStore is shutdown and a checkpoint hasn't been done "for a while"
	<LI> RawStore is recovered and a checkpoint hasn't been done "for a while"
	</OL>
	<P>
	This LogFactory is responsible for the formats of 2 kinds of file: the log
	file and the log control file.  And it is responsible for the format of the
	log record wrapper.
	<P> <PRE>

	Format of log control file 

	@format_id	FILE_STREAM_LOG_FILE
	@purpose	The log control file contains information about which log files
	are present and where the last checkpoint log record is located.
	@upgrade	
	@disk_layout
	(pre-v15)
		int format id
		int log file version
		long the log instant (LogCounter) of the last completed checkpoint
	(v15 onward)
		int format id
		int obsolete log file version
		long the log instant (LogCounter) of the last completed checkpoint
		int JBMS version
		int checkpoint interval
		long spare (value set to 0)
		long spare (value set to 0)
		long spare (value set to 0)

	@end_format
	</PRE>	
	<HR WIDTH="100%">
	<PRE>

	Format of the log file

	@format_id	FILE_STREAM_LOG_FILE
	@purpose	The log file contains log record which record all the changes
	to the database.  The complete transaction log is composed of a series of
	log files.
	@upgrade
	@disk_layout
		int format id - 	the format Id of this log file
		int obsolete log file version - not used
		long log file number - this number orders the log files in a
						series to form the complete transaction log
		long prevLogRecord - log instant of the previous log record, in the
				previous log file. 

		[log record wrapper]* one or more log records with wrapper

		int endMarker - value of zero.  The beginning of a log record wrapper
				is the length of the log record, therefore it is never zero
		[int fuzzy end]* zero or more int's of value 0, in case this log file
				has been recovered and any incomplete log record set to zero.
	@end_format
	</PRE>	
	<HR WIDTH="100%">
	<PRE>

	Format of the log record wrapper

	@format_id none.  The format is implied by the FILE_STREAM_LOG_FILE
	@purpose	The log record wrapper provides information for the log scan.
	@upgrade
	@disk_layout
		length(int) length of the log record (for forward scan)
		instant(long) LogInstant of the log record
		logRecord(byte[length]) byte array that is written by the FileLogger
		length(int) length of the log record (for backward scan)
	@end_format
	</PRE>


	<P>Multithreading considerations:<BR>
	Log Factory must be MT-safe.
	<P>
	Class is final as it has methods with privilege blocks
	and implements PrivilegedExceptionAction.
	*/

public final class LogToFile implements LogFactory, ModuleControl, ModuleSupportable,
								  Serviceable, java.security.PrivilegedExceptionAction
{

	private static int fid = StoredFormatIds.FILE_STREAM_LOG_FILE; 

	// format Id must fit in 4 bytes
 
	/**
		Return my format identifier.
	*/
	public int getTypeFormatId() {
		return StoredFormatIds.FILE_STREAM_LOG_FILE;
	}

	// at the beginning of every log file is the following information:
	// the log file formatId
	// the log file version (int)
	// the log file number (long)
	// the log instant at the end of the last log record in the previous file (long)
	public static final int LOG_FILE_HEADER_SIZE = 24;

	protected static final int LOG_FILE_HEADER_PREVIOUS_LOG_INSTANT_OFFSET = LOG_FILE_HEADER_SIZE-8;

	// Number of bytes overhead of each log record.
	// 4 bytes of length at the beginning, 8 bytes of log instant,
	// and 4 bytes of length at the end
	public static final int LOG_RECORD_OVERHEAD = 4+8+4;

	public static final String DBG_FLAG = SanityManager.DEBUG ? "LogTrace" : null;
	public static final String DUMP_LOG_ONLY = SanityManager.DEBUG ? "DumpLogOnly" : null;
	public static final String DUMP_LOG_FROM_LOG_FILE = 
		SanityManager.DEBUG ? "derby.storage.logDumpStart" : null;
	protected static final String LOG_SYNC_STATISTICS = "LogSyncStatistics";

	// If you change this number, then JBMS 1.1x and 1.2x will give a really
	// horrendous error message when booting against a db created by you.  When
	// we decided that we don't need to worry about people mis-using the
	// product that way, then we can change this.  Just remember, before we do,
	// all existing database will have the number 9 in there.
	private static final int OBSOLETE_LOG_VERSION_NUMBER = 9;

	/* how big the log file should be before checkpoint or log switch is taken */
	private static final int DEFAULT_LOG_SWITCH_INTERVAL = 1024*1024;		
	private static final int LOG_SWITCH_INTERVAL_MIN     = 100000;
	private static final int LOG_SWITCH_INTERVAL_MAX     = 128*1024*1024;
	private static final int CHECKPOINT_INTERVAL_MIN     = 100000;
	private static final int CHECKPOINT_INTERVAL_MAX     = 128*1024*1024;
	private static final int DEFAULT_CHECKPOINT_INTERVAL = 10*1024*1024;

	//log buffer size values
	private static final int DEFAULT_LOG_BUFFER_SIZE = 32768; //32K
	private static final int LOG_BUFFER_SIZE_MIN = 8192; //8k
	private static final int LOG_BUFFER_SIZE_MAX = LOG_SWITCH_INTERVAL_MAX;
	private int logBufferSize = DEFAULT_LOG_BUFFER_SIZE;

	/* Log Control file flags. */
	private static final byte IS_BETA_FLAG = 0x1;
	
    /**
     * When the derby.system.durability property is set to 'test', the store 
     * system will not force sync calls in the following cases
     * - for the log file at each commit
     * - for the log file before data page is forced to disk
     * - for page allocation when file is grown
     * - for data writes during checkpoint
     * This means it is possible that the recovery system may not work properly,
     * committed transactions may be lost, and/or database may not
     * be in a consistent state.
     * In order that we recognize this case that the database was previously
     * at any time booted in this mode, this value is written out
     * into the log control file. This will help prevent us from 
     * wasting time to resolve issues in such cases. 
     * @see org.apache.derby.iapi.reference.Property#DURABILITY_PROPERTY
     * This value is written as part of the log control file flags byte.
     */
    private static final byte IS_DURABILITY_TESTMODE_NO_SYNC_FLAG = 0x2;
	
    /**
     * keeps track of if the database was booted previously at any time with 
     * derby.system.durability=test
     */
    private static boolean wasDBInDurabilityTestModeNoSync = false;
    
	/* to err on the conservative side, unless otherwise set, assume log
	 *	archive is ON 
	 */
	private static final String DEFAULT_LOG_ARCHIVE_DIRECTORY = "DEFAULT";

	private int     logSwitchInterval   = DEFAULT_LOG_SWITCH_INTERVAL;
	private int     checkpointInterval  = DEFAULT_CHECKPOINT_INTERVAL;

	String dataDirectory; 					// where files are stored
    private WritableStorageFactory logStorageFactory;
    
	private boolean logBeingFlushed; // is the log in the middle of a flush
									 // (access of the variable should sync on this)

	protected LogAccessFile logOut;		// an output stream to the log file
								// (access of the variable should sync on this)
	private   StorageRandomAccessFile firstLog = null;
	protected long		     endPosition = -1; // end position of the current log file
	long					 lastFlush = 0;	// the position in the current log
											// file that has been flushed to disk

	long					 logFileNumber = -1; // current log file number
								// other than during boot and recovery time,
								// logFileNumber is only changed by
								// switchLogFile, which is synchronized.
								// 
								// MT - Anyone accessing this number should
								// synchronized on this if the current log file
								// must not be changed. If not synchronized,
								// the log file may have been switched.

	long					 firstLogFileNumber = -1;
								// first log file that makes up the active
								// portion (with active transactions) of the
								// log.  
								// 
								// MT - This value is set during recovery or
								// during log truncation.  In the former single
								// thread is assumed.  In the latter
								// must be synchronized with this to access
								// or change.
	
	private long              maxLogFileNumber = LogCounter.MAX_LOGFILE_NUMBER;
	private CheckpointOperation		 currentCheckpoint;
								// last checkpoint successfully taken
								// 
								// MT - only changed or access in recovery or
								// checkpoint, both are single thread access

	long					 checkpointInstant;
								// log instant of te curerntCheckpoint

	private DaemonService	 checkpointDaemon;	// the background worker thread who is going to
								// do checkpoints for this log factory.

	private	int			myClientNumber;	
								// use this number to talk to checkpoint Daemon

	private volatile boolean checkpointDaemonCalled; 
								// checkpoint Daemon called already - it is not
								// important that this value is correct, the
								// daemon just need to be called once in a
								// while.  Deamon can handle multiple posts.

	private long logWrittenFromLastCheckPoint = 0;
	                            // keeps track of the amout of log written between checkpoints
	private RawStoreFactory rawStoreFactory; 
								// use this only after recovery is finished

	protected DataFactory dataFactory;
								// use this only after revocery is finished

	protected boolean	ReadOnlyDB;	// true if this db is read only, i.e, cannot
								// append log records


	// DEBUG DEBUG - do not truncate log files
	private boolean keepAllLogs;

	// if database is encrypted, the content of the log files are encrypted
	private boolean databaseEncrypted; 

	// the following booleans are used to put the log factory into various
	// states
	private boolean			 recoveryNeeded = true; // log needs to be recovered
	private boolean			 inCheckpoint = false; 	// in the middle of a checkpoint
	private boolean			 inRedo = false;        // in the middle of redo loop
	private boolean          inLogSwitch = false;

	// make sure we don't do anything after the log factory has been stopped
	private boolean			 stopped = false;

	// if log is to go to another device, this variable is set.  If null, then
	// log goes to the log subdirectory underneath the data directory
	String logDevice;

    // disable syncing of log file when running in derby.system.durability=test
    private boolean logNotSynced = false;

	private boolean logArchived = false;
	private boolean logSwitchRequired = false;

	/** DEBUG test only */
	int test_logWritten = 0;
	int test_numRecordToFillLog = -1;
	private int mon_flushCalls;
	private int mon_syncCalls;
	private int mon_numLogFlushWaits;
	private boolean mon_LogSyncStatistics;
	private int mon_numBytesToLog;