utpersist.c

DataDraw is an ultra-fast persistent database for high performance programs written in C. It s so fa

/*
 * This file was written by Bill Cox.  It is hereby placed into the public domain.
 */

/*--------------------------------------------------------------------------------------------------
  Functions supporting database persistence.  It would really complicate things if this module
  depended in any way on itself, so it's coded bare-bones in C.
--------------------------------------------------------------------------------------------------*/
#include <stdlib.h>
#include "ddutil.h"
#include "utpersist.h"

/* This buffer reduces calls to fwrite */
/* #define UT_CHANGE_BUFFER_LENGTH 16384 */
/* temp - for debugging */
#define UT_CHANGE_BUFFER_LENGTH 16
static uint8 *utCommandBuffer;
static uint32 utBufferPosition, utBufferSize;
static FILE *utRecentChangesFile;
/* The utBufferStartPosition is relative to the first change in recentChanges if we are
 * persistent, and should be equal to 0 otherwise. */
static uint32 utLastTransactionPosition, utBufferStartPosition;
/* utRedoTransaction is the next transaction that would be redone if we did one.  It's usually
 * NULL, unless we undo one. */
static utTransaction utRedoTransaction;
/* These are used while undoing a transaction, since we have to compute the command sizes */
static uint32 *utTransactionCommands;
static uint32 utTransactionUsedCommands, utTransactionAllocatedCommands;
/* This says whether or not we are in the middle of a transaction */
static bool utTransactionInProgress;
/* These are used while registering classes and fields so we don't have to keep passing it. */
static utModule utCurrentModule;
static utClass utCurrentClass;
/* The size of the database file */
static uint64 utDatabaseSize;
/* When true, keep a copy of the previous database file */
static bool utKeepBackup;

/* The command buffer is only used while applying commands.  It holds one transaction at a time. */
static uint32 utAllocatedCommands;
static uint32 utUsedCommands;
static uint8 *utCommands;

/* This checksum is to verify a transaction has not been corrupted */
static uint32 utChecksum;

/* Globals that keep track of the allocated objects */
struct utModuleStruct *utModules;
uint8 utAllocatedModules, utUsedModules;
struct utClassStruct *utClasses;
uint16 utAllocatedClasses, utUsedClasses;
struct utFieldStruct *utFields;
uint16 utAllocatedFields, utUsedFields;
struct utTransactionStruct *utTransactions;
uint32 utUsedTransactions, utAllocatedTransactions;
struct utFieldStruct *utFields;
uint16 utAllocatedFields, utUsedFields;
struct utEnumStruct *utEnums;
uint16 utAllocatedEnums, utUsedEnums;
struct utEntryStruct *utEntries;
uint16 utAllocatedEntries, utUsedEntries;
struct utUnionStruct *utUnions;
uint16 utAllocatedUnions, utUsedUnions;
struct utUnionCaseStruct *utUnionCases;
uint16 utAllocatedUnionCases, utUsedUnionCases;

/* This keeps us from writing the the unopened changes file before starting persistence */
bool utPersistenceInitialized;
char *utDatabaseDirectory;
bool utUseTextDatabaseFormat;

/*--------------------------------------------------------------------------------------------------
  Open the recentChanges file with the mode.
--------------------------------------------------------------------------------------------------*/
static void openRecentChanges(
    char *mode)
{
    char *fileName = utSprintf("%s%crecentChanges", utDatabaseDirectory, UTDIRSEP);

    utRecentChangesFile = fopen(fileName, mode);
    if(utRecentChangesFile == NULL) {
        utExit("Could not open file %s", fileName);
    }
}

/*--------------------------------------------------------------------------------------------------
  Register a module.
--------------------------------------------------------------------------------------------------*/
uint8 utRegisterModule(
    char *prefix,
    uint32 hashValue,
    uint16 numClasses,
    uint16 numFields,
    uint16 numEnums,
    uint16 globalSize,
    void *globalData,
    void (*start)(void),
    void (*stop)(void))
{
    utModule module = utFindModule(prefix);

    if(module != NULL) {
        /* Already registered, so ignore it */
        utCurrentModule = NULL;
        module->initialized = true;
        return module - utModules;
    }
    if(utUsedModules == utAllocatedModules) {
        utAllocatedModules += utAllocatedModules >> 1;
        utResizeArray(utModules, utAllocatedModules);
    }
    module = utModules + utUsedModules;
    module->prefix = calloc(strlen(prefix) + 1, sizeof(char));
    strcpy(module->prefix, prefix);
    module->hashValue = hashValue;
    module->globalSize = globalSize;
    module->globalData = globalData;
    module->start = start;
    module->stop = stop;
    module->firstClassIndex = utUsedClasses;
    module->numClasses = numClasses;
    module->firstFieldIndex = utUsedFields;
    module->numFields = numFields;
    module->firstEnumIndex = utUsedEnums;
    module->numEnums = numEnums;
    module->initialized = true;
    utCurrentModule = module;
    return utUsedModules++;
}

/*--------------------------------------------------------------------------------------------------
  Set a module to uninitialized.
--------------------------------------------------------------------------------------------------*/
void utUnregisterModule(
    uint8 moduleID)
{
    utModule module = utModules + moduleID;

    module->initialized = false;
}

/*--------------------------------------------------------------------------------------------------
  Register a class.
--------------------------------------------------------------------------------------------------*/
void utRegisterClass(
    char *name,
    uint16 numFields,
    void *numUsedPtr,
    void *numAllocatedPtr,
    void *firstFreePtr,
    uint16 nextFreeFieldIndex,
    uint8 referenceSize,
    uint64 (*constructor)(void),
    void (*destructor)(uint64 objectIndex))
{
    utClass theClass;

    if(utCurrentModule == NULL) {
        return;
    }
    if(utUsedClasses == utAllocatedClasses) {
        utAllocatedClasses += utAllocatedClasses >> 1;
        utResizeArray(utClasses, utAllocatedClasses);
    }
    theClass = utClasses + utUsedClasses++;
    theClass->name = calloc(strlen(name) + 1, sizeof(char));
    strcpy(theClass->name, name);
    theClass->firstFieldIndex = utUsedFields;
    theClass->numFields = numFields;
    theClass->numUsedPtr = numUsedPtr;
    theClass->numAllocatedPtr = numAllocatedPtr;
    theClass->firstFreePtr = firstFreePtr;
    theClass->nextFreeFieldIndex = nextFreeFieldIndex;
    theClass->referenceSize = referenceSize;
    theClass->constructor = constructor;
    theClass->destructor = destructor;
    theClass->moduleIndex = utCurrentModule - utModules;
    theClass->numHiddenFields = 0;
    utCurrentClass = theClass;
}

/*--------------------------------------------------------------------------------------------------
  Register a field.
--------------------------------------------------------------------------------------------------*/
void utRegisterField(
    char *name,
    void *arrayPtr,
    uint32 size,
    utFieldType type,
    char *destName)
{
    utField field;

    if(utCurrentModule == NULL) {
        return;
    }
    if(utUsedFields == utAllocatedFields) {
        utAllocatedFields += utAllocatedFields >> 1;
        utResizeArray(utFields, utAllocatedFields);
    }
    field = utFields + utUsedFields++;
    field->name = calloc(strlen(name) + 1, sizeof(char));
    strcpy(field->name, name);
    field->arrayPtr = arrayPtr;
    field->size = size;
    field->type = type;
    field->classIndex = utCurrentClass - utClasses;
    if(destName == NULL) {
        field->destName = NULL;
    } else {
        field->destName = calloc(strlen(destName) + 1, sizeof(char));
        strcpy(field->destName, destName);
    }
    field->array = false;
}

/*--------------------------------------------------------------------------------------------------
  Set the previously registered field as hidden.
--------------------------------------------------------------------------------------------------*/
void utSetFieldHidden(void)
{
    utField field = utFields + utUsedFields - 1;

    if(utCurrentModule == NULL) {
        return;
    }
    field->hidden = true;
    utCurrentClass->numHiddenFields++;
}

/*--------------------------------------------------------------------------------------------------
  Set the field as an array.
--------------------------------------------------------------------------------------------------*/
void utRegisterArray(
    uint32 *numUsedPtr,
    uint32 *numAllocatedPtr,
    void *(*getValues)(uint64 objectNumber, uint32 *numValues),
    void *(*allocValues)(uint64 objectNumber, uint32 numValues))
{
    utField arrayField = utFields + utUsedFields - 1;

    if(utCurrentModule == NULL) {
        return;
    }
    arrayField->array = true;
    arrayField->numUsedPtr = numUsedPtr;
    arrayField->numAllocatedPtr = numAllocatedPtr;
    arrayField->getValues = getValues;
    arrayField->allocValues = allocValues;
}

/*--------------------------------------------------------------------------------------------------
  Flush recent changes to disk.
--------------------------------------------------------------------------------------------------*/
static void flushRecentChanges(void)
{
    if(utPersistenceInitialized) {
        if(utBufferPosition > 0) {
            fwrite((void *)utCommandBuffer, sizeof(uint8), utBufferPosition, utRecentChangesFile);
        }
        utBufferStartPosition += utBufferPosition;
        utBufferPosition = 0;
    } else if(utBufferPosition == utBufferSize) {
        utBufferSize += utBufferSize >> 1;
        utResizeArray(utCommandBuffer, utBufferSize);
    }
}

/*--------------------------------------------------------------------------------------------------
  Write a byte to the recentChanges buffer.
--------------------------------------------------------------------------------------------------*/
static void writeUint8(
    uint8 value)
{
    if(utBufferPosition == utBufferSize) {
        flushRecentChanges();
    }
    utAssert(utTransactionInProgress || utChecksum == 0);
    if(utRedoTransaction != NULL) {
        utBufferPosition = utRedoTransaction->position - utBufferStartPosition;
        utLastTransactionPosition = utBufferStartPosition + utBufferPosition;
        utUsedTransactions = utRedoTransaction - utTransactions;
        utRedoTransaction = NULL;
    }
    utCommandBuffer[utBufferPosition++] = value;
    utTransactionInProgress = true;
    utChecksum = (utChecksum ^ value)*1103515245 + 12345;
}

/*--------------------------------------------------------------------------------------------------
  Write a uint16 to the recentChanges buffer.
--------------------------------------------------------------------------------------------------*/
static void writeUint16(
    uint16 value)
{
    uint8 *values = (uint8 *)(void *)&value;

    writeUint8(*values++);
    writeUint8(*values);
}

/*--------------------------------------------------------------------------------------------------
  Write a uint32 to the recentChanges buffer.
--------------------------------------------------------------------------------------------------*/
static void writeUint32(
    uint32 value)
{
    uint8 *values = (uint8 *)(void *)&value;

    writeUint8(*values++);
    writeUint8(*values++);
    writeUint8(*values++);
    writeUint8(*values);
}

/*--------------------------------------------------------------------------------------------------
  Write a uint64 to the recentChanges buffer.
--------------------------------------------------------------------------------------------------*/
static void writeUint64(
    uint64 value)
{
    uint8 *values = (uint8 *)(void *)&value;

    writeUint8(*values++);
    writeUint8(*values++);
    writeUint8(*values++);
    writeUint8(*values++);
    writeUint8(*values++);
    writeUint8(*values++);
    writeUint8(*values++);
    writeUint8(*values);
}

/*--------------------------------------------------------------------------------------------------
  Write a number of bytes to the recentChanges buffer.
--------------------------------------------------------------------------------------------------*/
static void writeValues(
    uint8 *values,
    uint32 numBytes)
{
    while(numBytes-- != 0) {
        writeUint8(*values++);
    }
}

/*--------------------------------------------------------------------------------------------------
  Create a new transaction object to track a group of commands associated with a transaction.
--------------------------------------------------------------------------------------------------*/
static utTransaction utTransactionCreate(
    uint32 position,
    uint32 length)
{
    utTransaction transaction;

    if(utPersistenceInitialized && utBufferStartPosition == 0) {
        utAssert(utCommandBuffer[position] != 0);
    }
    if(utUsedTransactions == utAllocatedTransactions) {
        utAllocatedTransactions += utAllocatedTransactions >> 1;
        utResizeArray(utTransactions, utAllocatedTransactions);
    }
    transaction = utTransactions + utUsedTransactions++;
    transaction->position = position;
    transaction->length = length;
    utTransactionInProgress = false;
    return transaction;
}

/*--------------------------------------------------------------------------------------------------
  Empty the command buffer.
--------------------------------------------------------------------------------------------------*/
static void resetCommandBuffer(void)
{
    utBufferPosition = 0;
    utLastTransactionPosition = 0;
    utBufferStartPosition = 0;
    utChecksum = 0;
    utTransactionInProgress = false;
    utUsedTransactions = 0;
    utRedoTransaction = NULL;
}

/*--------------------------------------------------------------------------------------------------
  Compact the database, and truncate recentChanges.
--------------------------------------------------------------------------------------------------*/
void utCompactDatabase(void)
{
    char *fileName, *backupFileName;

    if(utKeepBackup) {
        fileName = utSprintf("%s%cdatabase", utDatabaseDirectory, UTDIRSEP);
        backupFileName = utSprintf("%s.old", fileName);
        rename(fileName, backupFileName);
    }
    if(utUseTextDatabaseFormat) {
        utSaveTextDatabase(NULL);
    } else {
        utSaveBinaryDatabase(NULL);
    }