utpersist.c

DataDraw is an ultra-fast persistent database for high performance programs written in C. It s so fa

/*--------------------------------------------------------------------------------------------------
  Load the persistent database into memory if it exists.  Return true if it does, otherwise false.
  If useTextDatabaseFormat is true, we'll keep the database in text instead of binary.  If
  keepBackup is true, we'll rename the old database file before writing the new one.
--------------------------------------------------------------------------------------------------*/
bool utStartPersistence(
    char *directory,
    bool useTextDatabaseFormat,
    bool keepBackup)
{
    char *fileName;

    if(!utAccess(directory, "w")) {
        utExit("Cannot write to directory %s\n", directory);
    }
    utDatabaseDirectory = utAllocString(directory);
    utUseTextDatabaseFormat = useTextDatabaseFormat;
    utKeepBackup = keepBackup;
    resetCommandBuffer();
    utTransactionInProgress = false;
    fileName = utSprintf("%s%cdatabase", directory, UTDIRSEP);
    if(utFileExists(fileName)) {
        utDatabaseSize = utFindFileSize(fileName);
        if(useTextDatabaseFormat) {
            utLoadTextDatabase(NULL);
        } else {
            utLoadBinaryDatabase(NULL);
        }
        resetCommandBuffer(); /* Loading a text database can issue record commands */
        openRecentChanges("rb");
        applyRecentChanges();
        utPersistenceInitialized = true;
        return true;
    }
    if(useTextDatabaseFormat) {
        utSaveTextDatabase(NULL);
    } else {
        utSaveBinaryDatabase(NULL);
    }
    utDatabaseSize = utFindFileSize(utSprintf("%s%cdatabase", directory, UTDIRSEP));
    openRecentChanges("wb");
    fclose(utRecentChangesFile);
    openRecentChanges("ab");
    utPersistenceInitialized = true;
    return false;
}

/*--------------------------------------------------------------------------------------------------
  Destroy the field.
--------------------------------------------------------------------------------------------------*/
void utFieldDestroy(
    utField field)
{
    free(field->name);
    if(field->destName != NULL) {
        free(field->destName);
    }
}

/*--------------------------------------------------------------------------------------------------
  Destroy the module.
--------------------------------------------------------------------------------------------------*/
void utClassDestroy(
    utClass theClass)
{
    free(theClass->name);
}

/*--------------------------------------------------------------------------------------------------
  Destroy the entry.
--------------------------------------------------------------------------------------------------*/
void utEntryDestroy(
    utEntry entry)
{
    free(entry->name);
}

/*--------------------------------------------------------------------------------------------------
  Destroy the enum.
--------------------------------------------------------------------------------------------------*/
void utEnumDestroy(
    utEnum theEnum)
{
    utEntry entry;
    uint16 xEntry;

    for(xEntry = 0; xEntry < theEnum->numEntries; xEntry++) {
        entry = utEntries + theEnum->firstEntryIndex + xEntry;
        utEntryDestroy(entry);
    }
    free(theEnum->name);
}

/*--------------------------------------------------------------------------------------------------
  Destroy the module.
--------------------------------------------------------------------------------------------------*/
void utModuleDestroy(
    utModule module)
{
    utClass theClass;
    utField field;
    utEnum theEnum;
    uint16 xField, xClass, xEnum;

    for(xField = 0; xField < module->numFields; xField++) {
        field = utFields + module->firstFieldIndex + xField;
        utFieldDestroy(field);
    }
    for(xClass = 0; xClass < module->numClasses; xClass++) {
        theClass = utClasses + module->firstClassIndex + xClass;
        utClassDestroy(theClass);
    }
    for(xEnum = 0; xEnum < module->numEnums; xEnum++) {
        theEnum = utEnums + module->firstEnumIndex + xEnum;
        utEnumDestroy(theEnum);
    }
    free(module->prefix);
}

/*--------------------------------------------------------------------------------------------------
  Stop the persistent database.  Flush changes to disk, and close files.
--------------------------------------------------------------------------------------------------*/
void utStopPersistence(void)
{
    flushRecentChanges();
    fclose(utRecentChangesFile);
    if(utRedoTransaction != NULL) {
        utTruncateFile(utSprintf("%s%crecentChanges", utDatabaseDirectory, UTDIRSEP),
            utRedoTransaction->position);
    }
    utFree(utDatabaseDirectory);
    utPersistenceInitialized = false;
}

/*--------------------------------------------------------------------------------------------------
  Load the persistent database into memory if it exists.  Return true if it does, otherwise false.
--------------------------------------------------------------------------------------------------*/
void utAllocPersistenceObjects(void)
{
    utAllocatedModules = 2;
    utUsedModules = 0;
    utModules = utNewA(struct utModuleStruct, utAllocatedModules);
    utAllocatedClasses = 2;
    utUsedClasses = 0;
    utClasses = utNewA(struct utClassStruct, utAllocatedClasses);
    utAllocatedFields = 2;
    utUsedFields = 0;
    utFields = utNewA(struct utFieldStruct, utAllocatedFields);
    utBufferPosition = 0;
    utBufferSize = UT_CHANGE_BUFFER_LENGTH;
    utCommandBuffer = utNewA(uint8, utBufferSize);
    utAllocatedTransactions = 2;
    utUsedTransactions = 0;
    utTransactions = utNewA(struct utTransactionStruct, utAllocatedTransactions);
    utTransactionAllocatedCommands = 2;
    utTransactionUsedCommands = 0;
    utTransactionCommands = utNewA(uint32, utTransactionAllocatedCommands);
    utAllocatedCommands = 2;
    utUsedCommands = 0;
    utCommands = utNewA(uint8, utAllocatedCommands);
    utAllocatedEnums = 2;
    utUsedEnums = 0;
    utEnums = utNewA(struct utEnumStruct, utAllocatedEnums);
    utAllocatedEntries = 2;
    utUsedEntries = 0;
    utEntries = utNewA(struct utEntryStruct, utAllocatedEntries);
    utAllocatedUnions = 2;
    utUsedUnions = 0;
    utUnions = utNewA(struct utUnionStruct, utAllocatedUnions);
    utAllocatedUnionCases = 2;
    utUsedUnionCases = 0;
    utUnionCases = utNewA(struct utUnionCaseStruct, utAllocatedUnionCases);
    utRedoTransaction = NULL;
}

/*--------------------------------------------------------------------------------------------------
  Stop the persistent database.  Flush changes to disk, and close files.
--------------------------------------------------------------------------------------------------*/
void utFreePersistenceObjects(void)
{
    utModule module;
    uint8 xModule;

    for(xModule = 0; xModule < utUsedModules; xModule++) {
        module = utModules + xModule;
        utModuleDestroy(module);
    }
    utUsedModules = 0;
    utUsedClasses = 0;
    utUsedFields = 0;
    utFree(utModules);
    utFree(utClasses);
    utFree(utFields);
    utFree(utCommandBuffer);
    utFree(utTransactions);
    utFree(utTransactionCommands);
    utFree(utCommands);
    utFree(utEnums);
    utFree(utEntries);
    utFree(utUnions);
    utFree(utUnionCases);
}

/*--------------------------------------------------------------------------------------------------
  Reset the database, so that we can read in a new one.
--------------------------------------------------------------------------------------------------*/
void utResetDatabase(void)
{
    utModule module;
    uint8 xModule;

    for(xModule = 0; xModule < utUsedModules; xModule++) {
        module = utModules + xModule;
        if(module->initialized) {
            module->stop();
            module->start();
        }
    }
    utInitSymTable();
}

/*--------------------------------------------------------------------------------------------------
  Dump a transaction complete command.
  TRANSACTION_COMPLETE <32-bit checksum>
--------------------------------------------------------------------------------------------------*/
static void dumpTransactionComplete(
    uint8 *command)
{
    uint32 checksum = readUint32(command);

    utDebug("transaction complete with checksum 0x%x\n", checksum);
}

/*--------------------------------------------------------------------------------------------------
  Dump a field command.
  WRITE_FIELD <16-bit field #> <32-bit index> <value>
--------------------------------------------------------------------------------------------------*/
static void dumpField(
    uint8 *command)
{
    uint16 xField = readUint16(command);
    utField field = utFields + xField;
    utClass theClass = utClasses + field->classIndex;
    utModule module = utModules + theClass->moduleIndex;
    uint64 objectNumber;

    command += 2;
    objectNumber = utFindIntValue(command, theClass->referenceSize);
    command += theClass->referenceSize;
    utDebug("%s%ss.%s[%llu] = 0x%s\n", module->prefix, theClass->name, field->name, objectNumber,
        utFindHexString(command, field->size));
}

/*--------------------------------------------------------------------------------------------------
  Dump an array command.
  WRITE_ARRAY <16-bit field #> <32-bit index> <32-bit numValues> <values>
--------------------------------------------------------------------------------------------------*/
static void dumpArray(
    uint8 *command)
{
    uint16 xField = readUint16(command);
    utField field = utFields + xField;
    utClass theClass = utClasses + field->classIndex;
    utModule module = utModules + theClass->moduleIndex;
    uint32 length, bytes;
    uint32 dataIndex;
    uint32 xValue;
    bool firstTime = true;

    command += 2;
    dataIndex = readUint32(command);
    command += 4;
    length = readUint32(command);
    command += 4;
    bytes = field->size*length;
    utDebug("%s%ss.%s[%u] = (", module->prefix, theClass->name, field->name, dataIndex);
    for(xValue = 0; xValue < length; xValue++) {
        if(!firstTime) {
            utDebug(", ");
        }
        firstTime = false;
        utDebug("0x%s", utFindHexString(command, field->size));
    }
    utDebug(")\n");
}

/*--------------------------------------------------------------------------------------------------
  Dump a global command.
  WRITE_GLOBAL <8-bit moduleID> <16-bit offset> <8-bit numBytes> <values>
--------------------------------------------------------------------------------------------------*/
static void dumpGlobal(
    uint8 *command)
{
    uint8 moduleID = readUint8(command);
    utModule module = utModules + moduleID;
    uint16 offset;
    uint8 numBytes;

    command++;
    offset = readUint16(command);
    command += 2;
    numBytes = readUint8(command);
    command++;
    utDebug("%sRootData[%u] = 0x%s\n", module->prefix, offset, utFindHexString(command, numBytes));
}

/*--------------------------------------------------------------------------------------------------
  Dump a resize command.
  RESIZE_FIELD <16-bit field #> <64-bit size>
--------------------------------------------------------------------------------------------------*/
static void dumpResize(
    uint8 *command)
{
    uint16 xField = readUint16(command);
    utField field = utFields + xField;
    utClass theClass = utClasses + field->classIndex;
    utModule module = utModules + theClass->moduleIndex;
    uint64 size;

    command += 2;
    size = readUint64(command);
    command += 8;
    utDebug("%s%ss.%s resized to %llu\n", module->prefix, theClass->name, field->name, size);
}

/*--------------------------------------------------------------------------------------------------
  Dump the command 
--------------------------------------------------------------------------------------------------*/
static void dumpCommand(
    uint8 *command)
{
    if((*command) & 0x80) {
        utDebug("undo ");
    }
    switch(*command++ & 0x7f) {
    case UT_TRANSACTION_COMPLETE:
        dumpTransactionComplete(command);
        break;
    case UT_WRITE_FIELD:
        dumpField(command);
        break;
    case UT_WRITE_ARRAY:
        dumpArray(command);
        break;
    case UT_WRITE_GLOBAL:
        dumpGlobal(command);
        break;
    case UT_RESIZE_FIELD:
        dumpResize(command);
        break;
    default:
        utExit("Unknown command in recentChanges file");
    }
}

/*--------------------------------------------------------------------------------------------------
  Dump the commands in the buffer to the log file.
--------------------------------------------------------------------------------------------------*/
static void dumpCommands(
    uint8 *commands,
    uint32 length)
{
    uint32 xCommand = 0;
    uint32 size;
    uint32 checksum = 0;

    while(xCommand < length) {
        utDebug("%8x %8x ", checksum, xCommand);
        size = findCommandSize(commands + xCommand);
        dumpCommand(commands + xCommand);
        if(commands[xCommand] == UT_TRANSACTION_COMPLETE) {
            checksum = 0;
        }
        while(size-- != 0) {
            checksum = (checksum ^ commands[xCommand++])*1103515245 + 12345;
        }
    }
}

/*--------------------------------------------------------------------------------------------------
  Dump the recent changes buffer to the log file in text.
--------------------------------------------------------------------------------------------------*/
void utDumpRecentChanges(void)
{
    char *fileName = utSprintf("%s%crecentChanges", utDatabaseDirectory, UTDIRSEP);
    uint32 fileSize = utFindFileSize(fileName);
    uint8 *commands = utNewA(uint8, fileSize);

    fclose(utRecentChangesFile);
    openRecentChanges("rb");
    fread(commands, sizeof(uint8), fileSize, utRecentChangesFile);
    fclose(utRecentChangesFile);
    dumpCommands(commands, fileSize);
    openRecentChanges("ab");
    utFree(commands);
}