engine.cpp

Emdros is a text database middleware-layer aimed at storage and retrieval of "text plus information

		case kForm_UTF32BE:
			CHECK_AVAIL(4);
			rval = DATA(dataPtr) << 24; dataPtr++;
			rval += DATA(dataPtr) << 16; dataPtr++;
			rval += DATA(dataPtr) << 8; dataPtr++;
			rval += DATA(dataPtr); dataPtr++;
			break;

		case kForm_UTF32LE:
			CHECK_AVAIL(4);
			rval = DATA(dataPtr); dataPtr++;
			rval += DATA(dataPtr) << 8; dataPtr++;
			rval += DATA(dataPtr) << 16; dataPtr++;
			rval += DATA(dataPtr) << 24; dataPtr++;
			break;
	}
	
	if (dataPtr >= savedCount) {
		dataPtr -= savedCount;
		savedCount = 0;
	}
	
	return rval;
}

void
Converter::_savePendingBytes()
{
	dataPtr -= savedCount;
	while (dataPtr < dataLen)
		savedBytes[savedCount++] = data[dataPtr++];
}

bool
Converter::IsForward() const
{
	return forward;
}

void
Converter::GetFlags(UInt32& sourceFlags, UInt32& targetFlags) const
{
	const FileHeader*	fh = (const FileHeader*)table;
	if (forward) {
		sourceFlags = READ(fh->formFlagsLHS);
		targetFlags = READ(fh->formFlagsRHS);
	}
	else {
		sourceFlags = READ(fh->formFlagsRHS);
		targetFlags = READ(fh->formFlagsLHS);
	}
}

static bool
getNamePtrFromTable(const Byte* table, UInt16 nameID, const Byte*& outNamePtr, UInt32& outNameLen)
{
	const FileHeader*	fh = (const FileHeader*)table;
	const UInt32*		nameOffsets = (const UInt32*)(table + sizeof(FileHeader));
	for (UInt32 i = 0; i < READ(fh->numNames); ++i) {
		const NameRec*	n = (const NameRec*)(table + READ(nameOffsets[i]));
		if (READ(n->nameID) == nameID) {
			outNameLen = READ(n->nameLength);
			outNamePtr = (const Byte*)n + sizeof(NameRec);
			return true;
		}
	}
	return false;
}

bool
Converter::GetNamePtr(UInt16 nameID, const Byte*& outNamePtr, UInt32& outNameLen) const
{
	return getNamePtrFromTable(table, nameID, outNamePtr, outNameLen);
}

TECkit_Status
Converter::ConvertBufferOpt(
	const Byte* inBuffer, UInt32 inLength, UInt32* inUsed,
	Byte* outBuffer, UInt32 outLength, UInt32* outUsed,
	UInt32 inOptions, UInt32* lookaheadCount)
{
	TECkit_Status	rval;
#undef RETURN
#define RETURN(returnStatus)	rval = returnStatus; goto RETURN_LABEL

	UInt32	outPtr = 0;
	
	data = inBuffer;
	dataLen = inLength;
	dataPtr = 0;
	inputComplete = ((inOptions & kOptionsMask_InputComplete) == kOptionsComplete_InputIsComplete);
	unmappedBehavior = (inOptions & kOptionsMask_UnmappedBehavior);
	
	UInt32	c;
	if (pendingOutputChar != kInvalidChar) {
		c = pendingOutputChar;
		pendingOutputChar = kInvalidChar;
		goto GOT_CHAR;
	}
	while (1) {
		c = finalStage->getChar();
	GOT_CHAR:
		switch (c) {
			case kEndOfText:
				RETURN(kStatus_NoError);

			case kNeedMoreInput:
				RETURN(kStatus_NeedMoreInput);

			case kInvalidChar:
				RETURN(kStatus_IncompleteChar);

			case kUnmappedChar:
				RETURN(kStatus_UnmappedChar);

			default:
				switch (outputForm) {
					case kForm_Bytes:
						if (outPtr == outLength) {
							pendingOutputChar = c;
							RETURN(kStatus_OutputBufferFull);
						}
						outBuffer[outPtr++] = c;
						break;

					case kForm_UTF8:
						{
							int	bytesToWrite;
							if (c < 0x80) {				bytesToWrite = 1;
							} else if (c < 0x800) {		bytesToWrite = 2;
							} else if (c < 0x10000) {	bytesToWrite = 3;
							} else if (c < 0x200000) {	bytesToWrite = 4;
							} else {					bytesToWrite = 2;
														c = 0x0000fffd;
							};
							if (outPtr + bytesToWrite > outLength) {
								pendingOutputChar = c;
								RETURN (kStatus_OutputBufferFull);
							}
							outPtr += bytesToWrite;
							switch (bytesToWrite) {	/* note: code falls through cases! */
								case 4:	outBuffer[--outPtr] = (c | byteMark) & byteMask; c >>= 6;
								case 3:	outBuffer[--outPtr] = (c | byteMark) & byteMask; c >>= 6;
								case 2:	outBuffer[--outPtr] = (c | byteMark) & byteMask; c >>= 6;
								case 1:	outBuffer[--outPtr] =  c | firstByteMark[bytesToWrite];
							};
							outPtr += bytesToWrite;
						}
						break;

					case kForm_UTF16BE:
						if (c < 0x00010000) {
							if (outPtr + 2 > outLength) {
								pendingOutputChar = c;
								RETURN (kStatus_OutputBufferFull);
							}
							outBuffer[outPtr++] = c >> 8;
							outBuffer[outPtr++] = c;
						}
						else {
							if (outPtr + 4 > outLength) {
								pendingOutputChar = c;
								RETURN (kStatus_OutputBufferFull);
							}
							c -= halfBase;
							UInt32	hi = (c >> halfShift) + kSurrogateHighStart;
							UInt32	lo = (c & halfMask) + kSurrogateLowStart;
							outBuffer[outPtr++] = hi >> 8;
							outBuffer[outPtr++] = hi;
							outBuffer[outPtr++] = lo >> 8;
							outBuffer[outPtr++] = lo;
						}
						break;

					case kForm_UTF16LE:
						if (c < 0x00010000) {
							if (outPtr + 2 > outLength) {
								pendingOutputChar = c;
								RETURN (kStatus_OutputBufferFull);
							}
							outBuffer[outPtr++] = c;
							outBuffer[outPtr++] = c >> 8;
						}
						else {
							if (outPtr + 4 > outLength) {
								pendingOutputChar = c;
								RETURN (kStatus_OutputBufferFull);
							}
							c -= halfBase;
							UInt32	hi = (c >> halfShift) + kSurrogateHighStart;
							UInt32	lo = (c & halfMask) + kSurrogateLowStart;
							outBuffer[outPtr++] = hi;
							outBuffer[outPtr++] = hi >> 8;
							outBuffer[outPtr++] = lo;
							outBuffer[outPtr++] = lo >> 8;
						}
						break;

					case kForm_UTF32BE:
						if (outPtr + 4 > outLength) {
							pendingOutputChar = c;
							RETURN (kStatus_OutputBufferFull);
						}
						outBuffer[outPtr++] = c >> 24;
						outBuffer[outPtr++] = c >> 16;
						outBuffer[outPtr++] = c >> 8;
						outBuffer[outPtr++] = c;
						break;

					case kForm_UTF32LE:
						if (outPtr + 4 > outLength) {
							pendingOutputChar = c;
							RETURN (kStatus_OutputBufferFull);
						}
						outBuffer[outPtr++] = c;
						outBuffer[outPtr++] = c >> 8;
						outBuffer[outPtr++] = c >> 16;
						outBuffer[outPtr++] = c >> 24;
						break;
				}
				break;
		}
	}
	RETURN (kStatus_NoError);

RETURN_LABEL:
	if (inUsed)
		*inUsed = dataPtr;
	if (outUsed)
		*outUsed = outPtr;
	if (lookaheadCount) {
		*lookaheadCount = 0;
		Stage*	s = finalStage;
		while (s != this) {
			*lookaheadCount += s->lookaheadCount();
			s = s->prevStage;
		}
	}

	rval |= warningStatus;
	if ((rval & kStatusMask_Basic) == kStatus_NoError)
		Reset();
	
	return rval;
}

void
Converter::Reset()
{
	pendingOutputChar = kInvalidChar;
	savedCount = 0;
	dataPtr = 0;
	dataLen = 0;
	warningStatus = 0;
	Stage*	s = finalStage;
	while (s != this) {
		s->Reset();
		s = s->prevStage;
	}
}

bool
Converter::Validate(const Converter* cnv)
{
	if (!cnv)
		return false;
	if (cnv->status != kStatus_NoError)
		return false;
	if (cnv->table != 0) {
		const FileHeader*	fh = (const FileHeader*)cnv->table;
		if (READ(fh->type) != kMagicNumber)
			return false;
	}
	return true;
}

#pragma mark --- Public "C" API functions ---

TECkit_Status
WINAPI
TECkit_CreateConverter(
	Byte*				mapping,
	UInt32				mappingSize,
	Byte				mapForward,
	UInt16				inputForm,
	UInt16				outputForm,
	TECkit_Converter*	converter)
{
	TECkit_Status	status = kStatus_NoError;
	Converter*	cnv = 0;
	*converter = 0;
	try {
		cnv = new Converter(mapping, mappingSize, mapForward, inputForm, outputForm);
		status = cnv->creationStatus();
		if (status == kStatus_NoError)
			*converter = (TECkit_Converter)cnv;
		else
			delete cnv;
	}
	catch (Converter::Exception e) {
		status = e.errorCode;
	}
	catch (...) {
		status = kStatus_Exception;
	}
	return status;
}

TECkit_Status
WINAPI
TECkit_DisposeConverter(
	TECkit_Converter	converter)
{
	TECkit_Status	status = kStatus_NoError;
	Converter*	cnv = (Converter*)converter;
	if (!Converter::Validate(cnv))
		status = kStatus_InvalidConverter;
	else
		delete cnv;
	return status;
}

TECkit_Status
WINAPI
TECkit_GetConverterName(
	TECkit_Converter	converter,
	UInt16				nameID,
	Byte*				nameBuffer,
	UInt32				bufferSize,
	UInt32*				nameLength)
{
	TECkit_Status	status = kStatus_NoError;
	Converter*	cnv = (Converter*)converter;
	if (!Converter::Validate(cnv))
		status = kStatus_InvalidConverter;
	else {
		const Byte*	namePtr;
		if (cnv->GetNamePtr(nameID, namePtr, *nameLength)) {
			UInt16	copyBytes = *nameLength < bufferSize ? *nameLength : bufferSize;
			if (copyBytes > 0)
				memcpy(nameBuffer, namePtr, copyBytes);
		}
		else
			status = kStatus_NameNotFound;
	}
	return status;
}

TECkit_Status
WINAPI
TECkit_GetConverterFlags(
	TECkit_Converter	converter,
	UInt32*				sourceFlags,
	UInt32*				targetFlags)
{
	TECkit_Status	status = kStatus_NoError;
	Converter*	cnv = (Converter*)converter;
	if (!Converter::Validate(cnv))
		status = kStatus_InvalidConverter;
	else
		cnv->GetFlags(*sourceFlags, *targetFlags);
	return status;
}

TECkit_Status
WINAPI
TECkit_ResetConverter(
	TECkit_Converter	converter)
{
	TECkit_Status	status = kStatus_NoError;
	Converter*	cnv = (Converter*)converter;
	if (!Converter::Validate(cnv))
		status = kStatus_InvalidConverter;
	else
		cnv->Reset();
	return status;
}

TECkit_Status
WINAPI
TECkit_ConvertBufferOpt(
	TECkit_Converter	converter,
	const Byte*			inBuffer,
	UInt32				inLength,
	UInt32*				inUsed,
	Byte*				outBuffer,
	UInt32				outLength,
	UInt32*				outUsed,
	UInt32				inOptions,
	UInt32*				lookaheadCount)
{
	TECkit_Status	status = kStatus_NoError;
	Converter*	cnv = (Converter*)converter;
	if (!Converter::Validate(cnv))
		status = kStatus_InvalidConverter;
	else
		status = cnv->ConvertBufferOpt(inBuffer, inLength, inUsed, outBuffer, outLength, outUsed, inOptions, lookaheadCount);
	return status;
}

TECkit_Status
WINAPI
TECkit_ConvertBuffer(
	TECkit_Converter	converter,
	const Byte*			inBuffer,
	UInt32				inLength,
	UInt32*				inUsed,
	Byte*				outBuffer,
	UInt32				outLength,
	UInt32*				outUsed,
	Byte				inputIsComplete)
{
	return TECkit_ConvertBufferOpt(converter, inBuffer, inLength, inUsed, outBuffer, outLength, outUsed,
			kOptionsUnmapped_UseReplacementCharSilently + (inputIsComplete ? kOptionsComplete_InputIsComplete : 0), 0);
}

TECkit_Status
WINAPI
TECkit_FlushOpt(
	TECkit_Converter	converter,
	Byte*				outBuffer,
	UInt32				outLength,
	UInt32*				outUsed,
	UInt32				inOptions,
	UInt32*				lookaheadCount)
{
	TECkit_Status	status = kStatus_NoError;
	Converter*	cnv = (Converter*)converter;
	if (!Converter::Validate(cnv))
		status = kStatus_InvalidConverter;
	else
		status = cnv->ConvertBufferOpt(0, 0, 0, outBuffer, outLength, outUsed,
			inOptions | kOptionsComplete_InputIsComplete, lookaheadCount);
	return status;
}

TECkit_Status
WINAPI
TECkit_Flush(
	TECkit_Converter	converter,
	Byte*				outBuffer,
	UInt32				outLength,
	UInt32*				outUsed)
{
	return TECkit_FlushOpt(converter, outBuffer, outLength, outUsed, kOptionsUnmapped_UseReplacementCharSilently, 0);
}

TECkit_Status
WINAPI
TECkit_GetMappingFlags(
	Byte*				mapping,
	UInt32				mappingSize,
	UInt32*				lhsFlags,
	UInt32*				rhsFlags)
{
	TECkit_Status	status = kStatus_NoError;
	if (mapping == 0)
		status = kStatus_InvalidMapping;
	else {
		const FileHeader*	fh = (const FileHeader*)mapping;
		FileHeader			header;
#ifndef NO_ZLIB
		if (READ(fh->type) == kMagicNumberCmp) {
			// compressed mapping, so we need to decompress enough of it to read the flags
			unsigned long	uncompressedLen = sizeof(FileHeader);
			int	result = uncompress((Byte*)&header, &uncompressedLen, mapping + 2 * sizeof(UInt32), mappingSize - 2 * sizeof(UInt32));
			if (result != Z_BUF_ERROR)
				status = kStatus_InvalidMapping;
			fh = &header;
		}
#endif
		if (status == kStatus_NoError && READ(fh->type) == kMagicNumber) {
			if ((READ(fh->version) & 0xFFFF0000) > (kCurrentFileVersion & 0xFFFF0000))
				status = kStatus_BadMappingVersion;
			else {
				*lhsFlags = READ(fh->formFlagsLHS);
				*rhsFlags = READ(fh->formFlagsRHS);
			}
		}
		else
			status = kStatus_InvalidMapping;
	}
	return status;
}

TECkit_Status
WINAPI
TECkit_GetMappingName(
	Byte*				mapping,
	UInt32				mappingSize,
	UInt16				nameID,
	Byte*				nameBuffer,
	UInt32				bufferSize,
	UInt32*				nameLength)
{
	void*	buf = 0;
	TECkit_Status	status = kStatus_NoError;
	if (mapping == 0)
		status = kStatus_InvalidMapping;
	else {
		const FileHeader*	fh = (const FileHeader*)mapping;
		FileHeader			header;
#ifndef NO_ZLIB
		if (READ(fh->type) == kMagicNumberCmp) {
			// compressed mapping, so we need to decompress the fixed header to read the headerLength field,
			// and then decompress the complete header to get the names
			unsigned long	uncompressedLen = sizeof(FileHeader);
			int	result = uncompress((Byte*)&header, &uncompressedLen, mapping + 2 * sizeof(UInt32), mappingSize - 2 * sizeof(UInt32));
			if (result != Z_BUF_ERROR)
				status = kStatus_InvalidMapping;
			else {
				fh = &header;
				uncompressedLen = READ(fh->headerLength);
				buf = malloc(uncompressedLen);
				if (buf == 0)
					status = kStatus_OutOfMemory;
				else {
					result = uncompress((Byte*)buf, &uncompressedLen, mapping + 2 * sizeof(UInt32), mappingSize - 2 * sizeof(UInt32));
					if (result != Z_BUF_ERROR)
						status = kStatus_InvalidMapping;
					fh = (const FileHeader*)buf;
				}
			}
		}
#endif
		if (status == kStatus_NoError && READ(fh->type) == kMagicNumber) {
			if ((READ(fh->version) & 0xFFFF0000) > (kCurrentFileVersion & 0xFFFF0000))
				status = kStatus_BadMappingVersion;
			else {
				const Byte*	namePtr;
				if (getNamePtrFromTable((Byte*)fh, nameID, namePtr, *nameLength)) {
					UInt16	copyBytes = *nameLength < bufferSize ? *nameLength : bufferSize;
					if (copyBytes > 0)
						memcpy(nameBuffer, namePtr, copyBytes);
				}
				else
					status = kStatus_NameNotFound;
			}
		}
		else
			status = kStatus_InvalidMapping;
		if (buf != 0)
			free(buf);
	}
	return status;
}

UInt32
WINAPI
TECkit_GetVersion()
{
	return kCurrentTECkitVersion;
}