copyhelper.cpp

Data Replication Prototype Using ADO

/*
 	Replication & Migration stuff description.

	There are four useful macros describing structure and order of replication:
1. COPY_ENTRY, 2. COPY_LINK, 3. COPY_XLINK, 4. COPY_RLINK.

	Replication graph structure is tree-like; but it is a little more complicated then tree. It consists of 
several trees, which can interfere and contain self-linked branches. COPY_ENTRY macro describes 
tree root items. In a single item replication case this means that all the data in SlaveTable 
with SlaveField value = lIdFrom should be replicated using fltFilter for search and cleanup. SlaveField in 
table being copied to will be replaced by lIdTo. For database replication stuff, there are 
COPY_ENTRY macros with fltFilter = fltNoFilter. It means that all data in SlaveTable should be 
copied. SlaveField in this case should be PK field. In this case lIdTo, lIdFrom will be ignored 
automatically. There is also COPY_ENTRY_PK(ArrayId, SlaveTable, SlaveField) macro, where 
ArrayId is pointer to a CArray containing Ids of records to be replicated. SlaveField here is PK.
	Entries start walking through tree structure described by COPY_LINK macros. This macro 
means that all SlaveTable records with SlaveField equal PK in MasterTable should be copied and 
SlaveField value in table being copied to will be replaced by PK in data receiving MasterTable 
record which was being copied earlier. There exist also "indirect" entries referring to link slave 
tables. These entries do not start process of walking through tree. They are used together with links 
to support filters consisting of several fields. Several entries can be applied to one table 
simultaneously, too. However links are always treated separately.
	COPY_XLINK macro describes links being out of this copy force tree (e. g denormalized tables). 
It requires to update SlaveField values in SlaveTable according to PK in MasterTable. This x-link requires 
for MasterTable to be replicated first. It is important that it can cause deadlock and failure of replication 
process if links and x-links are constructed in wrong way and cross each other. If  MasterTable 
record with proper PK is not available, and matching record is not found with fltUniqueIndex filter, 
new record in MasterTable is created and its data is copied from corresponding record on the input side.
	R-link created by COPY_RLINK macro is similar to one created by COPY_XLINK. The 
main difference is that it does not require for MasterTable to be replicated first. This macro is 
destined for links to reference tables. These links are out of scope of tree-like replication structure.
	X- and r-links share cache maps for input and output primary keys organized by master table 
type for replication optimization. During database migration, reference tables are copied, and, 
correspondingly, cache maps are filled before main replication routine.
	Most links require no customization. There is a set of template-based classes for other links. 
Base class CDataHandler contains virtual functions. They do nothing or implement default actions:

	virtual BOOL DeleteRecord()
	{ 
		return GetTblCopyTo()->DeleteRecord(); 
	}
	virtual void CorrectTableData() {}
	virtual BOOL PostUpdate() { return TRUE; }

	DeleteRecord() should be used for tables which records cannot be deleted automatically. In 
this case for COPY_ENTRY correct work link constructor should contain SetLeaveData(lkLeaveUnique) call, 
and only records which match by unique index will be deleted. 
	CorrectTableData() is used for tables where not all the data is set by replication routine. 
It can be also used for data transformation.
	PostUpdate() is destined for actions which should take place after actual table update. 
	This set of virtual functions can be easily extended.
*/

#include "stdafx.h"
#include "CopyHelper.h"
#include "resource.h"

#pragma warning(disable:4100)
#include <algorithm>
#pragma warning(default:4100)

#include "mapiter.h"
#include "TableHolder.h"
#include "DBTable.h"

#include "DlgUIChoicePrompt.h"

//////////////////////////////////////////////////////////////////////////


char* pchar_stack::get(size_t nSize) const
{
	if(!m_pBuf)
		return reinterpret_cast<char*>(operator new (nSize));
	char* result = reinterpret_cast<char*>(m_pBuf);
	m_pBuf = m_pBuf->pNext;
	return result;
}

void pchar_stack::put(void* p) const
{
	buffer* pBuf = reinterpret_cast<buffer*>(p);
	pBuf->pNext = m_pBuf;
	m_pBuf = pBuf;
}

void pchar_stack::RemoveAll()
{
	while(m_pBuf)
	{
		buffer* pBuf = m_pBuf->pNext;
		delete reinterpret_cast<char*>(m_pBuf);
		m_pBuf = pBuf;
	}
}


//////////////////////////////////////////////////////////////////////////

#ifdef _DEBUG
#define new DEBUG_NEW
#undef THIS_FILE
static char THIS_FILE[] = __FILE__;
#endif


long Compare(const DBASE_VERSION& V1, const DBASE_VERSION& V2)
{
	long lResult = V1.m_lVersion_Number - V2.m_lVersion_Number;
	if(!lResult)
	{
		lResult = V1.m_lSubVersion_Number - V2.m_lSubVersion_Number;
		if(!lResult)
		{
			lResult = V1.m_lRelease_Number - V2.m_lRelease_Number;
			if(!lResult)
				lResult = V1.m_lRevision_Number - V2.m_lRevision_Number;
		}
	}
	return lResult;
}

#define DECLARE_GET_pEntry(i)				\
	COrderEntry* pEntry = m_Entries[i];		\
	CHECK_ADDRESS(pEntry)

#define DECLARE_GET_pLink(i)				\
	COrderLink* pLink = m_Links[i];			\
	CHECK_ADDRESS(pLink)

#define DECLARE_GET_pXLink(i)				\
	CXLink* pXLink = m_XLinks[i];			\
	CHECK_ADDRESS(pXLink)

//////////////////////////////////////////////////////////////

CMapIdentities* CMapTbl2MapId::GetAtNew(CTableId pTableTo)
{
	if(!pTableTo)
	{
		ASSERT(0);
		return NULL;
	}
	CMapIdentities* pMapId;
	if(!Lookup(pTableTo, pMapId))
	{
		pMapId = new CMapIdentities;
		SetAt(pTableTo, pMapId);
	}
	ASSERT(pMapId);
	return pMapId;
}

//////////////////////////////////////////////////////////////

struct Delete
{
	template<class T> void operator ()(T* pOb)
	{
		delete pOb;
	}
};

//////////////////////////////////////////////////////////////

struct llmap : public mfciter::base_map<Identity, Identity, CMapIdentities>
    {
private:
    llmap();    // not constructable
    };

    // llmap returning freestanding functions
llmap::const_iterator begin(const CMapIdentities &mp)
    {
    return llmap::const_iterator(mp);
    }
llmap::iterator begin(CMapIdentities &mp)
    {
    return llmap::iterator(mp);
    }
llmap::const_iterator end(const CMapIdentities &)
    {
    return llmap::const_iterator();
    }
llmap::iterator end(CMapIdentities &)
    {
    return llmap::iterator();
    }

//////////////////////////////////////////////////////////////

class transfer_exception {};

void HandleTransferIds(CSubstRecArrayPtr& arrOutpSubstRec, 
								 Identity lIdTo, Identity lIdFrom, bool bAdded)
{
	if(bAdded)
		arrOutpSubstRec->Add(CSubstRec(lIdTo, lIdFrom));
}

template<typename T, typename C, typename H>
// Does not work with other result types
inline void TransferData(T* pHolder, const C& rContext, COrderVariant* pVar, int nCount, 
	 bool bForceAdd, CMapIdentities* pSubstId, H& rIdHandler)
{
	CHECK_ADDRESS(pHolder);
	CHECK_ADDRESS(pVar);
	CHECK_ADDRESS(pSubstId);

	CDBTable* pTblTo		= pVar->GetTblCopyTo();

	pVar->CopyData(false);
	pVar->SetIDs();

	if(!pHolder->DoConvertAndFilter(pVar, pSubstId, rContext))
		return;

	if(pVar->IsObsolete())
		return;
	if(!pHolder->GoUpstairs(pTblTo, nCount - 1))
		throw transfer_exception();

	pHolder->SetDefValues(pVar);
	pVar->CorrectTableData();
	if(pVar->IsObsoleteByRefs())
		return;

	bool bUpdate = false;
	bool bHandleDependants = false;

	Identity lIdTo = ID_NULL;

	if(bForceAdd || !pTblTo->HasUniqueFilter()
	|| !pVar->FindMatchByUI())
	{
		pHolder->DoAddDropRecord(pVar, bUpdate);
		bHandleDependants = bUpdate;
	}
	else
	{
		if(pHolder->HasSameDatabases() && pTblTo->HasPrimaryKey()
		&& pTblTo->GetPrimaryKey() == pVar->GetPrimaryKeyFrom())
			return;
		pHolder->DoHandleRecord(pVar, bUpdate, bHandleDependants);
		lIdTo = pTblTo->GetPrimaryKey();
	}
	if(bUpdate)
	{
		if(!pTblTo->Update() || !pVar->PostUpdate()) 
			throw transfer_exception();
		lIdTo = pTblTo->GetPrimaryKey();
	}
	if(IsValid(lIdTo))
	{
		pSubstId->SetAt(pVar->GetPrimaryKeyFrom(), lIdTo);
		HandleTransferIds(rIdHandler, lIdTo, pVar->GetPrimaryKeyFrom(), bHandleDependants);
	}
}

//////////////////////////////////////////////////////////////

void PumpPaintMsg()
{
	MSG msg;
	while (::PeekMessage( &msg, NULL, WM_PAINT, WM_PAINT, PM_REMOVE ))
		::DispatchMessage(&msg);
}

//////////////////////////////////////////////////////////////

CTblCopyHelper::CTblCopyHelper()
{
	m_pHolderTo	  = NULL;
	m_pHolderFrom = NULL;
	m_bAutoDelete = FALSE;
	m_pProgress	  = NULL;
	m_nReferenceTables = -1;
	m_dwCopyFlags = 0;
	m_eSameDBases = eNotDef;

	m_bLazyObjectBinding = true;

	m_nProgressDelay = 0;
}

CTblCopyHelper::~CTblCopyHelper()
{
	FreeArrays();
	FreeDBHolders();
}


void CTblCopyHelper::FreeArrays()
{
	int i;
	for(i = m_Entries.GetSize(); --i >= 0; ) 
		delete m_Entries[i];
	m_Entries.SetSize(0);

	std::for_each(m_WorkFlowEntries.begin(), m_WorkFlowEntries.end(), Delete());
	m_WorkFlowEntries.clear();

	for(i = m_Links.GetSize(); --i >= 0; )
		delete m_Links[i];
	m_Links.SetSize(0);

	for(i = m_XLinks.GetSize(); --i >= 0; )
		delete m_XLinks[i];
	m_XLinks.SetSize(0);

	std::for_each<mfciter::cmap<CTableId, CMapIdentities*, const CTableId&>::iterator>
		(mfciter::begin(m_mapTbl2MapId), mfciter::end(m_mapTbl2MapId), DeleteSecond());
	m_mapTbl2MapId.RemoveAll();

	std::for_each(m_mapOrderVariants.begin(), m_mapOrderVariants.end(), Delete());
	m_mapOrderVariants.clear();

	std::for_each(m_DataProviders.begin(), m_DataProviders.end(), Delete());
	m_DataProviders.clear();

	m_LongMapMemCache.RemoveAll();

	m_nReferenceTables = -1;
}

void CTblCopyHelper::FreeDBHolders()
{
	if(m_bAutoDelete) 
	{
		delete m_pHolderTo;
		delete m_pHolderFrom;
	}
	m_pHolderTo = NULL;
	m_pHolderFrom = NULL;
}

void CTblCopyHelper::SetDataSources(CTableHolder* pHolderTo, 
		CTableHolder* pHolderFrom /*= NULL*/, 
		BOOL bAutoDelete /*= FALSE*/)
{
	FreeArrays();
	FreeDBHolders();
	if(!pHolderFrom)
		pHolderFrom = pHolderTo;
	CHECK_ADDRESS(pHolderTo);
	CHECK_ADDRESS(pHolderFrom);
	m_pHolderTo	  = pHolderTo;
	m_pHolderFrom = pHolderFrom;
	m_bAutoDelete = bAutoDelete;
	m_pHolderTo->SetAddMissingFields(FALSE);
	m_pHolderFrom->SetAddMissingFields(FALSE);
	SetFormerVersion();
}

void CTblCopyHelper::SetFormerVersion()
{
}

void CTblCopyHelper::SetCurrentVersion()
{
}


BOOL CTblCopyHelper::CopyTables(CProgressCtrl* pProgressBar /*= NULL*/)
{
	CHECK_NULL_OR_ADDRESS(pProgressBar);
	AFX_MANAGE_STATE(AfxGetStaticModuleState());

	m_nProgressDelay = 0;
	
	CWaitCursor WaitCursor;

	PumpPaintMsg();

	GetHolderTo()->SetReadOnly(false);
	if (GetHolderFrom() != GetHolderTo())
		GetHolderFrom()->SetReadOnly(true);

//	SetAutoCommitOff();

	BOOL bOK = PreCopyTables(pProgressBar) 
				&& DoCopyTables(pProgressBar)
				&& PostCopyTables();

	//if(bOK)
	//	Commit();
	//else
	//	Rollback();
	//SetAutoCommitOn();

	return bOK;
}

#ifdef _DEBUG
static CStringArray g_arrVirginXLinks;
#endif// _DEBUG

enum { EXTRA_SPACE = 1000 };

BOOL CTblCopyHelper::DoCopyTables(CProgressCtrl* pProgress /*= NULL*/)
{
	m_pProgress = pProgress;

#ifdef _DEBUG
	int i;
	for(i = m_Links.GetSize(); --i >= 0; )
	{
		COrderLink* pLink1 = m_Links[i];
		CHECK_ADDRESS(pLink1);
		for(int j = i; --j >=0; )
		{
			COrderLink* pLink2 = m_Links[j];
			CHECK_ADDRESS(pLink2);
			if(pLink1->GetTblSlaveTo() == pLink2->GetTblSlaveTo())
			{
				TRACE3("Replication: Duplicate links to table \"%s\" from tables \"%s\" and \"%s\".\n", 
						pLink1->GetTblSlaveTo().GetTableName(),
						pLink1->GetTblMasterTo().GetTableName(), 
						pLink2->GetTblMasterTo().GetTableName());

			}
		}
	}
#endif//_DEBUG


	ASSERT(m_WorkFlowEntries.size() > 0);
	int nMaxCount = m_Links.GetSize() + m_XLinks.GetSize() + EXTRA_SPACE;
	BOOL bResult = TRUE;
	for(int nIteration = 0; m_WorkFlowEntries.size() > 0; nIteration++) 
	{
		bool bHandled = false;
		for(CMultiSetEntries::iterator iter = m_WorkFlowEntries.begin()
		; iter != m_WorkFlowEntries.end()
		; )// Increment in the operator body
		{
#ifdef _DEBUG
			g_arrVirginXLinks.RemoveAll();
#endif// _DEBUG
			COrderEntry* pEntry = static_cast<COrderEntry*>(*iter);
			CHECK_ADDRESS(pEntry);
			switch(HandleEntry(pEntry))
			{
			case handled:
				bHandled = true;
				m_WorkFlowEntries.erase(iter++);
				delete pEntry;
				ShowProgress();
				continue;
			case error:
				bResult = FALSE;
				ASSERT(0);
				break;
			case postpone:
				iter++;
				continue;
			}
			break;
		}
		if(!bResult || !bHandled || nIteration >= nMaxCount 
		|| int(m_WorkFlowEntries.size()) >= nMaxCount) 
		{
#ifdef _DEBUG
			for(CMultiSetEntries::const_iterator iter = m_WorkFlowEntries.begin()
			; iter != m_WorkFlowEntries.end(); ++iter)
				TRACE1(_T("Replication Error: Stuck table \"%s\"\n"), (*iter)->GetTblSlaveTo().GetTableName());

			TRACE0("Replication Error: List of virgin x-links\n");
			for(i = g_arrVirginXLinks.GetSize(); --i >= 0; )
				TRACE0(_T('\"') + g_arrVirginXLinks[i] + _T("\"\n"));
			TRACE0("Replication Error: End of List of virgin x-links\n");
#endif
			ASSERT(0); 
			bResult = FALSE;
			break;
		}
	}

#ifdef _DEBUG
	for(i = m_Links.GetSize(); --i >= 0; ) 
	{
		DECLARE_GET_pLink(i);
		if(!pLink->IsPassed())
		{
			TRACE2("Replication: Link from table \"%s\" to table \"%s\" isn't passed.\n", 
						pLink->GetTblMasterTo().GetTableName(), pLink->GetTblSlaveTo().GetTableName());
		}
	}
#endif

	return bResult;
}

EHandleResult CTblCopyHelper::HandleEntry (COrderEntry* pEntry)
{
	if(HasVirginXLinks(pEntry->GetTblSlaveTo(), m_Links.GetSize() + m_XLinks.GetSize())) 
		return postpone;