fastcopy.h

fastcopy是一个非常快速的数据拷贝软件

/* static char *fastcopy_id = 
	"@(#)Copyright (C) H.Shirouzu 2004-2008   fastcopy.h	Ver1.70"; */
/* ========================================================================
	Project  Name			: Fast Copy file and directory
	Create					: 2004-09-15(Wed)
	Update					: 2008-02-02(Sat)
	Copyright				: H.Shirouzu
	Reference				: 
	======================================================================== */

#include "tlib.h"
#include "resource.h"
#include "utility.h"
#include "regexp.h"

#define ALIGN_SIZE(all_size, block_size) (((all_size) + (block_size) -1) / (block_size) * (block_size))

#define MIN_SECTOR_SIZE		(512)
#define OPT_SECTOR_SIZE		(2048)
#define MAX_BUF				(1024 * 1024 * 1024)
#define MIN_BUF				(1024 * 1024)
#define BIGTRANS_ALIGN		(32 * 1024)
#define APP_MEMSIZE			(6 * 1024 * 1024)
#define PATH_LOCAL_PREFIX	L"\\\\?\\"
#define PATH_UNC_PREFIX		L"\\\\?\\UNC"
#define PATH_LOCAL_PREFIX_LEN	4
#define PATH_UNC_PREFIX_LEN		7
#define MIN_ATTR_BUF		(256 * 1024)
//#define MAX_ATTR_BUF		(128 * 1024 * 1024)
#define MIN_ATTRIDX_BUF		ALIGN_SIZE((MIN_ATTR_BUF / 4), PAGE_SIZE)
#define MAX_ATTRIDX_BUF(x)	ALIGN_SIZE(((x) / 4), PAGE_SIZE)
#define MIN_MKDIRQUEUE_BUF	(8 * 1024)
#define MAX_MKDIRQUEUE_BUF	(64 * 1024)
#define MIN_DSTDIREXT_BUF	(64 * 1024)
#define MAX_DSTDIREXT_BUF	(2 * 1024 * 1024)
#define MIN_ERR_BUF			(64 * 1024)
#define MAX_ERR_BUF			(4 * 1024 * 1024)
#define MAX_LIST_BUF		(128 * 1024)
#define MAX_PUTLIST_BUF		(128 * 1024 * 1024)
#define MIN_PUTLIST_BUF		(1024 * 1024)

#define FASTCOPY_MUTEX		"FastCopyMutex"
#define FASTCOPY_EVENT		"FastCopyEvent"

struct TotalTrans {
	BOOL	isPreSearch;
	int		preDirs;
	int		preFiles;
	_int64	preTrans;
	int		readDirs;
	int		readFiles;
	int		readAclStream;
	_int64	readTrans;
	int		writeDirs;
	int		writeFiles;
	int		writeAclStream;
	_int64	writeTrans;
	int		deleteDirs;
	int		deleteFiles;
	_int64	deleteTrans;
	int		skipDirs;
	int		skipFiles;
	_int64	skipTrans;
	int		filterSkips;
	int		errFiles;
	int		errAclStream;
	int		errDirs;
	_int64	errTrans;
	int		openRetry;
};

struct TransInfo {
	TotalTrans			total;
	DWORD				tickCount;
	BOOL				isSameDrv;
	BOOL				ignoreErr;
	DWORD				waitTick;

	VBuf				*errBuf;
	CRITICAL_SECTION	*errCs;

	VBuf				*listBuf;
	CRITICAL_SECTION	*listCs;

	WCHAR				curPath[MAX_PATH_EX];
};

struct FileStat {
	_int64		fileID;
	FILETIME	ftCreationTime;
	FILETIME	ftLastAccessTime;
	FILETIME	ftLastWriteTime;
	DWORD		nFileSizeLow;	// WIN32_FIND_DATA 偺 nFileSizeLow/High
	DWORD		nFileSizeHigh;	// 偲偼媡弴乮_int64 梡乯
	DWORD		dwFileAttributes;	// 0 == ALTSTREAM
	HANDLE		hFile;
	DWORD		lastError;
	int			renameCount;
	BOOL		isExists;
	int			size;
	int			minSize;		// upperName 暘傪娷傔側偄
	BYTE		*upperName;		// cFileName 廔抂+1傪巜偡
	DWORD		hashVal;		// upperName 偺 hash抣

	// extraData
	BYTE		*acl;
	int			aclSize;
	BYTE		*ead;
	int			eadSize;
	BYTE		*rep;	// reparse data
	int			repSize;

	FileStat	*next;			// for hashTable
	BYTE		cFileName[4];	// 4 == dummy

	_int64	FileSize() { return *(_int64 *)&nFileSizeLow; }
	void	SetFileSize(_int64 file_size) { *(_int64 *)&nFileSizeLow = file_size; }
};

class StatHash {
	FileStat	**hashTbl;
	int			hashNum;
	int			HashNum(int data_num);

public:
	StatHash() {}
	int		RequireSize(int data_num) { return	sizeof(FileStat *) * HashNum(data_num); }
	BOOL	Init(FileStat **data, int _data_num, void *tbl_buf);
	FileStat *Search(void *upperName, DWORD hash_val);
};

struct DirStatTag {
	int			statNum;
	int			oldStatSize;
	FileStat	*statBuf;
	FileStat	**statIndex;
};

class FastCopy {
public:
	enum Mode { DIFFCP_MODE, SYNCCP_MODE, MOVE_MODE, DELETE_MODE };
	enum OverWrite { BY_NAME, BY_ATTR, BY_LASTEST, BY_CONTENTS, BY_ALWAYS };
	enum FsType { FSTYPE_NONE, FSTYPE_NTFS, FSTYPE_FAT, FSTYPE_NETWORK };
	enum Flags { CREATE_OPENERR_FILE=1, USE_OSCACHE_READ=2, USE_OSCACHE_WRITE=4, PRE_SEARCH=8,
				SAMEDIR_RENAME=0x10, SKIP_EMPTYDIR=0x20, FIX_SAMEDISK=0x40, FIX_DIFFDISK=0x80,
				AUTOSLOW_IOLIMIT=0x100, WITH_ACL=0x200, WITH_ALTSTREAM=0x400,
				OVERWRITE_DELETE=0x800, OVERWRITE_DELETE_NSA=0x1000,
				FILE_REPARSE=0x2000, DIR_REPARSE=0x4000,
				OVERWRITE_PARANOIA=0x04000000,
				VERIFY_FILE=0x08000000, LISTING_ONLY=0x10000000,
				REPORT_ACL_ERROR=0x20000000, REPORT_STREAM_ERROR=0x40000000 };
	enum SpecialWaitTick { SUSPEND_WAITTICK=0x7fffffff };

	struct Info {
		BOOL	ignoreErr;		// (I/ )
		Mode	mode;			// (I/ )
		OverWrite overWrite;	// (I/ )
		BOOL	isPhysLock;		// (I/ )
		int		flags;			// (I/ )
		int		bufSize;		// (I/ )
		int		maxOpenFiles;	// (I/ )
		int		maxTransSize;	// (I/ )
		int		maxAttrSize;	// (I/ )
		int		maxDirSize;		// (I/ )
		int		nbMinSizeNtfs;	// (I/ ) FILE_FLAG_NO_BUFFERING 偱僆乕僾儞偡傞嵟彫僒僀僘
		int		nbMinSizeFat;	// (I/ ) FILE_FLAG_NO_BUFFERING 偱僆乕僾儞偡傞嵟彫僒僀僘 (FAT梡)
		HWND	hNotifyWnd;		// (I/ )
		UINT	uNotifyMsg;		// (I/ )
		int		lcid;			// (I/ )
		BOOL	isRenameMode;	// ( /O) ...乽暋惢偟傑偡乿僟僀傾儘僌僞僀僩儖梡忣曬乮巄掕乯
	};							//			 彨棃揑偵丄忣曬偑憹偊傟偽丄儊儞僶偐傜愗傝棧偟

	struct RandomDataBuf {	// 忋彂偒嶍彍梡
		BOOL	is_nsa;
		int		base_size;
		int		buf_size;
		BYTE	*buf[3];
	};

	enum Notify { END_NOTIFY, CONFIRM_NOTIFY, RENAME_NOTIFY };
	struct Confirm {
		enum Result { CANCEL_RESULT, IGNORE_RESULT, CONTINUE_RESULT };
		const void	*message;		// (I/ )
		BOOL		allow_continue;	// (I/ )
		const void	*path;			// (I/ )
		DWORD		err_code;		// (I/ )
		Result		result;			// ( /O)
	};

public:
	FastCopy(void);
	virtual ~FastCopy();

	BOOL RegisterInfo(const PathArray *_srcArray, const PathArray *_dstArray, Info *_info, const PathArray *_includeArray=NULL, const PathArray *_excludeArray=NULL);
	BOOL TakeExclusivePriv(void);
	BOOL Start(void);
	BOOL End(void);
	BOOL IsStarting(void) { return hReadThread || hWriteThread; }
	BOOL Suspend(void);
	BOOL Resume(void);
	void Aborting(void);
	BOOL WriteErrLog(void *message, int len=-1);
	BOOL IsAborting(void) { return isAbort; }
	void SetWaitTick(DWORD wait) { waitTick = wait; }
	DWORD GetWaitTick() { return waitTick; }
	BOOL GetTransInfo(TransInfo *ti, BOOL fullInfo=TRUE);

protected:
	enum Command { WRITE_FILE, WRITE_BACKUP_FILE, WRITE_FILE_CONT, WRITE_ABORT, WRITE_BACKUP_ACL, WRITE_BACKUP_EADATA, WRITE_BACKUP_ALTSTREAM, WRITE_BACKUP_END, REMOVE_FILES, MKDIR, INTODIR, DELETE_FILES, RETURN_PARENT, REQ_EOF /*, READDIR*/ };
	enum PutListOpt { PL_NORMAL=1, PL_DIRECTORY=2, PL_DELETE=4, PL_COMPARE=8 };
	struct ReqHeader : public TListObj {	// request header
		Command		command;
		BYTE		*buf;
		int			bufSize;
		int			reqSize;
		FileStat	stat;	// 壜曄挿
	};
	struct ReqBuf {
		BYTE		*buf;
		int			bufSize;
		ReqHeader	*req;
		int			reqSize;
	};

	// 婎杮忣曬
	DriveMng	driveMng;	// Drive 忣曬
	Info		info;		// 僆僾僔儑儞巜掕摍
	StatHash	hash;
	HashVal		hashVal;
	PathArray	srcArray;
	PathArray	dstArray;
	void	*src;			// src 僷僗奿擺梡
	void	*dst;			// dst 僷僗奿擺梡
	void	*confirmDst;	// 忋彂偒妋擣挷嵏梡
	int		srcBaseLen;		// src 僷僗偺屌掕晹暘偺挿偝
	int		dstBaseLen;		// dst 僷僗偺屌掕晹暘偺挿偝
	int		srcPrefixLen;	// \\?\ or \\?\UNC\ 偺挿偝
	int		dstPrefixLen;
	BOOL	isExtendDir;
	BOOL	isMetaSrc;
	int		maxStatSize;	// max size of FileStat
	int		nbMinSize;		// struct Info 嶲徠
	BOOL	enableAcl;
	BOOL	enableStream;

	// filter
	BOOL		isFilter;
	enum		{ FILE_EXP, DIR_EXP, MAX_FTYPE_EXP };
	enum		{ INC_EXP, EXC_EXP, MAX_KIND_EXP };
	RegExpEx	regExp[MAX_FTYPE_EXP][MAX_KIND_EXP];

	// 僶僢僼傽
	VBuf	mainBuf;		// Read/Write 梡 buffer
	VBuf	fileStatBuf;	// src file stat 梡 buffer
	VBuf	dirStatBuf;		// src dir stat 梡 buffer
	VBuf	dstStatBuf;		// dst dir/file stat 梡 buffer
	VBuf	dstStatIdxBuf;	// dstStatBuf 撪 entry 偺 index sort 梡
	VBuf	mkdirQueueBuf;
	VBuf	dstDirExtBuf;
	VBuf	srcDigestBuf;
	VBuf	dstDigestBuf;
	VBuf	errBuf;
	VBuf	listBuf;

	// 僙僋僞忣曬側偳
	int		srcSectorSize;
	int		dstSectorSize;
	int		sectorSize;
	int		maxTransSize;
	FsType	srcFsType;
	FsType	dstFsType;
	BYTE	src_root[MAX_PATH];

	TotalTrans	total;		// 僼傽僀儖僐僺乕摑寁忣曬

	DWORD	startTick;
	DWORD	endTick;
	DWORD	suspendTick;
	volatile DWORD	waitTick;

	BOOL	isAbort;
	BOOL	isSuspend;
	BOOL	isSameDrv;
	BOOL	isSameVol;
	BOOL	isRename;
	enum	DstReqKind { DSTREQ_NONE=0, DSTREQ_READSTAT=1, DSTREQ_DIGEST=2 } dstAsyncRequest;
	BOOL	dstRequestResult;

	TDigest	srcDigest;
	TDigest	dstDigest;
	BYTE	srcDigestVal[SHA1_SIZE];
	BYTE	dstDigestVal[SHA1_SIZE];

	HANDLE	hReadThread;
	HANDLE	hWriteThread;

	class TReqList : public TList {
	public:
		TReqList(void) {}
		ReqHeader *TopObj(void) { return (ReqHeader *)TList::TopObj(); }
		ReqHeader *NextObj(ReqHeader *obj) { return (ReqHeader *)TList::NextObj(obj); }
	};

	HANDLE		hRunMutex;
	Condition	cv;

	CRITICAL_SECTION errCs;
	CRITICAL_SECTION listCs;

	TReqList	readReqList;
	TReqList	writeReqList;
	BYTE		*usedOffset;
	BYTE		*freeOffset;
	ReqHeader	*writeReq;
	_int64		nextFileID;
	_int64		errFileID;

	FileStat	**openFiles;
	int			openFilesCnt;

	static unsigned WINAPI ReadThread(void *fastCopyObj);
	static unsigned WINAPI WriteThread(void *fastCopyObj);
	static unsigned WINAPI DeleteThread(void *fastCopyObj);

	BOOL ReadThreadCore(void);
	BOOL DeleteThreadCore(void);
	BOOL WriteThreadCore(void);
	BOOL FinishNotify(void);
	BOOL PreSearch(void);
	BOOL PreSearchProc(void *path, int prefix_len, int dir_len);
	BOOL ReadProc(int dir_len, BOOL confirm_dir=TRUE);
	BOOL GetDirExtData(ReqBuf *req_buf, FileStat *stat);
	BOOL IsOverWriteFile(FileStat *srcStat, FileStat *dstStat);
	int  MakeRenameName(void *new_fname, int count, void *org_fname);
	BOOL SetRenameCount(FileStat *stat);
	BOOL FilterCheck(const void *path, const void *fname, DWORD attr);
	BOOL ReadDirEntry(int dir_len, BOOL confirm_dir);
	BOOL OpenFileProc(FileStat *stat, int dir_len);
	BOOL OpenFileBackupProc(FileStat *stat, int src_len);
	BOOL ReadMultiFilesProc(int dir_len);
	BOOL CloseMultiFilesProc(void);
	void *RestoreOpenFilePath(void *path, int idx, int dir_len);
	BOOL ReadFileProc(int start_idx, int *end_idx, int dir_len);
	BOOL DeleteProc(void *path, int dir_len);
	BOOL DeleteDirProc(void *path, int dir_len, void *fname, FileStat *stat);
	BOOL DeleteFileProc(void *path, int dir_len, void *fname, FileStat *stat);

	void SetupRandomDataBuf(void);
	void GenRandomName(void *path, int fname_len, int ext_len);
	BOOL RenameRandomFname(void *org_path, void *rename_path, int dir_len, int fname_len);
	BOOL WriteRandomData(void *path, FileStat *stat, BOOL skip_hardlink);

	BOOL IsSameContents();
	BOOL MakeDigest(void *path, VBuf *vbuf, TDigest *digest, BYTE *val);

	void DstRequest(DstReqKind kind);
	BOOL WaitDstRequest(void);
	BOOL CheckDstRequest(void);
	BOOL ReadDstStat(void);
	BOOL MakeHashTable(void);
	void FreeDstStat(void);
	static int SortStatFunc(const void *stat1, const void *stat2);
	BOOL WriteProc(int dir_len);
	BOOL ExecDirQueue(void);
	BOOL SetDirExtData(FileStat *stat);
	BOOL WriteFileProc(int dst_len);
	BOOL WriteFileBackupProc(HANDLE fh, int dst_len);
	BOOL ChangeToWriteModeCore(void);
	BOOL ChangeToWriteMode(void);
	BOOL AllocReqBuf(int req_size, _int64 _data_size, ReqBuf *buf);
	BOOL PrepareReqBuf(int req_size, _int64 data_size, _int64 file_id, ReqBuf *buf);
	BOOL SendRequest(Command command, ReqBuf *buf=NULL, FileStat *stat=NULL);
	BOOL RecvRequest(void);
	void WriteReqDone(void);
	void SetErrFileID(_int64 file_id);

	BOOL InitSrcPath(int idx);
	BOOL InitDeletePath(int idx);
	BOOL InitDstPath(void);
	BOOL GetRootDir(const void *path, void *root_dir);
	FsType GetFsType(const void *root_dir);
	int GetSectorSize(const void *root_dir);
	BOOL IsSameDrive(const void *root1, const void *root2);
	int MakeUnlimitedPath(WCHAR *buf);
	BOOL PutList(void *path, DWORD opt);

	inline BOOL IsParentOrSelfDirs(void *name) { return *(BYTE *)name == '.' && (!strcmpV(name, DOT_V) || !strcmpV(name, DOTDOT_V)); }
	int FdatToFileStat(WIN32_FIND_DATAW *fdat, FileStat *stat, BOOL is_usehash);
	Confirm::Result ConfirmErr(const char *message, const void *path=NULL, BOOL allow_continue=TRUE);
	BOOL ConvertExternalPath(const void *path, void *buf, int buf_len);
	BOOL Wait(DWORD tick=0);
};

//	RegisterInfoProc
//	InitConfigProc
//	StartProc