hashmanager.cpp

来自「一个不错的关于手机模块程序This page contains everythi」· C++ 代码 · 共 678 行 · 第 1/2 页

		HashLoader l(*this);
		SimpleXMLReader(&l).fromXML(File(getIndexFile(), File::READ, File::OPEN).read());
	} catch(const Exception&) {
		// ...
	}
}

static const string sHashStore = "HashStore";
static const string sversion = "version";		// Oops, v1 was like this
static const string sVersion = "Version";
static const string sTrees = "Trees";
static const string sFiles = "Files";
static const string sFile = "File";
static const string sName = "Name";
static const string sSize = "Size";
static const string sHash = "Hash";
static const string sType = "Type";
static const string sTTH = "TTH";
static const string sIndex = "Index";
static const string sLeafSize = "LeafSize";		// Residue from v1 as well
static const string sBlockSize = "BlockSize";
static const string sTimeStamp = "TimeStamp";
static const string sRoot = "Root";

void HashLoader::startTag(const string& name, StringPairList& attribs, bool simple) {
	if(!inHashStore && name == sHashStore) {
		version = Util::toInt(getAttrib(attribs, sVersion, 0));
		if(version == 0) {
			version = Util::toInt(getAttrib(attribs, sversion, 0));
		}
		inHashStore = !simple;
	} else if(inHashStore && version == 2) {
		if(inTrees && name == sHash) {
			const string& type = getAttrib(attribs, sType, 0);
			int64_t index = Util::toInt64(getAttrib(attribs, sIndex, 1));
			int64_t blockSize = Util::toInt64(getAttrib(attribs, sBlockSize, 2));
			int64_t size = Util::toInt64(getAttrib(attribs, sSize, 3));
			const string& root = getAttrib(attribs, sRoot, 4);
			if(!root.empty() && type == sTTH && (index >= 8 || index == HashManager::SMALL_TREE) && blockSize >= 1024) {
				store.treeIndex[TTHValue(root)] = HashManager::HashStore::TreeInfo(size, index, blockSize);
			}
		} else if(inFiles && name == sFile) {
			file = getAttrib(attribs, sName, 0);
			timeStamp = Util::toUInt32(getAttrib(attribs, sTimeStamp, 1));
			const string& root = getAttrib(attribs, sRoot, 2);

			if(!file.empty() && size >= 0 && timeStamp > 0 && !root.empty()) {
				string fname = Text::toLower(Util::getFileName(file));
				string fpath = Text::toLower(Util::getFilePath(file));

				store.fileIndex[fpath].push_back(HashManager::HashStore::FileInfo(fname, TTHValue(root), timeStamp, false));
			}
		} else if(name == sTrees) {
			inTrees = !simple;
		} else if(name == sFiles) {
			inFiles = !simple;
		}
	}
}

void HashLoader::endTag(const string& name, const string&) {
	if(name == sFile) {
		file.clear();
	}
}

HashManager::HashStore::HashStore() : dirty(false) 
{ 
	if(File::getSize(getDataFile()) <= sizeof(int64_t)) {
		try {
			createDataFile(getDataFile());
		} catch(const FileException&) {
			// ?
		}
	}
}

/**
 * Creates the data files for storing hash values.
 * The data file is very simple in its format. The first 8 bytes
 * are filled with an int64_t (little endian) of the next write position
 * in the file counting from the start (so that file can be grown in chunks).
 * We start with a 1 mb file, and then grow it as needed to avoid fragmentation.
 * To find data inside the file, use the corresponding index file.
 * Since file is never deleted, space will eventually be wasted, so a rebuild
 * should occasionally be done.
 */
void HashManager::HashStore::createDataFile(const string& name) {
	try {
		File dat(name, File::WRITE, File::CREATE | File::TRUNCATE);
		dat.setPos(1024*1024);
		dat.setEOF();
		dat.setPos(0);
		int64_t start = sizeof(start);
		dat.write(&start, sizeof(start));

	} catch(const FileException& e) {
		LogManager::getInstance()->message(STRING(ERROR_CREATING_HASH_DATA_FILE) + e.getError());
	}
}

#define BUF_SIZE (256*1024)

#ifdef _WIN32
bool HashManager::Hasher::fastHash(const string& fname, u_int8_t* buf, TigerTree& tth, int64_t size, CRC32Filter* xcrc32) {
	HANDLE h = INVALID_HANDLE_VALUE;
	DWORD x, y;
	if(!GetDiskFreeSpace(Text::toT(Util::getFilePath(fname)).c_str(), &y, &x, &y, &y)) {
		return false;
	} else {
		if((BUF_SIZE % x) != 0) {
			return false;
		} else {
			h = ::CreateFile(Text::toT(fname).c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, NULL);
			if(h == INVALID_HANDLE_VALUE)
				return false;
		}
	}
	DWORD hn = 0;
	DWORD rn = 0;
	u_int8_t* hbuf = buf + BUF_SIZE;
	u_int8_t* rbuf = buf;

	OVERLAPPED over = { 0 };
	over.hEvent = CreateEvent(NULL, FALSE, TRUE, NULL);
	
	bool ok = false;

	u_int32_t lastRead = GET_TICK();
	if(!::ReadFile(h, hbuf, BUF_SIZE, &hn, &over)) {
		if(GetLastError() == ERROR_HANDLE_EOF) {
			hn = 0;
		} else if(GetLastError() == ERROR_IO_PENDING) {
			if(!GetOverlappedResult(h, &over, &hn, TRUE)) {
				if(GetLastError() == ERROR_HANDLE_EOF) {
					hn = 0;
				} else {
					goto cleanup;
				}
			}
		} else {
			goto cleanup;
		}
	}

	over.Offset = hn;
	size -= hn;
	BOOL res = TRUE;
	for(;;) {
		if(size > 0) {
			// Start a new overlapped read
			ResetEvent(over.hEvent);
			if(SETTING(MAX_HASH_SPEED) > 0) {
				u_int32_t now = GET_TICK();
				u_int32_t minTime = hn * 1000LL / (SETTING(MAX_HASH_SPEED) * 1024LL * 1024LL);
				if(lastRead + minTime > now) {
					u_int32_t diff = now - lastRead;
					Thread::sleep(minTime - diff);
				} 
				lastRead = lastRead + minTime;
			} else {
				lastRead = GET_TICK();
			}
			res = ReadFile(h, rbuf, BUF_SIZE, &rn, &over);
		} else {
			rn = 0;
		}

		tth.update(hbuf, hn);
		if(xcrc32) (*xcrc32)(hbuf, hn);

		{
			Lock l(cs);
			currentSize = max(currentSize - hn, _LL(0));
		}

		if(size == 0) {
			ok = true;
			break;
		}

		if (!res) { 
			// deal with the error code 
			switch (GetLastError()) { 
			case ERROR_IO_PENDING: 
				if(!GetOverlappedResult(h, &over, &rn, TRUE)) {
					dcdebug("Error 0x%x: %s\n", GetLastError(), Util::translateError(GetLastError()).c_str());
					goto cleanup;
				}
				break;
			default:
				dcdebug("Error 0x%x: %s\n", GetLastError(), Util::translateError(GetLastError()).c_str());
				goto cleanup;
			}
		}

		*((u_int64_t*)&over.Offset) += rn;
		size -= rn;

		swap(rbuf, hbuf);
		swap(rn, hn);
	}

cleanup:
	::CloseHandle(over.hEvent);
	::CloseHandle(h);
	return ok;
}
#endif

int HashManager::Hasher::run() {
	setThreadPriority(Thread::IDLE);

	u_int8_t* buf = NULL;
	bool virtualBuf = true;

	string fname;
	bool last = false;
	for(;;) {
		s.wait();
		if(stop)
			break;
		if(rebuild) {
			HashManager::getInstance()->doRebuild();
			rebuild = false;
			LogManager::getInstance()->message(STRING(HASH_REBUILT));
			continue;
		}
		{
			Lock l(cs);
			if(!w.empty()) {
				currentFile = fname = w.begin()->first;
				currentSize = w.begin()->second;
				w.erase(w.begin());
				last = w.empty();
			} else {
				last = true;
				fname.clear();
			}
		}
		running = true;

		if(!fname.empty()) {
			int64_t size = File::getSize(fname);
			int64_t sizeLeft = size;
#ifdef _WIN32
			if(buf == NULL) {
				virtualBuf = true;
				buf = (u_int8_t*)VirtualAlloc(NULL, 2*BUF_SIZE, MEM_COMMIT, PAGE_READWRITE);
			}
#endif
			if(buf == NULL) {
				virtualBuf = false;
				buf = new u_int8_t[BUF_SIZE];
			}
			try {
				File f(fname, File::READ, File::OPEN);
				int64_t bs = max(TigerTree::calcBlockSize(f.getSize(), 10), MIN_BLOCK_SIZE);
				u_int32_t start = GET_TICK();
				u_int32_t timestamp = f.getLastModified();
				TigerTree slowTTH(bs);
				TigerTree* tth = &slowTTH;

				CRC32Filter crc32;
				SFVReader sfv(fname);
				CRC32Filter* xcrc32 = 0;
				if(sfv.hasCRC())
					xcrc32 = &crc32;

				size_t n = 0;
#ifdef _WIN32
				TigerTree fastTTH(bs);
				tth = &fastTTH;
				if(!virtualBuf || !fastHash(fname, buf, fastTTH, size, xcrc32)) {
					tth = &slowTTH;
					crc32 = CRC32Filter();
#endif
					u_int32_t lastRead = GET_TICK();

					do {
						size_t bufSize = BUF_SIZE;
						if(SETTING(MAX_HASH_SPEED) > 0) {
							u_int32_t now = GET_TICK();
							u_int32_t minTime = n * 1000LL / (SETTING(MAX_HASH_SPEED) * 1024LL * 1024LL);
							if(lastRead + minTime > now) {
								Thread::sleep(minTime - (now - lastRead));
							}
						}
						n = f.read(buf, bufSize);
						tth->update(buf, n);
						if(xcrc32) (*xcrc32)(buf, n);

						{
							Lock l(cs);
							currentSize = max(currentSize - n, _LL(0));
						}
						sizeLeft -= n;
					} while (n > 0 && !stop);
#ifdef _WIN32
				} else {
					sizeLeft = 0;
				}
#endif
				f.close();
				tth->finalize();
				u_int32_t end = GET_TICK();
				int64_t speed = 0;
				if(end > start) {
					speed = size * _LL(1000) / (end - start);
				}
				if(xcrc32 && xcrc32->getValue() != sfv.getCRC()) {
					LogManager::getInstance()->message(fname + STRING(NO_CRC32_MATCH));
				} else {
					HashManager::getInstance()->hashDone(fname, timestamp, *tth, speed);
				}
			} catch(const FileException& e) {
				LogManager::getInstance()->message(STRING(ERROR_HASHING) + fname + ": " + e.getError());
			}
		}
		{
			Lock l(cs);
			currentFile.clear();
			currentSize = 0;
		}
		running = false;
		if(buf != NULL && (last || stop)) {
			if(virtualBuf) {
#ifdef _WIN32
				VirtualFree(buf, 0, MEM_RELEASE);
#endif
			} else {
				delete buf;
			}
			buf = NULL;
		}
	}
	return 0;
}