mydocument.cpp

本程序可实现网站资源下载

			pThis->m_fileLog.Write("[OK]\n",5);

			// Check for offsite links, don't follow the current page's
			// links if it is an offsite page
			if(bIsOffsite)
				nCount = 0;

			// Should we get multimedia files??
			if(pThis->m_Options.bMultimedia)
			{
				// Iterate through the list of multimedia links
				CString strMedia;
				for(int j = 0; j < pThis->m_aLinks[pThis->m_nLevel].arrMedia.GetSize(); j++)
				{
					strMedia = pThis->m_aLinks[pThis->m_nLevel].arrMedia.GetAt(j);

					// Should we get this file?
					if(pThis->ShouldGetMedia(strMedia,pMapEntry))
					{
						// Yep, make sure it has a unique name
						pThis->m_Inet.GenerateUniqueFileName(strMedia,
										strFileName,pThis->m_arrMediaDone,FALSE);

						// Log the info
						strLogData.Format("[%02d] Getting media %s ",pThis->m_nLevel,
														strMedia);
						pThis->m_fileLog.Write(strLogData,strLogData.GetLength());

						// We don't need to download EMAIL links so just make
						// them look like a successful file entry
						BOOL bMail;
						if(strMedia.Left(7) == "mailto:")
						{
							bMail = TRUE;
							strFileName = strMedia;
						}
						else bMail = FALSE;

						// Did everything work okay??
						if(bMail || pThis->GetMedia(strMedia,strFileName))
						{
							// Yep...add this file to our file list and to the tree
							pThis->SetMediaCacheEntry(strMedia,strFileName);
							pView->AddTreeMedia(strFileName.SpanExcluding("#"),
											CMyTreeCtrl::GetMediaType(strFileName));

							// Increment the statistics count
							if(!bMail)
								pThis->m_nGottenFileCount++;

							// Log the results
							pThis->m_fileLog.Write("[OK]\n",5);
						}
						else
						{
							// Log the results
							pThis->m_fileLog.Write("[FAILED] ",9);

							// Show a detailed error -- if possible
							CString strError = pThis->m_Inet.GetErrorText();
							pThis->m_fileLog.Write(strError,strError.GetLength());
							pThis->m_fileLog.Write("\n",1);
						}
					}
				}
			}
		}
		else
		{
			// Log the results
			pThis->m_fileLog.Write("[FAILED] ",9);

			// Show a detailed error -- if possible
			CString strError = pThis->m_Inet.GetErrorText();
			pThis->m_fileLog.Write(strError,strError.GetLength());
			pThis->m_fileLog.Write("\n",1);
			
			nCount = 0;
		}

		// Make sure the statistics window is updated properly

		// If we've hit the max page count then just get out
		if(pThis->m_Options.nMaxPages > 0 && 
				pThis->m_nGottenPageCount >= pThis->m_Options.nMaxPages) 
			break;

		// Continue recursion if we haven't hit maximum depth yet
		// and as long as we have links on this page
		if(pThis->m_nLevel < nMaxDepth && nCount > 0)
		{
			// Get the next page to parse
			strPage = pThis->m_aLinks[pThis->m_nLevel].arrLinks.GetAt(0);
			bIsOffsite = pThis->m_aLinks[pThis->m_nLevel].arrOffsite.GetAt(0);

			// Move to the next level, initialize its link info
			pThis->m_nLevel++;
			pThis->ResetLink(pThis->m_nLevel);

			// Queue the links
			pThis->m_nQueuedPageCount += nCount;
			continue;
		}

		// Finished will all links on this page, reset its link info
		pThis->ResetLink(pThis->m_nLevel);

		// Move back to the previous level
		pThis->m_nLevel--;

		// Find the next page on the second dimension
		if(pThis->m_nLevel >= 0)
		{
			int nMaxCount;

			// Find another page that has links
			while(pThis->m_nLevel >= 0)
			{
				// How many second dimension entries do we have??
				nMaxCount = pThis->m_aLinks[pThis->m_nLevel].arrLinks.GetSize();

				// Did we have another valid page at this level?
				if(pThis->m_aLinks[pThis->m_nLevel].nIndex < nMaxCount-1)
				{
					// Yes, get the next page
					pThis->m_aLinks[pThis->m_nLevel].nIndex++;
					pThis->m_nQueuedPageCount--;
					break;
				}
				else 
				{
					// No, back up a level in the tree
					pThis->m_nLevel--;
					pThis->m_nQueuedPageCount--;
				}
			}
		}
	}

	// Make sure the "stopping, please wait" message isn't displayed

	// Make sure that the tree was correctly parsed
	// nNodeCount should always be 0
	pThis->m_nLevel = pThis->m_nQueuedPageCount;

	// Should we fix up the links for browsing??
	if(pThis->m_Options.bFixupLinks)
	{
		// Yes, but the user has canceled the download so ask if they want
		// to fix up anyway
		if(false&&pView->ShowPrompt(IDS_FIXUP_PAGES,MB_ICONQUESTION|MB_YESNO) != IDYES)
			;
		else

		{
			// Everything was normal so just fixup the links
			pThis->FixupLinks();
		}

	}

	// Close the log file
	pThis->m_fileLog.Close();

	// Are we doing the TOC??
	if(pThis->m_Options.bContents)
	{
		// Yep...write the bottom of it and close the file
		CString strText;
		strText = "</UL>\r\n</BODY>\r\n</HTML>\r\n";
		pThis->m_fileContents.Write(strText,strText.GetLength());
		pThis->m_fileContents.Close();
	}

	// Let the user know that we've finished
	pView->SetSnagging(FALSE);

	// Finished with the maps, so free the memory
	pThis->ClearCacheMaps();

	// Make sure the "stopping, please wait" message isn't displayed

	// Close the main window if we're in automatic mode
	if(pThis->m_bAutoMode)
		AfxGetMainWnd()->PostMessage(WM_CLOSE);

	// Shutdown the WinInet Session
	try
	{
		pThis->m_Inet.CloseSession();
	}
	catch(...)
	{
	}

	pThis->m_nLevel = 0;

	return(0);
}

// Removes all of the entries in the page and file download lists.  These are
// CMap's so it must navigate the linked list and delete each of the CObject
// entries.
void CMyDoc::ClearCacheMaps()
{
	MAP_FILES *pMapEntry;
	CString strPage;
	POSITION pos;
	int i;

	// Zap the page list
	pos = m_arrPagesDone.GetStartPosition();
	for(i = 0; i < m_arrPagesDone.GetCount(); i++)
	{
		m_arrPagesDone.GetNextAssoc(pos,strPage,(CObject *&) pMapEntry);
		delete pMapEntry;
	}
	m_arrPagesDone.RemoveAll();

	// Zap the multimedia list
	pos = m_arrMediaDone.GetStartPosition();
	for(i = 0; i < m_arrMediaDone.GetCount(); i++)
	{
		m_arrMediaDone.GetNextAssoc(pos,strPage,(CObject *&) pMapEntry);
		delete pMapEntry;
	}
	m_arrMediaDone.RemoveAll();
}

// Adds a new entry to the retrieved page list
void CMyDoc::SetPageCacheEntry(LPCTSTR lpszPage, LPCTSTR lpszFileName, int nLevel)
{
	CString strTempPage = lpszPage;
	MAP_FILES *pMapEntry = new MAP_FILES();

	// The page name should always be saved as lowercase
	pMapEntry->strFileName = lpszFileName;
	pMapEntry->nMaxLevel = nLevel;
	strTempPage.MakeLower();
	strTempPage = strTempPage.SpanExcluding("#");

	// Assimilate it into the collective
	m_arrPagesDone.SetAt(strTempPage,(CObject *&) pMapEntry);

	// Check for a generated default page name...if we have one then add some
	// other default names so we don't break during link fixup
	if(pMapEntry->strFileName == "index.htm" && strTempPage.Find("index.htm") == -1)
	{
		// Common default page names, for links such as "www.mycompany.com/"
		char *aszDefPages[4] = { "index.htm", "index.html", "default.htm", 
										"default.html"};

		MAP_FILES *pTempMapEntry;

		CString strServer, strObject, strUser, strPassword;
		INTERNET_PORT nPort;
		DWORD dwServiceType;
		AfxParseURLEx(strTempPage,dwServiceType,strServer,strObject,nPort,
									strUser,strPassword,
									ICU_NO_ENCODE);

		// Add the default names
		if(!strServer.IsEmpty())
		{
			strServer = "http://"+strServer+"/";

			for(int i = 0; i < 4; i++)
			{
				pTempMapEntry = new MAP_FILES();
				pTempMapEntry->strFileName = pMapEntry->strFileName;
				pTempMapEntry->nMaxLevel = pMapEntry->nMaxLevel;
				m_arrPagesDone.SetAt(strServer+aszDefPages[i],(CObject *&) pTempMapEntry);
			}
		}
	}	
}

// Adds a new entry to the retrieved file list
void CMyDoc::SetMediaCacheEntry(LPCTSTR lpszMedia, LPCTSTR lpszFileName)
{
	CString strTempMedia = lpszMedia;
	MAP_FILES *pMapEntry = new MAP_FILES();

	// The file name should always be saved as lowercase
	pMapEntry->strFileName = lpszFileName;
	pMapEntry->nMaxLevel = -1;
	strTempMedia.MakeLower();
	strTempMedia = strTempMedia.SpanExcluding("#");

	// Assimilate it into the collective
	m_arrMediaDone.SetAt(strTempMedia,(CObject *&) pMapEntry);
}

// Sets up a snagging session and calls AfxBeginThread() to start the
// real download thread routine.  This routine returns immediately once that
// thread is started, freeing up the UI.
void CMyDoc::RecursiveDownload(LPCTSTR lpszURL)
{
	// Initialize the statistics
	m_nGottenPageCount = 0;
	m_nGottenFileCount = 0;
	m_nQueuedPageCount = 0;
	m_nTotalBytes = 0;
	m_strStartPage = lpszURL;
	ClearCacheMaps();

	// Initialize the statistics window display
	POSITION pos = GetFirstViewPosition();
	CMyView* pView = (CMyView *) GetNextView(pos);

	// Start the download thread
	CWinThread *pThread = AfxBeginThread(DownloadThread,this);
}

// Fixs up each of the downloaded pages to allow local browsing.  It
// basically goes through the downloaded page list and opens each 
// file specified then parses the page and fixes up each of the 
// links found.
void CMyDoc::FixupLinks()
{
	BYTE *pbyBuffer = m_byBuffer;
	int nLen;
	CFile fileIn, fileOut;
	CFileException ex;
	MAP_FILES* pMapEntry;
	int nIndex;
	int nBytesWritten;
	int nSectionLength;
	int nOffset;
	CString strFixupFileName;
	CString strPage;
	CString strFileName;
	CString strTempName = m_strDirectory+"$ssnag$.htm";
	BOOL bFixup;
	int nFiles = m_arrPagesDone.GetCount();
	BOOL bIndexDone = FALSE;

	// Update the statistics window to indicate the number of files
	// we have left to do

	// Iterate through the downloaded page list
	POSITION pos = m_arrPagesDone.GetStartPosition();
	for(int i = 0; i < m_arrPagesDone.GetCount(); i++)
	{
		m_arrPagesDone.GetNextAssoc(pos,strPage,(CObject *&) pMapEntry);

		if(pMapEntry->strFileName == "index.htm" || 
						pMapEntry->strFileName == "index.html")
		{
			if(bIndexDone)
			{
				// Decrement the queued file count in the stastics window
				continue;
			}	
			else bIndexDone = TRUE;
		}

		// Display the current file in that statistics window
		strFileName = m_strDirectory+pMapEntry->strFileName;

		// Open the local file for create 
		if(fileIn.Open(strFileName.SpanExcluding("#"),CFile::modeRead,&ex))
		{
			pbyBuffer = m_byBuffer;

			// Read the file into memory
			nLen = fileIn.Read(pbyBuffer,MAX_INET_BUFFER);
			fileIn.Close();

			// Parse it (tell the parser to look for only links, i.e. fixup mode)
			CHTMLFileParser Parser;
			Parser.SetFixupMode(TRUE);
			Parser.ResetArrays();
			Parser.SetGetMedia(m_Options.bMultimedia);
			Parser.SetPageURL(strPage);
			Parser.ParseText((char *)pbyBuffer,nLen);

			CString strServer, strObject, strUser, strPassword;
			INTERNET_PORT nPort;
			DWORD dwServiceType;
			AfxParseURLEx(strPage,dwServiceType,strServer,strObject,nPort,
									strUser,strPassword,
									ICU_NO_ENCODE);
			CString strOrigHost = "http://"+strServer+"/";

			// Open a temporary workfile
			if(fileOut.Open(strTempName,CFile::modeCreate|CFile::modeWrite,&ex))
			{
				CString strURL;

				nIndex = 0;
				nBytesWritten = 0;
				nOffset = 0;

				// Write the file and perform fix-ups (if necessary)
				for(int j = 0; j < Parser.GetFixups().arrURL.GetSize(); j++)
				{
					strURL = Parser.GetFixups().arrURL.GetAt(j);

					// Text page fixup??
					if(Parser.GetFixups().arrTextPage.GetAt(j))
					{
						// Did we download this page??
						if(!ShouldGetPage(strURL,pMapEntry))
						{
							// Yep...get it's local filename
							strFixupFileName = pMapEntry->strFileName;
							bFixup = TRUE;
						}
						else
						{							
							AfxParseURLEx(strURL,dwServiceType,strServer,
									strObject,nPort,strUser,strPassword,
									ICU_NO_ENCODE);

							if(!strServer.IsEmpty())
							{
								strFixupFileName = strURL;
								bFixup = TRUE;
							}
							else bFixup = FALSE;
						}
					}
					else // Media fixup
					{
						// Did we download this multimedia file??
						if(!ShouldGetMedia(strURL,pMapEntry))
						{
							// Yep...get it's local filename
							strFixupFileName = pMapEntry->strFileName;
							bFixup = TRUE;
						}
						else bFixup = FALSE;
					}

					// Write the data leading up to the reference that we are
					// going to fix up
					nSectionLength = Parser.GetFixups().arrIndex.GetAt(j)-nIndex-nOffset;

					try
					{
						fileOut.Write(pbyBuffer,nSectionLength);
					}
					catch(...)
					{
					}

					nBytesWritten += nSectionLength;
					pbyBuffer += nSectionLength;
					nOffset = 0;

					// Do we have an entry to fix up??
					if(bFixup)
					{
						// Yes, write the local filename
						try
						{
							fileOut.Write(" \"",2);
							fileOut.Write(strFixupFileName,strFixupFileName.GetLength());
						}
						catch(...)
						{
						}

						// Find the start of the old filename
						while(*pbyBuffer == ' ') 
						{
							pbyBuffer++;
							nBytesWritten++;
							nOffset++;
						}

						// Is the filename in quotes?
						if(*pbyBuffer == '"') 
						{
							// Yep, look for the end
							do
							{
								pbyBuffer++;
								nBytesWritten++;
								nOffset++;
							}
							while(*pbyBuffer != '"');

							pbyBuffer++;
							nBytesWritten++;
							nOffset++;
						}
						else
						{
							// Not in quotes, so just look for the first
							// trailing blank
							do
							{
								pbyBuffer++;
								nBytesWritten++;
								nOffset++;
							}
							while(*pbyBuffer != ' ');
						}

						// Write a trailing quote and blank, just to be safe
						fileOut.Write("\" ",2);
					}

					// Save the location where this fixup started
					nIndex = Parser.GetFixups().arrIndex.GetAt(j);
				}

				// Finished fixing up, so write the rest of the data left
				// in the file buffer
				try
				{
					if(nLen-nBytesWritten > 0)
						fileOut.Write(pbyBuffer,nLen-nBytesWritten);
				}
				catch(...)
				{
				}

				fileOut.Close();
			}
		}

		// Replace the original page with this new fixed up page
		remove(strFileName.SpanExcluding("#"));
		rename(strTempName,strFileName.SpanExcluding("#"));

		// Decrement the queued file count in the stastics window
	}

	// Get rid if any leftover work files (this might happen if the
	// user aborted in the middle)
	remove(strTempName);
}

/////////////////////////////////////////////////////////////////////////////
// CMyDoc diagnostics

#ifdef _DEBUG

// Check validity of object
void CMyDoc::AssertValid() const
{
	CDocument::AssertValid();
}

// Dump the object
void CMyDoc::Dump(CDumpContext& dc) const
{
	CDocument::Dump(dc);
}
#endif //_DEBUG