pg_backup_archiver.c

PostgreSQL7.4.6 for Linux

/*-------------------------------------------------------------------------
 *
 * pg_backup_archiver.c
 *
 *	Private implementation of the archiver routines.
 *
 *	See the headers to pg_restore for more details.
 *
 * Copyright (c) 2000, Philip Warner
 *	Rights are granted to use this software in any way so long
 *	as this notice is not removed.
 *
 *	The author is not responsible for loss or damages that may
 *	result from its use.
 *
 *
 * IDENTIFICATION
 *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.79.2.3 2004/07/19 21:02:42 tgl Exp $
 *
 *-------------------------------------------------------------------------
 */

#include "pg_backup.h"
#include "pg_dump.h"
#include "pg_backup_archiver.h"
#include "pg_backup_db.h"
#include "dumputils.h"

#include <ctype.h>
#include <errno.h>
#include <unistd.h>

#include "pqexpbuffer.h"
#include "libpq/libpq-fs.h"


typedef enum _teReqs_
{
	REQ_SCHEMA = 1,
	REQ_DATA = 2,
	REQ_ALL = REQ_SCHEMA + REQ_DATA
} teReqs;

static void _SortToc(ArchiveHandle *AH, TocSortCompareFn fn);
static int	_tocSortCompareByOIDNum(const void *p1, const void *p2);
static int	_tocSortCompareByIDNum(const void *p1, const void *p2);
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
		 const int compression, ArchiveMode mode);
static int	_printTocEntry(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt, bool isData);

static void _doSetFixedOutputState(ArchiveHandle *AH);
static void _doSetSessionAuth(ArchiveHandle *AH, const char *user);
static void _reconnectToDB(ArchiveHandle *AH, const char *dbname, const char *user);
static void _becomeUser(ArchiveHandle *AH, const char *user);
static void _becomeOwner(ArchiveHandle *AH, TocEntry *te);
static void _selectOutputSchema(ArchiveHandle *AH, const char *schemaName);

static teReqs _tocEntryRequired(TocEntry *te, RestoreOptions *ropt);
static void _disableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt);
static void _enableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt);
static TocEntry *_getTocEntry(ArchiveHandle *AH, int id);
static void _moveAfter(ArchiveHandle *AH, TocEntry *pos, TocEntry *te);
static void _moveBefore(ArchiveHandle *AH, TocEntry *pos, TocEntry *te);
static int	_discoverArchiveFormat(ArchiveHandle *AH);
static void _fixupOidInfo(TocEntry *te);
static Oid	_findMaxOID(const char *((*deps)[]));

const char *progname;
static char *modulename = gettext_noop("archiver");

static void _write_msg(const char *modulename, const char *fmt, va_list ap);
static void _die_horribly(ArchiveHandle *AH, const char *modulename, const char *fmt, va_list ap);

static int	_canRestoreBlobs(ArchiveHandle *AH);
static int	_restoringToDB(ArchiveHandle *AH);

/*
 *	Wrapper functions.
 *
 *	The objective it to make writing new formats and dumpers as simple
 *	as possible, if necessary at the expense of extra function calls etc.
 *
 */


/* Create a new archive */
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
			  const int compression)

{
	ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, archModeWrite);

	return (Archive *) AH;
}

/* Open an existing archive */
/* Public */
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
	ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, archModeRead);

	return (Archive *) AH;
}

/* Public */
void
CloseArchive(Archive *AHX)
{
	int			res = 0;
	ArchiveHandle *AH = (ArchiveHandle *) AHX;

	(*AH->ClosePtr) (AH);

	/* Close the output */
	if (AH->gzOut)
		res = GZCLOSE(AH->OF);
	else if (AH->OF != stdout)
		res = fclose(AH->OF);

	if (res != 0)
		die_horribly(AH, modulename, "could not close output archive file\n");
}

/* Public */
void
RestoreArchive(Archive *AHX, RestoreOptions *ropt)
{
	ArchiveHandle *AH = (ArchiveHandle *) AHX;
	TocEntry   *te = AH->toc->next;
	teReqs		reqs;
	OutputContext sav;
	int			impliedDataOnly;
	bool		defnDumped;

	AH->ropt = ropt;

	/*
	 * Check for nonsensical option combinations.
	 *
	 * NB: create+dropSchema is useless because if you're creating the DB,
	 * there's no need to drop individual items in it.  Moreover, if we
	 * tried to do that then we'd issue the drops in the database
	 * initially connected to, not the one we will create, which is very
	 * bad...
	 */
	if (ropt->create && ropt->dropSchema)
		die_horribly(AH, modulename, "-C and -c are incompatible options\n");

	/*
	 * If we're using a DB connection, then connect it.
	 */
	if (ropt->useDB)
	{
		ahlog(AH, 1, "connecting to database for restore\n");
		if (AH->version < K_VERS_1_3)
			die_horribly(AH, modulename, "direct database connections are not supported in pre-1.3 archives\n");

		/* XXX Should get this from the archive */
		AHX->minRemoteVersion = 070100;
		AHX->maxRemoteVersion = 999999;

		ConnectDatabase(AHX, ropt->dbname,
						ropt->pghost, ropt->pgport, ropt->username,
						ropt->requirePassword, ropt->ignoreVersion);
	}

	/*
	 * Work out if we have an implied data-only restore. This can happen
	 * if the dump was data only or if the user has used a toc list to
	 * exclude all of the schema data. All we do is look for schema
	 * entries - if none are found then we set the dataOnly flag.
	 *
	 * We could scan for wanted TABLE entries, but that is not the same as
	 * dataOnly. At this stage, it seems unnecessary (6-Mar-2001).
	 */
	if (!ropt->dataOnly)
	{
		te = AH->toc->next;
		impliedDataOnly = 1;
		while (te != AH->toc)
		{
			reqs = _tocEntryRequired(te, ropt);
			if ((reqs & REQ_SCHEMA) != 0)
			{					/* It's schema, and it's wanted */
				impliedDataOnly = 0;
				break;
			}
			te = te->next;
		}
		if (impliedDataOnly)
		{
			ropt->dataOnly = impliedDataOnly;
			ahlog(AH, 1, "implied data-only restore\n");
		}
	}

	/*
	 * Setup the output file if necessary.
	 */
	if (ropt->filename || ropt->compression)
		sav = SetOutput(AH, ropt->filename, ropt->compression);

	ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");

	/*
	 * Establish important parameter values right away.
	 */
	_doSetFixedOutputState(AH);

	/*
	 * Drop the items at the start, in reverse order
	 */
	if (ropt->dropSchema)
	{
		te = AH->toc->prev;
		while (te != AH->toc)
		{
			reqs = _tocEntryRequired(te, ropt);
			if (((reqs & REQ_SCHEMA) != 0) && te->dropStmt)
			{
				/* We want the schema */
				ahlog(AH, 1, "dropping %s %s\n", te->desc, te->tag);
				/* Select owner and schema as necessary */
				_becomeOwner(AH, te);
				_selectOutputSchema(AH, te->namespace);
				/* Drop it */
				ahprintf(AH, "%s", te->dropStmt);
			}
			te = te->prev;
		}
	}

	/*
	 * Now process each TOC entry
	 */
	te = AH->toc->next;
	while (te != AH->toc)
	{
		/* Work out what, if anything, we want from this entry */
		reqs = _tocEntryRequired(te, ropt);

		/* Dump any relevant dump warnings to stderr */
		if (!ropt->suppressDumpWarnings && strcmp(te->desc, "WARNING") == 0)
		{
			if (!ropt->dataOnly && te->defn != NULL && strlen(te->defn) != 0)
				write_msg(modulename, "warning from original dump file: %s\n", te->defn);
			else if (te->copyStmt != NULL && strlen(te->copyStmt) != 0)
				write_msg(modulename, "warning from original dump file: %s\n", te->copyStmt);
		}

		defnDumped = false;

		if ((reqs & REQ_SCHEMA) != 0)	/* We want the schema */
		{
			ahlog(AH, 1, "creating %s %s\n", te->desc, te->tag);
			_printTocEntry(AH, te, ropt, false);
			defnDumped = true;

			/* If we created a DB, connect to it... */
			if (strcmp(te->desc, "DATABASE") == 0)
			{
				ahlog(AH, 1, "connecting to new database \"%s\" as user \"%s\"\n", te->tag, te->owner);
				_reconnectToDB(AH, te->tag, te->owner);
			}
		}

		/*
		 * If we have a data component, then process it
		 */
		if ((reqs & REQ_DATA) != 0)
		{
			/*
			 * hadDumper will be set if there is genuine data component
			 * for this node. Otherwise, we need to check the defn field
			 * for statements that need to be executed in data-only
			 * restores.
			 */
			if (te->hadDumper)
			{
				/*
				 * If we can output the data, then restore it.
				 */
				if (AH->PrintTocDataPtr !=NULL && (reqs & REQ_DATA) != 0)
				{
#ifndef HAVE_LIBZ
					if (AH->compression != 0)
						die_horribly(AH, modulename, "cannot restore from compressed archive (not configured for compression support)\n");
#endif

					_printTocEntry(AH, te, ropt, true);

					/*
					 * Maybe we can't do BLOBS, so check if this node is
					 * for BLOBS
					 */
					if ((strcmp(te->desc, "BLOBS") == 0) &&
						!_canRestoreBlobs(AH))
					{
						ahprintf(AH, "--\n-- SKIPPED \n--\n\n");

						/*
						 * This is a bit nasty - we assume, for the
						 * moment, that if a custom output is used, then
						 * we don't want warnings.
						 */
						if (!AH->CustomOutPtr)
							write_msg(modulename, "WARNING: skipping large-object restoration\n");
					}
					else
					{
						_disableTriggersIfNecessary(AH, te, ropt);

						/* Select owner and schema as necessary */
						_becomeOwner(AH, te);
						_selectOutputSchema(AH, te->namespace);

						ahlog(AH, 1, "restoring data for table \"%s\"\n", te->tag);

						/*
						 * If we have a copy statement, use it. As of
						 * V1.3, these are separate to allow easy import
						 * from withing a database connection. Pre 1.3
						 * archives can not use DB connections and are
						 * sent to output only.
						 *
						 * For V1.3+, the table data MUST have a copy
						 * statement so that we can go into appropriate
						 * mode with libpq.
						 */
						if (te->copyStmt && strlen(te->copyStmt) > 0)
							ahprintf(AH, te->copyStmt);

						(*AH->PrintTocDataPtr) (AH, te, ropt);

						_enableTriggersIfNecessary(AH, te, ropt);
					}
				}
			}
			else if (!defnDumped)
			{
				/* If we haven't already dumped the defn part, do so now */
				ahlog(AH, 1, "executing %s %s\n", te->desc, te->tag);
				_printTocEntry(AH, te, ropt, false);
			}
		}
		te = te->next;
	}		/* end loop over TOC entries */

	/*
	 * Now use blobs_xref (if used) to fixup any refs for tables that we
	 * loaded
	 */
	if (_canRestoreBlobs(AH) && AH->createdBlobXref)
	{
		/* NULL parameter means disable ALL user triggers */
		_disableTriggersIfNecessary(AH, NULL, ropt);

		te = AH->toc->next;
		while (te != AH->toc)
		{
			/* Is it table data? */
			if (strcmp(te->desc, "TABLE DATA") == 0)
			{
				ahlog(AH, 2, "checking whether we loaded \"%s\"\n", te->tag);

				reqs = _tocEntryRequired(te, ropt);

				if ((reqs & REQ_DATA) != 0)		/* We loaded the data */
				{
					ahlog(AH, 1, "fixing up large-object cross-reference for \"%s\"\n", te->tag);
					FixupBlobRefs(AH, te);
				}
			}
			else
				ahlog(AH, 2, "ignoring large-object cross-references for %s %s\n", te->desc, te->tag);

			te = te->next;
		}

		/* NULL parameter means enable ALL user triggers */
		_enableTriggersIfNecessary(AH, NULL, ropt);
	}

	/*
	 * Clean up & we're done.
	 */
	if (ropt->filename || ropt->compression)
		ResetOutput(AH, sav);

	if (ropt->useDB)
	{
		PQfinish(AH->connection);
		AH->connection = NULL;

		if (AH->blobConnection)
		{
			PQfinish(AH->blobConnection);
			AH->blobConnection = NULL;
		}
	}
}

/*
 * Allocate a new RestoreOptions block.
 * This is mainly so we can initialize it, but also for future expansion,
 */
RestoreOptions *
NewRestoreOptions(void)
{
	RestoreOptions *opts;

	opts = (RestoreOptions *) calloc(1, sizeof(RestoreOptions));

	opts->format = archUnknown;
	opts->suppressDumpWarnings = false;

	return opts;
}

/*
 * Returns true if we're restoring directly to the database (and
 * aren't just making a psql script that can do the restoration).
 */
static int
_restoringToDB(ArchiveHandle *AH)
{
	return (AH->ropt->useDB && AH->connection);
}

static int
_canRestoreBlobs(ArchiveHandle *AH)
{
	return _restoringToDB(AH);
}

static void
_disableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
{
	/* This hack is only needed in a data-only restore */
	if (!ropt->dataOnly || !ropt->disable_triggers)
		return;

	/* Don't do it for the BLOBS TocEntry, either */
	if (te && strcmp(te->desc, "BLOBS") == 0)
		return;

	/*
	 * Become superuser if possible, since they are the only ones who can
	 * update pg_class.  If -S was not given, assume the initial user identity
	 * is a superuser.
	 */
	_becomeUser(AH, ropt->superuser);

	ahlog(AH, 1, "disabling triggers\n");

	/*
	 * Disable them. This is a hack. Needs to be done via an appropriate
	 * 'SET' command when one is available.
	 */
	ahprintf(AH, "-- Disable triggers\n");

	/*
	 * Just update the AFFECTED table, if known.  Otherwise update all
	 * non-system tables.
	 */
	if (te && te->tag && strlen(te->tag) > 0)
		ahprintf(AH, "UPDATE pg_catalog.pg_class SET reltriggers = 0 "
				 "WHERE oid = '%s'::pg_catalog.regclass;\n\n",
				 fmtId(te->tag));
	else
		ahprintf(AH, "UPDATE pg_catalog.pg_class SET reltriggers = 0 FROM pg_catalog.pg_namespace "
				 "WHERE relnamespace = pg_namespace.oid AND nspname !~ '^pg_';\n\n");
}

static void
_enableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
{
	/* This hack is only needed in a data-only restore */
	if (!ropt->dataOnly || !ropt->disable_triggers)
		return;

	/* Don't do it for the BLOBS TocEntry, either */
	if (te && strcmp(te->desc, "BLOBS") == 0)
		return;

	/*
	 * Become superuser if possible, since they are the only ones who can
	 * update pg_class.  If -S was not given, assume the initial user identity
	 * is a superuser.
	 */
	_becomeUser(AH, ropt->superuser);

	ahlog(AH, 1, "enabling triggers\n");

	/*
	 * Enable them. This is a hack. Needs to be done via an appropriate
	 * 'SET' command when one is available.
	 */
	ahprintf(AH, "-- Enable triggers\n");

	/*
	 * Just update the AFFECTED table, if known.  Otherwise update all
	 * non-system tables.
	 */
	if (te && te->tag && strlen(te->tag) > 0)
		ahprintf(AH, "UPDATE pg_catalog.pg_class SET reltriggers = "
				 "(SELECT pg_catalog.count(*) FROM pg_catalog.pg_trigger where pg_class.oid = tgrelid) "
				 "WHERE oid = '%s'::pg_catalog.regclass;\n\n",
				 fmtId(te->tag));
	else
		ahprintf(AH, "UPDATE pg_catalog.pg_class SET reltriggers = "
				 "(SELECT pg_catalog.count(*) FROM pg_catalog.pg_trigger where pg_class.oid = tgrelid) "
				 "FROM pg_catalog.pg_namespace "
				 "WHERE relnamespace = pg_namespace.oid AND nspname !~ '^pg_';\n\n");
}

/*
 * This is a routine that is part of the dumper interface, hence the 'Archive*' parameter.
 */

/* Public */
size_t
WriteData(Archive *AHX, const void *data, size_t dLen)
{
	ArchiveHandle *AH = (ArchiveHandle *) AHX;

	if (!AH->currToc)
		die_horribly(AH, modulename, "internal error -- WriteData cannot be called outside the context of a DataDumper routine\n");

	return (*AH->WriteDataPtr) (AH, data, dLen);
}

/*
 * Create a new TOC entry. The TOC was designed as a TOC, but is now the
 * repository for all metadata. But the name has stuck.
 */

/* Public */
void
ArchiveEntry(Archive *AHX, const char *oid, const char *tag,
			 const char *namespace, const char *owner,
			 const char *desc, const char *((*deps)[]),
			 const char *defn, const char *dropStmt,
			 const char *copyStmt,
			 DataDumperPtr dumpFn, void *dumpArg)
{
	ArchiveHandle *AH = (ArchiveHandle *) AHX;
	TocEntry   *newToc;

	AH->lastID++;
	AH->tocCount++;

	newToc = (TocEntry *) calloc(1, sizeof(TocEntry));
	if (!newToc)
		die_horribly(AH, modulename, "out of memory\n");

	newToc->prev = AH->toc->prev;
	newToc->next = AH->toc;
	AH->toc->prev->next = newToc;
	AH->toc->prev = newToc;

	newToc->id = AH->lastID;

	newToc->tag = strdup(tag);
	newToc->namespace = namespace ? strdup(namespace) : NULL;
	newToc->owner = strdup(owner);
	newToc->desc = strdup(desc);
	newToc->defn = strdup(defn);
	newToc->dropStmt = strdup(dropStmt);
	newToc->copyStmt = copyStmt ? strdup(copyStmt) : NULL;

	newToc->oid = strdup(oid);
	newToc->depOid = deps;		/* NB: not copied */
	_fixupOidInfo(newToc);

	newToc->printed = 0;
	newToc->formatData = NULL;
	newToc->dataDumper = dumpFn;
	newToc->dataDumperArg = dumpArg;

	newToc->hadDumper = dumpFn ? 1 : 0;

	if (AH->ArchiveEntryPtr !=NULL)
		(*AH->ArchiveEntryPtr) (AH, newToc);

	/*
	 * printf("New toc owned by '%s', oid %u\n", newToc->owner,
	 * newToc->oidVal);
	 */
}

/* Public */
void
PrintTOCSummary(Archive *AHX, RestoreOptions *ropt)
{
	ArchiveHandle *AH = (ArchiveHandle *) AHX;
	TocEntry   *te = AH->toc->next;
	OutputContext sav;
	char	   *fmtName;

	if (ropt->filename)
		sav = SetOutput(AH, ropt->filename, 0 /* no compression */);

	ahprintf(AH, ";\n; Archive created at %s", ctime(&AH->createDate));
	ahprintf(AH, ";     dbname: %s\n;     TOC Entries: %d\n;     Compression: %d\n",
			 AH->archdbname, AH->tocCount, AH->compression);

	switch (AH->format)
	{
		case archFiles:
			fmtName = "FILES";
			break;
		case archCustom:
			fmtName = "CUSTOM";
			break;
		case archTar:
			fmtName = "TAR";
			break;
		default:
			fmtName = "UNKNOWN";
	}

	ahprintf(AH, ";     Dump Version: %d.%d-%d\n", AH->vmaj, AH->vmin, AH->vrev);
	ahprintf(AH, ";     Format: %s\n", fmtName);
	ahprintf(AH, ";     Integer: %d bytes\n", (int) AH->intSize);
	ahprintf(AH, ";     Offset: %d bytes\n", (int) AH->offSize);

	ahprintf(AH, ";\n;\n; Selected TOC Entries:\n;\n");

	while (te != AH->toc)
	{
		if (_tocEntryRequired(te, ropt) != 0)
			ahprintf(AH, "%d; %d %s %s %s\n", te->id, te->oidVal, te->desc, te->tag, te->owner);
		te = te->next;
	}

	if (ropt->filename)
		ResetOutput(AH, sav);
}

/***********
 * BLOB Archival
 ***********/

/* Called by a dumper to signal start of a BLOB */
int
StartBlob(Archive *AHX, Oid oid)
{