flist.c

Rsync 3.0.5 source code

/*
 * Generate and receive file lists.
 *
 * Copyright (C) 1996 Andrew Tridgell
 * Copyright (C) 1996 Paul Mackerras
 * Copyright (C) 2001, 2002 Martin Pool <[e-mail]>
 * Copyright (C) 2002-2008 Wayne Davison
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, visit the http://fsf.org website.
 */

#include "rsync.h"
#include "ifuncs.h"
#include "rounding.h"
#include "io.h"

extern int verbose;
extern int am_root;
extern int am_server;
extern int am_daemon;
extern int am_sender;
extern int am_generator;
extern int inc_recurse;
extern int do_progress;
extern int always_checksum;
extern int module_id;
extern int ignore_errors;
extern int numeric_ids;
extern int recurse;
extern int use_qsort;
extern int xfer_dirs;
extern int filesfrom_fd;
extern int one_file_system;
extern int copy_dirlinks;
extern int keep_dirlinks;
extern int preserve_uid;
extern int preserve_gid;
extern int preserve_acls;
extern int preserve_xattrs;
extern int preserve_links;
extern int preserve_hard_links;
extern int preserve_devices;
extern int preserve_specials;
extern int uid_ndx;
extern int gid_ndx;
extern int eol_nulls;
extern int relative_paths;
extern int implied_dirs;
extern int file_extra_cnt;
extern int ignore_perishable;
extern int non_perishable_cnt;
extern int prune_empty_dirs;
extern int copy_links;
extern int copy_unsafe_links;
extern int protocol_version;
extern int sanitize_paths;
extern int munge_symlinks;
extern int need_unsorted_flist;
extern int sender_symlink_iconv;
extern int unsort_ndx;
extern struct stats stats;
extern char *filesfrom_host;

extern char curr_dir[MAXPATHLEN];

extern struct chmod_mode_struct *chmod_modes;

extern struct filter_list_struct filter_list;
extern struct filter_list_struct daemon_filter_list;

#ifdef ICONV_OPTION
extern int filesfrom_convert;
extern iconv_t ic_send, ic_recv;
#endif

#define PTR_SIZE (sizeof (struct file_struct *))

int io_error;
int checksum_len;
dev_t filesystem_dev; /* used to implement -x */

struct file_list *cur_flist, *first_flist, *dir_flist;
int send_dir_ndx = -1, send_dir_depth = -1;
int flist_cnt = 0; /* how many (non-tmp) file list objects exist */
int file_total = 0; /* total of all active items over all file-lists */
int flist_eof = 0; /* all the file-lists are now known */

#define NORMAL_NAME 0
#define SLASH_ENDING_NAME 1
#define DOTDIR_NAME 2

/* Starting from protocol version 26, we always use 64-bit ino_t and dev_t
 * internally, even if this platform does not allow files to have 64-bit inums.
 * The only exception is if we're on a platform with no 64-bit type at all.
 *
 * Because we use read_longint() to get these off the wire, if you transfer
 * devices or (for protocols < 30) hardlinks with dev or inum > 2**32 to a
 * machine with no 64-bit types then you will get an overflow error.
 *
 * Note that if you transfer devices from a 64-bit-devt machine (say, Solaris)
 * to a 32-bit-devt machine (say, Linux-2.2/x86) then the device numbers will
 * be truncated.  But it's a kind of silly thing to do anyhow. */

/* The tmp_* vars are used as a cache area by make_file() to store data
 * that the sender doesn't need to remember in its file list.  The data
 * will survive just long enough to be used by send_file_entry(). */
static dev_t tmp_rdev;
#ifdef SUPPORT_HARD_LINKS
static int64 tmp_dev, tmp_ino;
#endif
static char tmp_sum[MAX_DIGEST_LEN];

static char empty_sum[MAX_DIGEST_LEN];
static int flist_count_offset; /* for --delete --progress */
static int dir_count = 0;

static void flist_sort_and_clean(struct file_list *flist, int strip_root);
static void output_flist(struct file_list *flist);

void init_flist(void)
{
	if (verbose > 4) {
		rprintf(FINFO, "FILE_STRUCT_LEN=%d, EXTRA_LEN=%d\n",
			(int)FILE_STRUCT_LEN, (int)EXTRA_LEN);
	}
	checksum_len = protocol_version < 21 ? 2
		     : protocol_version < 30 ? MD4_DIGEST_LEN
		     : MD5_DIGEST_LEN;
}

static int show_filelist_p(void)
{
	return verbose && xfer_dirs && !am_server && !inc_recurse;
}

static void start_filelist_progress(char *kind)
{
	rprintf(FCLIENT, "%s ... ", kind);
	if (verbose > 1 || do_progress)
		rprintf(FCLIENT, "\n");
	rflush(FINFO);
}

static void emit_filelist_progress(int count)
{
	rprintf(FCLIENT, " %d files...\r", count);
}

static void maybe_emit_filelist_progress(int count)
{
	if (do_progress && show_filelist_p() && (count % 100) == 0)
		emit_filelist_progress(count);
}

static void finish_filelist_progress(const struct file_list *flist)
{
	if (do_progress) {
		/* This overwrites the progress line */
		rprintf(FINFO, "%d file%sto consider\n",
			flist->used, flist->used == 1 ? " " : "s ");
	} else
		rprintf(FINFO, "done\n");
}

void show_flist_stats(void)
{
	/* Nothing yet */
}

/* Stat either a symlink or its referent, depending on the settings of
 * copy_links, copy_unsafe_links, etc.  Returns -1 on error, 0 on success.
 *
 * If path is the name of a symlink, then the linkbuf buffer (which must hold
 * MAXPATHLEN chars) will be set to the symlink's target string.
 *
 * The stat structure pointed to by stp will contain information about the
 * link or the referent as appropriate, if they exist. */
static int readlink_stat(const char *path, STRUCT_STAT *stp, char *linkbuf)
{
#ifdef SUPPORT_LINKS
	if (link_stat(path, stp, copy_dirlinks) < 0)
		return -1;
	if (S_ISLNK(stp->st_mode)) {
		int llen = readlink(path, linkbuf, MAXPATHLEN - 1);
		if (llen < 0)
			return -1;
		linkbuf[llen] = '\0';
		if (copy_unsafe_links && unsafe_symlink(linkbuf, path)) {
			if (verbose > 1) {
				rprintf(FINFO,"copying unsafe symlink \"%s\" -> \"%s\"\n",
					path, linkbuf);
			}
			return x_stat(path, stp, NULL);
		}
		if (munge_symlinks && am_sender && llen > SYMLINK_PREFIX_LEN
		 && strncmp(linkbuf, SYMLINK_PREFIX, SYMLINK_PREFIX_LEN) == 0) {
			memmove(linkbuf, linkbuf + SYMLINK_PREFIX_LEN,
				llen - SYMLINK_PREFIX_LEN + 1);
		}
	}
	return 0;
#else
	return x_stat(path, stp, NULL);
#endif
}

int link_stat(const char *path, STRUCT_STAT *stp, int follow_dirlinks)
{
#ifdef SUPPORT_LINKS
	if (copy_links)
		return x_stat(path, stp, NULL);
	if (x_lstat(path, stp, NULL) < 0)
		return -1;
	if (follow_dirlinks && S_ISLNK(stp->st_mode)) {
		STRUCT_STAT st;
		if (x_stat(path, &st, NULL) == 0 && S_ISDIR(st.st_mode))
			*stp = st;
	}
	return 0;
#else
	return x_stat(path, stp, NULL);
#endif
}

static inline int is_daemon_excluded(const char *fname, int is_dir)
{
	if (daemon_filter_list.head
	 && check_filter(&daemon_filter_list, FLOG, fname, is_dir) < 0) {
		errno = ENOENT;
		return 1;
	}
	return 0;
}

static inline int path_is_daemon_excluded(char *path, int ignore_filename)
{
	if (daemon_filter_list.head) {
		char *slash = path;

		while ((slash = strchr(slash+1, '/')) != NULL) {
			int ret;
			*slash = '\0';
			ret = check_filter(&daemon_filter_list, FLOG, path, 1);
			*slash = '/';
			if (ret < 0) {
				errno = ENOENT;
				return 1;
			}
		}

		if (!ignore_filename
		 && check_filter(&daemon_filter_list, FLOG, path, 1) < 0) {
			errno = ENOENT;
			return 1;
		}
	}

	return 0;
}

/* This function is used to check if a file should be included/excluded
 * from the list of files based on its name and type etc.  The value of
 * filter_level is set to either SERVER_FILTERS or ALL_FILTERS. */
static int is_excluded(const char *fname, int is_dir, int filter_level)
{
#if 0 /* This currently never happens, so avoid a useless compare. */
	if (filter_level == NO_FILTERS)
		return 0;
#endif
	if (is_daemon_excluded(fname, is_dir))
		return 1;
	if (filter_level != ALL_FILTERS)
		return 0;
	if (filter_list.head
	    && check_filter(&filter_list, FINFO, fname, is_dir) < 0)
		return 1;
	return 0;
}

static void send_directory(int f, struct file_list *flist,
			   char *fbuf, int len, int flags);

static const char *pathname, *orig_dir;
static int pathname_len;

/* Make sure flist can hold at least flist->used + extra entries. */
static void flist_expand(struct file_list *flist, int extra)
{
	struct file_struct **new_ptr;

	if (flist->used + extra <= flist->malloced)
		return;

	if (flist->malloced < FLIST_START)
		flist->malloced = FLIST_START;
	else if (flist->malloced >= FLIST_LINEAR)
		flist->malloced += FLIST_LINEAR;
	else
		flist->malloced *= 2;

	/* In case count jumped or we are starting the list
	 * with a known size just set it. */
	if (flist->malloced < flist->used + extra)
		flist->malloced = flist->used + extra;

	new_ptr = realloc_array(flist->files, struct file_struct *,
				flist->malloced);

	if (verbose >= 2 && flist->malloced != FLIST_START) {
		rprintf(FCLIENT, "[%s] expand file_list pointer array to %.0f bytes, did%s move\n",
		    who_am_i(),
		    (double)sizeof flist->files[0] * flist->malloced,
		    (new_ptr == flist->files) ? " not" : "");
	}

	flist->files = new_ptr;

	if (!flist->files)
		out_of_memory("flist_expand");
}

static void flist_done_allocating(struct file_list *flist)
{
	void *ptr = pool_boundary(flist->file_pool, 8*1024);
	if (flist->pool_boundary == ptr)
		flist->pool_boundary = NULL; /* list didn't use any pool memory */
	else
		flist->pool_boundary = ptr;
}

/* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's
 * F_PATHNAME(), or (2) "NULL, dir, dirlen" to chdir() to the supplied dir,
 * with dir == NULL taken to be the starting directory, and dirlen < 0
 * indicating that strdup(dir) should be called and then the -dirlen length
 * value checked to ensure that it is not daemon-excluded. */
int change_pathname(struct file_struct *file, const char *dir, int dirlen)
{
	if (dirlen < 0) {
		char *cpy = strdup(dir);
		if (*cpy != '/')
			change_dir(orig_dir, CD_SKIP_CHDIR);
		if (path_is_daemon_excluded(cpy, 0))
			goto chdir_error;
		dir = cpy;
		dirlen = -dirlen;
	} else {
		if (file) {
			if (pathname == F_PATHNAME(file))
				return 1;
			dir = F_PATHNAME(file);
			if (dir)
				dirlen = strlen(dir);
		} else if (pathname == dir)
			return 1;
		if (dir && *dir != '/')
			change_dir(orig_dir, CD_SKIP_CHDIR);
	}

	pathname = dir;
	pathname_len = dirlen;

	if (!dir)
		dir = orig_dir;

	if (!change_dir(dir, CD_NORMAL)) {
	  chdir_error:
		io_error |= IOERR_GENERAL;
		rsyserr(FERROR_XFER, errno, "change_dir %s failed", full_fname(dir));
		if (dir != orig_dir)
			change_dir(orig_dir, CD_NORMAL);
		pathname = NULL;
		pathname_len = 0;
		return 0;
	}

	return 1;
}

static void send_file_entry(int f, const char *fname, struct file_struct *file,
#ifdef SUPPORT_LINKS
			    const char *symlink_name, int symlink_len,
#endif
			    int ndx, int first_ndx)
{
	static time_t modtime;
	static mode_t mode;
#ifdef SUPPORT_HARD_LINKS
	static int64 dev;
#endif
	static dev_t rdev;
	static uint32 rdev_major;
	static uid_t uid;
	static gid_t gid;
	static const char *user_name, *group_name;
	static char lastname[MAXPATHLEN];
	int first_hlink_ndx = -1;
	int l1, l2;
	int xflags;

	/* Initialize starting value of xflags. */
	if (protocol_version >= 30 && S_ISDIR(file->mode)) {
		dir_count++;
		if (file->flags & FLAG_CONTENT_DIR)
			xflags = file->flags & FLAG_TOP_DIR;
		else if (file->flags & FLAG_IMPLIED_DIR)
			xflags = XMIT_TOP_DIR | XMIT_NO_CONTENT_DIR;
		else
			xflags = XMIT_NO_CONTENT_DIR;
	} else
		xflags = file->flags & FLAG_TOP_DIR; /* FLAG_TOP_DIR == XMIT_TOP_DIR */

	if (file->mode == mode)
		xflags |= XMIT_SAME_MODE;
	else
		mode = file->mode;

	if ((preserve_devices && IS_DEVICE(mode))
	 || (preserve_specials && IS_SPECIAL(mode))) {
		if (protocol_version < 28) {
			if (tmp_rdev == rdev)
				xflags |= XMIT_SAME_RDEV_pre28;
			else
				rdev = tmp_rdev;
		} else {
			rdev = tmp_rdev;
			if ((uint32)major(rdev) == rdev_major)
				xflags |= XMIT_SAME_RDEV_MAJOR;
			else
				rdev_major = major(rdev);
			if (protocol_version < 30 && (uint32)minor(rdev) <= 0xFFu)
				xflags |= XMIT_RDEV_MINOR_8_pre30;
		}
	} else if (protocol_version < 28)
		rdev = MAKEDEV(0, 0);
	if (!preserve_uid || ((uid_t)F_OWNER(file) == uid && *lastname))
		xflags |= XMIT_SAME_UID;
	else {
		uid = F_OWNER(file);
		if (!numeric_ids) {
			user_name = add_uid(uid);
			if (inc_recurse && user_name)
				xflags |= XMIT_USER_NAME_FOLLOWS;
		}
	}
	if (!preserve_gid || ((gid_t)F_GROUP(file) == gid && *lastname))
		xflags |= XMIT_SAME_GID;
	else {
		gid = F_GROUP(file);
		if (!numeric_ids) {
			group_name = add_gid(gid);
			if (inc_recurse && group_name)
				xflags |= XMIT_GROUP_NAME_FOLLOWS;
		}
	}
	if (file->modtime == modtime)
		xflags |= XMIT_SAME_TIME;
	else
		modtime = file->modtime;

#ifdef SUPPORT_HARD_LINKS
	if (tmp_dev != 0) {
		if (protocol_version >= 30) {
			struct ht_int64_node *np = idev_find(tmp_dev, tmp_ino);
			first_hlink_ndx = (int32)(long)np->data - 1;
			if (first_hlink_ndx < 0) {
				np->data = (void*)(long)(first_ndx + ndx + 1);
				xflags |= XMIT_HLINK_FIRST;
			}
		} else {
			if (tmp_dev == dev) {
				if (protocol_version >= 28)
					xflags |= XMIT_SAME_DEV_pre30;
			} else
				dev = tmp_dev;