flist.c

Rsync 3.0.5 source code

		start_filelist_progress("building file list");
	else if (inc_recurse && verbose && !am_server)
		rprintf(FCLIENT, "sending incremental file list\n");

	start_write = stats.total_written;
	gettimeofday(&start_tv, NULL);

	if (relative_paths && protocol_version >= 30)
		implied_dirs = 1; /* We send flagged implied dirs */

#ifdef SUPPORT_HARD_LINKS
	if (preserve_hard_links && protocol_version >= 30 && !cur_flist)
		init_hard_links();
#endif

	flist = cur_flist = flist_new(0, "send_file_list");
	if (inc_recurse) {
		dir_flist = flist_new(FLIST_TEMP, "send_file_list");
		flags |= FLAG_DIVERT_DIRS;
	} else
		dir_flist = cur_flist;

	disable_buffering = io_start_buffering_out(f);
	if (filesfrom_fd >= 0) {
		if (argv[0] && !change_dir(argv[0], CD_NORMAL)) {
			rsyserr(FERROR_XFER, errno, "change_dir %s failed",
				full_fname(argv[0]));
			exit_cleanup(RERR_FILESELECT);
		}
		use_ff_fd = 1;
	}

	if (!orig_dir)
		orig_dir = strdup(curr_dir);

	while (1) {
		char fbuf[MAXPATHLEN], *fn, name_type;

		if (use_ff_fd) {
			if (read_line(filesfrom_fd, fbuf, sizeof fbuf, rl_flags) == 0)
				break;
			sanitize_path(fbuf, fbuf, "", 0, SP_KEEP_DOT_DIRS);
		} else {
			if (argc-- == 0)
				break;
			strlcpy(fbuf, *argv++, MAXPATHLEN);
			if (sanitize_paths)
				sanitize_path(fbuf, fbuf, "", 0, SP_KEEP_DOT_DIRS);
		}

		len = strlen(fbuf);
		if (relative_paths) {
			/* We clean up fbuf below. */
			name_type = NORMAL_NAME;
		} else if (!len || fbuf[len - 1] == '/') {
			if (len == 2 && fbuf[0] == '.') {
				/* Turn "./" into just "." rather than "./." */
				fbuf[--len] = '\0';
			} else {
				if (len + 1 >= MAXPATHLEN)
					overflow_exit("send_file_list");
				fbuf[len++] = '.';
				fbuf[len] = '\0';
			}
			name_type = DOTDIR_NAME;
		} else if (len > 1 && fbuf[len-1] == '.' && fbuf[len-2] == '.'
		    && (len == 2 || fbuf[len-3] == '/')) {
			if (len + 2 >= MAXPATHLEN)
				overflow_exit("send_file_list");
			fbuf[len++] = '/';
			fbuf[len++] = '.';
			fbuf[len] = '\0';
			name_type = DOTDIR_NAME;
		} else if (fbuf[len-1] == '.' && (len == 1 || fbuf[len-2] == '/'))
			name_type = DOTDIR_NAME;
		else
			name_type = NORMAL_NAME;

		dir = NULL;

		if (!relative_paths) {
			p = strrchr(fbuf, '/');
			if (p) {
				*p = '\0';
				if (p == fbuf)
					dir = "/";
				else
					dir = fbuf;
				len -= p - fbuf + 1;
				fn = p + 1;
			} else
				fn = fbuf;
		} else {
			if ((p = strstr(fbuf, "/./")) != NULL) {
				*p = '\0';
				if (p == fbuf)
					dir = "/";
				else {
					dir = fbuf;
					clean_fname(dir, 0);
				}
				fn = p + 3;
				while (*fn == '/')
					fn++;
				if (!*fn)
					*--fn = '\0'; /* ensure room for '.' */
			} else
				fn = fbuf;
			/* A leading ./ can be used in relative mode to affect
			 * the dest dir without its name being in the path. */
			if (*fn == '.' && fn[1] == '/' && !implied_dot_dir) {
				send_file_name(f, flist, ".", NULL,
				    (flags | FLAG_IMPLIED_DIR) & ~FLAG_CONTENT_DIR,
				    ALL_FILTERS);
				implied_dot_dir = 1;
			}
			len = clean_fname(fn, CFN_KEEP_TRAILING_SLASH
					    | CFN_DROP_TRAILING_DOT_DIR);
			if (len == 1) {
				if (fn[0] == '/') {
					fn = "/.";
					len = 2;
					name_type = DOTDIR_NAME;
				} else if (fn[0] == '.')
					name_type = DOTDIR_NAME;
			} else if (fn[len-1] == '/') {
				fn[--len] = '\0';
				if (len == 1 && *fn == '.')
					name_type = DOTDIR_NAME;
				else
					name_type = SLASH_ENDING_NAME;
			}
			/* Reject a ".." dir in the active part of the path. */
			for (p = fn; (p = strstr(p, "..")) != NULL; p += 2) {
				if ((p[2] == '/' || p[2] == '\0')
				 && (p == fn || p[-1] == '/')) {
					rprintf(FERROR,
					    "found \"..\" dir in relative path: %s\n",
					    fn);
					exit_cleanup(RERR_SYNTAX);
				}
			}
		}

		if (!*fn) {
			len = 1;
			fn = ".";
			name_type = DOTDIR_NAME;
		}

		dirlen = dir ? strlen(dir) : 0;
		if (dirlen != lastdir_len || memcmp(lastdir, dir, dirlen) != 0) {
			if (!change_pathname(NULL, dir, -dirlen))
				continue;
			lastdir = pathname;
			lastdir_len = pathname_len;
		} else if (!change_pathname(NULL, lastdir, lastdir_len))
			continue;

		if (fn != fbuf)
			memmove(fbuf, fn, len + 1);

		if (link_stat(fbuf, &st, copy_dirlinks || name_type != NORMAL_NAME) != 0
		 || (name_type != DOTDIR_NAME && is_daemon_excluded(fbuf, S_ISDIR(st.st_mode)))
		 || (relative_paths && path_is_daemon_excluded(fbuf, 1))) {
			io_error |= IOERR_GENERAL;
			rsyserr(FERROR_XFER, errno, "link_stat %s failed",
				full_fname(fbuf));
			continue;
		}

		/* A dot-dir should not be excluded! */
		if (name_type != DOTDIR_NAME
		 && is_excluded(fbuf, S_ISDIR(st.st_mode) != 0, ALL_FILTERS))
			continue;

		if (S_ISDIR(st.st_mode) && !xfer_dirs) {
			rprintf(FINFO, "skipping directory %s\n", fbuf);
			continue;
		}

		if (inc_recurse && relative_paths && *fbuf) {
			if ((p = strchr(fbuf+1, '/')) != NULL) {
				if (p - fbuf == 1 && *fbuf == '.') {
					if ((fn = strchr(p+1, '/')) != NULL)
						p = fn;
				} else
					fn = p;
				send_implied_dirs(f, flist, fbuf, fbuf, p, flags, name_type);
				if (fn == p)
					continue;
			}
		} else if (implied_dirs && (p=strrchr(fbuf,'/')) && p != fbuf) {
			/* Send the implied directories at the start of the
			 * source spec, so we get their permissions right. */
			send_implied_dirs(f, flist, fbuf, fbuf, p, flags, 0);
		}

		if (one_file_system)
			filesystem_dev = st.st_dev;

		if (recurse || (xfer_dirs && name_type != NORMAL_NAME)) {
			struct file_struct *file;
			file = send_file_name(f, flist, fbuf, &st,
					      FLAG_TOP_DIR | FLAG_CONTENT_DIR | flags,
					      NO_FILTERS);
			if (!file)
				continue;
			if (inc_recurse) {
				if (name_type == DOTDIR_NAME) {
					if (send_dir_depth < 0) {
						send_dir_depth = 0;
						change_local_filter_dir(fbuf, len, send_dir_depth);
					}
					send_directory(f, flist, fbuf, len, flags);
				}
			} else
				send_if_directory(f, flist, file, fbuf, len, flags);
		} else
			send_file_name(f, flist, fbuf, &st, flags, NO_FILTERS);
	}

	gettimeofday(&end_tv, NULL);
	stats.flist_buildtime = (int64)(end_tv.tv_sec - start_tv.tv_sec) * 1000
			      + (end_tv.tv_usec - start_tv.tv_usec) / 1000;
	if (stats.flist_buildtime == 0)
		stats.flist_buildtime = 1;
	start_tv = end_tv;

	write_byte(f, 0); /* Indicate end of file list */

#ifdef SUPPORT_HARD_LINKS
	if (preserve_hard_links && protocol_version >= 30 && !inc_recurse)
		idev_destroy();
#endif

	if (show_filelist_p())
		finish_filelist_progress(flist);

	gettimeofday(&end_tv, NULL);
	stats.flist_xfertime = (int64)(end_tv.tv_sec - start_tv.tv_sec) * 1000
			     + (end_tv.tv_usec - start_tv.tv_usec) / 1000;

	/* When converting names, both sides keep an unsorted file-list array
	 * because the names will differ on the sending and receiving sides
	 * (both sides will use the unsorted index number for each item). */

	/* Sort the list without removing any duplicates.  This allows the
	 * receiving side to ask for whatever name it kept.  For incremental
	 * recursion mode, the sender marks duplicate dirs so that it can
	 * send them together in a single file-list. */
	if (need_unsorted_flist) {
		if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
			out_of_memory("send_file_list");
		memcpy(flist->sorted, flist->files,
		       flist->used * sizeof (struct file_struct*));
	} else
		flist->sorted = flist->files;
	flist_sort_and_clean(flist, 0);
	file_total += flist->used;

	if (numeric_ids <= 0 && !inc_recurse)
		send_id_list(f);

	/* send the io_error flag */
	if (protocol_version < 30)
		write_int(f, ignore_errors ? 0 : io_error);
	else if (io_error && !ignore_errors)
		send_msg_int(MSG_IO_ERROR, io_error);

	if (disable_buffering)
		io_end_buffering_out();

	stats.flist_size = stats.total_written - start_write;
	stats.num_files = flist->used;

	if (verbose > 3)
		output_flist(flist);

	if (verbose > 2)
		rprintf(FINFO, "send_file_list done\n");

	if (inc_recurse) {
		send_dir_depth = 1;
		add_dirs_to_tree(-1, flist, dir_count);
		if (!file_total || strcmp(flist->sorted[flist->low]->basename, ".") != 0)
			flist->parent_ndx = -1;
		flist_done_allocating(flist);
		if (send_dir_ndx < 0) {
			write_ndx(f, NDX_FLIST_EOF);
			flist_eof = 1;
		}
		else if (file_total == 1) {
			/* If we're creating incremental file-lists and there
			 * was just 1 item in the first file-list, send 1 more
			 * file-list to check if this is a 1-file xfer. */
			send_extra_file_list(f, 1);
		}
	}

	return flist;
}

struct file_list *recv_file_list(int f)
{
	struct file_list *flist;
	int dstart, flags;
	int64 start_read;
	int save_verbose = verbose;

	if (!first_flist)
		rprintf(FLOG, "receiving file list\n");
	if (show_filelist_p())
		start_filelist_progress("receiving file list");
	else if (inc_recurse && verbose && !am_server && !first_flist)
		rprintf(FCLIENT, "receiving incremental file list\n");

	start_read = stats.total_read;

#ifdef SUPPORT_HARD_LINKS
	if (preserve_hard_links && !first_flist)
		init_hard_links();
#endif

	flist = flist_new(0, "recv_file_list");

	if (inc_recurse) {
		if (flist->ndx_start == 1)
			dir_flist = flist_new(FLIST_TEMP, "recv_file_list");
		dstart = dir_flist->used;
	} else {
		dir_flist = flist;
		dstart = 0;
	}

	if (am_server && verbose > 2)
		verbose = 2;
	while ((flags = read_byte(f)) != 0) {
		struct file_struct *file;

		flist_expand(flist, 1);

		if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
			flags |= read_byte(f) << 8;
		file = recv_file_entry(flist, flags, f);

		if (inc_recurse && S_ISDIR(file->mode)) {
			flist_expand(dir_flist, 1);
			dir_flist->files[dir_flist->used++] = file;
		}

		flist->files[flist->used++] = file;

		maybe_emit_filelist_progress(flist->used);

		if (verbose > 2) {
			char *name = f_name(file, NULL);
			rprintf(FINFO, "recv_file_name(%s)\n", NS(name));
		}
	}
	file_total += flist->used;
	verbose = save_verbose;

	if (verbose > 2)
		rprintf(FINFO, "received %d names\n", flist->used);

	if (show_filelist_p())
		finish_filelist_progress(flist);

	if (need_unsorted_flist) {
		/* Create an extra array of index pointers that we can sort for
		 * the generator's use (for wading through the files in sorted
		 * order and for calling flist_find()).  We keep the "files"
		 * list unsorted for our exchange of index numbers with the
		 * other side (since their names may not sort the same). */
		if (!(flist->sorted = new_array(struct file_struct *, flist->used)))
			out_of_memory("recv_file_list");
		memcpy(flist->sorted, flist->files,
		       flist->used * sizeof (struct file_struct*));
		if (inc_recurse && dir_flist->used > dstart) {
			static int dir_flist_malloced = 0;
			if (dir_flist_malloced < dir_flist->malloced) {
				dir_flist->sorted = realloc_array(dir_flist->sorted,
							struct file_struct *,
							dir_flist->malloced);
				dir_flist_malloced = dir_flist->malloced;
			}
			memcpy(dir_flist->sorted + dstart, dir_flist->files + dstart,
			       (dir_flist->used - dstart) * sizeof (struct file_struct*));
			fsort(dir_flist->sorted + dstart, dir_flist->used - dstart);
		}
	} else {
		flist->sorted = flist->files;
		if (inc_recurse && dir_flist->used > dstart) {
			dir_flist->sorted = dir_flist->files;
			fsort(dir_flist->sorted + dstart, dir_flist->used - dstart);
		}
	}

	if (inc_recurse)
		flist_done_allocating(flist);
	else if (f >= 0)
		recv_id_list(f, flist);

	flist_sort_and_clean(flist, relative_paths);

	if (protocol_version < 30) {
		/* Recv the io_error flag */
		if (ignore_errors)
			read_int(f);
		else
			io_error |= read_int(f);
	} else if (inc_recurse && flist->ndx_start == 1) {
		if (!file_total || strcmp(flist->sorted[flist->low]->basename, ".") != 0)
			flist->parent_ndx = -1;
	}

	if (verbose > 3)
		output_flist(flist);

	if (verbose > 2)
		rprintf(FINFO, "recv_file_list done\n");

	stats.flist_size += stats.total_read - start_read;
	stats.num_files += flist->used;

	return flist;
}

/* This is only used once by the receiver if the very first file-list
 * has exactly one item in it. */
void recv_additional_file_list(int f)
{
	struct file_list *flist;
	int ndx = read_ndx(f);
	if (ndx == NDX_FLIST_EOF) {
		flist_eof = 1;
		change_local_filter_dir(NULL, 0, 0);
	} else {
		ndx = NDX_FLIST_OFFSET - ndx;
		if (ndx < 0 || ndx >= dir_flist->used) {
			ndx = NDX_FLIST_OFFSET - ndx;
			rprintf(FERROR,
				"[%s] Invalid dir index: %d (%d - %d)\n",
				who_am_i(), ndx, NDX_FLIST_OFFSET,
				NDX_FLIST_OFFSET - dir_flist->used + 1);
			exit_cleanup(RERR_PROTOCOL);
		}
		if (verbose > 3) {
			rprintf(FINFO, "[%s] receiving flist for dir %d\n",
				who_am_i(), ndx);
		}
		flist = recv_file_list(f);
		flist->parent_ndx = ndx;
	}
}

/* Search for an identically-named item in the file list.  Note that the
 * items must agree in their directory-ness, or no match is returned. */
int flist_find(struct file_list *flist, struct file_struct *f)
{
	int low = flist->low, high = flist->high;
	int diff, mid, mid_up;

	while (low <= high) {
		mid = (low + high) / 2;
		if (F_IS_ACTIVE(flist->sorted[mid]))
			mid_up = mid;
		else {
			/* Scan for the next non-empty entry using the cached
			 * distance values.  If the value isn't fully up-to-
			 * date, update it. */
			mid_up = mid + F_DEPTH(flist->sorted[mid]);
			if (!F_IS_ACTIVE(flist->sorted[mid_up])) {
				do {
				    mid_up += F_DEPTH(flist->sorted[mid_up]);
				} while (!F_IS_ACTIVE(flist->sorted[mid_up]));
				F_DEPTH(flist->sorted[mid]) = mid_up - mid;
			}
			if (mid_up > high) {
				/* If there's nothing left above us, set high to
				 * a non-empty entry below us and continue. */
				high = mid - (int)flist->sorted[mid]->len32;
				if (!F_IS_ACTIVE(flist