store_dir.c

-

char *
storeDirSwapLogFile(int dirn, const char *ext)
{
    LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
    LOCAL_ARRAY(char, digit, 32);
    if (Config.Log.swap) {
	xstrncpy(path, Config.Log.swap, SQUID_MAXPATHLEN - 64);
	strcat(path, ".");
	snprintf(digit, 32, "%02d", dirn);
	strncat(path, digit, 3);
    } else {
	xstrncpy(path, storeSwapDir(dirn), SQUID_MAXPATHLEN - 64);
	strcat(path, "/swap.state");
    }
    if (ext)
	strncat(path, ext, 16);
    return path;
}

void
storeDirOpenSwapLogs(void)
{
    int i;
    char *path;
    int fd;
    SwapDir *SD;
    for (i = 0; i < Config.cacheSwap.n_configured; i++) {
	SD = &Config.cacheSwap.swapDirs[i];
	path = storeDirSwapLogFile(i, NULL);
	fd = file_open(path, O_WRONLY | O_CREAT, NULL, NULL, NULL);
	if (fd < 0) {
	    debug(50, 1) ("%s: %s\n", path, xstrerror());
	    fatal("storeDirOpenSwapLogs: Failed to open swap log.");
	}
	debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", i, fd);
	SD->swaplog_fd = fd;
    }
}

void
storeDirCloseSwapLogs(void)
{
    int i;
    SwapDir *SD;
    for (i = 0; i < Config.cacheSwap.n_configured; i++) {
	SD = &Config.cacheSwap.swapDirs[i];
	if (SD->swaplog_fd < 0)	/* not open */
	    continue;
	file_close(SD->swaplog_fd);
	debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", i, SD->swaplog_fd);
	SD->swaplog_fd = -1;
    }
}

FILE *
storeDirOpenTmpSwapLog(int dirn, int *clean_flag, int *zero_flag)
{
    char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL));
    char *clean_path = xstrdup(storeDirSwapLogFile(dirn, ".last-clean"));
    char *new_path = xstrdup(storeDirSwapLogFile(dirn, ".new"));
    struct stat log_sb;
    struct stat clean_sb;
    SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
    FILE *fp;
    int fd;
    if (stat(swaplog_path, &log_sb) < 0) {
	debug(47, 1) ("Cache Dir #%d: No log file\n", dirn);
	safe_free(swaplog_path);
	safe_free(clean_path);
	safe_free(new_path);
	return NULL;
    }
    *zero_flag = log_sb.st_size == 0 ? 1 : 0;
    /* close the existing write-only FD */
    if (SD->swaplog_fd >= 0)
	file_close(SD->swaplog_fd);
    /* open a write-only FD for the new log */
    fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL);
    if (fd < 0) {
	debug(50, 1) ("%s: %s\n", new_path, xstrerror());
	fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
    }
    SD->swaplog_fd = fd;
    /* open a read-only stream of the old log */
    fp = fopen(swaplog_path, "r");
    if (fp == NULL) {
	debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
	fatal("Failed to open swap log for reading");
    }
    memset(&clean_sb, '\0', sizeof(struct stat));
    if (stat(clean_path, &clean_sb) < 0)
	*clean_flag = 0;
    else if (clean_sb.st_mtime < log_sb.st_mtime)
	*clean_flag = 0;
    else
	*clean_flag = 1;
    safeunlink(clean_path, 1);
    safe_free(swaplog_path);
    safe_free(clean_path);
    safe_free(new_path);
    return fp;
}

void
storeDirCloseTmpSwapLog(int dirn)
{
    char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL));
    char *new_path = xstrdup(storeDirSwapLogFile(dirn, ".new"));
    SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
    int fd;
    file_close(SD->swaplog_fd);
#ifdef _SQUID_OS2_
    if (unlink(swaplog_path) < 0) {
	debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
	fatal("storeDirCloseTmpSwapLog: unlink failed");
    }
#endif
    if (rename(new_path, swaplog_path) < 0) {
	debug(50, 0) ("%s,%s: %s\n", new_path, swaplog_path, xstrerror());
	fatal("storeDirCloseTmpSwapLog: rename failed");
    }
    fd = file_open(swaplog_path, O_WRONLY | O_CREAT, NULL, NULL, NULL);
    if (fd < 0) {
	debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
	fatal("storeDirCloseTmpSwapLog: Failed to open swap log.");
    }
    safe_free(swaplog_path);
    safe_free(new_path);
    SD->swaplog_fd = fd;
    debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", dirn, fd);
}

void
storeDirUpdateSwapSize(int fn, size_t size, int sign)
{
    int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
    int k = ((size + 1023) >> 10) * sign;
    Config.cacheSwap.swapDirs[dirn].cur_size += k;
    store_swap_size += k;
    if (sign > 0)
	n_disk_objects++;
    else if (sign < 0)
	n_disk_objects--;
}

void
storeDirStats(StoreEntry * sentry)
{
    int i;
    SwapDir *SD;
#if HAVE_STATVFS
    struct statvfs sfs;
#endif
    storeAppendPrintf(sentry, "Store Directory Statistics:\n");
    storeAppendPrintf(sentry, "Store Entries          : %d\n",
	memInUse(MEM_STOREENTRY));
    storeAppendPrintf(sentry, "Maximum Swap Size      : %8d KB\n",
	Config.Swap.maxSize);
    storeAppendPrintf(sentry, "Current Store Swap Size: %8d KB\n",
	store_swap_size);
    storeAppendPrintf(sentry, "Current Capacity       : %d%% used, %d%% free\n",
	percent((int) store_swap_size, (int) Config.Swap.maxSize),
	percent((int) (Config.Swap.maxSize - store_swap_size), (int) Config.Swap.maxSize));
    for (i = 0; i < Config.cacheSwap.n_configured; i++) {
	SD = &Config.cacheSwap.swapDirs[i];
	storeAppendPrintf(sentry, "\n");
	storeAppendPrintf(sentry, "Store Directory #%d: %s\n", i, SD->path);
	storeAppendPrintf(sentry, "First level subdirectories: %d\n", SD->l1);
	storeAppendPrintf(sentry, "Second level subdirectories: %d\n", SD->l2);
	storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
	storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
	storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
	    100.0 * SD->cur_size / SD->max_size);
	storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
	    SD->map->n_files_in_map, SD->map->max_n_files,
	    percent(SD->map->n_files_in_map, SD->map->max_n_files));
#if HAVE_STATVFS
#define fsbtoblk(num, fsbs, bs) \
        (((fsbs) != 0 && (fsbs) < (bs)) ? \
                (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
	if (!statvfs(SD->path, &sfs)) {
	    storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
		fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024),
		fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024),
		percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks));
	    storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
		sfs.f_files - sfs.f_ffree, sfs.f_files,
		percent(sfs.f_files - sfs.f_ffree, sfs.f_files));
	}
#endif
	storeAppendPrintf(sentry, "Flags:");
	if (SD->flags.selected)
	    storeAppendPrintf(sentry, " SELECTED");
	if (SD->flags.read_only)
	    storeAppendPrintf(sentry, " READ-ONLY");
	storeAppendPrintf(sentry, "\n");
    }
}

int
storeDirMapBitsInUse(void)
{
    int i;
    int n = 0;
    for (i = 0; i < Config.cacheSwap.n_configured; i++)
	n += Config.cacheSwap.swapDirs[i].map->n_files_in_map;
    return n;
}

/*
 *  storeDirWriteCleanLogs
 * 
 *  Writes a "clean" swap log file from in-memory metadata.
 */
#define CLEAN_BUF_SZ 16384
int
storeDirWriteCleanLogs(int reopen)
{
    StoreEntry *e = NULL;
    int *fd;
    int n = 0;
    time_t start, stop, r;
    struct stat sb;
    char **cur;
    char **new;
    char **cln;
    int dirn;
    int N = Config.cacheSwap.n_configured;
    dlink_node *m;
    char **outbuf;
    off_t *outbufoffset;
    storeSwapLogData s;
    size_t ss = sizeof(storeSwapLogData);
    if (store_rebuilding) {
	debug(20, 1) ("Not currently OK to rewrite swap log.\n");
	debug(20, 1) ("storeDirWriteCleanLogs: Operation aborted.\n");
	return 0;
    }
    debug(20, 1) ("storeDirWriteCleanLogs: Starting...\n");
    start = squid_curtime;
    fd = xcalloc(N, sizeof(int));
    cur = xcalloc(N, sizeof(char *));
    new = xcalloc(N, sizeof(char *));
    cln = xcalloc(N, sizeof(char *));
    for (dirn = 0; dirn < N; dirn++) {
	fd[dirn] = -1;
	cur[dirn] = xstrdup(storeDirSwapLogFile(dirn, NULL));
	new[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".clean"));
	cln[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".last-clean"));
	unlink(new[dirn]);
	unlink(cln[dirn]);
	fd[dirn] = file_open(new[dirn],
	    O_WRONLY | O_CREAT | O_TRUNC,
	    NULL,
	    NULL,
	    NULL);
	if (fd[dirn] < 0) {
	    debug(50, 0) ("storeDirWriteCleanLogs: %s: %s\n", new[dirn], xstrerror());
	    continue;
	}
	debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
	    new[dirn], fd[dirn]);
#if HAVE_FCHMOD
	if (stat(cur[dirn], &sb) == 0)
	    fchmod(fd[dirn], sb.st_mode);
#endif
    }
    outbuf = xcalloc(N, sizeof(char *));
    outbufoffset = xcalloc(N, sizeof(*outbufoffset));
    for (dirn = 0; dirn < N; dirn++) {
	outbuf[dirn] = xcalloc(CLEAN_BUF_SZ, 1);
	outbufoffset[dirn] = 0;
    }
    for (m = store_list.tail; m; m = m->prev) {
	e = m->data;
	if (e->swap_file_number < 0)
	    continue;
	if (e->swap_status != SWAPOUT_DONE)
	    continue;
	if (e->swap_file_sz <= 0)
	    continue;
	if (EBIT_TEST(e->flags, RELEASE_REQUEST))
	    continue;
	if (EBIT_TEST(e->flags, KEY_PRIVATE))
	    continue;
	if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
	    continue;
	dirn = storeDirNumber(e->swap_file_number);
	assert(dirn < N);
	if (fd[dirn] < 0)
	    continue;
	memset(&s, '\0', ss);
	s.op = (char) SWAP_LOG_ADD;
	s.swap_file_number = e->swap_file_number;
	s.timestamp = e->timestamp;
	s.lastref = e->lastref;
	s.expires = e->expires;
	s.lastmod = e->lastmod;
	s.swap_file_sz = e->swap_file_sz;
	s.refcount = e->refcount;
	s.flags = e->flags;
	xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS);
	xmemcpy(outbuf[dirn] + outbufoffset[dirn], &s, ss);
	outbufoffset[dirn] += ss;
	/* buffered write */
	if (outbufoffset[dirn] + ss > CLEAN_BUF_SZ) {
	    if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) {
		debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
		    new[dirn], xstrerror());
		debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
		file_close(fd[dirn]);
		fd[dirn] = -1;
		unlink(new[dirn]);
		continue;
	    }
	    outbufoffset[dirn] = 0;
	}
	if ((++n & 0xFFFF) == 0) {
	    getCurrentTime();
	    debug(20, 1) ("  %7d entries written so far.\n", n);
	}
    }
    /* flush */
    for (dirn = 0; dirn < N; dirn++) {
	if (outbufoffset[dirn] == 0)
	    continue;
	if (fd[dirn] < 0)
	    continue;
	if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) {
	    debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
		new[dirn], xstrerror());
	    debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
	    file_close(fd[dirn]);
	    fd[dirn] = -1;
	    unlink(new[dirn]);
	    continue;
	}
	safe_free(outbuf[dirn]);
    }
    safe_free(outbuf);
    safe_free(outbufoffset);
    /*
     * You can't rename open files on Microsoft "operating systems"
     * so we have to close before renaming.
     */
    storeDirCloseSwapLogs();
    /* rename */
    for (dirn = 0; dirn < N; dirn++) {
	if (fd[dirn] < 0)
	    continue;
#ifdef _SQUID_OS2_
	file_close(fd[dirn]);
	fd[dirn] = -1;
	if (unlink(cur[dirn]) < 0)
	    debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n",
		xstrerror(), cur[dirn]);
#endif
	if (rename(new[dirn], cur[dirn]) < 0) {
	    debug(50, 0) ("storeDirWriteCleanLogs: rename failed: %s, %s -> %s\n",
		xstrerror(), new[dirn], cur[dirn]);
	}
    }
    if (reopen)
	storeDirOpenSwapLogs();
    stop = squid_curtime;
    r = stop - start;
    debug(20, 1) ("  Finished.  Wrote %d entries.\n", n);
    debug(20, 1) ("  Took %d seconds (%6.1f entries/sec).\n",
	r > 0 ? (int) r : 0,
	(double) n / (r > 0 ? r : 1));
    /* touch a timestamp file if we're not still validating */
    if (!store_rebuilding) {
	for (dirn = 0; dirn < N; dirn++) {
	    if (fd[dirn] < 0)
		continue;
	    file_close(file_open(cln[dirn],
		    O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL));
	}
    }
    /* close */
    for (dirn = 0; dirn < N; dirn++) {
	safe_free(cur[dirn]);
	safe_free(new[dirn]);
	safe_free(cln[dirn]);
	if (fd[dirn] < 0)
	    continue;
	file_close(fd[dirn]);
	fd[dirn] = -1;
    }
    safe_free(cur);
    safe_free(new);
    safe_free(cln);
    safe_free(fd);
    return n;
}
#undef CLEAN_BUF_SZ

void
storeDirConfigure(void)
{
    SwapDir *SD;
    int n;
    int i;
    fileMap *fm;
    Config.Swap.maxSize = 0;
    for (i = 0; i < Config.cacheSwap.n_configured; i++) {
	SD = &Config.cacheSwap.swapDirs[i];;
	Config.Swap.maxSize += SD->max_size;
	n = 2 * SD->max_size / Config.Store.avgObjectSize;
	if (NULL == SD->map) {
	    /* first time */
	    SD->map = file_map_create(n);
	} else if (n > SD->map->max_n_files) {
	    /* it grew, need to expand */
	    fm = file_map_create(n);
	    filemapCopy(SD->map, fm);
	    filemapFreeMemory(SD->map);
	    SD->map = fm;
	}
	/* else it shrunk, and we leave the old one in place */
    }
}

void
storeDirDiskFull(int fn)
{
    int dirn = fn >> SWAP_DIR_SHIFT;
    SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
    assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
    SD->max_size = SD->cur_size;
    debug(20, 1) ("WARNING: Shrinking cache_dir #%d to %d KB\n",
	dirn, SD->cur_size);
}