store_dir.c

-

/*
 * $Id: store_dir.c,v 1.84.2.1 1999/02/14 00:23:11 wessels Exp $
 *
 * DEBUG: section 47    Store Directory Routines
 * AUTHOR: Duane Wessels
 *
 * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
 * ----------------------------------------------------------
 *
 *  Squid is the result of efforts by numerous individuals from the
 *  Internet community.  Development is led by Duane Wessels of the
 *  National Laboratory for Applied Network Research and funded by the
 *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
 *  Duane Wessels and the University of California San Diego.  Please
 *  see the COPYRIGHT file for full details.  Squid incorporates
 *  software developed and/or copyrighted by other sources.  Please see
 *  the CREDITS file for full details.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *  
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *  
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
 *
 */

#include "squid.h"
#if HAVE_STATVFS
#if HAVE_SYS_STATVFS_H
#include <sys/statvfs.h>
#endif
#endif

#define SWAP_DIR_SHIFT 24
#define SWAP_FILE_MASK 0x00FFFFFF
#define DefaultLevelOneDirs     16
#define DefaultLevelTwoDirs     256

static char *storeSwapSubDir(int dirn, int subdirn);
static int storeDirSelectSwapDir(void);
static int storeVerifyDirectory(const char *path);
static int storeCreateDirectory(const char *path, int);
static void storeCreateSwapSubDirs(int j);

/* return full name to swapfile */
char *
storeSwapFullPath(int fn, char *fullpath)
{
    LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
    int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
    int filn = fn & SWAP_FILE_MASK;
    int L1 = Config.cacheSwap.swapDirs[dirn].l1;
    int L2 = Config.cacheSwap.swapDirs[dirn].l2;
    if (!fullpath)
	fullpath = fullfilename;
    fullpath[0] = '\0';
    snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
	Config.cacheSwap.swapDirs[dirn].path,
	((filn / L2) / L2) % L1,
	(filn / L2) % L2,
	filn);
    return fullpath;
}

static char *
storeSwapSubDir(int dirn, int subdirn)
{
    LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
    SwapDir *SD;
    assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
    SD = &Config.cacheSwap.swapDirs[dirn];
    assert(0 <= subdirn && subdirn < SD->l1);
    snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X",
	Config.cacheSwap.swapDirs[dirn].path,
	subdirn);
    return fullfilename;
}

char *
storeSwapSubSubDir(int fn, char *fullpath)
{
    LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
    int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
    int filn = fn & SWAP_FILE_MASK;
    int L1 = Config.cacheSwap.swapDirs[dirn].l1;
    int L2 = Config.cacheSwap.swapDirs[dirn].l2;
    if (!fullpath)
	fullpath = fullfilename;
    fullpath[0] = '\0';
    snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
	Config.cacheSwap.swapDirs[dirn].path,
	((filn / L2) / L2) % L1,
	(filn / L2) % L2);
    return fullpath;
}

/*
 * Does swapfile number 'fn' belong in cachedir #F0,
 * level1 dir #F1, level2 dir #F2?
 *
 * This is called by storeDirClean(), but placed here because
 * the algorithm needs to match storeSwapSubSubDir().
 *
 * Don't check that (fn >> SWAP_DIR_SHIFT) == F0 because
 * 'fn' may not have the directory bits set.
 */
int
storeFilenoBelongsHere(int fn, int F0, int F1, int F2)
{
    int D1, D2;
    int L1, L2;
    int filn = fn & SWAP_FILE_MASK;
    assert(F0 < Config.cacheSwap.n_configured);
    L1 = Config.cacheSwap.swapDirs[F0].l1;
    L2 = Config.cacheSwap.swapDirs[F0].l2;
    D1 = ((filn / L2) / L2) % L1;
    if (F1 != D1)
	return 0;
    D2 = (filn / L2) % L2;
    if (F2 != D2)
	return 0;
    return 1;
}

static int
storeCreateDirectory(const char *path, int should_exist)
{
    int created = 0;
    struct stat st;
    getCurrentTime();
    if (0 == stat(path, &st)) {
	if (S_ISDIR(st.st_mode)) {
	    debug(20, should_exist ? 3 : 1) ("%s exists\n", path);
	} else {
	    fatalf("Swap directory %s is not a directory.", path);
	}
    } else if (0 == mkdir(path, 0755)) {
	debug(20, should_exist ? 1 : 3) ("%s created\n", path);
	created = 1;
    } else {
	fatalf("Failed to make swap directory %s: %s",
	    path, xstrerror());
    }
    return created;
}

static int
storeVerifyDirectory(const char *path)
{
    struct stat sb;
    if (stat(path, &sb) < 0) {
	debug(20, 0) ("%s: %s\n", path, xstrerror());
	return -1;
    }
    if (S_ISDIR(sb.st_mode) == 0) {
	debug(20, 0) ("%s is not a directory\n", path);
	return -1;
    }
    return 0;
}

/*
 * This function is called by storeInit().  If this returns < 0,
 * then Squid exits, complains about swap directories not
 * existing, and instructs the admin to run 'squid -z'
 */
int
storeVerifyCacheDirs(void)
{
    int i;
    int j;
    const char *path;
    for (i = 0; i < Config.cacheSwap.n_configured; i++) {
	path = Config.cacheSwap.swapDirs[i].path;
	if (storeVerifyDirectory(path) < 0)
	    return -1;
	for (j = 0; j < Config.cacheSwap.swapDirs[i].l1; j++) {
	    path = storeSwapSubDir(i, j);
	    if (storeVerifyDirectory(path) < 0)
		return -1;
	}
    }
    return 0;
}

void
storeCreateSwapDirectories(void)
{
    int i;
    const char *path = NULL;
    for (i = 0; i < Config.cacheSwap.n_configured; i++) {
	path = Config.cacheSwap.swapDirs[i].path;
	debug(47, 3) ("Creating swap space in %s\n", path);
	storeCreateDirectory(path, 0);
	storeCreateSwapSubDirs(i);
    }
}

static void
storeCreateSwapSubDirs(int j)
{
    int i, k;
    int should_exist;
    SwapDir *SD = &Config.cacheSwap.swapDirs[j];
    LOCAL_ARRAY(char, name, MAXPATHLEN);
    for (i = 0; i < SD->l1; i++) {
	snprintf(name, MAXPATHLEN, "%s/%02X", SD->path, i);
	if (storeCreateDirectory(name, 0))
	    should_exist = 0;
	else
	    should_exist = 1;
	debug(47, 1) ("Making directories in %s\n", name);
	for (k = 0; k < SD->l2; k++) {
	    snprintf(name, MAXPATHLEN, "%s/%02X/%02X", SD->path, i, k);
	    storeCreateDirectory(name, should_exist);
	}
    }
}

/*
 *Spread load across least 3/4 of the store directories
 */
static int
storeDirSelectSwapDir(void)
{
    double least_used = 1.0;
    double high = (double) Config.Swap.highWaterMark / 100.0;
    double u;
    int dirn;
    int i, j;
    SwapDir *SD;
    static int nleast = 0;
    static int nconf = 0;
    static int *dirq = NULL;
    static double *diru = NULL;
    /*
     * Handle simplest case of a single swap directory immediately
     */
    if (Config.cacheSwap.n_configured == 1)
	return 0;
    /*
     * Initialise dirq on the first call or on change of number of dirs
     */
    if (nconf != Config.cacheSwap.n_configured) {
	nconf = Config.cacheSwap.n_configured;
	nleast = (nconf * 3) / 4;
	safe_free(dirq);
	dirq = (int *) xmalloc(sizeof(int) * nleast);
	safe_free(diru);
	diru = (double *) xmalloc(sizeof(double) * nconf);
	for (j = 0; j < nleast; j++)
	    dirq[j] = -1;
    }
    /*
     * Scan for a non-negative dirn in the dirq array and return that one
     */
    dirn = -1;
    for (j = 0; j < nleast; j++) {
	dirn = dirq[j];
	if (dirn < 0)
	    continue;
	dirq[j] = -1;
	break;
    }
    /*
     * If we found a valid dirn return it
     */
    if (dirn >= 0)
	return dirn;
    /*
     * Now for the real guts of the algorithm - building the dirq array
     */
    for (i = 0; i < nconf; i++) {
	diru[i] = 1.1;
	SD = &Config.cacheSwap.swapDirs[i];
	SD->flags.selected = 0;
	if (SD->flags.read_only)
	    continue;
	u = (double) SD->cur_size / SD->max_size;
	if (u > high)
	    continue;
	diru[i] = u;
    }
    for (j = 0; j < nleast; j++) {
	dirq[j] = -1;
	least_used = 1.0;
	dirn = -1;
	for (i = 0; i < nconf; i++) {
	    if (diru[i] < least_used) {
		least_used = diru[i];
		dirn = i;
	    }
	}
	if (dirn < 0)
	    break;
	dirq[j] = dirn;
	diru[dirn] = 1.1;
	/* set selected flag for debugging/cachemgr only */
	Config.cacheSwap.swapDirs[dirn].flags.selected = 1;
    }
    /*
     * Setup default return of 0 if no least found
     */
    if (dirq[0] < 0)
	dirq[0] = 0;
    dirn = dirq[0];
    dirq[0] = -1;
    return dirn;
}

int
storeDirValidFileno(int fn)
{
    int dirn = fn >> SWAP_DIR_SHIFT;
    int filn = fn & SWAP_FILE_MASK;
    if (dirn > Config.cacheSwap.n_configured)
	return 0;
    if (dirn < 0)
	return 0;
    if (filn < 0)
	return 0;
    if (filn > Config.cacheSwap.swapDirs[dirn].map->max_n_files)
	return 0;
    return 1;
}

int
storeDirMapBitTest(int fn)
{
    int dirn = fn >> SWAP_DIR_SHIFT;
    int filn = fn & SWAP_FILE_MASK;
    return file_map_bit_test(Config.cacheSwap.swapDirs[dirn].map, filn);
}

void
storeDirMapBitSet(int fn)
{
    int dirn = fn >> SWAP_DIR_SHIFT;
    int filn = fn & SWAP_FILE_MASK;
    file_map_bit_set(Config.cacheSwap.swapDirs[dirn].map, filn);
}

void
storeDirMapBitReset(int fn)
{
    int dirn = fn >> SWAP_DIR_SHIFT;
    int filn = fn & SWAP_FILE_MASK;
    file_map_bit_reset(Config.cacheSwap.swapDirs[dirn].map, filn);
}

int
storeDirMapAllocate(void)
{
    int dirn = storeDirSelectSwapDir();
    SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
    int filn = file_map_allocate(SD->map, SD->suggest);
    SD->suggest = filn + 1;
    return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK);
}

char *
storeSwapDir(int dirn)
{
    assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
    return Config.cacheSwap.swapDirs[dirn].path;
}

int
storeDirNumber(int swap_file_number)
{
    return swap_file_number >> SWAP_DIR_SHIFT;
}

int
storeDirProperFileno(int dirn, int fn)
{
    return (dirn << SWAP_DIR_SHIFT) | (fn & SWAP_FILE_MASK);
}

/*
 * An entry written to the swap log MUST have the following
 * properties.
 *   1.  It MUST be a public key.  It does no good to log
 *       a public ADD, change the key, then log a private
 *       DEL.  So we need to log a DEL before we change a
 *       key from public to private.
 *   2.  It MUST have a valid (> -1) swap_file_number.
 */
void
storeDirSwapLog(const StoreEntry * e, int op)
{
    storeSwapLogData *s;
    int dirn;
    dirn = e->swap_file_number >> SWAP_DIR_SHIFT;
    assert(dirn < Config.cacheSwap.n_configured);
    assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
    assert(e->swap_file_number >= 0);
    /*
     * icons and such; don't write them to the swap log
     */
    if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
	return;
    assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
    debug(20, 3) ("storeDirSwapLog: %s %s %08X\n",
	swap_log_op_str[op],
	storeKeyText(e->key),
	e->swap_file_number);
    s = xcalloc(1, sizeof(storeSwapLogData));
    s->op = (char) op;
    s->swap_file_number = e->swap_file_number;
    s->timestamp = e->timestamp;
    s->lastref = e->lastref;
    s->expires = e->expires;
    s->lastmod = e->lastmod;
    s->swap_file_sz = e->swap_file_sz;
    s->refcount = e->refcount;
    s->flags = e->flags;
    xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
    file_write(Config.cacheSwap.swapDirs[dirn].swaplog_fd,
	-1,
	s,
	sizeof(storeSwapLogData),
	NULL,
	NULL,
	xfree);
}