store_dir_diskd.c

一个功能非常全面的代理服务器源代码程序,

	     * and the validation procedure hasn't run. */
	    assert(rb->flags.need_to_validate);
	    rb->counts.clashcount++;
	    continue;
	} else if (e && !disk_entry_newer) {
	    /* key already exists, current entry is newer */
	    /* keep old, ignore new */
	    rb->counts.dupcount++;
	    continue;
	} else if (e) {
	    /* key already exists, this swapfile not being used */
	    /* junk old, load new */
	    storeRecycle(e);
	    rb->counts.dupcount++;
	} else {
	    /* URL doesnt exist, swapfile not in use */
	    /* load new */
	    (void) 0;
	}
	/* update store_swap_size */
	rb->counts.objcount++;
	e = storeDiskdDirAddDiskRestore(SD, s.key,
	    s.swap_filen,
	    s.swap_file_sz,
	    s.expires,
	    s.timestamp,
	    s.lastref,
	    s.lastmod,
	    s.refcount,
	    s.flags,
	    (int) rb->flags.clean);
	storeDirSwapLog(e, SWAP_LOG_ADD);
    }
    eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1);
}

#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
/* This is an exact copy of the above, but using storeSwapLogDataOld entry type */
static void
storeDiskdDirRebuildFromSwapLogOld(void *data)
{
    RebuildState *rb = data;
    SwapDir *SD = rb->sd;
    StoreEntry *e = NULL;
    storeSwapLogDataOld s;
    size_t ss = sizeof(storeSwapLogDataOld);
    int count;
    int used;			/* is swapfile already in use? */
    int disk_entry_newer;	/* is the log entry newer than current entry? */
    double x;
    assert(rb != NULL);
    /* load a number of objects per invocation */
    for (count = 0; count < rb->speed; count++) {
	if (fread(&s, ss, 1, rb->log) != 1) {
	    storeDiskdDirRebuildComplete(rb);
	    return;
	}
	rb->n_read++;
	/*
	 * BC: during 2.4 development, we changed the way swap file
	 * numbers are assigned and stored.  The high 16 bits used
	 * to encode the SD index number.  There used to be a call
	 * to storeDirProperFileno here that re-assigned the index
	 * bits.  Now, for backwards compatibility, we just need
	 * to mask it off.
	 */
	s.swap_filen &= 0x00FFFFFF;
	debug(20, 3) ("storeDiskdDirRebuildFromSwapLog: %s %s %08X\n",
	    swap_log_op_str[(int) s.op],
	    storeKeyText(s.key),
	    s.swap_filen);
	if (s.op == SWAP_LOG_ADD) {
	    /*
	     * Here we have some special checks for large files.
	     * I've been seeing a system crash followed by a reboot
	     * that seems to corrupt the swap log.  Squid believes
	     * that the disk holds some really large files.  It
	     * complains about using being over the high water mark
	     * and proceeds to delete files as fast as it can.  To
	     * prevent that, we call stat() on sufficiently large
	     * files (>128KB) and reject those that are missing or
	     * have the wrong size.
	     */
	    struct stat sb;
	    char *p = storeDiskdDirFullPath(SD, s.swap_filen, NULL);
	    if (s.swap_file_sz < (1 << 17)) {
		(void) 0;
	    } else if (stat(p, &sb) < 0) {
		debug(47, 2) ("its missing!: %s\n", p);
		continue;
	    } else if (sb.st_size != s.swap_file_sz) {
		debug(47, 2) ("size mismatch!: stat=%d, log=%d\n",
		    (int) sb.st_size, (int) s.swap_file_sz);
		continue;
	    } else {
		debug(47, 2) ("big file (%d bytes) checks out\n",
		    (int) s.swap_file_sz);
	    }
	} else if (s.op == SWAP_LOG_DEL) {
	    /* Delete unless we already have a newer copy */
	    if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) {
		/*
		 * Make sure we don't unlink the file, it might be
		 * in use by a subsequent entry.  Also note that
		 * we don't have to subtract from store_swap_size
		 * because adding to store_swap_size happens in
		 * the cleanup procedure.
		 */
		storeRecycle(e);
		rb->counts.objcount--;
		rb->counts.cancelcount++;
	    }
	    continue;
	} else {
	    x = log(++rb->counts.bad_log_op) / log(10.0);
	    if (0.0 == x - (double) (int) x)
		debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
		    rb->counts.bad_log_op);
	    rb->counts.invalid++;
	    continue;
	}
	if ((++rb->counts.scancount & 0xFFF) == 0) {
	    struct stat sb;
	    if (0 == fstat(fileno(rb->log), &sb))
		storeRebuildProgress(SD->index,
		    (int) sb.st_size / ss, rb->n_read);
	}
	if (!storeDiskdDirValidFileno(SD, s.swap_filen, 0)) {
	    rb->counts.invalid++;
	    continue;
	}
	if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
	    rb->counts.badflags++;
	    continue;
	}
	e = storeGet(s.key);
	used = storeDiskdDirMapBitTest(SD, s.swap_filen);
	/* If this URL already exists in the cache, does the swap log
	 * appear to have a newer entry?  Compare 'lastref' from the
	 * swap log to e->lastref. */
	disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
	if (used && !disk_entry_newer) {
	    /* log entry is old, ignore it */
	    rb->counts.clashcount++;
	    continue;
	} else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
	    /* swapfile taken, same URL, newer, update meta */
	    if (e->store_status == STORE_OK) {
		e->lastref = s.timestamp;
		e->timestamp = s.timestamp;
		e->expires = s.expires;
		e->lastmod = s.lastmod;
		e->flags = s.flags;
		e->refcount += s.refcount;
		storeDiskdDirUnrefObj(SD, e);
	    } else {
		debug_trap("storeDiskdDirRebuildFromSwapLog: bad condition");
		debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
	    }
	    continue;
	} else if (used) {
	    /* swapfile in use, not by this URL, log entry is newer */
	    /* This is sorta bad: the log entry should NOT be newer at this
	     * point.  If the log is dirty, the filesize check should have
	     * caught this.  If the log is clean, there should never be a
	     * newer entry. */
	    debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
		SD->index, s.swap_filen);
	    /* I'm tempted to remove the swapfile here just to be safe,
	     * but there is a bad race condition in the NOVM version if
	     * the swapfile has recently been opened for writing, but
	     * not yet opened for reading.  Because we can't map
	     * swapfiles back to StoreEntrys, we don't know the state
	     * of the entry using that file.  */
	    /* We'll assume the existing entry is valid, probably because
	     * were in a slow rebuild and the the swap file number got taken
	     * and the validation procedure hasn't run. */
	    assert(rb->flags.need_to_validate);
	    rb->counts.clashcount++;
	    continue;
	} else if (e && !disk_entry_newer) {
	    /* key already exists, current entry is newer */
	    /* keep old, ignore new */
	    rb->counts.dupcount++;
	    continue;
	} else if (e) {
	    /* key already exists, this swapfile not being used */
	    /* junk old, load new */
	    storeRecycle(e);
	    rb->counts.dupcount++;
	} else {
	    /* URL doesnt exist, swapfile not in use */
	    /* load new */
	    (void) 0;
	}
	/* update store_swap_size */
	rb->counts.objcount++;
	e = storeDiskdDirAddDiskRestore(SD, s.key,
	    s.swap_filen,
	    s.swap_file_sz,
	    s.expires,
	    s.timestamp,
	    s.lastref,
	    s.lastmod,
	    s.refcount,
	    s.flags,
	    (int) rb->flags.clean);
	storeDirSwapLog(e, SWAP_LOG_ADD);
    }
    eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLogOld, rb, 0.0, 1);
}

#endif

static void
storeDiskdDirRebuildFromSwapLogCheckVersion(void *data)
{
    RebuildState *rb = data;
    storeSwapLogHeader hdr;

    if (fread(&hdr, sizeof(hdr), 1, rb->log) != 1) {
	storeDiskdDirRebuildComplete(rb);
	return;
    }
    if (hdr.op == SWAP_LOG_VERSION) {
	if (fseek(rb->log, hdr.record_size, SEEK_SET) != 0) {
	    storeDiskdDirRebuildComplete(rb);
	    return;
	}
	if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogData)) {
	    eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1);
	    return;
	}
#if SIZEOF_SQUID_FILE_SZ != SIZEOF_SIZE_T
	if (hdr.version == 1 && hdr.record_size == sizeof(storeSwapLogDataOld)) {
	    debug(47, 1) ("storeDiskdDirRebuildFromSwapLog: Found current version but without large file support. Upgrading\n");
	    eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLogOld, rb, 0.0, 1);
	    return;
	}
#endif
	debug(47, 1) ("storeDiskdDirRebuildFromSwapLog: Unsupported swap.state version %d size %d\n",
	    hdr.version, hdr.record_size);
	storeDiskdDirRebuildComplete(rb);
	return;
    }
    rewind(rb->log);
    debug(47, 1) ("storeDiskdDirRebuildFromSwapLog: Old version detected. Upgrading\n");
#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
    eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1);
#else
    eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLogOld, rb, 0.0, 1);
#endif
}

static int
storeDiskdDirGetNextFile(RebuildState * rb, sfileno * filn_p, int *size)
{
    SwapDir *SD = rb->sd;
    diskdinfo_t *diskdinfo = SD->fsdata;
    int fd = -1;
    int used = 0;
    int dirs_opened = 0;
    debug(20, 3) ("storeDiskdDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
	rb->flags.init,
	rb->sd->index,
	rb->curlvl1,
	rb->curlvl2);
    if (rb->done)
	return -2;
    while (fd < 0 && rb->done == 0) {
	fd = -1;
	if (0 == rb->flags.init) {	/* initialize, open first file */
	    rb->done = 0;
	    rb->curlvl1 = 0;
	    rb->curlvl2 = 0;
	    rb->in_dir = 0;
	    rb->flags.init = 1;
	    assert(Config.cacheSwap.n_configured > 0);
	}
	if (0 == rb->in_dir) {	/* we need to read in a new directory */
	    snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
		rb->sd->path,
		rb->curlvl1, rb->curlvl2);
	    if (dirs_opened)
		return -1;
	    rb->td = opendir(rb->fullpath);
	    dirs_opened++;
	    if (rb->td == NULL) {
		debug(50, 1) ("storeDiskdDirGetNextFile: opendir: %s: %s\n",
		    rb->fullpath, xstrerror());
	    } else {
		rb->entry = readdir(rb->td);	/* skip . and .. */
		rb->entry = readdir(rb->td);
		if (rb->entry == NULL && errno == ENOENT)
		    debug(20, 1) ("storeDiskdDirGetNextFile: directory does not exist!.\n");
		debug(20, 3) ("storeDiskdDirGetNextFile: Directory %s\n", rb->fullpath);
	    }
	}
	if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) {
	    rb->in_dir++;
	    if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) {
		debug(20, 3) ("storeDiskdDirGetNextFile: invalid %s\n",
		    rb->entry->d_name);
		continue;
	    }
	    if (!storeDiskdFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) {
		debug(20, 3) ("storeDiskdDirGetNextFile: %08X does not belong in %d/%d/%d\n",
		    rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2);
		continue;
	    }
	    used = storeDiskdDirMapBitTest(SD, rb->fn);
	    if (used) {
		debug(20, 3) ("storeDiskdDirGetNextFile: Locked, continuing with next.\n");
		continue;
	    }
	    snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
		rb->fullpath, rb->entry->d_name);
	    debug(20, 3) ("storeDiskdDirGetNextFile: Opening %s\n", rb->fullfilename);
	    fd = file_open(rb->fullfilename, O_RDONLY | O_BINARY);
	    if (fd < 0)
		debug(50, 1) ("storeDiskdDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror());
	    else
		store_open_disk_fd++;
	    continue;
	}
	if (rb->td != NULL)
	    closedir(rb->td);
	rb->td = NULL;
	rb->in_dir = 0;
	if (++rb->curlvl2 < diskdinfo->l2)
	    continue;
	rb->curlvl2 = 0;
	if (++rb->curlvl1 < diskdinfo->l1)
	    continue;
	rb->curlvl1 = 0;
	rb->done = 1;
    }
    *filn_p = rb->fn;
    return fd;
}

/* Add a new object to the cache with empty memory copy and pointer to disk
 * use to rebuild store from disk. */
static StoreEntry *
storeDiskdDirAddDiskRestore(SwapDir * SD, const cache_key * key,
    int file_number,
    squid_file_sz swap_file_sz,
    time_t expires,
    time_t timestamp,
    time_t lastref,
    time_t lastmod,
    u_num32 refcount,
    u_short flags,
    int clean)
{
    StoreEntry *e = NULL;
    debug(20, 5) ("storeDiskdAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
    /* if you call this you'd better be sure file_number is not 
     * already in use! */
    e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
    e->store_status = STORE_OK;
    storeSetMemStatus(e, NOT_IN_MEMORY);
    e->swap_status = SWAPOUT_DONE;
    e->swap_filen = file_number;
    e->swap_dirn = SD->index;
    e->swap_file_sz = swap_file_sz;
    e->lock_count = 0;
    e->lastref = lastref;
    e->timestamp = timestamp;
    e->expires = expires;
    e->lastmod = lastmod;
    e->refcount = refcount;
    e->flags = flags;
    EBIT_SET(e->flags, ENTRY_CACHABLE);
    EBIT_CLR(e->flags, RELEASE_REQUEST);
    EBIT_CLR(e->flags, KEY_PRIVATE);
    e->ping_status = PING_NONE;
    EBIT_CLR(e->flags, ENTRY_VALIDATED);
    storeDiskdDirMapBitSet(SD, e->swap_filen);
    storeHashInsert(e, key);	/* do it after we clear KEY_PRIVATE */
    storeDiskdDirReplAdd(SD, e);
    return e;
}

CBDATA_TYPE(RebuildState);

static void
storeDiskdDirRebuild(SwapDir * sd)
{
    RebuildState *rb;
    int clean = 0;
    int zero = 0;
    FILE *fp;
    EVH *func = NULL;
    CBDATA_INIT_TYPE(RebuildState);
    rb = cbdataAlloc(RebuildState);
    rb->sd = sd;
    rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
    /*
     * If the swap.state file exists in the cache_dir, then
     * we'll use storeDiskdDirRebuildFromSwapLog(), otherwise we'll
     * use storeDiskdDirRebuildFromDirectory() to open up each file
     * and suck in the meta data.
     */
    fp = storeDiskdDirOpenTmpSwapLog(sd, &clean, &zero);
    if (fp == NULL || zero) {
	if (fp != NULL)
	    fclose(fp);
	func = storeDiskdDirRebuildFromDirectory;
    } else {
	func = storeDiskdDirRebuildFromSwapLogCheckVersion;
	rb->log = fp;
	rb->flags.clean = (unsigned int) clean;
    }
    if (!clean)
	rb->flags.need_to_validate = 1;
    debug(20, 1) ("Rebuilding storage in %s (%s)\n",
	sd->path, clean ? "CLEAN" : "DIRTY");
    store_dirs_rebuilding++;
    eventAdd("storeRebuild", func, rb, 0.0, 1);
}

static void
storeDiskdDirCloseTmpSwapLog(SwapDir * sd)
{
    diskdinfo_t *diskdinfo = sd->fsdata;
    char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL));
    char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new"));
    int fd;
    file_close(diskdinfo->swaplog_fd);
    if (xrename(new_path, swaplog_path) < 0) {
	fatal("storeDiskdDirCloseTmpSwapLog: rename failed");
    }
    fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY);