store_io_coss.c

一个功能非常全面的代理服务器源代码程序,

/*
 * $Id: store_io_coss.c,v 1.34 2006/09/23 10:34:41 serassio Exp $
 *
 * DEBUG: section 79    Storage Manager COSS Interface
 * AUTHOR: Eric Stern
 *
 * SQUID Web Proxy Cache          http://www.squid-cache.org/
 * ----------------------------------------------------------
 *
 *  Squid is the result of efforts by numerous individuals from
 *  the Internet community; see the CONTRIBUTORS file for full
 *  details.   Many organizations have provided support for Squid's
 *  development; see the SPONSORS file for full details.  Squid is
 *  Copyrighted (C) 2001 by the Regents of the University of
 *  California; see the COPYRIGHT file for full details.  Squid
 *  incorporates software developed and/or copyrighted by other
 *  sources; see the CREDITS file for full details.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *  
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *  
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
 *
 */

#include "squid.h"
#if HAVE_AIO_H
#include <aio.h>
#endif
#include "async_io.h"
#include "store_coss.h"
#if USE_AUFSOPS
#include "../aufs/async_io.h"
#endif

#if USE_AUFSOPS
static AIOCB storeCossWriteMemBufDone;
#else
static DWCB storeCossWriteMemBufDone;
#endif
static void storeCossIOCallback(storeIOState * sio, int errflag);
static char *storeCossMemPointerFromDiskOffset(CossInfo * cs, off_t offset, CossMemBuf ** mb);
static void storeCossMemBufLock(SwapDir * SD, storeIOState * e);
static void storeCossMemBufUnlock(SwapDir * SD, storeIOState * e);
static void storeCossWriteMemBuf(SwapDir * SD, CossMemBuf * t);
static CossMemBuf *storeCossCreateMemBuf(SwapDir * SD, int stripe, sfileno curfn, int *collision);
static CossMemBuf *storeCossCreateMemOnlyBuf(SwapDir * SD);
static CBDUNL storeCossIOFreeEntry;
static off_t storeCossFilenoToDiskOffset(sfileno f, CossInfo *);
static sfileno storeCossDiskOffsetToFileno(off_t o, CossInfo *);
static void storeCossMaybeWriteMemBuf(SwapDir * SD, CossMemBuf * t);
static void storeCossMaybeFreeBuf(CossInfo * cs, CossMemBuf * mb);
int storeCossFilenoToStripe(CossInfo * cs, sfileno filen);

static void membuf_describe(CossMemBuf * t, int level, int line);

/* Handle relocates - temporary routines until readops have been fleshed out */
void storeCossNewPendingRelocate(CossInfo * cs, storeIOState * sio, sfileno original_filen, sfileno new_filen);
CossPendingReloc *storeCossGetPendingReloc(CossInfo * cs, sfileno new_filen);
#if USE_AUFSOPS
AIOCB storeCossCompletePendingReloc;
#else
DRCB storeCossCompletePendingReloc;
#endif

/* Read operation code */
CossReadOp *storeCossCreateReadOp(CossInfo * cs, storeIOState * sio);
void storeCossCompleteReadOp(CossInfo * cs, CossReadOp * op, int error);
void storeCossKickReadOp(CossInfo * cs, CossReadOp * op);

CBDATA_TYPE(storeIOState);
CBDATA_TYPE(CossMemBuf);
CBDATA_TYPE(CossPendingReloc);

/* === PUBLIC =========================================================== */

static sfileno
storeCossMemOnlyAllocate(SwapDir * SD, const StoreEntry * e)
{
    CossInfo *cs = (CossInfo *) SD->fsdata;
    CossMemBuf *newmb;
    off_t retofs;
    size_t allocsize;
    sfileno f;

    coss_stats.alloc.memalloc++;
    allocsize = e->swap_file_sz;

    if (cs->current_memonly_membuf == NULL) {
	newmb = storeCossCreateMemOnlyBuf(SD);
	cs->current_memonly_membuf = newmb;

	if (newmb == NULL) {
	    return -1;
	}
	cs->current_memonly_offset = cs->current_memonly_membuf->diskstart;
    } else if ((cs->current_memonly_offset + allocsize) >= cs->current_memonly_membuf->diskend) {
	debug(79, 3) ("storeCossMemOnlyAllocate: overflow for buffer %d (%p)\n", cs->curmemstripe, cs->current_memonly_membuf);
	cs->current_memonly_membuf->flags.full = 1;
	storeCossMaybeWriteMemBuf(SD, cs->current_memonly_membuf);
	/* cs->current_memonly_membuf may be invalid at this point */

	newmb = storeCossCreateMemOnlyBuf(SD);
	cs->current_memonly_membuf = newmb;

	if (newmb == NULL) {
	    return -1;
	}
	cs->current_memonly_offset = cs->current_memonly_membuf->diskstart;
    }
    retofs = cs->current_memonly_offset;
    cs->current_memonly_offset = retofs + allocsize;
    cs->current_memonly_membuf->numobjs++;
    cs->current_memonly_offset = ((cs->current_memonly_offset + cs->blksz_mask) >> cs->blksz_bits) << cs->blksz_bits;
    f = storeCossDiskOffsetToFileno(retofs, cs);
    assert(f >= 0 && f <= 0xffffff);
    debug(79, 3) ("storeCossMemOnlyAllocate: offset %" PRId64 ", filen: %d\n", (int64_t) retofs, f);
    return f;

}

/*
 * This routine sucks. I want to rewrite it when possible, and I also think
 * that we should check after creatmembuf() to see if the object has a
 * RELEASE_REQUEST set on it (thanks Eric!) rather than this way which seems
 * to work..
 * -- Adrian
 */
static sfileno
storeCossAllocate(SwapDir * SD, const StoreEntry * e, int which)
{
    CossInfo *cs = (CossInfo *) SD->fsdata;
    CossMemBuf *newmb;
    off_t retofs;
    size_t allocsize;
    int coll = 0;
    sfileno f;
    sfileno checkf;

    /* Make sure we chcek collisions if reallocating */
    if (which == COSS_ALLOC_REALLOC) {
	checkf = e->swap_filen;
	coss_stats.alloc.realloc++;
    } else {
	checkf = -1;
	coss_stats.alloc.alloc++;
    }

    if (e->swap_file_sz > 0)
	allocsize = e->swap_file_sz;
    else
	allocsize = objectLen(e) + e->mem_obj->swap_hdr_sz;

    /* Since we're not supporting NOTIFY anymore, lets fail */
    assert(which != COSS_ALLOC_NOTIFY);

    /* Check to see if we need to allocate a membuf to start */
    if (cs->current_membuf == NULL) {
	if (cs->curstripe < (cs->numstripes - 1))
	    newmb = storeCossCreateMemBuf(SD, cs->curstripe + 1, checkf, &coll);
	else
	    newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll);

	cs->current_membuf = newmb;
	if (newmb == NULL) {
	    cs->sizerange_max = SD->max_objsize;
	    return -1;
	}
	cs->current_offset = cs->current_membuf->diskstart;

	/* Check if we have overflowed the disk .. */
    } else if ((cs->current_offset + allocsize) > ((off_t) SD->max_size << 10)) {
	/*
	 * tried to allocate past the end of the disk, so wrap
	 * back to the beginning
	 */
	coss_stats.disk_overflows++;
	cs->current_membuf->flags.full = 1;
	cs->numfullstripes++;
	cs->current_membuf->diskend = cs->current_offset;
	storeCossMaybeWriteMemBuf(SD, cs->current_membuf);
	/* cs->current_membuf may be invalid at this point */
	cs->current_offset = 0;	/* wrap back to beginning */
	debug(79, 2) ("storeCossAllocate: %s: wrap to 0\n", stripePath(SD));

	newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll);
	cs->current_membuf = newmb;
	if (newmb == NULL) {
	    cs->sizerange_max = SD->max_objsize;
	    return -1;
	}
	/* Check if we have overflowed the MemBuf */
    } else if ((cs->current_offset + allocsize) >= cs->current_membuf->diskend) {
	/*
	 * Skip the blank space at the end of the stripe. start over.
	 */
	coss_stats.stripe_overflows++;
	cs->current_membuf->flags.full = 1;
	cs->numfullstripes++;
	cs->current_offset = cs->current_membuf->diskend;
	storeCossMaybeWriteMemBuf(SD, cs->current_membuf);
	/* cs->current_membuf may be invalid at this point */
	debug(79, 3) ("storeCossAllocate: %s: New offset - %" PRId64 "\n", stripePath(SD),
	    (int64_t) cs->current_offset);
	assert(cs->curstripe < (cs->numstripes - 1));
	newmb = storeCossCreateMemBuf(SD, cs->curstripe + 1, checkf, &coll);
	cs->current_membuf = newmb;
	if (newmb == NULL) {
	    cs->sizerange_max = SD->max_objsize;
	    return -1;
	}
    }
    /* If we didn't get a collision, then update the current offset and return it */
    if (coll == 0) {
	retofs = cs->current_offset;
	cs->current_offset = retofs + allocsize;
	cs->current_membuf->numobjs++;
	/* round up to our blocksize */
	cs->current_offset = ((cs->current_offset + cs->blksz_mask) >> cs->blksz_bits) << cs->blksz_bits;
	f = storeCossDiskOffsetToFileno(retofs, cs);
	assert(f >= 0 && f <= 0xffffff);
	debug(79, 3) ("storeCossAllocate: offset %" PRId64 ", filen: %d\n", (int64_t) retofs, f);

	/* 
	 * Keep track of the largest object we can accept based on the
	 * max-wasted-space value
	 */
	cs->sizerange_max = cs->current_membuf->diskend - cs->current_offset;
	if (cs->sizerange_max < cs->sizerange_min)
	    cs->sizerange_max = cs->sizerange_min;

	return f;
    } else {
	/* Reset this to a safe value */
	cs->sizerange_max = SD->max_objsize;

	coss_stats.alloc.collisions++;
	debug(79, 3) ("storeCossAllocate: %s: Collision\n", stripePath(SD));
	return -1;
    }
}

void
storeCossUnlink(SwapDir * SD, StoreEntry * e)
{
    debug(79, 3) ("storeCossUnlink: %s: offset %d\n", stripePath(SD), e->swap_filen);
    coss_stats.unlink.ops++;
    coss_stats.unlink.success++;
    storeCossRemove(SD, e);
}

void
storeCossRecycle(SwapDir * SD, StoreEntry * e)
{
    debug(79, 3) ("storeCossRecycle: %s: offset %d\n", stripePath(SD), e->swap_filen);

    /* Expire the object */
    storeExpireNow(e);
    storeReleaseRequest(e);

    /* If there is a valid filen remove from COSS linked list */
    if (e->swap_filen > -1) {
	storeCossUnlink(SD, e);

	/* 
	 * Set filen and dirn to -1.  
	 * This makes storeRelease() treat the entry differently 
	 */
	e->swap_filen = -1;
	e->swap_dirn = -1;
    }
    /* Finally make the store layer forget about this object */
    storeRelease(e);
}

static int
storeCossRelocateRequired(CossInfo * cs, sfileno f)
{
    int stripes_written;
    int original_stripe = storeCossFilenoToStripe(cs, f);

    if (cs->curstripe > original_stripe)
	stripes_written = cs->curstripe - original_stripe;
    else
	stripes_written = cs->numstripes + cs->curstripe - original_stripe;

    /* Relocate if stripes_written > minimum_stripe_distance */
    return (stripes_written > cs->minimum_stripe_distance);
}

storeIOState *
storeCossCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, STIOCB * callback, void *callback_data)
{
    CossState *cstate;
    storeIOState *sio;
    CossInfo *cs = SD->fsdata;

    assert(cs->rebuild.rebuilding == 0);
    coss_stats.create.ops++;
    sio = cbdataAlloc(storeIOState);
    cstate = memPoolAlloc(coss_state_pool);
    sio->fsstate = cstate;
    sio->offset = 0;
    sio->mode = O_WRONLY | O_BINARY;

    /*
     * If we get handed an object with a size of -1,
     * the squid code is broken
     */
    assert(e->mem_obj->object_sz != -1);

    /*
     * this one is kinda strange - Eric called storeCossAllocate(), then
     * storeCossOpen(O_RDONLY) .. weird. Anyway, I'm allocating this now.
     */
    sio->st_size = objectLen(e) + e->mem_obj->swap_hdr_sz;
    sio->swap_dirn = SD->index;
    sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_ALLOCATE);
    debug(79, 3) ("storeCossCreate: %p: filen: %d\n", sio, sio->swap_filen);
    assert(-1 != sio->swap_filen);

    sio->callback = callback;
    sio->file_callback = file_callback;
    sio->callback_data = callback_data;
    cbdataLock(callback_data);
    sio->e = (StoreEntry *) e;

    cstate->flags.writing = 0;
    cstate->flags.reading = 0;
    cstate->reqdiskoffset = -1;

    /* Now add it into the index list */
    e->swap_filen = sio->swap_filen;
    e->swap_dirn = sio->swap_dirn;
    storeCossAdd(SD, e, cs->curstripe);

    storeCossMemBufLock(SD, sio);
    coss_stats.create.success++;
    return sio;
}

storeIOState *
storeCossOpen(SwapDir * SD, StoreEntry * e, STFNCB * file_callback,
    STIOCB * callback, void *callback_data)
{
    storeIOState *sio;
    char *p;
    CossState *cstate;
    sfileno f = e->swap_filen;
    sfileno nf;
    CossInfo *cs = (CossInfo *) SD->fsdata;

    assert(cs->rebuild.rebuilding == 0);

    sio = cbdataAlloc(storeIOState);
    cstate = memPoolAlloc(coss_state_pool);

    debug(79, 3) ("storeCossOpen: %p: offset %d\n", sio, f);
    coss_stats.open.ops++;

    sio->fsstate = cstate;
    sio->swap_filen = f;
    sio->swap_dirn = SD->index;
    sio->offset = 0;
    sio->mode = O_RDONLY | O_BINARY;
    sio->callback = callback;
    sio->file_callback = file_callback;
    sio->callback_data = callback_data;
    cbdataLock(callback_data);
    sio->st_size = e->swap_file_sz;
    sio->e = e;

    cstate->flags.writing = 0;
    cstate->flags.reading = 0;
    cstate->reqdiskoffset = -1;

    /* make local copy so we don't have to lock membuf */
    p = storeCossMemPointerFromDiskOffset(cs, storeCossFilenoToDiskOffset(f, cs), NULL);
    if (p) {
	coss_stats.open_mem_hits++;
	// This seems to cause a crash: either the membuf pointer is set wrong or the membuf
	// is deallocated from underneath us.
	storeCossMemBufLock(SD, sio);
	debug(79, 3) ("storeCossOpen: %s: memory hit!\n", stripePath(SD));
    } else {
	/* Do the allocation */
	/* this is the first time we've been called on a new sio
	 * read the whole object into memory, then return the 
	 * requested amount
	 */
	coss_stats.open_mem_misses++;
	/*
	 * This bit of code actually does the LRU disk thing - we realloc
	 * a place for the object here, and the file_read() reads the object
	 * into the cossmembuf for later writing ..
	 */
	cstate->reqdiskoffset = storeCossFilenoToDiskOffset(sio->swap_filen, cs);
	assert(cstate->reqdiskoffset >= 0);

	/* If the object is allocated too recently, make a memory-only copy */
	if (storeCossRelocateRequired(cs, sio->swap_filen)) {
	    debug(79, 3) ("storeCossOpen: %s: memory miss - doing reallocation (Current stripe : %d  Object in stripe : %d)\n", stripePath(SD), cs->curstripe, storeCossFilenoToStripe(cs, sio->swap_filen));
	    nf = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC);
	} else {
	    debug(79, 3) ("storeCossOpen: %s memory miss - not reallocating (Current stripe : %d  Object in stripe : %d)\n", stripePath(SD), cs->curstripe, storeCossFilenoToStripe(cs, sio->swap_filen));
	    nf = storeCossMemOnlyAllocate(SD, e);
	    if (nf == -1) {
		debug(79, 3) ("storeCossOpen: %s memory miss - reallocating because all membufs are in use\n", stripePath(SD));
		nf = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC);
	    }
	}
	if (nf == -1) {
	    /* We have to clean up neatly .. */
	    coss_stats.open.fail++;
	    cbdataFree(sio);
	    cs->numcollisions++;
	    debug(79, 3) ("storeCossOpen: Reallocation of %d/%d failed\n", e->swap_dirn, e->swap_filen);
	    /* XXX XXX XXX Will squid call storeUnlink for this object? */
	    return NULL;
	}
	if (nf < cs->max_disk_nf) {
	    /* Remove the object from its currently-allocated stripe */
	    storeCossRemove(SD, e);
	    storeCossNewPendingRelocate(cs, sio, sio->swap_filen, nf);
	    sio->swap_filen = nf;
	    cstate->flags.reloc = 1;
	    /* Notify the upper levels that we've changed file number */
	    sio->file_callback(sio->callback_data, 0, sio);
	    /*
	     * lock the new buffer so it doesn't get swapped out on us
	     * this will get unlocked in storeCossClose
	     */
	    storeCossMemBufLock(SD, sio);
	    /*
	     * Do the index magic to keep the disk and memory LRUs identical