erl_alloc_util.c

OTP是开放电信平台的简称

    return size;
}

static ERTS_INLINE void
link_carrier(CarrierList_t *cl, Carrier_t *crr)
{
    crr->next = NULL;
    if (!cl->last) {
	ASSERT(!cl->first);
	cl->first = cl->last = crr;
	crr->prev = NULL;
    }
    else {
	ASSERT(cl->first);
	ASSERT(!cl->first->prev);
	ASSERT(cl->last);
	ASSERT(!cl->last->next);
	crr->prev = cl->last;
	cl->last->next = crr;
	cl->last = crr;
    }
    ASSERT(crr->next != crr);
    ASSERT(crr->prev != crr);
}

static ERTS_INLINE void
relink_carrier(CarrierList_t *cl, Carrier_t *crr)
{
    if (crr->next) {
	if (crr->next->prev != crr)
	    crr->next->prev = crr;
    }
    else if (cl->last != crr)
	cl->last = crr;

    if (crr->prev) {
	if (crr->prev->next != crr)
	    crr->prev->next = crr;
    }
    else if (cl->first != crr)
	cl->first = crr;
}

static ERTS_INLINE void
unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
{
    ASSERT(crr->next != crr);
    ASSERT(crr->prev != crr);

    if (cl->first == crr) {
	ASSERT(!crr->prev);
	cl->first = crr->next;
    }
    else {
	ASSERT(crr->prev);
	crr->prev->next = crr->next;
    }

    if (cl->last == crr) {
	ASSERT(!crr->next);
	cl->last = crr->prev;
    }
    else {
	ASSERT(crr->next);
	crr->next->prev = crr->prev;
    }
}


static Block_t *create_carrier(Allctr_t *, Uint, Uint);
static void destroy_carrier(Allctr_t *, Block_t *);

/* Multi block carrier alloc/realloc/free ... */

/* NOTE! mbc_alloc() may in case of memory shortage place the requested
 * block in a sbc.
 */
static void *
mbc_alloc(Allctr_t *allctr, Uint size)
{
    Uint last_blk_flg;
    Uint blk_sz;
    Block_t *blk;
    Uint nxt_blk_sz;
    Block_t *nxt_blk;

    ASSERT(size);
    ASSERT(size < allctr->sbc_threshold);

    blk_sz = UMEMSZ2BLKSZ(allctr, size);

    blk = (*allctr->get_free_block)(allctr, blk_sz);

    if (!blk) {
	blk = create_carrier(allctr, blk_sz, CFLG_MBC);
	if (!blk) {
	    /* Emergency! We couldn't create the carrier as we wanted.
	       Try to place it in a sys_alloced sbc. */
	    blk = create_carrier(allctr,
				 size,
				 CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC);
	    return blk ? BLK2UMEM(blk) : NULL;
	}
    }

    ASSERT(BLK_SZ(blk) >= blk_sz);
    ASSERT(IS_FREE_BLK(blk));
    ASSERT(IS_MBC_BLK(blk));

#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
    (*allctr->link_free_block)(allctr, blk);
    HARD_CHECK_BLK_CARRIER(allctr, blk);
    (*allctr->unlink_free_block)(allctr, blk);
#endif


    ASSERT(blk);

    SET_BLK_ALLOCED(blk);

#ifdef DEBUG
    nxt_blk = NULL;
#endif

    last_blk_flg = GET_LAST_BLK_HDR_FLG(blk);

    if (BLK_SZ(blk) - allctr->min_block_size >= blk_sz) {
	/* Shrink block... */
	nxt_blk_sz = BLK_SZ(blk) - blk_sz;
	SET_BLK_SZ(blk, blk_sz);

	nxt_blk = NXT_BLK(blk);
	SET_BLK_HDR(nxt_blk,
		    nxt_blk_sz,
		    SBH_THIS_FREE|SBH_PREV_ALLOCED|last_blk_flg);

	if (last_blk_flg == LAST_BLK_HDR_FLG)
	    SET_NOT_LAST_BLK(blk);
	else
	    SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);

	(*allctr->link_free_block)(allctr, nxt_blk);

	ASSERT(IS_NOT_LAST_BLK(blk));
	ASSERT(IS_FREE_BLK(nxt_blk));
	ASSERT(last_blk_flg == LAST_BLK_HDR_FLG
	       ? IS_LAST_BLK(nxt_blk)
	       : IS_NOT_LAST_BLK(nxt_blk));
	ASSERT(last_blk_flg == LAST_BLK_HDR_FLG
	       || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk)));
	ASSERT(last_blk_flg == LAST_BLK_HDR_FLG
	       || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk)));
	ASSERT(nxt_blk_sz == BLK_SZ(nxt_blk));
	ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0);
	ASSERT(nxt_blk_sz >= allctr->min_block_size);
    }
    else {
	if (last_blk_flg != LAST_BLK_HDR_FLG) {
	    nxt_blk = NXT_BLK(blk);
	    SET_PREV_BLK_ALLOCED(nxt_blk);
	}
	blk_sz = BLK_SZ(blk);

	ASSERT(last_blk_flg == LAST_BLK_HDR_FLG
	       ? IS_LAST_BLK(blk)
	       : IS_NOT_LAST_BLK(blk));
    }

    STAT_MBC_BLK_ALLOC(allctr, blk_sz);

    ASSERT(IS_ALLOCED_BLK(blk));
    ASSERT(blk_sz == BLK_SZ(blk));
    ASSERT(blk_sz % sizeof(Unit_t) == 0);
    ASSERT(blk_sz >= allctr->min_block_size);
    ASSERT(blk_sz >= size + ABLK_HDR_SZ);
    ASSERT(IS_MBC_BLK(blk));

    ASSERT(!nxt_blk || IS_PREV_BLK_ALLOCED(nxt_blk));
    ASSERT(!nxt_blk || IS_MBC_BLK(nxt_blk));

    HARD_CHECK_BLK_CARRIER(allctr, blk);

    return BLK2UMEM(blk);
}

static void
mbc_free(Allctr_t *allctr, void *p)
{
    Uint is_first_blk;
    Uint is_last_blk;
    Uint blk_sz;
    Block_t *blk;
    Block_t *nxt_blk;


    ASSERT(p);

    blk = UMEM2BLK(p);
    blk_sz = BLK_SZ(blk);

    ASSERT(IS_MBC_BLK(blk));
    ASSERT(blk_sz >= allctr->min_block_size);

    HARD_CHECK_BLK_CARRIER(allctr, blk);

    STAT_MBC_BLK_FREE(allctr, blk_sz);

    is_first_blk = IS_FIRST_BLK(blk);
    is_last_blk = IS_LAST_BLK(blk);

    if (!is_first_blk && IS_PREV_BLK_FREE(blk)) {
	/* Coalesce with previous block... */
	blk = PREV_BLK(blk);
	(*allctr->unlink_free_block)(allctr, blk);

	blk_sz += BLK_SZ(blk);
	is_first_blk = IS_FIRST_BLK(blk);
	SET_BLK_SZ(blk, blk_sz);
    }
    else {
	SET_BLK_FREE(blk);
    }

    if (is_last_blk)
	SET_LAST_BLK(blk);
    else {
	nxt_blk = NXT_BLK(blk);
	if (IS_FREE_BLK(nxt_blk)) {
	    /* Coalesce with next block... */
	    (*allctr->unlink_free_block)(allctr, nxt_blk);
	    blk_sz += BLK_SZ(nxt_blk);
	    SET_BLK_SZ(blk, blk_sz);

	    is_last_blk = IS_LAST_BLK(nxt_blk);
	    if (is_last_blk) 
		SET_LAST_BLK(blk);
	    else {
		SET_NOT_LAST_BLK(blk);
		SET_BLK_SZ_FTR(blk, blk_sz);
	    }
	}
	else {
	    SET_PREV_BLK_FREE(nxt_blk);
	    SET_NOT_LAST_BLK(blk);
	    SET_BLK_SZ_FTR(blk, blk_sz);
	}

    }

    ASSERT(is_last_blk  ? IS_LAST_BLK(blk)  : IS_NOT_LAST_BLK(blk));
    ASSERT(is_first_blk ? IS_FIRST_BLK(blk) : IS_NOT_FIRST_BLK(blk));
    ASSERT(IS_FREE_BLK(blk));
    ASSERT(is_first_blk || IS_PREV_BLK_ALLOCED(blk));
    ASSERT(is_last_blk  || IS_PREV_BLK_FREE(NXT_BLK(blk)));
    ASSERT(blk_sz == BLK_SZ(blk));
    ASSERT(is_last_blk || blk == PREV_BLK(NXT_BLK(blk)));
    ASSERT(blk_sz % sizeof(Unit_t) == 0);
    ASSERT(IS_MBC_BLK(blk));

    if (is_first_blk
	&& is_last_blk
	&& allctr->main_carrier != FBLK2MBC(allctr, blk))
	destroy_carrier(allctr, blk);
    else {
	(*allctr->link_free_block)(allctr, blk);
	HARD_CHECK_BLK_CARRIER(allctr, blk);
    }
}

static void *
mbc_realloc(Allctr_t *allctr, void *p, Uint size)
{
    void *new_p;
    Uint old_blk_sz;
    Block_t *blk;
#ifndef MBC_REALLOC_ALWAYS_MOVES
    Uint blk_sz;
    Block_t *nxt_blk;
    Uint nxt_blk_sz;
    Uint is_last_blk;
#endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */

    ASSERT(p);
    ASSERT(size);
    ASSERT(size < allctr->sbc_threshold);

    blk = (Block_t *) UMEM2BLK(p);
    old_blk_sz = BLK_SZ(blk);

    ASSERT(old_blk_sz >= allctr->min_block_size);

#ifndef MBC_REALLOC_ALWAYS_MOVES

    blk_sz = UMEMSZ2BLKSZ(allctr, size);

    ASSERT(IS_ALLOCED_BLK(blk));
    ASSERT(IS_MBC_BLK(blk));

    if (old_blk_sz == blk_sz)
	return p;

    is_last_blk = IS_LAST_BLK(blk);

    if (blk_sz < old_blk_sz) {
	/* Shrink block... */
	Block_t *nxt_nxt_blk;

	nxt_blk_sz = old_blk_sz - blk_sz;

	if ((is_last_blk || IS_ALLOCED_BLK(NXT_BLK(blk)))
	    && (nxt_blk_sz < allctr->min_block_size))
	    return p;

	HARD_CHECK_BLK_CARRIER(allctr, blk);

	SET_BLK_SZ(blk, blk_sz);
	SET_NOT_LAST_BLK(blk);

	nxt_blk = NXT_BLK(blk);
	SET_BLK_HDR(nxt_blk,
		    nxt_blk_sz,
		    SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK);

	STAT_MBC_BLK_FREE(allctr, old_blk_sz);
	STAT_MBC_BLK_ALLOC(allctr, blk_sz);

	ASSERT(BLK_SZ(blk) >= allctr->min_block_size);

	if (is_last_blk)
	    SET_LAST_BLK(nxt_blk);
	else {
	    nxt_nxt_blk = NXT_BLK(nxt_blk);
	    if (IS_FREE_BLK(nxt_nxt_blk)) {
		/* Coalesce with next free block... */
		nxt_blk_sz += BLK_SZ(nxt_nxt_blk);
		(*allctr->unlink_free_block)(allctr, nxt_nxt_blk);
		SET_BLK_SZ(nxt_blk, nxt_blk_sz);

		is_last_blk = IS_LAST_BLK(nxt_nxt_blk);
		if (is_last_blk)
		    SET_LAST_BLK(nxt_blk);
		else
		    SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
	    }
	    else {
		SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
		SET_PREV_BLK_FREE(nxt_nxt_blk);
	    }
	}

	(*allctr->link_free_block)(allctr, nxt_blk);


	ASSERT(IS_ALLOCED_BLK(blk));
	ASSERT(blk_sz == BLK_SZ(blk));
	ASSERT(blk_sz % sizeof(Unit_t) == 0);
	ASSERT(blk_sz >= allctr->min_block_size);
	ASSERT(blk_sz >= size + ABLK_HDR_SZ);
	ASSERT(IS_MBC_BLK(blk));
    
	ASSERT(IS_FREE_BLK(nxt_blk));
	ASSERT(IS_PREV_BLK_ALLOCED(nxt_blk));
	ASSERT(nxt_blk_sz == BLK_SZ(nxt_blk));
	ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0);
	ASSERT(nxt_blk_sz >= allctr->min_block_size);
	ASSERT(IS_MBC_BLK(nxt_blk));
	ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk));
	ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk)));
	ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk)));

	HARD_CHECK_BLK_CARRIER(allctr, blk);

	return p;
    }

    /* Need larger block... */

    if (!is_last_blk) {
	nxt_blk = NXT_BLK(blk);
	nxt_blk_sz = BLK_SZ(nxt_blk);
	if (IS_FREE_BLK(nxt_blk) && blk_sz <= old_blk_sz + nxt_blk_sz) {
	    /* Grow into next block... */

	    HARD_CHECK_BLK_CARRIER(allctr, blk);

	    (*allctr->unlink_free_block)(allctr, nxt_blk);
	    nxt_blk_sz -= blk_sz - old_blk_sz;

	    is_last_blk = IS_LAST_BLK(nxt_blk);
	    if (nxt_blk_sz < allctr->min_block_size) {
		blk_sz += nxt_blk_sz;

		SET_BLK_SZ(blk, blk_sz);

		if (is_last_blk) {
		    SET_LAST_BLK(blk);
#ifdef DEBUG
		    nxt_blk = NULL;
#endif
		}
		else {
		    nxt_blk = NXT_BLK(blk);
		    SET_PREV_BLK_ALLOCED(nxt_blk);
#ifdef DEBUG
		    is_last_blk = IS_LAST_BLK(nxt_blk);
		    nxt_blk_sz = BLK_SZ(nxt_blk);
#endif
		}
	    }
	    else {
		SET_BLK_SZ(blk, blk_sz);

		nxt_blk = NXT_BLK(blk);
		SET_BLK_HDR(nxt_blk,
			    nxt_blk_sz,
			    SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK);

		if (is_last_blk)
		    SET_LAST_BLK(nxt_blk);
		else
		    SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);

		(*allctr->link_free_block)(allctr, nxt_blk);

		ASSERT(IS_FREE_BLK(nxt_blk));
	    }

	    STAT_MBC_BLK_FREE(allctr, old_blk_sz);
	    STAT_MBC_BLK_ALLOC(allctr, blk_sz);


	    ASSERT(IS_ALLOCED_BLK(blk));
	    ASSERT(blk_sz == BLK_SZ(blk));
	    ASSERT(blk_sz % sizeof(Unit_t) == 0);
	    ASSERT(blk_sz >= allctr->min_block_size);
	    ASSERT(blk_sz >= size + ABLK_HDR_SZ);
	    ASSERT(IS_MBC_BLK(blk));

	    ASSERT(!nxt_blk || IS_PREV_BLK_ALLOCED(nxt_blk));
	    ASSERT(!nxt_blk || nxt_blk_sz == BLK_SZ(nxt_blk));
	    ASSERT(!nxt_blk || nxt_blk_sz % sizeof(Unit_t) == 0);
	    ASSERT(!nxt_blk || nxt_blk_sz >= allctr->min_block_size);
	    ASSERT(!nxt_blk || IS_MBC_BLK(nxt_blk));
	    ASSERT(!nxt_blk || (is_last_blk
				? IS_LAST_BLK(nxt_blk)
				: IS_NOT_LAST_BLK(nxt_blk)));
	    ASSERT(!nxt_blk || is_last_blk
		   || IS_ALLOCED_BLK(nxt_blk)
		   || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk)));
	    ASSERT(!nxt_blk || is_last_blk
		   || IS_ALLOCED_BLK(nxt_blk)
		   || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk)));

	    HARD_CHECK_BLK_CARRIER(allctr, blk);

	    return p;
	}
    }

    /* Failed to grow; move into a new one... */

#endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */

    new_p = mbc_alloc(allctr, size);
    if (!new_p)
	return NULL;
    sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ));
    mbc_free(allctr, p);


    return new_p;
}

#ifdef DEBUG

#if HAVE_ERTS_MSEG
#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % mseg_unit_size == 0)
#else
#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ)
#endif

#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ)		\
do {									\
    ASSERT(IS_FIRST_BLK((B)));						\
    ASSERT(IS_LAST_BLK((B)));						\
    ASSERT((CSZ) == CARRIER_SZ((C)));					\
    ASSERT((BSZ) == BLK_SZ((B)));					\
    ASSERT((BSZ) % sizeof(Unit_t) == 0);				\
    if ((SBC)) {							\
	ASSERT(IS_SBC_BLK((B)));					\
	ASSERT(IS_SB_CARRIER((C)));					\
    }									\
    else {								\
	ASSERT(IS_MBC_BLK((B)));					\
	ASSERT(IS_MB_CARRIER((C)));					\
    }									\
    if ((MSEGED)) {							\
	ASSERT(IS_MSEG_CARRIER((C)));					\
	ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ));				\
    }									\
    else {								\
	ASSERT(IS_SYS_ALLOC_CARRIER((C)));					\
	ASSERT((CSZ) % sizeof(Unit_t) == 0);				\
    }									\
} while (0)

#else
#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ)
#endif


static Block_t *
create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
{
    Block_t *blk;