erl_mseg.c

来自「OTP是开放电信平台的简称」· C语言 代码 · 共 1,383 行 · 第 1/2 页

/* ``The contents of this file are subject to the Erlang Public License,
 * Version 1.1, (the "License"); you may not use this file except in
 * compliance with the License. You should have received a copy of the
 * Erlang Public License along with this software. If not, it can be
 * retrieved via the world wide web at http://www.erlang.org/.
 * 
 * Software distributed under the License is distributed on an "AS IS"
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 * the License for the specific language governing rights and limitations
 * under the License.
 * 
 * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
 * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
 * AB. All Rights Reserved.''
 * 
 *     $Id$
 */

/*
 * Description:	A memory segment allocator. Segments that are deallocated
 *              are kept for a while in a segment "cache" before they are
 *              destroyed. When segments are allocated, cached segments
 *              are used if possible instead of creating new segments.
 *
 * Author: 	Rickard Green
 */

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#include "sys.h"
#include "erl_mseg.h"
#include "global.h"
#include "erl_threads.h"
#include "erl_mtrace.h"

#if HAVE_ERTS_MSEG

#define SEGTYPE ERTS_MTRACE_SEGMENT_ID

#ifndef HAVE_GETPAGESIZE
#define HAVE_GETPAGESIZE 0
#endif

#ifdef _SC_PAGESIZE
#  define GET_PAGE_SIZE sysconf(_SC_PAGESIZE)
#elif HAVE_GETPAGESIZE
#  define GET_PAGE_SIZE getpagesize()
#else
#  error "Page size unknown"
     /* Implement some other way to get the real page size if needed! */
#endif

#define MAX_CACHE_SIZE 30

#undef MIN
#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
#undef MAX
#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))

#undef  PAGE_MASK
#define INV_PAGE_MASK	((Uint) (page_size - 1))
#define PAGE_MASK	(~INV_PAGE_MASK)
#define PAGE_FLOOR(X)	((X) & PAGE_MASK)
#define PAGE_CEILING(X)	PAGE_FLOOR((X) + INV_PAGE_MASK)
#define PAGES(X)	((X) >> page_shift)

static int atoms_initialized;

static Uint cache_check_interval;

static void check_cache(void *unused);
static void mseg_clear_cache(void);
static int is_cache_check_scheduled;

#if HAVE_MMAP
/* Mmap ... */

#define MMAP_PROT		(PROT_READ|PROT_WRITE)
#ifdef MAP_ANON
#  define MMAP_FLAGS		(MAP_ANON|MAP_PRIVATE)
#  define MMAP_FD		(-1)
#else
#  define MMAP_FLAGS		(MAP_PRIVATE)
#  define MMAP_FD		mmap_fd
static int mmap_fd;
#endif

#if HAVE_MREMAP
#  define HAVE_MSEG_RECREATE 1
#else
#  define HAVE_MSEG_RECREATE 0
#endif

#define CAN_PARTLY_DESTROY 1
#else  /* #if HAVE_MMAP */
#define CAN_PARTLY_DESTROY 0
#error "Not supported"
#endif /* #if HAVE_MMAP */


#if defined(ERTS_MSEG_FAKE_SEGMENTS)
#undef CAN_PARTLY_DESTROY
#define CAN_PARTLY_DESTROY 0
#endif

static const ErtsMsegOpt_t default_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER;

typedef struct cache_desc_t_ {
    void *seg;
    Uint size;
    struct cache_desc_t_ *next;
    struct cache_desc_t_ *prev;
} cache_desc_t;

typedef struct {
    Uint32 giga_no;
    Uint32 no;
} CallCounter;

static int is_init_done;
static Uint page_size;
static Uint page_shift;

static struct {
    CallCounter alloc;
    CallCounter dealloc;
    CallCounter realloc;
    CallCounter create;
    CallCounter destroy;
#if HAVE_MSEG_RECREATE
    CallCounter recreate;
#endif
    CallCounter clear_cache;
    CallCounter check_cache;
} calls;

static cache_desc_t cache_descs[MAX_CACHE_SIZE];
static cache_desc_t *free_cache_descs;
static cache_desc_t *cache;
static cache_desc_t *cache_end;
static Uint cache_hits;
static Uint cache_size;
static Uint min_cached_seg_size;
static Uint max_cached_seg_size;

static Uint max_cache_size;
static Uint abs_max_cache_bad_fit;
static Uint rel_max_cache_bad_fit;

#if CAN_PARTLY_DESTROY
static Uint min_seg_size;
#endif

static Sint no_of_segments;
static Sint no_of_segments_watermark;

#define ONE_GIGA (1000000000)

#define ZERO_CC(CC) (calls.CC.no = 0, calls.CC.giga_no = 0)

#define INC_CC(CC) (calls.CC.no == ONE_GIGA - 1				\
		    ? (calls.CC.giga_no++, calls.CC.no = 0)		\
		    : calls.CC.no++)

#define DEC_CC(CC) (calls.CC.no == 0					\
		    ? (calls.CC.giga_no--,				\
		       calls.CC.no = ONE_GIGA - 1)			\
		    : calls.CC.no--)


static erts_mtx_t mseg_mutex; /* Also needed when !USE_THREADS */
static erts_mtx_t init_atoms_mutex; /* Also needed when !USE_THREADS */

#ifdef USE_THREADS
#ifndef ERTS_SMP
static erts_cnd_t mseg_cond;
static int do_shutdown;
#endif

static void thread_safe_init(void)
{
    erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
    erts_mtx_init(&mseg_mutex, "mseg");
    erts_mtx_set_forksafe(&mseg_mutex);
#ifndef ERTS_SMP
    erts_cnd_init(&mseg_cond);
    do_shutdown = 0;
#endif
}

#endif

#if defined(USE_THREADS) && !defined(ERTS_SMP) 

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 * Non-SMP multi-threaded case                                             *
\*                                                                         */

static erts_thr_timeval_t check_time;
static erts_tid_t mseg_cc_tid;

static void *
mseg_cache_cleaner(void *unused)
{
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_set_thread_name("memory segment cache cleaner");
#endif

    erts_mtx_lock(&mseg_mutex);

    while (!do_shutdown) {

	while (!is_cache_check_scheduled && !do_shutdown)
	    erts_cnd_wait(&mseg_cond, &mseg_mutex);

	res = 0;
	while (res != ETIMEDOUT && !do_shutdown)
	    res = erts_cnd_timedwait(&mseg_cond, &mseg_mutex, &check_time);

	if (do_shutdown)
	    mseg_clear_cache();
	else
	    check_cache(NULL);
    }

    erts_mtx_unlock(&mseg_mutex);
    return NULL;
}

static void
mseg_shutdown(void)
{
    erts_mtx_lock(&mseg_mutex);
    do_shutdown = 1;
    erts_cnd_signal(&mseg_cond);
    erts_mtx_unlock(&mseg_mutex);
    erts_thr_join(mseg_cc_tid, NULL);
    erts_mtx_destroy(&mseg_mutex);
    erts_cnd_destroy(&mseg_cond);
}

static ERTS_INLINE void
schedule_cache_check(void)
{
    if (!is_cache_check_scheduled && is_init_done) {
	erts_thr_time_now(&check_time);
	check_time.tv_sec += cache_check_interval / 1000;
	check_time.tv_nsec += (cache_check_interval % 1000)*1000000;
	if (check_time.tv_nsec >= 1000000000) {
	    check_time.tv_sec++;
	    check_time.tv_nsec -= 1000000000;
	}
	ASSERT(check_time.tv_nsec < 1000000000);
	is_cache_check_scheduled = 1;
	erts_cnd_signal(&mseg_cond);
    }
}

static void
mseg_late_init(void)
{
    erts_thr_opts_t opts = ERTS_THR_OPTS_DEFAULT_INITER;
    opts.detached = 0;
    opts.suggested_stack_size = 8; /* kilo words */
    erts_thr_create(&mseg_cc_tid, mseg_cache_cleaner, NULL, &opts);
}

#else  /* #if defined(USE_THREADS) && !defined(ERTS_SMP) */

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 * Single-threaded and SMP case                                            *
\*                                                                         */

static ErlTimer cache_check_timer;

static ERTS_INLINE void
schedule_cache_check(void)
{
    if (!is_cache_check_scheduled && is_init_done) {
	cache_check_timer.active = 0;
	erl_set_timer(&cache_check_timer,
		      check_cache,
		      NULL,
		      NULL,
		      cache_check_interval);
	is_cache_check_scheduled = 1;
    }
}

static void
mseg_late_init(void)
{
}

static void
mseg_shutdown(void)
{
#ifdef ERTS_SMP
    erts_mtx_lock(&mseg_mutex);
#endif
    mseg_clear_cache();
#ifdef ERTS_SMP
    erts_mtx_unlock(&mseg_mutex);
#endif
}

#endif  /* #if defined(USE_THREADS) && !defined(ERTS_SMP) */

static ERTS_INLINE void *
mseg_create(Uint size)
{
    void *seg;

    ASSERT(size % page_size == 0);

#if defined(ERTS_MSEG_FAKE_SEGMENTS)
    seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size);
#elif HAVE_MMAP
    seg = (void *) mmap((void *) 0, (size_t) size,
			MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0);
    if (seg == (void *) MAP_FAILED)
	seg = NULL;
#else
#error "Missing mseg_create() implementation"
#endif

    INC_CC(create);

    return seg;
}

static ERTS_INLINE void
mseg_destroy(void *seg, Uint size)
{
#if defined(ERTS_MSEG_FAKE_SEGMENTS)
    erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg);
#elif HAVE_MMAP

#ifdef DEBUG
    int res =
#endif

	munmap((void *) seg, size);

    ASSERT(size % page_size == 0);
    ASSERT(res == 0);
#else
#error "Missing mseg_destroy() implementation"
#endif

    INC_CC(destroy);

}

#if HAVE_MSEG_RECREATE

static ERTS_INLINE void *
mseg_recreate(void *old_seg, Uint old_size, Uint new_size)
{
    void *new_seg;

    ASSERT(old_size % page_size == 0);
    ASSERT(new_size % page_size == 0);

#if defined(ERTS_MSEG_FAKE_SEGMENTS)
    new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size);
#elif HAVE_MREMAP
    new_seg = (void *) mremap((void *) old_seg,
			      (size_t) old_size,
			      (size_t) new_size,
			      MREMAP_MAYMOVE);
    if (new_seg == (void *) MAP_FAILED)
	new_seg = NULL;
#else
#error "Missing mseg_recreate() implementation"
#endif

    INC_CC(recreate);

    return new_seg;
}

#endif /* #if HAVE_MSEG_RECREATE */


static ERTS_INLINE cache_desc_t * 
alloc_cd(void)
{
    cache_desc_t *cd = free_cache_descs;
    if (cd)
	free_cache_descs = cd->next;
    return cd;
}

static ERTS_INLINE void
free_cd(cache_desc_t *cd)
{
    cd->next = free_cache_descs;
    free_cache_descs = cd;
}


static ERTS_INLINE void
link_cd(cache_desc_t *cd)
{
    if (cache)
	cache->prev = cd;
    cd->next = cache;
    cd->prev = NULL;
    cache = cd;

    if (!cache_end) {
	ASSERT(!cd->next);
	cache_end = cd;
    }

    cache_size++;
}

static ERTS_INLINE void
end_link_cd(cache_desc_t *cd)
{
    if (cache_end)
	cache_end->next = cd;
    cd->next = NULL;
    cd->prev = cache_end;
    cache_end = cd;

    if (!cache) {
	ASSERT(!cd->prev);
	cache = cd;
    }

    cache_size++;
}

static ERTS_INLINE void
unlink_cd(cache_desc_t *cd)
{

    if (cd->next)
	cd->next->prev = cd->prev;
    else
	cache_end = cd->prev;
    
    if (cd->prev)
	cd->prev->next = cd->next;
    else
	cache = cd->next;
    ASSERT(cache_size > 0);
    cache_size--;
}

static ERTS_INLINE void
check_cache_limits(void)
{
    cache_desc_t *cd;
    max_cached_seg_size = 0;
    min_cached_seg_size = ~((Uint) 0);
    for (cd = cache; cd; cd = cd->next) {
	if (cd->size < min_cached_seg_size)
	    min_cached_seg_size = cd->size;
	if (cd->size > max_cached_seg_size)
	    max_cached_seg_size = cd->size;
    }

}

static ERTS_INLINE void
adjust_cache_size(int force_check_limits)
{
    cache_desc_t *cd;
    int check_limits = force_check_limits;
    Sint max_cached = no_of_segments_watermark - no_of_segments;

    while (((Sint) cache_size) > max_cached && ((Sint) cache_size) > 0) {
	ASSERT(cache_end);
	cd = cache_end;
	if (!check_limits &&
	    !(min_cached_seg_size < cd->size
	      && cd->size < max_cached_seg_size)) {
	    check_limits = 1;
	}
	if (erts_mtrace_enabled)
	    erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
	mseg_destroy(cd->seg, cd->size);
	unlink_cd(cd);
	free_cd(cd);
    }

    if (check_limits)
	check_cache_limits();

}

static void
check_cache(void *unused)
{
#ifdef ERTS_SMP
    erts_mtx_lock(&mseg_mutex);
#endif

    is_cache_check_scheduled = 0;

    if (no_of_segments_watermark > no_of_segments)
	no_of_segments_watermark--;
    adjust_cache_size(0);

    if (cache_size)
	schedule_cache_check();

    INC_CC(check_cache);

#ifdef ERTS_SMP
    erts_mtx_unlock(&mseg_mutex);
#endif

}

static void
mseg_clear_cache(void)
{
    no_of_segments_watermark = 0;

    adjust_cache_size(1);

    ASSERT(!cache);
    ASSERT(!cache_end);
    ASSERT(!cache_size);

    no_of_segments_watermark = no_of_segments;

    INC_CC(clear_cache);
}

static void *
mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
{

    Uint max, min, diff_size, size;
    cache_desc_t *cd, *cand_cd;
    void *seg;

    INC_CC(alloc);

    size = PAGE_CEILING(*size_p);

    no_of_segments++;
    if (no_of_segments_watermark < no_of_segments)
	no_of_segments_watermark = no_of_segments;

#if CAN_PARTLY_DESTROY
    if (size < min_seg_size)	
	min_seg_size = size;
#endif

    if (!opt->cache) {
    create_seg:
	adjust_cache_size(0);
	seg = mseg_create(size);
	if (!seg) {
	    mseg_clear_cache();
	    seg = mseg_create(size);
	    if (!seg)
		size = 0;
	}

	*size_p = size;
	if (erts_mtrace_enabled)
	    erts_mtrace_crr_alloc(seg, atype, ERTS_MTRACE_SEGMENT_ID, size);
	return seg;
    }

    if (size > max_cached_seg_size)
	goto create_seg;

    if (size < min_cached_seg_size) {

	diff_size = min_cached_seg_size - size;

	if (diff_size > abs_max_cache_bad_fit)
	    goto create_seg;

	if (100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size))
	    goto create_seg;

    }

    max = 0;
    min = ~((Uint) 0);
    cand_cd = NULL;

    for (cd = cache; cd; cd = cd->next) {
	if (cd->size >= size) {
	    if (!cand_cd) {
		cand_cd = cd;
		continue;
	    }
	    else if (cd->size < cand_cd->size) {
		if (max < cand_cd->size)
		    max = cand_cd->size;
		if (min > cand_cd->size)
		    min = cand_cd->size;
		cand_cd = cd;
		continue;
	    }
	}
	if (max < cd->size)
	    max = cd->size;
	if (min > cd->size)
	    min = cd->size;
    }

    min_cached_seg_size = min;
    max_cached_seg_size = max;

    if (!cand_cd)
	goto create_seg;

    diff_size = cand_cd->size - size;

    if (diff_size > abs_max_cache_bad_fit
	|| 100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size)) {
	if (max_cached_seg_size < cand_cd->size)	
	    max_cached_seg_size = cand_cd->size;
	if (min_cached_seg_size > cand_cd->size)
	    min_cached_seg_size = cand_cd->size;
	goto create_seg;
    }

    cache_hits++;

    size = cand_cd->size;
    seg = cand_cd->seg;

    unlink_cd(cand_cd);
    free_cd(cand_cd);

    *size_p = size;

    if (erts_mtrace_enabled) {
	erts_mtrace_crr_free(SEGTYPE, SEGTYPE, seg);
	erts_mtrace_crr_alloc(seg, atype, SEGTYPE, size);
    }
    return seg;
}


static void
mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size,
	     const ErtsMsegOpt_t *opt)
{
    cache_desc_t *cd;

    no_of_segments--;

    if (!opt->cache || max_cache_size == 0) {
	if (erts_mtrace_enabled)
	    erts_mtrace_crr_free(atype, SEGTYPE, seg);
	mseg_destroy(seg, size);
    }
    else {
	int check_limits = 0;
	
	if (size < min_cached_seg_size)
	    min_cached_seg_size = size;
	if (size > max_cached_seg_size)
	    max_cached_seg_size = size;

	if (!free_cache_descs) {
	    cd = cache_end;
	    if (!(min_cached_seg_size < cd->size
		  && cd->size < max_cached_seg_size)) {
		check_limits = 1;
	    }
	    if (erts_mtrace_enabled)
		erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
	    mseg_destroy(cd->seg, cd->size);
	    unlink_cd(cd);
	    free_cd(cd);
	}

	cd = alloc_cd();
	ASSERT(cd);
	cd->seg = seg;
	cd->size = size;
	link_cd(cd);