ldlm_pool.c

来自「lustre 1.6.5 source code」· C语言 代码 · 共 1,146 行 · 第 1/3 页

static int ldlm_pools_shrink(ldlm_side_t client, int nr, 
                             unsigned int gfp_mask)
{
        int total = 0, cached = 0, nr_ns;
        struct ldlm_namespace *ns;

        if (nr != 0 && !(gfp_mask & __GFP_FS))
                return -1;

        CDEBUG(D_DLMTRACE, "request to shrink %d %s locks from all pools\n",
               nr, client == LDLM_NAMESPACE_CLIENT ? "client" : "server");

        /* Find out how many resources we may release. */
        for (nr_ns = atomic_read(ldlm_namespace_nr(client)); 
             nr_ns > 0; nr_ns--) 
        {
                mutex_down(ldlm_namespace_lock(client));
                if (list_empty(ldlm_namespace_list(client))) {
                        mutex_up(ldlm_namespace_lock(client));
                        return 0;
                }
                ns = ldlm_namespace_first(client);
                ldlm_namespace_get(ns);
                ldlm_namespace_move(ns, client);
                mutex_up(ldlm_namespace_lock(client));
                total += ldlm_pool_shrink(&ns->ns_pool, 0, gfp_mask);
                ldlm_namespace_put(ns, 1);
        }
 
        if (nr == 0 || total == 0)
                return total;

        /* Shrink at least ldlm_namespace_nr(client) namespaces. */
        for (nr_ns = atomic_read(ldlm_namespace_nr(client)); 
             nr_ns > 0; nr_ns--) 
        {
                int cancel, nr_locks;

                /* Do not call shrink under ldlm_namespace_lock(client) */
                mutex_down(ldlm_namespace_lock(client));
                if (list_empty(ldlm_namespace_list(client))) {
                        mutex_up(ldlm_namespace_lock(client));
                        /* If list is empty, we can't return any @cached > 0,
                         * that probably would cause needless shrinker
                         * call. */
                        cached = 0;
                        break;
                }
                ns = ldlm_namespace_first(client);
                ldlm_namespace_get(ns);
                ldlm_namespace_move(ns, client);
                mutex_up(ldlm_namespace_lock(client));
                
                nr_locks = ldlm_pool_granted(&ns->ns_pool);
                cancel = 1 + nr_locks * nr / total;
                ldlm_pool_shrink(&ns->ns_pool, cancel, gfp_mask);
                cached += ldlm_pool_granted(&ns->ns_pool);
                ldlm_namespace_put(ns, 1);
        }
        return cached;
}

static int ldlm_pools_srv_shrink(int nr, unsigned int gfp_mask)
{
        return ldlm_pools_shrink(LDLM_NAMESPACE_SERVER, nr, gfp_mask);
}

static int ldlm_pools_cli_shrink(int nr, unsigned int gfp_mask)
{
        return ldlm_pools_shrink(LDLM_NAMESPACE_CLIENT, nr, gfp_mask);
}

void ldlm_pools_recalc(ldlm_side_t client)
{
        __u32 nr_l = 0, nr_p = 0, l;
        struct ldlm_namespace *ns;
        int nr, equal = 0;

        /* No need to setup pool limit for client pools. */
        if (client == LDLM_NAMESPACE_SERVER) {
                /* Check all modest namespaces first. */
                mutex_down(ldlm_namespace_lock(client));
                list_for_each_entry(ns, ldlm_namespace_list(client), 
                                    ns_list_chain) 
                {
                        if (ns->ns_appetite != LDLM_NAMESPACE_MODEST)
                                continue;

                        l = ldlm_pool_granted(&ns->ns_pool);
                        if (l == 0)
                                l = 1;

                        /* Set the modest pools limit equal to their avg granted
                         * locks + 5%. */
                        l += dru(l * LDLM_POOLS_MODEST_MARGIN, 100);
                        ldlm_pool_setup(&ns->ns_pool, l);
                        nr_l += l;
                        nr_p++;
                }

                /* Make sure that modest namespaces did not eat more that 2/3 
                 * of limit */
                if (nr_l >= 2 * (LDLM_POOL_HOST_L / 3)) {
                        CWARN("\"Modest\" pools eat out 2/3 of server locks "
                              "limit (%d of %lu). This means that you have too "
                              "many clients for this amount of server RAM. "
                              "Upgrade server!\n", nr_l, LDLM_POOL_HOST_L);
                        equal = 1;
                }

                /* The rest is given to greedy namespaces. */
                list_for_each_entry(ns, ldlm_namespace_list(client), 
                                    ns_list_chain) 
                {
                        if (!equal && ns->ns_appetite != LDLM_NAMESPACE_GREEDY)
                                continue;

                        if (equal) {
                                /* In the case 2/3 locks are eaten out by
                                 * modest pools, we re-setup equal limit
                                 * for _all_ pools. */
                                l = LDLM_POOL_HOST_L /
                                        atomic_read(ldlm_namespace_nr(client));
                        } else {
                                /* All the rest of greedy pools will have
                                 * all locks in equal parts.*/
                                l = (LDLM_POOL_HOST_L - nr_l) /
                                        (atomic_read(ldlm_namespace_nr(client)) -
                                         nr_p);
                        }
                        ldlm_pool_setup(&ns->ns_pool, l);
                }
                mutex_up(ldlm_namespace_lock(client));
        }

        /* Recalc at least ldlm_namespace_nr(client) namespaces. */
        for (nr = atomic_read(ldlm_namespace_nr(client)); nr > 0; nr--) {
                /* Lock the list, get first @ns in the list, getref, move it
                 * to the tail, unlock and call pool recalc. This way we avoid
                 * calling recalc under @ns lock what is really good as we get
                 * rid of potential deadlock on client nodes when canceling
                 * locks synchronously. */
                mutex_down(ldlm_namespace_lock(client));
                if (list_empty(ldlm_namespace_list(client))) {
                        mutex_up(ldlm_namespace_lock(client));
                        break;
                }
                ns = ldlm_namespace_first(client);
                ldlm_namespace_get(ns);
                ldlm_namespace_move(ns, client);
                mutex_up(ldlm_namespace_lock(client));

                /* After setup is done - recalc the pool. */
                ldlm_pool_recalc(&ns->ns_pool);
                ldlm_namespace_put(ns, 1);
        }
}
EXPORT_SYMBOL(ldlm_pools_recalc);

static int ldlm_pools_thread_main(void *arg)
{
        struct ptlrpc_thread *thread = (struct ptlrpc_thread *)arg;
        char *t_name = "ldlm_poold";
        ENTRY;

        cfs_daemonize(t_name);
        thread->t_flags = SVC_RUNNING;
        cfs_waitq_signal(&thread->t_ctl_waitq);

        CDEBUG(D_DLMTRACE, "%s: pool thread starting, process %d\n",
               t_name, cfs_curproc_pid());

        while (1) {
                struct l_wait_info lwi;

                /* Recal all pools on this tick. */
                ldlm_pools_recalc(LDLM_NAMESPACE_SERVER);
                ldlm_pools_recalc(LDLM_NAMESPACE_CLIENT);
                
                /* Wait until the next check time, or until we're
                 * stopped. */
                lwi = LWI_TIMEOUT(cfs_time_seconds(LDLM_POOLS_THREAD_PERIOD),
                                  NULL, NULL);
                l_wait_event(thread->t_ctl_waitq, (thread->t_flags &
                                                   (SVC_STOPPING|SVC_EVENT)),
                             &lwi);

                if (thread->t_flags & SVC_STOPPING) {
                        thread->t_flags &= ~SVC_STOPPING;
                        break;
                } else if (thread->t_flags & SVC_EVENT) {
                        thread->t_flags &= ~SVC_EVENT;
                }
        }

        thread->t_flags = SVC_STOPPED;
        cfs_waitq_signal(&thread->t_ctl_waitq);

        CDEBUG(D_DLMTRACE, "%s: pool thread exiting, process %d\n",
               t_name, cfs_curproc_pid());

        complete_and_exit(&ldlm_pools_comp, 0);
}

static int ldlm_pools_thread_start(void)
{
        struct l_wait_info lwi = { 0 };
        int rc;
        ENTRY;

        if (ldlm_pools_thread != NULL)
                RETURN(-EALREADY);

        OBD_ALLOC_PTR(ldlm_pools_thread);
        if (ldlm_pools_thread == NULL)
                RETURN(-ENOMEM);

        init_completion(&ldlm_pools_comp);
        cfs_waitq_init(&ldlm_pools_thread->t_ctl_waitq);

        /* CLONE_VM and CLONE_FILES just avoid a needless copy, because we
         * just drop the VM and FILES in ptlrpc_daemonize() right away. */
        rc = cfs_kernel_thread(ldlm_pools_thread_main, ldlm_pools_thread,
                               CLONE_VM | CLONE_FILES);
        if (rc < 0) {
                CERROR("Can't start pool thread, error %d\n",
                       rc);
                OBD_FREE(ldlm_pools_thread, sizeof(*ldlm_pools_thread));
                ldlm_pools_thread = NULL;
                RETURN(rc);
        }
        l_wait_event(ldlm_pools_thread->t_ctl_waitq,
                     (ldlm_pools_thread->t_flags & SVC_RUNNING), &lwi);
        RETURN(0);
}

static void ldlm_pools_thread_stop(void)
{
        ENTRY;

        if (ldlm_pools_thread == NULL) {
                EXIT;
                return;
        }

        ldlm_pools_thread->t_flags = SVC_STOPPING;
        cfs_waitq_signal(&ldlm_pools_thread->t_ctl_waitq);

        /* Make sure that pools thread is finished before freeing @thread.
         * This fixes possible race and oops due to accessing freed memory
         * in pools thread. */
        wait_for_completion(&ldlm_pools_comp);
        OBD_FREE_PTR(ldlm_pools_thread);
        ldlm_pools_thread = NULL;
        EXIT;
}

int ldlm_pools_init(void)
{
        int rc;
        ENTRY;

        rc = ldlm_pools_thread_start();
        if (rc == 0) {
                ldlm_pools_srv_shrinker = set_shrinker(DEFAULT_SEEKS,
                                                       ldlm_pools_srv_shrink);
                ldlm_pools_cli_shrinker = set_shrinker(DEFAULT_SEEKS,
                                                       ldlm_pools_cli_shrink);
        }
        RETURN(rc);
}
EXPORT_SYMBOL(ldlm_pools_init);

void ldlm_pools_fini(void)
{
        if (ldlm_pools_srv_shrinker != NULL) {
                remove_shrinker(ldlm_pools_srv_shrinker);
                ldlm_pools_srv_shrinker = NULL;
        }
        if (ldlm_pools_cli_shrinker != NULL) {
                remove_shrinker(ldlm_pools_cli_shrinker);
                ldlm_pools_cli_shrinker = NULL;
        }
        ldlm_pools_thread_stop();
}
EXPORT_SYMBOL(ldlm_pools_fini);
#endif /* __KERNEL__ */

#else /* !HAVE_LRU_RESIZE_SUPPORT */
int ldlm_pool_setup(struct ldlm_pool *pl, int limit)
{
        return 0;
}
EXPORT_SYMBOL(ldlm_pool_setup);

int ldlm_pool_recalc(struct ldlm_pool *pl)
{
        return 0;
}
EXPORT_SYMBOL(ldlm_pool_recalc);

int ldlm_pool_shrink(struct ldlm_pool *pl,
                     int nr, unsigned int gfp_mask)
{
        return 0;
}
EXPORT_SYMBOL(ldlm_pool_shrink);

int ldlm_pool_init(struct ldlm_pool *pl, struct ldlm_namespace *ns,
                   int idx, ldlm_side_t client)
{
        return 0;
}
EXPORT_SYMBOL(ldlm_pool_init);

void ldlm_pool_fini(struct ldlm_pool *pl)
{
        return;
}
EXPORT_SYMBOL(ldlm_pool_fini);

void ldlm_pool_add(struct ldlm_pool *pl, struct ldlm_lock *lock)
{
        return;
}
EXPORT_SYMBOL(ldlm_pool_add);

void ldlm_pool_del(struct ldlm_pool *pl, struct ldlm_lock *lock)
{
        return;
}
EXPORT_SYMBOL(ldlm_pool_del);

__u64 ldlm_pool_get_slv(struct ldlm_pool *pl)
{
        return 1;
}
EXPORT_SYMBOL(ldlm_pool_get_slv);

void ldlm_pool_set_slv(struct ldlm_pool *pl, __u64 slv)
{
        return;
}
EXPORT_SYMBOL(ldlm_pool_set_slv);

__u32 ldlm_pool_get_limit(struct ldlm_pool *pl)
{
        return 0;
}
EXPORT_SYMBOL(ldlm_pool_get_limit);

void ldlm_pool_set_limit(struct ldlm_pool *pl, __u32 limit)
{
        return;
}
EXPORT_SYMBOL(ldlm_pool_set_limit);

int ldlm_pools_init(void)
{
        return 0;
}
EXPORT_SYMBOL(ldlm_pools_init);

void ldlm_pools_fini(void)
{
        return;
}
EXPORT_SYMBOL(ldlm_pools_fini);

void ldlm_pools_wakeup(void)
{
        return;
}
EXPORT_SYMBOL(ldlm_pools_wakeup);

void ldlm_pools_recalc(ldlm_side_t client)
{
        return;
}
EXPORT_SYMBOL(ldlm_pools_recalc);
#endif /* HAVE_LRU_RESIZE_SUPPORT */