threadpool.c

Apache 2.0.63 is the current stable version of the 2.0 series, and is recommended over any previous

/* Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* The purpose of this MPM is to fix the design flaws in the threaded
 * model.  Because of the way that pthreads and mutex locks interact,
 * it is basically impossible to cleanly gracefully shutdown a child
 * process if multiple threads are all blocked in accept.  This model
 * fixes those problems.
 */

#include "apr.h"
#include "apr_portable.h"
#include "apr_strings.h"
#include "apr_file_io.h"
#include "apr_thread_proc.h"
#include "apr_signal.h"
#include "apr_poll.h"
#include "apr_thread_mutex.h"
#include "apr_thread_cond.h"
#include "apr_proc_mutex.h"
#define APR_WANT_STRFUNC
#include "apr_want.h"

#if APR_HAVE_UNISTD_H
#include <unistd.h>
#endif
#if APR_HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#if APR_HAVE_SYS_WAIT_H
#include <sys/wait.h> 
#endif
#ifdef HAVE_SYS_PROCESSOR_H
#include <sys/processor.h> /* for bindprocessor() */
#endif

#if !APR_HAS_THREADS
#error The Worker MPM requires APR threads, but they are unavailable.
#endif

#define CORE_PRIVATE 
 
#include "ap_config.h"
#include "httpd.h" 
#include "http_main.h" 
#include "http_log.h" 
#include "http_config.h"        /* for read_config */ 
#include "http_core.h"          /* for get_remote_host */ 
#include "http_connection.h"
#include "ap_mpm.h"
#include "pod.h"
#include "mpm_common.h"
#include "ap_listen.h"
#include "scoreboard.h" 
#include "mpm_default.h"

#include <signal.h>
#include <limits.h>             /* for INT_MAX */

/* Limit on the total --- clients will be locked out if more servers than
 * this are needed.  It is intended solely to keep the server from crashing
 * when things get out of hand.
 *
 * We keep a hard maximum number of servers, for two reasons --- first off,
 * in case something goes seriously wrong, we want to stop the fork bomb
 * short of actually crashing the machine we're running on by filling some
 * kernel table.  Secondly, it keeps the size of the scoreboard file small
 * enough that we can read the whole thing without worrying too much about
 * the overhead.
 */
#ifndef DEFAULT_SERVER_LIMIT
#define DEFAULT_SERVER_LIMIT 16
#endif

/* Admin can't tune ServerLimit beyond MAX_SERVER_LIMIT.  We want
 * some sort of compile-time limit to help catch typos.
 */
#ifndef MAX_SERVER_LIMIT
#define MAX_SERVER_LIMIT 20000
#endif

/* Limit on the threads per process.  Clients will be locked out if more than
 * this  * server_limit are needed.
 *
 * We keep this for one reason it keeps the size of the scoreboard file small
 * enough that we can read the whole thing without worrying too much about
 * the overhead.
 */
#ifndef DEFAULT_THREAD_LIMIT
#define DEFAULT_THREAD_LIMIT 64 
#endif

/* Admin can't tune ThreadLimit beyond MAX_THREAD_LIMIT.  We want
 * some sort of compile-time limit to help catch typos.
 */
#ifndef MAX_THREAD_LIMIT
#define MAX_THREAD_LIMIT 20000
#endif

/*
 * Actual definitions of config globals
 */

int ap_threads_per_child = 0;         /* Worker threads per child */
static int ap_daemons_to_start = 0;
static int min_spare_threads = 0;
static int max_spare_threads = 0;
static int ap_daemons_limit = 0;
static int server_limit = DEFAULT_SERVER_LIMIT;
static int first_server_limit;
static int thread_limit = DEFAULT_THREAD_LIMIT;
static int first_thread_limit;
static int changed_limit_at_restart;
static int dying = 0;
static int workers_may_exit = 0;
static int start_thread_may_exit = 0;
static int listener_may_exit = 0;
static int requests_this_child;
static int num_listensocks = 0;
static int resource_shortage = 0;
static int mpm_state = AP_MPMQ_STARTING;

/* The structure used to pass unique initialization info to each thread */
typedef struct {
    int pid;
    int tid;
    int sd;
} proc_info;

/* Structure used to pass information to the thread responsible for 
 * creating the rest of the threads.
 */
typedef struct {
    apr_thread_t **threads;
    apr_thread_t *listener;
    int child_num_arg;
    apr_threadattr_t *threadattr;
} thread_starter;

#define ID_FROM_CHILD_THREAD(c, t)    ((c * thread_limit) + t)

/*
 * The max child slot ever assigned, preserved across restarts.  Necessary
 * to deal with MaxClients changes across AP_SIG_GRACEFUL restarts.  We 
 * use this value to optimize routines that have to scan the entire 
 * scoreboard.
 */
int ap_max_daemons_limit = -1;

static ap_pod_t *pod;

/* *Non*-shared http_main globals... */

server_rec *ap_server_conf;

/* The worker MPM respects a couple of runtime flags that can aid
 * in debugging. Setting the -DNO_DETACH flag will prevent the root process
 * from detaching from its controlling terminal. Additionally, setting
 * the -DONE_PROCESS flag (which implies -DNO_DETACH) will get you the
 * child_main loop running in the process which originally started up.
 * This gives you a pretty nice debugging environment.  (You'll get a SIGHUP
 * early in standalone_main; just continue through.  This is the server
 * trying to kill off any child processes which it might have lying
 * around --- Apache doesn't keep track of their pids, it just sends
 * SIGHUP to the process group, ignoring it in the root process.
 * Continue through and you'll be fine.).
 */

static int one_process = 0;

#ifdef DEBUG_SIGSTOP
int raise_sigstop_flags;
#endif

static apr_pool_t *pconf;                 /* Pool for config stuff */
static apr_pool_t *pchild;                /* Pool for httpd child stuff */

static pid_t ap_my_pid; /* Linux getpid() doesn't work except in main 
                           thread. Use this instead */
static pid_t parent_pid;
static apr_os_thread_t *listener_os_thread;

/* Locks for accept serialization */
static apr_proc_mutex_t *accept_mutex;

#if APR_O_NONBLOCK_INHERITED
#undef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
#endif /* APR_O_NONBLOCK_INHERITED */

#ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
#define SAFE_ACCEPT(stmt) (ap_listeners->next ? (stmt) : APR_SUCCESS)
#else
#define SAFE_ACCEPT(stmt) (stmt)
#endif

/* The LISTENER_SIGNAL signal will be sent from the main thread to the 
 * listener thread to wake it up for graceful termination (what a child 
 * process from an old generation does when the admin does "apachectl 
 * graceful").  This signal will be blocked in all threads of a child
 * process except for the listener thread.
 */
#define LISTENER_SIGNAL     SIGHUP


/* Possible states of a worker thread. */
typedef enum {
    WORKER_IDLE,
    WORKER_BUSY,
    WORKER_TERMINATED
} worker_state_e;

/* Structure used to wake up an idle worker thread
 */
typedef struct {
    apr_pool_t *pool;
    apr_socket_t *csd;
    worker_state_e state;
    apr_thread_cond_t *cond;
    apr_thread_mutex_t *mutex;
} worker_wakeup_info;

/* Structure used to hold a stack of idle worker threads 
 */
typedef struct {
    apr_thread_mutex_t *mutex;
    apr_thread_cond_t *cond;
    worker_wakeup_info **stack;
    apr_size_t nelts;
    apr_size_t nalloc;
    int terminated;
} worker_stack;

static worker_stack* worker_stack_create(apr_pool_t *pool, apr_size_t max)
{
    apr_status_t rv;
    worker_stack *stack = (worker_stack *)apr_palloc(pool, sizeof(*stack));

    if ((rv = apr_thread_mutex_create(&stack->mutex, APR_THREAD_MUTEX_DEFAULT,
                                      pool)) != APR_SUCCESS) {
        return NULL;
    }
    if ((rv = apr_thread_cond_create(&stack->cond, pool)) != APR_SUCCESS) {
        return NULL;
    }
    stack->nelts = 0;
    stack->nalloc = max;
    stack->stack =
        (worker_wakeup_info **)apr_palloc(pool, stack->nalloc *
                                          sizeof(worker_wakeup_info *));
    stack->terminated = 0;
    return stack;
}

static apr_status_t worker_stack_wait(worker_stack *stack,
                                      worker_wakeup_info *wakeup)
{
    apr_status_t rv;

    wakeup->state = WORKER_IDLE;
    
    if ((rv = apr_thread_mutex_lock(stack->mutex)) != APR_SUCCESS) {
        return rv;
    }
    if (stack->terminated) {
        if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
            return rv;
        }
        return APR_EOF;
    }
    if (stack->nelts == stack->nalloc) {
        if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
            return rv;
        }
        return APR_ENOSPC;
    }
    stack->stack[stack->nelts] = wakeup;
    /* Signal a blocking listener thread only if we just made the
     * stack non-empty. */
    if (stack->nelts++ == 0) {
        (void)apr_thread_cond_signal(stack->cond);
    }
    if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
        return rv;
    }

    /* At this point we've already added this worker to the stack, now
     * we just wait until the listener has accept()ed a connection
     * for us. */
    if ((rv = apr_thread_mutex_lock(wakeup->mutex)) != APR_SUCCESS) {
        return rv;
    }
    while (wakeup->state == WORKER_IDLE) {
        if ((rv = apr_thread_cond_wait(wakeup->cond, wakeup->mutex)) !=
            APR_SUCCESS) {
            return rv;
        }
    }
    if ((rv = apr_thread_mutex_unlock(wakeup->mutex)) != APR_SUCCESS) {
        return rv;
    }
    return APR_SUCCESS;
}

static apr_status_t worker_stack_pop(worker_stack *stack,
                                     worker_wakeup_info **worker)
{
    apr_status_t rv;
    if ((rv = apr_thread_mutex_lock(stack->mutex)) != APR_SUCCESS) {
        return rv;
    }
    AP_DEBUG_ASSERT(stack->nelts >= 0);
    while ((stack->nelts == 0) && (!stack->terminated)) {
        rv = apr_thread_cond_wait(stack->cond, stack->mutex);
        if (rv != APR_SUCCESS) {
            apr_status_t rv2;
            rv2 = apr_thread_mutex_unlock(stack->mutex);
            if (rv2 != APR_SUCCESS) {
                return rv2;
            }
            return rv;
        }
    }
    if (stack->terminated) {
        if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
            return rv;
        }
        return APR_EOF;
    }
    *worker = stack->stack[--stack->nelts];
    if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
        return rv;
    }
    return APR_SUCCESS;
}

static apr_status_t worker_stack_terminate(worker_stack *stack)
{
    apr_status_t rv;
    worker_wakeup_info *worker;

    if ((rv = apr_thread_mutex_lock(stack->mutex)) != APR_SUCCESS) {
        return rv;
    }
    stack->terminated = 1;
    /* Wake up the listener thread. Although there will never be
     * more than one thread blocking on this condition, broadcast
     * just in case. */
    apr_thread_cond_broadcast(stack->cond);
    while (stack->nelts) {
        worker = stack->stack[--stack->nelts];
        apr_thread_mutex_lock(worker->mutex);