smpboot.c

linux-2.6.15.6

/*
 *	x86 SMP booting functions
 *
 *	(c) 1995 Alan Cox, Building #3 <[e-mail]>
 *	(c) 1998, 1999, 2000 Ingo Molnar <[e-mail]>
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIP report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *					Various cleanups.
 *					Probably mostly hotplug CPU ready now.
 *	Ashok Raj			: CPU hotplug support
 */


#include <linux/config.h>
#include <linux/init.h>

#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/smp_lock.h>
#include <linux/bootmem.h>
#include <linux/thread_info.h>
#include <linux/module.h>

#include <linux/delay.h>
#include <linux/mc146818rtc.h>
#include <asm/mtrr.h>
#include <asm/pgalloc.h>
#include <asm/desc.h>
#include <asm/kdebug.h>
#include <asm/tlbflush.h>
#include <asm/proto.h>
#include <asm/nmi.h>
#include <asm/irq.h>
#include <asm/hw_irq.h>

/* Number of siblings per CPU package */
int smp_num_siblings = 1;
/* Package ID of each logical CPU */
u8 phys_proc_id[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
/* core ID of each logical CPU */
u8 cpu_core_id[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };

/* Bitmask of currently online CPUs */
cpumask_t cpu_online_map __read_mostly;

EXPORT_SYMBOL(cpu_online_map);

/*
 * Private maps to synchronize booting between AP and BP.
 * Probably not needed anymore, but it makes for easier debugging. -AK
 */
cpumask_t cpu_callin_map;
cpumask_t cpu_callout_map;

cpumask_t cpu_possible_map;
EXPORT_SYMBOL(cpu_possible_map);

/* Per CPU bogomips and other parameters */
struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;

/* Set when the idlers are all forked */
int smp_threads_ready;

/* representing HT siblings of each logical CPU */
cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;

/* representing HT and core siblings of each logical CPU */
cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(cpu_core_map);

/*
 * Trampoline 80x86 program as an array.
 */

extern unsigned char trampoline_data[];
extern unsigned char trampoline_end[];

/* State of each CPU */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

/*
 * Store all idle threads, this can be reused instead of creating
 * a new thread. Also avoids complicated thread destroy functionality
 * for idle threads.
 */
struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;

#define get_idle_for_cpu(x)     (idle_thread_array[(x)])
#define set_idle_for_cpu(x,p)   (idle_thread_array[(x)] = (p))

/*
 * Currently trivial. Write the real->protected mode
 * bootstrap into the page concerned. The caller
 * has made sure it's suitably aligned.
 */

static unsigned long __cpuinit setup_trampoline(void)
{
	void *tramp = __va(SMP_TRAMPOLINE_BASE); 
	memcpy(tramp, trampoline_data, trampoline_end - trampoline_data);
	return virt_to_phys(tramp);
}

/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */

static void __cpuinit smp_store_cpu_info(int id)
{
	struct cpuinfo_x86 *c = cpu_data + id;

	*c = boot_cpu_data;
	identify_cpu(c);
	print_cpu_info(c);
}

/*
 * New Funky TSC sync algorithm borrowed from IA64.
 * Main advantage is that it doesn't reset the TSCs fully and
 * in general looks more robust and it works better than my earlier
 * attempts. I believe it was written by David Mosberger. Some minor
 * adjustments for x86-64 by me -AK
 *
 * Original comment reproduced below.
 *
 * Synchronize TSC of the current (slave) CPU with the TSC of the
 * MASTER CPU (normally the time-keeper CPU).  We use a closed loop to
 * eliminate the possibility of unaccounted-for errors (such as
 * getting a machine check in the middle of a calibration step).  The
 * basic idea is for the slave to ask the master what itc value it has
 * and to read its own itc before and after the master responds.  Each
 * iteration gives us three timestamps:
 *
 *	slave		master
 *
 *	t0 ---\
 *             ---\
 *		   --->
 *			tm
 *		   /---
 *	       /---
 *	t1 <---
 *
 *
 * The goal is to adjust the slave's TSC such that tm falls exactly
 * half-way between t0 and t1.  If we achieve this, the clocks are
 * synchronized provided the interconnect between the slave and the
 * master is symmetric.  Even if the interconnect were asymmetric, we
 * would still know that the synchronization error is smaller than the
 * roundtrip latency (t0 - t1).
 *
 * When the interconnect is quiet and symmetric, this lets us
 * synchronize the TSC to within one or two cycles.  However, we can
 * only *guarantee* that the synchronization is accurate to within a
 * round-trip time, which is typically in the range of several hundred
 * cycles (e.g., ~500 cycles).  In practice, this means that the TSCs
 * are usually almost perfectly synchronized, but we shouldn't assume
 * that the accuracy is much better than half a micro second or so.
 *
 * [there are other errors like the latency of RDTSC and of the
 * WRMSR. These can also account to hundreds of cycles. So it's
 * probably worse. It claims 153 cycles error on a dual Opteron,
 * but I suspect the numbers are actually somewhat worse -AK]
 */

#define MASTER	0
#define SLAVE	(SMP_CACHE_BYTES/8)

/* Intentionally don't use cpu_relax() while TSC synchronization
   because we don't want to go into funky power save modi or cause
   hypervisors to schedule us away.  Going to sleep would likely affect
   latency and low latency is the primary objective here. -AK */
#define no_cpu_relax() barrier()

static __cpuinitdata DEFINE_SPINLOCK(tsc_sync_lock);
static volatile __cpuinitdata unsigned long go[SLAVE + 1];
static int notscsync __cpuinitdata;

#undef DEBUG_TSC_SYNC

#define NUM_ROUNDS	64	/* magic value */
#define NUM_ITERS	5	/* likewise */

/* Callback on boot CPU */
static __cpuinit void sync_master(void *arg)
{
	unsigned long flags, i;

	go[MASTER] = 0;

	local_irq_save(flags);
	{
		for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
			while (!go[MASTER])
				no_cpu_relax();
			go[MASTER] = 0;
			rdtscll(go[SLAVE]);
		}
	}
	local_irq_restore(flags);
}

/*
 * Return the number of cycles by which our tsc differs from the tsc
 * on the master (time-keeper) CPU.  A positive number indicates our
 * tsc is ahead of the master, negative that it is behind.
 */
static inline long
get_delta(long *rt, long *master)
{
	unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
	unsigned long tcenter, t0, t1, tm;
	int i;

	for (i = 0; i < NUM_ITERS; ++i) {
		rdtscll(t0);
		go[MASTER] = 1;
		while (!(tm = go[SLAVE]))
			no_cpu_relax();
		go[SLAVE] = 0;
		rdtscll(t1);

		if (t1 - t0 < best_t1 - best_t0)
			best_t0 = t0, best_t1 = t1, best_tm = tm;
	}

	*rt = best_t1 - best_t0;
	*master = best_tm - best_t0;

	/* average best_t0 and best_t1 without overflow: */
	tcenter = (best_t0/2 + best_t1/2);
	if (best_t0 % 2 + best_t1 % 2 == 2)
		++tcenter;
	return tcenter - best_tm;
}

static __cpuinit void sync_tsc(unsigned int master)
{
	int i, done = 0;
	long delta, adj, adjust_latency = 0;
	unsigned long flags, rt, master_time_stamp, bound;
#ifdef DEBUG_TSC_SYNC
	static struct syncdebug {
		long rt;	/* roundtrip time */
		long master;	/* master's timestamp */
		long diff;	/* difference between midpoint and master's timestamp */
		long lat;	/* estimate of tsc adjustment latency */
	} t[NUM_ROUNDS] __cpuinitdata;
#endif

	printk(KERN_INFO "CPU %d: Syncing TSC to CPU %u.\n",
		smp_processor_id(), master);

	go[MASTER] = 1;

	/* It is dangerous to broadcast IPI as cpus are coming up,
	 * as they may not be ready to accept them.  So since
	 * we only need to send the ipi to the boot cpu direct
	 * the message, and avoid the race.
	 */
	smp_call_function_single(master, sync_master, NULL, 1, 0);

	while (go[MASTER])	/* wait for master to be ready */
		no_cpu_relax();

	spin_lock_irqsave(&tsc_sync_lock, flags);
	{
		for (i = 0; i < NUM_ROUNDS; ++i) {
			delta = get_delta(&rt, &master_time_stamp);
			if (delta == 0) {
				done = 1;	/* let's lock on to this... */
				bound = rt;
			}

			if (!done) {
				unsigned long t;
				if (i > 0) {
					adjust_latency += -delta;
					adj = -delta + adjust_latency/4;
				} else
					adj = -delta;

				rdtscll(t);
				wrmsrl(MSR_IA32_TSC, t + adj);
			}
#ifdef DEBUG_TSC_SYNC
			t[i].rt = rt;
			t[i].master = master_time_stamp;
			t[i].diff = delta;
			t[i].lat = adjust_latency/4;
#endif
		}
	}
	spin_unlock_irqrestore(&tsc_sync_lock, flags);

#ifdef DEBUG_TSC_SYNC
	for (i = 0; i < NUM_ROUNDS; ++i)
		printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
		       t[i].rt, t[i].master, t[i].diff, t[i].lat);
#endif

	printk(KERN_INFO
	       "CPU %d: synchronized TSC with CPU %u (last diff %ld cycles, "
	       "maxerr %lu cycles)\n",
	       smp_processor_id(), master, delta, rt);
}

static void __cpuinit tsc_sync_wait(void)
{
	if (notscsync || !cpu_has_tsc)
		return;
	sync_tsc(0);
}

static __init int notscsync_setup(char *s)
{
	notscsync = 1;
	return 0;
}
__setup("notscsync", notscsync_setup);

static atomic_t init_deasserted __cpuinitdata;

/*
 * Report back to the Boot Processor.
 * Running on AP.
 */
void __cpuinit smp_callin(void)
{
	int cpuid, phys_id;
	unsigned long timeout;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
	 */
	while (!atomic_read(&init_deasserted))
		cpu_relax();

	/*
	 * (This works even if the APIC is not enabled.)
	 */
	phys_id = GET_APIC_ID(apic_read(APIC_ID));
	cpuid = smp_processor_id();
	if (cpu_isset(cpuid, cpu_callin_map)) {
		panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
					phys_id, cpuid);
	}
	Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
		if (cpu_isset(cpuid, cpu_callout_map))
			break;
		cpu_relax();
	}

	if (!time_before(jiffies, timeout)) {
		panic("smp_callin: CPU%d started up but did not get a callout!\n",
			cpuid);
	}

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */

	Dprintk("CALLIN, before setup_local_APIC().\n");