io_apic_32.c

linux 内核源代码

/*
 *	Intel IO-APIC support for multi-Pentium hosts.
 *
 *	Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo
 *
 *	Many thanks to Stig Venaas for trying out countless experimental
 *	patches and reporting/debugging problems patiently!
 *
 *	(c) 1999, Multiple IO-APIC support, developed by
 *	Ken-ichi Yaku <[e-mail]> and
 *      Hidemi Kishimoto <[e-mail]>,
 *	further tested and cleaned up by Zach Brown <[e-mail]>
 *	and Ingo Molnar <[e-mail]>
 *
 *	Fixes
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
 *					thanks to Eric Gilmore
 *					and Rolf G. Tews
 *					for testing these extensively
 *	Paul Diefenbaugh	:	Added full ACPI support
 */

#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
#include <linux/acpi.h>
#include <linux/module.h>
#include <linux/sysdev.h>
#include <linux/pci.h>
#include <linux/msi.h>
#include <linux/htirq.h>
#include <linux/freezer.h>
#include <linux/kthread.h>

#include <asm/io.h>
#include <asm/smp.h>
#include <asm/desc.h>
#include <asm/timer.h>
#include <asm/i8259.h>
#include <asm/nmi.h>
#include <asm/msidef.h>
#include <asm/hypertransport.h>

#include <mach_apic.h>
#include <mach_apicdef.h>

#include "io_ports.h"

int (*ioapic_renumber_irq)(int ioapic, int irq);
atomic_t irq_mis_count;

/* Where if anywhere is the i8259 connect in external int mode */
static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };

static DEFINE_SPINLOCK(ioapic_lock);
static DEFINE_SPINLOCK(vector_lock);

int timer_over_8254 __initdata = 1;

/*
 *	Is the SiS APIC rmw bug present ?
 *	-1 = don't know, 0 = no, 1 = yes
 */
int sis_apic_bug = -1;

/*
 * # of IRQ routing registers
 */
int nr_ioapic_registers[MAX_IO_APICS];

static int disable_timer_pin_1 __initdata;

/*
 * Rough estimation of how many shared IRQs there are, can
 * be changed anytime.
 */
#define MAX_PLUS_SHARED_IRQS NR_IRQS
#define PIN_MAP_SIZE (MAX_PLUS_SHARED_IRQS + NR_IRQS)

/*
 * This is performance-critical, we want to do it O(1)
 *
 * the indexing order of this array favors 1:1 mappings
 * between pins and IRQs.
 */

static struct irq_pin_list {
	int apic, pin, next;
} irq_2_pin[PIN_MAP_SIZE];

struct io_apic {
	unsigned int index;
	unsigned int unused[3];
	unsigned int data;
};

static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
{
	return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
		+ (mp_ioapics[idx].mpc_apicaddr & ~PAGE_MASK);
}

static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	return readl(&io_apic->data);
}

static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	writel(value, &io_apic->data);
}

/*
 * Re-write a value: to be used for read-modify-write
 * cycles where the read already set up the index register.
 *
 * Older SiS APIC requires we rewrite the index register
 */
static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
{
	volatile struct io_apic __iomem *io_apic = io_apic_base(apic);
	if (sis_apic_bug)
		writel(reg, &io_apic->index);
	writel(value, &io_apic->data);
}

union entry_union {
	struct { u32 w1, w2; };
	struct IO_APIC_route_entry entry;
};

static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
{
	union entry_union eu;
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
	eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
	spin_unlock_irqrestore(&ioapic_lock, flags);
	return eu.entry;
}

/*
 * When we write a new IO APIC routing entry, we need to write the high
 * word first! If the mask bit in the low word is clear, we will enable
 * the interrupt, and we need to make sure the entry is fully populated
 * before that happens.
 */
static void
__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
{
	union entry_union eu;
	eu.entry = e;
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
}

static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
{
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	__ioapic_write_entry(apic, pin, e);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

/*
 * When we mask an IO APIC routing entry, we need to write the low
 * word first, in order to set the mask bit before we change the
 * high bits!
 */
static void ioapic_mask_entry(int apic, int pin)
{
	unsigned long flags;
	union entry_union eu = { .entry.mask = 1 };

	spin_lock_irqsave(&ioapic_lock, flags);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

/*
 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
 * shared ISA-space IRQs, so we have to support them. We are super
 * fast in the common case, and fast for shared ISA-space IRQs.
 */
static void add_pin_to_irq(unsigned int irq, int apic, int pin)
{
	static int first_free_entry = NR_IRQS;
	struct irq_pin_list *entry = irq_2_pin + irq;

	while (entry->next)
		entry = irq_2_pin + entry->next;

	if (entry->pin != -1) {
		entry->next = first_free_entry;
		entry = irq_2_pin + entry->next;
		if (++first_free_entry >= PIN_MAP_SIZE)
			panic("io_apic.c: whoops");
	}
	entry->apic = apic;
	entry->pin = pin;
}

/*
 * Reroute an IRQ to a different pin.
 */
static void __init replace_pin_at_irq(unsigned int irq,
				      int oldapic, int oldpin,
				      int newapic, int newpin)
{
	struct irq_pin_list *entry = irq_2_pin + irq;

	while (1) {
		if (entry->apic == oldapic && entry->pin == oldpin) {
			entry->apic = newapic;
			entry->pin = newpin;
		}
		if (!entry->next)
			break;
		entry = irq_2_pin + entry->next;
	}
}

static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsigned long disable)
{
	struct irq_pin_list *entry = irq_2_pin + irq;
	unsigned int pin, reg;

	for (;;) {
		pin = entry->pin;
		if (pin == -1)
			break;
		reg = io_apic_read(entry->apic, 0x10 + pin*2);
		reg &= ~disable;
		reg |= enable;
		io_apic_modify(entry->apic, 0x10 + pin*2, reg);
		if (!entry->next)
			break;
		entry = irq_2_pin + entry->next;
	}
}

/* mask = 1 */
static void __mask_IO_APIC_irq (unsigned int irq)
{
	__modify_IO_APIC_irq(irq, 0x00010000, 0);
}

/* mask = 0 */
static void __unmask_IO_APIC_irq (unsigned int irq)
{
	__modify_IO_APIC_irq(irq, 0, 0x00010000);
}

/* mask = 1, trigger = 0 */
static void __mask_and_edge_IO_APIC_irq (unsigned int irq)
{
	__modify_IO_APIC_irq(irq, 0x00010000, 0x00008000);
}

/* mask = 0, trigger = 1 */
static void __unmask_and_level_IO_APIC_irq (unsigned int irq)
{
	__modify_IO_APIC_irq(irq, 0x00008000, 0x00010000);
}

static void mask_IO_APIC_irq (unsigned int irq)
{
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	__mask_IO_APIC_irq(irq);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

static void unmask_IO_APIC_irq (unsigned int irq)
{
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	__unmask_IO_APIC_irq(irq);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
{
	struct IO_APIC_route_entry entry;
	
	/* Check delivery_mode to be sure we're not clearing an SMI pin */
	entry = ioapic_read_entry(apic, pin);
	if (entry.delivery_mode == dest_SMI)
		return;

	/*
	 * Disable it in the IO-APIC irq-routing table:
	 */
	ioapic_mask_entry(apic, pin);
}

static void clear_IO_APIC (void)
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			clear_IO_APIC_pin(apic, pin);
}

#ifdef CONFIG_SMP
static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t cpumask)
{
	unsigned long flags;
	int pin;
	struct irq_pin_list *entry = irq_2_pin + irq;
	unsigned int apicid_value;
	cpumask_t tmp;
	
	cpus_and(tmp, cpumask, cpu_online_map);
	if (cpus_empty(tmp))
		tmp = TARGET_CPUS;

	cpus_and(cpumask, tmp, CPU_MASK_ALL);

	apicid_value = cpu_mask_to_apicid(cpumask);
	/* Prepare to do the io_apic_write */
	apicid_value = apicid_value << 24;
	spin_lock_irqsave(&ioapic_lock, flags);
	for (;;) {
		pin = entry->pin;
		if (pin == -1)
			break;
		io_apic_write(entry->apic, 0x10 + 1 + pin*2, apicid_value);
		if (!entry->next)
			break;
		entry = irq_2_pin + entry->next;
	}
	irq_desc[irq].affinity = cpumask;
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

#if defined(CONFIG_IRQBALANCE)
# include <asm/processor.h>	/* kernel_thread() */
# include <linux/kernel_stat.h>	/* kstat */
# include <linux/slab.h>		/* kmalloc() */
# include <linux/timer.h>	/* time_after() */
 
#define IRQBALANCE_CHECK_ARCH -999
#define MAX_BALANCED_IRQ_INTERVAL	(5*HZ)
#define MIN_BALANCED_IRQ_INTERVAL	(HZ/2)
#define BALANCED_IRQ_MORE_DELTA		(HZ/10)
#define BALANCED_IRQ_LESS_DELTA		(HZ)

static int irqbalance_disabled __read_mostly = IRQBALANCE_CHECK_ARCH;
static int physical_balance __read_mostly;
static long balanced_irq_interval __read_mostly = MAX_BALANCED_IRQ_INTERVAL;

static struct irq_cpu_info {
	unsigned long * last_irq;
	unsigned long * irq_delta;
	unsigned long irq;
} irq_cpu_data[NR_CPUS];

#define CPU_IRQ(cpu)		(irq_cpu_data[cpu].irq)
#define LAST_CPU_IRQ(cpu,irq)   (irq_cpu_data[cpu].last_irq[irq])
#define IRQ_DELTA(cpu,irq) 	(irq_cpu_data[cpu].irq_delta[irq])

#define IDLE_ENOUGH(cpu,now) \
	(idle_cpu(cpu) && ((now) - per_cpu(irq_stat, (cpu)).idle_timestamp > 1))

#define IRQ_ALLOWED(cpu, allowed_mask)	cpu_isset(cpu, allowed_mask)

#define CPU_TO_PACKAGEINDEX(i) (first_cpu(per_cpu(cpu_sibling_map, i)))

static cpumask_t balance_irq_affinity[NR_IRQS] = {
	[0 ... NR_IRQS-1] = CPU_MASK_ALL
};

void set_balance_irq_affinity(unsigned int irq, cpumask_t mask)
{
	balance_irq_affinity[irq] = mask;
}

static unsigned long move(int curr_cpu, cpumask_t allowed_mask,
			unsigned long now, int direction)
{
	int search_idle = 1;
	int cpu = curr_cpu;

	goto inside;

	do {
		if (unlikely(cpu == curr_cpu))
			search_idle = 0;
inside:
		if (direction == 1) {
			cpu++;
			if (cpu >= NR_CPUS)
				cpu = 0;
		} else {
			cpu--;
			if (cpu == -1)
				cpu = NR_CPUS-1;
		}
	} while (!cpu_online(cpu) || !IRQ_ALLOWED(cpu,allowed_mask) ||
			(search_idle && !IDLE_ENOUGH(cpu,now)));

	return cpu;
}

static inline void balance_irq(int cpu, int irq)
{
	unsigned long now = jiffies;
	cpumask_t allowed_mask;
	unsigned int new_cpu;
		
	if (irqbalance_disabled)
		return; 

	cpus_and(allowed_mask, cpu_online_map, balance_irq_affinity[irq]);
	new_cpu = move(cpu, allowed_mask, now, 1);
	if (cpu != new_cpu) {
		set_pending_irq(irq, cpumask_of_cpu(new_cpu));
	}
}

static inline void rotate_irqs_among_cpus(unsigned long useful_load_threshold)
{
	int i, j;

	for_each_online_cpu(i) {
		for (j = 0; j < NR_IRQS; j++) {
			if (!irq_desc[j].action)
				continue;
			/* Is it a significant load ?  */
			if (IRQ_DELTA(CPU_TO_PACKAGEINDEX(i),j) <
						useful_load_threshold)
				continue;
			balance_irq(i, j);
		}
	}
	balanced_irq_interval = max((long)MIN_BALANCED_IRQ_INTERVAL,
		balanced_irq_interval - BALANCED_IRQ_LESS_DELTA);	
	return;
}

static void do_irq_balance(void)
{
	int i, j;
	unsigned long max_cpu_irq = 0, min_cpu_irq = (~0);
	unsigned long move_this_load = 0;
	int max_loaded = 0, min_loaded = 0;
	int load;
	unsigned long useful_load_threshold = balanced_irq_interval + 10;
	int selected_irq;
	int tmp_loaded, first_attempt = 1;
	unsigned long tmp_cpu_irq;
	unsigned long imbalance = 0;
	cpumask_t allowed_mask, target_cpu_mask, tmp;

	for_each_possible_cpu(i) {
		int package_index;
		CPU_IRQ(i) = 0;
		if (!cpu_online(i))
			continue;
		package_index = CPU_TO_PACKAGEINDEX(i);
		for (j = 0; j < NR_IRQS; j++) {
			unsigned long value_now, delta;
			/* Is this an active IRQ or balancing disabled ? */
			if (!irq_desc[j].action || irq_balancing_disabled(j))
				continue;
			if ( package_index == i )
				IRQ_DELTA(package_index,j) = 0;
			/* Determine the total count per processor per IRQ */
			value_now = (unsigned long) kstat_cpu(i).irqs[j];

			/* Determine the activity per processor per IRQ */
			delta = value_now - LAST_CPU_IRQ(i,j);

			/* Update last_cpu_irq[][] for the next time */
			LAST_CPU_IRQ(i,j) = value_now;

			/* Ignore IRQs whose rate is less than the clock */
			if (delta < useful_load_threshold)
				continue;
			/* update the load for the processor or package total */
			IRQ_DELTA(package_index,j) += delta;

			/* Keep track of the higher numbered sibling as well */
			if (i != package_index)
				CPU_IRQ(i) += delta;
			/*
			 * We have sibling A and sibling B in the package
			 *
			 * cpu_irq[A] = load for cpu A + load for cpu B
			 * cpu_irq[B] = load for cpu B
			 */
			CPU_IRQ(package_index) += delta;
		}
	}
	/* Find the least loaded processor package */
	for_each_online_cpu(i) {
		if (i != CPU_TO_PACKAGEINDEX(i))
			continue;
		if (min_cpu_irq > CPU_IRQ(i)) {
			min_cpu_irq = CPU_IRQ(i);
			min_loaded = i;
		}
	}
	max_cpu_irq = ULONG_MAX;

tryanothercpu:
	/* Look for heaviest loaded processor.
	 * We may come back to get the next heaviest loaded processor.
	 * Skip processors with trivial loads.
	 */
	tmp_cpu_irq = 0;
	tmp_loaded = -1;
	for_each_online_cpu(i) {
		if (i != CPU_TO_PACKAGEINDEX(i))