smp.c

linux-2.6.15.6

	data.func = func;
	data.info = info;
	data.wait = wait;

	cpu_clear(smp_processor_id(), callmap);
	if (cpus_empty(callmap))
		goto out;

	data.pending = callmap;
	if (wait)
		data.unfinished = callmap;

	/*
	 * try to get the mutex on smp_call_function_data
	 */
	spin_lock(&smp_call_function_lock);
	smp_call_function_data = &data;

	send_ipi_message(callmap, IPI_CALL_FUNC);

	timeout = jiffies + HZ;
	while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
		barrier();

	/*
	 * did we time out?
	 */
	if (!cpus_empty(data.pending)) {
		/*
		 * this may be causing our panic - report it
		 */
		printk(KERN_CRIT
		       "CPU%u: smp_call_function timeout for %p(%p)\n"
		       "      callmap %lx pending %lx, %swait\n",
		       smp_processor_id(), func, info, *cpus_addr(callmap),
		       *cpus_addr(data.pending), wait ? "" : "no ");

		/*
		 * TRACE
		 */
		timeout = jiffies + (5 * HZ);
		while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
			barrier();

		if (cpus_empty(data.pending))
			printk(KERN_CRIT "     RESOLVED\n");
		else
			printk(KERN_CRIT "     STILL STUCK\n");
	}

	/*
	 * whatever happened, we're done with the data, so release it
	 */
	smp_call_function_data = NULL;
	spin_unlock(&smp_call_function_lock);

	if (!cpus_empty(data.pending)) {
		ret = -ETIMEDOUT;
		goto out;
	}

	if (wait)
		while (!cpus_empty(data.unfinished))
			barrier();
 out:

	return 0;
}

int smp_call_function(void (*func)(void *info), void *info, int retry,
                      int wait)
{
	return smp_call_function_on_cpu(func, info, retry, wait,
					cpu_online_map);
}

void show_ipi_list(struct seq_file *p)
{
	unsigned int cpu;

	seq_puts(p, "IPI:");

	for_each_present_cpu(cpu)
		seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count);

	seq_putc(p, '\n');
}

void show_local_irqs(struct seq_file *p)
{
	unsigned int cpu;

	seq_printf(p, "LOC: ");

	for_each_present_cpu(cpu)
		seq_printf(p, "%10u ", irq_stat[cpu].local_timer_irqs);

	seq_putc(p, '\n');
}

static void ipi_timer(struct pt_regs *regs)
{
	int user = user_mode(regs);

	irq_enter();
	profile_tick(CPU_PROFILING, regs);
	update_process_times(user);
	irq_exit();
}

#ifdef CONFIG_LOCAL_TIMERS
asmlinkage void do_local_timer(struct pt_regs *regs)
{
	int cpu = smp_processor_id();

	if (local_timer_ack()) {
		irq_stat[cpu].local_timer_irqs++;
		ipi_timer(regs);
	}
}
#endif

/*
 * ipi_call_function - handle IPI from smp_call_function()
 *
 * Note that we copy data out of the cross-call structure and then
 * let the caller know that we're here and have done with their data
 */
static void ipi_call_function(unsigned int cpu)
{
	struct smp_call_struct *data = smp_call_function_data;
	void (*func)(void *info) = data->func;
	void *info = data->info;
	int wait = data->wait;

	cpu_clear(cpu, data->pending);

	func(info);

	if (wait)
		cpu_clear(cpu, data->unfinished);
}

static DEFINE_SPINLOCK(stop_lock);

/*
 * ipi_cpu_stop - handle IPI from smp_send_stop()
 */
static void ipi_cpu_stop(unsigned int cpu)
{
	spin_lock(&stop_lock);
	printk(KERN_CRIT "CPU%u: stopping\n", cpu);
	dump_stack();
	spin_unlock(&stop_lock);

	cpu_clear(cpu, cpu_online_map);

	local_fiq_disable();
	local_irq_disable();

	while (1)
		cpu_relax();
}

/*
 * Main handler for inter-processor interrupts
 *
 * For ARM, the ipimask now only identifies a single
 * category of IPI (Bit 1 IPIs have been replaced by a
 * different mechanism):
 *
 *  Bit 0 - Inter-processor function call
 */
asmlinkage void do_IPI(struct pt_regs *regs)
{
	unsigned int cpu = smp_processor_id();
	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);

	ipi->ipi_count++;

	for (;;) {
		unsigned long msgs;

		spin_lock(&ipi->lock);
		msgs = ipi->bits;
		ipi->bits = 0;
		spin_unlock(&ipi->lock);

		if (!msgs)
			break;

		do {
			unsigned nextmsg;

			nextmsg = msgs & -msgs;
			msgs &= ~nextmsg;
			nextmsg = ffz(~nextmsg);

			switch (nextmsg) {
			case IPI_TIMER:
				ipi_timer(regs);
				break;

			case IPI_RESCHEDULE:
				/*
				 * nothing more to do - eveything is
				 * done on the interrupt return path
				 */
				break;

			case IPI_CALL_FUNC:
				ipi_call_function(cpu);
				break;

			case IPI_CPU_STOP:
				ipi_cpu_stop(cpu);
				break;

			default:
				printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
				       cpu, nextmsg);
				break;
			}
		} while (msgs);
	}
}

void smp_send_reschedule(int cpu)
{
	send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE);
}

void smp_send_timer(void)
{
	cpumask_t mask = cpu_online_map;
	cpu_clear(smp_processor_id(), mask);
	send_ipi_message(mask, IPI_TIMER);
}

void smp_send_stop(void)
{
	cpumask_t mask = cpu_online_map;
	cpu_clear(smp_processor_id(), mask);
	send_ipi_message(mask, IPI_CPU_STOP);
}

/*
 * not supported here
 */
int __init setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}

static int
on_each_cpu_mask(void (*func)(void *), void *info, int retry, int wait,
		 cpumask_t mask)
{
	int ret = 0;

	preempt_disable();

	ret = smp_call_function_on_cpu(func, info, retry, wait, mask);
	if (cpu_isset(smp_processor_id(), mask))
		func(info);

	preempt_enable();

	return ret;
}

/**********************************************************************/

/*
 * TLB operations
 */
struct tlb_args {
	struct vm_area_struct *ta_vma;
	unsigned long ta_start;
	unsigned long ta_end;
};

static inline void ipi_flush_tlb_all(void *ignored)
{
	local_flush_tlb_all();
}

static inline void ipi_flush_tlb_mm(void *arg)
{
	struct mm_struct *mm = (struct mm_struct *)arg;

	local_flush_tlb_mm(mm);
}

static inline void ipi_flush_tlb_page(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_page(ta->ta_vma, ta->ta_start);
}

static inline void ipi_flush_tlb_kernel_page(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_kernel_page(ta->ta_start);
}

static inline void ipi_flush_tlb_range(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
}

static inline void ipi_flush_tlb_kernel_range(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
}

void flush_tlb_all(void)
{
	on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1);
}

void flush_tlb_mm(struct mm_struct *mm)
{
	cpumask_t mask = mm->cpu_vm_mask;

	on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask);
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
{
	cpumask_t mask = vma->vm_mm->cpu_vm_mask;
	struct tlb_args ta;

	ta.ta_vma = vma;
	ta.ta_start = uaddr;

	on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, 1, mask);
}

void flush_tlb_kernel_page(unsigned long kaddr)
{
	struct tlb_args ta;

	ta.ta_start = kaddr;

	on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1, 1);
}

void flush_tlb_range(struct vm_area_struct *vma,
                     unsigned long start, unsigned long end)
{
	cpumask_t mask = vma->vm_mm->cpu_vm_mask;
	struct tlb_args ta;

	ta.ta_vma = vma;
	ta.ta_start = start;
	ta.ta_end = end;

	on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, 1, mask);
}

void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	struct tlb_args ta;

	ta.ta_start = start;
	ta.ta_end = end;

	on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1, 1);
}