switch.c

linux2.6.16版本

/*
 * spu_switch.c
 *
 * (C) Copyright IBM Corp. 2005
 *
 * Author: Mark Nutter <[e-mail]>
 *
 * Host-side part of SPU context switch sequence outlined in
 * Synergistic Processor Element, Book IV.
 *
 * A fully premptive switch of an SPE is very expensive in terms
 * of time and system resources.  SPE Book IV indicates that SPE
 * allocation should follow a "serially reusable device" model,
 * in which the SPE is assigned a task until it completes.  When
 * this is not possible, this sequence may be used to premptively
 * save, and then later (optionally) restore the context of a
 * program executing on an SPE.
 *
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include <linux/config.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <linux/smp.h>
#include <linux/smp_lock.h>
#include <linux/stddef.h>
#include <linux/unistd.h>

#include <asm/io.h>
#include <asm/spu.h>
#include <asm/spu_csa.h>
#include <asm/mmu_context.h>

#include "spu_save_dump.h"
#include "spu_restore_dump.h"

#if 0
#define POLL_WHILE_TRUE(_c) {				\
    do {						\
    } while (_c);					\
  }
#else
#define RELAX_SPIN_COUNT				1000
#define POLL_WHILE_TRUE(_c) {				\
    do {						\
	int _i;						\
	for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \
	    cpu_relax();				\
	}						\
	if (unlikely(_c)) yield();			\
	else break;					\
    } while (_c);					\
  }
#endif				/* debug */

#define POLL_WHILE_FALSE(_c) 	POLL_WHILE_TRUE(!(_c))

static inline void acquire_spu_lock(struct spu *spu)
{
	/* Save, Step 1:
	 * Restore, Step 1:
	 *    Acquire SPU-specific mutual exclusion lock.
	 *    TBD.
	 */
}

static inline void release_spu_lock(struct spu *spu)
{
	/* Restore, Step 76:
	 *    Release SPU-specific mutual exclusion lock.
	 *    TBD.
	 */
}

static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu)
{
	struct spu_problem __iomem *prob = spu->problem;
	u32 isolate_state;

	/* Save, Step 2:
	 * Save, Step 6:
	 *     If SPU_Status[E,L,IS] any field is '1', this
	 *     SPU is in isolate state and cannot be context
	 *     saved at this time.
	 */
	isolate_state = SPU_STATUS_ISOLATED_STATE |
	    SPU_STATUS_ISOLATED_LOAD_STAUTUS | SPU_STATUS_ISOLATED_EXIT_STAUTUS;
	return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0;
}

static inline void disable_interrupts(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 3:
	 * Restore, Step 2:
	 *     Save INT_Mask_class0 in CSA.
	 *     Write INT_MASK_class0 with value of 0.
	 *     Save INT_Mask_class1 in CSA.
	 *     Write INT_MASK_class1 with value of 0.
	 *     Save INT_Mask_class2 in CSA.
	 *     Write INT_MASK_class2 with value of 0.
	 */
	spin_lock_irq(&spu->register_lock);
	if (csa) {
		csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0);
		csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1);
		csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2);
	}
	spu_int_mask_set(spu, 0, 0ul);
	spu_int_mask_set(spu, 1, 0ul);
	spu_int_mask_set(spu, 2, 0ul);
	eieio();
	spin_unlock_irq(&spu->register_lock);
}

static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 4:
	 * Restore, Step 25.
	 *    Set a software watchdog timer, which specifies the
	 *    maximum allowable time for a context save sequence.
	 *
	 *    For present, this implementation will not set a global
	 *    watchdog timer, as virtualization & variable system load
	 *    may cause unpredictable execution times.
	 */
}

static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 5:
	 * Restore, Step 3:
	 *     Inhibit user-space access (if provided) to this
	 *     SPU by unmapping the virtual pages assigned to
	 *     the SPU memory-mapped I/O (MMIO) for problem
	 *     state. TBD.
	 */
}

static inline void set_switch_pending(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 7:
	 * Restore, Step 5:
	 *     Set a software context switch pending flag.
	 */
	set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
	mb();
}

static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu)
{
	struct spu_priv2 __iomem *priv2 = spu->priv2;

	/* Save, Step 8:
	 *     Suspend DMA and save MFC_CNTL.
	 */
	switch (in_be64(&priv2->mfc_control_RW) &
	       MFC_CNTL_SUSPEND_DMA_STATUS_MASK) {
	case MFC_CNTL_SUSPEND_IN_PROGRESS:
		POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
				  MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
				 MFC_CNTL_SUSPEND_COMPLETE);
		/* fall through */
	case MFC_CNTL_SUSPEND_COMPLETE:
		if (csa) {
			csa->priv2.mfc_control_RW =
				in_be64(&priv2->mfc_control_RW) |
				MFC_CNTL_SUSPEND_DMA_QUEUE;
		}
		break;
	case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION:
		out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE);
		POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
				  MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
				 MFC_CNTL_SUSPEND_COMPLETE);
		if (csa) {
			csa->priv2.mfc_control_RW =
				in_be64(&priv2->mfc_control_RW) &
				~MFC_CNTL_SUSPEND_DMA_QUEUE;
		}
		break;
	}
}

static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu)
{
	struct spu_problem __iomem *prob = spu->problem;

	/* Save, Step 9:
	 *     Save SPU_Runcntl in the CSA.  This value contains
	 *     the "Application Desired State".
	 */
	csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW);
}

static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 10:
	 *     Save MFC_SR1 in the CSA.
	 */
	csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu);
}

static inline void save_spu_status(struct spu_state *csa, struct spu *spu)
{
	struct spu_problem __iomem *prob = spu->problem;

	/* Save, Step 11:
	 *     Read SPU_Status[R], and save to CSA.
	 */
	if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) {
		csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
	} else {
		u32 stopped;

		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
		eieio();
		POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
				SPU_STATUS_RUNNING);
		stopped =
		    SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP |
		    SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
		if ((in_be32(&prob->spu_status_R) & stopped) == 0)
			csa->prob.spu_status_R = SPU_STATUS_RUNNING;
		else
			csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
	}
}

static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu)
{
	struct spu_priv2 __iomem *priv2 = spu->priv2;

	/* Save, Step 12:
	 *     Read MFC_CNTL[Ds].  Update saved copy of
	 *     CSA.MFC_CNTL[Ds].
	 */
	if (in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING) {
		csa->priv2.mfc_control_RW |= MFC_CNTL_DECREMENTER_RUNNING;
		csa->suspend_time = get_cycles();
		out_be64(&priv2->spu_chnlcntptr_RW, 7ULL);
		eieio();
		csa->spu_chnldata_RW[7] = in_be64(&priv2->spu_chnldata_RW);
		eieio();
	} else {
		csa->priv2.mfc_control_RW &= ~MFC_CNTL_DECREMENTER_RUNNING;
	}
}

static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu)
{
	struct spu_priv2 __iomem *priv2 = spu->priv2;

	/* Save, Step 13:
	 *     Write MFC_CNTL[Dh] set to a '1' to halt
	 *     the decrementer.
	 */
	out_be64(&priv2->mfc_control_RW, MFC_CNTL_DECREMENTER_HALTED);
	eieio();
}

static inline void save_timebase(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 14:
	 *    Read PPE Timebase High and Timebase low registers
	 *    and save in CSA.  TBD.
	 */
	csa->suspend_time = get_cycles();
}

static inline void remove_other_spu_access(struct spu_state *csa,
					   struct spu *spu)
{
	/* Save, Step 15:
	 *     Remove other SPU access to this SPU by unmapping
	 *     this SPU's pages from their address space.  TBD.
	 */
}

static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu)
{
	struct spu_problem __iomem *prob = spu->problem;

	/* Save, Step 16:
	 * Restore, Step 11.
	 *     Write SPU_MSSync register. Poll SPU_MSSync[P]
	 *     for a value of 0.
	 */
	out_be64(&prob->spc_mssync_RW, 1UL);
	POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING);
}

static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 17:
	 * Restore, Step 12.
	 * Restore, Step 48.
	 *     Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register.
	 *     Then issue a PPE sync instruction.
	 */
	spu_tlb_invalidate(spu);
	mb();
}

static inline void handle_pending_interrupts(struct spu_state *csa,
					     struct spu *spu)
{
	/* Save, Step 18:
	 *     Handle any pending interrupts from this SPU
	 *     here.  This is OS or hypervisor specific.  One
	 *     option is to re-enable interrupts to handle any
	 *     pending interrupts, with the interrupt handlers
	 *     recognizing the software Context Switch Pending
	 *     flag, to ensure the SPU execution or MFC command
	 *     queue is not restarted.  TBD.
	 */
}

static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu)
{
	struct spu_priv2 __iomem *priv2 = spu->priv2;
	int i;

	/* Save, Step 19:
	 *     If MFC_Cntl[Se]=0 then save
	 *     MFC command queues.
	 */
	if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) {
		for (i = 0; i < 8; i++) {
			csa->priv2.puq[i].mfc_cq_data0_RW =
			    in_be64(&priv2->puq[i].mfc_cq_data0_RW);
			csa->priv2.puq[i].mfc_cq_data1_RW =
			    in_be64(&priv2->puq[i].mfc_cq_data1_RW);
			csa->priv2.puq[i].mfc_cq_data2_RW =
			    in_be64(&priv2->puq[i].mfc_cq_data2_RW);
			csa->priv2.puq[i].mfc_cq_data3_RW =
			    in_be64(&priv2->puq[i].mfc_cq_data3_RW);
		}
		for (i = 0; i < 16; i++) {
			csa->priv2.spuq[i].mfc_cq_data0_RW =
			    in_be64(&priv2->spuq[i].mfc_cq_data0_RW);
			csa->priv2.spuq[i].mfc_cq_data1_RW =
			    in_be64(&priv2->spuq[i].mfc_cq_data1_RW);
			csa->priv2.spuq[i].mfc_cq_data2_RW =
			    in_be64(&priv2->spuq[i].mfc_cq_data2_RW);
			csa->priv2.spuq[i].mfc_cq_data3_RW =
			    in_be64(&priv2->spuq[i].mfc_cq_data3_RW);
		}
	}
}

static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu)
{
	struct spu_problem __iomem *prob = spu->problem;

	/* Save, Step 20:
	 *     Save the PPU_QueryMask register
	 *     in the CSA.
	 */
	csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW);
}

static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu)
{
	struct spu_problem __iomem *prob = spu->problem;

	/* Save, Step 21:
	 *     Save the PPU_QueryType register
	 *     in the CSA.
	 */
	csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW);
}

static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
{
	struct spu_priv2 __iomem *priv2 = spu->priv2;

	/* Save, Step 22:
	 *     Save the MFC_CSR_TSQ register
	 *     in the LSCSA.
	 */
	csa->priv2.spu_tag_status_query_RW =
	    in_be64(&priv2->spu_tag_status_query_RW);
}

static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
{
	struct spu_priv2 __iomem *priv2 = spu->priv2;

	/* Save, Step 23:
	 *     Save the MFC_CSR_CMD1 and MFC_CSR_CMD2
	 *     registers in the CSA.
	 */
	csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW);
	csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW);
}

static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
{
	struct spu_priv2 __iomem *priv2 = spu->priv2;

	/* Save, Step 24:
	 *     Save the MFC_CSR_ATO register in
	 *     the CSA.
	 */
	csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW);
}

static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 25:
	 *     Save the MFC_TCLASS_ID register in
	 *     the CSA.
	 */
	csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu);
}

static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
{
	/* Save, Step 26:
	 * Restore, Step 23.
	 *     Write the MFC_TCLASS_ID register with
	 *     the value 0x10000000.
	 */