sba_iommu.c

内核linux2.4.20,可跟rtlinux3.2打补丁 组成实时linux系统,编译内核

	struct ioc *ioc = (struct ioc *) ((struct pci_hba *) (dev->sysdata))->dma_data;
#else
	struct ioc *ioc = &sba_list->ioc[0];
#endif
	
	unsigned long flags; 
	dma_addr_t offset;
	offset = iova & ~IOVP_MASK;

	DBG_RUN("%s() iovp 0x%lx/%x\n", __FUNCTION__, (long) iova, size);

	iova ^= offset;        /* clear offset bits */
	size += offset;
	size = ROUNDUP(size, IOVP_SIZE);

	ASSERT(0 != iova);

	spin_lock_irqsave(&ioc->res_lock, flags);
#ifdef CONFIG_PROC_FS
	ioc->usingle_calls++;
	ioc->usingle_pages += size >> IOVP_SHIFT;
#endif
#ifdef DELAYED_RESOURCE_CNT
	if (ioc->saved_cnt < DELAYED_RESOURCE_CNT) {
		ioc->saved_iova[ioc->saved_cnt] = iova;
		ioc->saved_size[ioc->saved_cnt] = size;
		ioc_saved_cnt++;
	} else {
		do {
#endif
			sba_mark_invalid(ioc, iova, size);
			sba_free_range(ioc, iova, size);

#ifdef DELAYED_RESOURCE_CNT
			ioc->saved_cnt--;
			iova = ioc->saved_iova[ioc->saved_cnt];
			size = ioc->saved_size[ioc->saved_cnt];
		} while (ioc->saved_cnt)

		/* flush purges */
		(void) (volatile) READ_REG32(ioc->ioc_hpa+IOC_PCOM);
	}
#else
	/* flush purges */
	READ_REG32(ioc->ioc_hpa+IOC_PCOM);
#endif
	spin_unlock_irqrestore(&ioc->res_lock, flags);
}


static void *
sba_alloc_consistent(struct pci_dev *hwdev, size_t size, dma_addr_t *dma_handle)
{
	void *ret;

	if (!hwdev) {
		/* only support PCI */
		*dma_handle = 0;
		return 0;
	}

        ret = (void *) __get_free_pages(GFP_ATOMIC, get_order(size));

	if (ret) {
		memset(ret, 0, size);
		*dma_handle = sba_map_single(hwdev, ret, size, 0);
	}

	return ret;
}


static void
sba_free_consistent(struct pci_dev *hwdev, size_t size, void *vaddr, dma_addr_t dma_handle)
{
	sba_unmap_single(hwdev, dma_handle, size, 0);
	free_pages((unsigned long) vaddr, get_order(size));
}

/*
** Two address ranges are "virtually contiguous" iff:
** 1) end of prev == start of next, or...	 append case
** 3) end of next == start of prev		 prepend case
**
** and they are DMA contiguous *iff*:
** 2) end of prev and start of next are both on a page boundry
**
** (shift left is a quick trick to mask off upper bits)
*/
#define DMA_CONTIG(__X, __Y) \
	(((((unsigned long) __X) | ((unsigned long) __Y)) << (BITS_PER_LONG - PAGE_SHIFT)) == 0UL)

/*
** Assumption is two transactions are mutually exclusive.
** ie both go to different parts of memory.
** If both are true, then both transaction are on the same page.
*/
#define DMA_SAME_PAGE(s1,e1,s2,e2) \
	( ((((s1) ^ (s2)) >> PAGE_SHIFT) == 0) \
		&& ((((e1) ^ (e2)) >> PAGE_SHIFT) == 0) )

/*
** Since 0 is a valid pdir_base index value, can't use that
** to determine if a value is valid or not. Use a flag to indicate
** the SG list entry contains a valid pdir index.
*/
#define PIDE_FLAG 0x80000000UL

#ifdef DEBUG_LARGE_SG_ENTRIES
int dump_run_sg = 0;
#endif

static SBA_INLINE int
sba_fill_pdir(
	struct ioc *ioc,
	struct scatterlist *startsg,
	int nents)
{
	struct scatterlist *dma_sg = startsg;	/* pointer to current DMA */
	int n_mappings = 0;
	u64 *pdirp = 0;
	unsigned long dma_offset = 0;

	dma_sg--;
	while (nents-- > 0) {
		int     cnt = sg_dma_len(startsg);
		sg_dma_len(startsg) = 0;

#ifdef DEBUG_LARGE_SG_ENTRIES
		if (dump_run_sg)
			printk(" %d : %08lx/%05x %p/%05x\n",
				nents,
				(unsigned long) sg_dma_address(startsg), cnt,
				startsg->address, startsg->length
		);
#else
		DBG_RUN_SG(" %d : %08lx/%05x %p/%05x\n",
				nents,
				(unsigned long) sg_dma_address(startsg), cnt,
				startsg->address, startsg->length
		);
#endif
		/*
		** Look for the start of a new DMA stream
		*/
		if (sg_dma_address(startsg) & PIDE_FLAG) {
			u32 pide = sg_dma_address(startsg) & ~PIDE_FLAG;
			dma_offset = (unsigned long) pide & ~IOVP_MASK;
			pide >>= IOVP_SHIFT;
			pdirp = &(ioc->pdir_base[pide]);
			sg_dma_address(startsg) = 0;
			++dma_sg;
			sg_dma_address(dma_sg) = (pide << IOVP_SHIFT) + dma_offset;
			n_mappings++;
		}

		/*
		** Look for a VCONTIG chunk
		*/
		if (cnt) {
			unsigned long vaddr = (unsigned long) startsg->address;
			ASSERT(pdirp);

			sg_dma_len(dma_sg) += cnt;
			cnt += dma_offset;
			dma_offset=0;	/* only want offset on first chunk */
			cnt = ROUNDUP(cnt, IOVP_SIZE);
#ifdef CONFIG_PROC_FS
			ioc->msg_pages += cnt >> IOVP_SHIFT;
#endif
			do {
				sba_io_pdir_entry(pdirp, KERNEL_SPACE, vaddr);
				vaddr += IOVP_SIZE;
				cnt -= IOVP_SIZE;
				pdirp++;
			} while (cnt > 0);
		}
		startsg++;
	}
#ifdef DEBUG_LARGE_SG_ENTRIES
	dump_run_sg = 0;
#endif
	return(n_mappings);
}



/*
** First pass is to walk the SG list and determine where the breaks are
** in the DMA stream. Allocates PDIR entries but does not fill them.
** Returns the number of DMA chunks.
**
** Doing the fill seperate from the coalescing/allocation keeps the
** code simpler. Future enhancement could make one pass through
** the sglist do both.
*/
static SBA_INLINE int
sba_coalesce_chunks( struct ioc *ioc,
	struct scatterlist *startsg,
	int nents)
{
	int n_mappings = 0;

	while (nents > 0) {
		struct scatterlist *dma_sg;  /* next DMA stream head */
		unsigned long dma_offset, dma_len;   /* start/len of DMA stream */
		struct scatterlist *chunksg; /* virtually contig chunk head */
		unsigned long chunk_addr, chunk_len; /* start/len of VCONTIG chunk */

		/*
		** Prepare for first/next DMA stream
		*/
		dma_sg = chunksg = startsg;
		dma_len = chunk_len  = startsg->length;
		chunk_addr = (unsigned long) startsg->address;
		dma_offset = 0UL;

		/*
		** This loop terminates one iteration "early" since
		** it's always looking one "ahead".
		*/
		while (--nents > 0) {
			/* ptr to coalesce prev and next */
			struct scatterlist *prev_sg = startsg;
			unsigned long prev_end = (unsigned long) prev_sg->address + prev_sg->length;
			unsigned long current_end;

			/* PARANOID: clear entries */
			sg_dma_address(startsg) = 0;
			sg_dma_len(startsg) = 0;

			/* Now start looking ahead */
			startsg++;
			current_end  = (unsigned long) startsg->address + startsg->length;

			/*
			** First look for virtually contiguous blocks.
			** PARISC needs this since it's cache is virtually
			** indexed and we need the associated virtual
			** address for each I/O address we map.
			**
			** 1) can we *prepend* the next transaction?
			*/
			if (current_end == (unsigned long) prev_sg->address)
			{
				/* prepend : get new offset */
				chunksg = startsg;
				chunk_addr = (unsigned long) prev_sg->address;
				chunk_len += startsg->length;
				dma_len   += startsg->length;
				continue;
			}

			/*
			** 2) or append the next transaction?
			*/
			if  (prev_end == (unsigned long) startsg->address)
			{
				chunk_len += startsg->length;
				dma_len   += startsg->length;
				continue;
			}

#ifdef DEBUG_LARGE_SG_ENTRIES
			dump_run_sg = (chunk_len > IOVP_SIZE);
#endif
			/*
			** Not virtually contigous.
			** Terminate prev chunk.
			** Start a new chunk.
			**
			** Once we start a new VCONTIG chunk, the offset
			** can't change. And we need the offset from the first
			** chunk - not the last one. Ergo Successive chunks
			** must start on page boundaries and dove tail
			** with it's predecessor.
			*/
			sg_dma_len(prev_sg) = chunk_len;

			chunk_len = startsg->length;
			dma_offset |= (chunk_addr & ~IOVP_MASK);
			ASSERT((0 == (chunk_addr & ~IOVP_MASK)) ||
				(dma_offset == (chunk_addr & ~IOVP_MASK)));

#if 0
			/*
			** 4) do the chunks end/start on page boundaries?
			**  Easier than 3 since no offsets are involved.
			*/
			if (DMA_CONTIG(prev_end, startsg->address))
			{
				/*
				** Yes.
				** Reset chunk ptr.
				*/
				chunksg = startsg;
				chunk_addr = (unsigned long) startsg->address;

				continue;
			} else
#endif
			{
				break;
			}
		}

		/*
		** End of DMA Stream
		** Terminate chunk.
		** Allocate space for DMA stream.
		*/
		sg_dma_len(startsg) = chunk_len;
		dma_len = (dma_len + dma_offset + ~IOVP_MASK) & IOVP_MASK;
		sg_dma_address(dma_sg) =
			PIDE_FLAG 
			| (sba_alloc_range(ioc, dma_len) << IOVP_SHIFT)
			| dma_offset;
		n_mappings++;
	}

	return n_mappings;
}


/*
** And this algorithm still generally only ends up coalescing entries
** that happens to be on the same page due to how sglists are assembled.
*/
static int
sba_map_sg(struct pci_dev *dev, struct scatterlist *sglist, int nents, int direction)
{
	struct ioc *ioc = &sba_list->ioc[0];  /* FIXME : see Multi-IOC below */
	int coalesced, filled = 0;
	unsigned long flags;

	DBG_RUN_SG("%s() START %d entries\n", __FUNCTION__, nents);

	/* Fast path single entry scatterlists. */
	if (nents == 1) {
		sg_dma_address(sglist)= sba_map_single(dev, sglist->address,
						sglist->length, direction);
		sg_dma_len(sglist)= sglist->length;
		return 1;
	}

	spin_lock_irqsave(&ioc->res_lock, flags);

#ifdef ASSERT_PDIR_SANITY
	if (sba_check_pdir(ioc,"Check before sba_map_sg()"))
	{
		sba_dump_sg(ioc, sglist, nents);
		panic("Check before sba_map_sg()");
	}
#endif

#ifdef CONFIG_PROC_FS
	ioc->msg_calls++;
#endif

	/*
	** First coalesce the chunks and allocate I/O pdir space
	**
	** If this is one DMA stream, we can properly map using the
	** correct virtual address associated with each DMA page.
	** w/o this association, we wouldn't have coherent DMA!
	** Access to the virtual address is what forces a two pass algorithm.
	*/
	coalesced = sba_coalesce_chunks(ioc, sglist, nents);

	/*
	** Program the I/O Pdir
	**
	** map the virtual addresses to the I/O Pdir
	** o dma_address will contain the pdir index
	** o dma_len will contain the number of bytes to map 
	** o address contains the virtual address.
	*/
	filled = sba_fill_pdir(ioc, sglist, nents);

#ifdef ASSERT_PDIR_SANITY
	if (sba_check_pdir(ioc,"Check after sba_map_sg()"))
	{
		sba_dump_sg(ioc, sglist, nents);
		panic("Check after sba_map_sg()\n");
	}
#endif

	spin_unlock_irqrestore(&ioc->res_lock, flags);

	ASSERT(coalesced == filled);
	DBG_RUN_SG("%s() DONE %d mappings\n", __FUNCTION__, filled);

	return filled;
}


static void 
sba_unmap_sg(struct pci_dev *dev, struct scatterlist *sglist, int nents, int direction)
{
	struct ioc *ioc = &sba_list->ioc[0];  /* FIXME : see Multi-IOC below */
#ifdef ASSERT_PDIR_SANITY
	unsigned long flags;
#endif

	DBG_RUN_SG("%s() START %d entries,  %p,%x\n",
		__FUNCTION__, nents, sglist->address, sglist->length);

#ifdef CONFIG_PROC_FS
	ioc->usg_calls++;
#endif

#ifdef ASSERT_PDIR_SANITY
	spin_lock_irqsave(&ioc->res_lock, flags);
	sba_check_pdir(ioc,"Check before sba_unmap_sg()");
	spin_unlock_irqrestore(&ioc->res_lock, flags);
#endif

	while (sg_dma_len(sglist) && nents--) {

#ifdef CONFIG_PROC_FS
	ioc->usg_pages += sg_dma_len(sglist) >> PAGE_SHIFT;
#endif
		sba_unmap_single(dev, sg_dma_address(sglist), sg_dma_len(sglist), direction);
		++sglist;
	}

	DBG_RUN_SG("%s() DONE (nents %d)\n", __FUNCTION__,  nents);

#ifdef ASSERT_PDIR_SANITY
	spin_lock_irqsave(&ioc->res_lock, flags);
	sba_check_pdir(ioc,"Check after sba_unmap_sg()");
	spin_unlock_irqrestore(&ioc->res_lock, flags);
#endif

}

static struct pci_dma_ops sba_ops = {
	sba_dma_supported,
	sba_alloc_consistent,	/* allocate cacheable host mem */