mm.c

linux 内核源代码

	unsigned long aligned_len = _ALIGN_UP(len+bus_addr-aligned_bus,
					      1 << r->page_size);

	list_for_each_entry(c, &r->chunk_list.head, link) {
		/* intersection */
		if (aligned_bus >= c->bus_addr &&
		    aligned_bus + aligned_len <= c->bus_addr + c->len)
			return c;

		/* below */
		if (aligned_bus + aligned_len <= c->bus_addr)
			continue;

		/* above */
		if (aligned_bus >= c->bus_addr + c->len)
			continue;

		/* we don't handle the multi-chunk case for now */
		dma_dump_chunk(c);
		BUG();
	}
	return NULL;
}

static struct dma_chunk *dma_find_chunk_lpar(struct ps3_dma_region *r,
	unsigned long lpar_addr, unsigned long len)
{
	struct dma_chunk *c;
	unsigned long aligned_lpar = _ALIGN_DOWN(lpar_addr, 1 << r->page_size);
	unsigned long aligned_len = _ALIGN_UP(len + lpar_addr - aligned_lpar,
					      1 << r->page_size);

	list_for_each_entry(c, &r->chunk_list.head, link) {
		/* intersection */
		if (c->lpar_addr <= aligned_lpar &&
		    aligned_lpar < c->lpar_addr + c->len) {
			if (aligned_lpar + aligned_len <= c->lpar_addr + c->len)
				return c;
			else {
				dma_dump_chunk(c);
				BUG();
			}
		}
		/* below */
		if (aligned_lpar + aligned_len <= c->lpar_addr) {
			continue;
		}
		/* above */
		if (c->lpar_addr + c->len <= aligned_lpar) {
			continue;
		}
	}
	return NULL;
}

static int dma_sb_free_chunk(struct dma_chunk *c)
{
	int result = 0;

	if (c->bus_addr) {
		result = lv1_unmap_device_dma_region(c->region->dev->bus_id,
			c->region->dev->dev_id, c->bus_addr, c->len);
		BUG_ON(result);
	}

	kfree(c);
	return result;
}

static int dma_ioc0_free_chunk(struct dma_chunk *c)
{
	int result = 0;
	int iopage;
	unsigned long offset;
	struct ps3_dma_region *r = c->region;

	DBG("%s:start\n", __func__);
	for (iopage = 0; iopage < (c->len >> r->page_size); iopage++) {
		offset = (1 << r->page_size) * iopage;
		/* put INVALID entry */
		result = lv1_put_iopte(0,
				       c->bus_addr + offset,
				       c->lpar_addr + offset,
				       r->ioid,
				       0);
		DBG("%s: bus=%#lx, lpar=%#lx, ioid=%d\n", __func__,
		    c->bus_addr + offset,
		    c->lpar_addr + offset,
		    r->ioid);

		if (result) {
			DBG("%s:%d: lv1_put_iopte failed: %s\n", __func__,
			    __LINE__, ps3_result(result));
		}
	}
	kfree(c);
	DBG("%s:end\n", __func__);
	return result;
}

/**
 * dma_sb_map_pages - Maps dma pages into the io controller bus address space.
 * @r: Pointer to a struct ps3_dma_region.
 * @phys_addr: Starting physical address of the area to map.
 * @len: Length in bytes of the area to map.
 * c_out: A pointer to receive an allocated struct dma_chunk for this area.
 *
 * This is the lowest level dma mapping routine, and is the one that will
 * make the HV call to add the pages into the io controller address space.
 */

static int dma_sb_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,
	    unsigned long len, struct dma_chunk **c_out, u64 iopte_flag)
{
	int result;
	struct dma_chunk *c;

	c = kzalloc(sizeof(struct dma_chunk), GFP_ATOMIC);

	if (!c) {
		result = -ENOMEM;
		goto fail_alloc;
	}

	c->region = r;
	c->lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
	c->bus_addr = dma_sb_lpar_to_bus(r, c->lpar_addr);
	c->len = len;

	BUG_ON(iopte_flag != 0xf800000000000000UL);
	result = lv1_map_device_dma_region(c->region->dev->bus_id,
					   c->region->dev->dev_id, c->lpar_addr,
					   c->bus_addr, c->len, iopte_flag);
	if (result) {
		DBG("%s:%d: lv1_map_device_dma_region failed: %s\n",
			__func__, __LINE__, ps3_result(result));
		goto fail_map;
	}

	list_add(&c->link, &r->chunk_list.head);

	*c_out = c;
	return 0;

fail_map:
	kfree(c);
fail_alloc:
	*c_out = NULL;
	DBG(" <- %s:%d\n", __func__, __LINE__);
	return result;
}

static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,
			      unsigned long len, struct dma_chunk **c_out,
			      u64 iopte_flag)
{
	int result;
	struct dma_chunk *c, *last;
	int iopage, pages;
	unsigned long offset;

	DBG(KERN_ERR "%s: phy=%#lx, lpar%#lx, len=%#lx\n", __func__,
	    phys_addr, ps3_mm_phys_to_lpar(phys_addr), len);
	c = kzalloc(sizeof(struct dma_chunk), GFP_ATOMIC);

	if (!c) {
		result = -ENOMEM;
		goto fail_alloc;
	}

	c->region = r;
	c->len = len;
	c->lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
	/* allocate IO address */
	if (list_empty(&r->chunk_list.head)) {
		/* first one */
		c->bus_addr = r->bus_addr;
	} else {
		/* derive from last bus addr*/
		last  = list_entry(r->chunk_list.head.next,
				   struct dma_chunk, link);
		c->bus_addr = last->bus_addr + last->len;
		DBG("%s: last bus=%#lx, len=%#lx\n", __func__,
		    last->bus_addr, last->len);
	}

	/* FIXME: check whether length exceeds region size */

	/* build ioptes for the area */
	pages = len >> r->page_size;
	DBG("%s: pgsize=%#x len=%#lx pages=%#x iopteflag=%#lx\n", __func__,
	    r->page_size, r->len, pages, iopte_flag);
	for (iopage = 0; iopage < pages; iopage++) {
		offset = (1 << r->page_size) * iopage;
		result = lv1_put_iopte(0,
				       c->bus_addr + offset,
				       c->lpar_addr + offset,
				       r->ioid,
				       iopte_flag);
		if (result) {
			printk(KERN_WARNING "%s:%d: lv1_map_device_dma_region "
				"failed: %s\n", __func__, __LINE__,
				ps3_result(result));
			goto fail_map;
		}
		DBG("%s: pg=%d bus=%#lx, lpar=%#lx, ioid=%#x\n", __func__,
		    iopage, c->bus_addr + offset, c->lpar_addr + offset,
		    r->ioid);
	}

	/* be sure that last allocated one is inserted at head */
	list_add(&c->link, &r->chunk_list.head);

	*c_out = c;
	DBG("%s: end\n", __func__);
	return 0;

fail_map:
	for (iopage--; 0 <= iopage; iopage--) {
		lv1_put_iopte(0,
			      c->bus_addr + offset,
			      c->lpar_addr + offset,
			      r->ioid,
			      0);
	}
	kfree(c);
fail_alloc:
	*c_out = NULL;
	return result;
}

/**
 * dma_sb_region_create - Create a device dma region.
 * @r: Pointer to a struct ps3_dma_region.
 *
 * This is the lowest level dma region create routine, and is the one that
 * will make the HV call to create the region.
 */

static int dma_sb_region_create(struct ps3_dma_region *r)
{
	int result;

	pr_info(" -> %s:%d:\n", __func__, __LINE__);

	BUG_ON(!r);

	if (!r->dev->bus_id) {
		pr_info("%s:%d: %u:%u no dma\n", __func__, __LINE__,
			r->dev->bus_id, r->dev->dev_id);
		return 0;
	}

	DBG("%s:%u: len = 0x%lx, page_size = %u, offset = 0x%lx\n", __func__,
	    __LINE__, r->len, r->page_size, r->offset);

	BUG_ON(!r->len);
	BUG_ON(!r->page_size);
	BUG_ON(!r->region_ops);

	INIT_LIST_HEAD(&r->chunk_list.head);
	spin_lock_init(&r->chunk_list.lock);

	result = lv1_allocate_device_dma_region(r->dev->bus_id, r->dev->dev_id,
		roundup_pow_of_two(r->len), r->page_size, r->region_type,
		&r->bus_addr);

	if (result) {
		DBG("%s:%d: lv1_allocate_device_dma_region failed: %s\n",
			__func__, __LINE__, ps3_result(result));
		r->len = r->bus_addr = 0;
	}

	return result;
}

static int dma_ioc0_region_create(struct ps3_dma_region *r)
{
	int result;

	INIT_LIST_HEAD(&r->chunk_list.head);
	spin_lock_init(&r->chunk_list.lock);

	result = lv1_allocate_io_segment(0,
					 r->len,
					 r->page_size,
					 &r->bus_addr);
	if (result) {
		DBG("%s:%d: lv1_allocate_io_segment failed: %s\n",
			__func__, __LINE__, ps3_result(result));
		r->len = r->bus_addr = 0;
	}
	DBG("%s: len=%#lx, pg=%d, bus=%#lx\n", __func__,
	    r->len, r->page_size, r->bus_addr);
	return result;
}

/**
 * dma_region_free - Free a device dma region.
 * @r: Pointer to a struct ps3_dma_region.
 *
 * This is the lowest level dma region free routine, and is the one that
 * will make the HV call to free the region.
 */

static int dma_sb_region_free(struct ps3_dma_region *r)
{
	int result;
	struct dma_chunk *c;
	struct dma_chunk *tmp;

	BUG_ON(!r);

	if (!r->dev->bus_id) {
		pr_info("%s:%d: %u:%u no dma\n", __func__, __LINE__,
			r->dev->bus_id, r->dev->dev_id);
		return 0;
	}

	list_for_each_entry_safe(c, tmp, &r->chunk_list.head, link) {
		list_del(&c->link);
		dma_sb_free_chunk(c);
	}

	result = lv1_free_device_dma_region(r->dev->bus_id, r->dev->dev_id,
		r->bus_addr);

	if (result)
		DBG("%s:%d: lv1_free_device_dma_region failed: %s\n",
			__func__, __LINE__, ps3_result(result));

	r->bus_addr = 0;

	return result;
}

static int dma_ioc0_region_free(struct ps3_dma_region *r)
{
	int result;
	struct dma_chunk *c, *n;

	DBG("%s: start\n", __func__);
	list_for_each_entry_safe(c, n, &r->chunk_list.head, link) {
		list_del(&c->link);
		dma_ioc0_free_chunk(c);
	}

	result = lv1_release_io_segment(0, r->bus_addr);

	if (result)
		DBG("%s:%d: lv1_free_device_dma_region failed: %s\n",
			__func__, __LINE__, ps3_result(result));

	r->bus_addr = 0;
	DBG("%s: end\n", __func__);

	return result;
}

/**
 * dma_sb_map_area - Map an area of memory into a device dma region.
 * @r: Pointer to a struct ps3_dma_region.
 * @virt_addr: Starting virtual address of the area to map.
 * @len: Length in bytes of the area to map.
 * @bus_addr: A pointer to return the starting ioc bus address of the area to
 * map.
 *
 * This is the common dma mapping routine.
 */

static int dma_sb_map_area(struct ps3_dma_region *r, unsigned long virt_addr,
	   unsigned long len, unsigned long *bus_addr,
	   u64 iopte_flag)
{
	int result;
	unsigned long flags;
	struct dma_chunk *c;
	unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr)
		: virt_addr;
	unsigned long aligned_phys = _ALIGN_DOWN(phys_addr, 1 << r->page_size);
	unsigned long aligned_len = _ALIGN_UP(len + phys_addr - aligned_phys,
					      1 << r->page_size);
	*bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr));

	if (!USE_DYNAMIC_DMA) {
		unsigned long lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
		DBG(" -> %s:%d\n", __func__, __LINE__);
		DBG("%s:%d virt_addr %lxh\n", __func__, __LINE__,
			virt_addr);
		DBG("%s:%d phys_addr %lxh\n", __func__, __LINE__,
			phys_addr);
		DBG("%s:%d lpar_addr %lxh\n", __func__, __LINE__,
			lpar_addr);
		DBG("%s:%d len       %lxh\n", __func__, __LINE__, len);
		DBG("%s:%d bus_addr  %lxh (%lxh)\n", __func__, __LINE__,
		*bus_addr, len);
	}

	spin_lock_irqsave(&r->chunk_list.lock, flags);
	c = dma_find_chunk(r, *bus_addr, len);

	if (c) {
		DBG("%s:%d: reusing mapped chunk", __func__, __LINE__);
		dma_dump_chunk(c);
		c->usage_count++;
		spin_unlock_irqrestore(&r->chunk_list.lock, flags);
		return 0;
	}

	result = dma_sb_map_pages(r, aligned_phys, aligned_len, &c, iopte_flag);

	if (result) {