mthca_memfree.c

linux-2.6.15.6

}

void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table)
{
	int i;
	u8 status;

	for (i = 0; i < table->num_icm; ++i)
		if (table->icm[i]) {
			mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE,
					MTHCA_TABLE_CHUNK_SIZE >> 12, &status);
			mthca_free_icm(dev, table->icm[i]);
		}

	kfree(table);
}

static u64 mthca_uarc_virt(struct mthca_dev *dev, struct mthca_uar *uar, int page)
{
	return dev->uar_table.uarc_base +
		uar->index * dev->uar_table.uarc_size +
		page * 4096;
}

int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
		      struct mthca_user_db_table *db_tab, int index, u64 uaddr)
{
	int ret = 0;
	u8 status;
	int i;

	if (!mthca_is_memfree(dev))
		return 0;

	if (index < 0 || index > dev->uar_table.uarc_size / 8)
		return -EINVAL;

	down(&db_tab->mutex);

	i = index / MTHCA_DB_REC_PER_PAGE;

	if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE)       ||
	    (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) ||
	    (uaddr & 4095)) {
		ret = -EINVAL;
		goto out;
	}

	if (db_tab->page[i].refcount) {
		++db_tab->page[i].refcount;
		goto out;
	}

	ret = get_user_pages(current, current->mm, uaddr & PAGE_MASK, 1, 1, 0,
			     &db_tab->page[i].mem.page, NULL);
	if (ret < 0)
		goto out;

	db_tab->page[i].mem.length = 4096;
	db_tab->page[i].mem.offset = uaddr & ~PAGE_MASK;

	ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
	if (ret < 0) {
		put_page(db_tab->page[i].mem.page);
		goto out;
	}

	ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem),
				 mthca_uarc_virt(dev, uar, i), &status);
	if (!ret && status)
		ret = -EINVAL;
	if (ret) {
		pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
		put_page(db_tab->page[i].mem.page);
		goto out;
	}

	db_tab->page[i].uvirt    = uaddr;
	db_tab->page[i].refcount = 1;

out:
	up(&db_tab->mutex);
	return ret;
}

void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
			 struct mthca_user_db_table *db_tab, int index)
{
	if (!mthca_is_memfree(dev))
		return;

	/*
	 * To make our bookkeeping simpler, we don't unmap DB
	 * pages until we clean up the whole db table.
	 */

	down(&db_tab->mutex);

	--db_tab->page[index / MTHCA_DB_REC_PER_PAGE].refcount;

	up(&db_tab->mutex);
}

struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev)
{
	struct mthca_user_db_table *db_tab;
	int npages;
	int i;

	if (!mthca_is_memfree(dev))
		return NULL;

	npages = dev->uar_table.uarc_size / 4096;
	db_tab = kmalloc(sizeof *db_tab + npages * sizeof *db_tab->page, GFP_KERNEL);
	if (!db_tab)
		return ERR_PTR(-ENOMEM);

	init_MUTEX(&db_tab->mutex);
	for (i = 0; i < npages; ++i) {
		db_tab->page[i].refcount = 0;
		db_tab->page[i].uvirt    = 0;
	}

	return db_tab;
}

void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar,
			       struct mthca_user_db_table *db_tab)
{
	int i;
	u8 status;

	if (!mthca_is_memfree(dev))
		return;

	for (i = 0; i < dev->uar_table.uarc_size / 4096; ++i) {
		if (db_tab->page[i].uvirt) {
			mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1, &status);
			pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
			put_page(db_tab->page[i].mem.page);
		}
	}
}

int mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type,
		   u32 qn, __be32 **db)
{
	int group;
	int start, end, dir;
	int i, j;
	struct mthca_db_page *page;
	int ret = 0;
	u8 status;

	down(&dev->db_tab->mutex);

	switch (type) {
	case MTHCA_DB_TYPE_CQ_ARM:
	case MTHCA_DB_TYPE_SQ:
		group = 0;
		start = 0;
		end   = dev->db_tab->max_group1;
		dir   = 1;
		break;

	case MTHCA_DB_TYPE_CQ_SET_CI:
	case MTHCA_DB_TYPE_RQ:
	case MTHCA_DB_TYPE_SRQ:
		group = 1;
		start = dev->db_tab->npages - 1;
		end   = dev->db_tab->min_group2;
		dir   = -1;
		break;

	default:
		ret = -EINVAL;
		goto out;
	}

	for (i = start; i != end; i += dir)
		if (dev->db_tab->page[i].db_rec &&
		    !bitmap_full(dev->db_tab->page[i].used,
				 MTHCA_DB_REC_PER_PAGE)) {
			page = dev->db_tab->page + i;
			goto found;
		}

	for (i = start; i != end; i += dir)
		if (!dev->db_tab->page[i].db_rec) {
			page = dev->db_tab->page + i;
			goto alloc;
		}

	if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) {
		ret = -ENOMEM;
		goto out;
	}

	if (group == 0)
		++dev->db_tab->max_group1;
	else
		--dev->db_tab->min_group2;

	page = dev->db_tab->page + end;

alloc:
	page->db_rec = dma_alloc_coherent(&dev->pdev->dev, 4096,
					  &page->mapping, GFP_KERNEL);
	if (!page->db_rec) {
		ret = -ENOMEM;
		goto out;
	}
	memset(page->db_rec, 0, 4096);

	ret = mthca_MAP_ICM_page(dev, page->mapping,
				 mthca_uarc_virt(dev, &dev->driver_uar, i), &status);
	if (!ret && status)
		ret = -EINVAL;
	if (ret) {
		dma_free_coherent(&dev->pdev->dev, 4096,
				  page->db_rec, page->mapping);
		goto out;
	}

	bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE);

found:
	j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE);
	set_bit(j, page->used);

	if (group == 1)
		j = MTHCA_DB_REC_PER_PAGE - 1 - j;

	ret = i * MTHCA_DB_REC_PER_PAGE + j;

	page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5));

	*db = (__be32 *) &page->db_rec[j];

out:
	up(&dev->db_tab->mutex);

	return ret;
}

void mthca_free_db(struct mthca_dev *dev, int type, int db_index)
{
	int i, j;
	struct mthca_db_page *page;
	u8 status;

	i = db_index / MTHCA_DB_REC_PER_PAGE;
	j = db_index % MTHCA_DB_REC_PER_PAGE;

	page = dev->db_tab->page + i;

	down(&dev->db_tab->mutex);

	page->db_rec[j] = 0;
	if (i >= dev->db_tab->min_group2)
		j = MTHCA_DB_REC_PER_PAGE - 1 - j;
	clear_bit(j, page->used);

	if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) &&
	    i >= dev->db_tab->max_group1 - 1) {
		mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status);

		dma_free_coherent(&dev->pdev->dev, 4096,
				  page->db_rec, page->mapping);
		page->db_rec = NULL;

		if (i == dev->db_tab->max_group1) {
			--dev->db_tab->max_group1;
			/* XXX may be able to unmap more pages now */
		}
		if (i == dev->db_tab->min_group2)
			++dev->db_tab->min_group2;
	}

	up(&dev->db_tab->mutex);
}

int mthca_init_db_tab(struct mthca_dev *dev)
{
	int i;

	if (!mthca_is_memfree(dev))
		return 0;

	dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL);
	if (!dev->db_tab)
		return -ENOMEM;

	init_MUTEX(&dev->db_tab->mutex);

	dev->db_tab->npages     = dev->uar_table.uarc_size / 4096;
	dev->db_tab->max_group1 = 0;
	dev->db_tab->min_group2 = dev->db_tab->npages - 1;

	dev->db_tab->page = kmalloc(dev->db_tab->npages *
				    sizeof *dev->db_tab->page,
				    GFP_KERNEL);
	if (!dev->db_tab->page) {
		kfree(dev->db_tab);
		return -ENOMEM;
	}

	for (i = 0; i < dev->db_tab->npages; ++i)
		dev->db_tab->page[i].db_rec = NULL;

	return 0;
}

void mthca_cleanup_db_tab(struct mthca_dev *dev)
{
	int i;
	u8 status;

	if (!mthca_is_memfree(dev))
		return;

	/*
	 * Because we don't always free our UARC pages when they
	 * become empty to make mthca_free_db() simpler we need to
	 * make a sweep through the doorbell pages and free any
	 * leftover pages now.
	 */
	for (i = 0; i < dev->db_tab->npages; ++i) {
		if (!dev->db_tab->page[i].db_rec)
			continue;

		if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE))
			mthca_warn(dev, "Kernel UARC page %d not empty\n", i);

		mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status);

		dma_free_coherent(&dev->pdev->dev, 4096,
				  dev->db_tab->page[i].db_rec,
				  dev->db_tab->page[i].mapping);
	}

	kfree(dev->db_tab->page);
	kfree(dev->db_tab);
}