mthca_provider.c

来自「底层驱动开发」· C语言 代码 · 共 1,145 行 · 第 1/2 页

		/* Don't allow userspace to create special QPs */
		if (pd->uobject)
			return ERR_PTR(-EINVAL);

		qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
		if (!qp)
			return ERR_PTR(-ENOMEM);

		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;

		err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
				      to_mcq(init_attr->send_cq),
				      to_mcq(init_attr->recv_cq),
				      init_attr->sq_sig_type, &init_attr->cap,
				      qp->ibqp.qp_num, init_attr->port_num,
				      to_msqp(qp));
		break;
	}
	default:
		/* Don't support raw QPs */
		return ERR_PTR(-ENOSYS);
	}

	if (err) {
		kfree(qp);
		return ERR_PTR(err);
	}

	init_attr->cap.max_inline_data = 0;
	init_attr->cap.max_send_wr     = qp->sq.max;
	init_attr->cap.max_recv_wr     = qp->rq.max;
	init_attr->cap.max_send_sge    = qp->sq.max_gs;
	init_attr->cap.max_recv_sge    = qp->rq.max_gs;

	return &qp->ibqp;
}

static int mthca_destroy_qp(struct ib_qp *qp)
{
	if (qp->uobject) {
		mthca_unmap_user_db(to_mdev(qp->device),
				    &to_mucontext(qp->uobject->context)->uar,
				    to_mucontext(qp->uobject->context)->db_tab,
				    to_mqp(qp)->sq.db_index);
		mthca_unmap_user_db(to_mdev(qp->device),
				    &to_mucontext(qp->uobject->context)->uar,
				    to_mucontext(qp->uobject->context)->db_tab,
				    to_mqp(qp)->rq.db_index);
	}
	mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
	kfree(qp);
	return 0;
}

static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries,
				     struct ib_ucontext *context,
				     struct ib_udata *udata)
{
	struct mthca_create_cq ucmd;
	struct mthca_cq *cq;
	int nent;
	int err;

	if (context) {
		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
			return ERR_PTR(-EFAULT);

		err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
					to_mucontext(context)->db_tab,
					ucmd.set_db_index, ucmd.set_db_page);
		if (err)
			return ERR_PTR(err);

		err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
					to_mucontext(context)->db_tab,
					ucmd.arm_db_index, ucmd.arm_db_page);
		if (err)
			goto err_unmap_set;
	}

	cq = kmalloc(sizeof *cq, GFP_KERNEL);
	if (!cq) {
		err = -ENOMEM;
		goto err_unmap_arm;
	}

	if (context) {
		cq->mr.ibmr.lkey    = ucmd.lkey;
		cq->set_ci_db_index = ucmd.set_db_index;
		cq->arm_db_index    = ucmd.arm_db_index;
	}

	for (nent = 1; nent <= entries; nent <<= 1)
		; /* nothing */

	err = mthca_init_cq(to_mdev(ibdev), nent,
			    context ? to_mucontext(context) : NULL,
			    context ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
			    cq);
	if (err)
		goto err_free;

	if (context && ib_copy_to_udata(udata, &cq->cqn, sizeof (__u32))) {
		mthca_free_cq(to_mdev(ibdev), cq);
		goto err_free;
	}

	return &cq->ibcq;

err_free:
	kfree(cq);

err_unmap_arm:
	if (context)
		mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
				    to_mucontext(context)->db_tab, ucmd.arm_db_index);

err_unmap_set:
	if (context)
		mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
				    to_mucontext(context)->db_tab, ucmd.set_db_index);

	return ERR_PTR(err);
}

static int mthca_destroy_cq(struct ib_cq *cq)
{
	if (cq->uobject) {
		mthca_unmap_user_db(to_mdev(cq->device),
				    &to_mucontext(cq->uobject->context)->uar,
				    to_mucontext(cq->uobject->context)->db_tab,
				    to_mcq(cq)->arm_db_index);
		mthca_unmap_user_db(to_mdev(cq->device),
				    &to_mucontext(cq->uobject->context)->uar,
				    to_mucontext(cq->uobject->context)->db_tab,
				    to_mcq(cq)->set_ci_db_index);
	}
	mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
	kfree(cq);

	return 0;
}

static inline u32 convert_access(int acc)
{
	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC       : 0) |
	       (acc & IB_ACCESS_REMOTE_WRITE  ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) |
	       (acc & IB_ACCESS_REMOTE_READ   ? MTHCA_MPT_FLAG_REMOTE_READ  : 0) |
	       (acc & IB_ACCESS_LOCAL_WRITE   ? MTHCA_MPT_FLAG_LOCAL_WRITE  : 0) |
	       MTHCA_MPT_FLAG_LOCAL_READ;
}

static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc)
{
	struct mthca_mr *mr;
	int err;

	mr = kmalloc(sizeof *mr, GFP_KERNEL);
	if (!mr)
		return ERR_PTR(-ENOMEM);

	err = mthca_mr_alloc_notrans(to_mdev(pd->device),
				     to_mpd(pd)->pd_num,
				     convert_access(acc), mr);

	if (err) {
		kfree(mr);
		return ERR_PTR(err);
	}

	return &mr->ibmr;
}

static struct ib_mr *mthca_reg_phys_mr(struct ib_pd       *pd,
				       struct ib_phys_buf *buffer_list,
				       int                 num_phys_buf,
				       int                 acc,
				       u64                *iova_start)
{
	struct mthca_mr *mr;
	u64 *page_list;
	u64 total_size;
	u64 mask;
	int shift;
	int npages;
	int err;
	int i, j, n;

	/* First check that we have enough alignment */
	if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK))
		return ERR_PTR(-EINVAL);

	if (num_phys_buf > 1 &&
	    ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK))
		return ERR_PTR(-EINVAL);

	mask = 0;
	total_size = 0;
	for (i = 0; i < num_phys_buf; ++i) {
		if (i != 0 && buffer_list[i].addr & ~PAGE_MASK)
			return ERR_PTR(-EINVAL);
		if (i != 0 && i != num_phys_buf - 1 &&
		    (buffer_list[i].size & ~PAGE_MASK))
			return ERR_PTR(-EINVAL);

		total_size += buffer_list[i].size;
		if (i > 0)
			mask |= buffer_list[i].addr;
	}

	/* Find largest page shift we can use to cover buffers */
	for (shift = PAGE_SHIFT; shift < 31; ++shift)
		if (num_phys_buf > 1) {
			if ((1ULL << shift) & mask)
				break;
		} else {
			if (1ULL << shift >=
			    buffer_list[0].size +
			    (buffer_list[0].addr & ((1ULL << shift) - 1)))
				break;
		}

	buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1);
	buffer_list[0].addr &= ~0ull << shift;

	mr = kmalloc(sizeof *mr, GFP_KERNEL);
	if (!mr)
		return ERR_PTR(-ENOMEM);

	npages = 0;
	for (i = 0; i < num_phys_buf; ++i)
		npages += (buffer_list[i].size + (1ULL << shift) - 1) >> shift;

	if (!npages)
		return &mr->ibmr;

	page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL);
	if (!page_list) {
		kfree(mr);
		return ERR_PTR(-ENOMEM);
	}

	n = 0;
	for (i = 0; i < num_phys_buf; ++i)
		for (j = 0;
		     j < (buffer_list[i].size + (1ULL << shift) - 1) >> shift;
		     ++j)
			page_list[n++] = buffer_list[i].addr + ((u64) j << shift);

	mthca_dbg(to_mdev(pd->device), "Registering memory at %llx (iova %llx) "
		  "in PD %x; shift %d, npages %d.\n",
		  (unsigned long long) buffer_list[0].addr,
		  (unsigned long long) *iova_start,
		  to_mpd(pd)->pd_num,
		  shift, npages);

	err = mthca_mr_alloc_phys(to_mdev(pd->device),
				  to_mpd(pd)->pd_num,
				  page_list, shift, npages,
				  *iova_start, total_size,
				  convert_access(acc), mr);

	if (err) {
		kfree(page_list);
		kfree(mr);
		return ERR_PTR(err);
	}

	kfree(page_list);
	return &mr->ibmr;
}

static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, struct ib_umem *region,
				       int acc, struct ib_udata *udata)
{
	struct mthca_dev *dev = to_mdev(pd->device);
	struct ib_umem_chunk *chunk;
	struct mthca_mr *mr;
	u64 *pages;
	int shift, n, len;
	int i, j, k;
	int err = 0;

	shift = ffs(region->page_size) - 1;

	mr = kmalloc(sizeof *mr, GFP_KERNEL);
	if (!mr)
		return ERR_PTR(-ENOMEM);

	n = 0;
	list_for_each_entry(chunk, &region->chunk_list, list)
		n += chunk->nents;

	mr->mtt = mthca_alloc_mtt(dev, n);
	if (IS_ERR(mr->mtt)) {
		err = PTR_ERR(mr->mtt);
		goto err;
	}

	pages = (u64 *) __get_free_page(GFP_KERNEL);
	if (!pages) {
		err = -ENOMEM;
		goto err_mtt;
	}

	i = n = 0;

	list_for_each_entry(chunk, &region->chunk_list, list)
		for (j = 0; j < chunk->nmap; ++j) {
			len = sg_dma_len(&chunk->page_list[j]) >> shift;
			for (k = 0; k < len; ++k) {
				pages[i++] = sg_dma_address(&chunk->page_list[j]) +
					region->page_size * k;
				/*
				 * Be friendly to WRITE_MTT command
				 * and leave two empty slots for the
				 * index and reserved fields of the
				 * mailbox.
				 */
				if (i == PAGE_SIZE / sizeof (u64) - 2) {
					err = mthca_write_mtt(dev, mr->mtt,
							      n, pages, i);
					if (err)
						goto mtt_done;
					n += i;
					i = 0;
				}
			}
		}

	if (i)
		err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
mtt_done:
	free_page((unsigned long) pages);
	if (err)
		goto err_mtt;

	err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, shift, region->virt_base,
			     region->length, convert_access(acc), mr);

	if (err)
		goto err_mtt;

	return &mr->ibmr;

err_mtt:
	mthca_free_mtt(dev, mr->mtt);

err:
	kfree(mr);
	return ERR_PTR(err);
}

static int mthca_dereg_mr(struct ib_mr *mr)
{
	struct mthca_mr *mmr = to_mmr(mr);
	mthca_free_mr(to_mdev(mr->device), mmr);
	kfree(mmr);
	return 0;
}

static struct ib_fmr *mthca_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
				      struct ib_fmr_attr *fmr_attr)
{
	struct mthca_fmr *fmr;
	int err;

	fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
	if (!fmr)
		return ERR_PTR(-ENOMEM);

	memcpy(&fmr->attr, fmr_attr, sizeof *fmr_attr);
	err = mthca_fmr_alloc(to_mdev(pd->device), to_mpd(pd)->pd_num,
			     convert_access(mr_access_flags), fmr);

	if (err) {
		kfree(fmr);
		return ERR_PTR(err);
	}

	return &fmr->ibmr;
}

static int mthca_dealloc_fmr(struct ib_fmr *fmr)
{
	struct mthca_fmr *mfmr = to_mfmr(fmr);
	int err;

	err = mthca_free_fmr(to_mdev(fmr->device), mfmr);
	if (err)
		return err;

	kfree(mfmr);
	return 0;
}

static int mthca_unmap_fmr(struct list_head *fmr_list)
{
	struct ib_fmr *fmr;
	int err;
	u8 status;
	struct mthca_dev *mdev = NULL;

	list_for_each_entry(fmr, fmr_list, list) {
		if (mdev && to_mdev(fmr->device) != mdev)
			return -EINVAL;
		mdev = to_mdev(fmr->device);
	}

	if (!mdev)
		return 0;

	if (mthca_is_memfree(mdev)) {
		list_for_each_entry(fmr, fmr_list, list)
			mthca_arbel_fmr_unmap(mdev, to_mfmr(fmr));

		wmb();
	} else
		list_for_each_entry(fmr, fmr_list, list)
			mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr));

	err = mthca_SYNC_TPT(mdev, &status);
	if (err)
		return err;
	if (status)
		return -EINVAL;
	return 0;
}

static ssize_t show_rev(struct class_device *cdev, char *buf)
{
	struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev);
	return sprintf(buf, "%x\n", dev->rev_id);
}

static ssize_t show_fw_ver(struct class_device *cdev, char *buf)
{
	struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev);
	return sprintf(buf, "%x.%x.%x\n", (int) (dev->fw_ver >> 32),
		       (int) (dev->fw_ver >> 16) & 0xffff,
		       (int) dev->fw_ver & 0xffff);
}

static ssize_t show_hca(struct class_device *cdev, char *buf)
{
	struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev);
	switch (dev->pdev->device) {
	case PCI_DEVICE_ID_MELLANOX_TAVOR:
		return sprintf(buf, "MT23108\n");
	case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT:
		return sprintf(buf, "MT25208 (MT23108 compat mode)\n");
	case PCI_DEVICE_ID_MELLANOX_ARBEL:
		return sprintf(buf, "MT25208\n");
	case PCI_DEVICE_ID_MELLANOX_SINAI:
	case PCI_DEVICE_ID_MELLANOX_SINAI_OLD:
		return sprintf(buf, "MT25204\n");
	default:
		return sprintf(buf, "unknown\n");
	}
}

static ssize_t show_board(struct class_device *cdev, char *buf)
{
	struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev);
	return sprintf(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id);
}

static CLASS_DEVICE_ATTR(hw_rev,   S_IRUGO, show_rev,    NULL);
static CLASS_DEVICE_ATTR(fw_ver,   S_IRUGO, show_fw_ver, NULL);
static CLASS_DEVICE_ATTR(hca_type, S_IRUGO, show_hca,    NULL);
static CLASS_DEVICE_ATTR(board_id, S_IRUGO, show_board,  NULL);

static struct class_device_attribute *mthca_class_attributes[] = {
	&class_device_attr_hw_rev,
	&class_device_attr_fw_ver,
	&class_device_attr_hca_type,
	&class_device_attr_board_id
};

int mthca_register_device(struct mthca_dev *dev)
{
	int ret;
	int i;

	strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX);
	dev->ib_dev.owner                = THIS_MODULE;

	dev->ib_dev.node_type            = IB_NODE_CA;
	dev->ib_dev.phys_port_cnt        = dev->limits.num_ports;
	dev->ib_dev.dma_device           = &dev->pdev->dev;
	dev->ib_dev.class_dev.dev        = &dev->pdev->dev;
	dev->ib_dev.query_device         = mthca_query_device;
	dev->ib_dev.query_port           = mthca_query_port;
	dev->ib_dev.modify_port          = mthca_modify_port;
	dev->ib_dev.query_pkey           = mthca_query_pkey;
	dev->ib_dev.query_gid            = mthca_query_gid;
	dev->ib_dev.alloc_ucontext       = mthca_alloc_ucontext;
	dev->ib_dev.dealloc_ucontext     = mthca_dealloc_ucontext;
	dev->ib_dev.mmap                 = mthca_mmap_uar;
	dev->ib_dev.alloc_pd             = mthca_alloc_pd;
	dev->ib_dev.dealloc_pd           = mthca_dealloc_pd;
	dev->ib_dev.create_ah            = mthca_ah_create;
	dev->ib_dev.destroy_ah           = mthca_ah_destroy;

	if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
		dev->ib_dev.create_srq           = mthca_create_srq;
		dev->ib_dev.destroy_srq          = mthca_destroy_srq;

		if (mthca_is_memfree(dev))
			dev->ib_dev.post_srq_recv = mthca_arbel_post_srq_recv;
		else
			dev->ib_dev.post_srq_recv = mthca_tavor_post_srq_recv;
	}

	dev->ib_dev.create_qp            = mthca_create_qp;
	dev->ib_dev.modify_qp            = mthca_modify_qp;
	dev->ib_dev.destroy_qp           = mthca_destroy_qp;
	dev->ib_dev.create_cq            = mthca_create_cq;
	dev->ib_dev.destroy_cq           = mthca_destroy_cq;
	dev->ib_dev.poll_cq              = mthca_poll_cq;
	dev->ib_dev.get_dma_mr           = mthca_get_dma_mr;
	dev->ib_dev.reg_phys_mr          = mthca_reg_phys_mr;
	dev->ib_dev.reg_user_mr          = mthca_reg_user_mr;
	dev->ib_dev.dereg_mr             = mthca_dereg_mr;

	if (dev->mthca_flags & MTHCA_FLAG_FMR) {
		dev->ib_dev.alloc_fmr            = mthca_alloc_fmr;
		dev->ib_dev.unmap_fmr            = mthca_unmap_fmr;
		dev->ib_dev.dealloc_fmr          = mthca_dealloc_fmr;
		if (mthca_is_memfree(dev))
			dev->ib_dev.map_phys_fmr = mthca_arbel_map_phys_fmr;
		else
			dev->ib_dev.map_phys_fmr = mthca_tavor_map_phys_fmr;
	}

	dev->ib_dev.attach_mcast         = mthca_multicast_attach;
	dev->ib_dev.detach_mcast         = mthca_multicast_detach;
	dev->ib_dev.process_mad          = mthca_process_mad;

	if (mthca_is_memfree(dev)) {
		dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq;
		dev->ib_dev.post_send     = mthca_arbel_post_send;
		dev->ib_dev.post_recv     = mthca_arbel_post_receive;
	} else {
		dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq;
		dev->ib_dev.post_send     = mthca_tavor_post_send;
		dev->ib_dev.post_recv     = mthca_tavor_post_receive;
	}

	init_MUTEX(&dev->cap_mask_mutex);

	ret = ib_register_device(&dev->ib_dev);
	if (ret)
		return ret;

	for (i = 0; i < ARRAY_SIZE(mthca_class_attributes); ++i) {
		ret = class_device_create_file(&dev->ib_dev.class_dev,
					       mthca_class_attributes[i]);
		if (ret) {
			ib_unregister_device(&dev->ib_dev);
			return ret;
		}
	}

	return 0;
}

void mthca_unregister_device(struct mthca_dev *dev)
{
	ib_unregister_device(&dev->ib_dev);
}