mthca_provider.c

linux内核源码

	srq = kmalloc(sizeof *srq, GFP_KERNEL);
	if (!srq)
		return ERR_PTR(-ENOMEM);

	if (pd->uobject) {
		context = to_mucontext(pd->uobject->context);

		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
			err = -EFAULT;
			goto err_free;
		}

		err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
					context->db_tab, ucmd.db_index,
					ucmd.db_page);

		if (err)
			goto err_free;

		srq->mr.ibmr.lkey = ucmd.lkey;
		srq->db_index     = ucmd.db_index;
	}

	err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd),
			      &init_attr->attr, srq);

	if (err && pd->uobject)
		mthca_unmap_user_db(to_mdev(pd->device), &context->uar,
				    context->db_tab, ucmd.db_index);

	if (err)
		goto err_free;

	if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof (__u32))) {
		mthca_free_srq(to_mdev(pd->device), srq);
		err = -EFAULT;
		goto err_free;
	}

	return &srq->ibsrq;

err_free:
	kfree(srq);

	return ERR_PTR(err);
}

static int mthca_destroy_srq(struct ib_srq *srq)
{
	struct mthca_ucontext *context;

	if (srq->uobject) {
		context = to_mucontext(srq->uobject->context);

		mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
				    context->db_tab, to_msrq(srq)->db_index);
	}

	mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
	kfree(srq);

	return 0;
}

static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
				     struct ib_qp_init_attr *init_attr,
				     struct ib_udata *udata)
{
	struct mthca_create_qp ucmd;
	struct mthca_qp *qp;
	int err;

	switch (init_attr->qp_type) {
	case IB_QPT_RC:
	case IB_QPT_UC:
	case IB_QPT_UD:
	{
		struct mthca_ucontext *context;

		qp = kmalloc(sizeof *qp, GFP_KERNEL);
		if (!qp)
			return ERR_PTR(-ENOMEM);

		if (pd->uobject) {
			context = to_mucontext(pd->uobject->context);

			if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
				kfree(qp);
				return ERR_PTR(-EFAULT);
			}

			err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
						context->db_tab,
						ucmd.sq_db_index, ucmd.sq_db_page);
			if (err) {
				kfree(qp);
				return ERR_PTR(err);
			}

			err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
						context->db_tab,
						ucmd.rq_db_index, ucmd.rq_db_page);
			if (err) {
				mthca_unmap_user_db(to_mdev(pd->device),
						    &context->uar,
						    context->db_tab,
						    ucmd.sq_db_index);
				kfree(qp);
				return ERR_PTR(err);
			}

			qp->mr.ibmr.lkey = ucmd.lkey;
			qp->sq.db_index  = ucmd.sq_db_index;
			qp->rq.db_index  = ucmd.rq_db_index;
		}

		err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
				     to_mcq(init_attr->send_cq),
				     to_mcq(init_attr->recv_cq),
				     init_attr->qp_type, init_attr->sq_sig_type,
				     &init_attr->cap, qp);

		if (err && pd->uobject) {
			context = to_mucontext(pd->uobject->context);

			mthca_unmap_user_db(to_mdev(pd->device),
					    &context->uar,
					    context->db_tab,
					    ucmd.sq_db_index);
			mthca_unmap_user_db(to_mdev(pd->device),
					    &context->uar,
					    context->db_tab,
					    ucmd.rq_db_index);
		}

		qp->ibqp.qp_num = qp->qpn;
		break;
	}
	case IB_QPT_SMI:
	case IB_QPT_GSI:
	{
		/* Don't allow userspace to create special QPs */
		if (pd->uobject)
			return ERR_PTR(-EINVAL);

		qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
		if (!qp)
			return ERR_PTR(-ENOMEM);

		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;

		err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
				      to_mcq(init_attr->send_cq),
				      to_mcq(init_attr->recv_cq),
				      init_attr->sq_sig_type, &init_attr->cap,
				      qp->ibqp.qp_num, init_attr->port_num,
				      to_msqp(qp));
		break;
	}
	default:
		/* Don't support raw QPs */
		return ERR_PTR(-ENOSYS);
	}

	if (err) {
		kfree(qp);
		return ERR_PTR(err);
	}

	init_attr->cap.max_send_wr     = qp->sq.max;
	init_attr->cap.max_recv_wr     = qp->rq.max;
	init_attr->cap.max_send_sge    = qp->sq.max_gs;
	init_attr->cap.max_recv_sge    = qp->rq.max_gs;
	init_attr->cap.max_inline_data = qp->max_inline_data;

	return &qp->ibqp;
}

static int mthca_destroy_qp(struct ib_qp *qp)
{
	if (qp->uobject) {
		mthca_unmap_user_db(to_mdev(qp->device),
				    &to_mucontext(qp->uobject->context)->uar,
				    to_mucontext(qp->uobject->context)->db_tab,
				    to_mqp(qp)->sq.db_index);
		mthca_unmap_user_db(to_mdev(qp->device),
				    &to_mucontext(qp->uobject->context)->uar,
				    to_mucontext(qp->uobject->context)->db_tab,
				    to_mqp(qp)->rq.db_index);
	}
	mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
	kfree(qp);
	return 0;
}

static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries,
				     int comp_vector,
				     struct ib_ucontext *context,
				     struct ib_udata *udata)
{
	struct mthca_create_cq ucmd;
	struct mthca_cq *cq;
	int nent;
	int err;

	if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
		return ERR_PTR(-EINVAL);

	if (context) {
		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
			return ERR_PTR(-EFAULT);

		err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
					to_mucontext(context)->db_tab,
					ucmd.set_db_index, ucmd.set_db_page);
		if (err)
			return ERR_PTR(err);

		err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
					to_mucontext(context)->db_tab,
					ucmd.arm_db_index, ucmd.arm_db_page);
		if (err)
			goto err_unmap_set;
	}

	cq = kmalloc(sizeof *cq, GFP_KERNEL);
	if (!cq) {
		err = -ENOMEM;
		goto err_unmap_arm;
	}

	if (context) {
		cq->buf.mr.ibmr.lkey = ucmd.lkey;
		cq->set_ci_db_index  = ucmd.set_db_index;
		cq->arm_db_index     = ucmd.arm_db_index;
	}

	for (nent = 1; nent <= entries; nent <<= 1)
		; /* nothing */

	err = mthca_init_cq(to_mdev(ibdev), nent,
			    context ? to_mucontext(context) : NULL,
			    context ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
			    cq);
	if (err)
		goto err_free;

	if (context && ib_copy_to_udata(udata, &cq->cqn, sizeof (__u32))) {
		mthca_free_cq(to_mdev(ibdev), cq);
		goto err_free;
	}

	cq->resize_buf = NULL;

	return &cq->ibcq;

err_free:
	kfree(cq);

err_unmap_arm:
	if (context)
		mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
				    to_mucontext(context)->db_tab, ucmd.arm_db_index);

err_unmap_set:
	if (context)
		mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
				    to_mucontext(context)->db_tab, ucmd.set_db_index);

	return ERR_PTR(err);
}

static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq,
				  int entries)
{
	int ret;

	spin_lock_irq(&cq->lock);
	if (cq->resize_buf) {
		ret = -EBUSY;
		goto unlock;
	}

	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC);
	if (!cq->resize_buf) {
		ret = -ENOMEM;
		goto unlock;
	}

	cq->resize_buf->state = CQ_RESIZE_ALLOC;

	ret = 0;

unlock:
	spin_unlock_irq(&cq->lock);

	if (ret)
		return ret;

	ret = mthca_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);
	if (ret) {
		spin_lock_irq(&cq->lock);
		kfree(cq->resize_buf);
		cq->resize_buf = NULL;
		spin_unlock_irq(&cq->lock);
		return ret;
	}

	cq->resize_buf->cqe = entries - 1;

	spin_lock_irq(&cq->lock);
	cq->resize_buf->state = CQ_RESIZE_READY;
	spin_unlock_irq(&cq->lock);

	return 0;
}

static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
{
	struct mthca_dev *dev = to_mdev(ibcq->device);
	struct mthca_cq *cq = to_mcq(ibcq);
	struct mthca_resize_cq ucmd;
	u32 lkey;
	u8 status;
	int ret;

	if (entries < 1 || entries > dev->limits.max_cqes)
		return -EINVAL;

	mutex_lock(&cq->mutex);

	entries = roundup_pow_of_two(entries + 1);
	if (entries == ibcq->cqe + 1) {
		ret = 0;
		goto out;
	}

	if (cq->is_kernel) {
		ret = mthca_alloc_resize_buf(dev, cq, entries);
		if (ret)
			goto out;
		lkey = cq->resize_buf->buf.mr.ibmr.lkey;
	} else {
		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
			ret = -EFAULT;
			goto out;
		}
		lkey = ucmd.lkey;
	}

	ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, ilog2(entries), &status);
	if (status)
		ret = -EINVAL;

	if (ret) {
		if (cq->resize_buf) {
			mthca_free_cq_buf(dev, &cq->resize_buf->buf,
					  cq->resize_buf->cqe);
			kfree(cq->resize_buf);
			spin_lock_irq(&cq->lock);
			cq->resize_buf = NULL;
			spin_unlock_irq(&cq->lock);
		}
		goto out;
	}

	if (cq->is_kernel) {
		struct mthca_cq_buf tbuf;
		int tcqe;

		spin_lock_irq(&cq->lock);
		if (cq->resize_buf->state == CQ_RESIZE_READY) {
			mthca_cq_resize_copy_cqes(cq);
			tbuf         = cq->buf;
			tcqe         = cq->ibcq.cqe;
			cq->buf      = cq->resize_buf->buf;
			cq->ibcq.cqe = cq->resize_buf->cqe;
		} else {
			tbuf = cq->resize_buf->buf;
			tcqe = cq->resize_buf->cqe;
		}

		kfree(cq->resize_buf);
		cq->resize_buf = NULL;
		spin_unlock_irq(&cq->lock);

		mthca_free_cq_buf(dev, &tbuf, tcqe);
	} else
		ibcq->cqe = entries - 1;

out:
	mutex_unlock(&cq->mutex);

	return ret;
}

static int mthca_destroy_cq(struct ib_cq *cq)
{
	if (cq->uobject) {
		mthca_unmap_user_db(to_mdev(cq->device),
				    &to_mucontext(cq->uobject->context)->uar,
				    to_mucontext(cq->uobject->context)->db_tab,
				    to_mcq(cq)->arm_db_index);
		mthca_unmap_user_db(to_mdev(cq->device),
				    &to_mucontext(cq->uobject->context)->uar,
				    to_mucontext(cq->uobject->context)->db_tab,
				    to_mcq(cq)->set_ci_db_index);
	}
	mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
	kfree(cq);

	return 0;
}

static inline u32 convert_access(int acc)
{
	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC       : 0) |
	       (acc & IB_ACCESS_REMOTE_WRITE  ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) |
	       (acc & IB_ACCESS_REMOTE_READ   ? MTHCA_MPT_FLAG_REMOTE_READ  : 0) |
	       (acc & IB_ACCESS_LOCAL_WRITE   ? MTHCA_MPT_FLAG_LOCAL_WRITE  : 0) |
	       MTHCA_MPT_FLAG_LOCAL_READ;
}

static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc)
{
	struct mthca_mr *mr;
	int err;

	mr = kmalloc(sizeof *mr, GFP_KERNEL);
	if (!mr)
		return ERR_PTR(-ENOMEM);

	err = mthca_mr_alloc_notrans(to_mdev(pd->device),
				     to_mpd(pd)->pd_num,
				     convert_access(acc), mr);

	if (err) {
		kfree(mr);
		return ERR_PTR(err);
	}

	mr->umem = NULL;

	return &mr->ibmr;
}

static struct ib_mr *mthca_reg_phys_mr(struct ib_pd       *pd,
				       struct ib_phys_buf *buffer_list,
				       int                 num_phys_buf,
				       int                 acc,
				       u64                *iova_start)
{
	struct mthca_mr *mr;
	u64 *page_list;
	u64 total_size;
	u64 mask;
	int shift;
	int npages;
	int err;
	int i, j, n;

	/* First check that we have enough alignment */
	if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK))
		return ERR_PTR(-EINVAL);

	mask = 0;
	total_size = 0;
	for (i = 0; i < num_phys_buf; ++i) {
		if (i != 0)
			mask |= buffer_list[i].addr;