cxio_hal.c

linux内核源码

{
	static int cnt;
	struct cxio_rdev *rdev_p = NULL;
	struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) skb->data;
	PDBG("%d: %s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x"
	     " se %0x notify %0x cqbranch %0x creditth %0x\n",
	     cnt, __FUNCTION__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg),
	     RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg),
	     RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg),
	     RSPQ_CREDIT_THRESH(rsp_msg));
	PDBG("CQE: QPID 0x%0x genbit %0x type 0x%0x status 0x%0x opcode %d "
	     "len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",
	     CQE_QPID(rsp_msg->cqe), CQE_GENBIT(rsp_msg->cqe),
	     CQE_TYPE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe),
	     CQE_OPCODE(rsp_msg->cqe), CQE_LEN(rsp_msg->cqe),
	     CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe));
	rdev_p = (struct cxio_rdev *)t3cdev_p->ulp;
	if (!rdev_p) {
		PDBG("%s called by t3cdev %p with null ulp\n", __FUNCTION__,
		     t3cdev_p);
		return 0;
	}
	if (CQE_QPID(rsp_msg->cqe) == T3_CTRL_QP_ID) {
		rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;
		wake_up_interruptible(&rdev_p->ctrl_qp.waitq);
		dev_kfree_skb_irq(skb);
	} else if (CQE_QPID(rsp_msg->cqe) == 0xfff8)
		dev_kfree_skb_irq(skb);
	else if (cxio_ev_cb)
		(*cxio_ev_cb) (rdev_p, skb);
	else
		dev_kfree_skb_irq(skb);
	cnt++;
	return 0;
}

/* Caller takes care of locking if needed */
int cxio_rdev_open(struct cxio_rdev *rdev_p)
{
	struct net_device *netdev_p = NULL;
	int err = 0;
	if (strlen(rdev_p->dev_name)) {
		if (cxio_hal_find_rdev_by_name(rdev_p->dev_name)) {
			return -EBUSY;
		}
		netdev_p = dev_get_by_name(&init_net, rdev_p->dev_name);
		if (!netdev_p) {
			return -EINVAL;
		}
		dev_put(netdev_p);
	} else if (rdev_p->t3cdev_p) {
		if (cxio_hal_find_rdev_by_t3cdev(rdev_p->t3cdev_p)) {
			return -EBUSY;
		}
		netdev_p = rdev_p->t3cdev_p->lldev;
		strncpy(rdev_p->dev_name, rdev_p->t3cdev_p->name,
			T3_MAX_DEV_NAME_LEN);
	} else {
		PDBG("%s t3cdev_p or dev_name must be set\n", __FUNCTION__);
		return -EINVAL;
	}

	list_add_tail(&rdev_p->entry, &rdev_list);

	PDBG("%s opening rnic dev %s\n", __FUNCTION__, rdev_p->dev_name);
	memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));
	if (!rdev_p->t3cdev_p)
		rdev_p->t3cdev_p = dev2t3cdev(netdev_p);
	rdev_p->t3cdev_p->ulp = (void *) rdev_p;
	err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_GET_PARAMS,
					 &(rdev_p->rnic_info));
	if (err) {
		printk(KERN_ERR "%s t3cdev_p(%p)->ctl returned error %d.\n",
		     __FUNCTION__, rdev_p->t3cdev_p, err);
		goto err1;
	}
	err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_PORTS,
				    &(rdev_p->port_info));
	if (err) {
		printk(KERN_ERR "%s t3cdev_p(%p)->ctl returned error %d.\n",
		     __FUNCTION__, rdev_p->t3cdev_p, err);
		goto err1;
	}

	/*
	 * qpshift is the number of bits to shift the qpid left in order
	 * to get the correct address of the doorbell for that qp.
	 */
	cxio_init_ucontext(rdev_p, &rdev_p->uctx);
	rdev_p->qpshift = PAGE_SHIFT -
			  ilog2(65536 >>
			            ilog2(rdev_p->rnic_info.udbell_len >>
					      PAGE_SHIFT));
	rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;
	rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;
	PDBG("%s rnic %s info: tpt_base 0x%0x tpt_top 0x%0x num stags %d "
	     "pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x\n",
	     __FUNCTION__, rdev_p->dev_name, rdev_p->rnic_info.tpt_base,
	     rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p),
	     rdev_p->rnic_info.pbl_base,
	     rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,
	     rdev_p->rnic_info.rqt_top);
	PDBG("udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu "
	     "qpnr %d qpmask 0x%x\n",
	     rdev_p->rnic_info.udbell_len,
	     rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,
	     rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);

	err = cxio_hal_init_ctrl_qp(rdev_p);
	if (err) {
		printk(KERN_ERR "%s error %d initializing ctrl_qp.\n",
		       __FUNCTION__, err);
		goto err1;
	}
	err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,
				     0, T3_MAX_NUM_QP, T3_MAX_NUM_CQ,
				     T3_MAX_NUM_PD);
	if (err) {
		printk(KERN_ERR "%s error %d initializing hal resources.\n",
		       __FUNCTION__, err);
		goto err2;
	}
	err = cxio_hal_pblpool_create(rdev_p);
	if (err) {
		printk(KERN_ERR "%s error %d initializing pbl mem pool.\n",
		       __FUNCTION__, err);
		goto err3;
	}
	err = cxio_hal_rqtpool_create(rdev_p);
	if (err) {
		printk(KERN_ERR "%s error %d initializing rqt mem pool.\n",
		       __FUNCTION__, err);
		goto err4;
	}
	return 0;
err4:
	cxio_hal_pblpool_destroy(rdev_p);
err3:
	cxio_hal_destroy_resource(rdev_p->rscp);
err2:
	cxio_hal_destroy_ctrl_qp(rdev_p);
err1:
	list_del(&rdev_p->entry);
	return err;
}

void cxio_rdev_close(struct cxio_rdev *rdev_p)
{
	if (rdev_p) {
		cxio_hal_pblpool_destroy(rdev_p);
		cxio_hal_rqtpool_destroy(rdev_p);
		list_del(&rdev_p->entry);
		rdev_p->t3cdev_p->ulp = NULL;
		cxio_hal_destroy_ctrl_qp(rdev_p);
		cxio_hal_destroy_resource(rdev_p->rscp);
	}
}

int __init cxio_hal_init(void)
{
	if (cxio_hal_init_rhdl_resource(T3_MAX_NUM_RI))
		return -ENOMEM;
	t3_register_cpl_handler(CPL_ASYNC_NOTIF, cxio_hal_ev_handler);
	return 0;
}

void __exit cxio_hal_exit(void)
{
	struct cxio_rdev *rdev, *tmp;

	t3_register_cpl_handler(CPL_ASYNC_NOTIF, NULL);
	list_for_each_entry_safe(rdev, tmp, &rdev_list, entry)
		cxio_rdev_close(rdev);
	cxio_hal_destroy_rhdl_resource();
}

static void flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
{
	struct t3_swsq *sqp;
	__u32 ptr = wq->sq_rptr;
	int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);

	sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
	while (count--)
		if (!sqp->signaled) {
			ptr++;
			sqp = wq->sq + Q_PTR2IDX(ptr,  wq->sq_size_log2);
		} else if (sqp->complete) {

			/*
			 * Insert this completed cqe into the swcq.
			 */
			PDBG("%s moving cqe into swcq sq idx %ld cq idx %ld\n",
			     __FUNCTION__, Q_PTR2IDX(ptr,  wq->sq_size_log2),
			     Q_PTR2IDX(cq->sw_wptr, cq->size_log2));
			sqp->cqe.header |= htonl(V_CQE_SWCQE(1));
			*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2))
				= sqp->cqe;
			cq->sw_wptr++;
			sqp->signaled = 0;
			break;
		} else
			break;
}

static void create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe,
				struct t3_cqe *read_cqe)
{
	read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr;
	read_cqe->len = wq->oldest_read->read_len;
	read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
				 V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
				 V_CQE_OPCODE(T3_READ_REQ) |
				 V_CQE_TYPE(1));
}

/*
 * Return a ptr to the next read wr in the SWSQ or NULL.
 */
static void advance_oldest_read(struct t3_wq *wq)
{

	u32 rptr = wq->oldest_read - wq->sq + 1;
	u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);

	while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
		wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);

		if (wq->oldest_read->opcode == T3_READ_REQ)
			return;
		rptr++;
	}
	wq->oldest_read = NULL;
}

/*
 * cxio_poll_cq
 *
 * Caller must:
 *     check the validity of the first CQE,
 *     supply the wq assicated with the qpid.
 *
 * credit: cq credit to return to sge.
 * cqe_flushed: 1 iff the CQE is flushed.
 * cqe: copy of the polled CQE.
 *
 * return value:
 *     0       CQE returned,
 *    -1       CQE skipped, try again.
 */
int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
		     u8 *cqe_flushed, u64 *cookie, u32 *credit)
{
	int ret = 0;
	struct t3_cqe *hw_cqe, read_cqe;

	*cqe_flushed = 0;
	*credit = 0;
	hw_cqe = cxio_next_cqe(cq);

	PDBG("%s CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x"
	     " opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",
	     __FUNCTION__, CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe),
	     CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe),
	     CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe),
	     CQE_WRID_LOW(*hw_cqe));

	/*
	 * skip cqe's not affiliated with a QP.
	 */
	if (wq == NULL) {
		ret = -1;
		goto skip_cqe;
	}

	/*
	 * Gotta tweak READ completions:
	 *	1) the cqe doesn't contain the sq_wptr from the wr.
	 *	2) opcode not reflected from the wr.
	 *	3) read_len not reflected from the wr.
	 *	4) cq_type is RQ_TYPE not SQ_TYPE.
	 */
	if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {

		/*
		 * Don't write to the HWCQ, so create a new read req CQE
		 * in local memory.
		 */
		create_read_req_cqe(wq, hw_cqe, &read_cqe);
		hw_cqe = &read_cqe;
		advance_oldest_read(wq);
	}

	/*
	 * T3A: Discard TERMINATE CQEs.
	 */
	if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) {
		ret = -1;
		wq->error = 1;
		goto skip_cqe;
	}

	if (CQE_STATUS(*hw_cqe) || wq->error) {
		*cqe_flushed = wq->error;
		wq->error = 1;

		/*
		 * T3A inserts errors into the CQE.  We cannot return
		 * these as work completions.
		 */
		/* incoming write failures */
		if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE)
		     && RQ_TYPE(*hw_cqe)) {
			ret = -1;
			goto skip_cqe;
		}
		/* incoming read request failures */
		if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) && SQ_TYPE(*hw_cqe)) {
			ret = -1;
			goto skip_cqe;
		}

		/* incoming SEND with no receive posted failures */
		if ((CQE_OPCODE(*hw_cqe) == T3_SEND) && RQ_TYPE(*hw_cqe) &&
		    Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
			ret = -1;
			goto skip_cqe;
		}
		goto proc_cqe;
	}

	/*
	 * RECV completion.
	 */
	if (RQ_TYPE(*hw_cqe)) {

		/*
		 * HW only validates 4 bits of MSN.  So we must validate that
		 * the MSN in the SEND is the next expected MSN.  If its not,
		 * then we complete this with TPT_ERR_MSN and mark the wq in
		 * error.
		 */
		if (unlikely((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) {
			wq->error = 1;
			hw_cqe->header |= htonl(V_CQE_STATUS(TPT_ERR_MSN));
			goto proc_cqe;
		}
		goto proc_cqe;
	}

	/*
	 * If we get here its a send completion.
	 *
	 * Handle out of order completion. These get stuffed
	 * in the SW SQ. Then the SW SQ is walked to move any
	 * now in-order completions into the SW CQ.  This handles
	 * 2 cases:
	 *	1) reaping unsignaled WRs when the first subsequent
	 *	   signaled WR is completed.
	 *	2) out of order read completions.
	 */
	if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
		struct t3_swsq *sqp;

		PDBG("%s out of order completion going in swsq at idx %ld\n",
		     __FUNCTION__,
		     Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2));
		sqp = wq->sq +
		      Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
		sqp->cqe = *hw_cqe;
		sqp->complete = 1;
		ret = -1;
		goto flush_wq;
	}

proc_cqe:
	*cqe = *hw_cqe;

	/*
	 * Reap the associated WR(s) that are freed up with this
	 * completion.
	 */
	if (SQ_TYPE(*hw_cqe)) {
		wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
		PDBG("%s completing sq idx %ld\n", __FUNCTION__,
		     Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2));
		*cookie = (wq->sq +
			   Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2))->wr_id;
		wq->sq_rptr++;
	} else {
		PDBG("%s completing rq idx %ld\n", __FUNCTION__,
		     Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
		*cookie = *(wq->rq + Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
		wq->rq_rptr++;
	}

flush_wq:
	/*
	 * Flush any completed cqes that are now in-order.
	 */
	flush_completed_wrs(wq, cq);

skip_cqe:
	if (SW_CQE(*hw_cqe)) {
		PDBG("%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x\n",
		     __FUNCTION__, cq, cq->cqid, cq->sw_rptr);
		++cq->sw_rptr;
	} else {
		PDBG("%s cq %p cqid 0x%x skip hw cqe rptr 0x%x\n",
		     __FUNCTION__, cq, cq->cqid, cq->rptr);
		++cq->rptr;

		/*
		 * T3A: compute credits.
		 */
		if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))
		    || ((cq->rptr - cq->wptr) >= 128)) {
			*credit = cq->rptr - cq->wptr;
			cq->wptr = cq->rptr;
		}
	}
	return ret;
}