ehca_reqs.c

linux内核源码

			/* WQE purged */
			*wc_status = IB_WC_WR_FLUSH_ERR;
			break;
		default:
			*wc_status = IB_WC_FATAL_ERR;

		}
	} else
		*wc_status = IB_WC_SUCCESS;
}

int ehca_post_send(struct ib_qp *qp,
		   struct ib_send_wr *send_wr,
		   struct ib_send_wr **bad_send_wr)
{
	struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
	struct ib_send_wr *cur_send_wr;
	struct ehca_wqe *wqe_p;
	int wqe_cnt = 0;
	int ret = 0;
	unsigned long flags;

	/* LOCK the QUEUE */
	spin_lock_irqsave(&my_qp->spinlock_s, flags);

	/* loop processes list of send reqs */
	for (cur_send_wr = send_wr; cur_send_wr != NULL;
	     cur_send_wr = cur_send_wr->next) {
		u64 start_offset = my_qp->ipz_squeue.current_q_offset;
		/* get pointer next to free WQE */
		wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
		if (unlikely(!wqe_p)) {
			/* too many posted work requests: queue overflow */
			if (bad_send_wr)
				*bad_send_wr = cur_send_wr;
			if (wqe_cnt == 0) {
				ret = -ENOMEM;
				ehca_err(qp->device, "Too many posted WQEs "
					 "qp_num=%x", qp->qp_num);
			}
			goto post_send_exit0;
		}
		/* write a SEND WQE into the QUEUE */
		ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr);
		/*
		 * if something failed,
		 * reset the free entry pointer to the start value
		 */
		if (unlikely(ret)) {
			my_qp->ipz_squeue.current_q_offset = start_offset;
			*bad_send_wr = cur_send_wr;
			if (wqe_cnt == 0) {
				ret = -EINVAL;
				ehca_err(qp->device, "Could not write WQE "
					 "qp_num=%x", qp->qp_num);
			}
			goto post_send_exit0;
		}
		wqe_cnt++;
		ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
			 my_qp, qp->qp_num, wqe_cnt);
	} /* eof for cur_send_wr */

post_send_exit0:
	iosync(); /* serialize GAL register access */
	hipz_update_sqa(my_qp, wqe_cnt);
	spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
	return ret;
}

static int internal_post_recv(struct ehca_qp *my_qp,
			      struct ib_device *dev,
			      struct ib_recv_wr *recv_wr,
			      struct ib_recv_wr **bad_recv_wr)
{
	struct ib_recv_wr *cur_recv_wr;
	struct ehca_wqe *wqe_p;
	int wqe_cnt = 0;
	int ret = 0;
	unsigned long flags;

	if (unlikely(!HAS_RQ(my_qp))) {
		ehca_err(dev, "QP has no RQ  ehca_qp=%p qp_num=%x ext_type=%d",
			 my_qp, my_qp->real_qp_num, my_qp->ext_type);
		return -ENODEV;
	}

	/* LOCK the QUEUE */
	spin_lock_irqsave(&my_qp->spinlock_r, flags);

	/* loop processes list of send reqs */
	for (cur_recv_wr = recv_wr; cur_recv_wr != NULL;
	     cur_recv_wr = cur_recv_wr->next) {
		u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
		/* get pointer next to free WQE */
		wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
		if (unlikely(!wqe_p)) {
			/* too many posted work requests: queue overflow */
			if (bad_recv_wr)
				*bad_recv_wr = cur_recv_wr;
			if (wqe_cnt == 0) {
				ret = -ENOMEM;
				ehca_err(dev, "Too many posted WQEs "
					 "qp_num=%x", my_qp->real_qp_num);
			}
			goto post_recv_exit0;
		}
		/* write a RECV WQE into the QUEUE */
		ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, cur_recv_wr);
		/*
		 * if something failed,
		 * reset the free entry pointer to the start value
		 */
		if (unlikely(ret)) {
			my_qp->ipz_rqueue.current_q_offset = start_offset;
			*bad_recv_wr = cur_recv_wr;
			if (wqe_cnt == 0) {
				ret = -EINVAL;
				ehca_err(dev, "Could not write WQE "
					 "qp_num=%x", my_qp->real_qp_num);
			}
			goto post_recv_exit0;
		}
		wqe_cnt++;
		ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
			 my_qp, my_qp->real_qp_num, wqe_cnt);
	} /* eof for cur_recv_wr */

post_recv_exit0:
	iosync(); /* serialize GAL register access */
	hipz_update_rqa(my_qp, wqe_cnt);
	spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
	return ret;
}

int ehca_post_recv(struct ib_qp *qp,
		   struct ib_recv_wr *recv_wr,
		   struct ib_recv_wr **bad_recv_wr)
{
	return internal_post_recv(container_of(qp, struct ehca_qp, ib_qp),
				  qp->device, recv_wr, bad_recv_wr);
}

int ehca_post_srq_recv(struct ib_srq *srq,
		       struct ib_recv_wr *recv_wr,
		       struct ib_recv_wr **bad_recv_wr)
{
	return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
				  srq->device, recv_wr, bad_recv_wr);
}

/*
 * ib_wc_opcode table converts ehca wc opcode to ib
 * Since we use zero to indicate invalid opcode, the actual ib opcode must
 * be decremented!!!
 */
static const u8 ib_wc_opcode[255] = {
	[0x01] = IB_WC_RECV+1,
	[0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
	[0x04] = IB_WC_BIND_MW+1,
	[0x08] = IB_WC_FETCH_ADD+1,
	[0x10] = IB_WC_COMP_SWAP+1,
	[0x20] = IB_WC_RDMA_WRITE+1,
	[0x40] = IB_WC_RDMA_READ+1,
	[0x80] = IB_WC_SEND+1
};

/* internal function to poll one entry of cq */
static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
{
	int ret = 0;
	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	struct ehca_cqe *cqe;
	struct ehca_qp *my_qp;
	int cqe_count = 0;

poll_cq_one_read_cqe:
	cqe = (struct ehca_cqe *)
		ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
	if (!cqe) {
		ret = -EAGAIN;
		ehca_dbg(cq->device, "Completion queue is empty ehca_cq=%p "
			 "cq_num=%x ret=%i", my_cq, my_cq->cq_number, ret);
		goto  poll_cq_one_exit0;
	}

	/* prevents loads being reordered across this point */
	rmb();

	cqe_count++;
	if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
		struct ehca_qp *qp;
		int purgeflag;
		unsigned long flags;

		qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
		if (!qp) {
			ehca_err(cq->device, "cq_num=%x qp_num=%x "
				 "could not find qp -> ignore cqe",
				 my_cq->cq_number, cqe->local_qp_number);
			ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
				 my_cq->cq_number, cqe->local_qp_number);
			/* ignore this purged cqe */
			goto poll_cq_one_read_cqe;
		}
		spin_lock_irqsave(&qp->spinlock_s, flags);
		purgeflag = qp->sqerr_purgeflag;
		spin_unlock_irqrestore(&qp->spinlock_s, flags);

		if (purgeflag) {
			ehca_dbg(cq->device,
				 "Got CQE with purged bit qp_num=%x src_qp=%x",
				 cqe->local_qp_number, cqe->remote_qp_number);
			if (ehca_debug_level)
				ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
					 cqe->local_qp_number,
					 cqe->remote_qp_number);
			/*
			 * ignore this to avoid double cqes of bad wqe
			 * that caused sqe and turn off purge flag
			 */
			qp->sqerr_purgeflag = 0;
			goto poll_cq_one_read_cqe;
		}
	}

	/* tracing cqe */
	if (unlikely(ehca_debug_level)) {
		ehca_dbg(cq->device,
			 "Received COMPLETION ehca_cq=%p cq_num=%x -----",
			 my_cq, my_cq->cq_number);
		ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
			 my_cq, my_cq->cq_number);
		ehca_dbg(cq->device,
			 "ehca_cq=%p cq_num=%x -------------------------",
			 my_cq, my_cq->cq_number);
	}

	/* we got a completion! */
	wc->wr_id = cqe->work_request_id;

	/* eval ib_wc_opcode */
	wc->opcode = ib_wc_opcode[cqe->optype]-1;
	if (unlikely(wc->opcode == -1)) {
		ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
			 "ehca_cq=%p cq_num=%x",
			 cqe->optype, cqe->status, my_cq, my_cq->cq_number);
		/* dump cqe for other infos */
		ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
			 my_cq, my_cq->cq_number);
		/* update also queue adder to throw away this entry!!! */
		goto poll_cq_one_exit0;
	}
	/* eval ib_wc_status */
	if (unlikely(cqe->status & WC_STATUS_ERROR_BIT)) {
		/* complete with errors */
		map_ib_wc_status(cqe->status, &wc->status);
		wc->vendor_err = wc->status;
	} else
		wc->status = IB_WC_SUCCESS;

	read_lock(&ehca_qp_idr_lock);
	my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
	wc->qp = &my_qp->ib_qp;
	read_unlock(&ehca_qp_idr_lock);

	wc->byte_len = cqe->nr_bytes_transferred;
	wc->pkey_index = cqe->pkey_index;
	wc->slid = cqe->rlid;
	wc->dlid_path_bits = cqe->dlid;
	wc->src_qp = cqe->remote_qp_number;
	wc->wc_flags = cqe->w_completion_flags;
	wc->imm_data = cpu_to_be32(cqe->immediate_data);
	wc->sl = cqe->service_level;

	if (unlikely(wc->status != IB_WC_SUCCESS))
		ehca_dbg(cq->device,
			 "ehca_cq=%p cq_num=%x WARNING unsuccessful cqe "
			 "OPType=%x status=%x qp_num=%x src_qp=%x wr_id=%lx "
			 "cqe=%p", my_cq, my_cq->cq_number, cqe->optype,
			 cqe->status, cqe->local_qp_number,
			 cqe->remote_qp_number, cqe->work_request_id, cqe);

poll_cq_one_exit0:
	if (cqe_count > 0)
		hipz_update_feca(my_cq, cqe_count);

	return ret;
}

int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
{
	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	int nr;
	struct ib_wc *current_wc = wc;
	int ret = 0;
	unsigned long flags;

	if (num_entries < 1) {
		ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
			 "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
		ret = -EINVAL;
		goto poll_cq_exit0;
	}

	spin_lock_irqsave(&my_cq->spinlock, flags);
	for (nr = 0; nr < num_entries; nr++) {
		ret = ehca_poll_cq_one(cq, current_wc);
		if (ret)
			break;
		current_wc++;
	} /* eof for nr */
	spin_unlock_irqrestore(&my_cq->spinlock, flags);
	if (ret == -EAGAIN  || !ret)
		ret = nr;

poll_cq_exit0:
	return ret;
}

int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
{
	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	int ret = 0;

	switch (notify_flags & IB_CQ_SOLICITED_MASK) {
	case IB_CQ_SOLICITED:
		hipz_set_cqx_n0(my_cq, 1);
		break;
	case IB_CQ_NEXT_COMP:
		hipz_set_cqx_n1(my_cq, 1);
		break;
	default:
		return -EINVAL;
	}

	if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
		unsigned long spl_flags;
		spin_lock_irqsave(&my_cq->spinlock, spl_flags);
		ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
		spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
	}

	return ret;
}