ipath_verbs.c

来自「linux 内核源代码」· C语言 代码 · 共 1,887 行 · 第 1/4 页

	case IB_QPT_RC:
		ipath_rc_rcv(dev, hdr, has_grh, data, tlen, qp);
		break;

	case IB_QPT_UC:
		ipath_uc_rcv(dev, hdr, has_grh, data, tlen, qp);
		break;

	default:
		break;
	}
}

/**
 * ipath_ib_rcv - process an incoming packet
 * @arg: the device pointer
 * @rhdr: the header of the packet
 * @data: the packet data
 * @tlen: the packet length
 *
 * This is called from ipath_kreceive() to process an incoming packet at
 * interrupt level. Tlen is the length of the header + data + CRC in bytes.
 */
void ipath_ib_rcv(struct ipath_ibdev *dev, void *rhdr, void *data,
		  u32 tlen)
{
	struct ipath_ib_header *hdr = rhdr;
	struct ipath_other_headers *ohdr;
	struct ipath_qp *qp;
	u32 qp_num;
	int lnh;
	u8 opcode;
	u16 lid;

	if (unlikely(dev == NULL))
		goto bail;

	if (unlikely(tlen < 24)) {	/* LRH+BTH+CRC */
		dev->rcv_errors++;
		goto bail;
	}

	/* Check for a valid destination LID (see ch. 7.11.1). */
	lid = be16_to_cpu(hdr->lrh[1]);
	if (lid < IPATH_MULTICAST_LID_BASE) {
		lid &= ~((1 << dev->dd->ipath_lmc) - 1);
		if (unlikely(lid != dev->dd->ipath_lid)) {
			dev->rcv_errors++;
			goto bail;
		}
	}

	/* Check for GRH */
	lnh = be16_to_cpu(hdr->lrh[0]) & 3;
	if (lnh == IPATH_LRH_BTH)
		ohdr = &hdr->u.oth;
	else if (lnh == IPATH_LRH_GRH)
		ohdr = &hdr->u.l.oth;
	else {
		dev->rcv_errors++;
		goto bail;
	}

	opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
	dev->opstats[opcode].n_bytes += tlen;
	dev->opstats[opcode].n_packets++;

	/* Get the destination QP number. */
	qp_num = be32_to_cpu(ohdr->bth[1]) & IPATH_QPN_MASK;
	if (qp_num == IPATH_MULTICAST_QPN) {
		struct ipath_mcast *mcast;
		struct ipath_mcast_qp *p;

		if (lnh != IPATH_LRH_GRH) {
			dev->n_pkt_drops++;
			goto bail;
		}
		mcast = ipath_mcast_find(&hdr->u.l.grh.dgid);
		if (mcast == NULL) {
			dev->n_pkt_drops++;
			goto bail;
		}
		dev->n_multicast_rcv++;
		list_for_each_entry_rcu(p, &mcast->qp_list, list)
			ipath_qp_rcv(dev, hdr, 1, data, tlen, p->qp);
		/*
		 * Notify ipath_multicast_detach() if it is waiting for us
		 * to finish.
		 */
		if (atomic_dec_return(&mcast->refcount) <= 1)
			wake_up(&mcast->wait);
	} else {
		qp = ipath_lookup_qpn(&dev->qp_table, qp_num);
		if (qp) {
			dev->n_unicast_rcv++;
			ipath_qp_rcv(dev, hdr, lnh == IPATH_LRH_GRH, data,
				     tlen, qp);
			/*
			 * Notify ipath_destroy_qp() if it is waiting
			 * for us to finish.
			 */
			if (atomic_dec_and_test(&qp->refcount))
				wake_up(&qp->wait);
		} else
			dev->n_pkt_drops++;
	}

bail:;
}

/**
 * ipath_ib_timer - verbs timer
 * @arg: the device pointer
 *
 * This is called from ipath_do_rcv_timer() at interrupt level to check for
 * QPs which need retransmits and to collect performance numbers.
 */
static void ipath_ib_timer(struct ipath_ibdev *dev)
{
	struct ipath_qp *resend = NULL;
	struct list_head *last;
	struct ipath_qp *qp;
	unsigned long flags;

	if (dev == NULL)
		return;

	spin_lock_irqsave(&dev->pending_lock, flags);
	/* Start filling the next pending queue. */
	if (++dev->pending_index >= ARRAY_SIZE(dev->pending))
		dev->pending_index = 0;
	/* Save any requests still in the new queue, they have timed out. */
	last = &dev->pending[dev->pending_index];
	while (!list_empty(last)) {
		qp = list_entry(last->next, struct ipath_qp, timerwait);
		list_del_init(&qp->timerwait);
		qp->timer_next = resend;
		resend = qp;
		atomic_inc(&qp->refcount);
	}
	last = &dev->rnrwait;
	if (!list_empty(last)) {
		qp = list_entry(last->next, struct ipath_qp, timerwait);
		if (--qp->s_rnr_timeout == 0) {
			do {
				list_del_init(&qp->timerwait);
				tasklet_hi_schedule(&qp->s_task);
				if (list_empty(last))
					break;
				qp = list_entry(last->next, struct ipath_qp,
						timerwait);
			} while (qp->s_rnr_timeout == 0);
		}
	}
	/*
	 * We should only be in the started state if pma_sample_start != 0
	 */
	if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_STARTED &&
	    --dev->pma_sample_start == 0) {
		dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
		ipath_snapshot_counters(dev->dd, &dev->ipath_sword,
					&dev->ipath_rword,
					&dev->ipath_spkts,
					&dev->ipath_rpkts,
					&dev->ipath_xmit_wait);
	}
	if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
		if (dev->pma_sample_interval == 0) {
			u64 ta, tb, tc, td, te;

			dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
			ipath_snapshot_counters(dev->dd, &ta, &tb,
						&tc, &td, &te);

			dev->ipath_sword = ta - dev->ipath_sword;
			dev->ipath_rword = tb - dev->ipath_rword;
			dev->ipath_spkts = tc - dev->ipath_spkts;
			dev->ipath_rpkts = td - dev->ipath_rpkts;
			dev->ipath_xmit_wait = te - dev->ipath_xmit_wait;
		}
		else
			dev->pma_sample_interval--;
	}
	spin_unlock_irqrestore(&dev->pending_lock, flags);

	/* XXX What if timer fires again while this is running? */
	for (qp = resend; qp != NULL; qp = qp->timer_next) {
		struct ib_wc wc;

		spin_lock_irqsave(&qp->s_lock, flags);
		if (qp->s_last != qp->s_tail && qp->state == IB_QPS_RTS) {
			dev->n_timeouts++;
			ipath_restart_rc(qp, qp->s_last_psn + 1, &wc);
		}
		spin_unlock_irqrestore(&qp->s_lock, flags);

		/* Notify ipath_destroy_qp() if it is waiting. */
		if (atomic_dec_and_test(&qp->refcount))
			wake_up(&qp->wait);
	}
}

static void update_sge(struct ipath_sge_state *ss, u32 length)
{
	struct ipath_sge *sge = &ss->sge;

	sge->vaddr += length;
	sge->length -= length;
	sge->sge_length -= length;
	if (sge->sge_length == 0) {
		if (--ss->num_sge)
			*sge = *ss->sg_list++;
	} else if (sge->length == 0 && sge->mr != NULL) {
		if (++sge->n >= IPATH_SEGSZ) {
			if (++sge->m >= sge->mr->mapsz)
				return;
			sge->n = 0;
		}
		sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
		sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
	}
}

#ifdef __LITTLE_ENDIAN
static inline u32 get_upper_bits(u32 data, u32 shift)
{
	return data >> shift;
}

static inline u32 set_upper_bits(u32 data, u32 shift)
{
	return data << shift;
}

static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off)
{
	data <<= ((sizeof(u32) - n) * BITS_PER_BYTE);
	data >>= ((sizeof(u32) - n - off) * BITS_PER_BYTE);
	return data;
}
#else
static inline u32 get_upper_bits(u32 data, u32 shift)
{
	return data << shift;
}

static inline u32 set_upper_bits(u32 data, u32 shift)
{
	return data >> shift;
}

static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off)
{
	data >>= ((sizeof(u32) - n) * BITS_PER_BYTE);
	data <<= ((sizeof(u32) - n - off) * BITS_PER_BYTE);
	return data;
}
#endif

static void copy_io(u32 __iomem *piobuf, struct ipath_sge_state *ss,
		    u32 length, unsigned flush_wc)
{
	u32 extra = 0;
	u32 data = 0;
	u32 last;

	while (1) {
		u32 len = ss->sge.length;
		u32 off;

		if (len > length)
			len = length;
		if (len > ss->sge.sge_length)
			len = ss->sge.sge_length;
		BUG_ON(len == 0);
		/* If the source address is not aligned, try to align it. */
		off = (unsigned long)ss->sge.vaddr & (sizeof(u32) - 1);
		if (off) {
			u32 *addr = (u32 *)((unsigned long)ss->sge.vaddr &
					    ~(sizeof(u32) - 1));
			u32 v = get_upper_bits(*addr, off * BITS_PER_BYTE);
			u32 y;

			y = sizeof(u32) - off;
			if (len > y)
				len = y;
			if (len + extra >= sizeof(u32)) {
				data |= set_upper_bits(v, extra *
						       BITS_PER_BYTE);
				len = sizeof(u32) - extra;
				if (len == length) {
					last = data;
					break;
				}
				__raw_writel(data, piobuf);
				piobuf++;
				extra = 0;
				data = 0;
			} else {
				/* Clear unused upper bytes */
				data |= clear_upper_bytes(v, len, extra);
				if (len == length) {
					last = data;
					break;
				}
				extra += len;
			}
		} else if (extra) {
			/* Source address is aligned. */
			u32 *addr = (u32 *) ss->sge.vaddr;
			int shift = extra * BITS_PER_BYTE;
			int ushift = 32 - shift;
			u32 l = len;

			while (l >= sizeof(u32)) {
				u32 v = *addr;

				data |= set_upper_bits(v, shift);
				__raw_writel(data, piobuf);
				data = get_upper_bits(v, ushift);
				piobuf++;
				addr++;
				l -= sizeof(u32);
			}
			/*
			 * We still have 'extra' number of bytes leftover.
			 */
			if (l) {
				u32 v = *addr;

				if (l + extra >= sizeof(u32)) {
					data |= set_upper_bits(v, shift);
					len -= l + extra - sizeof(u32);
					if (len == length) {
						last = data;
						break;
					}
					__raw_writel(data, piobuf);
					piobuf++;
					extra = 0;
					data = 0;
				} else {
					/* Clear unused upper bytes */
					data |= clear_upper_bytes(v, l,
								  extra);
					if (len == length) {
						last = data;
						break;
					}
					extra += l;
				}
			} else if (len == length) {
				last = data;
				break;
			}
		} else if (len == length) {
			u32 w;

			/*
			 * Need to round up for the last dword in the
			 * packet.
			 */
			w = (len + 3) >> 2;
			__iowrite32_copy(piobuf, ss->sge.vaddr, w - 1);
			piobuf += w - 1;
			last = ((u32 *) ss->sge.vaddr)[w - 1];
			break;
		} else {
			u32 w = len >> 2;

			__iowrite32_copy(piobuf, ss->sge.vaddr, w);
			piobuf += w;

			extra = len & (sizeof(u32) - 1);
			if (extra) {
				u32 v = ((u32 *) ss->sge.vaddr)[w];

				/* Clear unused upper bytes */
				data = clear_upper_bytes(v, extra, 0);
			}
		}
		update_sge(ss, len);
		length -= len;
	}
	/* Update address before sending packet. */
	update_sge(ss, length);
	if (flush_wc) {
		/* must flush early everything before trigger word */
		ipath_flush_wc();
		__raw_writel(last, piobuf);
		/* be sure trigger word is written */
		ipath_flush_wc();
	} else
		__raw_writel(last, piobuf);
}

static int ipath_verbs_send_pio(struct ipath_qp *qp, u32 *hdr, u32 hdrwords,
				struct ipath_sge_state *ss, u32 len,
				u32 plen, u32 dwords)
{
	struct ipath_devdata *dd = to_idev(qp->ibqp.device)->dd;
	u32 __iomem *piobuf;
	unsigned flush_wc;
	int ret;

	piobuf = ipath_getpiobuf(dd, NULL);
	if (unlikely(piobuf == NULL)) {
		ret = -EBUSY;
		goto bail;
	}

	/*
	 * Write len to control qword, no flags.
	 * We have to flush after the PBC for correctness on some cpus
	 * or WC buffer can be written out of order.
	 */
	writeq(plen, piobuf);
	piobuf += 2;

	flush_wc = dd->ipath_flags & IPATH_PIO_FLUSH_WC;
	if (len == 0) {
		/*
		 * If there is just the header portion, must flush before
		 * writing last word of header for correctness, and after
		 * the last header word (trigger word).
		 */
		if (flush_wc) {
			ipath_flush_wc();
			__iowrite32_copy(piobuf, hdr, hdrwords - 1);
			ipath_flush_wc();
			__raw_writel(hdr[hdrwords - 1], piobuf + hdrwords - 1);
			ipath_flush_wc();
		} else
			__iowrite32_copy(piobuf, hdr, hdrwords);
		goto done;
	}

	if (flush_wc)
		ipath_flush_wc();
	__iowrite32_copy(piobuf, hdr, hdrwords);
	piobuf += hdrwords;

	/* The common case is aligned and contained in one segment. */
	if (likely(ss->num_sge == 1 && len <= ss->sge.length &&
		   !((unsigned long)ss->sge.vaddr & (sizeof(u32) - 1)))) {
		u32 *addr = (u32 *) ss->sge.vaddr;

		/* Update address before sending packet. */
		update_sge(ss, len);
		if (flush_wc) {
			__iowrite32_copy(piobuf, addr, dwords - 1);
			/* must flush early everything before trigger word */
			ipath_flush_wc();
			__raw_writel(addr[dwords - 1], piobuf + dwords - 1);
			/* be sure trigger word is written */
			ipath_flush_wc();
		} else
			__iowrite32_copy(piobuf, addr, dwords);
		goto done;
	}
	copy_io(piobuf, ss, len, flush_wc);
done:
	if (qp->s_wqe)
		ipath_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS);
	ret = 0;
bail:
	return ret;
}

/**
 * ipath_verbs_send - send a packet
 * @qp: the QP to send on