ipath_driver.c

来自「LINUX 2.6.17.4的源码」· C语言 代码 · 共 1,993 行 · 第 1/4 页

				int ret = __ipath_verbs_rcv(dd, rc + 1,
							    ebuf, tlen);
				if (ret == -ENODEV)
					ipath_cdbg(VERBOSE,
						   "received IB packet, "
						   "not SMA (QP=%x)\n", qp);
		} else if (etype == RCVHQ_RCV_TYPE_EAGER) {
			if (qp == IPATH_KD_QP &&
			    bthbytes[0] == ipath_layer_rcv_opcode &&
			    ebuf)
				ipath_rcv_layer(dd, etail, tlen,
						(struct ether_header *)hdr);
			else
				ipath_cdbg(PKT, "typ %x, opcode %x (eager, "
					   "qp=%x), len %x; ignored\n",
					   etype, bthbytes[0], qp, tlen);
		}
		else if (etype == RCVHQ_RCV_TYPE_EXPECTED)
			ipath_dbg("Bug: Expected TID, opcode %x; ignored\n",
				  be32_to_cpu(hdr->bth[0]) & 0xff);
		else if (eflags & (INFINIPATH_RHF_H_TIDERR |
				   INFINIPATH_RHF_H_IHDRERR)) {
			/*
			 * This is a type 3 packet, only the LRH is in the
			 * rcvhdrq, the rest of the header is in the eager
			 * buffer.
			 */
			u8 opcode;
			if (ebuf) {
				bthbytes = (u8 *) ebuf;
				opcode = *bthbytes;
			}
			else
				opcode = 0;
			get_rhf_errstring(eflags, emsg, sizeof emsg);
			ipath_dbg("Err %x (%s), opcode %x, egrbuf %x, "
				  "len %x\n", eflags, emsg, opcode, etail,
				  tlen);
		} else {
			/*
			 * error packet, type of error	unknown.
			 * Probably type 3, but we don't know, so don't
			 * even try to print the opcode, etc.
			 */
			ipath_dbg("Error Pkt, but no eflags! egrbuf %x, "
				  "len %x\nhdrq@%lx;hdrq+%x rhf: %llx; "
				  "hdr %llx %llx %llx %llx %llx\n",
				  etail, tlen, (unsigned long) rc, l,
				  (unsigned long long) rc[0],
				  (unsigned long long) rc[1],
				  (unsigned long long) rc[2],
				  (unsigned long long) rc[3],
				  (unsigned long long) rc[4],
				  (unsigned long long) rc[5]);
		}
		l += rsize;
		if (l >= maxcnt)
			l = 0;
		/*
		 * update for each packet, to help prevent overflows if we
		 * have lots of packets.
		 */
		(void)ipath_write_ureg(dd, ur_rcvhdrhead,
				       dd->ipath_rhdrhead_intr_off | l, 0);
		if (etype != RCVHQ_RCV_TYPE_EXPECTED)
			(void)ipath_write_ureg(dd, ur_rcvegrindexhead,
					       etail, 0);
	}

	pkttot += i;

	dd->ipath_port0head = l;

	if (hdrqtail != (u32)le64_to_cpu(*dd->ipath_hdrqtailptr))
		/* more arrived while we handled first batch */
		goto gotmore;

	if (pkttot > ipath_stats.sps_maxpkts_call)
		ipath_stats.sps_maxpkts_call = pkttot;
	ipath_stats.sps_port0pkts += pkttot;
	ipath_stats.sps_avgpkts_call =
		ipath_stats.sps_port0pkts / ++totcalls;

done:
	clear_bit(0, &dd->ipath_rcv_pending);
	smp_mb__after_clear_bit();

bail:;
}

/**
 * ipath_update_pio_bufs - update shadow copy of the PIO availability map
 * @dd: the infinipath device
 *
 * called whenever our local copy indicates we have run out of send buffers
 * NOTE: This can be called from interrupt context by some code
 * and from non-interrupt context by ipath_getpiobuf().
 */

static void ipath_update_pio_bufs(struct ipath_devdata *dd)
{
	unsigned long flags;
	int i;
	const unsigned piobregs = (unsigned)dd->ipath_pioavregs;

	/* If the generation (check) bits have changed, then we update the
	 * busy bit for the corresponding PIO buffer.  This algorithm will
	 * modify positions to the value they already have in some cases
	 * (i.e., no change), but it's faster than changing only the bits
	 * that have changed.
	 *
	 * We would like to do this atomicly, to avoid spinlocks in the
	 * critical send path, but that's not really possible, given the
	 * type of changes, and that this routine could be called on
	 * multiple cpu's simultaneously, so we lock in this routine only,
	 * to avoid conflicting updates; all we change is the shadow, and
	 * it's a single 64 bit memory location, so by definition the update
	 * is atomic in terms of what other cpu's can see in testing the
	 * bits.  The spin_lock overhead isn't too bad, since it only
	 * happens when all buffers are in use, so only cpu overhead, not
	 * latency or bandwidth is affected.
	 */
#define _IPATH_ALL_CHECKBITS 0x5555555555555555ULL
	if (!dd->ipath_pioavailregs_dma) {
		ipath_dbg("Update shadow pioavail, but regs_dma NULL!\n");
		return;
	}
	if (ipath_debug & __IPATH_VERBDBG) {
		/* only if packet debug and verbose */
		volatile __le64 *dma = dd->ipath_pioavailregs_dma;
		unsigned long *shadow = dd->ipath_pioavailshadow;

		ipath_cdbg(PKT, "Refill avail, dma0=%llx shad0=%lx, "
			   "d1=%llx s1=%lx, d2=%llx s2=%lx, d3=%llx "
			   "s3=%lx\n",
			   (unsigned long long) le64_to_cpu(dma[0]),
			   shadow[0],
			   (unsigned long long) le64_to_cpu(dma[1]),
			   shadow[1],
			   (unsigned long long) le64_to_cpu(dma[2]),
			   shadow[2],
			   (unsigned long long) le64_to_cpu(dma[3]),
			   shadow[3]);
		if (piobregs > 4)
			ipath_cdbg(
				PKT, "2nd group, dma4=%llx shad4=%lx, "
				"d5=%llx s5=%lx, d6=%llx s6=%lx, "
				"d7=%llx s7=%lx\n",
				(unsigned long long) le64_to_cpu(dma[4]),
				shadow[4],
				(unsigned long long) le64_to_cpu(dma[5]),
				shadow[5],
				(unsigned long long) le64_to_cpu(dma[6]),
				shadow[6],
				(unsigned long long) le64_to_cpu(dma[7]),
				shadow[7]);
	}
	spin_lock_irqsave(&ipath_pioavail_lock, flags);
	for (i = 0; i < piobregs; i++) {
		u64 pchbusy, pchg, piov, pnew;
		/*
		 * Chip Errata: bug 6641; even and odd qwords>3 are swapped
		 */
		if (i > 3) {
			if (i & 1)
				piov = le64_to_cpu(
					dd->ipath_pioavailregs_dma[i - 1]);
			else
				piov = le64_to_cpu(
					dd->ipath_pioavailregs_dma[i + 1]);
		} else
			piov = le64_to_cpu(dd->ipath_pioavailregs_dma[i]);
		pchg = _IPATH_ALL_CHECKBITS &
			~(dd->ipath_pioavailshadow[i] ^ piov);
		pchbusy = pchg << INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT;
		if (pchg && (pchbusy & dd->ipath_pioavailshadow[i])) {
			pnew = dd->ipath_pioavailshadow[i] & ~pchbusy;
			pnew |= piov & pchbusy;
			dd->ipath_pioavailshadow[i] = pnew;
		}
	}
	spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
}

/**
 * ipath_setrcvhdrsize - set the receive header size
 * @dd: the infinipath device
 * @rhdrsize: the receive header size
 *
 * called from user init code, and also layered driver init
 */
int ipath_setrcvhdrsize(struct ipath_devdata *dd, unsigned rhdrsize)
{
	int ret = 0;

	if (dd->ipath_flags & IPATH_RCVHDRSZ_SET) {
		if (dd->ipath_rcvhdrsize != rhdrsize) {
			dev_info(&dd->pcidev->dev,
				 "Error: can't set protocol header "
				 "size %u, already %u\n",
				 rhdrsize, dd->ipath_rcvhdrsize);
			ret = -EAGAIN;
		} else
			ipath_cdbg(VERBOSE, "Reuse same protocol header "
				   "size %u\n", dd->ipath_rcvhdrsize);
	} else if (rhdrsize > (dd->ipath_rcvhdrentsize -
			       (sizeof(u64) / sizeof(u32)))) {
		ipath_dbg("Error: can't set protocol header size %u "
			  "(> max %u)\n", rhdrsize,
			  dd->ipath_rcvhdrentsize -
			  (u32) (sizeof(u64) / sizeof(u32)));
		ret = -EOVERFLOW;
	} else {
		dd->ipath_flags |= IPATH_RCVHDRSZ_SET;
		dd->ipath_rcvhdrsize = rhdrsize;
		ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvhdrsize,
				 dd->ipath_rcvhdrsize);
		ipath_cdbg(VERBOSE, "Set protocol header size to %u\n",
			   dd->ipath_rcvhdrsize);
	}
	return ret;
}

/**
 * ipath_getpiobuf - find an available pio buffer
 * @dd: the infinipath device
 * @pbufnum: the buffer number is placed here
 *
 * do appropriate marking as busy, etc.
 * returns buffer number if one found (>=0), negative number is error.
 * Used by ipath_sma_send_pkt and ipath_layer_send
 */
u32 __iomem *ipath_getpiobuf(struct ipath_devdata *dd, u32 * pbufnum)
{
	int i, j, starti, updated = 0;
	unsigned piobcnt, iter;
	unsigned long flags;
	unsigned long *shadow = dd->ipath_pioavailshadow;
	u32 __iomem *buf;

	piobcnt = (unsigned)(dd->ipath_piobcnt2k
			     + dd->ipath_piobcnt4k);
	starti = dd->ipath_lastport_piobuf;
	iter = piobcnt - starti;
	if (dd->ipath_upd_pio_shadow) {
		/*
		 * Minor optimization.  If we had no buffers on last call,
		 * start out by doing the update; continue and do scan even
		 * if no buffers were updated, to be paranoid
		 */
		ipath_update_pio_bufs(dd);
		/* we scanned here, don't do it at end of scan */
		updated = 1;
		i = starti;
	} else
		i = dd->ipath_lastpioindex;

rescan:
	/*
	 * while test_and_set_bit() is atomic, we do that and then the
	 * change_bit(), and the pair is not.  See if this is the cause
	 * of the remaining armlaunch errors.
	 */
	spin_lock_irqsave(&ipath_pioavail_lock, flags);
	for (j = 0; j < iter; j++, i++) {
		if (i >= piobcnt)
			i = starti;
		/*
		 * To avoid bus lock overhead, we first find a candidate
		 * buffer, then do the test and set, and continue if that
		 * fails.
		 */
		if (test_bit((2 * i) + 1, shadow) ||
		    test_and_set_bit((2 * i) + 1, shadow))
			continue;
		/* flip generation bit */
		change_bit(2 * i, shadow);
		break;
	}
	spin_unlock_irqrestore(&ipath_pioavail_lock, flags);

	if (j == iter) {
		volatile __le64 *dma = dd->ipath_pioavailregs_dma;

		/*
		 * first time through; shadow exhausted, but may be real
		 * buffers available, so go see; if any updated, rescan
		 * (once)
		 */
		if (!updated) {
			ipath_update_pio_bufs(dd);
			updated = 1;
			i = starti;
			goto rescan;
		}
		dd->ipath_upd_pio_shadow = 1;
		/*
		 * not atomic, but if we lose one once in a while, that's OK
		 */
		ipath_stats.sps_nopiobufs++;
		if (!(++dd->ipath_consec_nopiobuf % 100000)) {
			ipath_dbg(
				"%u pio sends with no bufavail; dmacopy: "
				"%llx %llx %llx %llx; shadow:  "
				"%lx %lx %lx %lx\n",
				dd->ipath_consec_nopiobuf,
				(unsigned long long) le64_to_cpu(dma[0]),
				(unsigned long long) le64_to_cpu(dma[1]),
				(unsigned long long) le64_to_cpu(dma[2]),
				(unsigned long long) le64_to_cpu(dma[3]),
				shadow[0], shadow[1], shadow[2],
				shadow[3]);
			/*
			 * 4 buffers per byte, 4 registers above, cover rest
			 * below
			 */
			if ((dd->ipath_piobcnt2k + dd->ipath_piobcnt4k) >
			    (sizeof(shadow[0]) * 4 * 4))
				ipath_dbg("2nd group: dmacopy: %llx %llx "
					  "%llx %llx; shadow: %lx %lx "
					  "%lx %lx\n",
					  (unsigned long long)
					  le64_to_cpu(dma[4]),
					  (unsigned long long)
					  le64_to_cpu(dma[5]),
					  (unsigned long long)
					  le64_to_cpu(dma[6]),
					  (unsigned long long)
					  le64_to_cpu(dma[7]),
					  shadow[4], shadow[5],
					  shadow[6], shadow[7]);
		}
		buf = NULL;
		goto bail;
	}

	if (updated)
		/*
		 * ran out of bufs, now some (at least this one we just
		 * got) are now available, so tell the layered driver.
		 */
		__ipath_layer_intr(dd, IPATH_LAYER_INT_SEND_CONTINUE);

	/*
	 * set next starting place.  Since it's just an optimization,
	 * it doesn't matter who wins on this, so no locking
	 */
	dd->ipath_lastpioindex = i + 1;
	if (dd->ipath_upd_pio_shadow)
		dd->ipath_upd_pio_shadow = 0;
	if (dd->ipath_consec_nopiobuf)
		dd->ipath_consec_nopiobuf = 0;
	if (i < dd->ipath_piobcnt2k)
		buf = (u32 __iomem *) (dd->ipath_pio2kbase +
				       i * dd->ipath_palign);
	else
		buf = (u32 __iomem *)
			(dd->ipath_pio4kbase +
			 (i - dd->ipath_piobcnt2k) * dd->ipath_4kalign);
	ipath_cdbg(VERBOSE, "Return piobuf%u %uk @ %p\n",
		   i, (i < dd->ipath_piobcnt2k) ? 2 : 4, buf);
	if (pbufnum)
		*pbufnum = i;

bail:
	return buf;
}

/**
 * ipath_create_rcvhdrq - create a receive header queue
 * @dd: the infinipath device
 * @pd: the port data
 *
 * this *must* be physically contiguous memory, and for now,
 * that limits it to what kmalloc can do.
 */
int ipath_create_rcvhdrq(struct ipath_devdata *dd,
			 struct ipath_portdata *pd)
{
	int ret = 0, amt;

	amt = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
		    sizeof(u32), PAGE_SIZE);
	if (!pd->port_rcvhdrq) {
		/*
		 * not using REPEAT isn't viable; at 128KB, we can easily
		 * fail this.  The problem with REPEAT is we can block here
		 * "forever".  There isn't an inbetween, unfortunately.  We
		 * could reduce the risk by never freeing the rcvhdrq except
		 * at unload, but even then, the first time a port is used,
		 * we could delay for some time...
		 */
		gfp_t gfp_flags = GFP_USER | __GFP_COMP;

		pd->port_rcvhdrq = dma_alloc_coherent(
			&dd->pcidev->dev, amt, &pd->port_rcvhdrq_phys,
			gfp_flags);

		if (!pd->port_rcvhdrq) {
			ipath_dev_err(dd, "attempt to allocate %d bytes "
				      "for port %u rcvhdrq failed\n",
				      amt, pd->port_port);
			ret = -ENOMEM;
			goto bail;
		}

		pd->port_rcvhdrq_size = amt;

		ipath_cdbg(VERBOSE, "%d pages at %p (phys %lx) size=%lu "
			   "for port %u rcvhdr Q\n",
			   amt >> PAGE_SHIFT, pd->port_rcvhdrq,
			   (unsigned long) pd->port_rcvhdrq_phys,
			   (unsigned long) pd->port_rcvhdrq_size,
			   pd->port_port);
	} else {
		/*
		 * clear for security, sanity, and/or debugging, each
		 * time we reuse
		 */
		memset(pd->port_rcvhdrq, 0, amt);
	}

	/*
	 * tell chip each time we init it, even if we are re-using previous
	 * memory (we zero it at process close)
	 */
	ipath_cdbg(VERBOSE, "writing port %d rcvhdraddr as %lx\n",
		   pd->port_port, (unsigned long) pd->port_rcvhdrq_phys);
	ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
			      pd->port_port, pd->port_rcvhdrq_phys);

	ret = 0;
bail:
	return ret;
}

int ipath_waitfor_complete(struct ipath_devdata *dd, ipath_kreg reg_id,
			   u64 bits_to_wait_for, u64 * valp)
{
	unsigned long timeout;
	u64 lastval, val;
	int ret;

	lastval = ipath_read_kreg64(dd, reg_id);
	/* wait a ridiculously long time */
	timeout = jiffies + msecs_to_jiffies(5);
	do {
		val = ipath_read_kreg64(dd, reg_id);
		/* set so they have something, even on failures. */
		*valp = val;
		if ((val & bits_to_wait_for) == bits_to_wait_for) {
			ret = 0;
			break;
		}
		if (val != lastval)
			ipath_cdbg(VERBOSE, "Changed from %llx to %llx, "
				   "waiting for %llx bits\n",
				   (unsigned long long) lastval,
				   (unsigned long long) val,
				   (unsigned long long) bits_to_wait_for);
		cond_resched();
		if (time_after(jiffies, timeout)) {
			ipath_dbg("Didn't get bits %llx in register 0x%x, "
				  "got %llx\n",
				  (unsigned long long) bits_to_wait_for,
				  reg_id, (unsigned long long) *valp);
			ret = -ENODEV;
			break;
		}
	} while (1);

	return ret;
}

/**
 * ipath_waitfor_mdio_cmdready - wait for last command to complete
 * @dd: the infinipath device
 *
 * Like ipath_waitfor_complete(), but we wait for the CMDVALID bit to go
 * away indicating the last command has completed.  It doesn't return data
 */
int ipath_waitfor_mdio_cmdready(struct ipath_devdata *dd)
{
	unsigned long timeout;
	u64 val;
	int ret;

	/* wait a ridiculously long time */
	timeout = jiffies + msecs_to_jiffies(5);
	do {
		val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_mdio);
		if (!(val & IPATH_MDIO_CMDVALID)) {
			ret = 0;
			break;
		}
		cond_resched();
		if (time_after(jiffies, timeout)) {
			ipath_dbg("CMDVALID stuck in mdio reg? (%llx)\n",
				  (unsigned long long) val);