sge.c

来自「linux 内核源代码」· C语言 代码 · 共 2,209 行 · 第 1/5 页

	if (!skb) {
		if (!s->num)
			return NULL;
	} else {
		skbq = &s->p[skb->dev->if_port].skbq;
		__skb_queue_tail(skbq, skb);
		s->num++;
		skb = NULL;
	}

	if (credits < MAX_SKB_FRAGS + 1)
		goto out;

again:
	for (i = 0; i < MAX_NPORTS; i++) {
		s->port = ++s->port & (MAX_NPORTS - 1);
		skbq = &s->p[s->port].skbq;

		skb = skb_peek(skbq);

		if (!skb)
			continue;

		len = skb->len;
		if (len <= s->p[s->port].avail) {
			s->p[s->port].avail -= len;
			s->num--;
			__skb_unlink(skb, skbq);
			goto out;
		}
		skb = NULL;
	}

	if (update-- && sched_update_avail(sge))
		goto again;

out:
	/* If there are more pending skbs, we use the hardware to schedule us
	 * again.
	 */
	if (s->num && !skb) {
		struct cmdQ *q = &sge->cmdQ[0];
		clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
		if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
			set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
			writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL);
		}
	}
	pr_debug("sched_skb ret %p\n", skb);

	return skb;
}

/*
 * PIO to indicate that memory mapped Q contains valid descriptor(s).
 */
static inline void doorbell_pio(struct adapter *adapter, u32 val)
{
	wmb();
	writel(val, adapter->regs + A_SG_DOORBELL);
}

/*
 * Frees all RX buffers on the freelist Q. The caller must make sure that
 * the SGE is turned off before calling this function.
 */
static void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *q)
{
	unsigned int cidx = q->cidx;

	while (q->credits--) {
		struct freelQ_ce *ce = &q->centries[cidx];

		pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
				 pci_unmap_len(ce, dma_len),
				 PCI_DMA_FROMDEVICE);
		dev_kfree_skb(ce->skb);
		ce->skb = NULL;
		if (++cidx == q->size)
			cidx = 0;
	}
}

/*
 * Free RX free list and response queue resources.
 */
static void free_rx_resources(struct sge *sge)
{
	struct pci_dev *pdev = sge->adapter->pdev;
	unsigned int size, i;

	if (sge->respQ.entries) {
		size = sizeof(struct respQ_e) * sge->respQ.size;
		pci_free_consistent(pdev, size, sge->respQ.entries,
				    sge->respQ.dma_addr);
	}

	for (i = 0; i < SGE_FREELQ_N; i++) {
		struct freelQ *q = &sge->freelQ[i];

		if (q->centries) {
			free_freelQ_buffers(pdev, q);
			kfree(q->centries);
		}
		if (q->entries) {
			size = sizeof(struct freelQ_e) * q->size;
			pci_free_consistent(pdev, size, q->entries,
					    q->dma_addr);
		}
	}
}

/*
 * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
 * response queue.
 */
static int alloc_rx_resources(struct sge *sge, struct sge_params *p)
{
	struct pci_dev *pdev = sge->adapter->pdev;
	unsigned int size, i;

	for (i = 0; i < SGE_FREELQ_N; i++) {
		struct freelQ *q = &sge->freelQ[i];

		q->genbit = 1;
		q->size = p->freelQ_size[i];
		q->dma_offset = sge->rx_pkt_pad ? 0 : NET_IP_ALIGN;
		size = sizeof(struct freelQ_e) * q->size;
		q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);
		if (!q->entries)
			goto err_no_mem;

		size = sizeof(struct freelQ_ce) * q->size;
		q->centries = kzalloc(size, GFP_KERNEL);
		if (!q->centries)
			goto err_no_mem;
	}

	/*
	 * Calculate the buffer sizes for the two free lists.  FL0 accommodates
	 * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
	 * including all the sk_buff overhead.
	 *
	 * Note: For T2 FL0 and FL1 are reversed.
	 */
	sge->freelQ[!sge->jumbo_fl].rx_buffer_size = SGE_RX_SM_BUF_SIZE +
		sizeof(struct cpl_rx_data) +
		sge->freelQ[!sge->jumbo_fl].dma_offset;

		size = (16 * 1024) -
		    SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

	sge->freelQ[sge->jumbo_fl].rx_buffer_size = size;

	/*
	 * Setup which skb recycle Q should be used when recycling buffers from
	 * each free list.
	 */
	sge->freelQ[!sge->jumbo_fl].recycleq_idx = 0;
	sge->freelQ[sge->jumbo_fl].recycleq_idx = 1;

	sge->respQ.genbit = 1;
	sge->respQ.size = SGE_RESPQ_E_N;
	sge->respQ.credits = 0;
	size = sizeof(struct respQ_e) * sge->respQ.size;
	sge->respQ.entries =
		pci_alloc_consistent(pdev, size, &sge->respQ.dma_addr);
	if (!sge->respQ.entries)
		goto err_no_mem;
	return 0;

err_no_mem:
	free_rx_resources(sge);
	return -ENOMEM;
}

/*
 * Reclaims n TX descriptors and frees the buffers associated with them.
 */
static void free_cmdQ_buffers(struct sge *sge, struct cmdQ *q, unsigned int n)
{
	struct cmdQ_ce *ce;
	struct pci_dev *pdev = sge->adapter->pdev;
	unsigned int cidx = q->cidx;

	q->in_use -= n;
	ce = &q->centries[cidx];
	while (n--) {
		if (likely(pci_unmap_len(ce, dma_len))) {
			pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
					 pci_unmap_len(ce, dma_len),
					 PCI_DMA_TODEVICE);
			if (q->sop)
				q->sop = 0;
		}
		if (ce->skb) {
			dev_kfree_skb_any(ce->skb);
			q->sop = 1;
		}
		ce++;
		if (++cidx == q->size) {
			cidx = 0;
			ce = q->centries;
		}
	}
	q->cidx = cidx;
}

/*
 * Free TX resources.
 *
 * Assumes that SGE is stopped and all interrupts are disabled.
 */
static void free_tx_resources(struct sge *sge)
{
	struct pci_dev *pdev = sge->adapter->pdev;
	unsigned int size, i;

	for (i = 0; i < SGE_CMDQ_N; i++) {
		struct cmdQ *q = &sge->cmdQ[i];

		if (q->centries) {
			if (q->in_use)
				free_cmdQ_buffers(sge, q, q->in_use);
			kfree(q->centries);
		}
		if (q->entries) {
			size = sizeof(struct cmdQ_e) * q->size;
			pci_free_consistent(pdev, size, q->entries,
					    q->dma_addr);
		}
	}
}

/*
 * Allocates basic TX resources, consisting of memory mapped command Qs.
 */
static int alloc_tx_resources(struct sge *sge, struct sge_params *p)
{
	struct pci_dev *pdev = sge->adapter->pdev;
	unsigned int size, i;

	for (i = 0; i < SGE_CMDQ_N; i++) {
		struct cmdQ *q = &sge->cmdQ[i];

		q->genbit = 1;
		q->sop = 1;
		q->size = p->cmdQ_size[i];
		q->in_use = 0;
		q->status = 0;
		q->processed = q->cleaned = 0;
		q->stop_thres = 0;
		spin_lock_init(&q->lock);
		size = sizeof(struct cmdQ_e) * q->size;
		q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);
		if (!q->entries)
			goto err_no_mem;

		size = sizeof(struct cmdQ_ce) * q->size;
		q->centries = kzalloc(size, GFP_KERNEL);
		if (!q->centries)
			goto err_no_mem;
	}

	/*
	 * CommandQ 0 handles Ethernet and TOE packets, while queue 1 is TOE
	 * only.  For queue 0 set the stop threshold so we can handle one more
	 * packet from each port, plus reserve an additional 24 entries for
	 * Ethernet packets only.  Queue 1 never suspends nor do we reserve
	 * space for Ethernet packets.
	 */
	sge->cmdQ[0].stop_thres = sge->adapter->params.nports *
		(MAX_SKB_FRAGS + 1);
	return 0;

err_no_mem:
	free_tx_resources(sge);
	return -ENOMEM;
}

static inline void setup_ring_params(struct adapter *adapter, u64 addr,
				     u32 size, int base_reg_lo,
				     int base_reg_hi, int size_reg)
{
	writel((u32)addr, adapter->regs + base_reg_lo);
	writel(addr >> 32, adapter->regs + base_reg_hi);
	writel(size, adapter->regs + size_reg);
}

/*
 * Enable/disable VLAN acceleration.
 */
void t1_set_vlan_accel(struct adapter *adapter, int on_off)
{
	struct sge *sge = adapter->sge;

	sge->sge_control &= ~F_VLAN_XTRACT;
	if (on_off)
		sge->sge_control |= F_VLAN_XTRACT;
	if (adapter->open_device_map) {
		writel(sge->sge_control, adapter->regs + A_SG_CONTROL);
		readl(adapter->regs + A_SG_CONTROL);   /* flush */
	}
}

/*
 * Programs the various SGE registers. However, the engine is not yet enabled,
 * but sge->sge_control is setup and ready to go.
 */
static void configure_sge(struct sge *sge, struct sge_params *p)
{
	struct adapter *ap = sge->adapter;

	writel(0, ap->regs + A_SG_CONTROL);
	setup_ring_params(ap, sge->cmdQ[0].dma_addr, sge->cmdQ[0].size,
			  A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
	setup_ring_params(ap, sge->cmdQ[1].dma_addr, sge->cmdQ[1].size,
			  A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
	setup_ring_params(ap, sge->freelQ[0].dma_addr,
			  sge->freelQ[0].size, A_SG_FL0BASELWR,
			  A_SG_FL0BASEUPR, A_SG_FL0SIZE);
	setup_ring_params(ap, sge->freelQ[1].dma_addr,
			  sge->freelQ[1].size, A_SG_FL1BASELWR,
			  A_SG_FL1BASEUPR, A_SG_FL1SIZE);

	/* The threshold comparison uses <. */
	writel(SGE_RX_SM_BUF_SIZE + 1, ap->regs + A_SG_FLTHRESHOLD);

	setup_ring_params(ap, sge->respQ.dma_addr, sge->respQ.size,
			  A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
	writel((u32)sge->respQ.size - 1, ap->regs + A_SG_RSPQUEUECREDIT);

	sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
		F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
		V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
		V_RX_PKT_OFFSET(sge->rx_pkt_pad);

#if defined(__BIG_ENDIAN_BITFIELD)
	sge->sge_control |= F_ENABLE_BIG_ENDIAN;
#endif

	/* Initialize no-resource timer */
	sge->intrtimer_nres = SGE_INTRTIMER_NRES * core_ticks_per_usec(ap);

	t1_sge_set_coalesce_params(sge, p);
}

/*
 * Return the payload capacity of the jumbo free-list buffers.
 */
static inline unsigned int jumbo_payload_capacity(const struct sge *sge)
{
	return sge->freelQ[sge->jumbo_fl].rx_buffer_size -
		sge->freelQ[sge->jumbo_fl].dma_offset -
		sizeof(struct cpl_rx_data);
}

/*
 * Frees all SGE related resources and the sge structure itself
 */
void t1_sge_destroy(struct sge *sge)
{
	int i;

	for_each_port(sge->adapter, i)
		free_percpu(sge->port_stats[i]);

	kfree(sge->tx_sched);
	free_tx_resources(sge);
	free_rx_resources(sge);
	kfree(sge);
}

/*
 * Allocates new RX buffers on the freelist Q (and tracks them on the freelist
 * context Q) until the Q is full or alloc_skb fails.
 *
 * It is possible that the generation bits already match, indicating that the
 * buffer is already valid and nothing needs to be done. This happens when we
 * copied a received buffer into a new sk_buff during the interrupt processing.
 *
 * If the SGE doesn't automatically align packets properly (!sge->rx_pkt_pad),
 * we specify a RX_OFFSET in order to make sure that the IP header is 4B
 * aligned.
 */
static void refill_free_list(struct sge *sge, struct freelQ *q)
{
	struct pci_dev *pdev = sge->adapter->pdev;
	struct freelQ_ce *ce = &q->centries[q->pidx];
	struct freelQ_e *e = &q->entries[q->pidx];
	unsigned int dma_len = q->rx_buffer_size - q->dma_offset;

	while (q->credits < q->size) {
		struct sk_buff *skb;
		dma_addr_t mapping;

		skb = alloc_skb(q->rx_buffer_size, GFP_ATOMIC);
		if (!skb)
			break;

		skb_reserve(skb, q->dma_offset);
		mapping = pci_map_single(pdev, skb->data, dma_len,
					 PCI_DMA_FROMDEVICE);
		skb_reserve(skb, sge->rx_pkt_pad);

		ce->skb = skb;
		pci_unmap_addr_set(ce, dma_addr, mapping);
		pci_unmap_len_set(ce, dma_len, dma_len);
		e->addr_lo = (u32)mapping;
		e->addr_hi = (u64)mapping >> 32;
		e->len_gen = V_CMD_LEN(dma_len) | V_CMD_GEN1(q->genbit);
		wmb();
		e->gen2 = V_CMD_GEN2(q->genbit);

		e++;
		ce++;
		if (++q->pidx == q->size) {
			q->pidx = 0;
			q->genbit ^= 1;
			ce = q->centries;
			e = q->entries;
		}
		q->credits++;
	}
}

/*
 * Calls refill_free_list for both free lists. If we cannot fill at least 1/4
 * of both rings, we go into 'few interrupt mode' in order to give the system
 * time to free up resources.
 */
static void freelQs_empty(struct sge *sge)
{
	struct adapter *adapter = sge->adapter;
	u32 irq_reg = readl(adapter->regs + A_SG_INT_ENABLE);
	u32 irqholdoff_reg;

	refill_free_list(sge, &sge->freelQ[0]);
	refill_free_list(sge, &sge->freelQ[1]);

	if (sge->freelQ[0].credits > (sge->freelQ[0].size >> 2) &&
	    sge->freelQ[1].credits > (sge->freelQ[1].size >> 2)) {