sge.c

来自「linux 内核源代码」· C语言 代码 · 共 2,293 行 · 第 1/5 页

	t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
}

static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
{
	refill_fl(adap, fl, min(16U, fl->size - fl->credits), GFP_ATOMIC);
}

/**
 *	recycle_rx_buf - recycle a receive buffer
 *	@adapter: the adapter
 *	@q: the SGE free list
 *	@idx: index of buffer to recycle
 *
 *	Recycles the specified buffer on the given free list by adding it at
 *	the next available slot on the list.
 */
static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q,
			   unsigned int idx)
{
	struct rx_desc *from = &q->desc[idx];
	struct rx_desc *to = &q->desc[q->pidx];

	q->sdesc[q->pidx] = q->sdesc[idx];
	to->addr_lo = from->addr_lo;	/* already big endian */
	to->addr_hi = from->addr_hi;	/* likewise */
	wmb();
	to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen));
	to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen));
	q->credits++;

	if (++q->pidx == q->size) {
		q->pidx = 0;
		q->gen ^= 1;
	}
	t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
}

/**
 *	alloc_ring - allocate resources for an SGE descriptor ring
 *	@pdev: the PCI device
 *	@nelem: the number of descriptors
 *	@elem_size: the size of each descriptor
 *	@sw_size: the size of the SW state associated with each ring element
 *	@phys: the physical address of the allocated ring
 *	@metadata: address of the array holding the SW state for the ring
 *
 *	Allocates resources for an SGE descriptor ring, such as Tx queues,
 *	free buffer lists, or response queues.  Each SGE ring requires
 *	space for its HW descriptors plus, optionally, space for the SW state
 *	associated with each HW entry (the metadata).  The function returns
 *	three values: the virtual address for the HW ring (the return value
 *	of the function), the physical address of the HW ring, and the address
 *	of the SW ring.
 */
static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size,
			size_t sw_size, dma_addr_t * phys, void *metadata)
{
	size_t len = nelem * elem_size;
	void *s = NULL;
	void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL);

	if (!p)
		return NULL;
	if (sw_size) {
		s = kcalloc(nelem, sw_size, GFP_KERNEL);

		if (!s) {
			dma_free_coherent(&pdev->dev, len, p, *phys);
			return NULL;
		}
	}
	if (metadata)
		*(void **)metadata = s;
	memset(p, 0, len);
	return p;
}

/**
 *	free_qset - free the resources of an SGE queue set
 *	@adapter: the adapter owning the queue set
 *	@q: the queue set
 *
 *	Release the HW and SW resources associated with an SGE queue set, such
 *	as HW contexts, packet buffers, and descriptor rings.  Traffic to the
 *	queue set must be quiesced prior to calling this.
 */
static void t3_free_qset(struct adapter *adapter, struct sge_qset *q)
{
	int i;
	struct pci_dev *pdev = adapter->pdev;

	if (q->tx_reclaim_timer.function)
		del_timer_sync(&q->tx_reclaim_timer);

	for (i = 0; i < SGE_RXQ_PER_SET; ++i)
		if (q->fl[i].desc) {
			spin_lock(&adapter->sge.reg_lock);
			t3_sge_disable_fl(adapter, q->fl[i].cntxt_id);
			spin_unlock(&adapter->sge.reg_lock);
			free_rx_bufs(pdev, &q->fl[i]);
			kfree(q->fl[i].sdesc);
			dma_free_coherent(&pdev->dev,
					  q->fl[i].size *
					  sizeof(struct rx_desc), q->fl[i].desc,
					  q->fl[i].phys_addr);
		}

	for (i = 0; i < SGE_TXQ_PER_SET; ++i)
		if (q->txq[i].desc) {
			spin_lock(&adapter->sge.reg_lock);
			t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0);
			spin_unlock(&adapter->sge.reg_lock);
			if (q->txq[i].sdesc) {
				free_tx_desc(adapter, &q->txq[i],
					     q->txq[i].in_use);
				kfree(q->txq[i].sdesc);
			}
			dma_free_coherent(&pdev->dev,
					  q->txq[i].size *
					  sizeof(struct tx_desc),
					  q->txq[i].desc, q->txq[i].phys_addr);
			__skb_queue_purge(&q->txq[i].sendq);
		}

	if (q->rspq.desc) {
		spin_lock(&adapter->sge.reg_lock);
		t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id);
		spin_unlock(&adapter->sge.reg_lock);
		dma_free_coherent(&pdev->dev,
				  q->rspq.size * sizeof(struct rsp_desc),
				  q->rspq.desc, q->rspq.phys_addr);
	}

	memset(q, 0, sizeof(*q));
}

/**
 *	init_qset_cntxt - initialize an SGE queue set context info
 *	@qs: the queue set
 *	@id: the queue set id
 *
 *	Initializes the TIDs and context ids for the queues of a queue set.
 */
static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)
{
	qs->rspq.cntxt_id = id;
	qs->fl[0].cntxt_id = 2 * id;
	qs->fl[1].cntxt_id = 2 * id + 1;
	qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
	qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
	qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
	qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
	qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
}

/**
 *	sgl_len - calculates the size of an SGL of the given capacity
 *	@n: the number of SGL entries
 *
 *	Calculates the number of flits needed for a scatter/gather list that
 *	can hold the given number of entries.
 */
static inline unsigned int sgl_len(unsigned int n)
{
	/* alternatively: 3 * (n / 2) + 2 * (n & 1) */
	return (3 * n) / 2 + (n & 1);
}

/**
 *	flits_to_desc - returns the num of Tx descriptors for the given flits
 *	@n: the number of flits
 *
 *	Calculates the number of Tx descriptors needed for the supplied number
 *	of flits.
 */
static inline unsigned int flits_to_desc(unsigned int n)
{
	BUG_ON(n >= ARRAY_SIZE(flit_desc_map));
	return flit_desc_map[n];
}

/**
 *	get_packet - return the next ingress packet buffer from a free list
 *	@adap: the adapter that received the packet
 *	@fl: the SGE free list holding the packet
 *	@len: the packet length including any SGE padding
 *	@drop_thres: # of remaining buffers before we start dropping packets
 *
 *	Get the next packet from a free list and complete setup of the
 *	sk_buff.  If the packet is small we make a copy and recycle the
 *	original buffer, otherwise we use the original buffer itself.  If a
 *	positive drop threshold is supplied packets are dropped and their
 *	buffers recycled if (a) the number of remaining buffers is under the
 *	threshold and the packet is too big to copy, or (b) the packet should
 *	be copied but there is no memory for the copy.
 */
static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl,
				  unsigned int len, unsigned int drop_thres)
{
	struct sk_buff *skb = NULL;
	struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];

	prefetch(sd->skb->data);
	fl->credits--;

	if (len <= SGE_RX_COPY_THRES) {
		skb = alloc_skb(len, GFP_ATOMIC);
		if (likely(skb != NULL)) {
			__skb_put(skb, len);
			pci_dma_sync_single_for_cpu(adap->pdev,
					    pci_unmap_addr(sd, dma_addr), len,
					    PCI_DMA_FROMDEVICE);
			memcpy(skb->data, sd->skb->data, len);
			pci_dma_sync_single_for_device(adap->pdev,
					    pci_unmap_addr(sd, dma_addr), len,
					    PCI_DMA_FROMDEVICE);
		} else if (!drop_thres)
			goto use_orig_buf;
recycle:
		recycle_rx_buf(adap, fl, fl->cidx);
		return skb;
	}

	if (unlikely(fl->credits < drop_thres))
		goto recycle;

use_orig_buf:
	pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr),
			 fl->buf_size, PCI_DMA_FROMDEVICE);
	skb = sd->skb;
	skb_put(skb, len);
	__refill_fl(adap, fl);
	return skb;
}

/**
 *	get_packet_pg - return the next ingress packet buffer from a free list
 *	@adap: the adapter that received the packet
 *	@fl: the SGE free list holding the packet
 *	@len: the packet length including any SGE padding
 *	@drop_thres: # of remaining buffers before we start dropping packets
 *
 *	Get the next packet from a free list populated with page chunks.
 *	If the packet is small we make a copy and recycle the original buffer,
 *	otherwise we attach the original buffer as a page fragment to a fresh
 *	sk_buff.  If a positive drop threshold is supplied packets are dropped
 *	and their buffers recycled if (a) the number of remaining buffers is
 *	under the threshold and the packet is too big to copy, or (b) there's
 *	no system memory.
 *
 * 	Note: this function is similar to @get_packet but deals with Rx buffers
 * 	that are page chunks rather than sk_buffs.
 */
static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
				     unsigned int len, unsigned int drop_thres)
{
	struct sk_buff *skb = NULL;
	struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];

	if (len <= SGE_RX_COPY_THRES) {
		skb = alloc_skb(len, GFP_ATOMIC);
		if (likely(skb != NULL)) {
			__skb_put(skb, len);
			pci_dma_sync_single_for_cpu(adap->pdev,
					    pci_unmap_addr(sd, dma_addr), len,
					    PCI_DMA_FROMDEVICE);
			memcpy(skb->data, sd->pg_chunk.va, len);
			pci_dma_sync_single_for_device(adap->pdev,
					    pci_unmap_addr(sd, dma_addr), len,
					    PCI_DMA_FROMDEVICE);
		} else if (!drop_thres)
			return NULL;
recycle:
		fl->credits--;
		recycle_rx_buf(adap, fl, fl->cidx);
		return skb;
	}

	if (unlikely(fl->credits <= drop_thres))
		goto recycle;

	skb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
	if (unlikely(!skb)) {
		if (!drop_thres)
			return NULL;
		goto recycle;
	}

	pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr),
			 fl->buf_size, PCI_DMA_FROMDEVICE);
	__skb_put(skb, SGE_RX_PULL_LEN);
	memcpy(skb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
	skb_fill_page_desc(skb, 0, sd->pg_chunk.page,
			   sd->pg_chunk.offset + SGE_RX_PULL_LEN,
			   len - SGE_RX_PULL_LEN);
	skb->len = len;
	skb->data_len = len - SGE_RX_PULL_LEN;
	skb->truesize += skb->data_len;

	fl->credits--;
	/*
	 * We do not refill FLs here, we let the caller do it to overlap a
	 * prefetch.
	 */
	return skb;
}

/**
 *	get_imm_packet - return the next ingress packet buffer from a response
 *	@resp: the response descriptor containing the packet data
 *
 *	Return a packet containing the immediate data of the given response.
 */
static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp)
{
	struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC);

	if (skb) {
		__skb_put(skb, IMMED_PKT_SIZE);
		skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE);
	}
	return skb;
}

/**
 *	calc_tx_descs - calculate the number of Tx descriptors for a packet
 *	@skb: the packet
 *
 * 	Returns the number of Tx descriptors needed for the given Ethernet
 * 	packet.  Ethernet packets require addition of WR and CPL headers.
 */
static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
{
	unsigned int flits;

	if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt))
		return 1;

	flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2;
	if (skb_shinfo(skb)->gso_size)
		flits++;
	return flits_to_desc(flits);
}

/**
 *	make_sgl - populate a scatter/gather list for a packet
 *	@skb: the packet
 *	@sgp: the SGL to populate
 *	@start: start address of skb main body data to include in the SGL
 *	@len: length of skb main body data to include in the SGL
 *	@pdev: the PCI device
 *
 *	Generates a scatter/gather list for the buffers that make up a packet
 *	and returns the SGL size in 8-byte words.  The caller must size the SGL
 *	appropriately.
 */
static inline unsigned int make_sgl(const struct sk_buff *skb,
				    struct sg_ent *sgp, unsigned char *start,
				    unsigned int len, struct pci_dev *pdev)
{
	dma_addr_t mapping;
	unsigned int i, j = 0, nfrags;

	if (len) {
		mapping = pci_map_single(pdev, start, len, PCI_DMA_TODEVICE);
		sgp->len[0] = cpu_to_be32(len);
		sgp->addr[0] = cpu_to_be64(mapping);
		j = 1;
	}

	nfrags = skb_shinfo(skb)->nr_frags;
	for (i = 0; i < nfrags; i++) {
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];

		mapping = pci_map_page(pdev, frag->page, frag->page_offset,
				       frag->size, PCI_DMA_TODEVICE);
		sgp->len[j] = cpu_to_be32(frag->size);
		sgp->addr[j] = cpu_to_be64(mapping);
		j ^= 1;
		if (j == 0)
			++sgp;
	}
	if (j)
		sgp->len[j] = 0;
	return ((nfrags + (len != 0)) * 3) / 2 + j;
}

/**
 *	check_ring_tx_db - check and potentially ring a Tx queue's doorbell
 *	@adap: the adapter
 *	@q: the Tx queue
 *
 *	Ring the doorbel if a Tx queue is asleep.  There is a natural race,
 *	where the HW is going to sleep just after we checked, however,
 *	then the interrupt handler will detect the outstanding TX packet
 *	and ring the doorbell for us.
 *
 *	When GTS is disabled we unconditionally ring the doorbell.
 */
static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q)
{
#if USE_GTS
	clear_bit(TXQ_LAST_PKT_DB, &q->flags);
	if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
		set_bit(TXQ_LAST_PKT_DB, &q->flags);
		t3_write_reg(adap, A_SG_KDOORBELL,
			     F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
	}
#else
	wmb();			/* write descriptors before telling HW */
	t3_write_reg(adap, A_SG_KDOORBELL,
		     F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
#endif
}

static inline void wr_gen2(struct tx_desc *d, unsigned int gen)
{
#if SGE_NUM_GENBITS == 2
	d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen);
#endif
}

/**
 *	write_wr_hdr_sgl - write a WR header and, optionally, SGL
 *	@ndesc: number of Tx descriptors spanned by the SGL
 *	@skb: the packet corresponding to the WR
 *	@d: first Tx descriptor to be written
 *	@pidx: index of above descriptors
 *	@q: the SGE Tx queue
 *	@sgl: the SGL
 *	@flits: number of flits to the start of the SGL in the first descriptor
 *	@sgl_flits: the SGL size in flits
 *	@gen: the Tx descriptor generation
 *	@wr_hi: top 32 bits of WR header based on WR type (big endian)
 *	@wr_lo: low 32 bits of WR header based on WR type (big endian)
 *
 *	Write a work request header and an associated SGL.  If the SGL is
 *	small enough to fit into one Tx descriptor it has already been written
 *	and we just need to write the WR header.  Otherwise we distribute the
 *	SGL across the number of descriptors it spans.
 */
static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb,
			     struct tx_desc *d, unsigned int pidx,
			     const struct sge_txq *q,
			     const struct sg_ent *sgl,
			     unsigned int flits, unsigned int sgl_flits,
			     unsigned int gen, __be32 wr_hi,
			     __be32 wr_lo)
{
	struct work_request_hdr *wrp = (struct work_request_hdr *)d;
	struct tx_sw_desc *sd = &q->sdesc[pidx];

	sd->skb = skb;
	if (need_skb_unmap()) {
		struct unmap_info *ui = (struct unmap_info *)skb->cb;

		ui->fragidx = 0;
		ui->addr_idx = 0;