ehea_main.c

来自「linux 内核源代码」· C语言 代码 · 共 2,668 行 · 第 1/5 页

/*
 *  linux/drivers/net/ehea/ehea_main.c
 *
 *  eHEA ethernet device driver for IBM eServer System p
 *
 *  (C) Copyright IBM Corp. 2006
 *
 *  Authors:
 *       Christoph Raisch <[e-mail]>
 *       Jan-Bernd Themann <[e-mail]>
 *       Thomas Klein <[e-mail]>
 *
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include <linux/in.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/if.h>
#include <linux/list.h>
#include <linux/if_ether.h>
#include <linux/notifier.h>
#include <linux/reboot.h>

#include <net/ip.h>

#include "ehea.h"
#include "ehea_qmr.h"
#include "ehea_phyp.h"


MODULE_LICENSE("GPL");
MODULE_AUTHOR("Christoph Raisch <[e-mail]>");
MODULE_DESCRIPTION("IBM eServer HEA Driver");
MODULE_VERSION(DRV_VERSION);


static int msg_level = -1;
static int rq1_entries = EHEA_DEF_ENTRIES_RQ1;
static int rq2_entries = EHEA_DEF_ENTRIES_RQ2;
static int rq3_entries = EHEA_DEF_ENTRIES_RQ3;
static int sq_entries = EHEA_DEF_ENTRIES_SQ;
static int use_mcs = 0;
static int use_lro = 0;
static int lro_max_aggr = EHEA_LRO_MAX_AGGR;
static int num_tx_qps = EHEA_NUM_TX_QP;
static int prop_carrier_state = 0;

module_param(msg_level, int, 0);
module_param(rq1_entries, int, 0);
module_param(rq2_entries, int, 0);
module_param(rq3_entries, int, 0);
module_param(sq_entries, int, 0);
module_param(prop_carrier_state, int, 0);
module_param(use_mcs, int, 0);
module_param(use_lro, int, 0);
module_param(lro_max_aggr, int, 0);
module_param(num_tx_qps, int, 0);

MODULE_PARM_DESC(num_tx_qps, "Number of TX-QPS");
MODULE_PARM_DESC(msg_level, "msg_level");
MODULE_PARM_DESC(prop_carrier_state, "Propagate carrier state of physical "
		 "port to stack. 1:yes, 0:no.  Default = 0 ");
MODULE_PARM_DESC(rq3_entries, "Number of entries for Receive Queue 3 "
		 "[2^x - 1], x = [6..14]. Default = "
		 __MODULE_STRING(EHEA_DEF_ENTRIES_RQ3) ")");
MODULE_PARM_DESC(rq2_entries, "Number of entries for Receive Queue 2 "
		 "[2^x - 1], x = [6..14]. Default = "
		 __MODULE_STRING(EHEA_DEF_ENTRIES_RQ2) ")");
MODULE_PARM_DESC(rq1_entries, "Number of entries for Receive Queue 1 "
		 "[2^x - 1], x = [6..14]. Default = "
		 __MODULE_STRING(EHEA_DEF_ENTRIES_RQ1) ")");
MODULE_PARM_DESC(sq_entries, " Number of entries for the Send Queue  "
		 "[2^x - 1], x = [6..14]. Default = "
		 __MODULE_STRING(EHEA_DEF_ENTRIES_SQ) ")");
MODULE_PARM_DESC(use_mcs, " 0:NAPI, 1:Multiple receive queues, Default = 0 ");

MODULE_PARM_DESC(lro_max_aggr, " LRO: Max packets to be aggregated. Default = "
		 __MODULE_STRING(EHEA_LRO_MAX_AGGR));
MODULE_PARM_DESC(use_lro, " Large Receive Offload, 1: enable, 0: disable, "
		 "Default = 0");

static int port_name_cnt = 0;
static LIST_HEAD(adapter_list);
u64 ehea_driver_flags = 0;
struct work_struct ehea_rereg_mr_task;

struct semaphore dlpar_mem_lock;

static int __devinit ehea_probe_adapter(struct of_device *dev,
					const struct of_device_id *id);

static int __devexit ehea_remove(struct of_device *dev);

static struct of_device_id ehea_device_table[] = {
	{
		.name = "lhea",
		.compatible = "IBM,lhea",
	},
	{},
};

static struct of_platform_driver ehea_driver = {
	.name = "ehea",
	.match_table = ehea_device_table,
	.probe = ehea_probe_adapter,
	.remove = ehea_remove,
};

void ehea_dump(void *adr, int len, char *msg) {
	int x;
	unsigned char *deb = adr;
	for (x = 0; x < len; x += 16) {
		printk(DRV_NAME " %s adr=%p ofs=%04x %016lx %016lx\n", msg,
			  deb, x, *((u64*)&deb[0]), *((u64*)&deb[8]));
		deb += 16;
	}
}

static struct net_device_stats *ehea_get_stats(struct net_device *dev)
{
	struct ehea_port *port = netdev_priv(dev);
	struct net_device_stats *stats = &port->stats;
	struct hcp_ehea_port_cb2 *cb2;
	u64 hret, rx_packets, tx_packets;
	int i;

	memset(stats, 0, sizeof(*stats));

	cb2 = kzalloc(PAGE_SIZE, GFP_KERNEL);
	if (!cb2) {
		ehea_error("no mem for cb2");
		goto out;
	}

	hret = ehea_h_query_ehea_port(port->adapter->handle,
				      port->logical_port_id,
				      H_PORT_CB2, H_PORT_CB2_ALL, cb2);
	if (hret != H_SUCCESS) {
		ehea_error("query_ehea_port failed");
		goto out_herr;
	}

	if (netif_msg_hw(port))
		ehea_dump(cb2, sizeof(*cb2), "net_device_stats");

	rx_packets = 0;
	for (i = 0; i < port->num_def_qps; i++)
		rx_packets += port->port_res[i].rx_packets;

	tx_packets = 0;
	for (i = 0; i < port->num_def_qps + port->num_add_tx_qps; i++)
		tx_packets += port->port_res[i].tx_packets;

	stats->tx_packets = tx_packets;
	stats->multicast = cb2->rxmcp;
	stats->rx_errors = cb2->rxuerr;
	stats->rx_bytes = cb2->rxo;
	stats->tx_bytes = cb2->txo;
	stats->rx_packets = rx_packets;

out_herr:
	kfree(cb2);
out:
	return stats;
}

static void ehea_refill_rq1(struct ehea_port_res *pr, int index, int nr_of_wqes)
{
	struct sk_buff **skb_arr_rq1 = pr->rq1_skba.arr;
	struct net_device *dev = pr->port->netdev;
	int max_index_mask = pr->rq1_skba.len - 1;
	int fill_wqes = pr->rq1_skba.os_skbs + nr_of_wqes;
	int adder = 0;
	int i;

	pr->rq1_skba.os_skbs = 0;

	if (unlikely(test_bit(__EHEA_STOP_XFER, &ehea_driver_flags))) {
		pr->rq1_skba.index = index;
		pr->rq1_skba.os_skbs = fill_wqes;
		return;
	}

	for (i = 0; i < fill_wqes; i++) {
		if (!skb_arr_rq1[index]) {
			skb_arr_rq1[index] = netdev_alloc_skb(dev,
							      EHEA_L_PKT_SIZE);
			if (!skb_arr_rq1[index]) {
				pr->rq1_skba.os_skbs = fill_wqes - i;
				ehea_error("%s: no mem for skb/%d wqes filled",
					   dev->name, i);
				break;
			}
		}
		index--;
		index &= max_index_mask;
		adder++;
	}

	if (adder == 0)
		return;

	/* Ring doorbell */
	ehea_update_rq1a(pr->qp, adder);
}

static int ehea_init_fill_rq1(struct ehea_port_res *pr, int nr_rq1a)
{
	int ret = 0;
	struct sk_buff **skb_arr_rq1 = pr->rq1_skba.arr;
	struct net_device *dev = pr->port->netdev;
	int i;

	for (i = 0; i < pr->rq1_skba.len; i++) {
		skb_arr_rq1[i] = netdev_alloc_skb(dev, EHEA_L_PKT_SIZE);
		if (!skb_arr_rq1[i]) {
			ehea_error("%s: no mem for skb/%d wqes filled",
				   dev->name, i);
			ret = -ENOMEM;
			goto out;
		}
	}
	/* Ring doorbell */
	ehea_update_rq1a(pr->qp, nr_rq1a);
out:
	return ret;
}

static int ehea_refill_rq_def(struct ehea_port_res *pr,
			      struct ehea_q_skb_arr *q_skba, int rq_nr,
			      int num_wqes, int wqe_type, int packet_size)
{
	struct net_device *dev = pr->port->netdev;
	struct ehea_qp *qp = pr->qp;
	struct sk_buff **skb_arr = q_skba->arr;
	struct ehea_rwqe *rwqe;
	int i, index, max_index_mask, fill_wqes;
	int adder = 0;
	int ret = 0;

	fill_wqes = q_skba->os_skbs + num_wqes;
	q_skba->os_skbs = 0;

	if (unlikely(test_bit(__EHEA_STOP_XFER, &ehea_driver_flags))) {
		q_skba->os_skbs = fill_wqes;
		return ret;
	}

	index = q_skba->index;
	max_index_mask = q_skba->len - 1;
	for (i = 0; i < fill_wqes; i++) {
		u64 tmp_addr;
		struct sk_buff *skb = netdev_alloc_skb(dev, packet_size);
		if (!skb) {
			ehea_error("%s: no mem for skb/%d wqes filled",
				   pr->port->netdev->name, i);
			q_skba->os_skbs = fill_wqes - i;
			ret = -ENOMEM;
			break;
		}
		skb_reserve(skb, NET_IP_ALIGN);

		skb_arr[index] = skb;
		tmp_addr = ehea_map_vaddr(skb->data);
		if (tmp_addr == -1) {
			dev_kfree_skb(skb);
			q_skba->os_skbs = fill_wqes - i;
			ret = 0;
			break;
		}

		rwqe = ehea_get_next_rwqe(qp, rq_nr);
		rwqe->wr_id = EHEA_BMASK_SET(EHEA_WR_ID_TYPE, wqe_type)
			    | EHEA_BMASK_SET(EHEA_WR_ID_INDEX, index);
		rwqe->sg_list[0].l_key = pr->recv_mr.lkey;
		rwqe->sg_list[0].vaddr = tmp_addr;
		rwqe->sg_list[0].len = packet_size;
		rwqe->data_segments = 1;

		index++;
		index &= max_index_mask;
		adder++;
	}

	q_skba->index = index;
	if (adder == 0)
		goto out;

	/* Ring doorbell */
	iosync();
	if (rq_nr == 2)
		ehea_update_rq2a(pr->qp, adder);
	else
		ehea_update_rq3a(pr->qp, adder);
out:
	return ret;
}


static int ehea_refill_rq2(struct ehea_port_res *pr, int nr_of_wqes)
{
	return ehea_refill_rq_def(pr, &pr->rq2_skba, 2,
				  nr_of_wqes, EHEA_RWQE2_TYPE,
				  EHEA_RQ2_PKT_SIZE + NET_IP_ALIGN);
}


static int ehea_refill_rq3(struct ehea_port_res *pr, int nr_of_wqes)
{
	return ehea_refill_rq_def(pr, &pr->rq3_skba, 3,
				  nr_of_wqes, EHEA_RWQE3_TYPE,
				  EHEA_MAX_PACKET_SIZE + NET_IP_ALIGN);
}

static inline int ehea_check_cqe(struct ehea_cqe *cqe, int *rq_num)
{
	*rq_num = (cqe->type & EHEA_CQE_TYPE_RQ) >> 5;
	if ((cqe->status & EHEA_CQE_STAT_ERR_MASK) == 0)
		return 0;
	if (((cqe->status & EHEA_CQE_STAT_ERR_TCP) != 0) &&
	    (cqe->header_length == 0))
		return 0;
	return -EINVAL;
}

static inline void ehea_fill_skb(struct net_device *dev,
				 struct sk_buff *skb, struct ehea_cqe *cqe)
{
	int length = cqe->num_bytes_transfered - 4;	/*remove CRC */

	skb_put(skb, length);
	skb->ip_summed = CHECKSUM_UNNECESSARY;
	skb->protocol = eth_type_trans(skb, dev);
}

static inline struct sk_buff *get_skb_by_index(struct sk_buff **skb_array,
					       int arr_len,
					       struct ehea_cqe *cqe)
{
	int skb_index = EHEA_BMASK_GET(EHEA_WR_ID_INDEX, cqe->wr_id);
	struct sk_buff *skb;
	void *pref;
	int x;

	x = skb_index + 1;
	x &= (arr_len - 1);

	pref = skb_array[x];
	prefetchw(pref);
	prefetchw(pref + EHEA_CACHE_LINE);

	pref = (skb_array[x]->data);
	prefetch(pref);
	prefetch(pref + EHEA_CACHE_LINE);
	prefetch(pref + EHEA_CACHE_LINE * 2);
	prefetch(pref + EHEA_CACHE_LINE * 3);
	skb = skb_array[skb_index];
	skb_array[skb_index] = NULL;
	return skb;
}

static inline struct sk_buff *get_skb_by_index_ll(struct sk_buff **skb_array,
						  int arr_len, int wqe_index)
{
	struct sk_buff *skb;
	void *pref;
	int x;

	x = wqe_index + 1;
	x &= (arr_len - 1);

	pref = skb_array[x];
	prefetchw(pref);
	prefetchw(pref + EHEA_CACHE_LINE);

	pref = (skb_array[x]->data);
	prefetchw(pref);
	prefetchw(pref + EHEA_CACHE_LINE);

	skb = skb_array[wqe_index];
	skb_array[wqe_index] = NULL;
	return skb;
}

static int ehea_treat_poll_error(struct ehea_port_res *pr, int rq,
				 struct ehea_cqe *cqe, int *processed_rq2,
				 int *processed_rq3)
{
	struct sk_buff *skb;

	if (cqe->status & EHEA_CQE_STAT_ERR_TCP)
		pr->p_stats.err_tcp_cksum++;
	if (cqe->status & EHEA_CQE_STAT_ERR_IP)
		pr->p_stats.err_ip_cksum++;
	if (cqe->status & EHEA_CQE_STAT_ERR_CRC)
		pr->p_stats.err_frame_crc++;

	if (rq == 2) {
		*processed_rq2 += 1;
		skb = get_skb_by_index(pr->rq2_skba.arr, pr->rq2_skba.len, cqe);
		dev_kfree_skb(skb);
	} else if (rq == 3) {
		*processed_rq3 += 1;
		skb = get_skb_by_index(pr->rq3_skba.arr, pr->rq3_skba.len, cqe);
		dev_kfree_skb(skb);
	}

	if (cqe->status & EHEA_CQE_STAT_FAT_ERR_MASK) {
		if (netif_msg_rx_err(pr->port)) {
			ehea_error("Critical receive error for QP %d. "
				   "Resetting port.", pr->qp->init_attr.qp_nr);
			ehea_dump(cqe, sizeof(*cqe), "CQE");
		}
		schedule_work(&pr->port->reset_task);
		return 1;
	}

	return 0;
}

static int get_skb_hdr(struct sk_buff *skb, void **iphdr,
		       void **tcph, u64 *hdr_flags, void *priv)
{
	struct ehea_cqe *cqe = priv;
	unsigned int ip_len;
	struct iphdr *iph;

	/* non tcp/udp packets */
	if (!cqe->header_length)
		return -1;

	/* non tcp packet */
	skb_reset_network_header(skb);
	iph = ip_hdr(skb);
	if (iph->protocol != IPPROTO_TCP)
		return -1;

	ip_len = ip_hdrlen(skb);
	skb_set_transport_header(skb, ip_len);
	*tcph = tcp_hdr(skb);

	/* check if ip header and tcp header are complete */
	if (iph->tot_len < ip_len + tcp_hdrlen(skb))
		return -1;

	*hdr_flags = LRO_IPV4 | LRO_TCP;
	*iphdr = iph;

	return 0;
}

static void ehea_proc_skb(struct ehea_port_res *pr, struct ehea_cqe *cqe,
			  struct sk_buff *skb)
{
	int vlan_extracted = (cqe->status & EHEA_CQE_VLAN_TAG_XTRACT)
		&& pr->port->vgrp;

	if (use_lro) {
		if (vlan_extracted)
			lro_vlan_hwaccel_receive_skb(&pr->lro_mgr, skb,
						     pr->port->vgrp,
						     cqe->vlan_tag,
						     cqe);
		else
			lro_receive_skb(&pr->lro_mgr, skb, cqe);
	} else {
		if (vlan_extracted)
			vlan_hwaccel_receive_skb(skb, pr->port->vgrp,
						 cqe->vlan_tag);
		else
			netif_receive_skb(skb);
	}
}

static int ehea_proc_rwqes(struct net_device *dev,
			   struct ehea_port_res *pr,
			   int budget)
{
	struct ehea_port *port = pr->port;
	struct ehea_qp *qp = pr->qp;
	struct ehea_cqe *cqe;
	struct sk_buff *skb;
	struct sk_buff **skb_arr_rq1 = pr->rq1_skba.arr;
	struct sk_buff **skb_arr_rq2 = pr->rq2_skba.arr;
	struct sk_buff **skb_arr_rq3 = pr->rq3_skba.arr;
	int skb_arr_rq1_len = pr->rq1_skba.len;
	int skb_arr_rq2_len = pr->rq2_skba.len;
	int skb_arr_rq3_len = pr->rq3_skba.len;
	int processed, processed_rq1, processed_rq2, processed_rq3;
	int wqe_index, last_wqe_index, rq, port_reset;

	processed = processed_rq1 = processed_rq2 = processed_rq3 = 0;
	last_wqe_index = 0;

	cqe = ehea_poll_rq1(qp, &wqe_index);
	while ((processed < budget) && cqe) {
		ehea_inc_rq1(qp);
		processed_rq1++;
		processed++;
		if (netif_msg_rx_status(port))
			ehea_dump(cqe, sizeof(*cqe), "CQE");

		last_wqe_index = wqe_index;
		rmb();
		if (!ehea_check_cqe(cqe, &rq)) {
			if (rq == 1) {	/* LL RQ1 */
				skb = get_skb_by_index_ll(skb_arr_rq1,
							  skb_arr_rq1_len,
							  wqe_index);
				if (unlikely(!skb)) {
					if (netif_msg_rx_err(port))
						ehea_error("LL rq1: skb=NULL");

					skb = netdev_alloc_skb(dev,
							       EHEA_L_PKT_SIZE);
					if (!skb)
						break;
				}
				skb_copy_to_linear_data(skb, ((char*)cqe) + 64,