mthca_provider.c

linux-2.6.15.6

/*
 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
 * Copyright (c) 2005 Cisco Systems. All rights reserved.
 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * $Id: mthca_provider.c 1397 2004-12-28 05:09:00Z roland $
 */

#include <rdma/ib_smi.h>
#include <rdma/ib_user_verbs.h>
#include <linux/mm.h>

#include "mthca_dev.h"
#include "mthca_cmd.h"
#include "mthca_user.h"
#include "mthca_memfree.h"

static int mthca_query_device(struct ib_device *ibdev,
			      struct ib_device_attr *props)
{
	struct ib_smp *in_mad  = NULL;
	struct ib_smp *out_mad = NULL;
	int err = -ENOMEM;
	struct mthca_dev* mdev = to_mdev(ibdev);

	u8 status;

	in_mad  = kmalloc(sizeof *in_mad, GFP_KERNEL);
	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
	if (!in_mad || !out_mad)
		goto out;

	memset(props, 0, sizeof *props);

	props->fw_ver              = mdev->fw_ver;

	memset(in_mad, 0, sizeof *in_mad);
	in_mad->base_version       = 1;
	in_mad->mgmt_class     	   = IB_MGMT_CLASS_SUBN_LID_ROUTED;
	in_mad->class_version  	   = 1;
	in_mad->method         	   = IB_MGMT_METHOD_GET;
	in_mad->attr_id   	   = IB_SMP_ATTR_NODE_INFO;

	err = mthca_MAD_IFC(mdev, 1, 1,
			    1, NULL, NULL, in_mad, out_mad,
			    &status);
	if (err)
		goto out;
	if (status) {
		err = -EINVAL;
		goto out;
	}

	props->device_cap_flags    = mdev->device_cap_flags;
	props->vendor_id           = be32_to_cpup((__be32 *) (out_mad->data + 36)) &
		0xffffff;
	props->vendor_part_id      = be16_to_cpup((__be16 *) (out_mad->data + 30));
	props->hw_ver              = be32_to_cpup((__be32 *) (out_mad->data + 32));
	memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
	memcpy(&props->node_guid,      out_mad->data + 12, 8);

	props->max_mr_size         = ~0ull;
	props->page_size_cap       = mdev->limits.page_size_cap;
	props->max_qp              = mdev->limits.num_qps - mdev->limits.reserved_qps;
	props->max_qp_wr           = mdev->limits.max_wqes;
	props->max_sge             = mdev->limits.max_sg;
	props->max_cq              = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
	props->max_cqe             = mdev->limits.max_cqes;
	props->max_mr              = mdev->limits.num_mpts - mdev->limits.reserved_mrws;
	props->max_pd              = mdev->limits.num_pds - mdev->limits.reserved_pds;
	props->max_qp_rd_atom      = 1 << mdev->qp_table.rdb_shift;
	props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma;
	props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
	props->max_srq             = mdev->limits.num_srqs - mdev->limits.reserved_srqs;
	props->max_srq_wr          = mdev->limits.max_srq_wqes;
	props->max_srq_sge         = mdev->limits.max_sg;
	props->local_ca_ack_delay  = mdev->limits.local_ca_ack_delay;
	props->atomic_cap          = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? 
					IB_ATOMIC_HCA : IB_ATOMIC_NONE;
	props->max_pkeys           = mdev->limits.pkey_table_len;
	props->max_mcast_grp       = mdev->limits.num_mgms + mdev->limits.num_amgms;
	props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 
					   props->max_mcast_grp;

	err = 0;
 out:
	kfree(in_mad);
	kfree(out_mad);
	return err;
}

static int mthca_query_port(struct ib_device *ibdev,
			    u8 port, struct ib_port_attr *props)
{
	struct ib_smp *in_mad  = NULL;
	struct ib_smp *out_mad = NULL;
	int err = -ENOMEM;
	u8 status;

	in_mad  = kmalloc(sizeof *in_mad, GFP_KERNEL);
	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
	if (!in_mad || !out_mad)
		goto out;

	memset(props, 0, sizeof *props);

	memset(in_mad, 0, sizeof *in_mad);
	in_mad->base_version       = 1;
	in_mad->mgmt_class     	   = IB_MGMT_CLASS_SUBN_LID_ROUTED;
	in_mad->class_version  	   = 1;
	in_mad->method         	   = IB_MGMT_METHOD_GET;
	in_mad->attr_id   	   = IB_SMP_ATTR_PORT_INFO;
	in_mad->attr_mod           = cpu_to_be32(port);

	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
			    port, NULL, NULL, in_mad, out_mad,
			    &status);
	if (err)
		goto out;
	if (status) {
		err = -EINVAL;
		goto out;
	}

	props->lid               = be16_to_cpup((__be16 *) (out_mad->data + 16));
	props->lmc               = out_mad->data[34] & 0x7;
	props->sm_lid            = be16_to_cpup((__be16 *) (out_mad->data + 18));
	props->sm_sl             = out_mad->data[36] & 0xf;
	props->state             = out_mad->data[32] & 0xf;
	props->phys_state        = out_mad->data[33] >> 4;
	props->port_cap_flags    = be32_to_cpup((__be32 *) (out_mad->data + 20));
	props->gid_tbl_len       = to_mdev(ibdev)->limits.gid_table_len;
	props->max_msg_sz        = 0x80000000;
	props->pkey_tbl_len      = to_mdev(ibdev)->limits.pkey_table_len;
	props->bad_pkey_cntr     = be16_to_cpup((__be16 *) (out_mad->data + 46));
	props->qkey_viol_cntr    = be16_to_cpup((__be16 *) (out_mad->data + 48));
	props->active_width      = out_mad->data[31] & 0xf;
	props->active_speed      = out_mad->data[35] >> 4;
	props->max_mtu           = out_mad->data[41] & 0xf;
	props->active_mtu        = out_mad->data[36] >> 4;
	props->subnet_timeout    = out_mad->data[51] & 0x1f;

 out:
	kfree(in_mad);
	kfree(out_mad);
	return err;
}

static int mthca_modify_port(struct ib_device *ibdev,
			     u8 port, int port_modify_mask,
			     struct ib_port_modify *props)
{
	struct mthca_set_ib_param set_ib;
	struct ib_port_attr attr;
	int err;
	u8 status;

	if (down_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
		return -ERESTARTSYS;

	err = mthca_query_port(ibdev, port, &attr);
	if (err)
		goto out;

	set_ib.set_si_guid     = 0;
	set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);

	set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
		~props->clr_port_cap_mask;

	err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status);
	if (err)
		goto out;
	if (status) {
		err = -EINVAL;
		goto out;
	}

out:
	up(&to_mdev(ibdev)->cap_mask_mutex);
	return err;
}

static int mthca_query_pkey(struct ib_device *ibdev,
			    u8 port, u16 index, u16 *pkey)
{
	struct ib_smp *in_mad  = NULL;
	struct ib_smp *out_mad = NULL;
	int err = -ENOMEM;
	u8 status;

	in_mad  = kmalloc(sizeof *in_mad, GFP_KERNEL);
	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
	if (!in_mad || !out_mad)
		goto out;

	memset(in_mad, 0, sizeof *in_mad);
	in_mad->base_version       = 1;
	in_mad->mgmt_class     	   = IB_MGMT_CLASS_SUBN_LID_ROUTED;
	in_mad->class_version  	   = 1;
	in_mad->method         	   = IB_MGMT_METHOD_GET;
	in_mad->attr_id   	   = IB_SMP_ATTR_PKEY_TABLE;
	in_mad->attr_mod           = cpu_to_be32(index / 32);

	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
			    port, NULL, NULL, in_mad, out_mad,
			    &status);
	if (err)
		goto out;
	if (status) {
		err = -EINVAL;
		goto out;
	}

	*pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]);

 out:
	kfree(in_mad);
	kfree(out_mad);
	return err;
}

static int mthca_query_gid(struct ib_device *ibdev, u8 port,
			   int index, union ib_gid *gid)
{
	struct ib_smp *in_mad  = NULL;
	struct ib_smp *out_mad = NULL;
	int err = -ENOMEM;
	u8 status;

	in_mad  = kmalloc(sizeof *in_mad, GFP_KERNEL);
	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
	if (!in_mad || !out_mad)
		goto out;

	memset(in_mad, 0, sizeof *in_mad);
	in_mad->base_version       = 1;
	in_mad->mgmt_class     	   = IB_MGMT_CLASS_SUBN_LID_ROUTED;
	in_mad->class_version  	   = 1;
	in_mad->method         	   = IB_MGMT_METHOD_GET;
	in_mad->attr_id   	   = IB_SMP_ATTR_PORT_INFO;
	in_mad->attr_mod           = cpu_to_be32(port);

	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
			    port, NULL, NULL, in_mad, out_mad,
			    &status);
	if (err)
		goto out;
	if (status) {
		err = -EINVAL;
		goto out;
	}

	memcpy(gid->raw, out_mad->data + 8, 8);

	memset(in_mad, 0, sizeof *in_mad);
	in_mad->base_version       = 1;
	in_mad->mgmt_class     	   = IB_MGMT_CLASS_SUBN_LID_ROUTED;
	in_mad->class_version  	   = 1;
	in_mad->method         	   = IB_MGMT_METHOD_GET;
	in_mad->attr_id   	   = IB_SMP_ATTR_GUID_INFO;
	in_mad->attr_mod           = cpu_to_be32(index / 8);

	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
			    port, NULL, NULL, in_mad, out_mad,
			    &status);
	if (err)
		goto out;
	if (status) {
		err = -EINVAL;
		goto out;
	}

	memcpy(gid->raw + 8, out_mad->data + (index % 8) * 16, 8);

 out:
	kfree(in_mad);
	kfree(out_mad);
	return err;
}

static struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev,
						struct ib_udata *udata)
{
	struct mthca_alloc_ucontext_resp uresp;
	struct mthca_ucontext           *context;
	int                              err;

	memset(&uresp, 0, sizeof uresp);

	uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
	if (mthca_is_memfree(to_mdev(ibdev)))
		uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size;
	else
		uresp.uarc_size = 0;

	context = kmalloc(sizeof *context, GFP_KERNEL);
	if (!context)
		return ERR_PTR(-ENOMEM);

	err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
	if (err) {
		kfree(context);
		return ERR_PTR(err);
	}

	context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
	if (IS_ERR(context->db_tab)) {
		err = PTR_ERR(context->db_tab);
		mthca_uar_free(to_mdev(ibdev), &context->uar);
		kfree(context);
		return ERR_PTR(err);
	}

	if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) {
		mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab);
		mthca_uar_free(to_mdev(ibdev), &context->uar);
		kfree(context);
		return ERR_PTR(-EFAULT);
	}

	return &context->ibucontext;
}

static int mthca_dealloc_ucontext(struct ib_ucontext *context)
{
	mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar,
				  to_mucontext(context)->db_tab);
	mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar);
	kfree(to_mucontext(context));

	return 0;
}

static int mthca_mmap_uar(struct ib_ucontext *context,
			  struct vm_area_struct *vma)
{
	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
		return -EINVAL;

	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

	if (io_remap_pfn_range(vma, vma->vm_start,
			       to_mucontext(context)->uar.pfn,
			       PAGE_SIZE, vma->vm_page_prot))
		return -EAGAIN;

	return 0;
}

static struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev,
				    struct ib_ucontext *context,
				    struct ib_udata *udata)
{
	struct mthca_pd *pd;
	int err;

	pd = kmalloc(sizeof *pd, GFP_KERNEL);
	if (!pd)
		return ERR_PTR(-ENOMEM);

	err = mthca_pd_alloc(to_mdev(ibdev), !context, pd);
	if (err) {
		kfree(pd);
		return ERR_PTR(err);
	}