eq.c

来自「linux 内核源代码」· C语言 代码 · 共 657 行 · 第 1/2 页

	struct mlx4_eq_context *eq_context;
	int npages;
	u64 *dma_list = NULL;
	dma_addr_t t;
	u64 mtt_addr;
	int err = -ENOMEM;
	int i;

	eq->dev   = dev;
	eq->nent  = roundup_pow_of_two(max(nent, 2));
	npages = PAGE_ALIGN(eq->nent * MLX4_EQ_ENTRY_SIZE) / PAGE_SIZE;

	eq->page_list = kmalloc(npages * sizeof *eq->page_list,
				GFP_KERNEL);
	if (!eq->page_list)
		goto err_out;

	for (i = 0; i < npages; ++i)
		eq->page_list[i].buf = NULL;

	dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
	if (!dma_list)
		goto err_out_free;

	mailbox = mlx4_alloc_cmd_mailbox(dev);
	if (IS_ERR(mailbox))
		goto err_out_free;
	eq_context = mailbox->buf;

	for (i = 0; i < npages; ++i) {
		eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev,
							  PAGE_SIZE, &t, GFP_KERNEL);
		if (!eq->page_list[i].buf)
			goto err_out_free_pages;

		dma_list[i] = t;
		eq->page_list[i].map = t;

		memset(eq->page_list[i].buf, 0, PAGE_SIZE);
	}

	eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap);
	if (eq->eqn == -1)
		goto err_out_free_pages;

	eq->doorbell = mlx4_get_eq_uar(dev, eq);
	if (!eq->doorbell) {
		err = -ENOMEM;
		goto err_out_free_eq;
	}

	err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt);
	if (err)
		goto err_out_free_eq;

	err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list);
	if (err)
		goto err_out_free_mtt;

	memset(eq_context, 0, sizeof *eq_context);
	eq_context->flags	  = cpu_to_be32(MLX4_EQ_STATUS_OK   |
						MLX4_EQ_STATE_ARMED);
	eq_context->log_eq_size	  = ilog2(eq->nent);
	eq_context->intr	  = intr;
	eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT;

	mtt_addr = mlx4_mtt_addr(dev, &eq->mtt);
	eq_context->mtt_base_addr_h = mtt_addr >> 32;
	eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);

	err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn);
	if (err) {
		mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err);
		goto err_out_free_mtt;
	}

	kfree(dma_list);
	mlx4_free_cmd_mailbox(dev, mailbox);

	eq->cons_index = 0;

	return err;

err_out_free_mtt:
	mlx4_mtt_cleanup(dev, &eq->mtt);

err_out_free_eq:
	mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn);

err_out_free_pages:
	for (i = 0; i < npages; ++i)
		if (eq->page_list[i].buf)
			dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
					  eq->page_list[i].buf,
					  eq->page_list[i].map);

	mlx4_free_cmd_mailbox(dev, mailbox);

err_out_free:
	kfree(eq->page_list);
	kfree(dma_list);

err_out:
	return err;
}

static void mlx4_free_eq(struct mlx4_dev *dev,
			 struct mlx4_eq *eq)
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	struct mlx4_cmd_mailbox *mailbox;
	int err;
	int npages = PAGE_ALIGN(MLX4_EQ_ENTRY_SIZE * eq->nent) / PAGE_SIZE;
	int i;

	mailbox = mlx4_alloc_cmd_mailbox(dev);
	if (IS_ERR(mailbox))
		return;

	err = mlx4_HW2SW_EQ(dev, mailbox, eq->eqn);
	if (err)
		mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err);

	if (0) {
		mlx4_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn);
		for (i = 0; i < sizeof (struct mlx4_eq_context) / 4; ++i) {
			if (i % 4 == 0)
				printk("[%02x] ", i * 4);
			printk(" %08x", be32_to_cpup(mailbox->buf + i * 4));
			if ((i + 1) % 4 == 0)
				printk("\n");
		}
	}

	mlx4_mtt_cleanup(dev, &eq->mtt);
	for (i = 0; i < npages; ++i)
		pci_free_consistent(dev->pdev, PAGE_SIZE,
				    eq->page_list[i].buf,
				    eq->page_list[i].map);

	kfree(eq->page_list);
	mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn);
	mlx4_free_cmd_mailbox(dev, mailbox);
}

static void mlx4_free_irqs(struct mlx4_dev *dev)
{
	struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table;
	int i;

	if (eq_table->have_irq)
		free_irq(dev->pdev->irq, dev);
	for (i = 0; i < MLX4_NUM_EQ; ++i)
		if (eq_table->eq[i].have_irq)
			free_irq(eq_table->eq[i].irq, eq_table->eq + i);
}

static int mlx4_map_clr_int(struct mlx4_dev *dev)
{
	struct mlx4_priv *priv = mlx4_priv(dev);

	priv->clr_base = ioremap(pci_resource_start(dev->pdev, priv->fw.clr_int_bar) +
				 priv->fw.clr_int_base, MLX4_CLR_INT_SIZE);
	if (!priv->clr_base) {
		mlx4_err(dev, "Couldn't map interrupt clear register, aborting.\n");
		return -ENOMEM;
	}

	return 0;
}

static void mlx4_unmap_clr_int(struct mlx4_dev *dev)
{
	struct mlx4_priv *priv = mlx4_priv(dev);

	iounmap(priv->clr_base);
}

int mlx4_map_eq_icm(struct mlx4_dev *dev, u64 icm_virt)
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	int ret;

	/*
	 * We assume that mapping one page is enough for the whole EQ
	 * context table.  This is fine with all current HCAs, because
	 * we only use 32 EQs and each EQ uses 64 bytes of context
	 * memory, or 1 KB total.
	 */
	priv->eq_table.icm_virt = icm_virt;
	priv->eq_table.icm_page = alloc_page(GFP_HIGHUSER);
	if (!priv->eq_table.icm_page)
		return -ENOMEM;
	priv->eq_table.icm_dma  = pci_map_page(dev->pdev, priv->eq_table.icm_page, 0,
					       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
	if (pci_dma_mapping_error(priv->eq_table.icm_dma)) {
		__free_page(priv->eq_table.icm_page);
		return -ENOMEM;
	}

	ret = mlx4_MAP_ICM_page(dev, priv->eq_table.icm_dma, icm_virt);
	if (ret) {
		pci_unmap_page(dev->pdev, priv->eq_table.icm_dma, PAGE_SIZE,
			       PCI_DMA_BIDIRECTIONAL);
		__free_page(priv->eq_table.icm_page);
	}

	return ret;
}

void mlx4_unmap_eq_icm(struct mlx4_dev *dev)
{
	struct mlx4_priv *priv = mlx4_priv(dev);

	mlx4_UNMAP_ICM(dev, priv->eq_table.icm_virt, 1);
	pci_unmap_page(dev->pdev, priv->eq_table.icm_dma, PAGE_SIZE,
		       PCI_DMA_BIDIRECTIONAL);
	__free_page(priv->eq_table.icm_page);
}

int mlx4_init_eq_table(struct mlx4_dev *dev)
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	int err;
	int i;

	err = mlx4_bitmap_init(&priv->eq_table.bitmap, dev->caps.num_eqs,
			       dev->caps.num_eqs - 1, dev->caps.reserved_eqs);
	if (err)
		return err;

	for (i = 0; i < ARRAY_SIZE(priv->eq_table.uar_map); ++i)
		priv->eq_table.uar_map[i] = NULL;

	err = mlx4_map_clr_int(dev);
	if (err)
		goto err_out_free;

	priv->eq_table.clr_mask =
		swab32(1 << (priv->eq_table.inta_pin & 31));
	priv->eq_table.clr_int  = priv->clr_base +
		(priv->eq_table.inta_pin < 32 ? 4 : 0);

	err = mlx4_create_eq(dev, dev->caps.num_cqs + MLX4_NUM_SPARE_EQE,
			     (dev->flags & MLX4_FLAG_MSI_X) ? MLX4_EQ_COMP : 0,
			     &priv->eq_table.eq[MLX4_EQ_COMP]);
	if (err)
		goto err_out_unmap;

	err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE,
			     (dev->flags & MLX4_FLAG_MSI_X) ? MLX4_EQ_ASYNC : 0,
			     &priv->eq_table.eq[MLX4_EQ_ASYNC]);
	if (err)
		goto err_out_comp;

	if (dev->flags & MLX4_FLAG_MSI_X) {
		static const char *eq_name[] = {
			[MLX4_EQ_COMP]  = DRV_NAME " (comp)",
			[MLX4_EQ_ASYNC] = DRV_NAME " (async)"
		};

		for (i = 0; i < MLX4_NUM_EQ; ++i) {
			err = request_irq(priv->eq_table.eq[i].irq,
					  mlx4_msi_x_interrupt,
					  0, eq_name[i], priv->eq_table.eq + i);
			if (err)
				goto err_out_async;

			priv->eq_table.eq[i].have_irq = 1;
		}

	} else {
		err = request_irq(dev->pdev->irq, mlx4_interrupt,
				  IRQF_SHARED, DRV_NAME, dev);
		if (err)
			goto err_out_async;

		priv->eq_table.have_irq = 1;
	}

	err = mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 0,
			  priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
	if (err)
		mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
			   priv->eq_table.eq[MLX4_EQ_ASYNC].eqn, err);

	for (i = 0; i < MLX4_NUM_EQ; ++i)
		eq_set_ci(&priv->eq_table.eq[i], 1);

	return 0;

err_out_async:
	mlx4_free_eq(dev, &priv->eq_table.eq[MLX4_EQ_ASYNC]);

err_out_comp:
	mlx4_free_eq(dev, &priv->eq_table.eq[MLX4_EQ_COMP]);

err_out_unmap:
	mlx4_unmap_clr_int(dev);
	mlx4_free_irqs(dev);

err_out_free:
	mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
	return err;
}

void mlx4_cleanup_eq_table(struct mlx4_dev *dev)
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	int i;

	mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 1,
		    priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);

	mlx4_free_irqs(dev);

	for (i = 0; i < MLX4_NUM_EQ; ++i)
		mlx4_free_eq(dev, &priv->eq_table.eq[i]);

	mlx4_unmap_clr_int(dev);

	for (i = 0; i < ARRAY_SIZE(priv->eq_table.uar_map); ++i)
		if (priv->eq_table.uar_map[i])
			iounmap(priv->eq_table.uar_map[i]);

	mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
}