ntb_hw.c

}

static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
{
	struct ntb_db_cb *db_cb = data;
	struct ntb_device *ndev = db_cb->ndev;
	unsigned long mask;

	dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
		db_cb->db_num);

	mask = readw(ndev->reg_ofs.ldb_mask);
	set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
	writew(mask, ndev->reg_ofs.ldb_mask);

	tasklet_schedule(&db_cb->irq_work);

	/* On Sandybridge, there are 16 bits in the interrupt register
	 * but only 4 vectors.  So, 5 bits are assigned to the first 3
	 * vectors, with the 4th having a single bit for link
	 * interrupts.
	 */
	writew(((1 << ndev->bits_per_vector) - 1) <<
	       (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);

	return IRQ_HANDLED;
}

/* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
{
	struct ntb_device *ndev = dev;
	int rc;

	dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);

	rc = ntb_link_status(ndev);
	if (rc)
		dev_err(&ndev->pdev->dev, "Error determining link status\n");

	/* bit 15 is always the link bit */
	writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb);

	return IRQ_HANDLED;
}

static irqreturn_t ntb_interrupt(int irq, void *dev)
{
	struct ntb_device *ndev = dev;
	unsigned int i = 0;

	if (ndev->hw_type == BWD_HW) {
		u64 ldb = readq(ndev->reg_ofs.ldb);

		dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);

		while (ldb) {
			i = __ffs(ldb);
			ldb &= ldb - 1;
			bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
		}
	} else {
		u16 ldb = readw(ndev->reg_ofs.ldb);

		dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);

		if (ldb & SNB_DB_HW_LINK) {
			xeon_event_msix_irq(irq, dev);
			ldb &= ~SNB_DB_HW_LINK;
		}

		while (ldb) {
			i = __ffs(ldb);
			ldb &= ldb - 1;
			xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
		}
	}

	return IRQ_HANDLED;
}

static int ntb_setup_msix(struct ntb_device *ndev)
{
	struct pci_dev *pdev = ndev->pdev;
	struct msix_entry *msix;
	int msix_entries;
	int rc, i;
	u16 val;

	if (!pdev->msix_cap) {
		rc = -EIO;
		goto err;
	}

	rc = pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &val);
	if (rc)
		goto err;

	msix_entries = msix_table_size(val);
	if (msix_entries > ndev->limits.msix_cnt) {
		rc = -EINVAL;
		goto err;
	}

	ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
				     GFP_KERNEL);
	if (!ndev->msix_entries) {
		rc = -ENOMEM;
		goto err;
	}

	for (i = 0; i < msix_entries; i++)
		ndev->msix_entries[i].entry = i;

	rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);
	if (rc < 0)
		goto err1;
	if (rc > 0) {
		/* On SNB, the link interrupt is always tied to 4th vector.  If
		 * we can't get all 4, then we can't use MSI-X.
		 */
		if (ndev->hw_type != BWD_HW) {
			rc = -EIO;
			goto err1;
		}

		dev_warn(&pdev->dev,
			 "Only %d MSI-X vectors.  Limiting the number of queues to that number.\n",
			 rc);
		msix_entries = rc;

		rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);
		if (rc)
			goto err1;
	}

	for (i = 0; i < msix_entries; i++) {
		msix = &ndev->msix_entries[i];
		WARN_ON(!msix->vector);

		/* Use the last MSI-X vector for Link status */
		if (ndev->hw_type == BWD_HW) {
			rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
					 "ntb-callback-msix", &ndev->db_cb[i]);
			if (rc)
				goto err2;
		} else {
			if (i == msix_entries - 1) {
				rc = request_irq(msix->vector,
						 xeon_event_msix_irq, 0,
						 "ntb-event-msix", ndev);
				if (rc)
					goto err2;
			} else {
				rc = request_irq(msix->vector,
						 xeon_callback_msix_irq, 0,
						 "ntb-callback-msix",
						 &ndev->db_cb[i]);
				if (rc)
					goto err2;
			}
		}
	}

	ndev->num_msix = msix_entries;
	if (ndev->hw_type == BWD_HW)
		ndev->max_cbs = msix_entries;
	else
		ndev->max_cbs = msix_entries - 1;

	return 0;

err2:
	while (--i >= 0) {
		msix = &ndev->msix_entries[i];
		if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
			free_irq(msix->vector, ndev);
		else
			free_irq(msix->vector, &ndev->db_cb[i]);
	}
	pci_disable_msix(pdev);
err1:
	kfree(ndev->msix_entries);
	dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
err:
	ndev->num_msix = 0;
	return rc;
}

static int ntb_setup_msi(struct ntb_device *ndev)
{
	struct pci_dev *pdev = ndev->pdev;
	int rc;

	rc = pci_enable_msi(pdev);
	if (rc)
		return rc;

	rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
	if (rc) {
		pci_disable_msi(pdev);
		dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
		return rc;
	}

	return 0;
}

static int ntb_setup_intx(struct ntb_device *ndev)
{
	struct pci_dev *pdev = ndev->pdev;
	int rc;

	pci_msi_off(pdev);

	/* Verify intx is enabled */
	pci_intx(pdev, 1);

	rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
			 ndev);
	if (rc)
		return rc;

	return 0;
}

static int ntb_setup_interrupts(struct ntb_device *ndev)
{
	int rc;

	/* On BWD, disable all interrupts.  On SNB, disable all but Link
	 * Interrupt.  The rest will be unmasked as callbacks are registered.
	 */
	if (ndev->hw_type == BWD_HW)
		writeq(~0, ndev->reg_ofs.ldb_mask);
	else {
		u16 var = 1 << SNB_LINK_DB;
		writew(~var, ndev->reg_ofs.ldb_mask);
	}

	rc = ntb_setup_msix(ndev);
	if (!rc)
		goto done;

	ndev->bits_per_vector = 1;
	ndev->max_cbs = ndev->limits.max_db_bits;

	rc = ntb_setup_msi(ndev);
	if (!rc)
		goto done;

	rc = ntb_setup_intx(ndev);
	if (rc) {
		dev_err(&ndev->pdev->dev, "no usable interrupts\n");
		return rc;
	}

done:
	return 0;
}

static void ntb_free_interrupts(struct ntb_device *ndev)
{
	struct pci_dev *pdev = ndev->pdev;

	/* mask interrupts */
	if (ndev->hw_type == BWD_HW)
		writeq(~0, ndev->reg_ofs.ldb_mask);
	else
		writew(~0, ndev->reg_ofs.ldb_mask);

	if (ndev->num_msix) {
		struct msix_entry *msix;
		u32 i;

		for (i = 0; i < ndev->num_msix; i++) {
			msix = &ndev->msix_entries[i];
			if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
				free_irq(msix->vector, ndev);
			else
				free_irq(msix->vector, &ndev->db_cb[i]);
		}
		pci_disable_msix(pdev);
		kfree(ndev->msix_entries);
	} else {
		free_irq(pdev->irq, ndev);

		if (pci_dev_msi_enabled(pdev))
			pci_disable_msi(pdev);
	}
}

static int ntb_create_callbacks(struct ntb_device *ndev)
{
	int i;

	/* Chicken-egg issue.  We won't know how many callbacks are necessary
	 * until we see how many MSI-X vectors we get, but these pointers need
	 * to be passed into the MSI-X register function.  So, we allocate the
	 * max, knowing that they might not all be used, to work around this.
	 */
	ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
			      sizeof(struct ntb_db_cb),
			      GFP_KERNEL);
	if (!ndev->db_cb)
		return -ENOMEM;

	for (i = 0; i < ndev->limits.max_db_bits; i++) {
		ndev->db_cb[i].db_num = i;
		ndev->db_cb[i].ndev = ndev;
	}

	return 0;
}

static void ntb_free_callbacks(struct ntb_device *ndev)
{
	int i;

	for (i = 0; i < ndev->limits.max_db_bits; i++)
		ntb_unregister_db_callback(ndev, i);

	kfree(ndev->db_cb);
}

static void ntb_setup_debugfs(struct ntb_device *ndev)
{
	if (!debugfs_initialized())
		return;

	if (!debugfs_dir)
		debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);

	ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
					       debugfs_dir);
}

static void ntb_free_debugfs(struct ntb_device *ndev)
{
	debugfs_remove_recursive(ndev->debugfs_dir);

	if (debugfs_dir && simple_empty(debugfs_dir)) {
		debugfs_remove_recursive(debugfs_dir);
		debugfs_dir = NULL;
	}
}

static void ntb_hw_link_up(struct ntb_device *ndev)
{
	if (ndev->conn_type == NTB_CONN_TRANSPARENT)
		ntb_link_event(ndev, NTB_LINK_UP);
	else {
		u32 ntb_cntl;

		/* Let's bring the NTB link up */
		ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
		ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
		ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
		ntb_cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP;
		writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
	}
}

static void ntb_hw_link_down(struct ntb_device *ndev)
{
	u32 ntb_cntl;

	if (ndev->conn_type == NTB_CONN_TRANSPARENT) {
		ntb_link_event(ndev, NTB_LINK_DOWN);
		return;
	}

	/* Bring NTB link down */
	ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
	ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
	ntb_cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP);
	ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
	writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
}

static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	struct ntb_device *ndev;
	int rc, i;

	ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
	if (!ndev)
		return -ENOMEM;

	ndev->pdev = pdev;
	ndev->link_status = NTB_LINK_DOWN;
	pci_set_drvdata(pdev, ndev);
	ntb_setup_debugfs(ndev);

	rc = pci_enable_device(pdev);
	if (rc)
		goto err;

	pci_set_master(ndev->pdev);

	rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME);
	if (rc)
		goto err1;

	ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
	if (!ndev->reg_base) {
		dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
		rc = -EIO;
		goto err2;
	}

	for (i = 0; i < NTB_MAX_NUM_MW; i++) {
		ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
		ndev->mw[i].vbase =
		    ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)),
			       ndev->mw[i].bar_sz);
		dev_info(&pdev->dev, "MW %d size %llu\n", i,
			 (unsigned long long) ndev->mw[i].bar_sz);
		if (!ndev->mw[i].vbase) {
			dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
				 MW_TO_BAR(i));
			rc = -EIO;
			goto err3;
		}
	}

	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
	if (rc) {
		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
		if (rc)
			goto err3;

		dev_warn(&pdev->dev, "Cannot DMA highmem\n");
	}

	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	if (rc) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
		if (rc)
			goto err3;

		dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
	}

	rc = ntb_device_setup(ndev);
	if (rc)
		goto err3;

	rc = ntb_create_callbacks(ndev);
	if (rc)
		goto err4;

	rc = ntb_setup_interrupts(ndev);
	if (rc)
		goto err5;

	/* The scratchpad registers keep the values between rmmod/insmod,
	 * blast them now
	 */
	for (i = 0; i < ndev->limits.max_spads; i++) {
		ntb_write_local_spad(ndev, i, 0);
		ntb_write_remote_spad(ndev, i, 0);
	}

	rc = ntb_transport_init(pdev);
	if (rc)
		goto err6;

	ntb_hw_link_up(ndev);

	return 0;

err6:
	ntb_free_interrupts(ndev);
err5:
	ntb_free_callbacks(ndev);
err4:
	ntb_device_free(ndev);
err3:
	for (i--; i >= 0; i--)
		iounmap(ndev->mw[i].vbase);
	iounmap(ndev->reg_base);
err2:
	pci_release_selected_regions(pdev, NTB_BAR_MASK);
err1:
	pci_disable_device(pdev);
err:
	ntb_free_debugfs(ndev);
	kfree(ndev);

	dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
	return rc;
}

static void ntb_pci_remove(struct pci_dev *pdev)
{
	struct ntb_device *ndev = pci_get_drvdata(pdev);
	int i;

	ntb_hw_link_down(ndev);

	ntb_transport_free(ndev->ntb_transport);

	ntb_free_interrupts(ndev);
	ntb_free_callbacks(ndev);
	ntb_device_free(ndev);

	for (i = 0; i < NTB_MAX_NUM_MW; i++)
		iounmap(ndev->mw[i].vbase);

	iounmap(ndev->reg_base);
	pci_release_selected_regions(pdev, NTB_BAR_MASK);
	pci_disable_device(pdev);
	ntb_free_debugfs(ndev);
	kfree(ndev);
}

static struct pci_driver ntb_pci_driver = {
	.name = KBUILD_MODNAME,
	.id_table = ntb_pci_tbl,
	.probe = ntb_pci_probe,
	.remove = ntb_pci_remove,
};
module_pci_driver(ntb_pci_driver);