drm.c

		 * attached to a domain at this point because their group may
		 * have been detached by an earlier client.
		 */
		domain = iommu_get_domain_for_dev(client->dev);
		if (domain)
			iommu_detach_group(tegra->domain, client->group);

		iommu_group_put(client->group);
		client->group = NULL;
	}
}

void *tegra_drm_alloc(struct tegra_drm *tegra, size_t size, dma_addr_t *dma)
{
	struct iova *alloc;
	void *virt;
	gfp_t gfp;
	int err;

	if (tegra->domain)
		size = iova_align(&tegra->carveout.domain, size);
	else
		size = PAGE_ALIGN(size);

	gfp = GFP_KERNEL | __GFP_ZERO;
	if (!tegra->domain) {
		/*
		 * Many units only support 32-bit addresses, even on 64-bit
		 * SoCs. If there is no IOMMU to translate into a 32-bit IO
		 * virtual address space, force allocations to be in the
		 * lower 32-bit range.
		 */
		gfp |= GFP_DMA;
	}

	virt = (void *)__get_free_pages(gfp, get_order(size));
	if (!virt)
		return ERR_PTR(-ENOMEM);

	if (!tegra->domain) {
		/*
		 * If IOMMU is disabled, devices address physical memory
		 * directly.
		 */
		*dma = virt_to_phys(virt);
		return virt;
	}

	alloc = alloc_iova(&tegra->carveout.domain,
			   size >> tegra->carveout.shift,
			   tegra->carveout.limit, true);
	if (!alloc) {
		err = -EBUSY;
		goto free_pages;
	}

	*dma = iova_dma_addr(&tegra->carveout.domain, alloc);
	err = iommu_map(tegra->domain, *dma, virt_to_phys(virt),
			size, IOMMU_READ | IOMMU_WRITE);
	if (err < 0)
		goto free_iova;

	return virt;

free_iova:
	__free_iova(&tegra->carveout.domain, alloc);
free_pages:
	free_pages((unsigned long)virt, get_order(size));

	return ERR_PTR(err);
}

void tegra_drm_free(struct tegra_drm *tegra, size_t size, void *virt,
		    dma_addr_t dma)
{
	if (tegra->domain)
		size = iova_align(&tegra->carveout.domain, size);
	else
		size = PAGE_ALIGN(size);

	if (tegra->domain) {
		iommu_unmap(tegra->domain, dma, size);
		free_iova(&tegra->carveout.domain,
			  iova_pfn(&tegra->carveout.domain, dma));
	}

	free_pages((unsigned long)virt, get_order(size));
}

static bool host1x_drm_wants_iommu(struct host1x_device *dev)
{
	struct host1x *host1x = dev_get_drvdata(dev->dev.parent);
	struct iommu_domain *domain;

	/*
	 * If the Tegra DRM clients are backed by an IOMMU, push buffers are
	 * likely to be allocated beyond the 32-bit boundary if sufficient
	 * system memory is available. This is problematic on earlier Tegra
	 * generations where host1x supports a maximum of 32 address bits in
	 * the GATHER opcode. In this case, unless host1x is behind an IOMMU
	 * as well it won't be able to process buffers allocated beyond the
	 * 32-bit boundary.
	 *
	 * The DMA API will use bounce buffers in this case, so that could
	 * perhaps still be made to work, even if less efficient, but there
	 * is another catch: in order to perform cache maintenance on pages
	 * allocated for discontiguous buffers we need to map and unmap the
	 * SG table representing these buffers. This is fine for something
	 * small like a push buffer, but it exhausts the bounce buffer pool
	 * (typically on the order of a few MiB) for framebuffers (many MiB
	 * for any modern resolution).
	 *
	 * Work around this by making sure that Tegra DRM clients only use
	 * an IOMMU if the parent host1x also uses an IOMMU.
	 *
	 * Note that there's still a small gap here that we don't cover: if
	 * the DMA API is backed by an IOMMU there's no way to control which
	 * device is attached to an IOMMU and which isn't, except via wiring
	 * up the device tree appropriately. This is considered an problem
	 * of integration, so care must be taken for the DT to be consistent.
	 */
	domain = iommu_get_domain_for_dev(dev->dev.parent);

	/*
	 * Tegra20 and Tegra30 don't support addressing memory beyond the
	 * 32-bit boundary, so the regular GATHER opcodes will always be
	 * sufficient and whether or not the host1x is attached to an IOMMU
	 * doesn't matter.
	 */
	if (!domain && host1x_get_dma_mask(host1x) <= DMA_BIT_MASK(32))
		return true;

	return domain != NULL;
}

static int host1x_drm_probe(struct host1x_device *dev)
{
	struct tegra_drm *tegra;
	struct drm_device *drm;
	int err;

	drm = drm_dev_alloc(&tegra_drm_driver, &dev->dev);
	if (IS_ERR(drm))
		return PTR_ERR(drm);

	tegra = kzalloc(sizeof(*tegra), GFP_KERNEL);
	if (!tegra) {
		err = -ENOMEM;
		goto put;
	}

	if (host1x_drm_wants_iommu(dev) && iommu_present(&platform_bus_type)) {
		tegra->domain = iommu_domain_alloc(&platform_bus_type);
		if (!tegra->domain) {
			err = -ENOMEM;
			goto free;
		}

		err = iova_cache_get();
		if (err < 0)
			goto domain;
	}

	mutex_init(&tegra->clients_lock);
	INIT_LIST_HEAD(&tegra->clients);

	dev_set_drvdata(&dev->dev, drm);
	drm->dev_private = tegra;
	tegra->drm = drm;

	drm_mode_config_init(drm);

	drm->mode_config.min_width = 0;
	drm->mode_config.min_height = 0;
	drm->mode_config.max_width = 0;
	drm->mode_config.max_height = 0;

	drm->mode_config.normalize_zpos = true;

	drm->mode_config.funcs = &tegra_drm_mode_config_funcs;
	drm->mode_config.helper_private = &tegra_drm_mode_config_helpers;

	err = tegra_drm_fb_prepare(drm);
	if (err < 0)
		goto config;

	drm_kms_helper_poll_init(drm);

	err = host1x_device_init(dev);
	if (err < 0)
		goto fbdev;

	/*
	 * Now that all display controller have been initialized, the maximum
	 * supported resolution is known and the bitmask for horizontal and
	 * vertical bitfields can be computed.
	 */
	tegra->hmask = drm->mode_config.max_width - 1;
	tegra->vmask = drm->mode_config.max_height - 1;

	if (tegra->use_explicit_iommu) {
		u64 carveout_start, carveout_end, gem_start, gem_end;
		u64 dma_mask = dma_get_mask(&dev->dev);
		dma_addr_t start, end;
		unsigned long order;

		start = tegra->domain->geometry.aperture_start & dma_mask;
		end = tegra->domain->geometry.aperture_end & dma_mask;

		gem_start = start;
		gem_end = end - CARVEOUT_SZ;
		carveout_start = gem_end + 1;
		carveout_end = end;

		order = __ffs(tegra->domain->pgsize_bitmap);
		init_iova_domain(&tegra->carveout.domain, 1UL << order,
				 carveout_start >> order);

		tegra->carveout.shift = iova_shift(&tegra->carveout.domain);
		tegra->carveout.limit = carveout_end >> tegra->carveout.shift;

		drm_mm_init(&tegra->mm, gem_start, gem_end - gem_start + 1);
		mutex_init(&tegra->mm_lock);

		DRM_DEBUG_DRIVER("IOMMU apertures:\n");
		DRM_DEBUG_DRIVER("  GEM: %#llx-%#llx\n", gem_start, gem_end);
		DRM_DEBUG_DRIVER("  Carveout: %#llx-%#llx\n", carveout_start,
				 carveout_end);
	} else if (tegra->domain) {
		iommu_domain_free(tegra->domain);
		tegra->domain = NULL;
		iova_cache_put();
	}

	if (tegra->hub) {
		err = tegra_display_hub_prepare(tegra->hub);
		if (err < 0)
			goto device;
	}

	/* syncpoints are used for full 32-bit hardware VBLANK counters */
	drm->max_vblank_count = 0xffffffff;

	err = drm_vblank_init(drm, drm->mode_config.num_crtc);
	if (err < 0)
		goto hub;

	drm_mode_config_reset(drm);

	err = drm_aperture_remove_framebuffers(false, &tegra_drm_driver);
	if (err < 0)
		goto hub;

	err = tegra_drm_fb_init(drm);
	if (err < 0)
		goto hub;

	err = drm_dev_register(drm, 0);
	if (err < 0)
		goto fb;

	return 0;

fb:
	tegra_drm_fb_exit(drm);
hub:
	if (tegra->hub)
		tegra_display_hub_cleanup(tegra->hub);
device:
	if (tegra->domain) {
		mutex_destroy(&tegra->mm_lock);
		drm_mm_takedown(&tegra->mm);
		put_iova_domain(&tegra->carveout.domain);
		iova_cache_put();
	}

	host1x_device_exit(dev);
fbdev:
	drm_kms_helper_poll_fini(drm);
	tegra_drm_fb_free(drm);
config:
	drm_mode_config_cleanup(drm);
domain:
	if (tegra->domain)
		iommu_domain_free(tegra->domain);
free:
	kfree(tegra);
put:
	drm_dev_put(drm);
	return err;
}

static int host1x_drm_remove(struct host1x_device *dev)
{
	struct drm_device *drm = dev_get_drvdata(&dev->dev);
	struct tegra_drm *tegra = drm->dev_private;
	int err;

	drm_dev_unregister(drm);

	drm_kms_helper_poll_fini(drm);
	tegra_drm_fb_exit(drm);
	drm_atomic_helper_shutdown(drm);
	drm_mode_config_cleanup(drm);

	if (tegra->hub)
		tegra_display_hub_cleanup(tegra->hub);

	err = host1x_device_exit(dev);
	if (err < 0)
		dev_err(&dev->dev, "host1x device cleanup failed: %d\n", err);

	if (tegra->domain) {
		mutex_destroy(&tegra->mm_lock);
		drm_mm_takedown(&tegra->mm);
		put_iova_domain(&tegra->carveout.domain);
		iova_cache_put();
		iommu_domain_free(tegra->domain);
	}

	kfree(tegra);
	drm_dev_put(drm);

	return 0;
}

#ifdef CONFIG_PM_SLEEP
static int host1x_drm_suspend(struct device *dev)
{
	struct drm_device *drm = dev_get_drvdata(dev);

	return drm_mode_config_helper_suspend(drm);
}

static int host1x_drm_resume(struct device *dev)
{
	struct drm_device *drm = dev_get_drvdata(dev);

	return drm_mode_config_helper_resume(drm);
}
#endif

static SIMPLE_DEV_PM_OPS(host1x_drm_pm_ops, host1x_drm_suspend,
			 host1x_drm_resume);

static const struct of_device_id host1x_drm_subdevs[] = {
	{ .compatible = "nvidia,tegra20-dc", },
	{ .compatible = "nvidia,tegra20-hdmi", },
	{ .compatible = "nvidia,tegra20-gr2d", },
	{ .compatible = "nvidia,tegra20-gr3d", },
	{ .compatible = "nvidia,tegra30-dc", },
	{ .compatible = "nvidia,tegra30-hdmi", },
	{ .compatible = "nvidia,tegra30-gr2d", },
	{ .compatible = "nvidia,tegra30-gr3d", },
	{ .compatible = "nvidia,tegra114-dc", },
	{ .compatible = "nvidia,tegra114-dsi", },
	{ .compatible = "nvidia,tegra114-hdmi", },
	{ .compatible = "nvidia,tegra114-gr2d", },
	{ .compatible = "nvidia,tegra114-gr3d", },
	{ .compatible = "nvidia,tegra124-dc", },
	{ .compatible = "nvidia,tegra124-sor", },
	{ .compatible = "nvidia,tegra124-hdmi", },
	{ .compatible = "nvidia,tegra124-dsi", },
	{ .compatible = "nvidia,tegra124-vic", },
	{ .compatible = "nvidia,tegra132-dsi", },
	{ .compatible = "nvidia,tegra210-dc", },
	{ .compatible = "nvidia,tegra210-dsi", },
	{ .compatible = "nvidia,tegra210-sor", },
	{ .compatible = "nvidia,tegra210-sor1", },
	{ .compatible = "nvidia,tegra210-vic", },
	{ .compatible = "nvidia,tegra210-nvdec", },
	{ .compatible = "nvidia,tegra186-display", },
	{ .compatible = "nvidia,tegra186-dc", },
	{ .compatible = "nvidia,tegra186-sor", },
	{ .compatible = "nvidia,tegra186-sor1", },
	{ .compatible = "nvidia,tegra186-vic", },
	{ .compatible = "nvidia,tegra186-nvdec", },
	{ .compatible = "nvidia,tegra194-display", },
	{ .compatible = "nvidia,tegra194-dc", },
	{ .compatible = "nvidia,tegra194-sor", },
	{ .compatible = "nvidia,tegra194-vic", },
	{ .compatible = "nvidia,tegra194-nvdec", },
	{ /* sentinel */ }
};

static struct host1x_driver host1x_drm_driver = {
	.driver = {
		.name = "drm",
		.pm = &host1x_drm_pm_ops,
	},
	.probe = host1x_drm_probe,
	.remove = host1x_drm_remove,
	.subdevs = host1x_drm_subdevs,
};

static struct platform_driver * const drivers[] = {
	&tegra_display_hub_driver,
	&tegra_dc_driver,
	&tegra_hdmi_driver,
	&tegra_dsi_driver,
	&tegra_dpaux_driver,
	&tegra_sor_driver,
	&tegra_gr2d_driver,
	&tegra_gr3d_driver,
	&tegra_vic_driver,
	&tegra_nvdec_driver,
};

static int __init host1x_drm_init(void)
{
	int err;

	if (drm_firmware_drivers_only())
		return -ENODEV;

	err = host1x_driver_register(&host1x_drm_driver);
	if (err < 0)
		return err;

	err = platform_register_drivers(drivers, ARRAY_SIZE(drivers));
	if (err < 0)
		goto unregister_host1x;

	return 0;

unregister_host1x:
	host1x_driver_unregister(&host1x_drm_driver);
	return err;
}
module_init(host1x_drm_init);

static void __exit host1x_drm_exit(void)
{
	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
	host1x_driver_unregister(&host1x_drm_driver);
}
module_exit(host1x_drm_exit);

MODULE_AUTHOR("Thierry Reding <thierry.reding@avionic-design.de>");
MODULE_DESCRIPTION("NVIDIA Tegra DRM driver");
MODULE_LICENSE("GPL v2");