amd_iommu.c

{
	if (!dev || !dev->dma_mask)
		return false;

	return true;
}

/*
 * In this function the list of preallocated protection domains is traversed to
 * find the domain for a specific device
 */
static struct dma_ops_domain *find_protection_domain(u16 devid)
{
	struct dma_ops_domain *entry, *ret = NULL;
	unsigned long flags;

	if (list_empty(&iommu_pd_list))
		return NULL;

	spin_lock_irqsave(&iommu_pd_list_lock, flags);

	list_for_each_entry(entry, &iommu_pd_list, list) {
		if (entry->target_dev == devid) {
			ret = entry;
			break;
		}
	}

	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);

	return ret;
}

/*
 * In the dma_ops path we only have the struct device. This function
 * finds the corresponding IOMMU, the protection domain and the
 * requestor id for a given device.
 * If the device is not yet associated with a domain this is also done
 * in this function.
 */
static int get_device_resources(struct device *dev,
				struct amd_iommu **iommu,
				struct protection_domain **domain,
				u16 *bdf)
{
	struct dma_ops_domain *dma_dom;
	struct pci_dev *pcidev;
	u16 _bdf;

	*iommu = NULL;
	*domain = NULL;
	*bdf = 0xffff;

	if (dev->bus != &pci_bus_type)
		return 0;

	pcidev = to_pci_dev(dev);
	_bdf = calc_devid(pcidev->bus->number, pcidev->devfn);

	/* device not translated by any IOMMU in the system? */
	if (_bdf > amd_iommu_last_bdf)
		return 0;

	*bdf = amd_iommu_alias_table[_bdf];

	*iommu = amd_iommu_rlookup_table[*bdf];
	if (*iommu == NULL)
		return 0;
	*domain = domain_for_device(*bdf);
	if (*domain == NULL) {
		dma_dom = find_protection_domain(*bdf);
		if (!dma_dom)
			dma_dom = (*iommu)->default_dom;
		*domain = &dma_dom->domain;
		attach_device(*iommu, *domain, *bdf);
		printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
				"device %s\n", (*domain)->id, dev_name(dev));
	}

	if (domain_for_device(_bdf) == NULL)
		attach_device(*iommu, *domain, _bdf);

	return 1;
}

/*
 * This is the generic map function. It maps one 4kb page at paddr to
 * the given address in the DMA address space for the domain.
 */
static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
				     struct dma_ops_domain *dom,
				     unsigned long address,
				     phys_addr_t paddr,
				     int direction)
{
	u64 *pte, __pte;

	WARN_ON(address > dom->aperture_size);

	paddr &= PAGE_MASK;

	pte  = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
	pte += IOMMU_PTE_L0_INDEX(address);

	__pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;

	if (direction == DMA_TO_DEVICE)
		__pte |= IOMMU_PTE_IR;
	else if (direction == DMA_FROM_DEVICE)
		__pte |= IOMMU_PTE_IW;
	else if (direction == DMA_BIDIRECTIONAL)
		__pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;

	WARN_ON(*pte);

	*pte = __pte;

	return (dma_addr_t)address;
}

/*
 * The generic unmapping function for on page in the DMA address space.
 */
static void dma_ops_domain_unmap(struct amd_iommu *iommu,
				 struct dma_ops_domain *dom,
				 unsigned long address)
{
	u64 *pte;

	if (address >= dom->aperture_size)
		return;

	WARN_ON(address & ~PAGE_MASK || address >= dom->aperture_size);

	pte  = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
	pte += IOMMU_PTE_L0_INDEX(address);

	WARN_ON(!*pte);

	*pte = 0ULL;
}

/*
 * This function contains common code for mapping of a physically
 * contiguous memory region into DMA address space. It is used by all
 * mapping functions provided with this IOMMU driver.
 * Must be called with the domain lock held.
 */
static dma_addr_t __map_single(struct device *dev,
			       struct amd_iommu *iommu,
			       struct dma_ops_domain *dma_dom,
			       phys_addr_t paddr,
			       size_t size,
			       int dir,
			       bool align,
			       u64 dma_mask)
{
	dma_addr_t offset = paddr & ~PAGE_MASK;
	dma_addr_t address, start;
	unsigned int pages;
	unsigned long align_mask = 0;
	int i;

	pages = iommu_num_pages(paddr, size, PAGE_SIZE);
	paddr &= PAGE_MASK;

	if (align)
		align_mask = (1UL << get_order(size)) - 1;

	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
					  dma_mask);
	if (unlikely(address == bad_dma_address))
		goto out;

	start = address;
	for (i = 0; i < pages; ++i) {
		dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
		paddr += PAGE_SIZE;
		start += PAGE_SIZE;
	}
	address += offset;

	if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
		iommu_flush_tlb(iommu, dma_dom->domain.id);
		dma_dom->need_flush = false;
	} else if (unlikely(iommu_has_npcache(iommu)))
		iommu_flush_pages(iommu, dma_dom->domain.id, address, size);

out:
	return address;
}

/*
 * Does the reverse of the __map_single function. Must be called with
 * the domain lock held too
 */
static void __unmap_single(struct amd_iommu *iommu,
			   struct dma_ops_domain *dma_dom,
			   dma_addr_t dma_addr,
			   size_t size,
			   int dir)
{
	dma_addr_t i, start;
	unsigned int pages;

	if ((dma_addr == bad_dma_address) ||
	    (dma_addr + size > dma_dom->aperture_size))
		return;

	pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
	dma_addr &= PAGE_MASK;
	start = dma_addr;

	for (i = 0; i < pages; ++i) {
		dma_ops_domain_unmap(iommu, dma_dom, start);
		start += PAGE_SIZE;
	}

	dma_ops_free_addresses(dma_dom, dma_addr, pages);

	if (amd_iommu_unmap_flush || dma_dom->need_flush) {
		iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
		dma_dom->need_flush = false;
	}
}

/*
 * The exported map_single function for dma_ops.
 */
static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
			     size_t size, int dir)
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;
	dma_addr_t addr;
	u64 dma_mask;

	if (!check_device(dev))
		return bad_dma_address;

	dma_mask = *dev->dma_mask;

	get_device_resources(dev, &iommu, &domain, &devid);

	if (iommu == NULL || domain == NULL)
		/* device not handled by any AMD IOMMU */
		return (dma_addr_t)paddr;

	if (!dma_ops_domain(domain))
		return bad_dma_address;

	spin_lock_irqsave(&domain->lock, flags);
	addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
			    dma_mask);
	if (addr == bad_dma_address)
		goto out;

	iommu_completion_wait(iommu);

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return addr;
}

/*
 * The exported unmap_single function for dma_ops.
 */
static void unmap_single(struct device *dev, dma_addr_t dma_addr,
			 size_t size, int dir)
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;

	if (!check_device(dev) ||
	    !get_device_resources(dev, &iommu, &domain, &devid))
		/* device not handled by any AMD IOMMU */
		return;

	if (!dma_ops_domain(domain))
		return;

	spin_lock_irqsave(&domain->lock, flags);

	__unmap_single(iommu, domain->priv, dma_addr, size, dir);

	iommu_completion_wait(iommu);

	spin_unlock_irqrestore(&domain->lock, flags);
}

/*
 * This is a special map_sg function which is used if we should map a
 * device which is not handled by an AMD IOMMU in the system.
 */
static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
			   int nelems, int dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sglist, s, nelems, i) {
		s->dma_address = (dma_addr_t)sg_phys(s);
		s->dma_length  = s->length;
	}

	return nelems;
}

/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
static int map_sg(struct device *dev, struct scatterlist *sglist,
		  int nelems, int dir)
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;
	int i;
	struct scatterlist *s;
	phys_addr_t paddr;
	int mapped_elems = 0;
	u64 dma_mask;

	if (!check_device(dev))
		return 0;

	dma_mask = *dev->dma_mask;

	get_device_resources(dev, &iommu, &domain, &devid);

	if (!iommu || !domain)
		return map_sg_no_iommu(dev, sglist, nelems, dir);

	if (!dma_ops_domain(domain))
		return 0;

	spin_lock_irqsave(&domain->lock, flags);

	for_each_sg(sglist, s, nelems, i) {
		paddr = sg_phys(s);

		s->dma_address = __map_single(dev, iommu, domain->priv,
					      paddr, s->length, dir, false,
					      dma_mask);

		if (s->dma_address) {
			s->dma_length = s->length;
			mapped_elems++;
		} else
			goto unmap;
	}

	iommu_completion_wait(iommu);

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return mapped_elems;
unmap:
	for_each_sg(sglist, s, mapped_elems, i) {
		if (s->dma_address)
			__unmap_single(iommu, domain->priv, s->dma_address,
				       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

	mapped_elems = 0;

	goto out;
}

/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
static void unmap_sg(struct device *dev, struct scatterlist *sglist,
		     int nelems, int dir)
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	struct scatterlist *s;
	u16 devid;
	int i;

	if (!check_device(dev) ||
	    !get_device_resources(dev, &iommu, &domain, &devid))
		return;

	if (!dma_ops_domain(domain))
		return;

	spin_lock_irqsave(&domain->lock, flags);

	for_each_sg(sglist, s, nelems, i) {
		__unmap_single(iommu, domain->priv, s->dma_address,
			       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

	iommu_completion_wait(iommu);

	spin_unlock_irqrestore(&domain->lock, flags);
}

/*
 * The exported alloc_coherent function for dma_ops.
 */
static void *alloc_coherent(struct device *dev, size_t size,
			    dma_addr_t *dma_addr, gfp_t flag)
{
	unsigned long flags;
	void *virt_addr;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;
	phys_addr_t paddr;
	u64 dma_mask = dev->coherent_dma_mask;

	if (!check_device(dev))
		return NULL;

	if (!get_device_resources(dev, &iommu, &domain, &devid))
		flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);

	flag |= __GFP_ZERO;
	virt_addr = (void *)__get_free_pages(flag, get_order(size));
	if (!virt_addr)
		return 0;

	paddr = virt_to_phys(virt_addr);

	if (!iommu || !domain) {
		*dma_addr = (dma_addr_t)paddr;
		return virt_addr;
	}

	if (!dma_ops_domain(domain))
		goto out_free;

	if (!dma_mask)
		dma_mask = *dev->dma_mask;

	spin_lock_irqsave(&domain->lock, flags);

	*dma_addr = __map_single(dev, iommu, domain->priv, paddr,
				 size, DMA_BIDIRECTIONAL, true, dma_mask);

	if (*dma_addr == bad_dma_address)
		goto out_free;

	iommu_completion_wait(iommu);

	spin_unlock_irqrestore(&domain->lock, flags);

	return virt_addr;

out_free:

	free_pages((unsigned long)virt_addr, get_order(size));

	return NULL;
}

/*
 * The exported free_coherent function for dma_ops.
 */
static void free_coherent(struct device *dev, size_t size,
			  void *virt_addr, dma_addr_t dma_addr)
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;

	if (!check_device(dev))
		return;

	get_device_resources(dev, &iommu, &domain, &devid);

	if (!iommu || !domain)
		goto free_mem;

	if (!dma_ops_domain(domain))
		goto free_mem;

	spin_lock_irqsave(&domain->lock, flags);

	__unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);

	iommu_completion_wait(iommu);

	spin_unlock_irqrestore(&domain->lock, flags);

free_mem:
	free_pages((unsigned long)virt_addr, get_order(size));
}

/*
 * This function is called by the DMA layer to find out if we can handle a
 * particular device. It is part of the dma_ops.
 */
static int amd_iommu_dma_supported(struct device *dev, u64 mask)
{
	u16 bdf;
	struct pci_dev *pcidev;

	/* No device or no PCI device */
	if (!dev || dev->bus != &pci_bus_type)
		return 0;

	pcidev = to_pci_dev(dev);

	bdf = calc_devid(pcidev->bus->number, pcidev->devfn);

	/* Out of our scope? */
	if (bdf > amd_iommu_last_bdf)
		return 0;

	return 1;
}

/*
 * The function for pre-allocating protection domains.
 *
 * If the driver core informs the DMA layer if a driver grabs a device
 * we don't need to preallocate the protection domains anymore.
 * For now we have to.
 */
void prealloc_protection_domains(void)
{
	struct pci_dev *dev = NULL;
	struct dma_ops_domain *dma_dom;
	struct amd_iommu *iommu;
	int order = amd_iommu_aperture_order;
	u16 devid;

	while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
		devid = (dev->bus->number << 8) | dev->devfn;
		if (devid > amd_iommu_last_bdf)
			continue;
		devid = amd_iommu_alias_table[devid];
		if (domain_for_device(devid))
			continue;
		iommu = amd_iommu_rlookup_table[devid];
		if (!iommu)
			continue;
		dma_dom = dma_ops_domain_alloc(iommu, order);
		if (!dma_dom)
			continue;
		init_unity_mappings_for_device(dma_dom, devid);
		dma_dom->target_dev = devid;

		list_add_tail(&dma_dom->list, &iommu_pd_list);
	}
}

static struct dma_mapping_ops amd_iommu_dma_ops = {
	.alloc_coherent = alloc_coherent,
	.free_coherent = free_coherent,
	.map_single = map_single,
	.unmap_single = unmap_single,
	.map_sg = map_sg,
	.unmap_sg = unmap_sg,
	.dma_supported = amd_iommu_dma_supported,
};

/*
 * The function which clues the AMD IOMMU driver into dma_ops.
 */
int __init amd_iommu_init_dma_ops(void)
{
	struct amd_iommu *iommu;
	int order = amd_iommu_aperture_order;
	int ret;

	/*
	 * first allocate a default protection domain for every IOMMU we
	 * found in the system. Devices not assigned to any other
	 * protection domain will be assigned to the default one.
	 */
	list_for_each_entry(iommu, &amd_iommu_list, list) {
		iommu->default_dom = dma_ops_domain_alloc(iommu, order);
		if (iommu->default_dom == NULL)
			return -ENOMEM;
		iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
		ret = iommu_init_unity_mappings(iommu);
		if (ret)
			goto free_domains;
	}

	/*
	 * If device isolation is enabled, pre-allocate the protection
	 * domains for each device.
	 */
	if (amd_iommu_isolate)
		prealloc_protection_domains();

	iommu_detected = 1;
	force_iommu = 1;
	bad_dma_address = 0;
#ifdef CONFIG_GART_IOMMU
	gart_iommu_aperture_disabled = 1;
	gart_iommu_aperture = 0;
#endif

	/* Make the driver finally visible to the drivers */
	dma_ops = &amd_iommu_dma_ops;

#ifdef CONFIG_IOMMU_API
	register_iommu(&amd_iommu_ops);
#endif

	bus_register_notifier(&pci_bus_type, &device_nb);

	return 0;

free_domains:

	list_for_each_entry(iommu, &amd_iommu_list, list) {
		if (iommu->default_dom)
			dma_ops_domain_free(iommu->default_dom);
	}

	return ret;
}

/*****************************************************************************
 *
 * The following functions belong to the exported interface of AMD IOMMU
 *
 * This interface allows access to lower level functions of the IOMMU
 * like protection domain handling and assignement of devices to domains
 * which is not possible with the dma_ops interface.
 *
 *****************************************************************************/

#ifdef CONFIG_IOMMU_API

static void cleanup_domain(struct protection_domain *domain)
{
	unsigned long flags;
	u16 devid;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
		if (amd_iommu_pd_table[devid] == domain)
			__detach_device(domain, devid);

	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

static int amd_iommu_domain_init(struct iommu_domain *dom)
{
	struct protection_domain *domain;

	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
	if (!domain)
		return -ENOMEM;

	spin_lock_init(&domain->lock);
	domain->mode = PAGE_MODE_3_LEVEL;
	domain->id = domain_id_alloc();
	if (!domain->id)
		goto out_free;
	domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
	if (!domain->pt_root)
		goto out_free;

	dom->priv = domain;

	return 0;

out_free:
	kfree(domain);

	return -ENOMEM;
}

static void amd_iommu_domain_destroy(struct iommu_domain *dom)
{
	struct protection_domain *domain = dom->priv;

	if (!domain)
		return;

	if (domain->dev_cnt > 0)
		cleanup_domain(domain);

	BUG_ON(domain->dev_cnt != 0);

	free_pagetable(domain);

	domain_id_free(domain->id);

	kfree(domain);

	dom->priv = NULL;
}

static void amd_iommu_detach_device(struct iommu_domain *dom,
				    struct device *dev)
{
	struct protection_domain *domain = dom->priv;
	struct amd_iommu *iommu;
	struct pci_dev *pdev;
	u16 devid;

	if (dev->bus != &pci_bus_type)
		return;

	pdev = to_pci_dev(dev);

	devid = calc_devid(pdev->bus->number, pdev->devfn);

	if (devid > 0)
		detach_device(domain, devid);

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		return;

	iommu_queue_inv_dev_entry(iommu, devid);
	iommu_completion_wait(iommu);
}

static int amd_iommu_attach_device(struct iommu_domain *dom,
				   struct device *dev)
{
	struct protection_domain *domain = dom->priv;
	struct protection_domain *old_domain;
	struct amd_iommu *iommu;
	struct pci_dev *pdev;
	u16 devid;

	if (dev->bus != &pci_bus_type)
		return -EINVAL;

	pdev = to_pci_dev(dev);

	devid = calc_devid(pdev->bus->number, pdev->devfn);

	if (devid >= amd_iommu_last_bdf ||
			devid != amd_iommu_alias_table[devid])
		return -EINVAL;

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		return -EINVAL;

	old_domain = domain_for_device(devid);
	if (old_domain)
		return -EBUSY;

	attach_device(iommu, domain, devid);

	iommu_completion_wait(iommu);

	return 0;
}

static int amd_iommu_map_range(struct iommu_domain *dom,
			       unsigned long iova, phys_addr_t paddr,
			       size_t size, int iommu_prot)
{
	struct protection_domain *domain = dom->priv;
	unsigned long i,  npages = iommu_num_pages(paddr, size, PAGE_SIZE);
	int prot = 0;
	int ret;

	if (iommu_prot & IOMMU_READ)
		prot |= IOMMU_PROT_IR;
	if (iommu_prot & IOMMU_WRITE)
		prot |= IOMMU_PROT_IW;

	iova  &= PAGE_MASK;
	paddr &= PAGE_MASK;

	for (i = 0; i < npages; ++i) {
		ret = iommu_map_page(domain, iova, paddr, prot);
		if (ret)
			return ret;

		iova  += PAGE_SIZE;
		paddr += PAGE_SIZE;
	}

	return 0;
}

static void amd_iommu_unmap_range(struct iommu_domain *dom,
				  unsigned long iova, size_t size)
{

	struct protection_domain *domain = dom->priv;
	unsigned long i,  npages = iommu_num_pages(iova, size, PAGE_SIZE);

	iova  &= PAGE_MASK;

	for (i = 0; i < npages; ++i) {
		iommu_unmap_page(domain, iova);
		iova  += PAGE_SIZE;
	}

	iommu_flush_domain(domain->id);
}

static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
					  unsigned long iova)
{
	struct protection_domain *domain = dom->priv;
	unsigned long offset = iova & ~PAGE_MASK;
	phys_addr_t paddr;
	u64 *pte;

	pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(iova)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return 0;

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L1_INDEX(iova)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return 0;

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L0_INDEX(iova)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return 0;

	paddr  = *pte & IOMMU_PAGE_MASK;
	paddr |= offset;

	return paddr;
}

static struct iommu_ops amd_iommu_ops = {
	.domain_init = amd_iommu_domain_init,
	.domain_destroy = amd_iommu_domain_destroy,
	.attach_dev = amd_iommu_attach_device,
	.detach_dev = amd_iommu_detach_device,
	.map = amd_iommu_map_range,
	.unmap = amd_iommu_unmap_range,
	.iova_to_phys = amd_iommu_iova_to_phys,
};

#endif