Commit 7e7b6965 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'dma-mapping-5.17' of git://git.infradead.org/users/hch/dma-mapping 

Pull dma-mapping updates from Christoph Hellwig:

 - refactor the dma-direct coherent allocator

 - turn an macro into an inline in scatterlist.h (Logan Gunthorpe)

* tag 'dma-mapping-5.17' of git://git.infradead.org/users/hch/dma-mapping:
  lib/scatterlist: cleanup macros into static inline functions
  dma-direct: add a dma_direct_use_pool helper
  dma-direct: factor the swiotlb code out of __dma_direct_alloc_pages
  dma-direct: drop two CONFIG_DMA_RESTRICTED_POOL conditionals
  dma-direct: warn if there is no pool for force unencrypted allocations
  dma-direct: fail allocations that can't be made coherent
  dma-direct: refactor the !coherent checks in dma_direct_alloc
  dma-direct: factor out a helper for DMA_ATTR_NO_KERNEL_MAPPING allocations
  dma-direct: clean up the remapping checks in dma_direct_alloc
  dma-direct: always leak memory that can't be re-encrypted
  dma-direct: don't call dma_set_decrypted for remapped allocations
  dma-direct: factor out dma_set_{de,en}crypted helpers
parents daadb3bd f857acfc

include/linux/scatterlist.h

+23 −6
Original line number Diff line number Diff line
@@ -69,10 +69,27 @@ struct sg_append_table { 
 * a valid sg entry, or whether it points to the start of a new scatterlist.

 * Those low bits are there for everyone! (thanks mason :-)

 */

#define sg_is_chain(sg)		((sg)->page_link & SG_CHAIN)

#define sg_is_last(sg)		((sg)->page_link & SG_END)

#define sg_chain_ptr(sg)	\

	((struct scatterlist *) ((sg)->page_link & ~(SG_CHAIN | SG_END)))

#define SG_PAGE_LINK_MASK (SG_CHAIN | SG_END)



static inline unsigned int __sg_flags(struct scatterlist *sg)

{

	return sg->page_link & SG_PAGE_LINK_MASK;

}



static inline struct scatterlist *sg_chain_ptr(struct scatterlist *sg)

{

	return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK);

}



static inline bool sg_is_chain(struct scatterlist *sg)

{

	return __sg_flags(sg) & SG_CHAIN;

}



static inline bool sg_is_last(struct scatterlist *sg)

{

	return __sg_flags(sg) & SG_END;

}



/**

 * sg_assign_page - Assign a given page to an SG entry
@@ -92,7 +109,7 @@ static inline void sg_assign_page(struct scatterlist *sg, struct page *page) 
	 * In order for the low bit stealing approach to work, pages

	 * must be aligned at a 32-bit boundary as a minimum.

	 */

	BUG_ON((unsigned long) page & (SG_CHAIN | SG_END));

	BUG_ON((unsigned long)page & SG_PAGE_LINK_MASK);

#ifdef CONFIG_DEBUG_SG

	BUG_ON(sg_is_chain(sg));

#endif
@@ -126,7 +143,7 @@ static inline struct page *sg_page(struct scatterlist *sg) 
#ifdef CONFIG_DEBUG_SG

	BUG_ON(sg_is_chain(sg));

#endif

	return (struct page *)((sg)->page_link & ~(SG_CHAIN | SG_END));

	return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK);

}



/**

kernel/dma/direct.c

+140 −100
Original line number Diff line number Diff line
@@ -75,15 +75,45 @@ static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) 
		min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);

}



static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size)

{

	if (!force_dma_unencrypted(dev))

		return 0;

	return set_memory_decrypted((unsigned long)vaddr, 1 << get_order(size));

}



static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)

{

	int ret;



	if (!force_dma_unencrypted(dev))

		return 0;

	ret = set_memory_encrypted((unsigned long)vaddr, 1 << get_order(size));

	if (ret)

		pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");

	return ret;

}



static void __dma_direct_free_pages(struct device *dev, struct page *page,

				    size_t size)

{

	if (IS_ENABLED(CONFIG_DMA_RESTRICTED_POOL) &&

	    swiotlb_free(dev, page, size))

	if (swiotlb_free(dev, page, size))

		return;

	dma_free_contiguous(dev, page, size);

}



static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)

{

	struct page *page = swiotlb_alloc(dev, size);



	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {

		swiotlb_free(dev, page, size);

		return NULL;

	}



	return page;

}



static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,

		gfp_t gfp)

{
@@ -93,18 +123,11 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, 


	WARN_ON_ONCE(!PAGE_ALIGNED(size));



	if (is_swiotlb_for_alloc(dev))

		return dma_direct_alloc_swiotlb(dev, size);



	gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,

					   &phys_limit);

	if (IS_ENABLED(CONFIG_DMA_RESTRICTED_POOL) &&

	    is_swiotlb_for_alloc(dev)) {

		page = swiotlb_alloc(dev, size);

		if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {

			__dma_direct_free_pages(dev, page, size);

			return NULL;

		}

		return page;

	}



	page = dma_alloc_contiguous(dev, size, gfp);

	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {

		dma_free_contiguous(dev, page, size);
@@ -133,6 +156,15 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, 
	return page;

}



/*

 * Check if a potentially blocking operations needs to dip into the atomic

 * pools for the given device/gfp.

 */

static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)

{

	return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev);

}



static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,

		dma_addr_t *dma_handle, gfp_t gfp)

{
@@ -140,6 +172,9 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, 
	u64 phys_mask;

	void *ret;



	if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)))

		return NULL;



	gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,

					   &phys_mask);

	page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);
@@ -149,64 +184,103 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, 
	return ret;

}



void *dma_direct_alloc(struct device *dev, size_t size,

		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)

static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,

		dma_addr_t *dma_handle, gfp_t gfp)

{

	struct page *page;

	void *ret;

	int err;



	size = PAGE_ALIGN(size);

	if (attrs & DMA_ATTR_NO_WARN)

		gfp |= __GFP_NOWARN;



	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&

	    !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {

	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);

	if (!page)

		return NULL;



	/* remove any dirty cache lines on the kernel alias */

	if (!PageHighMem(page))

		arch_dma_prep_coherent(page, size);

		*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));



	/* return the page pointer as the opaque cookie */

	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));

	return page;

}



void *dma_direct_alloc(struct device *dev, size_t size,

		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)

{

	bool remap = false, set_uncached = false;

	struct page *page;

	void *ret;



	size = PAGE_ALIGN(size);

	if (attrs & DMA_ATTR_NO_WARN)

		gfp |= __GFP_NOWARN;



	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&

	    !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))

		return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);



	if (!dev_is_dma_coherent(dev)) {

		/*

		 * Fallback to the arch handler if it exists.  This should

		 * eventually go away.

		 */

		if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&

		    !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&

		    !IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&

	    !dev_is_dma_coherent(dev) &&

		    !is_swiotlb_for_alloc(dev))

		return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);

			return arch_dma_alloc(dev, size, dma_handle, gfp,

					      attrs);



	if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&

	    !dev_is_dma_coherent(dev))

		return dma_alloc_from_global_coherent(dev, size, dma_handle);

		/*

		 * If there is a global pool, always allocate from it for

		 * non-coherent devices.

		 */

		if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL))

			return dma_alloc_from_global_coherent(dev, size,

					dma_handle);



		/*

	 * Remapping or decrypting memory may block. If either is required and

	 * we can't block, allocate the memory from the atomic pools.

	 * If restricted DMA (i.e., is_swiotlb_for_alloc) is required, one must

	 * set up another device coherent pool by shared-dma-pool and use

	 * dma_alloc_from_dev_coherent instead.

		 * Otherwise remap if the architecture is asking for it.  But

		 * given that remapping memory is a blocking operation we'll

		 * instead have to dip into the atomic pools.

		 */

	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&

	    !gfpflags_allow_blocking(gfp) &&

	    (force_dma_unencrypted(dev) ||

	     (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&

	      !dev_is_dma_coherent(dev))) &&

	    !is_swiotlb_for_alloc(dev))

		remap = IS_ENABLED(CONFIG_DMA_DIRECT_REMAP);

		if (remap) {

			if (dma_direct_use_pool(dev, gfp))

				return dma_direct_alloc_from_pool(dev, size,

						dma_handle, gfp);

		} else {

			if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED))

				return NULL;

			set_uncached = true;

		}

	}



	/*

	 * Decrypting memory may block, so allocate the memory from the atomic

	 * pools if we can't block.

	 */

	if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))

		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);



	/* we always manually zero the memory once we are done */

	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);

	if (!page)

		return NULL;

	if (PageHighMem(page)) {

		/*

		 * Depending on the cma= arguments and per-arch setup,

		 * dma_alloc_contiguous could return highmem pages.

		 * Without remapping there is no way to return them here, so

		 * log an error and fail.

		 */

		if (!IS_ENABLED(CONFIG_DMA_REMAP)) {

			dev_info(dev, "Rejecting highmem page from CMA.\n");

			goto out_free_pages;

		}

		remap = true;

		set_uncached = false;

	}



	if ((IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&

	     !dev_is_dma_coherent(dev)) ||

	    (IS_ENABLED(CONFIG_DMA_REMAP) && PageHighMem(page))) {

	if (remap) {

		/* remove any dirty cache lines on the kernel alias */

		arch_dma_prep_coherent(page, size);
@@ -216,56 +290,27 @@ void *dma_direct_alloc(struct device *dev, size_t size, 
				__builtin_return_address(0));

		if (!ret)

			goto out_free_pages;

		if (force_dma_unencrypted(dev)) {

			err = set_memory_decrypted((unsigned long)ret,

						   1 << get_order(size));

			if (err)

				goto out_free_pages;

		}

		memset(ret, 0, size);

		goto done;

	}



	if (PageHighMem(page)) {

		/*

		 * Depending on the cma= arguments and per-arch setup

		 * dma_alloc_contiguous could return highmem pages.

		 * Without remapping there is no way to return them here,

		 * so log an error and fail.

		 */

		dev_info(dev, "Rejecting highmem page from CMA.\n");

		goto out_free_pages;

	}



	} else {

		ret = page_address(page);

	if (force_dma_unencrypted(dev)) {

		err = set_memory_decrypted((unsigned long)ret,

					   1 << get_order(size));

		if (err)

		if (dma_set_decrypted(dev, ret, size))

			goto out_free_pages;

	}



	memset(ret, 0, size);



	if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&

	    !dev_is_dma_coherent(dev)) {

	if (set_uncached) {

		arch_dma_prep_coherent(page, size);

		ret = arch_dma_set_uncached(ret, size);

		if (IS_ERR(ret))

			goto out_encrypt_pages;

	}

done:



	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));

	return ret;



out_encrypt_pages:

	if (force_dma_unencrypted(dev)) {

		err = set_memory_encrypted((unsigned long)page_address(page),

					   1 << get_order(size));

		/* If memory cannot be re-encrypted, it must be leaked */

		if (err)

	if (dma_set_encrypted(dev, page_address(page), size))

		return NULL;

	}

out_free_pages:

	__dma_direct_free_pages(dev, page, size);

	return NULL;
@@ -304,13 +349,14 @@ void dma_direct_free(struct device *dev, size_t size, 
	    dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))

		return;



	if (force_dma_unencrypted(dev))

		set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);



	if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr))

	if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {

		vunmap(cpu_addr);

	else if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))

	} else {

		if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))

			arch_dma_clear_uncached(cpu_addr, size);

		if (dma_set_encrypted(dev, cpu_addr, 1 << page_order))

			return;

	}



	__dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size);

}
@@ -321,9 +367,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, 
	struct page *page;

	void *ret;



	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&

	    force_dma_unencrypted(dev) && !gfpflags_allow_blocking(gfp) &&

	    !is_swiotlb_for_alloc(dev))

	if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))

		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);



	page = __dma_direct_alloc_pages(dev, size, gfp);
@@ -341,11 +385,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, 
	}



	ret = page_address(page);

	if (force_dma_unencrypted(dev)) {

		if (set_memory_decrypted((unsigned long)ret,

				1 << get_order(size)))

	if (dma_set_decrypted(dev, ret, size))

		goto out_free_pages;

	}

	memset(ret, 0, size);

	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));

	return page;
@@ -366,9 +407,8 @@ void dma_direct_free_pages(struct device *dev, size_t size, 
	    dma_free_from_pool(dev, vaddr, size))

		return;



	if (force_dma_unencrypted(dev))

		set_memory_encrypted((unsigned long)vaddr, 1 << page_order);



	if (dma_set_encrypted(dev, vaddr, 1 << page_order))

		return;

	__dma_direct_free_pages(dev, page, size);

}