Commit d3fa60d7 authored by Christoph Hellwig's avatar Christoph Hellwig
Browse files

dma-mapping: move the remaining DMA API calls out of line 



For a long time the DMA API has been implemented inline in dma-mapping.h,
but the function bodies can be quite large.  Move them all out of line.

This also removes all the dma_direct_* exports as those are just
implementation details and should never be used by drivers directly.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Tested-by: default avatarAlexey Kardashevskiy <aik@ozlabs.ru>
Reviewed-by: default avatarAlexey Kardashevskiy <aik@ozlabs.ru>
parent d9765e41

include/linux/dma-direct.h

+58 −0
Original line number Diff line number Diff line
@@ -87,4 +87,62 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, 
		unsigned long attrs);

int dma_direct_supported(struct device *dev, u64 mask);

bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);

dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,

		unsigned long offset, size_t size, enum dma_data_direction dir,

		unsigned long attrs);

int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,

		enum dma_data_direction dir, unsigned long attrs);

dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,

		size_t size, enum dma_data_direction dir, unsigned long attrs);



#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \

    defined(CONFIG_SWIOTLB)

void dma_direct_sync_single_for_device(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir);

void dma_direct_sync_sg_for_device(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir);

#else

static inline void dma_direct_sync_single_for_device(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

}

static inline void dma_direct_sync_sg_for_device(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)

{

}

#endif



#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \

    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \

    defined(CONFIG_SWIOTLB)

void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs);

void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,

		int nents, enum dma_data_direction dir, unsigned long attrs);

void dma_direct_sync_single_for_cpu(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir);

void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir);

#else

static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

}

static inline void dma_direct_unmap_sg(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir,

		unsigned long attrs)

{

}

static inline void dma_direct_sync_single_for_cpu(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

}

static inline void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)

{

}

#endif



size_t dma_direct_max_mapping_size(struct device *dev);



#endif /* _LINUX_DMA_DIRECT_H */

include/linux/dma-mapping.h

+22 −225
Original line number Diff line number Diff line
@@ -188,73 +188,6 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma, 
}

#endif /* CONFIG_DMA_DECLARE_COHERENT */



static inline bool dma_is_direct(const struct dma_map_ops *ops)

{

	return likely(!ops);

}



/*

 * All the dma_direct_* declarations are here just for the indirect call bypass,

 * and must not be used directly drivers!

 */

dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,

		unsigned long offset, size_t size, enum dma_data_direction dir,

		unsigned long attrs);

int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,

		enum dma_data_direction dir, unsigned long attrs);

dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,

		size_t size, enum dma_data_direction dir, unsigned long attrs);



#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \

    defined(CONFIG_SWIOTLB)

void dma_direct_sync_single_for_device(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir);

void dma_direct_sync_sg_for_device(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir);

#else

static inline void dma_direct_sync_single_for_device(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

}

static inline void dma_direct_sync_sg_for_device(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)

{

}

#endif



#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \

    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \

    defined(CONFIG_SWIOTLB)

void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs);

void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,

		int nents, enum dma_data_direction dir, unsigned long attrs);

void dma_direct_sync_single_for_cpu(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir);

void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir);

#else

static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

}

static inline void dma_direct_unmap_sg(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir,

		unsigned long attrs)

{

}

static inline void dma_direct_sync_single_for_cpu(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

}

static inline void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)

{

}

#endif



size_t dma_direct_max_mapping_size(struct device *dev);



#ifdef CONFIG_HAS_DMA

#include <asm/dma-mapping.h>
@@ -271,164 +204,6 @@ static inline void set_dma_ops(struct device *dev, 
	dev->dma_ops = dma_ops;

}



static inline dma_addr_t dma_map_page_attrs(struct device *dev,

		struct page *page, size_t offset, size_t size,

		enum dma_data_direction dir, unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);

	dma_addr_t addr;



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);

	else

		addr = ops->map_page(dev, page, offset, size, dir, attrs);

	debug_dma_map_page(dev, page, offset, size, dir, addr);



	return addr;

}



static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_unmap_page(dev, addr, size, dir, attrs);

	else if (ops->unmap_page)

		ops->unmap_page(dev, addr, size, dir, attrs);

	debug_dma_unmap_page(dev, addr, size, dir);

}



/*

 * dma_maps_sg_attrs returns 0 on error and > 0 on success.

 * It should never return a value < 0.

 */

static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,

				   int nents, enum dma_data_direction dir,

				   unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);

	int ents;



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);

	else

		ents = ops->map_sg(dev, sg, nents, dir, attrs);

	BUG_ON(ents < 0);

	debug_dma_map_sg(dev, sg, nents, ents, dir);



	return ents;

}



static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,

				      int nents, enum dma_data_direction dir,

				      unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	debug_dma_unmap_sg(dev, sg, nents, dir);

	if (dma_is_direct(ops))

		dma_direct_unmap_sg(dev, sg, nents, dir, attrs);

	else if (ops->unmap_sg)

		ops->unmap_sg(dev, sg, nents, dir, attrs);

}



static inline dma_addr_t dma_map_resource(struct device *dev,

					  phys_addr_t phys_addr,

					  size_t size,

					  enum dma_data_direction dir,

					  unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);

	dma_addr_t addr = DMA_MAPPING_ERROR;



	BUG_ON(!valid_dma_direction(dir));



	/* Don't allow RAM to be mapped */

	if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))

		return DMA_MAPPING_ERROR;



	if (dma_is_direct(ops))

		addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);

	else if (ops->map_resource)

		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);



	debug_dma_map_resource(dev, phys_addr, size, dir, addr);

	return addr;

}



static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,

				      size_t size, enum dma_data_direction dir,

				      unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (!dma_is_direct(ops) && ops->unmap_resource)

		ops->unmap_resource(dev, addr, size, dir, attrs);

	debug_dma_unmap_resource(dev, addr, size, dir);

}



static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,

					   size_t size,

					   enum dma_data_direction dir)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_sync_single_for_cpu(dev, addr, size, dir);

	else if (ops->sync_single_for_cpu)

		ops->sync_single_for_cpu(dev, addr, size, dir);

	debug_dma_sync_single_for_cpu(dev, addr, size, dir);

}



static inline void dma_sync_single_for_device(struct device *dev,

					      dma_addr_t addr, size_t size,

					      enum dma_data_direction dir)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_sync_single_for_device(dev, addr, size, dir);

	else if (ops->sync_single_for_device)

		ops->sync_single_for_device(dev, addr, size, dir);

	debug_dma_sync_single_for_device(dev, addr, size, dir);

}



static inline void

dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,

		    int nelems, enum dma_data_direction dir)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);

	else if (ops->sync_sg_for_cpu)

		ops->sync_sg_for_cpu(dev, sg, nelems, dir);

	debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);

}



static inline void

dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,

		       int nelems, enum dma_data_direction dir)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_sync_sg_for_device(dev, sg, nelems, dir);

	else if (ops->sync_sg_for_device)

		ops->sync_sg_for_device(dev, sg, nelems, dir);

	debug_dma_sync_sg_for_device(dev, sg, nelems, dir);



}



static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)

{
@@ -439,6 +214,28 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 
	return 0;

}



dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,

		size_t offset, size_t size, enum dma_data_direction dir,

		unsigned long attrs);

void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,

		enum dma_data_direction dir, unsigned long attrs);

int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,

		enum dma_data_direction dir, unsigned long attrs);

void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,

				      int nents, enum dma_data_direction dir,

				      unsigned long attrs);

dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs);

void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,

		enum dma_data_direction dir, unsigned long attrs);

void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,

		enum dma_data_direction dir);

void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir);

void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,

		    int nelems, enum dma_data_direction dir);

void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,

		       int nelems, enum dma_data_direction dir);

void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,

		gfp_t flag, unsigned long attrs);

void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,

kernel/dma/direct.c

+0 −9
Original line number Diff line number Diff line
@@ -315,7 +315,6 @@ void dma_direct_sync_single_for_device(struct device *dev, 
	if (!dev_is_dma_coherent(dev))

		arch_sync_dma_for_device(paddr, size, dir);

}

EXPORT_SYMBOL(dma_direct_sync_single_for_device);



void dma_direct_sync_sg_for_device(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)
@@ -335,7 +334,6 @@ void dma_direct_sync_sg_for_device(struct device *dev, 
					dir);

	}

}

EXPORT_SYMBOL(dma_direct_sync_sg_for_device);

#endif



#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
@@ -354,7 +352,6 @@ void dma_direct_sync_single_for_cpu(struct device *dev, 
	if (unlikely(is_swiotlb_buffer(paddr)))

		swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);

}

EXPORT_SYMBOL(dma_direct_sync_single_for_cpu);



void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)
@@ -376,7 +373,6 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, 
	if (!dev_is_dma_coherent(dev))

		arch_sync_dma_for_cpu_all();

}

EXPORT_SYMBOL(dma_direct_sync_sg_for_cpu);



void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)
@@ -389,7 +385,6 @@ void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, 
	if (unlikely(is_swiotlb_buffer(phys)))

		swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);

}

EXPORT_SYMBOL(dma_direct_unmap_page);



void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,

		int nents, enum dma_data_direction dir, unsigned long attrs)
@@ -401,7 +396,6 @@ void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, 
		dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir,

			     attrs);

}

EXPORT_SYMBOL(dma_direct_unmap_sg);

#endif



dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
@@ -428,7 +422,6 @@ dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, 
		arch_sync_dma_for_device(phys, size, dir);

	return dma_addr;

}

EXPORT_SYMBOL(dma_direct_map_page);



int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,

		enum dma_data_direction dir, unsigned long attrs)
@@ -450,7 +443,6 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, 
	dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);

	return 0;

}

EXPORT_SYMBOL(dma_direct_map_sg);



dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)
@@ -467,7 +459,6 @@ dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, 


	return dma_addr;

}

EXPORT_SYMBOL(dma_direct_map_resource);



int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,

		void *cpu_addr, dma_addr_t dma_addr, size_t size,

kernel/dma/mapping.c

+164 −0
Original line number Diff line number Diff line
@@ -105,6 +105,170 @@ void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 
}

EXPORT_SYMBOL(dmam_alloc_attrs);



static inline bool dma_is_direct(const struct dma_map_ops *ops)

{

	return likely(!ops);

}



dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,

		size_t offset, size_t size, enum dma_data_direction dir,

		unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);

	dma_addr_t addr;



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);

	else

		addr = ops->map_page(dev, page, offset, size, dir, attrs);

	debug_dma_map_page(dev, page, offset, size, dir, addr);



	return addr;

}

EXPORT_SYMBOL(dma_map_page_attrs);



void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,

		enum dma_data_direction dir, unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_unmap_page(dev, addr, size, dir, attrs);

	else if (ops->unmap_page)

		ops->unmap_page(dev, addr, size, dir, attrs);

	debug_dma_unmap_page(dev, addr, size, dir);

}

EXPORT_SYMBOL(dma_unmap_page_attrs);



/*

 * dma_maps_sg_attrs returns 0 on error and > 0 on success.

 * It should never return a value < 0.

 */

int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,

		enum dma_data_direction dir, unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);

	int ents;



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);

	else

		ents = ops->map_sg(dev, sg, nents, dir, attrs);

	BUG_ON(ents < 0);

	debug_dma_map_sg(dev, sg, nents, ents, dir);



	return ents;

}

EXPORT_SYMBOL(dma_map_sg_attrs);



void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,

				      int nents, enum dma_data_direction dir,

				      unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	debug_dma_unmap_sg(dev, sg, nents, dir);

	if (dma_is_direct(ops))

		dma_direct_unmap_sg(dev, sg, nents, dir, attrs);

	else if (ops->unmap_sg)

		ops->unmap_sg(dev, sg, nents, dir, attrs);

}

EXPORT_SYMBOL(dma_unmap_sg_attrs);



dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);

	dma_addr_t addr = DMA_MAPPING_ERROR;



	BUG_ON(!valid_dma_direction(dir));



	/* Don't allow RAM to be mapped */

	if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))

		return DMA_MAPPING_ERROR;



	if (dma_is_direct(ops))

		addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);

	else if (ops->map_resource)

		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);



	debug_dma_map_resource(dev, phys_addr, size, dir, addr);

	return addr;

}

EXPORT_SYMBOL(dma_map_resource);



void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,

		enum dma_data_direction dir, unsigned long attrs)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (!dma_is_direct(ops) && ops->unmap_resource)

		ops->unmap_resource(dev, addr, size, dir, attrs);

	debug_dma_unmap_resource(dev, addr, size, dir);

}

EXPORT_SYMBOL(dma_unmap_resource);



void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,

		enum dma_data_direction dir)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_sync_single_for_cpu(dev, addr, size, dir);

	else if (ops->sync_single_for_cpu)

		ops->sync_single_for_cpu(dev, addr, size, dir);

	debug_dma_sync_single_for_cpu(dev, addr, size, dir);

}

EXPORT_SYMBOL(dma_sync_single_for_cpu);



void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_sync_single_for_device(dev, addr, size, dir);

	else if (ops->sync_single_for_device)

		ops->sync_single_for_device(dev, addr, size, dir);

	debug_dma_sync_single_for_device(dev, addr, size, dir);

}

EXPORT_SYMBOL(dma_sync_single_for_device);



void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,

		    int nelems, enum dma_data_direction dir)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);

	else if (ops->sync_sg_for_cpu)

		ops->sync_sg_for_cpu(dev, sg, nelems, dir);

	debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);

}

EXPORT_SYMBOL(dma_sync_sg_for_cpu);



void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,

		       int nelems, enum dma_data_direction dir)

{

	const struct dma_map_ops *ops = get_dma_ops(dev);



	BUG_ON(!valid_dma_direction(dir));

	if (dma_is_direct(ops))

		dma_direct_sync_sg_for_device(dev, sg, nelems, dir);

	else if (ops->sync_sg_for_device)

		ops->sync_sg_for_device(dev, sg, nelems, dir);

	debug_dma_sync_sg_for_device(dev, sg, nelems, dir);

}

EXPORT_SYMBOL(dma_sync_sg_for_device);



/*

 * Create scatter-list for the already allocated DMA buffer.

 */