crypto: caam - Remove GFP_DMA and add DMA alignment padding (199354d7) · Commits · jan.koester / Linux

drivers/crypto/caam/blob_gen.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -83,7 +83,7 @@ int caam_process_blob(struct caam_blob_priv *priv,
	output_len = info->input_len - CAAM_BLOB_OVERHEAD;		output_len = info->input_len - CAAM_BLOB_OVERHEAD;
	}		}

	desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL \| GFP_DMA);		desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL);
	if (!desc)		if (!desc)
	return -ENOMEM;		return -ENOMEM;

drivers/crypto/caam/caamalg.c

+10 −6

Original line number	Original line	Diff line number	Diff line
	@@ -59,6 +59,8 @@
	#include <crypto/engine.h>		#include <crypto/engine.h>
	#include <crypto/xts.h>		#include <crypto/xts.h>
	#include <asm/unaligned.h>		#include <asm/unaligned.h>
			#include <linux/dma-mapping.h>
			#include <linux/kernel.h>

	/*		/*
	* crypto alg		* crypto alg
	@@ -1379,8 +1381,7 @@ static struct aead_edesc aead_edesc_alloc(struct aead_request req,
	sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);		sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);

	/* allocate space for base edesc and hw desc commands, link tables */		/* allocate space for base edesc and hw desc commands, link tables */
	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,		edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, flags);
	GFP_DMA \| flags);
	if (!edesc) {		if (!edesc) {
	caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
	0, 0, 0);		0, 0, 0);
	@@ -1608,6 +1609,7 @@ static struct skcipher_edesc skcipher_edesc_alloc(struct skcipher_request req,
	u8 *iv;		u8 *iv;
	int ivsize = crypto_skcipher_ivsize(skcipher);		int ivsize = crypto_skcipher_ivsize(skcipher);
	int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;		int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
			unsigned int aligned_size;

	src_nents = sg_nents_for_len(req->src, req->cryptlen);		src_nents = sg_nents_for_len(req->src, req->cryptlen);
	if (unlikely(src_nents < 0)) {		if (unlikely(src_nents < 0)) {
	@@ -1681,15 +1683,18 @@ static struct skcipher_edesc skcipher_edesc_alloc(struct skcipher_request req,
	/*		/*
	* allocate space for base edesc and hw desc commands, link tables, IV		* allocate space for base edesc and hw desc commands, link tables, IV
	*/		*/
	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,		aligned_size = ALIGN(ivsize, __alignof__(*edesc));
	GFP_DMA \| flags);		aligned_size += sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
	if (!edesc) {		aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());
			iv = kzalloc(aligned_size, flags);
			if (!iv) {
	dev_err(jrdev, "could not allocate extended descriptor\n");		dev_err(jrdev, "could not allocate extended descriptor\n");
	caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
	0, 0, 0);		0, 0, 0);
	return ERR_PTR(-ENOMEM);		return ERR_PTR(-ENOMEM);
	}		}

			edesc = (void )(iv + ALIGN(ivsize, __alignof__(edesc)));
	edesc->src_nents = src_nents;		edesc->src_nents = src_nents;
	edesc->dst_nents = dst_nents;		edesc->dst_nents = dst_nents;
	edesc->mapped_src_nents = mapped_src_nents;		edesc->mapped_src_nents = mapped_src_nents;
	@@ -1701,7 +1706,6 @@ static struct skcipher_edesc skcipher_edesc_alloc(struct skcipher_request req,

	/* Make sure IV is located in a DMAable area */		/* Make sure IV is located in a DMAable area */
	if (ivsize) {		if (ivsize) {
	iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
	memcpy(iv, req->iv, ivsize);		memcpy(iv, req->iv, ivsize);

	iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);		iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);

drivers/crypto/caam/caamalg_qi.c

+10 −6

Original line number	Original line	Diff line number	Diff line
	@@ -20,6 +20,8 @@
	#include "caamalg_desc.h"		#include "caamalg_desc.h"
	#include <crypto/xts.h>		#include <crypto/xts.h>
	#include <asm/unaligned.h>		#include <asm/unaligned.h>
			#include <linux/dma-mapping.h>
			#include <linux/kernel.h>

	/*		/*
	* crypto alg		* crypto alg
	@@ -959,7 +961,7 @@ static struct aead_edesc aead_edesc_alloc(struct aead_request req,
	return (struct aead_edesc *)drv_ctx;		return (struct aead_edesc *)drv_ctx;

	/* allocate space for base edesc and hw desc commands, link tables */		/* allocate space for base edesc and hw desc commands, link tables */
	edesc = qi_cache_alloc(GFP_DMA \| flags);		edesc = qi_cache_alloc(flags);
	if (unlikely(!edesc)) {		if (unlikely(!edesc)) {
	dev_err(qidev, "could not allocate extended descriptor\n");		dev_err(qidev, "could not allocate extended descriptor\n");
	return ERR_PTR(-ENOMEM);		return ERR_PTR(-ENOMEM);
	@@ -1317,8 +1319,9 @@ static struct skcipher_edesc skcipher_edesc_alloc(struct skcipher_request req,
	qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);		qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);

	qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);		qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
	if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +		if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) +
	ivsize > CAAM_QI_MEMCACHE_SIZE)) {		offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes >
			CAAM_QI_MEMCACHE_SIZE)) {
	dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",		dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
	qm_sg_ents, ivsize);		qm_sg_ents, ivsize);
	caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
	@@ -1327,17 +1330,18 @@ static struct skcipher_edesc skcipher_edesc_alloc(struct skcipher_request req,
	}		}

	/* allocate space for base edesc, link tables and IV */		/* allocate space for base edesc, link tables and IV */
	edesc = qi_cache_alloc(GFP_DMA \| flags);		iv = qi_cache_alloc(flags);
	if (unlikely(!edesc)) {		if (unlikely(!iv)) {
	dev_err(qidev, "could not allocate extended descriptor\n");		dev_err(qidev, "could not allocate extended descriptor\n");
	caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
	0, DMA_NONE, 0, 0);		0, DMA_NONE, 0, 0);
	return ERR_PTR(-ENOMEM);		return ERR_PTR(-ENOMEM);
	}		}

			edesc = (void )(iv + ALIGN(ivsize, __alignof__(edesc)));

	/* Make sure IV is located in a DMAable area */		/* Make sure IV is located in a DMAable area */
	sg_table = &edesc->sgt[0];		sg_table = &edesc->sgt[0];
	iv = (u8 *)(sg_table + qm_sg_ents);
	memcpy(iv, req->iv, ivsize);		memcpy(iv, req->iv, ivsize);

	iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);		iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);

drivers/crypto/caam/caamalg_qi2.c

+29 −19

Original line number	Original line	Diff line number	Diff line
	@@ -16,7 +16,9 @@
	#include "caamalg_desc.h"		#include "caamalg_desc.h"
	#include "caamhash_desc.h"		#include "caamhash_desc.h"
	#include "dpseci-debugfs.h"		#include "dpseci-debugfs.h"
			#include <linux/dma-mapping.h>
	#include <linux/fsl/mc.h>		#include <linux/fsl/mc.h>
			#include <linux/kernel.h>
	#include <soc/fsl/dpaa2-io.h>		#include <soc/fsl/dpaa2-io.h>
	#include <soc/fsl/dpaa2-fd.h>		#include <soc/fsl/dpaa2-fd.h>
	#include <crypto/xts.h>		#include <crypto/xts.h>
	@@ -370,7 +372,7 @@ static struct aead_edesc aead_edesc_alloc(struct aead_request req,
	struct dpaa2_sg_entry *sg_table;		struct dpaa2_sg_entry *sg_table;

	/* allocate space for base edesc, link tables and IV */		/* allocate space for base edesc, link tables and IV */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (unlikely(!edesc)) {		if (unlikely(!edesc)) {
	dev_err(dev, "could not allocate extended descriptor\n");		dev_err(dev, "could not allocate extended descriptor\n");
	return ERR_PTR(-ENOMEM);		return ERR_PTR(-ENOMEM);
	@@ -1189,7 +1191,7 @@ static struct skcipher_edesc skcipher_edesc_alloc(struct skcipher_request req)
	}		}

	/* allocate space for base edesc, link tables and IV */		/* allocate space for base edesc, link tables and IV */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (unlikely(!edesc)) {		if (unlikely(!edesc)) {
	dev_err(dev, "could not allocate extended descriptor\n");		dev_err(dev, "could not allocate extended descriptor\n");
	caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
	@@ -3220,14 +3222,14 @@ static int hash_digest_key(struct caam_hash_ctx ctx, u32 keylen, u8 *key,
	int ret = -ENOMEM;		int ret = -ENOMEM;
	struct dpaa2_fl_entry in_fle, out_fle;		struct dpaa2_fl_entry in_fle, out_fle;

	req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL \| GFP_DMA);		req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL);
	if (!req_ctx)		if (!req_ctx)
	return -ENOMEM;		return -ENOMEM;

	in_fle = &req_ctx->fd_flt[1];		in_fle = &req_ctx->fd_flt[1];
	out_fle = &req_ctx->fd_flt[0];		out_fle = &req_ctx->fd_flt[0];

	flc = kzalloc(sizeof(*flc), GFP_KERNEL \| GFP_DMA);		flc = kzalloc(sizeof(*flc), GFP_KERNEL);
	if (!flc)		if (!flc)
	goto err_flc;		goto err_flc;

	@@ -3316,7 +3318,13 @@ static int ahash_setkey(struct crypto_ahash ahash, const u8 key,
	dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);		dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);

	if (keylen > blocksize) {		if (keylen > blocksize) {
	hashed_key = kmemdup(key, keylen, GFP_KERNEL \| GFP_DMA);		unsigned int aligned_len =
			ALIGN(keylen, dma_get_cache_alignment());

			if (aligned_len < keylen)
			return -EOVERFLOW;

			hashed_key = kmemdup(key, aligned_len, GFP_KERNEL);
	if (!hashed_key)		if (!hashed_key)
	return -ENOMEM;		return -ENOMEM;
	ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);		ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
	@@ -3560,7 +3568,7 @@ static int ahash_update_ctx(struct ahash_request *req)
	}		}

	/* allocate space for base edesc and link tables */		/* allocate space for base edesc and link tables */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (!edesc) {		if (!edesc) {
	dma_unmap_sg(ctx->dev, req->src, src_nents,		dma_unmap_sg(ctx->dev, req->src, src_nents,
	DMA_TO_DEVICE);		DMA_TO_DEVICE);
	@@ -3654,7 +3662,7 @@ static int ahash_final_ctx(struct ahash_request *req)
	int ret;		int ret;

	/* allocate space for base edesc and link tables */		/* allocate space for base edesc and link tables */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (!edesc)		if (!edesc)
	return -ENOMEM;		return -ENOMEM;

	@@ -3743,7 +3751,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
	}		}

	/* allocate space for base edesc and link tables */		/* allocate space for base edesc and link tables */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (!edesc) {		if (!edesc) {
	dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
	return -ENOMEM;		return -ENOMEM;
	@@ -3836,7 +3844,7 @@ static int ahash_digest(struct ahash_request *req)
	}		}

	/* allocate space for base edesc and link tables */		/* allocate space for base edesc and link tables */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (!edesc) {		if (!edesc) {
	dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
	return ret;		return ret;
	@@ -3913,7 +3921,7 @@ static int ahash_final_no_ctx(struct ahash_request *req)
	int ret = -ENOMEM;		int ret = -ENOMEM;

	/* allocate space for base edesc and link tables */		/* allocate space for base edesc and link tables */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (!edesc)		if (!edesc)
	return ret;		return ret;

	@@ -4012,7 +4020,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
	}		}

	/* allocate space for base edesc and link tables */		/* allocate space for base edesc and link tables */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (!edesc) {		if (!edesc) {
	dma_unmap_sg(ctx->dev, req->src, src_nents,		dma_unmap_sg(ctx->dev, req->src, src_nents,
	DMA_TO_DEVICE);		DMA_TO_DEVICE);
	@@ -4125,7 +4133,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
	}		}

	/* allocate space for base edesc and link tables */		/* allocate space for base edesc and link tables */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (!edesc) {		if (!edesc) {
	dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
	return ret;		return ret;
	@@ -4230,7 +4238,7 @@ static int ahash_update_first(struct ahash_request *req)
	}		}

	/* allocate space for base edesc and link tables */		/* allocate space for base edesc and link tables */
	edesc = qi_cache_zalloc(GFP_DMA \| flags);		edesc = qi_cache_zalloc(flags);
	if (!edesc) {		if (!edesc) {
	dma_unmap_sg(ctx->dev, req->src, src_nents,		dma_unmap_sg(ctx->dev, req->src, src_nents,
	DMA_TO_DEVICE);		DMA_TO_DEVICE);
	@@ -4926,6 +4934,7 @@ static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
	{		{
	struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };		struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
	struct device *dev = priv->dev;		struct device *dev = priv->dev;
			unsigned int alignmask;
	int err;		int err;

	/*		/*
	@@ -4936,13 +4945,14 @@ static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
	!(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))		!(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
	return 0;		return 0;

	priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,		alignmask = DPAA2_CSCN_ALIGN - 1;
	GFP_KERNEL \| GFP_DMA);		alignmask \|= dma_get_cache_alignment() - 1;
			priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1),
			GFP_KERNEL);
	if (!priv->cscn_mem)		if (!priv->cscn_mem)
	return -ENOMEM;		return -ENOMEM;

	priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);		priv->cscn_dma = dma_map_single(dev, priv->cscn_mem,
	priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
	DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);		DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
	if (dma_mapping_error(dev, priv->cscn_dma)) {		if (dma_mapping_error(dev, priv->cscn_dma)) {
	dev_err(dev, "Error mapping CSCN memory area\n");		dev_err(dev, "Error mapping CSCN memory area\n");
	@@ -5174,7 +5184,7 @@ static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
	priv->domain = iommu_get_domain_for_dev(dev);		priv->domain = iommu_get_domain_for_dev(dev);

	qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,		qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
	0, SLAB_CACHE_DMA, NULL);		0, 0, NULL);
	if (!qi_cache) {		if (!qi_cache) {
	dev_err(dev, "Can't allocate SEC cache\n");		dev_err(dev, "Can't allocate SEC cache\n");
	return -ENOMEM;		return -ENOMEM;
	@@ -5451,7 +5461,7 @@ int dpaa2_caam_enqueue(struct device dev, struct caam_request req)
	dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,		dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
	DPAA2_CSCN_SIZE,		DPAA2_CSCN_SIZE,
	DMA_FROM_DEVICE);		DMA_FROM_DEVICE);
	if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {		if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) {
	dev_dbg_ratelimited(dev, "Dropping request\n");		dev_dbg_ratelimited(dev, "Dropping request\n");
	return -EBUSY;		return -EBUSY;
	}		}

drivers/crypto/caam/caamalg_qi2.h

+4 −6

Original line number	Original line	Diff line number	Diff line
	@@ -7,13 +7,14 @@
	#ifndef _CAAMALG_QI2_H_		#ifndef _CAAMALG_QI2_H_
	#define _CAAMALG_QI2_H_		#define _CAAMALG_QI2_H_

			#include <crypto/internal/skcipher.h>
			#include <linux/compiler_attributes.h>
	#include <soc/fsl/dpaa2-io.h>		#include <soc/fsl/dpaa2-io.h>
	#include <soc/fsl/dpaa2-fd.h>		#include <soc/fsl/dpaa2-fd.h>
	#include <linux/threads.h>		#include <linux/threads.h>
	#include <linux/netdevice.h>		#include <linux/netdevice.h>
	#include "dpseci.h"		#include "dpseci.h"
	#include "desc_constr.h"		#include "desc_constr.h"
	#include <crypto/skcipher.h>

	#define DPAA2_CAAM_STORE_SIZE 16		#define DPAA2_CAAM_STORE_SIZE 16
	/* NAPI weight must be a multiple of the store size. */		/* NAPI weight must be a multiple of the store size. */
	@@ -36,8 +37,6 @@
	* @tx_queue_attr: array of Tx queue attributes		* @tx_queue_attr: array of Tx queue attributes
	* @cscn_mem: pointer to memory region containing the congestion SCN		* @cscn_mem: pointer to memory region containing the congestion SCN
	* it's size is larger than to accommodate alignment		* it's size is larger than to accommodate alignment
	* @cscn_mem_aligned: pointer to congestion SCN; it is computed as
	* PTR_ALIGN(cscn_mem, DPAA2_CSCN_ALIGN)
	* @cscn_dma: dma address used by the QMAN to write CSCN messages		* @cscn_dma: dma address used by the QMAN to write CSCN messages
	* @dev: device associated with the DPSECI object		* @dev: device associated with the DPSECI object
	* @mc_io: pointer to MC portal's I/O object		* @mc_io: pointer to MC portal's I/O object
	@@ -58,7 +57,6 @@ struct dpaa2_caam_priv {

	/* congestion */		/* congestion */
	void *cscn_mem;		void *cscn_mem;
	void *cscn_mem_aligned;
	dma_addr_t cscn_dma;		dma_addr_t cscn_dma;

	struct device *dev;		struct device *dev;
	@@ -158,7 +156,7 @@ struct ahash_edesc {
	struct caam_flc {		struct caam_flc {
	u32 flc[16];		u32 flc[16];
	u32 sh_desc[MAX_SDLEN];		u32 sh_desc[MAX_SDLEN];
	} ____cacheline_aligned;		} __aligned(CRYPTO_DMA_ALIGN);

	enum optype {		enum optype {
	ENCRYPT = 0,		ENCRYPT = 0,
	@@ -180,7 +178,7 @@ enum optype {
	* @edesc: extended descriptor; points to one of {skcipher,aead}_edesc		* @edesc: extended descriptor; points to one of {skcipher,aead}_edesc
	*/		*/
	struct caam_request {		struct caam_request {
	struct dpaa2_fl_entry fd_flt[2];		struct dpaa2_fl_entry fd_flt[2] __aligned(CRYPTO_DMA_ALIGN);
	dma_addr_t fd_flt_dma;		dma_addr_t fd_flt_dma;
	struct caam_flc *flc;		struct caam_flc *flc;
	dma_addr_t flc_dma;		dma_addr_t flc_dma;