crypto: tegra: Use separate buffer for each host1x command

Allocate separate buffer for each host1x command. The single buffer used
for host1x commands can get corrupted when independent crypto operations
overlap.

Bug 4883011

Signed-off-by: Akhil R <akhilrajeev@nvidia.com>
Change-Id: I43029364c8e65e5014a5b7068cb45225c039aaf8
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/3248913
(cherry picked from commit f6c3d49e92)
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/3262770
Reviewed-by: Brad Griffis <bgriffis@nvidia.com>
GVS: buildbot_gerritrpt <buildbot_gerritrpt@nvidia.com>
Tested-by: Brad Griffis <bgriffis@nvidia.com>
Reviewed-by: Amulya Yarlagadda <ayarlagadda@nvidia.com>

drivers/crypto/tegra/tegra-se-aes.c

@@ -42,6 +42,7 @@ struct tegra_aes_reqctx {
 	u32 crypto_config;
 	u32 len;
 	u32 *iv;
+	u32 *cmdbuf;
 };
 
 struct tegra_aead_ctx {
@@ -69,6 +70,7 @@ struct tegra_aead_reqctx {
 	u32 key_id;
 	u32 iv[4];
 	u8 authdata[16];
+	u32 *cmdbuf;
 };
 
 struct tegra_cmac_ctx {
@@ -93,6 +95,7 @@ struct tegra_cmac_reqctx {
 	u32 key_id;
 	u32 *iv;
 	u32 result[CMAC_RESULT_REG_COUNT];
+	u32 *cmdbuf;
 };
 
 /* increment counter (128-bit int) */
@@ -213,7 +216,7 @@ static int tegra234_aes_cfg(u32 alg, bool encrypt)
 static unsigned int tegra_aes_prep_cmd(struct tegra_se *se, struct tegra_aes_reqctx *rctx)
 {
 	unsigned int data_count, res_bits, i = 0, j;
-	u32 *cpuvaddr = se->cmdbuf->addr;
+	u32 *cpuvaddr = rctx->cmdbuf;
 	dma_addr_t addr = rctx->datbuf.addr;
 
 	data_count = rctx->len / AES_BLOCK_SIZE;
@@ -292,16 +295,17 @@ static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq)
 
 	/* Prepare the command and submit for execution */
 	cmdlen = tegra_aes_prep_cmd(se, rctx);
-	ret = tegra_se_host1x_submit(se, cmdlen);
+	ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen);
 
 	/* Copy the result */
 	tegra_aes_update_iv(req, ctx);
 	scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1);
 
 	/* Free the buffer */
-	dma_free_coherent(ctx->se->dev, rctx->datbuf.size,
+	dma_free_coherent(se->dev, rctx->datbuf.size,
 			  rctx->datbuf.buf, rctx->datbuf.addr);
 
+	kfree(rctx->cmdbuf);
 	crypto_finalize_skcipher_request(se->engine, req, ret);
 
 	return 0;
@@ -459,6 +463,13 @@ static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt)
 	if (!req->cryptlen)
 		return 0;
 
+	if (ctx->alg == SE_ALG_ECB)
+		req->iv = NULL;
+
+	rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL);
+	if (!rctx->cmdbuf)
+		return -ENOMEM;
+
 	rctx->encrypt = encrypt;
 	rctx->config = tegra234_aes_cfg(ctx->alg, encrypt);
 	rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, encrypt);
@@ -597,7 +608,7 @@ static struct tegra_se_alg tegra_aes_algs[] = {
 static unsigned int tegra_gmac_prep_cmd(struct tegra_se *se, struct tegra_aead_reqctx *rctx)
 {
 	unsigned int data_count, res_bits, i = 0;
-	u32 *cpuvaddr = se->cmdbuf->addr;
+	u32 *cpuvaddr = rctx->cmdbuf;
 
 	data_count = (rctx->assoclen / AES_BLOCK_SIZE);
 	res_bits = (rctx->assoclen % AES_BLOCK_SIZE) * 8;
@@ -635,7 +646,7 @@ static unsigned int tegra_gmac_prep_cmd(struct tegra_se *se, struct tegra_aead_r
 static unsigned int tegra_gcm_crypt_prep_cmd(struct tegra_se *se, struct tegra_aead_reqctx *rctx)
 {
 	unsigned int data_count, res_bits, i = 0, j;
-	u32 *cpuvaddr = se->cmdbuf->addr, op;
+	u32 *cpuvaddr = rctx->cmdbuf, op;
 
 	data_count = (rctx->cryptlen / AES_BLOCK_SIZE);
 	res_bits = (rctx->cryptlen % AES_BLOCK_SIZE) * 8;
@@ -755,7 +766,7 @@ static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqct
 
 	cmdlen = tegra_gmac_prep_cmd(se, rctx);
 
-	return tegra_se_host1x_submit(se, cmdlen);
+	return tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen);
 }
 
 static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
@@ -772,7 +783,7 @@ static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc
 
 	/* Prepare command and submit */
 	cmdlen = tegra_gcm_crypt_prep_cmd(se, rctx);
-	ret = tegra_se_host1x_submit(se, cmdlen);
+	ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen);
 	if (ret)
 		return ret;
 
@@ -786,7 +797,7 @@ static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc
 static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
 {
 	struct tegra_se *se = ctx->se;
-	u32 *cpuvaddr = se->cmdbuf->addr;
+	u32 *cpuvaddr = rctx->cmdbuf;
 	int cmdlen, ret, offset;
 
 	rctx->config = tegra234_aes_cfg(SE_ALG_GCM_FINAL, rctx->encrypt);
@@ -795,7 +806,7 @@ static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc
 
 	/* Prepare command and submit */
 	cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx);
-	ret = tegra_se_host1x_submit(se, cmdlen);
+	ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen);
 	if (ret)
 		return ret;
 
@@ -840,7 +851,7 @@ static inline int tegra_ccm_check_iv(const u8 *iv)
 static unsigned int tegra_cbcmac_prep_cmd(struct tegra_se *se, struct tegra_aead_reqctx *rctx)
 {
 	unsigned int data_count, i = 0;
-	u32 *cpuvaddr = se->cmdbuf->addr;
+	u32 *cpuvaddr = rctx->cmdbuf;
 
 	data_count = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1;
 
@@ -873,7 +884,7 @@ static unsigned int tegra_cbcmac_prep_cmd(struct tegra_se *se, struct tegra_aead
 static unsigned int tegra_ctr_prep_cmd(struct tegra_se *se, struct tegra_aead_reqctx *rctx)
 {
 	unsigned int i = 0, j;
-	u32 *cpuvaddr = se->cmdbuf->addr;
+	u32 *cpuvaddr = rctx->cmdbuf;
 
 	cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
 	cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
@@ -923,7 +934,7 @@ static int tegra_ccm_do_cbcmac(struct tegra_aead_ctx *ctx, struct tegra_aead_req
 	/* Prepare command and submit */
 	cmdlen = tegra_cbcmac_prep_cmd(se, rctx);
 
-	return tegra_se_host1x_submit(se, cmdlen);
+	return tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen);
 }
 
 static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize)
@@ -1130,7 +1141,7 @@ static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx
 
 	/* Prepare command and submit */
 	cmdlen = tegra_ctr_prep_cmd(se, rctx);
-	ret = tegra_se_host1x_submit(se, cmdlen);
+	ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen);
 	if (ret)
 		return ret;
 
@@ -1231,6 +1242,7 @@ outbuf_err:
 	dma_free_coherent(ctx->se->dev, rctx->inbuf.size,
 			  rctx->inbuf.buf, rctx->inbuf.addr);
 
+	kfree(rctx->cmdbuf);
 	crypto_finalize_aead_request(ctx->se->engine, req, ret);
 
 	return 0;
@@ -1304,6 +1316,7 @@ outbuf_err:
 			  rctx->inbuf.buf, rctx->inbuf.addr);
 
 	/* Finalize the request if there are no errors */
+	kfree(rctx->cmdbuf);
 	crypto_finalize_aead_request(ctx->se->engine, req, ret);
 
 	return 0;
@@ -1434,6 +1447,10 @@ static int tegra_aead_crypt(struct aead_request *req, bool encrypt)
 	struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm);
 	struct tegra_aead_reqctx *rctx = aead_request_ctx(req);
 
+	rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL);
+	if (!rctx->cmdbuf)
+		return -ENOMEM;
+
 	rctx->encrypt = encrypt;
 
 	return crypto_transfer_aead_request_to_engine(ctx->se->engine, req);
@@ -1465,7 +1482,7 @@ static int tegra_aead_setkey(struct crypto_aead *tfm,
 static unsigned int tegra_cmac_prep_cmd(struct tegra_se *se, struct tegra_cmac_reqctx *rctx)
 {
 	unsigned int data_count, res_bits = 0, i = 0, j;
-	u32 *cpuvaddr = se->cmdbuf->addr, op;
+	u32 *cpuvaddr = rctx->cmdbuf, op;
 
 	data_count = (rctx->datbuf.size / AES_BLOCK_SIZE);
 
@@ -1600,7 +1617,7 @@ static int tegra_cmac_do_update(struct ahash_request *req)
 
 	cmdlen = tegra_cmac_prep_cmd(se, rctx);
 
-	ret = tegra_se_host1x_submit(se, cmdlen);
+	ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen);
 	/*
 	 * If this is not the final update, copy the intermediate results
 	 * from the registers so that it can be used in the next 'update'
@@ -1645,7 +1662,7 @@ static int tegra_cmac_do_final(struct ahash_request *req)
 
 	/* Prepare command and submit */
 	cmdlen = tegra_cmac_prep_cmd(se, rctx);
-	ret = tegra_se_host1x_submit(se, cmdlen);
+	ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen);
 	if (ret)
 		goto out;
 
@@ -1663,6 +1680,8 @@ out:
 out_free:
 	dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm) * 2,
 			  rctx->residue.buf, rctx->residue.addr);
+	kfree(rctx->cmdbuf);
+
 	return ret;
 }
 
@@ -1773,10 +1792,14 @@ static int tegra_cmac_init(struct ahash_request *req)
 	rctx->task = SHA_FIRST;
 	rctx->blk_size = crypto_ahash_blocksize(tfm);
 
+	rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL);
+	if (!rctx->cmdbuf)
+		return -ENOMEM;
+
 	rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2,
 					       &rctx->residue.addr, GFP_KERNEL);
 	if (!rctx->residue.buf)
-		return -ENOMEM;
+		goto cmdbuf_free;
 
 	rctx->residue.size = 0;
 	rctx->datbuf.size = 0;
@@ -1786,6 +1809,11 @@ static int tegra_cmac_init(struct ahash_request *req)
 		writel(0, se->base + se->hw->regs->result + (i * 4));
 
 	return 0;
+
+cmdbuf_free:
+	kfree(rctx->cmdbuf);
+
+	return -ENOMEM;
 }
 
 static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,

drivers/crypto/tegra/tegra-se-hash.c

@@ -44,6 +44,7 @@ struct tegra_sha_reqctx {
 	unsigned int blk_size;
 	unsigned int task;
 	u32 key_id;
+	u32 *cmdbuf;
 	u32 result[HASH_RESULT_REG_COUNT];
 	struct ahash_request fallback_req;
 };
@@ -305,7 +306,7 @@ static int tegra_sha_do_update(struct ahash_request *req)
 	struct tegra_sha_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
 	struct tegra_sha_reqctx *rctx = ahash_request_ctx(req);
 	unsigned int nblks, nresidue, size, ret;
-	u32 *cpuvaddr = ctx->se->cmdbuf->addr;
+	u32 *cpuvaddr = rctx->cmdbuf;
 
 	nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size;
 	nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size;
@@ -366,7 +367,7 @@ static int tegra_sha_do_update(struct ahash_request *req)
 
 	size = tegra_sha_prep_cmd(ctx->se, cpuvaddr, rctx);
 
-	ret = tegra_se_host1x_submit(ctx->se, size);
+	ret = tegra_se_host1x_submit(ctx->se, rctx->cmdbuf, size);
 
 	/*
 	 * If this is not the final update, copy the intermediate results
@@ -388,7 +389,7 @@ static int tegra_sha_do_final(struct ahash_request *req)
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm);
 	struct tegra_se *se = ctx->se;
-	u32 *cpuvaddr = se->cmdbuf->addr;
+	u32 *cpuvaddr = rctx->cmdbuf;
 	int size, ret = 0;
 
 	rctx->datbuf.size = rctx->residue.size;
@@ -409,7 +410,7 @@ static int tegra_sha_do_final(struct ahash_request *req)
 	}
 
 	size = tegra_sha_prep_cmd(se, cpuvaddr, rctx);
-	ret = tegra_se_host1x_submit(se, size);
+	ret = tegra_se_host1x_submit(se, rctx->cmdbuf, size);
 	if (ret)
 		goto out;
 
@@ -425,6 +426,8 @@ out_free:
 			  rctx->residue.buf, rctx->residue.addr);
 	dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf,
 			  rctx->digest.addr);
+	kfree(rctx->cmdbuf);
+
 	return ret;
 }
 
@@ -549,6 +552,10 @@ static int tegra_sha_init(struct ahash_request *req)
 	rctx->blk_size = crypto_ahash_blocksize(tfm);
 	rctx->digest.size = crypto_ahash_digestsize(tfm);
 
+	rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL);
+	if (!rctx->cmdbuf)
+		return -ENOMEM;
+
 	rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size,
 					      &rctx->digest.addr, GFP_KERNEL);
 	if (!rctx->digest.buf)
@@ -565,6 +572,8 @@ resbuf_fail:
 	dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf,
 			  rctx->digest.addr);
 digbuf_fail:
+	kfree(rctx->cmdbuf);
+
 	return -ENOMEM;
 }
 

drivers/crypto/tegra/tegra-se-key.c

@@ -115,11 +115,11 @@ static int tegra_key_insert(struct tegra_se *se, const u8 *key,
 			    u32 keylen, u16 slot, u32 alg)
 {
 	const u32 *keyval = (u32 *)key;
-	u32 *addr = se->cmdbuf->addr, size;
+	u32 addr[100], size;
 
 	size = tegra_key_prep_ins_cmd(se, addr, keyval, keylen, slot, alg);
 
-	return tegra_se_host1x_submit(se, size);
+	return tegra_se_host1x_submit(se, addr, size);
 }
 
 void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg)

drivers/crypto/tegra/tegra-se-main.c

@@ -141,9 +141,10 @@ static struct tegra_se_cmdbuf *tegra_se_host1x_bo_alloc(struct tegra_se *se, ssi
 	return cmdbuf;
 }
 
-int tegra_se_host1x_submit(struct tegra_se *se, u32 size)
+int tegra_se_host1x_submit(struct tegra_se *se, u32 *cpuvaddr, u32 size)
 {
 	struct host1x_job *job;
+	struct tegra_se_cmdbuf *cmdbuf;
 	int ret;
 
 	job = host1x_job_alloc(se->channel, 1, 0, true);
@@ -152,6 +153,12 @@ int tegra_se_host1x_submit(struct tegra_se *se, u32 size)
 		return -ENOMEM;
 	}
 
+	cmdbuf = tegra_se_host1x_bo_alloc(se, SZ_4K);
+	if (!cmdbuf)
+		goto job_put;
+
+	memcpy(cmdbuf->addr, cpuvaddr, size * 4);
+
 	job->syncpt = host1x_syncpt_get(se->syncpt);
 	job->syncpt_incrs = 1;
 	job->client = &se->client;
@@ -160,14 +167,14 @@ int tegra_se_host1x_submit(struct tegra_se *se, u32 size)
 	job->engine_fallback_streamid = se->stream_id;
 	job->engine_streamid_offset = SE_STREAM_ID;
 
-	se->cmdbuf->words = size;
+	cmdbuf->words = size;
 
-	host1x_job_add_gather(job, &se->cmdbuf->bo, size, 0);
+	host1x_job_add_gather(job, &cmdbuf->bo, size, 0);
 
 	ret = host1x_job_pin(job, se->dev);
 	if (ret) {
 		dev_err(se->dev, "failed to pin host1x job\n");
-		goto job_put;
+		goto cmdbuf_put;
 	}
 
 	ret = host1x_job_submit(job);
@@ -184,10 +191,14 @@ int tegra_se_host1x_submit(struct tegra_se *se, u32 size)
 	}
 
 	host1x_job_put(job);
+	tegra_se_cmdbuf_put(&cmdbuf->bo);
+
 	return 0;
 
 job_unpin:
 	host1x_job_unpin(job);
+cmdbuf_put:
+	tegra_se_cmdbuf_put(&cmdbuf->bo);
 job_put:
 	host1x_job_put(job);
 
@@ -214,22 +225,14 @@ static int tegra_se_client_init(struct host1x_client *client)
 
 	se->syncpt_id =  host1x_syncpt_id(se->syncpt);
 
-	se->cmdbuf = tegra_se_host1x_bo_alloc(se, SZ_4K);
-	if (!se->cmdbuf) {
-		ret = -ENOMEM;
-		goto syncpt_put;
-	}
-
 	ret = se->hw->init_alg(se);
 	if (ret) {
 		dev_err(se->dev, "failed to register algorithms\n");
-		goto cmdbuf_put;
+		goto syncpt_put;
 	}
 
 	return 0;
 
-cmdbuf_put:
-	tegra_se_cmdbuf_put(&se->cmdbuf->bo);
 syncpt_put:
 	host1x_syncpt_put(se->syncpt);
 channel_put:
@@ -243,7 +246,6 @@ static int tegra_se_client_deinit(struct host1x_client *client)
 	struct tegra_se *se = container_of(client, struct tegra_se, client);
 
 	se->hw->deinit_alg(se);
-	tegra_se_cmdbuf_put(&se->cmdbuf->bo);
 	host1x_syncpt_put(se->syncpt);
 	host1x_channel_put(se->channel);
 

drivers/crypto/tegra/tegra-se.h

@@ -23,6 +23,7 @@
 #define SE_OWNERSHIP_UID(x)				FIELD_GET(GENMASK(7, 0), x)
 #define TEGRA_GPSE_ID					3
 
+#define SE_MAX_CMDLEN					(100 * 4) /* max 100 commands of 4 bytes each */
 #define SE_STREAM_ID					0x90
 
 #define SE_SHA_CFG					0x4004
@@ -441,7 +442,6 @@ struct tegra_se {
 	const struct tegra_se_hw *hw;
 	struct host1x_client client;
 	struct host1x_channel *channel;
-	struct tegra_se_cmdbuf *cmdbuf;
 	struct crypto_engine *engine;
 	struct host1x_syncpt *syncpt;
 	struct device *dev;
@@ -524,7 +524,7 @@ void tegra_deinit_hash(struct tegra_se *se);
 int tegra_key_submit(struct tegra_se *se, const u8 *key,
 		     u32 keylen, u32 alg, u32 *keyid);
 void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg);
-int tegra_se_host1x_submit(struct tegra_se *se, u32 size);
+int tegra_se_host1x_submit(struct tegra_se *se, u32 *cpuvaddr, u32 size);
 
 /* HOST1x OPCODES */
 static inline u32 host1x_opcode_setpayload(unsigned int payload)