From a49c68b4e2191d62a0504061a438e2cd9c977e9e Mon Sep 17 00:00:00 2001 From: Akhil R Date: Thu, 14 Nov 2024 18:58:30 +0530 Subject: [PATCH] crypto: tegra: Use separate buffer for each host1x command Allocate separate buffer for each host1x command. The single buffer used for host1x commands can get corrupted when independent crypto operations overlap. Bug 4883011 Signed-off-by: Akhil R Change-Id: I43029364c8e65e5014a5b7068cb45225c039aaf8 Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/3251597 Reviewed-by: Laxman Dewangan GVS: buildbot_gerritrpt --- drivers/crypto/tegra/tegra-se-aes.c | 70 +++++++++++++++++++--------- drivers/crypto/tegra/tegra-se-hash.c | 25 ++++++---- drivers/crypto/tegra/tegra-se-key.c | 24 +++++----- drivers/crypto/tegra/tegra-se-main.c | 30 ++++++------ drivers/crypto/tegra/tegra-se-sm4.c | 64 +++++++++++++++++-------- drivers/crypto/tegra/tegra-se.h | 4 +- 6 files changed, 140 insertions(+), 77 deletions(-) diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index 58761d3a..b619c42c 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -44,6 +44,7 @@ struct tegra_aes_reqctx { u32 *iv; u32 key1_id; u32 key2_id; + u32 *cmdbuf; }; struct tegra_aead_ctx { @@ -74,6 +75,7 @@ struct tegra_aead_reqctx { u32 key_id; u32 iv[4]; u8 authdata[16]; + u32 *cmdbuf; }; struct tegra_cmac_ctx { @@ -100,6 +102,7 @@ struct tegra_cmac_reqctx { u32 key_id; u32 *iv; u32 result[CMAC_RESULT_REG_COUNT]; + u32 *cmdbuf; }; /* increment counter (128-bit int) */ @@ -225,7 +228,7 @@ static int tegra234_aes_cfg(u32 alg, bool encrypt) static unsigned int tegra_aes_prep_cmd(struct tegra_se *se, struct tegra_aes_reqctx *rctx) { unsigned int data_count, res_bits, i = 0, j; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; dma_addr_t addr = rctx->datbuf.addr; data_count = rctx->len / AES_BLOCK_SIZE; @@ -329,14 +332,14 @@ static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) /* Prepare the command and submit for execution */ cmdlen = tegra_aes_prep_cmd(se, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen); /* Copy the result */ tegra_aes_update_iv(req, ctx); scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1); /* Free the buffer */ - dma_free_coherent(ctx->se->dev, rctx->datbuf.size, + dma_free_coherent(se->dev, rctx->datbuf.size, rctx->datbuf.buf, rctx->datbuf.addr); key2_free: @@ -346,6 +349,8 @@ key1_free: if (rctx->key1_id != ctx->key1_id) tegra_key_invalidate(ctx->se, rctx->key1_id, ctx->alg); out: + kfree(rctx->cmdbuf); + crypto_finalize_skcipher_request(se->engine, req, ret); return 0; @@ -638,6 +643,13 @@ static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt) if (!req->cryptlen) return 0; + if (ctx->alg == SE_ALG_ECB) + req->iv = NULL; + + rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL); + if (!rctx->cmdbuf) + return -ENOMEM; + rctx->encrypt = encrypt; rctx->config = se->regcfg->cfg(ctx->alg, encrypt); rctx->crypto_config = se->regcfg->crypto_cfg(ctx->alg, encrypt); @@ -791,7 +803,7 @@ static void tegra_aes_set_regcfg(struct tegra_se *se) static unsigned int tegra_gmac_prep_cmd(struct tegra_se *se, struct tegra_aead_reqctx *rctx) { unsigned int data_count, res_bits, i = 0, j; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; data_count = (rctx->assoclen / AES_BLOCK_SIZE); res_bits = (rctx->assoclen % AES_BLOCK_SIZE) * 8; @@ -837,7 +849,7 @@ static unsigned int tegra_gmac_prep_cmd(struct tegra_se *se, struct tegra_aead_r static unsigned int tegra_gcm_crypt_prep_cmd(struct tegra_se *se, struct tegra_aead_reqctx *rctx) { unsigned int data_count, res_bits, i = 0, j; - u32 *cpuvaddr = se->cmdbuf->addr, op; + u32 *cpuvaddr = rctx->cmdbuf, op; data_count = (rctx->cryptlen / AES_BLOCK_SIZE); res_bits = (rctx->cryptlen % AES_BLOCK_SIZE) * 8; @@ -959,7 +971,7 @@ static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqct cmdlen = tegra_gmac_prep_cmd(se, rctx); - return tegra_se_host1x_submit(se, cmdlen); + return tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen); } static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) @@ -976,7 +988,7 @@ static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc /* Prepare command and submit */ cmdlen = tegra_gcm_crypt_prep_cmd(se, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen); if (ret) return ret; @@ -990,7 +1002,7 @@ static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) { struct tegra_se *se = ctx->se; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; int cmdlen, ret, offset; rctx->config = se->regcfg->cfg(ctx->final_alg, rctx->encrypt); @@ -999,7 +1011,7 @@ static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc /* Prepare command and submit */ cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen); if (ret) return ret; @@ -1016,7 +1028,7 @@ static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc static int tegra_gcm_hw_verify(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx, u8 *mac) { struct tegra_se *se = ctx->se; - u32 result, *cpuvaddr = se->cmdbuf->addr; + u32 result, *cpuvaddr = rctx->cmdbuf; int size, ret; memcpy(rctx->inbuf.buf, mac, rctx->authsize); @@ -1028,7 +1040,7 @@ static int tegra_gcm_hw_verify(struct tegra_aead_ctx *ctx, struct tegra_aead_req /* Prepare command and submit */ size = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(se, size); + ret = tegra_se_host1x_submit(se, cpuvaddr, size); if (ret) return ret; @@ -1077,7 +1089,7 @@ static inline int tegra_ccm_check_iv(const u8 *iv) static unsigned int tegra_cbcmac_prep_cmd(struct tegra_se *se, struct tegra_aead_reqctx *rctx) { unsigned int data_count, i = 0; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; data_count = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1; @@ -1111,7 +1123,7 @@ static unsigned int tegra_cbcmac_prep_cmd(struct tegra_se *se, struct tegra_aead static unsigned int tegra_ctr_prep_cmd(struct tegra_se *se, struct tegra_aead_reqctx *rctx) { unsigned int i = 0, j; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); @@ -1159,7 +1171,7 @@ static int tegra_ccm_do_cbcmac(struct tegra_se *se, struct tegra_aead_reqctx *rc /* Prepare command and submit */ cmdlen = tegra_cbcmac_prep_cmd(se, rctx); - return tegra_se_host1x_submit(se, cmdlen); + return tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen); } static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize) @@ -1364,7 +1376,7 @@ static int tegra_ccm_do_ctr(struct tegra_se *se, struct tegra_aead_reqctx *rctx) /* Prepare command and submit */ cmdlen = tegra_ctr_prep_cmd(se, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen); if (ret) return ret; @@ -1475,7 +1487,9 @@ key_free: /* Free the keyslot if it is cloned for this request */ if (rctx->key_id != ctx->key_id) tegra_key_invalidate(ctx->se, rctx->key_id, ctx->alg); + out: + kfree(rctx->cmdbuf); crypto_finalize_aead_request(ctx->se->engine, req, ret); return 0; @@ -1561,6 +1575,7 @@ outbuf_err: rctx->inbuf.buf, rctx->inbuf.addr); /* Finalize the request if there are no errors */ + kfree(rctx->cmdbuf); crypto_finalize_aead_request(ctx->se->engine, req, ret); return 0; @@ -1684,6 +1699,10 @@ static int tegra_aead_crypt(struct aead_request *req, bool encrypt) struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL); + if (!rctx->cmdbuf) + return -ENOMEM; + rctx->encrypt = encrypt; return crypto_transfer_aead_request_to_engine(ctx->se->engine, req); @@ -1715,7 +1734,7 @@ static int tegra_aead_setkey(struct crypto_aead *tfm, static unsigned int tegra_cmac_prep_cmd(struct tegra_se *se, struct tegra_cmac_reqctx *rctx) { unsigned int data_count, res_bits = 0, i = 0, j; - u32 *cpuvaddr = se->cmdbuf->addr, op; + u32 *cpuvaddr = rctx->cmdbuf, op; data_count = (rctx->datbuf.size / AES_BLOCK_SIZE); @@ -1862,7 +1881,7 @@ static int tegra_cmac_do_update(struct ahash_request *req) cmdlen = tegra_cmac_prep_cmd(se, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen); /* * If this is not the final update, copy the intermediate results * from the registers so that it can be used in the next 'update' @@ -1906,7 +1925,7 @@ static int tegra_cmac_do_final(struct ahash_request *req) /* Prepare command and submit */ cmdlen = tegra_cmac_prep_cmd(se, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, cmdlen); if (ret) goto out; @@ -1927,6 +1946,7 @@ out_free: rctx->residue.buf, rctx->residue.addr); dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, rctx->digest.addr); + kfree(rctx->cmdbuf); return ret; } @@ -2041,15 +2061,19 @@ static int tegra_cmac_init(struct ahash_request *req) return -ENOMEM; } + rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL); + if (!rctx->cmdbuf) + goto key_free; + rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, &rctx->digest.addr, GFP_KERNEL); if (!rctx->digest.buf) - goto digbuf_fail; + goto cmdbuf_free; rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, &rctx->residue.addr, GFP_KERNEL); if (!rctx->residue.buf) - goto resbuf_fail; + goto digbuf_free; rctx->residue.size = 0; rctx->datbuf.size = 0; @@ -2060,10 +2084,12 @@ static int tegra_cmac_init(struct ahash_request *req) return 0; -resbuf_fail: +digbuf_free: dma_free_coherent(se->dev, rctx->blk_size, rctx->digest.buf, rctx->digest.addr); -digbuf_fail: +cmdbuf_free: + kfree(rctx->cmdbuf); +key_free: if (rctx->key_id != ctx->key_id) tegra_key_invalidate(ctx->se, rctx->key_id, ctx->alg); diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index 2ad081c1..870e1795 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -45,6 +45,7 @@ struct tegra_sha_reqctx { unsigned int blk_size; unsigned int task; u32 key_id; + u32 *cmdbuf; u32 result[HASH_RESULT_REG_COUNT]; struct ahash_request fallback_req; }; @@ -311,7 +312,7 @@ static int tegra_sha_do_update(struct ahash_request *req) struct tegra_sha_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); unsigned int nblks, nresidue, size, ret; - u32 *cpuvaddr = ctx->se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; @@ -372,7 +373,7 @@ static int tegra_sha_do_update(struct ahash_request *req) size = tegra_sha_prep_cmd(ctx->se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(ctx->se, size); + ret = tegra_se_host1x_submit(ctx->se, rctx->cmdbuf, size); /* * If this is not the final update, copy the intermediate results @@ -394,7 +395,7 @@ static int tegra_sha_do_final(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); struct tegra_se *se = ctx->se; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; int size, ret = 0; rctx->datbuf.size = rctx->residue.size; @@ -415,7 +416,7 @@ static int tegra_sha_do_final(struct ahash_request *req) } size = tegra_sha_prep_cmd(se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(se, size); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, size); if (ret) goto out; @@ -431,6 +432,8 @@ out_free: rctx->residue.buf, rctx->residue.addr); dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, rctx->digest.addr); + kfree(rctx->cmdbuf); + return ret; } @@ -565,22 +568,28 @@ static int tegra_sha_init(struct ahash_request *req) return -ENOMEM; } + rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL); + if (!rctx->cmdbuf) + goto key_free; + rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, &rctx->digest.addr, GFP_KERNEL); if (!rctx->digest.buf) - goto digbuf_fail; + goto cmdbuf_free; rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size, &rctx->residue.addr, GFP_KERNEL); if (!rctx->residue.buf) - goto resbuf_fail; + goto digbuf_free; return 0; -resbuf_fail: +digbuf_free: dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, rctx->digest.addr); -digbuf_fail: +cmdbuf_free: + kfree(rctx->cmdbuf); +key_free: if (rctx->key_id != ctx->key_id) tegra_key_invalidate(ctx->se, rctx->key_id, ctx->alg); diff --git a/drivers/crypto/tegra/tegra-se-key.c b/drivers/crypto/tegra/tegra-se-key.c index 91ea9e24..e2b09ce2 100644 --- a/drivers/crypto/tegra/tegra-se-key.c +++ b/drivers/crypto/tegra/tegra-se-key.c @@ -177,22 +177,22 @@ static int tegra_key_insert(struct tegra_se *se, const u8 *key, u32 keylen, u16 slot, u32 alg) { const u32 *keyval = (u32 *)key; - u32 *addr = se->cmdbuf->addr, size; + u32 cmdbuf[100], size; - size = tegra_key_prep_ins_cmd(se, addr, keyval, keylen, slot, alg); + size = tegra_key_prep_ins_cmd(se, cmdbuf, keyval, keylen, slot, alg); - return tegra_se_host1x_submit(se, size); + return tegra_se_host1x_submit(se, cmdbuf, size); } static int tegra_key_move_to_kds(struct tegra_se *se, u32 slot, u32 kds_id) { - u32 src_keyid, size; + u32 src_keyid, size, cmdbuf[100]; int ret; src_keyid = SE_KSLT_REGION_ID_SYM | slot; - size = tegra_key_prep_mov_cmd(se, se->cmdbuf->addr, src_keyid, kds_id); + size = tegra_key_prep_mov_cmd(se, cmdbuf, src_keyid, kds_id); - ret = tegra_se_host1x_submit(se, size); + ret = tegra_se_host1x_submit(se, cmdbuf, size); if (ret) return ret; @@ -201,13 +201,13 @@ static int tegra_key_move_to_kds(struct tegra_se *se, u32 slot, u32 kds_id) static unsigned int tegra_kac_get_from_kds(struct tegra_se *se, u32 keyid, u16 slot) { - u32 tgt_keyid, size; + u32 tgt_keyid, size, cmdbuf[100]; int ret; tgt_keyid = SE_KSLT_REGION_ID_SYM | slot; - size = tegra_key_prep_mov_cmd(se, se->cmdbuf->addr, keyid, tgt_keyid); + size = tegra_key_prep_mov_cmd(se, cmdbuf, keyid, tgt_keyid); - ret = tegra_se_host1x_submit(se, size); + ret = tegra_se_host1x_submit(se, cmdbuf, size); if (ret) tegra_keyslot_free(slot); @@ -216,10 +216,10 @@ static unsigned int tegra_kac_get_from_kds(struct tegra_se *se, u32 keyid, u16 s static void tegra_key_kds_invalidate(struct tegra_se *se, u32 keyid) { - unsigned int size; + unsigned int size, cmdbuf[100]; - size = tegra_key_prep_invld_cmd(se, se->cmdbuf->addr, keyid); - tegra_se_host1x_submit(se, size); + size = tegra_key_prep_invld_cmd(se, cmdbuf, keyid); + tegra_se_host1x_submit(se, cmdbuf, size); tegra_kds_free_id(keyid); } diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c index d4b80218..10e37bd1 100644 --- a/drivers/crypto/tegra/tegra-se-main.c +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -143,9 +143,10 @@ static struct tegra_se_cmdbuf *tegra_se_host1x_bo_alloc(struct tegra_se *se, ssi return cmdbuf; } -int tegra_se_host1x_submit(struct tegra_se *se, u32 size) +int tegra_se_host1x_submit(struct tegra_se *se, u32 *cpuvaddr, u32 size) { struct host1x_job *job; + struct tegra_se_cmdbuf *cmdbuf; int ret; job = host1x_job_alloc(se->channel, 1, 0, true); @@ -154,6 +155,12 @@ int tegra_se_host1x_submit(struct tegra_se *se, u32 size) return -ENOMEM; } + cmdbuf = tegra_se_host1x_bo_alloc(se, SZ_4K); + if (!cmdbuf) + goto job_put; + + memcpy(cmdbuf->addr, cpuvaddr, size * 4); + job->syncpt = host1x_syncpt_get(se->syncpt); job->syncpt_incrs = 1; job->client = &se->client; @@ -162,14 +169,14 @@ int tegra_se_host1x_submit(struct tegra_se *se, u32 size) job->engine_fallback_streamid = se->stream_id; job->engine_streamid_offset = SE_STREAM_ID; - se->cmdbuf->words = size; + cmdbuf->words = size; - host1x_job_add_gather(job, &se->cmdbuf->bo, size, 0); + host1x_job_add_gather(job, &cmdbuf->bo, size, 0); ret = host1x_job_pin(job, se->dev); if (ret) { dev_err(se->dev, "failed to pin host1x job\n"); - goto job_put; + goto cmdbuf_put; } ret = host1x_job_submit(job); @@ -186,10 +193,14 @@ int tegra_se_host1x_submit(struct tegra_se *se, u32 size) } host1x_job_put(job); + tegra_se_cmdbuf_put(&cmdbuf->bo); + return 0; job_unpin: host1x_job_unpin(job); +cmdbuf_put: + tegra_se_cmdbuf_put(&cmdbuf->bo); job_put: host1x_job_put(job); @@ -216,22 +227,14 @@ static int tegra_se_client_init(struct host1x_client *client) se->syncpt_id = host1x_syncpt_id(se->syncpt); - se->cmdbuf = tegra_se_host1x_bo_alloc(se, SZ_4K); - if (!se->cmdbuf) { - ret = -ENOMEM; - goto syncpt_put; - } - ret = se->hw->init_alg(se); if (ret) { dev_err(se->dev, "failed to register algorithms\n"); - goto cmdbuf_put; + goto syncpt_put; } return 0; -cmdbuf_put: - tegra_se_cmdbuf_put(&se->cmdbuf->bo); syncpt_put: host1x_syncpt_put(se->syncpt); channel_put: @@ -245,7 +248,6 @@ static int tegra_se_client_deinit(struct host1x_client *client) struct tegra_se *se = container_of(client, struct tegra_se, client); se->hw->deinit_alg(se); - tegra_se_cmdbuf_put(&se->cmdbuf->bo); host1x_syncpt_put(se->syncpt); host1x_channel_put(se->channel); diff --git a/drivers/crypto/tegra/tegra-se-sm4.c b/drivers/crypto/tegra/tegra-se-sm4.c index ca9f2afd..9216f660 100644 --- a/drivers/crypto/tegra/tegra-se-sm4.c +++ b/drivers/crypto/tegra/tegra-se-sm4.c @@ -2,7 +2,7 @@ /* * Crypto driver for NVIDIA Security Engine for block cipher operations. * - * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. */ #include @@ -45,6 +45,7 @@ struct tegra_sm4_reqctx { u32 crypto_cfg; u32 key1_id; u32 key2_id; + u32 *cmdbuf; }; struct tegra_sm4_gcm_ctx { @@ -59,6 +60,7 @@ struct tegra_sm4_gcm_ctx { u32 verify_alg; u32 keylen; u32 key_id; + u32 *cmdbuf; }; struct tegra_sm4_gcm_reqctx { @@ -73,6 +75,7 @@ struct tegra_sm4_gcm_reqctx { u32 config; u32 crypto_config; u32 key_id; + u32 *cmdbuf; u32 iv[4]; u8 authdata[16]; }; @@ -100,6 +103,7 @@ struct tegra_sm4_cmac_reqctx { u32 crypto_config; u32 key_id; u32 result[CMAC_RESULT_REG_COUNT]; + u32 *cmdbuf; u32 *iv; }; @@ -225,7 +229,7 @@ static int tegra_sm4_do_one_req(struct crypto_engine *engine, void *areq) goto key2_free; } - cpuvaddr = se->cmdbuf->addr; + cpuvaddr = rctx->cmdbuf; len = req->cryptlen; /* Pad input to AES block size */ @@ -245,7 +249,7 @@ static int tegra_sm4_do_one_req(struct crypto_engine *engine, void *areq) size = tegra_sm4_prep_cmd(se, cpuvaddr, iv, len, rctx->datbuf.addr, config, crypto_config); - ret = tegra_se_host1x_submit(se, size); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, size); /* Copy the result */ dst_nents = sg_nents(req->dst); @@ -263,6 +267,7 @@ key1_free: if (rctx->key1_id != ctx->key1_id) tegra_key_invalidate(ctx->se, rctx->key1_id, ctx->alg); out: + kfree(rctx->cmdbuf); crypto_finalize_skcipher_request(se->engine, req, ret); return ret; @@ -501,6 +506,10 @@ static int tegra_sm4_encrypt(struct skcipher_request *req) rctx = skcipher_request_ctx(req); rctx->encrypt = true; + rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL); + if (!rctx->cmdbuf) + return -ENOMEM; + return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req); } @@ -513,6 +522,10 @@ static int tegra_sm4_decrypt(struct skcipher_request *req) rctx = skcipher_request_ctx(req); rctx->encrypt = false; + rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL); + if (!rctx->cmdbuf) + return -ENOMEM; + return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req); } @@ -829,7 +842,7 @@ static int tegra_sm4_gcm_prep_final_cmd(struct tegra_se *se, u32 *cpuvaddr, static int tegra_sm4_gcm_do_gmac(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm4_gcm_reqctx *rctx) { struct tegra_se *se = ctx->se; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; unsigned int nents, size; nents = sg_nents(rctx->src_sg); @@ -842,13 +855,13 @@ static int tegra_sm4_gcm_do_gmac(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm4 size = tegra_sm4_gmac_prep_cmd(se, cpuvaddr, rctx); - return tegra_se_host1x_submit(se, size); + return tegra_se_host1x_submit(se, cpuvaddr, size); } static int tegra_sm4_gcm_do_crypt(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm4_gcm_reqctx *rctx) { struct tegra_se *se = ctx->se; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; int size, ret; scatterwalk_map_and_copy(rctx->inbuf.buf, rctx->src_sg, @@ -860,7 +873,7 @@ static int tegra_sm4_gcm_do_crypt(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm /* Prepare command and submit */ size = tegra_sm4_gcm_crypt_prep_cmd(se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(se, size); + ret = tegra_se_host1x_submit(se, cpuvaddr, size); if (ret) return ret; @@ -874,7 +887,7 @@ static int tegra_sm4_gcm_do_crypt(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm static int tegra_sm4_gcm_do_final(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm4_gcm_reqctx *rctx) { struct tegra_se *se = ctx->se; - u32 *cpuvaddr = se->cmdbuf->addr; + u32 *cpuvaddr = rctx->cmdbuf; int size, ret, off; rctx->config = se->regcfg->cfg(ctx->final_alg, rctx->encrypt); @@ -883,7 +896,7 @@ static int tegra_sm4_gcm_do_final(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm /* Prepare command and submit */ size = tegra_sm4_gcm_prep_final_cmd(se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(se, size); + ret = tegra_se_host1x_submit(se, cpuvaddr, size); if (ret) return ret; @@ -900,7 +913,7 @@ static int tegra_sm4_gcm_do_final(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm static int tegra_sm4_gcm_hw_verify(struct tegra_sm4_gcm_ctx *ctx, struct tegra_sm4_gcm_reqctx *rctx, u8 *mac) { struct tegra_se *se = ctx->se; - u32 result, *cpuvaddr = se->cmdbuf->addr; + u32 result, *cpuvaddr = rctx->cmdbuf; int size, ret; memcpy(rctx->inbuf.buf, mac, rctx->authsize); @@ -912,7 +925,7 @@ static int tegra_sm4_gcm_hw_verify(struct tegra_sm4_gcm_ctx *ctx, struct tegra_s /* Prepare command and submit */ size = tegra_sm4_gcm_prep_final_cmd(se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(se, size); + ret = tegra_se_host1x_submit(se, cpuvaddr, size); if (ret) return ret; @@ -1022,6 +1035,7 @@ key_free: tegra_key_invalidate(ctx->se, rctx->key_id, ctx->alg); out: + kfree(rctx->cmdbuf); crypto_finalize_aead_request(se->engine, req, ret); return 0; @@ -1085,6 +1099,10 @@ static int tegra_sm4_gcm_crypt(struct aead_request *req, bool encrypt) rctx->encrypt = encrypt; + rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL); + if (!rctx->cmdbuf) + return -ENOMEM; + return crypto_transfer_aead_request_to_engine(ctx->se->engine, req); } @@ -1226,9 +1244,9 @@ static int tegra_sm4_cmac_do_update(struct ahash_request *req) /* Update residue value with the residue after current block */ rctx->residue.size = nresidue; - size = tegra_sm4_cmac_prep_cmd(se, se->cmdbuf->addr, rctx); + size = tegra_sm4_cmac_prep_cmd(se, rctx->cmdbuf, rctx); - return tegra_se_host1x_submit(se, size); + return tegra_se_host1x_submit(se, rctx->cmdbuf, size); } static int tegra_sm4_cmac_do_final(struct ahash_request *req) @@ -1255,8 +1273,8 @@ static int tegra_sm4_cmac_do_final(struct ahash_request *req) } /* Prepare command and submit */ - size = tegra_sm4_cmac_prep_cmd(se, se->cmdbuf->addr, rctx); - ret = tegra_se_host1x_submit(se, size); + size = tegra_sm4_cmac_prep_cmd(se, rctx->cmdbuf, rctx); + ret = tegra_se_host1x_submit(se, rctx->cmdbuf, size); if (ret) goto out; @@ -1279,6 +1297,8 @@ out_free: dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, rctx->digest.addr); + kfree(rctx->cmdbuf); + return ret; } @@ -1366,15 +1386,19 @@ static int tegra_sm4_cmac_init(struct ahash_request *req) return -ENOMEM; } + rctx->cmdbuf = kzalloc(SE_MAX_CMDLEN, GFP_KERNEL); + if (!rctx->cmdbuf) + goto key_free; + rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, &rctx->digest.addr, GFP_KERNEL); if (!rctx->digest.buf) - goto digbuf_fail; + goto cmdbuf_free; rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, &rctx->residue.addr, GFP_KERNEL); if (!rctx->residue.buf) - goto resbuf_fail; + goto digbuf_free; rctx->residue.size = 0; rctx->datbuf.size = 0; @@ -1385,10 +1409,12 @@ static int tegra_sm4_cmac_init(struct ahash_request *req) return 0; -resbuf_fail: +digbuf_free: dma_free_coherent(se->dev, rctx->blk_size, rctx->digest.buf, rctx->digest.addr); -digbuf_fail: +cmdbuf_free: + kfree(rctx->cmdbuf); +key_free: if (rctx->key_id != ctx->key_id) tegra_key_invalidate(ctx->se, rctx->key_id, ctx->alg); diff --git a/drivers/crypto/tegra/tegra-se.h b/drivers/crypto/tegra/tegra-se.h index 56b29495..961b6e41 100644 --- a/drivers/crypto/tegra/tegra-se.h +++ b/drivers/crypto/tegra/tegra-se.h @@ -23,6 +23,7 @@ #define SE_OWNERSHIP_UID(x) FIELD_GET(GENMASK(7, 0), x) #define TEGRA_GPSE_ID 3 +#define SE_MAX_CMDLEN (100 * 4) /* max 100 commands of 4 bytes each */ #define SE_STREAM_ID 0x90 #define SE_SHA_CFG 0x4004 @@ -655,7 +656,6 @@ struct tegra_se { struct host1x_client client; struct host1x_channel *channel; struct tegra_se_regcfg *regcfg; - struct tegra_se_cmdbuf *cmdbuf; struct crypto_engine *engine; struct host1x_syncpt *syncpt; struct device *dev; @@ -770,7 +770,7 @@ int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u32 *keyid); void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg); unsigned int tegra_key_get_idx(struct tegra_se *se, u32 keyid); -int tegra_se_host1x_submit(struct tegra_se *se, u32 size); +int tegra_se_host1x_submit(struct tegra_se *se, u32 *cpuvaddr, u32 size); u32 tegra_kds_get_id(void); void tegra_kds_free_id(u32 keyid);