/* * Cryptographic API. * drivers/crypto/tegra-se-nvhost.c * * Support for Tegra Security Engine hardware crypto algorithms. * * Copyright (c) 2015-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include //#include #include #include #include "nvhost.h" #include "tegra-se-nvhost.h" #include "hardware_t194.h" #define DRIVER_NAME "tegra-se-nvhost" #define NV_SE1_CLASS_ID 0x3A #define NV_SE2_CLASS_ID 0x3B #define NV_SE3_CLASS_ID 0x3C #define NV_SE4_CLASS_ID 0x3D #define NUM_SE_ALGO 6 #define MIN_DH_SZ_BITS 1536 #define GCM_IV_SIZE 12 #define __nvhost_opcode_nonincr(x, y) nvhost_opcode_nonincr((x) / 4, (y)) #define __nvhost_opcode_incr(x, y) nvhost_opcode_incr((x) / 4, (y)) #define __nvhost_opcode_nonincr_w(x) nvhost_opcode_nonincr_w((x) / 4) #define __nvhost_opcode_incr_w(x) nvhost_opcode_incr_w((x) / 4) /* Security Engine operation modes */ enum tegra_se_aes_op_mode { SE_AES_OP_MODE_CBC, /* Cipher Block Chaining (CBC) mode */ SE_AES_OP_MODE_ECB, /* Electronic Codebook (ECB) mode */ SE_AES_OP_MODE_CTR, /* Counter (CTR) mode */ SE_AES_OP_MODE_OFB, /* Output feedback (CFB) mode */ SE_AES_OP_MODE_CMAC, /* Cipher-based MAC (CMAC) mode */ SE_AES_OP_MODE_RNG_DRBG, /* Deterministic Random Bit Generator */ SE_AES_OP_MODE_SHA1, /* Secure Hash Algorithm-1 (SHA1) mode */ SE_AES_OP_MODE_SHA224, /* Secure Hash Algorithm-224 (SHA224) mode */ SE_AES_OP_MODE_SHA256, /* Secure Hash Algorithm-256 (SHA256) mode */ SE_AES_OP_MODE_SHA384, /* Secure Hash Algorithm-384 (SHA384) mode */ SE_AES_OP_MODE_SHA512, /* Secure Hash Algorithm-512 (SHA512) mode */ SE_AES_OP_MODE_SHA3_224,/* Secure Hash Algorithm3-224 (SHA3-224) mode */ SE_AES_OP_MODE_SHA3_256,/* Secure Hash Algorithm3-256 (SHA3-256) mode */ SE_AES_OP_MODE_SHA3_384,/* Secure Hash Algorithm3-384 (SHA3-384) mode */ SE_AES_OP_MODE_SHA3_512,/* Secure Hash Algorithm3-512 (SHA3-512) mode */ SE_AES_OP_MODE_SHAKE128,/* Secure Hash Algorithm3 (SHAKE128) mode */ SE_AES_OP_MODE_SHAKE256,/* Secure Hash Algorithm3 (SHAKE256) mode */ SE_AES_OP_MODE_HMAC_SHA224, /* Hash based MAC (HMAC) - 224 */ SE_AES_OP_MODE_HMAC_SHA256, /* Hash based MAC (HMAC) - 256 */ SE_AES_OP_MODE_HMAC_SHA384, /* Hash based MAC (HMAC) - 384 */ SE_AES_OP_MODE_HMAC_SHA512, /* Hash based MAC (HMAC) - 512 */ SE_AES_OP_MODE_XTS, /* XTS mode */ SE_AES_OP_MODE_INS, /* key insertion */ SE_AES_OP_MODE_CBC_MAC, /* CBC MAC mode */ SE_AES_OP_MODE_GCM /* GCM mode */ }; enum tegra_se_aes_gcm_mode { SE_AES_GMAC, SE_AES_GCM_ENC, SE_AES_GCM_DEC, SE_AES_GCM_FINAL }; /* Security Engine key table type */ enum tegra_se_key_table_type { SE_KEY_TABLE_TYPE_KEY, /* Key */ SE_KEY_TABLE_TYPE_KEY_IN_MEM, /* Key in Memory */ SE_KEY_TABLE_TYPE_ORGIV, /* Original IV */ SE_KEY_TABLE_TYPE_UPDTDIV, /* Updated IV */ SE_KEY_TABLE_TYPE_XTS_KEY1, /* XTS Key1 */ SE_KEY_TABLE_TYPE_XTS_KEY2, /* XTS Key2 */ SE_KEY_TABLE_TYPE_XTS_KEY1_IN_MEM, /* XTS Key1 in Memory */ SE_KEY_TABLE_TYPE_XTS_KEY2_IN_MEM, /* XTS Key2 in Memory */ SE_KEY_TABLE_TYPE_CMAC, SE_KEY_TABLE_TYPE_HMAC, SE_KEY_TABLE_TYPE_GCM /* GCM */ }; /* Key access control type */ enum tegra_se_kac_type { SE_KAC_T18X, SE_KAC_T23X }; struct tegra_se_chipdata { unsigned long aes_freq; unsigned int cpu_freq_mhz; enum tegra_se_kac_type kac_type; }; /* Security Engine Linked List */ struct tegra_se_ll { dma_addr_t addr; /* DMA buffer address */ u32 data_len; /* Data length in DMA buffer */ }; enum tegra_se_algo { SE_DRBG, SE_AES, SE_CMAC, SE_RSA, SE_SHA, SE_AEAD, }; enum tegra_se_callback { NONE, AES_CB, SHA_CB, }; struct tegra_se_cmdbuf { dma_addr_t iova; u32 *addr; struct device *dev; struct kref ref; struct host1x_bo bo; u32 words; ssize_t size; }; struct tegra_se_dev { struct platform_device *pdev; struct device *dev; void __iomem *io_regs; /* se device memory/io */ void __iomem *pmc_io_reg; /* pmc device memory/io */ struct mutex lock; /* Protect request queue */ /* Mutex lock (mtx) is to protect Hardware, as it can be used * in parallel by different threads. For example, set_key * request can come from a different thread and access HW */ struct mutex mtx; struct clk *pclk; /* Security Engine clock */ struct clk *enclk; /* Security Engine clock */ struct crypto_queue queue; /* Security Engine crypto queue */ struct tegra_se_slot *slot_list; /* pointer to key slots */ struct tegra_se_rsa_slot *rsa_slot_list; /* rsa key slot pointer */ struct tegra_se_chipdata *chipdata; /* chip specific data */ u32 *src_ll_buf; /* pointer to source linked list buffer */ dma_addr_t src_ll_buf_adr; /* Source linked list buffer dma address */ u32 src_ll_size; /* Size of source linked list buffer */ u32 *dst_ll_buf; /* pointer to destination linked list buffer */ dma_addr_t dst_ll_buf_adr; /* Destination linked list dma address */ u32 dst_ll_size; /* Size of destination linked list buffer */ struct tegra_se_ll *src_ll; struct tegra_se_ll *dst_ll; struct tegra_se_ll *aes_src_ll; struct tegra_se_ll *aes_dst_ll; u32 *dh_buf1, *dh_buf2; struct skcipher_request *reqs[SE_MAX_TASKS_PER_SUBMIT]; struct ahash_request *sha_req; unsigned int req_cnt; u32 stream_id; u32 syncpt_id; u32 opcode_addr; bool work_q_busy; /* Work queue busy status */ struct host1x_channel *channel; struct work_struct se_work; struct workqueue_struct *se_work_q; struct scatterlist sg; struct host1x_client client; bool dynamic_mem; u32 *total_aes_buf; dma_addr_t total_aes_buf_addr; void *aes_buf; dma_addr_t aes_buf_addr; void *aes_bufs[SE_MAX_AESBUF_ALLOC]; dma_addr_t aes_buf_addrs[SE_MAX_AESBUF_ALLOC]; atomic_t aes_buf_stat[SE_MAX_AESBUF_ALLOC]; dma_addr_t aes_addr; dma_addr_t aes_cur_addr; unsigned int cmdbuf_cnt; unsigned int src_bytes_mapped; unsigned int dst_bytes_mapped; unsigned int gather_buf_sz; unsigned int aesbuf_entry; struct pm_qos_request boost_cpufreq_req; /* Lock to protect cpufreq boost status */ struct mutex boost_cpufreq_lock; struct delayed_work restore_cpufreq_work; unsigned long cpufreq_last_boosted; bool cpufreq_boosted; bool ioc; bool sha_last; bool sha_src_mapped; bool sha_dst_mapped; }; static struct tegra_se_dev *se_devices[NUM_SE_ALGO]; /* Security Engine request context */ struct tegra_se_req_context { enum tegra_se_aes_op_mode op_mode; /* Security Engine operation mode */ bool encrypt; /* Operation type */ u32 config; u32 crypto_config; bool init; /* For GCM */ u8 *hash_result; /* Hash result buffer */ struct tegra_se_dev *se_dev; }; struct tegra_se_priv_data { struct skcipher_request *reqs[SE_MAX_TASKS_PER_SUBMIT]; struct ahash_request *sha_req; struct tegra_se_dev *se_dev; unsigned int req_cnt; unsigned int src_bytes_mapped; unsigned int dst_bytes_mapped; unsigned int gather_buf_sz; struct scatterlist sg; void *buf; bool dynmem; bool sha_last; bool sha_src_mapped; bool sha_dst_mapped; dma_addr_t buf_addr; dma_addr_t iova; unsigned int aesbuf_entry; }; /* Security Engine AES context */ struct tegra_se_aes_context { struct tegra_se_dev *se_dev; /* Security Engine device */ struct skcipher_request *req; struct tegra_se_slot *slot; /* Security Engine key slot */ struct tegra_se_slot *slot2; /* Security Engine key slot 2 */ u32 keylen; /* key length in bits */ u32 op_mode; /* AES operation mode */ bool is_key_in_mem; /* Whether key is in memory */ u8 key[64]; /* To store key if is_key_in_mem set */ }; /* Security Engine random number generator context */ struct tegra_se_rng_context { struct tegra_se_dev *se_dev; /* Security Engine device */ struct skcipher_request *req; struct tegra_se_slot *slot; /* Security Engine key slot */ u32 *dt_buf; /* Destination buffer pointer */ dma_addr_t dt_buf_adr; /* Destination buffer dma address */ u32 *rng_buf; /* RNG buffer pointer */ dma_addr_t rng_buf_adr; /* RNG buffer dma address */ }; /* Security Engine SHA context */ struct tegra_se_sha_context { struct tegra_se_dev *se_dev; /* Security Engine device */ u32 op_mode; /* SHA operation mode */ bool is_first; /* Represents first block */ u8 *sha_buf[2]; /* Buffer to store residual data */ dma_addr_t sha_buf_addr[2]; /* DMA address to residual data */ u32 total_count; /* Total bytes in all the requests */ u32 residual_bytes; /* Residual byte count */ u32 blk_size; /* SHA block size */ bool is_final; /* To know it is final/finup request */ /* HMAC Support*/ struct tegra_se_slot *slot; /* Security Engine key slot */ u32 keylen; /* key length in bits */ }; struct tegra_se_sha_zero_length_vector { unsigned int size; char *digest; }; /* Security Engine AES CMAC context */ struct tegra_se_aes_cmac_context { struct tegra_se_dev *se_dev; /* Security Engine device */ struct tegra_se_slot *slot; /* Security Engine key slot */ struct tegra_se_cmdbuf *cmdbuf; u32 keylen; /* key length in bits */ u8 K1[TEGRA_SE_KEY_128_SIZE]; /* Key1 */ u8 K2[TEGRA_SE_KEY_128_SIZE]; /* Key2 */ u8 *buf; /* Buffer pointer to store update request*/ dma_addr_t buf_dma_addr; /* DMA address of buffer */ u32 nbytes; /* Total bytes in input buffer */ }; struct tegra_se_dh_context { struct tegra_se_dev *se_dev; /* Security Engine device */ struct tegra_se_rsa_slot *slot; /* Security Engine rsa key slot */ void *key; void *p; void *g; unsigned int key_size; unsigned int p_size; unsigned int g_size; }; /* Security Engine AES CCM context */ struct tegra_se_aes_ccm_ctx { struct tegra_se_dev *se_dev; /* Security Engine device */ struct tegra_se_slot *slot; /* Security Engine key slot */ u32 keylen; /* key length in bits */ u8 *mac; dma_addr_t mac_addr; u8 *enc_mac; /* Used during encryption */ dma_addr_t enc_mac_addr; u8 *dec_mac; /* Used during decryption */ dma_addr_t dec_mac_addr; u8 *buf[4]; /* Temporary buffers */ dma_addr_t buf_addr[4]; /* DMA address for buffers */ unsigned int authsize; /* MAC size */ }; /* Security Engine AES GCM context */ struct tegra_se_aes_gcm_ctx { struct tegra_se_dev *se_dev; /* Security Engine device */ struct tegra_se_slot *slot; /* Security Engine key slot */ u32 keylen; /* key length in bits */ u8 *mac; dma_addr_t mac_addr; unsigned int authsize; /* MAC size */ }; /* Security Engine key slot */ struct tegra_se_slot { struct list_head node; u8 slot_num; /* Key slot number */ bool available; /* Tells whether key slot is free to use */ }; static struct tegra_se_slot ssk_slot = { .slot_num = 15, .available = false, }; static struct tegra_se_slot keymem_slot = { .slot_num = 14, .available = false, }; static struct tegra_se_slot srk_slot = { .slot_num = 0, .available = false, }; static struct tegra_se_slot pre_allocated_slot = { .slot_num = 0, .available = false, }; static LIST_HEAD(key_slot); static LIST_HEAD(rsa_key_slot); static DEFINE_SPINLOCK(rsa_key_slot_lock); static DEFINE_SPINLOCK(key_slot_lock); #define RNG_RESEED_INTERVAL 0x00773594 /* create a work for handling the async transfers */ static void tegra_se_work_handler(struct work_struct *work); static int force_reseed_count; #define GET_MSB(x) ((x) >> (8 * sizeof(x) - 1)) #define BOOST_PERIOD (msecs_to_jiffies(2 * 1000)) /* 2 seconds */ static unsigned int boost_cpu_freq; module_param(boost_cpu_freq, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(boost_cpu_freq, "CPU frequency (in MHz) to boost"); static void tegra_se_restore_cpu_freq_fn(struct work_struct *work) { struct tegra_se_dev *se_dev = container_of( work, struct tegra_se_dev, restore_cpufreq_work.work); unsigned long delay = BOOST_PERIOD; mutex_lock(&se_dev->boost_cpufreq_lock); if (time_is_after_jiffies(se_dev->cpufreq_last_boosted + delay)) { schedule_delayed_work(&se_dev->restore_cpufreq_work, delay); } else { #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0) pm_qos_update_request(&se_dev->boost_cpufreq_req, PM_QOS_DEFAULT_VALUE); #else cpu_latency_qos_update_request(&se_dev->boost_cpufreq_req, PM_QOS_DEFAULT_VALUE); #endif se_dev->cpufreq_boosted = false; } mutex_unlock(&se_dev->boost_cpufreq_lock); } static void tegra_se_boost_cpu_freq(struct tegra_se_dev *se_dev) { unsigned long delay = BOOST_PERIOD; s32 cpufreq_hz = boost_cpu_freq * 1000; mutex_lock(&se_dev->boost_cpufreq_lock); if (!se_dev->cpufreq_boosted) { #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0) pm_qos_update_request(&se_dev->boost_cpufreq_req, cpufreq_hz); #else cpu_latency_qos_update_request(&se_dev->boost_cpufreq_req, cpufreq_hz); #endif schedule_delayed_work(&se_dev->restore_cpufreq_work, delay); se_dev->cpufreq_boosted = true; } se_dev->cpufreq_last_boosted = jiffies; mutex_unlock(&se_dev->boost_cpufreq_lock); } static void tegra_se_boost_cpu_init(struct tegra_se_dev *se_dev) { boost_cpu_freq = se_dev->chipdata->cpu_freq_mhz; INIT_DELAYED_WORK(&se_dev->restore_cpufreq_work, tegra_se_restore_cpu_freq_fn); #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0) pm_qos_add_request(&se_dev->boost_cpufreq_req, PM_QOS_CPU_FREQ_MIN, PM_QOS_DEFAULT_VALUE); #else cpu_latency_qos_add_request(&se_dev->boost_cpufreq_req, PM_QOS_DEFAULT_VALUE); #endif mutex_init(&se_dev->boost_cpufreq_lock); } static void tegra_se_boost_cpu_deinit(struct tegra_se_dev *se_dev) { mutex_destroy(&se_dev->boost_cpufreq_lock); #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0) pm_qos_remove_request(&se_dev->boost_cpufreq_req); #else cpu_latency_qos_remove_request(&se_dev->boost_cpufreq_req); #endif cancel_delayed_work_sync(&se_dev->restore_cpufreq_work); } static void tegra_se_leftshift_onebit(u8 *in_buf, u32 size, u8 *org_msb) { u8 carry; u32 i; *org_msb = GET_MSB(in_buf[0]); /* left shift one bit */ in_buf[0] <<= 1; for (carry = 0, i = 1; i < size; i++) { carry = GET_MSB(in_buf[i]); in_buf[i - 1] |= carry; in_buf[i] <<= 1; } } static inline void se_writel(struct tegra_se_dev *se_dev, unsigned int val, unsigned int reg_offset) { writel(val, se_dev->io_regs + reg_offset); } static inline unsigned int se_readl(struct tegra_se_dev *se_dev, unsigned int reg_offset) { unsigned int val; val = readl(se_dev->io_regs + reg_offset); return val; } static void tegra_se_free_key_slot(struct tegra_se_slot *slot) { if (slot) { spin_lock(&key_slot_lock); slot->available = true; spin_unlock(&key_slot_lock); } } static struct tegra_se_slot *tegra_se_alloc_key_slot(void) { struct tegra_se_slot *slot = NULL; bool found = false; spin_lock(&key_slot_lock); list_for_each_entry(slot, &key_slot, node) { if (slot->available && (slot->slot_num != pre_allocated_slot.slot_num)) { slot->available = false; found = true; break; } } spin_unlock(&key_slot_lock); return found ? slot : NULL; } static int tegra_init_key_slot(struct tegra_se_dev *se_dev) { int i; spin_lock_init(&key_slot_lock); spin_lock(&key_slot_lock); /* *To avoid multiple secure engine initializing *key-slots. */ if (key_slot.prev != key_slot.next) { spin_unlock(&key_slot_lock); return 0; } spin_unlock(&key_slot_lock); se_dev->slot_list = devm_kzalloc(se_dev->dev, sizeof(struct tegra_se_slot) * TEGRA_SE_KEYSLOT_COUNT, GFP_KERNEL); if (!se_dev->slot_list) return -ENOMEM; spin_lock(&key_slot_lock); for (i = 0; i < TEGRA_SE_KEYSLOT_COUNT; i++) { /* * Slot 0, 14 and 15 are reserved and will not be added to the * free slots pool. Slot 0 is used for SRK generation, Slot 14 * for handling keys which are stored in memories and Slot 15 is * is used for SSK operation. */ if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if ((i == srk_slot.slot_num) || (i == ssk_slot.slot_num) || (i == keymem_slot.slot_num)) continue; } se_dev->slot_list[i].available = true; se_dev->slot_list[i].slot_num = i; INIT_LIST_HEAD(&se_dev->slot_list[i].node); list_add_tail(&se_dev->slot_list[i].node, &key_slot); } spin_unlock(&key_slot_lock); return 0; } static int tegra_se_alloc_ll_buf(struct tegra_se_dev *se_dev, u32 num_src_sgs, u32 num_dst_sgs) { if (se_dev->src_ll_buf || se_dev->dst_ll_buf) { dev_err(se_dev->dev, "trying to allocate memory to allocated memory\n"); return -EBUSY; } if (num_src_sgs) { se_dev->src_ll_size = sizeof(struct tegra_se_ll) * num_src_sgs; se_dev->src_ll_buf = dma_alloc_coherent( se_dev->dev, se_dev->src_ll_size, &se_dev->src_ll_buf_adr, GFP_KERNEL); if (!se_dev->src_ll_buf) { dev_err(se_dev->dev, "can not allocate src lldma buffer\n"); return -ENOMEM; } } if (num_dst_sgs) { se_dev->dst_ll_size = sizeof(struct tegra_se_ll) * num_dst_sgs; se_dev->dst_ll_buf = dma_alloc_coherent( se_dev->dev, se_dev->dst_ll_size, &se_dev->dst_ll_buf_adr, GFP_KERNEL); if (!se_dev->dst_ll_buf) { dev_err(se_dev->dev, "can not allocate dst ll dma buffer\n"); return -ENOMEM; } } return 0; } static void tegra_se_free_ll_buf(struct tegra_se_dev *se_dev) { if (se_dev->src_ll_buf) { dma_free_coherent(se_dev->dev, se_dev->src_ll_size, se_dev->src_ll_buf, se_dev->src_ll_buf_adr); se_dev->src_ll_buf = NULL; } if (se_dev->dst_ll_buf) { dma_free_coherent(se_dev->dev, se_dev->dst_ll_size, se_dev->dst_ll_buf, se_dev->dst_ll_buf_adr); se_dev->dst_ll_buf = NULL; } } static u32 tegra_se_get_config(struct tegra_se_dev *se_dev, enum tegra_se_aes_op_mode mode, bool encrypt, u32 data) { u32 val = 0, key_len, is_last; switch (mode) { case SE_AES_OP_MODE_CBC: case SE_AES_OP_MODE_CMAC: key_len = data; if (encrypt) { val = SE_CONFIG_ENC_ALG(ALG_AES_ENC); switch (se_dev->chipdata->kac_type) { case SE_KAC_T23X: if (mode == SE_AES_OP_MODE_CMAC) val |= SE_CONFIG_ENC_MODE(MODE_CMAC); break; case SE_KAC_T18X: if (key_len == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY256); else if (key_len == TEGRA_SE_KEY_192_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY192); else val |= SE_CONFIG_ENC_MODE(MODE_KEY128); break; default: break; } val |= SE_CONFIG_DEC_ALG(ALG_NOP); } else { val = SE_CONFIG_DEC_ALG(ALG_AES_DEC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if (key_len == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY256); else if (key_len == TEGRA_SE_KEY_192_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY192); else val |= SE_CONFIG_DEC_MODE(MODE_KEY128); } } if (mode == SE_AES_OP_MODE_CMAC) val |= SE_CONFIG_DST(DST_HASHREG); else val |= SE_CONFIG_DST(DST_MEMORY); break; case SE_AES_OP_MODE_CBC_MAC: val = SE_CONFIG_ENC_ALG(ALG_AES_ENC) | SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_GCM: if (data == SE_AES_GMAC) { if (encrypt) { val = SE_CONFIG_ENC_ALG(ALG_AES_ENC) | SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_MODE(MODE_GMAC); } else { val = SE_CONFIG_ENC_ALG(ALG_NOP) | SE_CONFIG_DEC_ALG(ALG_AES_DEC) | SE_CONFIG_DEC_MODE(MODE_GMAC); } } else if (data == SE_AES_GCM_ENC) { val = SE_CONFIG_ENC_ALG(ALG_AES_ENC) | SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_MODE(MODE_GCM); } else if (data == SE_AES_GCM_DEC) { val = SE_CONFIG_ENC_ALG(ALG_NOP) | SE_CONFIG_DEC_ALG(ALG_AES_DEC) | SE_CONFIG_DEC_MODE(MODE_GCM); } else if (data == SE_AES_GCM_FINAL) { if (encrypt) { val = SE_CONFIG_ENC_ALG(ALG_AES_ENC) | SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_MODE(MODE_GCM_FINAL); } else { val = SE_CONFIG_ENC_ALG(ALG_NOP) | SE_CONFIG_DEC_ALG(ALG_AES_DEC) | SE_CONFIG_DEC_MODE(MODE_GCM_FINAL); } } break; case SE_AES_OP_MODE_RNG_DRBG: if (se_dev->chipdata->kac_type == SE_KAC_T23X) { val = SE_CONFIG_ENC_ALG(ALG_RNG) | SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_DST(DST_MEMORY); } else { val = SE_CONFIG_ENC_ALG(ALG_RNG) | SE_CONFIG_ENC_MODE(MODE_KEY192) | SE_CONFIG_DST(DST_MEMORY); } break; case SE_AES_OP_MODE_ECB: key_len = data; if (encrypt) { val = SE_CONFIG_ENC_ALG(ALG_AES_ENC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if (key_len == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY256); else if (key_len == TEGRA_SE_KEY_192_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY192); else val |= SE_CONFIG_ENC_MODE(MODE_KEY128); } } else { val = SE_CONFIG_DEC_ALG(ALG_AES_DEC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if (key_len == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY256); else if (key_len == TEGRA_SE_KEY_192_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY192); else val |= SE_CONFIG_DEC_MODE(MODE_KEY128); } } val |= SE_CONFIG_DST(DST_MEMORY); break; case SE_AES_OP_MODE_CTR: key_len = data; if (encrypt) { val = SE_CONFIG_ENC_ALG(ALG_AES_ENC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if (key_len == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY256); else if (key_len == TEGRA_SE_KEY_192_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY192); else val |= SE_CONFIG_ENC_MODE(MODE_KEY128); } } else { val = SE_CONFIG_DEC_ALG(ALG_AES_DEC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if (key_len == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY256); else if (key_len == TEGRA_SE_KEY_192_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY192); else val |= SE_CONFIG_DEC_MODE(MODE_KEY128); } } val |= SE_CONFIG_DST(DST_MEMORY); break; case SE_AES_OP_MODE_OFB: key_len = data; if (encrypt) { val = SE_CONFIG_ENC_ALG(ALG_AES_ENC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if (key_len == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY256); else if (key_len == TEGRA_SE_KEY_192_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY192); else val |= SE_CONFIG_ENC_MODE(MODE_KEY128); } } else { val = SE_CONFIG_DEC_ALG(ALG_AES_DEC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if (key_len == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY256); else if (key_len == TEGRA_SE_KEY_192_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY192); else val |= SE_CONFIG_DEC_MODE(MODE_KEY128); } } val |= SE_CONFIG_DST(DST_MEMORY); break; case SE_AES_OP_MODE_SHA1: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA1) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHA224: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA224) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHA256: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA256) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHA384: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA384) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHA512: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA512) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHA3_224: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA3_224) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHA3_256: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA3_256) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHA3_384: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA3_384) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHA3_512: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHA3_512) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHAKE128: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHAKE128) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_SHAKE256: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_SHA) | SE_CONFIG_ENC_MODE(MODE_SHAKE256) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_HMAC_SHA224: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_HMAC) | SE_CONFIG_ENC_MODE(MODE_SHA224) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_HMAC_SHA256: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_HMAC) | SE_CONFIG_ENC_MODE(MODE_SHA256) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_HMAC_SHA384: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_HMAC) | SE_CONFIG_ENC_MODE(MODE_SHA384) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_HMAC_SHA512: is_last = data; val = SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_ENC_ALG(ALG_HMAC) | SE_CONFIG_ENC_MODE(MODE_SHA512) | SE_CONFIG_DST(DST_MEMORY); if (!is_last) val |= SE_CONFIG_DST(DST_HASHREG); break; case SE_AES_OP_MODE_XTS: key_len = data; if (encrypt) { val = SE_CONFIG_ENC_ALG(ALG_AES_ENC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if ((key_len / 2) == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_ENC_MODE(MODE_KEY256); else val |= SE_CONFIG_ENC_MODE(MODE_KEY128); } val |= SE_CONFIG_DEC_ALG(ALG_NOP); } else { val = SE_CONFIG_DEC_ALG(ALG_AES_DEC); if (se_dev->chipdata->kac_type == SE_KAC_T18X) { if (key_len / 2 == TEGRA_SE_KEY_256_SIZE) val |= SE_CONFIG_DEC_MODE(MODE_KEY256); else val |= SE_CONFIG_DEC_MODE(MODE_KEY128); } val |= SE_CONFIG_ENC_ALG(ALG_NOP); } val |= SE_CONFIG_DST(DST_MEMORY); break; case SE_AES_OP_MODE_INS: val = SE_CONFIG_ENC_ALG(ALG_INS); val |= SE_CONFIG_DEC_ALG(ALG_NOP); break; default: dev_warn(se_dev->dev, "Invalid operation mode\n"); break; } pr_debug("%s:%d config val = 0x%x\n", __func__, __LINE__, val); return val; } static void tegra_unmap_sg(struct device *dev, struct scatterlist *sg, enum dma_data_direction dir, u32 total) { u32 total_loop; total_loop = total; while (sg && (total_loop > 0)) { dma_unmap_sg(dev, sg, 1, dir); total_loop -= min_t(size_t, (size_t)sg->length, (size_t)total_loop); sg = sg_next(sg); } } static unsigned int tegra_se_count_sgs(struct scatterlist *sl, u32 nbytes) { unsigned int sg_nents = 0; while (sl) { sg_nents++; nbytes -= min((size_t)sl->length, (size_t)nbytes); if (!nbytes) break; sl = sg_next(sl); } return sg_nents; } static void tegra_se_sha_complete_callback(void *priv, int nr_completed) { struct tegra_se_priv_data *priv_data = priv; struct ahash_request *req; struct tegra_se_dev *se_dev; struct crypto_ahash *tfm; struct tegra_se_req_context *req_ctx; struct tegra_se_sha_context *sha_ctx; int dst_len; pr_debug("%s:%d sha callback", __func__, __LINE__); se_dev = priv_data->se_dev; req = priv_data->sha_req; if (!req) { dev_err(se_dev->dev, "Invalid request for callback\n"); devm_kfree(se_dev->dev, priv_data); return; } tfm = crypto_ahash_reqtfm(req); req_ctx = ahash_request_ctx(req); dst_len = crypto_ahash_digestsize(tfm); sha_ctx = crypto_ahash_ctx(tfm); #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) /* For SHAKE128/SHAKE256, digest size can vary */ // if (sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE128 // || sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE256) // dst_len = req->dst_size; #endif /* copy result */ if (sha_ctx->is_final) memcpy(req->result, req_ctx->hash_result, dst_len); if (priv_data->sha_src_mapped) tegra_unmap_sg(se_dev->dev, req->src, DMA_TO_DEVICE, priv_data->src_bytes_mapped); if (priv_data->sha_dst_mapped) tegra_unmap_sg(se_dev->dev, &priv_data->sg, DMA_FROM_DEVICE, priv_data->dst_bytes_mapped); devm_kfree(se_dev->dev, req_ctx->hash_result); req->base.complete(&req->base, 0); devm_kfree(se_dev->dev, priv_data); pr_debug("%s:%d sha callback complete", __func__, __LINE__); } static void tegra_se_aes_complete_callback(void *priv, int nr_completed) { int i = 0; struct tegra_se_priv_data *priv_data = priv; struct skcipher_request *req; struct tegra_se_dev *se_dev; void *buf; u32 num_sgs; pr_debug("%s(%d) aes callback\n", __func__, __LINE__); se_dev = priv_data->se_dev; if (!priv_data->req_cnt) { devm_kfree(se_dev->dev, priv_data); return; } if (!se_dev->ioc) dma_sync_single_for_cpu(se_dev->dev, priv_data->buf_addr, priv_data->gather_buf_sz, DMA_BIDIRECTIONAL); buf = priv_data->buf; for (i = 0; i < priv_data->req_cnt; i++) { req = priv_data->reqs[i]; if (!req) { dev_err(se_dev->dev, "Invalid request for callback\n"); if (priv_data->dynmem) kfree(priv_data->buf); devm_kfree(se_dev->dev, priv_data); return; } num_sgs = tegra_se_count_sgs(req->dst, req->cryptlen); if (num_sgs == 1) memcpy(sg_virt(req->dst), buf, req->cryptlen); else sg_copy_from_buffer(req->dst, num_sgs, buf, req->cryptlen); buf += req->cryptlen; req->base.complete(&req->base, 0); } if (!se_dev->ioc) dma_unmap_sg(se_dev->dev, &priv_data->sg, 1, DMA_BIDIRECTIONAL); if (unlikely(priv_data->dynmem)) { if (se_dev->ioc) dma_free_coherent(se_dev->dev, priv_data->gather_buf_sz, priv_data->buf, priv_data->buf_addr); else kfree(priv_data->buf); } else { atomic_set(&se_dev->aes_buf_stat[priv_data->aesbuf_entry], 1); } devm_kfree(se_dev->dev, priv_data); pr_debug("%s(%d) aes callback complete\n", __func__, __LINE__); } static void se_nvhost_write_method(u32 *buf, u32 op1, u32 op2, u32 *offset) { int i = 0; buf[i++] = op1; buf[i++] = op2; *offset = *offset + 2; } static struct host1x_bo *tegra_se_cmdbuf_get(struct host1x_bo *host_bo) { struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); kref_get(&cmdbuf->ref); return host_bo; } static void tegra_se_cmdbuf_release(struct kref *ref) { struct tegra_se_cmdbuf *cmdbuf = container_of(ref, struct tegra_se_cmdbuf, ref); dma_free_attrs(cmdbuf->dev->parent, cmdbuf->size, cmdbuf->addr, cmdbuf->iova, 0); kfree(cmdbuf); } static void tegra_se_cmdbuf_put(struct host1x_bo *host_bo) { struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); kref_put(&cmdbuf->ref, tegra_se_cmdbuf_release); } static struct host1x_bo_mapping * tegra_se_cmdbuf_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) { struct tegra_se_cmdbuf *cmdbuf = container_of(bo, struct tegra_se_cmdbuf, bo); struct host1x_bo_mapping *map; int err; map = kzalloc(sizeof(*map), GFP_KERNEL); if (!map) return ERR_PTR(-ENOMEM); kref_init(&map->ref); map->bo = host1x_bo_get(bo); map->direction = direction; map->dev = dev; map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); if (!map->sgt) { err = -ENOMEM; goto free; } err = dma_get_sgtable(cmdbuf->dev->parent, map->sgt, cmdbuf->addr, cmdbuf->iova, cmdbuf->words * 4); if (err) goto free_sgt; err = dma_map_sgtable(dev, map->sgt, direction, 0); if (err) goto free_sgt; map->phys = sg_dma_address(map->sgt->sgl); map->size = cmdbuf->words * 4; map->chunks = err; return map; free_sgt: sg_free_table(map->sgt); kfree(map->sgt); free: kfree(map); return ERR_PTR(err); } static void tegra_se_cmdbuf_unpin(struct host1x_bo_mapping *map) { if (!map) return; dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); sg_free_table(map->sgt); kfree(map->sgt); host1x_bo_put(map->bo); kfree(map); } static void *tegra_se_cmdbuf_mmap(struct host1x_bo *host_bo) { struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); return cmdbuf->addr; } static void tegra_se_cmdbuf_munmap(struct host1x_bo *host_bo, void *addr) { } static const struct host1x_bo_ops tegra_se_cmdbuf_ops = { .get = tegra_se_cmdbuf_get, .put = tegra_se_cmdbuf_put, .pin = tegra_se_cmdbuf_pin, .unpin = tegra_se_cmdbuf_unpin, .mmap = tegra_se_cmdbuf_mmap, .munmap = tegra_se_cmdbuf_munmap, }; static struct tegra_se_cmdbuf *tegra_se_alloc_cmdbuf(struct device *dev, ssize_t size) { struct tegra_se_cmdbuf *cmdbuf; cmdbuf = devm_kzalloc(dev, sizeof(struct tegra_se_cmdbuf), GFP_KERNEL); if (!cmdbuf) return ERR_PTR(-ENOMEM); cmdbuf->addr = dma_alloc_attrs(dev, size, &cmdbuf->iova, GFP_KERNEL, 0); if (!cmdbuf->addr) return ERR_PTR(-ENOMEM); cmdbuf->size = size; cmdbuf->dev = dev; host1x_bo_init(&cmdbuf->bo, &tegra_se_cmdbuf_ops); kref_init(&cmdbuf->ref); return cmdbuf; } static int tegra_se_channel_submit_gather(struct tegra_se_dev *se_dev, struct tegra_se_cmdbuf *cmdbuf, u32 offset, u32 num_words, enum tegra_se_callback callback) { int i = 0; struct host1x_job *job = NULL; u32 syncpt_id = 0; int err = 0; struct tegra_se_priv_data *priv = NULL; struct nvhost_device_data *pdata = platform_get_drvdata(se_dev->pdev); if (callback) { priv = devm_kzalloc(se_dev->dev, sizeof(struct tegra_se_priv_data), GFP_KERNEL); if (!priv) return -ENOMEM; } if (!se_dev->channel) { se_dev->channel = host1x_channel_request(&se_dev->client); if (err) { dev_err(se_dev->dev, "Nvhost Channel map failed\n"); goto exit; } } job = host1x_job_alloc(se_dev->channel, 1, 0, true); if (!job) { dev_err(se_dev->dev, "Nvhost Job allocation failed\n"); err = -ENOMEM; goto exit; } if (!se_dev->syncpt_id) { se_dev->syncpt_id = nvhost_get_syncpt_host_managed( se_dev->pdev, 0, se_dev->pdev->name); if (!se_dev->syncpt_id) { dev_err(se_dev->dev, "Cannot get syncpt_id for SE(%s)\n", se_dev->pdev->name); err = -ENOMEM; goto error; } } syncpt_id = se_dev->syncpt_id; /* initialize job data */ /* FIXME: Remove 'pdata' dependency here. Since 'pdata' is a static * variable this causes failure when multiple instances of driver * with same compatible is run. * */ job->syncpt = host1x_syncpt_get_by_id(pdata->host1x, syncpt_id); job->syncpt_incrs = 1; job->client = &se_dev->client; job->class = se_dev->client.class; job->serialize = true; job->engine_fallback_streamid = se_dev->stream_id; job->engine_streamid_offset = SE_STREAMID_REG_OFFSET; se_dev->client.num_syncpts = 1; cmdbuf->dev = se_dev->dev; cmdbuf->words = num_words; /* push increment after work has been completed */ se_nvhost_write_method(&cmdbuf->addr[num_words], nvhost_opcode_nonincr( host1x_uclass_incr_syncpt_r(), 1), nvhost_class_host_incr_syncpt( 0x00000001, syncpt_id), &num_words); host1x_job_add_gather(job, &cmdbuf->bo, num_words, offset); #if 0 err = nvhost_job_add_client_gather_address(job, num_words, pdata->class, iova); if (err) { dev_err(se_dev->dev, "Nvhost failed to add gather\n"); goto error; } #endif err = host1x_job_pin(job, se_dev->dev); if (err) { dev_err(se_dev->dev, "pin job failed\n"); goto error; } err = host1x_job_submit(job); if (err) { host1x_job_unpin(job); dev_err(se_dev->dev, "Nvhost submit failed\n"); goto error; } if (callback == AES_CB) { priv->se_dev = se_dev; for (i = 0; i < se_dev->req_cnt; i++) priv->reqs[i] = se_dev->reqs[i]; if (!se_dev->ioc) priv->sg = se_dev->sg; if (unlikely(se_dev->dynamic_mem)) { priv->buf = se_dev->aes_buf; priv->dynmem = se_dev->dynamic_mem; } else { priv->buf = se_dev->aes_bufs[se_dev->aesbuf_entry]; priv->aesbuf_entry = se_dev->aesbuf_entry; } priv->buf_addr = se_dev->aes_addr; priv->req_cnt = se_dev->req_cnt; priv->gather_buf_sz = se_dev->gather_buf_sz; /* Register callback to be called once * syncpt value has been reached */ err = nvhost_intr_register_notifier( se_dev->pdev, syncpt_id, job->syncpt_end, tegra_se_aes_complete_callback, priv); if (err) { dev_warn(se_dev->dev, "add nvhost interrupt action failed (%d) for AES\n", err); tegra_se_aes_complete_callback(priv, 0); err = 0; // goto error; } } else if (callback == SHA_CB) { priv->se_dev = se_dev; priv->sha_req = se_dev->sha_req; priv->sg = se_dev->sg; priv->src_bytes_mapped = se_dev->src_bytes_mapped; priv->dst_bytes_mapped = se_dev->dst_bytes_mapped; priv->sha_src_mapped = se_dev->sha_src_mapped; priv->sha_dst_mapped = se_dev->sha_dst_mapped; priv->sha_last = se_dev->sha_last; priv->buf_addr = se_dev->dst_ll->addr; err = nvhost_intr_register_notifier( se_dev->pdev, syncpt_id, job->syncpt_end, tegra_se_sha_complete_callback, priv); if (err) { dev_warn(se_dev->dev, "add nvhost interrupt action failed for SHA\n"); tegra_se_sha_complete_callback(priv, 0); err = 0; // goto error; } } else { /* wait until host1x has processed work */ host1x_syncpt_wait( job->syncpt, job->syncpt_end, MAX_SCHEDULE_TIMEOUT, NULL); } se_dev->req_cnt = 0; se_dev->gather_buf_sz = 0; se_dev->cmdbuf_cnt = 0; se_dev->src_bytes_mapped = 0; se_dev->dst_bytes_mapped = 0; se_dev->sha_src_mapped = false; se_dev->sha_dst_mapped = false; se_dev->sha_last = false; error: host1x_job_put(job); exit: if (err) devm_kfree(se_dev->dev, priv); return err; } static void tegra_se_send_ctr_seed(struct tegra_se_dev *se_dev, u32 *pdata, unsigned int opcode_addr, u32 *cpuvaddr) { u32 j; u32 cmdbuf_num_words = 0, i = 0; i = se_dev->cmdbuf_cnt; if (se_dev->chipdata->kac_type != SE_KAC_T23X) { cpuvaddr[i++] = __nvhost_opcode_nonincr(opcode_addr + SE_AES_CRYPTO_CTR_SPARE, 1); cpuvaddr[i++] = SE_AES_CTR_LITTLE_ENDIAN; } cpuvaddr[i++] = nvhost_opcode_setpayload(4); cpuvaddr[i++] = __nvhost_opcode_incr_w(opcode_addr + SE_AES_CRYPTO_LINEAR_CTR); for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) cpuvaddr[i++] = pdata[j]; cmdbuf_num_words = i; se_dev->cmdbuf_cnt = i; } static int tegra_se_aes_ins_op(struct tegra_se_dev *se_dev, u8 *pdata, u32 data_len, u8 slot_num, enum tegra_se_key_table_type type, unsigned int opcode_addr, struct tegra_se_cmdbuf *cmdbuf, enum tegra_se_callback callback) { u32 *pdata_buf = (u32 *)pdata; u32 *cpuvaddr = cmdbuf->addr; u32 val = 0, j; u32 cmdbuf_num_words = 0, i = 0; int err = 0; struct { u32 OPERATION; u32 CRYPTO_KEYTABLE_KEYMANIFEST; u32 CRYPTO_KEYTABLE_DST; u32 CRYPTO_KEYTABLE_ADDR; u32 CRYPTO_KEYTABLE_DATA; } OFFSETS; if (!pdata_buf) { dev_err(se_dev->dev, "No key data available\n"); return -ENODATA; } pr_debug("%s(%d) data_len = %d slot_num = %d\n", __func__, __LINE__, data_len, slot_num); /* Initialize register offsets based on SHA/AES operation. */ switch (type) { case SE_KEY_TABLE_TYPE_HMAC: OFFSETS.OPERATION = SE_SHA_OPERATION_OFFSET; OFFSETS.CRYPTO_KEYTABLE_KEYMANIFEST = SE_SHA_CRYPTO_KEYTABLE_KEYMANIFEST_OFFSET; OFFSETS.CRYPTO_KEYTABLE_DST = SE_SHA_CRYPTO_KEYTABLE_DST_OFFSET; OFFSETS.CRYPTO_KEYTABLE_ADDR = SE_SHA_CRYPTO_KEYTABLE_ADDR_OFFSET; OFFSETS.CRYPTO_KEYTABLE_DATA = SE_SHA_CRYPTO_KEYTABLE_DATA_OFFSET; break; default: /* Use default case for AES operations. */ OFFSETS.OPERATION = SE_AES_OPERATION_OFFSET; OFFSETS.CRYPTO_KEYTABLE_KEYMANIFEST = SE_AES_CRYPTO_KEYTABLE_KEYMANIFEST_OFFSET; OFFSETS.CRYPTO_KEYTABLE_DST = SE_AES_CRYPTO_KEYTABLE_DST_OFFSET; OFFSETS.CRYPTO_KEYTABLE_ADDR = SE_AES_CRYPTO_KEYTABLE_ADDR_OFFSET; OFFSETS.CRYPTO_KEYTABLE_DATA = SE_AES_CRYPTO_KEYTABLE_DATA_OFFSET; break; } i = se_dev->cmdbuf_cnt; if (!se_dev->cmdbuf_cnt) { cpuvaddr[i++] = nvhost_opcode_setpayload(1); cpuvaddr[i++] = __nvhost_opcode_incr_w( opcode_addr + OFFSETS.OPERATION); cpuvaddr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE) | SE_OPERATION_OP(OP_DUMMY); } /* key manifest */ /* user */ val = SE_KEYMANIFEST_USER(NS); /* purpose */ switch (type) { case SE_KEY_TABLE_TYPE_XTS_KEY1: case SE_KEY_TABLE_TYPE_XTS_KEY2: val |= SE_KEYMANIFEST_PURPOSE(XTS); break; case SE_KEY_TABLE_TYPE_CMAC: val |= SE_KEYMANIFEST_PURPOSE(CMAC); break; case SE_KEY_TABLE_TYPE_HMAC: val |= SE_KEYMANIFEST_PURPOSE(HMAC); break; case SE_KEY_TABLE_TYPE_GCM: val |= SE_KEYMANIFEST_PURPOSE(GCM); break; default: val |= SE_KEYMANIFEST_PURPOSE(ENC); break; } /* size */ if (data_len == 16) val |= SE_KEYMANIFEST_SIZE(KEY128); else if (data_len == 24) val |= SE_KEYMANIFEST_SIZE(KEY192); else if (data_len == 32) val |= SE_KEYMANIFEST_SIZE(KEY256); /* exportable */ val |= SE_KEYMANIFEST_EX(false); cpuvaddr[i++] = nvhost_opcode_setpayload(1); cpuvaddr[i++] = __nvhost_opcode_incr_w(opcode_addr + OFFSETS.CRYPTO_KEYTABLE_KEYMANIFEST); cpuvaddr[i++] = val; pr_debug("%s(%d) key manifest = 0x%x\n", __func__, __LINE__, val); if (type != SE_KEY_TABLE_TYPE_HMAC) { /* configure slot number */ cpuvaddr[i++] = nvhost_opcode_setpayload(1); cpuvaddr[i++] = __nvhost_opcode_incr_w( opcode_addr + OFFSETS.CRYPTO_KEYTABLE_DST); cpuvaddr[i++] = SE_AES_KEY_INDEX(slot_num); } /* write key data */ for (j = 0; j < data_len; j += 4) { pr_debug("%s(%d) data_len = %d j = %d\n", __func__, __LINE__, data_len, j); cpuvaddr[i++] = nvhost_opcode_setpayload(1); cpuvaddr[i++] = __nvhost_opcode_incr_w( opcode_addr + OFFSETS.CRYPTO_KEYTABLE_ADDR); val = (j / 4); /* program quad */ cpuvaddr[i++] = val; cpuvaddr[i++] = nvhost_opcode_setpayload(1); cpuvaddr[i++] = __nvhost_opcode_incr_w( opcode_addr + OFFSETS.CRYPTO_KEYTABLE_DATA); cpuvaddr[i++] = *pdata_buf++; } /* configure INS operation */ cpuvaddr[i++] = nvhost_opcode_setpayload(1); cpuvaddr[i++] = __nvhost_opcode_incr_w(opcode_addr); cpuvaddr[i++] = tegra_se_get_config(se_dev, SE_AES_OP_MODE_INS, 0, 0); /* initiate key write operation to key slot */ cpuvaddr[i++] = nvhost_opcode_setpayload(1); cpuvaddr[i++] = __nvhost_opcode_incr_w( opcode_addr + OFFSETS.OPERATION); cpuvaddr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE) | SE_OPERATION_OP(OP_START) | SE_OPERATION_LASTBUF(LASTBUF_TRUE); cmdbuf_num_words = i; se_dev->cmdbuf_cnt = i; err = tegra_se_channel_submit_gather( se_dev, cmdbuf, 0, cmdbuf_num_words, callback); pr_debug("%s(%d) complete\n", __func__, __LINE__); return err; } static int tegra_se_send_key_data(struct tegra_se_dev *se_dev, u8 *pdata, u32 data_len, u8 slot_num, enum tegra_se_key_table_type type, unsigned int opcode_addr, struct tegra_se_cmdbuf *cmdbuf, enum tegra_se_callback callback) { u32 data_size; u32 *pdata_buf = (u32 *)pdata; u32 *cpuvaddr = cmdbuf->addr; u8 pkt = 0, quad = 0; u32 val = 0, j; u32 cmdbuf_num_words = 0, i = 0; int err = 0; if (!pdata_buf) { dev_err(se_dev->dev, "No Key Data available\n"); return -ENODATA; } if (se_dev->chipdata->kac_type == SE_KAC_T23X) { if (type == SE_KEY_TABLE_TYPE_ORGIV || type == SE_KEY_TABLE_TYPE_UPDTDIV) { pr_debug("%s(%d) IV programming\n", __func__, __LINE__); /* Program IV using CTR registers */ tegra_se_send_ctr_seed(se_dev, pdata_buf, opcode_addr, cpuvaddr); } else { /* Program key using INS operation */ err = tegra_se_aes_ins_op(se_dev, pdata, data_len, slot_num, type, opcode_addr, cmdbuf, callback); } return err; } if ((type == SE_KEY_TABLE_TYPE_KEY) && (slot_num == ssk_slot.slot_num)) { dev_err(se_dev->dev, "SSK Key Slot used\n"); return -EINVAL; } if ((type == SE_KEY_TABLE_TYPE_ORGIV) || (type == SE_KEY_TABLE_TYPE_XTS_KEY2) || (type == SE_KEY_TABLE_TYPE_XTS_KEY2_IN_MEM)) quad = QUAD_ORG_IV; else if (type == SE_KEY_TABLE_TYPE_UPDTDIV) quad = QUAD_UPDTD_IV; else if ((type == SE_KEY_TABLE_TYPE_KEY) || (type == SE_KEY_TABLE_TYPE_XTS_KEY1) || (type == SE_KEY_TABLE_TYPE_KEY_IN_MEM) || (type == SE_KEY_TABLE_TYPE_XTS_KEY1_IN_MEM)) quad = QUAD_KEYS_128; i = se_dev->cmdbuf_cnt; if (!se_dev->cmdbuf_cnt) { cpuvaddr[i++] = __nvhost_opcode_nonincr( opcode_addr + SE_AES_OPERATION_OFFSET, 1); cpuvaddr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE) | SE_OPERATION_OP(OP_DUMMY); } data_size = SE_KEYTABLE_QUAD_SIZE_BYTES; do { if (type == SE_KEY_TABLE_TYPE_XTS_KEY2 || type == SE_KEY_TABLE_TYPE_XTS_KEY2_IN_MEM) pkt = SE_CRYPTO_KEYIV_PKT_SUBKEY_SEL(SUBKEY_SEL_KEY2); else if (type == SE_KEY_TABLE_TYPE_XTS_KEY1 || type == SE_KEY_TABLE_TYPE_XTS_KEY1_IN_MEM) pkt = SE_CRYPTO_KEYIV_PKT_SUBKEY_SEL(SUBKEY_SEL_KEY1); pkt |= (SE_KEYTABLE_SLOT(slot_num) | SE_KEYTABLE_QUAD(quad)); for (j = 0; j < data_size; j += 4, data_len -= 4) { cpuvaddr[i++] = __nvhost_opcode_nonincr( opcode_addr + SE_AES_CRYPTO_KEYTABLE_ADDR_OFFSET, 1); val = (SE_KEYTABLE_PKT(pkt) | (j / 4)); cpuvaddr[i++] = val; cpuvaddr[i++] = __nvhost_opcode_incr( opcode_addr + SE_AES_CRYPTO_KEYTABLE_DATA_OFFSET, 1); cpuvaddr[i++] = *pdata_buf++; } data_size = data_len; if ((type == SE_KEY_TABLE_TYPE_KEY) || (type == SE_KEY_TABLE_TYPE_XTS_KEY1) || (type == SE_KEY_TABLE_TYPE_KEY_IN_MEM) || (type == SE_KEY_TABLE_TYPE_XTS_KEY1_IN_MEM)) quad = QUAD_KEYS_256; else if ((type == SE_KEY_TABLE_TYPE_XTS_KEY2) || (type == SE_KEY_TABLE_TYPE_XTS_KEY2_IN_MEM)) quad = QUAD_UPDTD_IV; } while (data_len); if ((type != SE_KEY_TABLE_TYPE_ORGIV) && (type != SE_KEY_TABLE_TYPE_UPDTDIV) && (type != SE_KEY_TABLE_TYPE_KEY_IN_MEM) && (type != SE_KEY_TABLE_TYPE_XTS_KEY1_IN_MEM) && (type != SE_KEY_TABLE_TYPE_XTS_KEY2_IN_MEM)) { cpuvaddr[i++] = __nvhost_opcode_nonincr( opcode_addr + SE_AES_OPERATION_OFFSET, 1); cpuvaddr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE) | SE_OPERATION_OP(OP_DUMMY); } cmdbuf_num_words = i; se_dev->cmdbuf_cnt = i; if ((type != SE_KEY_TABLE_TYPE_ORGIV) && (type != SE_KEY_TABLE_TYPE_UPDTDIV) && (type != SE_KEY_TABLE_TYPE_KEY_IN_MEM) && (type != SE_KEY_TABLE_TYPE_XTS_KEY1_IN_MEM) && (type != SE_KEY_TABLE_TYPE_XTS_KEY2_IN_MEM)) err = tegra_se_channel_submit_gather( se_dev, cmdbuf, 0, cmdbuf_num_words, callback); return err; } static u32 tegra_se_get_crypto_config(struct tegra_se_dev *se_dev, enum tegra_se_aes_op_mode mode, bool encrypt, u8 slot_num, u8 slot2_num, bool org_iv) { u32 val = 0; unsigned long freq = 0; switch (mode) { case SE_AES_OP_MODE_XTS: if (encrypt) { val = SE_CRYPTO_INPUT_SEL(INPUT_MEMORY) | SE_CRYPTO_VCTRAM_SEL(VCTRAM_TWEAK) | SE_CRYPTO_XOR_POS(XOR_BOTH) | SE_CRYPTO_CORE_SEL(CORE_ENCRYPT); } else { val = SE_CRYPTO_INPUT_SEL(INPUT_MEMORY) | SE_CRYPTO_VCTRAM_SEL(VCTRAM_TWEAK) | SE_CRYPTO_XOR_POS(XOR_BOTH) | SE_CRYPTO_CORE_SEL(CORE_DECRYPT); } if (se_dev->chipdata->kac_type == SE_KAC_T23X) val |= SE_CRYPTO_KEY2_INDEX(slot2_num); freq = se_dev->chipdata->aes_freq; break; case SE_AES_OP_MODE_CMAC: case SE_AES_OP_MODE_CBC: if (encrypt) { if (se_dev->chipdata->kac_type == SE_KAC_T18X || (se_dev->chipdata->kac_type == SE_KAC_T23X && mode == SE_AES_OP_MODE_CBC)) { val = SE_CRYPTO_INPUT_SEL(INPUT_MEMORY) | SE_CRYPTO_VCTRAM_SEL(VCTRAM_AESOUT) | SE_CRYPTO_XOR_POS(XOR_TOP) | SE_CRYPTO_CORE_SEL(CORE_ENCRYPT); } } else { val = SE_CRYPTO_INPUT_SEL(INPUT_MEMORY) | SE_CRYPTO_VCTRAM_SEL(VCTRAM_PREVAHB) | SE_CRYPTO_XOR_POS(XOR_BOTTOM) | SE_CRYPTO_CORE_SEL(CORE_DECRYPT); } freq = se_dev->chipdata->aes_freq; break; case SE_AES_OP_MODE_CBC_MAC: val = SE_CRYPTO_XOR_POS(XOR_TOP) | SE_CRYPTO_INPUT_SEL(INPUT_MEMORY) | SE_CRYPTO_VCTRAM_SEL(VCTRAM_AESOUT) | SE_CRYPTO_HASH(HASH_ENABLE); if (se_dev->chipdata->kac_type == SE_KAC_T23X) val |= SE_CRYPTO_CORE_SEL(CORE_ENCRYPT); break; case SE_AES_OP_MODE_RNG_DRBG: val = SE_CRYPTO_INPUT_SEL(INPUT_RANDOM) | SE_CRYPTO_XOR_POS(XOR_BYPASS) | SE_CRYPTO_CORE_SEL(CORE_ENCRYPT); break; case SE_AES_OP_MODE_ECB: if (encrypt) { val = SE_CRYPTO_INPUT_SEL(INPUT_MEMORY) | SE_CRYPTO_XOR_POS(XOR_BYPASS) | SE_CRYPTO_CORE_SEL(CORE_ENCRYPT); } else { val = SE_CRYPTO_INPUT_SEL(INPUT_MEMORY) | SE_CRYPTO_XOR_POS(XOR_BYPASS) | SE_CRYPTO_CORE_SEL(CORE_DECRYPT); } freq = se_dev->chipdata->aes_freq; break; case SE_AES_OP_MODE_CTR: val = SE_CRYPTO_INPUT_SEL(INPUT_LNR_CTR) | SE_CRYPTO_VCTRAM_SEL(VCTRAM_MEMORY) | SE_CRYPTO_XOR_POS(XOR_BOTTOM) | SE_CRYPTO_CORE_SEL(CORE_ENCRYPT); freq = se_dev->chipdata->aes_freq; break; case SE_AES_OP_MODE_OFB: val = SE_CRYPTO_INPUT_SEL(INPUT_AESOUT) | SE_CRYPTO_VCTRAM_SEL(VCTRAM_MEMORY) | SE_CRYPTO_XOR_POS(XOR_BOTTOM) | SE_CRYPTO_CORE_SEL(CORE_ENCRYPT); freq = se_dev->chipdata->aes_freq; break; case SE_AES_OP_MODE_GCM: break; default: dev_warn(se_dev->dev, "Invalid operation mode\n"); break; } if (mode == SE_AES_OP_MODE_CTR) { val |= SE_CRYPTO_HASH(HASH_DISABLE) | SE_CRYPTO_KEY_INDEX(slot_num) | SE_CRYPTO_CTR_CNTN(1); if (se_dev->chipdata->kac_type == SE_KAC_T23X) val |= SE_CRYPTO_IV_SEL(IV_REG); } else { val |= SE_CRYPTO_HASH(HASH_DISABLE) | SE_CRYPTO_KEY_INDEX(slot_num); if (se_dev->chipdata->kac_type == SE_KAC_T23X) { if (mode != SE_AES_OP_MODE_ECB && mode != SE_AES_OP_MODE_CMAC && mode != SE_AES_OP_MODE_GCM) val |= SE_CRYPTO_IV_SEL(IV_REG); } else { val |= (org_iv ? SE_CRYPTO_IV_SEL(IV_ORIGINAL) : SE_CRYPTO_IV_SEL(IV_UPDATED)); } } /* Enable hash for CMAC if running on T18X. */ if (mode == SE_AES_OP_MODE_CMAC && se_dev->chipdata->kac_type == SE_KAC_T18X) val |= SE_CRYPTO_HASH(HASH_ENABLE); if (mode == SE_AES_OP_MODE_RNG_DRBG) { if (force_reseed_count <= 0) { se_writel(se_dev, SE_RNG_CONFIG_MODE(DRBG_MODE_FORCE_RESEED) | SE_RNG_CONFIG_SRC(DRBG_SRC_ENTROPY), SE_RNG_CONFIG_REG_OFFSET); force_reseed_count = RNG_RESEED_INTERVAL; } else { se_writel(se_dev, SE_RNG_CONFIG_MODE(DRBG_MODE_NORMAL) | SE_RNG_CONFIG_SRC(DRBG_SRC_ENTROPY), SE_RNG_CONFIG_REG_OFFSET); } --force_reseed_count; se_writel(se_dev, RNG_RESEED_INTERVAL, SE_RNG_RESEED_INTERVAL_REG_OFFSET); } pr_debug("%s:%d crypto_config val = 0x%x\n", __func__, __LINE__, val); return val; } static int tegra_se_send_sha_data(struct tegra_se_dev *se_dev, struct ahash_request *req, struct tegra_se_sha_context *sha_ctx, u32 count, bool last) { int err = 0; u32 cmdbuf_num_words = 0, i = 0; u32 *cmdbuf_cpuvaddr = NULL; dma_addr_t cmdbuf_iova = 0; struct tegra_se_req_context *req_ctx = ahash_request_ctx(req); struct tegra_se_ll *src_ll = se_dev->src_ll; struct tegra_se_ll *dst_ll = se_dev->dst_ll; struct tegra_se_cmdbuf *cmdbuf; unsigned int total = count, val; u64 msg_len; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K * SE_MAX_SUBMIT_CHAIN_SZ); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); return PTR_ERR(cmdbuf); } cmdbuf_cpuvaddr = cmdbuf->addr; cmdbuf_iova = cmdbuf->iova; while (total) { if (src_ll->data_len & SE_BUFF_SIZE_MASK) { return -EINVAL; } if (total == count) { cmdbuf_cpuvaddr[i++] = nvhost_opcode_setpayload(8); cmdbuf_cpuvaddr[i++] = __nvhost_opcode_incr_w( se_dev->opcode_addr + SE_SHA_MSG_LENGTH_OFFSET); /* Program message length in next 4 four registers */ if (sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE128 || sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE256) { /* Reduce 4 extra bits in padded 8 bits */ msg_len = (count * 8) - 4; } else msg_len = (count * 8); cmdbuf_cpuvaddr[i++] = (sha_ctx->total_count * 8); cmdbuf_cpuvaddr[i++] = (u32)(msg_len >> 32); cmdbuf_cpuvaddr[i++] = 0; cmdbuf_cpuvaddr[i++] = 0; /* Program message left in next 4 four registers */ /* If it is not last request, length of message left * should be more than input buffer length. */ if (!last) cmdbuf_cpuvaddr[i++] = (u32)(msg_len + 8) & 0xFFFFFFFFULL; else cmdbuf_cpuvaddr[i++] = (u32)((msg_len) & 0xFFFFFFFFULL); cmdbuf_cpuvaddr[i++] = (u32)(msg_len >> 32); cmdbuf_cpuvaddr[i++] = 0; cmdbuf_cpuvaddr[i++] = 0; cmdbuf_cpuvaddr[i++] = nvhost_opcode_setpayload(6); cmdbuf_cpuvaddr[i++] = __nvhost_opcode_incr_w( se_dev->opcode_addr); cmdbuf_cpuvaddr[i++] = req_ctx->config; if (sha_ctx->is_first) cmdbuf_cpuvaddr[i++] = SE4_HW_INIT_HASH(HW_INIT_HASH_ENABLE); else cmdbuf_cpuvaddr[i++] = SE4_HW_INIT_HASH(HW_INIT_HASH_DISABLE); } else { cmdbuf_cpuvaddr[i++] = nvhost_opcode_setpayload(4); cmdbuf_cpuvaddr[i++] = __nvhost_opcode_incr_w( se_dev->opcode_addr + SE4_SHA_IN_ADDR_OFFSET); } cmdbuf_cpuvaddr[i++] = src_ll->addr; cmdbuf_cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(src_ll->addr)) | SE_ADDR_HI_SZ(src_ll->data_len)); cmdbuf_cpuvaddr[i++] = dst_ll->addr; cmdbuf_cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(dst_ll->addr)) | SE_ADDR_HI_SZ(dst_ll->data_len)); #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) /* For SHAKE128/SHAKE256 program digest size */ #if 0 if (sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE128 || sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE256) { cmdbuf_cpuvaddr[i++] = nvhost_opcode_setpayload(1); cmdbuf_cpuvaddr[i++] = __nvhost_opcode_nonincr_w( se_dev->opcode_addr + SE_SHA_HASH_LENGTH); cmdbuf_cpuvaddr[i++] = (req->dst_size * 8) << 2; } #endif #endif cmdbuf_cpuvaddr[i++] = nvhost_opcode_setpayload(1); cmdbuf_cpuvaddr[i++] = __nvhost_opcode_nonincr_w( se_dev->opcode_addr + SE_SHA_OPERATION_OFFSET); val = SE_OPERATION_WRSTALL(WRSTALL_TRUE); if (total == count) { if (total == src_ll->data_len) val |= SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(OP_START); else val |= SE_OPERATION_LASTBUF(LASTBUF_FALSE) | SE_OPERATION_OP(OP_START); } else { if (total == src_ll->data_len) val |= SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(OP_RESTART_IN); else val |= SE_OPERATION_LASTBUF(LASTBUF_FALSE) | SE_OPERATION_OP(OP_RESTART_IN); } cmdbuf_cpuvaddr[i++] = val; total -= src_ll->data_len; src_ll++; } cmdbuf_num_words = i; err = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, cmdbuf_num_words, SHA_CB); if (err) dev_err(se_dev->dev, "Channel submission fail err = %d\n", err); return err; } static int tegra_se_read_cmac_result(struct tegra_se_dev *se_dev, u8 *pdata, u32 nbytes, bool swap32) { u32 *result = (u32 *)pdata; u32 i; for (i = 0; i < nbytes / 4; i++) { if (se_dev->chipdata->kac_type == SE_KAC_T23X) { result[i] = se_readl(se_dev, se_dev->opcode_addr + T234_SE_CMAC_RESULT_REG_OFFSET + (i * sizeof(u32))); } else { result[i] = se_readl(se_dev, SE_CMAC_RESULT_REG_OFFSET + (i * sizeof(u32))); } if (swap32) result[i] = be32_to_cpu(result[i]); } return 0; } static int tegra_se_clear_cmac_result(struct tegra_se_dev *se_dev, u32 nbytes) { u32 i; for (i = 0; i < nbytes / 4; i++) { if (se_dev->chipdata->kac_type == SE_KAC_T23X) { se_writel(se_dev, 0x0, se_dev->opcode_addr + T234_SE_CMAC_RESULT_REG_OFFSET + (i * sizeof(u32))); } else { se_writel(se_dev, 0x0, SE_CMAC_RESULT_REG_OFFSET + (i * sizeof(u32))); } } return 0; } static void tegra_se_send_data(struct tegra_se_dev *se_dev, struct tegra_se_req_context *req_ctx, struct skcipher_request *req, u32 nbytes, unsigned int opcode_addr, u32 *cpuvaddr) { u32 cmdbuf_num_words = 0, i = 0; u32 total, val, rbits; unsigned int restart_op; struct tegra_se_ll *src_ll; struct tegra_se_ll *dst_ll; if (req) { src_ll = se_dev->aes_src_ll; dst_ll = se_dev->aes_dst_ll; src_ll->addr = se_dev->aes_cur_addr; dst_ll->addr = se_dev->aes_cur_addr; src_ll->data_len = req->cryptlen; dst_ll->data_len = req->cryptlen; } else { src_ll = se_dev->src_ll; dst_ll = se_dev->dst_ll; } i = se_dev->cmdbuf_cnt; total = nbytes; /* Create Gather Buffer Command */ while (total) { if (total == nbytes) { cpuvaddr[i++] = __nvhost_opcode_nonincr( opcode_addr + SE_AES_CRYPTO_LAST_BLOCK_OFFSET, 1); val = ((nbytes / TEGRA_SE_AES_BLOCK_SIZE)) - 1; if (req_ctx->op_mode == SE_AES_OP_MODE_CMAC && se_dev->chipdata->kac_type == SE_KAC_T23X) { rbits = (nbytes % TEGRA_SE_AES_BLOCK_SIZE) * 8; if (rbits) { val++; val |= SE_LAST_BLOCK_RESIDUAL_BITS(rbits); } } cpuvaddr[i++] = val; cpuvaddr[i++] = __nvhost_opcode_incr(opcode_addr, 6); cpuvaddr[i++] = req_ctx->config; cpuvaddr[i++] = req_ctx->crypto_config; } else { cpuvaddr[i++] = __nvhost_opcode_incr( opcode_addr + SE_AES_IN_ADDR_OFFSET, 4); } cpuvaddr[i++] = (u32)(src_ll->addr); cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(src_ll->addr)) | SE_ADDR_HI_SZ(src_ll->data_len)); cpuvaddr[i++] = (u32)(dst_ll->addr); cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(dst_ll->addr)) | SE_ADDR_HI_SZ(dst_ll->data_len)); if (req_ctx->op_mode == SE_AES_OP_MODE_CMAC || req_ctx->op_mode == SE_AES_OP_MODE_CBC_MAC) restart_op = OP_RESTART_IN; else if (req_ctx->op_mode == SE_AES_OP_MODE_RNG_DRBG) restart_op = OP_RESTART_OUT; else restart_op = OP_RESTART_INOUT; cpuvaddr[i++] = __nvhost_opcode_nonincr( opcode_addr + SE_AES_OPERATION_OFFSET, 1); val = SE_OPERATION_WRSTALL(WRSTALL_TRUE); if (total == nbytes) { if (total == src_ll->data_len) { val |= SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(OP_START); if ((req_ctx->op_mode == SE_AES_OP_MODE_CMAC || req_ctx->op_mode == SE_AES_OP_MODE_CBC_MAC) && se_dev->chipdata->kac_type == SE_KAC_T23X) val |= SE_OPERATION_FINAL(FINAL_TRUE); } else { val |= SE_OPERATION_LASTBUF(LASTBUF_FALSE) | SE_OPERATION_OP(OP_START); } } else { if (total == src_ll->data_len) { val |= SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(restart_op); if ((req_ctx->op_mode == SE_AES_OP_MODE_CMAC || req_ctx->op_mode == SE_AES_OP_MODE_CBC_MAC) && se_dev->chipdata->kac_type == SE_KAC_T23X) val |= SE_OPERATION_FINAL(FINAL_TRUE); } else { val |= SE_OPERATION_LASTBUF(LASTBUF_FALSE) | SE_OPERATION_OP(restart_op); } } cpuvaddr[i++] = val; total -= src_ll->data_len; src_ll++; dst_ll++; } cmdbuf_num_words = i; se_dev->cmdbuf_cnt = i; if (req) se_dev->aes_cur_addr += req->cryptlen; } static void tegra_se_send_gcm_data(struct tegra_se_dev *se_dev, struct tegra_se_req_context *req_ctx, u32 nbytes, unsigned int opcode_addr, u32 *cpuvaddr, enum tegra_se_aes_gcm_mode sub_mode) { u32 cmdbuf_num_words = 0, i = 0; u32 total, val, rbits; struct tegra_se_ll *src_ll; struct tegra_se_ll *dst_ll; unsigned int restart_op; src_ll = se_dev->src_ll; dst_ll = se_dev->dst_ll; i = se_dev->cmdbuf_cnt; total = nbytes; /* Create Gather Buffer Commands in multiple buffer */ /* (not multiple task) format. */ /* Program LAST_BLOCK */ cpuvaddr[i++] = __nvhost_opcode_nonincr( opcode_addr + SE_AES_CRYPTO_LAST_BLOCK_OFFSET, 1); val = ((nbytes / TEGRA_SE_AES_BLOCK_SIZE)) - 1; rbits = (nbytes % TEGRA_SE_AES_BLOCK_SIZE) * 8; if (rbits) { val++; val |= SE_LAST_BLOCK_RESIDUAL_BITS(rbits); } cpuvaddr[i++] = val; /* Program config and crypto config */ cpuvaddr[i++] = __nvhost_opcode_incr(opcode_addr, 2); cpuvaddr[i++] = req_ctx->config; cpuvaddr[i++] = req_ctx->crypto_config; while (total) { /* Program IN and OUT addresses */ if (sub_mode != SE_AES_GMAC) { cpuvaddr[i++] = __nvhost_opcode_incr( opcode_addr + SE_AES_IN_ADDR_OFFSET, 4); } else { cpuvaddr[i++] = __nvhost_opcode_incr( opcode_addr + SE_AES_IN_ADDR_OFFSET, 2); } cpuvaddr[i++] = (u32)(src_ll->addr); cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(src_ll->addr)) | SE_ADDR_HI_SZ(src_ll->data_len)); if (sub_mode != SE_AES_GMAC) { cpuvaddr[i++] = (u32)(dst_ll->addr); cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(dst_ll->addr)) | SE_ADDR_HI_SZ(dst_ll->data_len)); } /* Program operation register */ if (sub_mode == SE_AES_GCM_ENC || sub_mode == SE_AES_GCM_DEC) restart_op = OP_RESTART_INOUT; else restart_op = OP_RESTART_IN; cpuvaddr[i++] = __nvhost_opcode_nonincr( opcode_addr + SE_AES_OPERATION_OFFSET, 1); val = SE_OPERATION_WRSTALL(WRSTALL_TRUE); if (req_ctx->init != true) {/* GMAC not operated */ val |= SE_OPERATION_INIT(INIT_TRUE); req_ctx->init = true; } val |= SE_OPERATION_FINAL(FINAL_TRUE); if (total == nbytes) { if (total == src_ll->data_len) { val |= SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(OP_START); } else { val |= SE_OPERATION_LASTBUF(LASTBUF_FALSE) | SE_OPERATION_OP(OP_START); } } else { if (total == src_ll->data_len) { val |= SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(restart_op); } else { val |= SE_OPERATION_LASTBUF(LASTBUF_FALSE) | SE_OPERATION_OP(restart_op); } } cpuvaddr[i++] = val; total -= src_ll->data_len; src_ll++; dst_ll++; } cmdbuf_num_words = i; se_dev->cmdbuf_cnt = i; } static int tegra_map_sg(struct device *dev, struct scatterlist *sg, unsigned int nents, enum dma_data_direction dir, struct tegra_se_ll *se_ll, u32 total) { u32 total_loop = 0; int ret = 0; total_loop = total; while (sg && (total_loop > 0)) { ret = dma_map_sg(dev, sg, nents, dir); if (!ret) { dev_err(dev, "dma_map_sg error\n"); return ret; } se_ll->addr = sg_dma_address(sg); se_ll->data_len = min((size_t)sg->length, (size_t)total_loop); total_loop -= min((size_t)sg->length, (size_t)total_loop); sg = sg_next(sg); se_ll++; } return ret; } static int tegra_se_setup_ablk_req(struct tegra_se_dev *se_dev) { struct skcipher_request *req; void *buf; int i, ret = 0; u32 num_sgs; unsigned int index = 0; if (unlikely(se_dev->dynamic_mem)) { if (se_dev->ioc) se_dev->aes_buf = dma_alloc_coherent( se_dev->dev, se_dev->gather_buf_sz, &se_dev->aes_buf_addr, GFP_KERNEL); else se_dev->aes_buf = kmalloc(se_dev->gather_buf_sz, GFP_KERNEL); if (!se_dev->aes_buf) return -ENOMEM; buf = se_dev->aes_buf; } else { index = se_dev->aesbuf_entry + 1; for (i = 0; i < SE_MAX_AESBUF_TIMEOUT; i++, index++) { index = index % SE_MAX_AESBUF_ALLOC; if (atomic_read(&se_dev->aes_buf_stat[index])) { se_dev->aesbuf_entry = index; atomic_set(&se_dev->aes_buf_stat[index], 0); break; } if (i % SE_MAX_AESBUF_ALLOC == 0) udelay(SE_WAIT_UDELAY); } if (i == SE_MAX_AESBUF_TIMEOUT) { pr_err("aes_buffer not available\n"); return -ETIMEDOUT; } buf = se_dev->aes_bufs[index]; } for (i = 0; i < se_dev->req_cnt; i++) { req = se_dev->reqs[i]; num_sgs = tegra_se_count_sgs(req->src, req->cryptlen); if (num_sgs == 1) memcpy(buf, sg_virt(req->src), req->cryptlen); else sg_copy_to_buffer(req->src, num_sgs, buf, req->cryptlen); buf += req->cryptlen; } if (se_dev->ioc) { if (unlikely(se_dev->dynamic_mem)) se_dev->aes_addr = se_dev->aes_buf_addr; else se_dev->aes_addr = se_dev->aes_buf_addrs[index]; } else { if (unlikely(se_dev->dynamic_mem)) sg_init_one(&se_dev->sg, se_dev->aes_buf, se_dev->gather_buf_sz); else sg_init_one(&se_dev->sg, se_dev->aes_bufs[index], se_dev->gather_buf_sz); ret = dma_map_sg(se_dev->dev, &se_dev->sg, 1, DMA_BIDIRECTIONAL); if (!ret) { dev_err(se_dev->dev, "dma_map_sg error\n"); if (unlikely(se_dev->dynamic_mem)) kfree(se_dev->aes_buf); else atomic_set(&se_dev->aes_buf_stat[index], 1); return ret; } se_dev->aes_addr = sg_dma_address(&se_dev->sg); } se_dev->aes_cur_addr = se_dev->aes_addr; return 0; } static int tegra_se_prepare_cmdbuf(struct tegra_se_dev *se_dev, struct tegra_se_cmdbuf *cmdbuf) { int i, ret = 0; u32 *cpuvaddr = cmdbuf->addr; struct tegra_se_aes_context *aes_ctx; struct skcipher_request *req; struct tegra_se_req_context *req_ctx; struct crypto_skcipher *tfm; u32 keylen; pr_debug("%s:%d req_cnt = %d\n", __func__, __LINE__, se_dev->req_cnt); for (i = 0; i < se_dev->req_cnt; i++) { req = se_dev->reqs[i]; tfm = crypto_skcipher_reqtfm(req); aes_ctx = crypto_skcipher_ctx(tfm); /* Ensure there is valid slot info */ if (!aes_ctx->slot) { dev_err(se_dev->dev, "Invalid AES Ctx Slot\n"); return -EINVAL; } if (aes_ctx->is_key_in_mem) { if (strcmp(crypto_tfm_alg_name(&tfm->base), "xts(aes)")) { ret = tegra_se_send_key_data( se_dev, aes_ctx->key, aes_ctx->keylen, aes_ctx->slot->slot_num, SE_KEY_TABLE_TYPE_KEY_IN_MEM, se_dev->opcode_addr, cmdbuf, AES_CB); } else { keylen = aes_ctx->keylen / 2; ret = tegra_se_send_key_data(se_dev, aes_ctx->key, keylen, aes_ctx->slot->slot_num, SE_KEY_TABLE_TYPE_XTS_KEY1_IN_MEM, se_dev->opcode_addr, cmdbuf, AES_CB); if (ret) { dev_err(se_dev->dev, "Error in setting Key\n"); goto out; } ret = tegra_se_send_key_data(se_dev, aes_ctx->key + keylen, keylen, aes_ctx->slot->slot_num, SE_KEY_TABLE_TYPE_XTS_KEY2_IN_MEM, se_dev->opcode_addr, cmdbuf, AES_CB); } if (ret) { dev_err(se_dev->dev, "Error in setting Key\n"); goto out; } } req_ctx = skcipher_request_ctx(req); if (req->iv) { if (req_ctx->op_mode == SE_AES_OP_MODE_CTR || req_ctx->op_mode == SE_AES_OP_MODE_XTS) { tegra_se_send_ctr_seed(se_dev, (u32 *)req->iv, se_dev->opcode_addr, cpuvaddr); } else { ret = tegra_se_send_key_data( se_dev, req->iv, TEGRA_SE_AES_IV_SIZE, aes_ctx->slot->slot_num, SE_KEY_TABLE_TYPE_UPDTDIV, se_dev->opcode_addr, cmdbuf, AES_CB); } } if (ret) return ret; req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, req_ctx->encrypt, aes_ctx->keylen); if (aes_ctx->slot2 != NULL) { req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, req_ctx->encrypt, aes_ctx->slot->slot_num, aes_ctx->slot2->slot_num, false); } else { req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, req_ctx->encrypt, aes_ctx->slot->slot_num, 0, false); } tegra_se_send_data(se_dev, req_ctx, req, req->cryptlen, se_dev->opcode_addr, cpuvaddr); } out: return ret; } static void tegra_se_process_new_req(struct tegra_se_dev *se_dev) { struct skcipher_request *req; u32 *cpuvaddr = NULL; dma_addr_t iova = 0; struct tegra_se_cmdbuf *cmdbuf; int err = 0, i = 0; pr_debug("%s:%d start req_cnt = %d\n", __func__, __LINE__, se_dev->req_cnt); tegra_se_boost_cpu_freq(se_dev); for (i = 0; i < se_dev->req_cnt; i++) { req = se_dev->reqs[i]; if (req->cryptlen != SE_STATIC_MEM_ALLOC_BUFSZ) { se_dev->dynamic_mem = true; break; } } err = tegra_se_setup_ablk_req(se_dev); if (err) goto mem_out; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K * SE_MAX_SUBMIT_CHAIN_SZ); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); goto mem_out; } cpuvaddr = cmdbuf->addr; iova = cmdbuf->iova; err = tegra_se_prepare_cmdbuf(se_dev, cmdbuf); if (err) goto cmdbuf_out; err = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, AES_CB); if (err) goto cmdbuf_out; se_dev->dynamic_mem = false; pr_debug("%s:%d complete\n", __func__, __LINE__); return; cmdbuf_out: dma_unmap_sg(se_dev->dev, &se_dev->sg, 1, DMA_BIDIRECTIONAL); //if (se_dev->dynamic_mem) // kfree(se_dev->aes_buf); mem_out: for (i = 0; i < se_dev->req_cnt; i++) { req = se_dev->reqs[i]; req->base.complete(&req->base, err); } se_dev->req_cnt = 0; se_dev->gather_buf_sz = 0; se_dev->cmdbuf_cnt = 0; se_dev->dynamic_mem = false; } static void tegra_se_work_handler(struct work_struct *work) { struct tegra_se_dev *se_dev = container_of(work, struct tegra_se_dev, se_work); struct crypto_async_request *async_req = NULL; struct crypto_async_request *backlog = NULL; struct skcipher_request *req; bool process_requests; mutex_lock(&se_dev->mtx); do { process_requests = false; mutex_lock(&se_dev->lock); do { backlog = crypto_get_backlog(&se_dev->queue); async_req = crypto_dequeue_request(&se_dev->queue); if (!async_req) se_dev->work_q_busy = false; if (backlog) { backlog->complete(backlog, -EINPROGRESS); backlog = NULL; } if (async_req) { req = skcipher_request_cast(async_req); se_dev->reqs[se_dev->req_cnt] = req; se_dev->gather_buf_sz += req->cryptlen; se_dev->req_cnt++; process_requests = true; } else { break; } } while (se_dev->queue.qlen && (se_dev->req_cnt < SE_MAX_TASKS_PER_SUBMIT)); mutex_unlock(&se_dev->lock); if (process_requests) tegra_se_process_new_req(se_dev); } while (se_dev->work_q_busy); mutex_unlock(&se_dev->mtx); } static int tegra_se_aes_queue_req(struct tegra_se_dev *se_dev, struct skcipher_request *req) { int err = 0; mutex_lock(&se_dev->lock); err = crypto_enqueue_request(&se_dev->queue, &req->base); if (!se_dev->work_q_busy) { se_dev->work_q_busy = true; mutex_unlock(&se_dev->lock); queue_work(se_dev->se_work_q, &se_dev->se_work); } else { mutex_unlock(&se_dev->lock); } return err; } static int tegra_se_aes_xts_encrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; if (!req_ctx->se_dev) { pr_err("Device is NULL\n"); return -ENODEV; } req_ctx->encrypt = true; req_ctx->op_mode = SE_AES_OP_MODE_XTS; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_xts_decrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; if (!req_ctx->se_dev) { pr_err("Device is NULL\n"); return -ENODEV; } req_ctx->encrypt = false; req_ctx->op_mode = SE_AES_OP_MODE_XTS; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_cbc_encrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; req_ctx->encrypt = true; req_ctx->op_mode = SE_AES_OP_MODE_CBC; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_cbc_decrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; req_ctx->encrypt = false; req_ctx->op_mode = SE_AES_OP_MODE_CBC; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_ecb_encrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; req_ctx->encrypt = true; req_ctx->op_mode = SE_AES_OP_MODE_ECB; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_ecb_decrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; req_ctx->encrypt = false; req_ctx->op_mode = SE_AES_OP_MODE_ECB; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_ctr_encrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; req_ctx->encrypt = true; req_ctx->op_mode = SE_AES_OP_MODE_CTR; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_ctr_decrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; req_ctx->encrypt = false; req_ctx->op_mode = SE_AES_OP_MODE_CTR; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_ofb_encrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; req_ctx->encrypt = true; req_ctx->op_mode = SE_AES_OP_MODE_OFB; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static int tegra_se_aes_ofb_decrypt(struct skcipher_request *req) { struct tegra_se_req_context *req_ctx = skcipher_request_ctx(req); req_ctx->se_dev = se_devices[SE_AES]; req_ctx->encrypt = false; req_ctx->op_mode = SE_AES_OP_MODE_OFB; return tegra_se_aes_queue_req(req_ctx->se_dev, req); } static void tegra_se_init_aesbuf(struct tegra_se_dev *se_dev) { int i; void *buf = se_dev->total_aes_buf; dma_addr_t buf_addr = se_dev->total_aes_buf_addr; for (i = 0; i < SE_MAX_AESBUF_ALLOC; i++) { se_dev->aes_bufs[i] = buf + (i * SE_MAX_GATHER_BUF_SZ); if (se_dev->ioc) se_dev->aes_buf_addrs[i] = buf_addr + (i * SE_MAX_GATHER_BUF_SZ); atomic_set(&se_dev->aes_buf_stat[i], 1); } } static int tegra_se_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, u32 keylen) { struct tegra_se_aes_context *ctx = crypto_tfm_ctx(&tfm->base); struct tegra_se_dev *se_dev; struct tegra_se_slot *pslot; struct tegra_se_cmdbuf *cmdbuf; u8 *pdata = (u8 *)key; int ret = 0; u32 *cpuvaddr = NULL; se_dev = se_devices[SE_AES]; if (!ctx || !se_dev) { pr_err("invalid context or dev"); return -EINVAL; } ctx->se_dev = se_dev; if (((keylen & SE_KEY_LEN_MASK) != TEGRA_SE_KEY_128_SIZE) && ((keylen & SE_KEY_LEN_MASK) != TEGRA_SE_KEY_192_SIZE) && ((keylen & SE_KEY_LEN_MASK) != TEGRA_SE_KEY_256_SIZE) && ((keylen & SE_KEY_LEN_MASK) != TEGRA_SE_KEY_512_SIZE)) { dev_err(se_dev->dev, "invalid key size"); return -EINVAL; } if ((keylen >> SE_MAGIC_PATTERN_OFFSET) == SE_STORE_KEY_IN_MEM) { ctx->is_key_in_mem = true; ctx->keylen = (keylen & SE_KEY_LEN_MASK); ctx->slot = &keymem_slot; memcpy(ctx->key, key, ctx->keylen); return 0; } ctx->is_key_in_mem = false; mutex_lock(&se_dev->mtx); if (key) { if (!ctx->slot || (ctx->slot && ctx->slot->slot_num == ssk_slot.slot_num)) { pslot = tegra_se_alloc_key_slot(); if (!pslot) { dev_err(se_dev->dev, "no free key slot\n"); mutex_unlock(&se_dev->mtx); return -ENOMEM; } ctx->slot = pslot; ctx->slot2 = NULL; if (!strcmp(crypto_tfm_alg_name(&tfm->base), "xts(aes)")) { if (se_dev->chipdata->kac_type == SE_KAC_T23X) { pslot = tegra_se_alloc_key_slot(); if (!pslot) { dev_err(se_dev->dev, "no free key slot\n"); mutex_unlock(&se_dev->mtx); return -ENOMEM; } ctx->slot2 = pslot; } } } ctx->keylen = keylen; } else if ((keylen >> SE_MAGIC_PATTERN_OFFSET) == SE_MAGIC_PATTERN) { ctx->slot = &pre_allocated_slot; spin_lock(&key_slot_lock); pre_allocated_slot.slot_num = ((keylen & SE_SLOT_NUM_MASK) >> SE_SLOT_POSITION); spin_unlock(&key_slot_lock); ctx->keylen = (keylen & SE_KEY_LEN_MASK); goto out; } else { tegra_se_free_key_slot(ctx->slot); ctx->slot = &ssk_slot; ctx->keylen = AES_KEYSIZE_128; goto out; } cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K * SE_MAX_SUBMIT_CHAIN_SZ); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto out; } cpuvaddr = cmdbuf->addr; /* load the key */ if (strcmp(crypto_tfm_alg_name(&tfm->base), "xts(aes)")) { ret = tegra_se_send_key_data( se_dev, pdata, keylen, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_KEY, se_dev->opcode_addr, cmdbuf, AES_CB); } else { keylen = keylen / 2; ret = tegra_se_send_key_data( se_dev, pdata, keylen, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_XTS_KEY1, se_dev->opcode_addr, cmdbuf, AES_CB); if (ret) goto keyslt_free; if (se_dev->chipdata->kac_type == SE_KAC_T23X) { ret = tegra_se_send_key_data(se_dev, pdata + keylen, keylen, ctx->slot2->slot_num, SE_KEY_TABLE_TYPE_XTS_KEY2, se_dev->opcode_addr, cmdbuf, AES_CB); } else { ret = tegra_se_send_key_data(se_dev, pdata + keylen, keylen, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_XTS_KEY2, se_dev->opcode_addr, cmdbuf, AES_CB); } } keyslt_free: if (ret) { tegra_se_free_key_slot(ctx->slot); tegra_se_free_key_slot(ctx->slot2); } out: mutex_unlock(&se_dev->mtx); return ret; } static int tegra_se_aes_cra_init(struct crypto_skcipher *tfm) { tfm->reqsize = sizeof(struct tegra_se_req_context); return 0; } static void tegra_se_aes_cra_exit(struct crypto_skcipher *tfm) { struct tegra_se_aes_context *ctx = crypto_tfm_ctx(&tfm->base); tegra_se_free_key_slot(ctx->slot); tegra_se_free_key_slot(ctx->slot2); ctx->slot = NULL; ctx->slot2 = NULL; } static int tegra_se_rng_drbg_init(struct crypto_tfm *tfm) { struct tegra_se_rng_context *rng_ctx = crypto_tfm_ctx(tfm); struct tegra_se_dev *se_dev; se_dev = se_devices[SE_DRBG]; mutex_lock(&se_dev->mtx); rng_ctx->se_dev = se_dev; rng_ctx->dt_buf = dma_alloc_coherent(se_dev->dev, TEGRA_SE_RNG_DT_SIZE, &rng_ctx->dt_buf_adr, GFP_KERNEL); if (!rng_ctx->dt_buf) { dev_err(se_dev->dev, "can not allocate rng dma buffer"); mutex_unlock(&se_dev->mtx); return -ENOMEM; } rng_ctx->rng_buf = dma_alloc_coherent( rng_ctx->se_dev->dev, TEGRA_SE_RNG_DT_SIZE, &rng_ctx->rng_buf_adr, GFP_KERNEL); if (!rng_ctx->rng_buf) { dev_err(se_dev->dev, "can not allocate rng dma buffer"); dma_free_coherent(rng_ctx->se_dev->dev, TEGRA_SE_RNG_DT_SIZE, rng_ctx->dt_buf, rng_ctx->dt_buf_adr); mutex_unlock(&se_dev->mtx); return -ENOMEM; } mutex_unlock(&se_dev->mtx); return 0; } static int tegra_se_rng_drbg_get_random(struct crypto_rng *tfm, const u8 *src, unsigned int slen, u8 *rdata, unsigned int dlen) { struct tegra_se_rng_context *rng_ctx = crypto_rng_ctx(tfm); struct tegra_se_dev *se_dev = rng_ctx->se_dev; u8 *rdata_addr; int ret = 0, j; unsigned int num_blocks, data_len = 0; struct tegra_se_cmdbuf *cmdbuf; struct tegra_se_req_context *req_ctx = devm_kzalloc(se_dev->dev, sizeof(struct tegra_se_req_context), GFP_KERNEL); if (!req_ctx) return -ENOMEM; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K * SE_MAX_SUBMIT_CHAIN_SZ); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto free_ctx; } num_blocks = (dlen / TEGRA_SE_RNG_DT_SIZE); data_len = (dlen % TEGRA_SE_RNG_DT_SIZE); if (data_len == 0) num_blocks = num_blocks - 1; mutex_lock(&se_dev->mtx); req_ctx->op_mode = SE_AES_OP_MODE_RNG_DRBG; se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); se_dev->dst_ll = (struct tegra_se_ll *)(se_dev->dst_ll_buf); req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, true, TEGRA_SE_KEY_128_SIZE); if (se_dev->chipdata->kac_type != SE_KAC_T23X) req_ctx->crypto_config = tegra_se_get_crypto_config(se_dev, req_ctx->op_mode, true, 0, 0, true); for (j = 0; j <= num_blocks; j++) { se_dev->src_ll->addr = rng_ctx->dt_buf_adr; se_dev->src_ll->data_len = TEGRA_SE_RNG_DT_SIZE; se_dev->dst_ll->addr = rng_ctx->rng_buf_adr; se_dev->dst_ll->data_len = TEGRA_SE_RNG_DT_SIZE; tegra_se_send_data(se_dev, req_ctx, NULL, TEGRA_SE_RNG_DT_SIZE, se_dev->opcode_addr, cmdbuf->addr); ret = tegra_se_channel_submit_gather( se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); if (ret) break; rdata_addr = (rdata + (j * TEGRA_SE_RNG_DT_SIZE)); if (data_len && num_blocks == j) memcpy(rdata_addr, rng_ctx->rng_buf, data_len); else memcpy(rdata_addr, rng_ctx->rng_buf, TEGRA_SE_RNG_DT_SIZE); } if (!ret) ret = dlen; mutex_unlock(&se_dev->mtx); free_ctx: devm_kfree(se_dev->dev, req_ctx); return ret; } static int tegra_se_rng_drbg_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) { return 0; } static void tegra_se_rng_drbg_exit(struct crypto_tfm *tfm) { struct tegra_se_rng_context *rng_ctx = crypto_tfm_ctx(tfm); if (rng_ctx->dt_buf) dma_free_coherent(rng_ctx->se_dev->dev, TEGRA_SE_RNG_DT_SIZE, rng_ctx->dt_buf, rng_ctx->dt_buf_adr); if (rng_ctx->rng_buf) dma_free_coherent(rng_ctx->se_dev->dev, TEGRA_SE_RNG_DT_SIZE, rng_ctx->rng_buf, rng_ctx->rng_buf_adr); rng_ctx->se_dev = NULL; } static void tegra_se_sha_copy_residual_data( struct ahash_request *req, struct tegra_se_sha_context *sha_ctx, u32 bytes_to_copy) { struct sg_mapping_iter miter; unsigned int sg_flags, total = 0; u32 num_sgs, last_block_bytes = bytes_to_copy; unsigned long flags; struct scatterlist *src_sg; u8 *temp_buffer = NULL; src_sg = req->src; num_sgs = tegra_se_count_sgs(req->src, req->nbytes); sg_flags = SG_MITER_ATOMIC | SG_MITER_FROM_SG; sg_miter_start(&miter, req->src, num_sgs, sg_flags); local_irq_save(flags); temp_buffer = sha_ctx->sha_buf[0]; while (sg_miter_next(&miter) && total < req->nbytes) { unsigned int len; len = min(miter.length, (size_t)(req->nbytes - total)); if ((req->nbytes - (total + len)) <= last_block_bytes) { bytes_to_copy = last_block_bytes - (req->nbytes - (total + len)); memcpy(temp_buffer, miter.addr + (len - bytes_to_copy), bytes_to_copy); last_block_bytes -= bytes_to_copy; temp_buffer += bytes_to_copy; } total += len; } sg_miter_stop(&miter); local_irq_restore(flags); } static int tegra_se_sha_process_buf(struct ahash_request *req, bool is_last, bool process_cur_req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_se_sha_context *sha_ctx = crypto_ahash_ctx(tfm); struct tegra_se_req_context *req_ctx = ahash_request_ctx(req); struct tegra_se_dev *se_dev = se_devices[SE_SHA]; struct scatterlist *src_sg = req->src; struct tegra_se_ll *src_ll; struct tegra_se_ll *dst_ll; u32 current_total = 0, num_sgs, bytes_process_in_req = 0, num_blks; int err = 0, dst_len = crypto_ahash_digestsize(tfm); pr_debug("%s:%d process sha buffer\n", __func__, __LINE__); num_sgs = tegra_se_count_sgs(req->src, req->nbytes); if (num_sgs > SE_MAX_SRC_SG_COUNT) { dev_err(se_dev->dev, "num of SG buffers are more\n"); return -ENOTSUPP; } #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) #if 0 /* For SHAKE128/SHAKE256, digest size can vary */ if (sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE128 || sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE256) { dst_len = req->dst_size; if (dst_len == 0) { req->base.complete(&req->base, 0); return 0; } } #endif #endif req_ctx->hash_result = devm_kzalloc(se_dev->dev, dst_len, GFP_KERNEL); if (!req_ctx->hash_result) return -ENOMEM; sg_init_one(&se_dev->sg, req_ctx->hash_result, dst_len); se_dev->sha_last = is_last; if (is_last) { /* Prepare buf for residual and current data */ /* Fill sgs entries */ se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); src_ll = se_dev->src_ll; se_dev->dst_ll = (struct tegra_se_ll *)(se_dev->dst_ll_buf); dst_ll = se_dev->dst_ll; if (sha_ctx->residual_bytes) { src_ll->addr = sha_ctx->sha_buf_addr[0]; src_ll->data_len = sha_ctx->residual_bytes; src_ll++; } if (process_cur_req) { bytes_process_in_req = req->nbytes; err = tegra_map_sg(se_dev->dev, src_sg, 1, DMA_TO_DEVICE, src_ll, bytes_process_in_req); if (!err) return -EINVAL; current_total = req->nbytes + sha_ctx->residual_bytes; sha_ctx->total_count += current_total; err = tegra_map_sg(se_dev->dev, &se_dev->sg, 1, DMA_FROM_DEVICE, dst_ll, dst_len); if (!err) return -EINVAL; se_dev->src_bytes_mapped = bytes_process_in_req; se_dev->dst_bytes_mapped = dst_len; se_dev->sha_src_mapped = true; se_dev->sha_dst_mapped = true; } else { current_total = sha_ctx->residual_bytes; sha_ctx->total_count += current_total; if (!current_total) { req->base.complete(&req->base, 0); return 0; } err = tegra_map_sg(se_dev->dev, &se_dev->sg, 1, DMA_FROM_DEVICE, dst_ll, dst_len); if (!err) return -EINVAL; se_dev->dst_bytes_mapped = dst_len; se_dev->sha_dst_mapped = true; } /* Pad last byte to be 0xff for SHAKE128/256 */ if (sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE128 || sha_ctx->op_mode == SE_AES_OP_MODE_SHAKE256) { while (num_sgs && process_cur_req) { src_ll++; num_sgs--; } sha_ctx->sha_buf[1][0] = 0xff; src_ll->addr = sha_ctx->sha_buf_addr[1]; src_ll->data_len = 1; src_ll++; current_total += 1; } } else { current_total = req->nbytes + sha_ctx->residual_bytes; num_blks = current_total / sha_ctx->blk_size; /* If total bytes is less than or equal to one blk, * copy to residual and return. */ if (num_blks <= 1) { sg_copy_to_buffer(req->src, num_sgs, sha_ctx->sha_buf[0] + sha_ctx->residual_bytes, req->nbytes); sha_ctx->residual_bytes += req->nbytes; req->base.complete(&req->base, 0); return 0; } /* Number of bytes to be processed from given request buffers */ bytes_process_in_req = (num_blks * sha_ctx->blk_size) - sha_ctx->residual_bytes; sha_ctx->total_count += bytes_process_in_req; /* Fill sgs entries */ /* If residual bytes are present copy it to second buffer */ if (sha_ctx->residual_bytes) memcpy(sha_ctx->sha_buf[1], sha_ctx->sha_buf[0], sha_ctx->residual_bytes); sha_ctx->total_count += sha_ctx->residual_bytes; se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); src_ll = se_dev->src_ll; if (sha_ctx->residual_bytes) { src_ll->addr = sha_ctx->sha_buf_addr[1]; src_ll->data_len = sha_ctx->residual_bytes; src_ll++; } se_dev->dst_ll = (struct tegra_se_ll *)(se_dev->dst_ll_buf); dst_ll = se_dev->dst_ll; /* Map required bytes to process in given request */ err = tegra_map_sg(se_dev->dev, src_sg, 1, DMA_TO_DEVICE, src_ll, bytes_process_in_req); if (!err) return -EINVAL; /* Copy residual data */ sha_ctx->residual_bytes = current_total - (num_blks * sha_ctx->blk_size); tegra_se_sha_copy_residual_data(req, sha_ctx, sha_ctx->residual_bytes); /* Total bytes to be processed */ current_total = (num_blks * sha_ctx->blk_size); err = tegra_map_sg(se_dev->dev, &se_dev->sg, 1, DMA_FROM_DEVICE, dst_ll, dst_len); if (!err) return -EINVAL; se_dev->src_bytes_mapped = bytes_process_in_req; se_dev->dst_bytes_mapped = dst_len; se_dev->sha_src_mapped = true; se_dev->sha_dst_mapped = true; } req_ctx->config = tegra_se_get_config(se_dev, sha_ctx->op_mode, false, is_last); err = tegra_se_send_sha_data(se_dev, req, sha_ctx, current_total, is_last); if (err) { if (se_dev->sha_src_mapped) tegra_unmap_sg(se_dev->dev, src_sg, DMA_TO_DEVICE, bytes_process_in_req); if (se_dev->sha_dst_mapped) tegra_unmap_sg(se_dev->dev, &se_dev->sg, DMA_FROM_DEVICE, dst_len); return err; } sha_ctx->is_first = false; pr_debug("%s:%d process sha buffer complete\n", __func__, __LINE__); return 0; } static int tegra_se_sha_op(struct ahash_request *req, bool is_last, bool process_cur_req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_se_sha_context *sha_ctx = crypto_ahash_ctx(tfm); struct tegra_se_dev *se_dev = se_devices[SE_SHA]; u32 mode; int ret; struct tegra_se_sha_zero_length_vector zero_vec[] = { { .size = SHA1_DIGEST_SIZE, .digest = "\xda\x39\xa3\xee\x5e\x6b\x4b\x0d" "\x32\x55\xbf\xef\x95\x60\x18\x90" "\xaf\xd8\x07\x09", }, { .size = SHA224_DIGEST_SIZE, .digest = "\xd1\x4a\x02\x8c\x2a\x3a\x2b\xc9" "\x47\x61\x02\xbb\x28\x82\x34\xc4" "\x15\xa2\xb0\x1f\x82\x8e\xa6\x2a" "\xc5\xb3\xe4\x2f", }, { .size = SHA256_DIGEST_SIZE, .digest = "\xe3\xb0\xc4\x42\x98\xfc\x1c\x14" "\x9a\xfb\xf4\xc8\x99\x6f\xb9\x24" "\x27\xae\x41\xe4\x64\x9b\x93\x4c" "\xa4\x95\x99\x1b\x78\x52\xb8\x55", }, { .size = SHA384_DIGEST_SIZE, .digest = "\x38\xb0\x60\xa7\x51\xac\x96\x38" "\x4c\xd9\x32\x7e\xb1\xb1\xe3\x6a" "\x21\xfd\xb7\x11\x14\xbe\x07\x43" "\x4c\x0c\xc7\xbf\x63\xf6\xe1\xda" "\x27\x4e\xde\xbf\xe7\x6f\x65\xfb" "\xd5\x1a\xd2\xf1\x48\x98\xb9\x5b", }, { .size = SHA512_DIGEST_SIZE, .digest = "\xcf\x83\xe1\x35\x7e\xef\xb8\xbd" "\xf1\x54\x28\x50\xd6\x6d\x80\x07" "\xd6\x20\xe4\x05\x0b\x57\x15\xdc" "\x83\xf4\xa9\x21\xd3\x6c\xe9\xce" "\x47\xd0\xd1\x3c\x5d\x85\xf2\xb0" "\xff\x83\x18\xd2\x87\x7e\xec\x2f" "\x63\xb9\x31\xbd\x47\x41\x7a\x81" "\xa5\x38\x32\x7a\xf9\x27\xda\x3e", }, { .size = SHA3_224_DIGEST_SIZE, .digest = "\x6b\x4e\x03\x42\x36\x67\xdb\xb7" "\x3b\x6e\x15\x45\x4f\x0e\xb1\xab" "\xd4\x59\x7f\x9a\x1b\x07\x8e\x3f" "\x5b\x5a\x6b\xc7", }, { .size = SHA3_256_DIGEST_SIZE, .digest = "\xa7\xff\xc6\xf8\xbf\x1e\xd7\x66" "\x51\xc1\x47\x56\xa0\x61\xd6\x62" "\xf5\x80\xff\x4d\xe4\x3b\x49\xfa" "\x82\xd8\x0a\x4b\x80\xf8\x43\x4a", }, { .size = SHA3_384_DIGEST_SIZE, .digest = "\x0c\x63\xa7\x5b\x84\x5e\x4f\x7d" "\x01\x10\x7d\x85\x2e\x4c\x24\x85" "\xc5\x1a\x50\xaa\xaa\x94\xfc\x61" "\x99\x5e\x71\xbb\xee\x98\x3a\x2a" "\xc3\x71\x38\x31\x26\x4a\xdb\x47" "\xfb\x6b\xd1\xe0\x58\xd5\xf0\x04", }, { .size = SHA3_512_DIGEST_SIZE, .digest = "\xa6\x9f\x73\xcc\xa2\x3a\x9a\xc5" "\xc8\xb5\x67\xdc\x18\x5a\x75\x6e" "\x97\xc9\x82\x16\x4f\xe2\x58\x59" "\xe0\xd1\xdc\xc1\x47\x5c\x80\xa6" "\x15\xb2\x12\x3a\xf1\xf5\xf9\x4c" "\x11\xe3\xe9\x40\x2c\x3a\xc5\x58" "\xf5\x00\x19\x9d\x95\xb6\xd3\xe3" "\x01\x75\x85\x86\x28\x1d\xcd\x26", } }; if (strcmp(crypto_ahash_alg_name(tfm), "sha1") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA1; } else if (strcmp(crypto_ahash_alg_name(tfm), "sha224") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA224; } else if (strcmp(crypto_ahash_alg_name(tfm), "sha256") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA256; } else if (strcmp(crypto_ahash_alg_name(tfm), "sha384") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA384; } else if (strcmp(crypto_ahash_alg_name(tfm), "sha512") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA512; } else if (strcmp(crypto_ahash_alg_name(tfm), "sha3-224") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA3_224; } else if (strcmp(crypto_ahash_alg_name(tfm), "sha3-256") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA3_256; } else if (strcmp(crypto_ahash_alg_name(tfm), "sha3-384") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA3_384; } else if (strcmp(crypto_ahash_alg_name(tfm), "sha3-512") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHA3_512; } else if (strcmp(crypto_ahash_alg_name(tfm), "shake128") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHAKE128; } else if (strcmp(crypto_ahash_alg_name(tfm), "shake256") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_SHAKE256; } else if (strcmp(crypto_ahash_alg_name(tfm), "hmac(sha224)") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_HMAC_SHA224; } else if (strcmp(crypto_ahash_alg_name(tfm), "hmac(sha256)") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_HMAC_SHA256; } else if (strcmp(crypto_ahash_alg_name(tfm), "hmac(sha384)") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_HMAC_SHA384; } else if (strcmp(crypto_ahash_alg_name(tfm), "hmac(sha512)") == 0) { sha_ctx->op_mode = SE_AES_OP_MODE_HMAC_SHA512; } else { dev_err(se_dev->dev, "Invalid SHA digest size\n"); return -EINVAL; } /* If the request length is zero, */ if (!req->nbytes) { if (sha_ctx->total_count) { req->base.complete(&req->base, 0); return 0; /* allow empty packets */ } else { /* total_count equals zero */ if (is_last) { /* If this is the last packet SW WAR for zero * length SHA operation since SE HW can't accept * zero length SHA operation. */ mode = sha_ctx->op_mode - SE_AES_OP_MODE_SHA1; if (mode < ARRAY_SIZE(zero_vec)) memcpy(req->result, zero_vec[mode].digest, zero_vec[mode].size); req->base.complete(&req->base, 0); return 0; } else { req->base.complete(&req->base, 0); return 0; /* allow empty first packet */ } } } sha_ctx->is_final = false; if (is_last) sha_ctx->is_final = true; ret = tegra_se_sha_process_buf(req, is_last, process_cur_req); if (ret) { mutex_unlock(&se_dev->mtx); return ret; } return 0; } static int tegra_se_sha_init(struct ahash_request *req) { struct tegra_se_req_context *req_ctx; struct crypto_ahash *tfm = NULL; struct tegra_se_sha_context *sha_ctx; struct tegra_se_dev *se_dev = se_devices[SE_SHA]; pr_debug("%s:%d start\n", __func__, __LINE__); if (!req) { dev_err(se_dev->dev, "SHA request not valid\n"); return -EINVAL; } tfm = crypto_ahash_reqtfm(req); if (!tfm) { dev_err(se_dev->dev, "SHA transform not valid\n"); return -EINVAL; } sha_ctx = crypto_ahash_ctx(tfm); if (!sha_ctx) { dev_err(se_dev->dev, "SHA context not valid\n"); return -EINVAL; } req_ctx = ahash_request_ctx(req); if (!req_ctx) { dev_err(se_dev->dev, "Request context not valid\n"); return -EINVAL; } mutex_lock(&se_dev->mtx); sha_ctx->total_count = 0; sha_ctx->is_first = true; sha_ctx->blk_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); sha_ctx->residual_bytes = 0; mutex_unlock(&se_dev->mtx); pr_debug("%s:%d end\n", __func__, __LINE__); return 0; } static int tegra_se_sha_update(struct ahash_request *req) { struct tegra_se_dev *se_dev = se_devices[SE_SHA]; int ret = 0; pr_debug("%s:%d start\n", __func__, __LINE__); if (!req) { dev_err(se_dev->dev, "SHA request not valid\n"); return -EINVAL; } mutex_lock(&se_dev->mtx); se_dev->sha_req = req; ret = tegra_se_sha_op(req, false, false); if (ret) dev_err(se_dev->dev, "tegra_se_sha_update failed - %d\n", ret); else ret = -EBUSY; mutex_unlock(&se_dev->mtx); pr_debug("%s:%d end\n", __func__, __LINE__); return ret; } static int tegra_se_sha_finup(struct ahash_request *req) { struct tegra_se_dev *se_dev = se_devices[SE_SHA]; int ret = 0; pr_debug("%s:%d start\n", __func__, __LINE__); if (!req) { dev_err(se_dev->dev, "SHA request not valid\n"); return -EINVAL; } mutex_lock(&se_dev->mtx); se_dev->sha_req = req; ret = tegra_se_sha_op(req, true, true); if (ret) dev_err(se_dev->dev, "tegra_se_sha_finup failed - %d\n", ret); else ret = -EBUSY; mutex_unlock(&se_dev->mtx); pr_debug("%s:%d end\n", __func__, __LINE__); return ret; } static int tegra_se_sha_final(struct ahash_request *req) { struct tegra_se_dev *se_dev = se_devices[SE_SHA]; int ret = 0; pr_debug("%s:%d start\n", __func__, __LINE__); if (!req) { dev_err(se_dev->dev, "SHA request not valid\n"); return -EINVAL; } mutex_lock(&se_dev->mtx); se_dev->sha_req = req; /* Do not process data in given request */ ret = tegra_se_sha_op(req, true, false); if (ret) dev_err(se_dev->dev, "tegra_se_sha_final failed - %d\n", ret); else ret = -EBUSY; mutex_unlock(&se_dev->mtx); pr_debug("%s:%d end\n", __func__, __LINE__); return ret; } static int tegra_se_sha_digest(struct ahash_request *req) { struct tegra_se_dev *se_dev = se_devices[SE_SHA]; int ret = 0; pr_debug("%s:%d start\n", __func__, __LINE__); ret = tegra_se_sha_init(req); if (ret) return ret; mutex_lock(&se_dev->mtx); se_dev->sha_req = req; ret = tegra_se_sha_op(req, true, true); if (ret) dev_err(se_dev->dev, "tegra_se_sha_digest failed - %d\n", ret); else ret = -EBUSY; mutex_unlock(&se_dev->mtx); pr_debug("%s:%d end\n", __func__, __LINE__); return ret; } static int tegra_se_sha_export(struct ahash_request *req, void *out) { return 0; } static int tegra_se_sha_import(struct ahash_request *req, const void *in) { return 0; } static int tegra_se_sha_cra_init(struct crypto_tfm *tfm) { struct tegra_se_sha_context *sha_ctx; struct tegra_se_dev *se_dev = se_devices[SE_SHA]; crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct tegra_se_sha_context)); sha_ctx = crypto_tfm_ctx(tfm); if (!sha_ctx) { dev_err(se_dev->dev, "SHA context not valid\n"); return -EINVAL; } mutex_lock(&se_dev->mtx); sha_ctx->sha_buf[0] = dma_alloc_coherent( se_dev->dev, (TEGRA_SE_SHA_MAX_BLOCK_SIZE * 2), &sha_ctx->sha_buf_addr[0], GFP_KERNEL); if (!sha_ctx->sha_buf[0]) { dev_err(se_dev->dev, "Cannot allocate memory to sha_buf[0]\n"); mutex_unlock(&se_dev->mtx); return -ENOMEM; } sha_ctx->sha_buf[1] = dma_alloc_coherent( se_dev->dev, (TEGRA_SE_SHA_MAX_BLOCK_SIZE * 2), &sha_ctx->sha_buf_addr[1], GFP_KERNEL); if (!sha_ctx->sha_buf[1]) { dma_free_coherent( se_dev->dev, (TEGRA_SE_SHA_MAX_BLOCK_SIZE * 2), sha_ctx->sha_buf[0], sha_ctx->sha_buf_addr[0]); sha_ctx->sha_buf[0] = NULL; dev_err(se_dev->dev, "Cannot allocate memory to sha_buf[1]\n"); mutex_unlock(&se_dev->mtx); return -ENOMEM; } mutex_unlock(&se_dev->mtx); return 0; } static void tegra_se_sha_cra_exit(struct crypto_tfm *tfm) { struct tegra_se_sha_context *sha_ctx = crypto_tfm_ctx(tfm); struct tegra_se_dev *se_dev = se_devices[SE_SHA]; int i; mutex_lock(&se_dev->mtx); for (i = 0; i < 2; i++) { /* dma_free_coherent does not panic if addr is NULL */ dma_free_coherent( se_dev->dev, (TEGRA_SE_SHA_MAX_BLOCK_SIZE * 2), sha_ctx->sha_buf[i], sha_ctx->sha_buf_addr[i]); sha_ctx->sha_buf[i] = NULL; } mutex_unlock(&se_dev->mtx); } static int tegra_se_aes_cmac_export(struct ahash_request *req, void *out) { return 0; } static int tegra_se_aes_cmac_import(struct ahash_request *req, const void *in) { return 0; } static int tegra_t23x_se_aes_cmac_op(struct ahash_request *req, bool process_cur_req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_se_aes_cmac_context *cmac_ctx = crypto_ahash_ctx(tfm); struct tegra_se_req_context *req_ctx = ahash_request_ctx(req); struct tegra_se_dev *se_dev; struct tegra_se_ll *src_ll; u32 num_sgs, total_bytes = 0; struct tegra_se_cmdbuf *cmdbuf; int ret = 0; se_dev = se_devices[SE_CMAC]; mutex_lock(&se_dev->mtx); /* * SE doesn't support CMAC input where message length is 0 bytes. * See Bug 200472457. */ if (!(cmac_ctx->nbytes + req->nbytes)) { ret = -EINVAL; goto out; } if (se_dev->cmdbuf_cnt) { cmdbuf = cmac_ctx->cmdbuf; } else { cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K * SE_MAX_SUBMIT_CHAIN_SZ); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto out; } } if (process_cur_req) { if ((cmac_ctx->nbytes + req->nbytes) > (TEGRA_SE_AES_BLOCK_SIZE * 20)) { dev_err(se_dev->dev, "num of SG buffers bytes are more\n"); mutex_unlock(&se_dev->mtx); return -EOPNOTSUPP; } num_sgs = tegra_se_count_sgs(req->src, req->nbytes); if (num_sgs > SE_MAX_SRC_SG_COUNT) { dev_err(se_dev->dev, "num of SG buffers are more\n"); ret = -EDOM; goto out; } sg_copy_to_buffer(req->src, num_sgs, cmac_ctx->buf + cmac_ctx->nbytes, req->nbytes); cmac_ctx->nbytes += req->nbytes; } se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); src_ll = se_dev->src_ll; src_ll->addr = cmac_ctx->buf_dma_addr; src_ll->data_len = cmac_ctx->nbytes; src_ll++; total_bytes = cmac_ctx->nbytes; req_ctx->op_mode = SE_AES_OP_MODE_CMAC; req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, true, cmac_ctx->keylen); req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, true, cmac_ctx->slot->slot_num, 0, true); tegra_se_send_data(se_dev, req_ctx, NULL, total_bytes, se_dev->opcode_addr, cmdbuf->addr); ret = tegra_se_channel_submit_gather( se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); if (ret) goto out; ret = tegra_se_read_cmac_result(se_dev, req->result, TEGRA_SE_AES_CMAC_DIGEST_SIZE, false); if (ret) { dev_err(se_dev->dev, "failed to read cmac result\n"); goto out; } ret = tegra_se_clear_cmac_result(se_dev, TEGRA_SE_AES_CMAC_DIGEST_SIZE); if (ret) { dev_err(se_dev->dev, "failed to clear cmac result\n"); goto out; } out: mutex_unlock(&se_dev->mtx); return ret; } static int tegra_se_aes_cmac_op(struct ahash_request *req, bool process_cur_req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_se_aes_cmac_context *cmac_ctx = crypto_ahash_ctx(tfm); struct tegra_se_req_context *req_ctx = ahash_request_ctx(req); struct tegra_se_dev *se_dev; struct tegra_se_ll *src_ll; struct tegra_se_cmdbuf *cmdbuf; u32 num_sgs, blocks_to_process, last_block_bytes = 0, offset = 0; u8 piv[TEGRA_SE_AES_IV_SIZE]; unsigned int total; int ret = 0, i = 0; bool padding_needed = false; bool use_orig_iv = true; se_dev = se_devices[SE_CMAC]; if (se_dev->chipdata->kac_type == SE_KAC_T23X) return tegra_t23x_se_aes_cmac_op(req, process_cur_req); mutex_lock(&se_dev->mtx); cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K * SE_MAX_SUBMIT_CHAIN_SZ); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto out; } if (process_cur_req) { if ((cmac_ctx->nbytes + req->nbytes) > (TEGRA_SE_AES_BLOCK_SIZE * 20)) { dev_err(se_dev->dev, "num of SG buffers bytes are more\n"); mutex_unlock(&se_dev->mtx); ret = -EOPNOTSUPP; goto out; } num_sgs = tegra_se_count_sgs(req->src, req->nbytes); if (num_sgs > SE_MAX_SRC_SG_COUNT) { dev_err(se_dev->dev, "num of SG buffers are more\n"); ret = -EDOM; goto out; } sg_copy_to_buffer(req->src, num_sgs, cmac_ctx->buf + cmac_ctx->nbytes, req->nbytes); cmac_ctx->nbytes += req->nbytes; } req_ctx->op_mode = SE_AES_OP_MODE_CMAC; blocks_to_process = cmac_ctx->nbytes / TEGRA_SE_AES_BLOCK_SIZE; /* num of bytes less than block size */ if ((cmac_ctx->nbytes % TEGRA_SE_AES_BLOCK_SIZE) || !blocks_to_process) { padding_needed = true; } else { /* decrement num of blocks */ blocks_to_process--; } /* first process all blocks except last block */ if (blocks_to_process) { total = blocks_to_process * TEGRA_SE_AES_BLOCK_SIZE; se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); src_ll = se_dev->src_ll; src_ll->addr = cmac_ctx->buf_dma_addr; src_ll->data_len = total; src_ll++; req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, true, cmac_ctx->keylen); /* write zero IV */ memset(piv, 0, TEGRA_SE_AES_IV_SIZE); ret = tegra_se_send_key_data( se_dev, piv, TEGRA_SE_AES_IV_SIZE, cmac_ctx->slot->slot_num, SE_KEY_TABLE_TYPE_ORGIV, se_dev->opcode_addr, cmdbuf, NONE); if (ret) goto out; req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, true, cmac_ctx->slot->slot_num, 0, true); tegra_se_send_data(se_dev, req_ctx, NULL, total, se_dev->opcode_addr, cmdbuf->addr); ret = tegra_se_channel_submit_gather( se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); if (ret) goto out; ret = tegra_se_read_cmac_result(se_dev, piv, TEGRA_SE_AES_CMAC_DIGEST_SIZE, false); if (ret) { dev_err(se_dev->dev, "failed to read cmac result\n"); goto out; } use_orig_iv = false; } /* process last block */ offset = blocks_to_process * TEGRA_SE_AES_BLOCK_SIZE; if (padding_needed) { /* pad with 0x80, 0, 0 ... */ last_block_bytes = cmac_ctx->nbytes % TEGRA_SE_AES_BLOCK_SIZE; cmac_ctx->buf[cmac_ctx->nbytes] = 0x80; for (i = last_block_bytes + 1; i < TEGRA_SE_AES_BLOCK_SIZE; i++) cmac_ctx->buf[offset + i] = 0; /* XOR with K2 */ for (i = 0; i < TEGRA_SE_AES_BLOCK_SIZE; i++) cmac_ctx->buf[offset + i] ^= cmac_ctx->K2[i]; } else { /* XOR with K1 */ for (i = 0; i < TEGRA_SE_AES_BLOCK_SIZE; i++) cmac_ctx->buf[offset + i] ^= cmac_ctx->K1[i]; } se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); se_dev->src_ll->addr = cmac_ctx->buf_dma_addr + offset; se_dev->src_ll->data_len = TEGRA_SE_AES_BLOCK_SIZE; if (use_orig_iv) { /* use zero IV, this is when num of bytes is * less <= block size */ memset(piv, 0, TEGRA_SE_AES_IV_SIZE); ret = tegra_se_send_key_data( se_dev, piv, TEGRA_SE_AES_IV_SIZE, cmac_ctx->slot->slot_num, SE_KEY_TABLE_TYPE_ORGIV, se_dev->opcode_addr, cmdbuf, NONE); } else { ret = tegra_se_send_key_data( se_dev, piv, TEGRA_SE_AES_IV_SIZE, cmac_ctx->slot->slot_num, SE_KEY_TABLE_TYPE_UPDTDIV, se_dev->opcode_addr, cmdbuf, NONE); } if (ret) goto out; req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, true, cmac_ctx->keylen); req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, true, cmac_ctx->slot->slot_num, 0, use_orig_iv); tegra_se_send_data(se_dev, req_ctx, NULL, TEGRA_SE_AES_BLOCK_SIZE, se_dev->opcode_addr, cmdbuf->addr); ret = tegra_se_channel_submit_gather( se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); if (ret) goto out; ret = tegra_se_read_cmac_result(se_dev, req->result, TEGRA_SE_AES_CMAC_DIGEST_SIZE, false); if (ret) { dev_err(se_dev->dev, "failed to read cmac result\n"); goto out; } cmac_ctx->nbytes = 0; out: mutex_unlock(&se_dev->mtx); return ret; } static int tegra_se_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct tegra_se_aes_cmac_context *ctx = crypto_ahash_ctx(tfm); struct tegra_se_req_context *req_ctx = NULL; struct tegra_se_dev *se_dev; struct tegra_se_slot *pslot; struct tegra_se_cmdbuf *cmdbuf; u8 piv[TEGRA_SE_AES_IV_SIZE]; u32 *pbuf; dma_addr_t pbuf_adr; int ret = 0; u8 const rb = 0x87; u8 msb; se_dev = se_devices[SE_CMAC]; mutex_lock(&se_dev->mtx); if (!ctx) { dev_err(se_dev->dev, "invalid context"); mutex_unlock(&se_dev->mtx); return -EINVAL; } req_ctx = devm_kzalloc(se_dev->dev, sizeof(struct tegra_se_req_context), GFP_KERNEL); if (!req_ctx) { mutex_unlock(&se_dev->mtx); return -ENOMEM; } if ((keylen != TEGRA_SE_KEY_128_SIZE) && (keylen != TEGRA_SE_KEY_192_SIZE) && (keylen != TEGRA_SE_KEY_256_SIZE)) { dev_err(se_dev->dev, "invalid key size"); ret = -EINVAL; goto free_ctx; } cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K * SE_MAX_SUBMIT_CHAIN_SZ); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto free_ctx; } ctx->cmdbuf = cmdbuf; if (key) { if (!ctx->slot || (ctx->slot && ctx->slot->slot_num == ssk_slot.slot_num)) { pslot = tegra_se_alloc_key_slot(); if (!pslot) { dev_err(se_dev->dev, "no free key slot\n"); ret = -ENOMEM; goto free_ctx; } ctx->slot = pslot; } ctx->keylen = keylen; } else { tegra_se_free_key_slot(ctx->slot); ctx->slot = &ssk_slot; ctx->keylen = AES_KEYSIZE_128; } pbuf = dma_alloc_coherent(se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, &pbuf_adr, GFP_KERNEL); if (!pbuf) { dev_err(se_dev->dev, "can not allocate dma buffer"); ret = -ENOMEM; goto keyslt_free; } memset(pbuf, 0, TEGRA_SE_AES_BLOCK_SIZE); se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); se_dev->dst_ll = (struct tegra_se_ll *)(se_dev->dst_ll_buf); se_dev->src_ll->addr = pbuf_adr; se_dev->src_ll->data_len = TEGRA_SE_AES_BLOCK_SIZE; se_dev->dst_ll->addr = pbuf_adr; se_dev->dst_ll->data_len = TEGRA_SE_AES_BLOCK_SIZE; /* load the key */ if (se_dev->chipdata->kac_type == SE_KAC_T23X) { ret = tegra_se_send_key_data( se_dev, (u8 *)key, keylen, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_CMAC, se_dev->opcode_addr, cmdbuf, NONE); } else { ret = tegra_se_send_key_data( se_dev, (u8 *)key, keylen, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_KEY, se_dev->opcode_addr, cmdbuf, NONE); } if (ret) { dev_err(se_dev->dev, "tegra_se_send_key_data for loading cmac key failed\n"); goto out; } /* write zero IV */ memset(piv, 0, TEGRA_SE_AES_IV_SIZE); /* load IV */ ret = tegra_se_send_key_data( se_dev, piv, TEGRA_SE_AES_IV_SIZE, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_ORGIV, se_dev->opcode_addr, cmdbuf, NONE); if (ret) { dev_err(se_dev->dev, "tegra_se_send_key_data for loading cmac iv failed\n"); goto out; } if (se_dev->chipdata->kac_type == SE_KAC_T23X) { ret = tegra_se_clear_cmac_result(se_dev, TEGRA_SE_AES_CMAC_DIGEST_SIZE); if (ret) { dev_err(se_dev->dev, "failed to clear cmac result\n"); goto out; } } else { /* config crypto algo */ req_ctx->config = tegra_se_get_config(se_dev, SE_AES_OP_MODE_CBC, true, keylen); req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, SE_AES_OP_MODE_CBC, true, ctx->slot->slot_num, 0, true); tegra_se_send_data(se_dev, req_ctx, NULL, TEGRA_SE_AES_BLOCK_SIZE, se_dev->opcode_addr, cmdbuf->addr); ret = tegra_se_channel_submit_gather( se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); if (ret) { dev_err(se_dev->dev, "tegra_se_aes_cmac_setkey:: start op failed\n"); goto out; } /* compute K1 subkey */ memcpy(ctx->K1, pbuf, TEGRA_SE_AES_BLOCK_SIZE); tegra_se_leftshift_onebit(ctx->K1, TEGRA_SE_AES_BLOCK_SIZE, &msb); if (msb) ctx->K1[TEGRA_SE_AES_BLOCK_SIZE - 1] ^= rb; /* compute K2 subkey */ memcpy(ctx->K2, ctx->K1, TEGRA_SE_AES_BLOCK_SIZE); tegra_se_leftshift_onebit(ctx->K2, TEGRA_SE_AES_BLOCK_SIZE, &msb); if (msb) ctx->K2[TEGRA_SE_AES_BLOCK_SIZE - 1] ^= rb; } out: if (pbuf) dma_free_coherent(se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, pbuf, pbuf_adr); keyslt_free: if (ret) tegra_se_free_key_slot(ctx->slot); free_ctx: devm_kfree(se_dev->dev, req_ctx); mutex_unlock(&se_dev->mtx); return ret; } static int tegra_se_aes_cmac_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_se_aes_cmac_context *cmac_ctx = crypto_ahash_ctx(tfm); cmac_ctx->nbytes = 0; return 0; } static int tegra_se_aes_cmac_update(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_se_aes_cmac_context *cmac_ctx = crypto_ahash_ctx(tfm); struct tegra_se_dev *se_dev; u32 num_sgs; se_dev = se_devices[SE_CMAC]; if ((cmac_ctx->nbytes + req->nbytes) > (TEGRA_SE_AES_BLOCK_SIZE * 20)) { dev_err(se_dev->dev, "num of SG buffers bytes are more\n"); return -EOPNOTSUPP; } num_sgs = tegra_se_count_sgs(req->src, req->nbytes); sg_copy_to_buffer(req->src, num_sgs, cmac_ctx->buf + cmac_ctx->nbytes, req->nbytes); cmac_ctx->nbytes += req->nbytes; return 0; } static int tegra_se_aes_cmac_digest(struct ahash_request *req) { return tegra_se_aes_cmac_init(req) ?: tegra_se_aes_cmac_op(req, true); } static int tegra_se_aes_cmac_final(struct ahash_request *req) { return tegra_se_aes_cmac_op(req, false); } static int tegra_se_aes_cmac_finup(struct ahash_request *req) { return tegra_se_aes_cmac_op(req, true); } static int tegra_se_aes_cmac_cra_init(struct crypto_tfm *tfm) { struct tegra_se_aes_cmac_context *cmac_ctx; struct tegra_se_dev *se_dev = se_devices[SE_CMAC]; crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct tegra_se_aes_cmac_context)); cmac_ctx = crypto_tfm_ctx(tfm); if (!cmac_ctx) { dev_err(se_dev->dev, "CMAC context not valid\n"); return -EINVAL; } mutex_lock(&se_dev->mtx); cmac_ctx->buf = dma_alloc_coherent(se_dev->dev, (TEGRA_SE_AES_BLOCK_SIZE * 20), &cmac_ctx->buf_dma_addr, GFP_KERNEL); if (!cmac_ctx->buf) { dev_err(se_dev->dev, "Cannot allocate memory to buf\n"); mutex_unlock(&se_dev->mtx); return -ENOMEM; } cmac_ctx->nbytes = 0; mutex_unlock(&se_dev->mtx); return 0; } static void tegra_se_aes_cmac_cra_exit(struct crypto_tfm *tfm) { struct tegra_se_aes_cmac_context *cmac_ctx = crypto_tfm_ctx(tfm); struct tegra_se_dev *se_dev = se_devices[SE_CMAC]; tegra_se_free_key_slot(cmac_ctx->slot); cmac_ctx->slot = NULL; mutex_lock(&se_dev->mtx); dma_free_coherent(se_dev->dev, (TEGRA_SE_AES_BLOCK_SIZE * 20), cmac_ctx->buf, cmac_ctx->buf_dma_addr); mutex_unlock(&se_dev->mtx); } static int tegra_se_sha_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct tegra_se_sha_context *ctx = crypto_ahash_ctx(tfm); struct tegra_se_req_context *req_ctx = NULL; struct tegra_se_dev *se_dev; struct tegra_se_cmdbuf *cmdbuf; dma_addr_t iova; u32 *cpuvaddr; u8 _key[TEGRA_SE_KEY_256_SIZE] = { 0 }; int ret = 0; se_dev = se_devices[SE_SHA]; mutex_lock(&se_dev->mtx); req_ctx = devm_kzalloc(se_dev->dev, sizeof(struct tegra_se_req_context), GFP_KERNEL); if (!req_ctx) { ret = -ENOMEM; goto out_mutex; } if (keylen > TEGRA_SE_KEY_256_SIZE) { dev_err(se_dev->dev, "invalid key size"); ret = -EINVAL; goto free_req_ctx; } /* If keylen < TEGRA_SE_KEY_256_SIZE. Appened 0's to the end of key. * This will allow the key to be stored in the keyslot. * Since, TEGRA_SE_KEY_256_SIZE < 512-bits which is the minimum sha * block size. */ memcpy(_key, key, keylen); if (!key) { ret = -EINVAL; goto free_req_ctx; } if (!ctx->slot) { ctx->slot = tegra_se_alloc_key_slot(); if (!ctx->slot) { dev_err(se_dev->dev, "no free key slot\n"); ret = -ENOMEM; goto free_req_ctx; } } ctx->keylen = keylen; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto free_req_ctx; } cpuvaddr = cmdbuf->addr; iova = cmdbuf->iova; ret = tegra_se_send_key_data(se_dev, _key, TEGRA_SE_KEY_256_SIZE, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_HMAC, se_dev->opcode_addr, cmdbuf, NONE); if (ret) dev_err(se_dev->dev, "tegra_se_send_key_data for loading HMAC key failed\n"); if (ret) tegra_se_free_key_slot(ctx->slot); free_req_ctx: devm_kfree(se_dev->dev, req_ctx); out_mutex: mutex_unlock(&se_dev->mtx); return ret; } /* Security Engine rsa key slot */ struct tegra_se_rsa_slot { struct list_head node; u8 slot_num; /* Key slot number */ bool available; /* Tells whether key slot is free to use */ }; /* Security Engine AES RSA context */ struct tegra_se_aes_rsa_context { struct tegra_se_dev *se_dev; /* Security Engine device */ struct tegra_se_rsa_slot *slot; /* Security Engine rsa key slot */ u32 mod_len; u32 exp_len; }; static void tegra_se_rsa_free_key_slot(struct tegra_se_rsa_slot *slot) { if (slot) { spin_lock(&rsa_key_slot_lock); slot->available = true; spin_unlock(&rsa_key_slot_lock); } } static struct tegra_se_rsa_slot *tegra_se_alloc_rsa_key_slot(void) { struct tegra_se_rsa_slot *slot = NULL; bool found = false; spin_lock(&rsa_key_slot_lock); list_for_each_entry(slot, &rsa_key_slot, node) { if (slot->available) { slot->available = false; found = true; break; } } spin_unlock(&rsa_key_slot_lock); return found ? slot : NULL; } static int tegra_init_rsa_key_slot(struct tegra_se_dev *se_dev) { int i; se_dev->rsa_slot_list = devm_kzalloc( se_dev->dev, sizeof(struct tegra_se_rsa_slot) * TEGRA_SE_RSA_KEYSLOT_COUNT, GFP_KERNEL); if (!se_dev->rsa_slot_list) return -ENOMEM; spin_lock_init(&rsa_key_slot_lock); spin_lock(&rsa_key_slot_lock); for (i = 0; i < TEGRA_SE_RSA_KEYSLOT_COUNT; i++) { se_dev->rsa_slot_list[i].available = true; se_dev->rsa_slot_list[i].slot_num = i; INIT_LIST_HEAD(&se_dev->rsa_slot_list[i].node); list_add_tail(&se_dev->rsa_slot_list[i].node, &rsa_key_slot); } spin_unlock(&rsa_key_slot_lock); return 0; } #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) static unsigned int tegra_se_rsa_max_size(struct crypto_akcipher *tfm) #else static int tegra_se_rsa_max_size(struct crypto_akcipher *tfm) #endif { struct tegra_se_aes_rsa_context *ctx = akcipher_tfm_ctx(tfm); if (!ctx) { pr_err("No RSA context\n"); return -EINVAL; } return ctx->mod_len; } static int tegra_se_send_rsa_data(struct tegra_se_dev *se_dev, struct tegra_se_aes_rsa_context *rsa_ctx) { struct tegra_se_cmdbuf *cmdbuf; u32 cmdbuf_num_words = 0, i = 0; int err = 0; u32 val = 0; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_4K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); return PTR_ERR(cmdbuf); } cmdbuf->size = SZ_4K; cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_OPERATION_OFFSET, 1); cmdbuf->addr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE); val = SE_CONFIG_ENC_ALG(ALG_RSA) | SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_DST(DST_MEMORY); cmdbuf->addr[i++] = __nvhost_opcode_incr(se_dev->opcode_addr, 8); cmdbuf->addr[i++] = val; cmdbuf->addr[i++] = RSA_KEY_SLOT(rsa_ctx->slot->slot_num); cmdbuf->addr[i++] = (rsa_ctx->mod_len / 64) - 1; cmdbuf->addr[i++] = (rsa_ctx->exp_len / 4); cmdbuf->addr[i++] = (u32)(se_dev->src_ll->addr); cmdbuf->addr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(se_dev->src_ll->addr)) | SE_ADDR_HI_SZ(se_dev->src_ll->data_len)); cmdbuf->addr[i++] = (u32)(se_dev->dst_ll->addr); cmdbuf->addr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(se_dev->dst_ll->addr)) | SE_ADDR_HI_SZ(se_dev->dst_ll->data_len)); cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_OPERATION_OFFSET, 1); cmdbuf->addr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE) | SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(OP_START); cmdbuf_num_words = i; err = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, cmdbuf_num_words, NONE); return err; } static int tegra_se_rsa_setkey(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { struct tegra_se_aes_rsa_context *ctx = akcipher_tfm_ctx(tfm); struct tegra_se_dev *se_dev; struct tegra_se_cmdbuf *cmdbuf; u32 module_key_length = 0; u32 exponent_key_length = 0; u32 pkt, val; u32 key_size_words; u32 key_word_size = 4; u32 *pkeydata = (u32 *)key; s32 j = 0; struct tegra_se_rsa_slot *pslot; u32 cmdbuf_num_words = 0, i = 0; int err = 0; int timeout = 0; se_dev = se_devices[SE_RSA]; if (!ctx || !key) { dev_err(se_dev->dev, "No RSA context or Key\n"); return -EINVAL; } /* Allocate rsa key slot */ if (!ctx->slot) { for (timeout = 0; timeout < SE_KEYSLOT_TIMEOUT; timeout++) { pslot = tegra_se_alloc_rsa_key_slot(); if (!pslot) { mdelay(SE_KEYSLOT_MDELAY); continue; } else { break; } } if (!pslot) { dev_err(se_dev->dev, "no free key slot\n"); return -ENOMEM; } else { ctx->slot = pslot; } } module_key_length = (keylen >> 16); exponent_key_length = (keylen & (0xFFFF)); if (!(((module_key_length / 64) >= 1) && ((module_key_length / 64) <= 4))) { tegra_se_rsa_free_key_slot(ctx->slot); dev_err(se_dev->dev, "Invalid RSA modulus length\n"); return -EDOM; } ctx->mod_len = module_key_length; ctx->exp_len = exponent_key_length; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_64K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); return PTR_ERR(cmdbuf); } cmdbuf->size = SZ_64K; cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_OPERATION_OFFSET, 1); cmdbuf->addr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE); if (exponent_key_length) { key_size_words = (exponent_key_length / key_word_size); /* Write exponent */ for (j = (key_size_words - 1); j >= 0; j--) { pkt = RSA_KEY_NUM(ctx->slot->slot_num) | RSA_KEY_TYPE(RSA_KEY_TYPE_EXP) | RSA_KEY_PKT_WORD_ADDR(j); val = SE_RSA_KEYTABLE_PKT(pkt); cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_KEYTABLE_ADDR_OFFSET, 1); cmdbuf->addr[i++] = val; cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_KEYTABLE_DATA_OFFSET, 1); cmdbuf->addr[i++] = *pkeydata++; } } if (module_key_length) { key_size_words = (module_key_length / key_word_size); /* Write modulus */ for (j = (key_size_words - 1); j >= 0; j--) { pkt = RSA_KEY_NUM(ctx->slot->slot_num) | RSA_KEY_TYPE(RSA_KEY_TYPE_MOD) | RSA_KEY_PKT_WORD_ADDR(j); val = SE_RSA_KEYTABLE_PKT(pkt); cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_KEYTABLE_ADDR_OFFSET, 1); cmdbuf->addr[i++] = val; cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_KEYTABLE_DATA_OFFSET, 1); cmdbuf->addr[i++] = *pkeydata++; } } cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_OPERATION_OFFSET, 1); cmdbuf->addr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE) | SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(OP_DUMMY); cmdbuf_num_words = i; mutex_lock(&se_dev->mtx); err = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, cmdbuf_num_words, NONE); mutex_unlock(&se_dev->mtx); if (err) tegra_se_rsa_free_key_slot(ctx->slot); return err; } static int tegra_se_rsa_op(struct akcipher_request *req) { struct crypto_akcipher *tfm = NULL; struct tegra_se_aes_rsa_context *rsa_ctx = NULL; struct tegra_se_dev *se_dev; u32 num_src_sgs, num_dst_sgs; int ret1 = 0, ret2 = 0; se_dev = se_devices[SE_RSA]; if (!req) { dev_err(se_dev->dev, "Invalid RSA request\n"); return -EINVAL; } tfm = crypto_akcipher_reqtfm(req); if (!tfm) { dev_err(se_dev->dev, "Invalid RSA transform\n"); return -EINVAL; } rsa_ctx = akcipher_tfm_ctx(tfm); if (!rsa_ctx || !rsa_ctx->slot) { dev_err(se_dev->dev, "Invalid RSA context\n"); return -EINVAL; } if ((req->src_len < TEGRA_SE_RSA512_INPUT_SIZE) || (req->src_len > TEGRA_SE_RSA2048_INPUT_SIZE)) { dev_err(se_dev->dev, "RSA src input length not in range\n"); return -EDOM; } if ((req->dst_len < TEGRA_SE_RSA512_INPUT_SIZE) || (req->dst_len > TEGRA_SE_RSA2048_INPUT_SIZE)) { dev_err(se_dev->dev, "RSA dst input length not in range\n"); return -EDOM; } if (req->src_len != rsa_ctx->mod_len) { dev_err(se_dev->dev, "Invalid RSA src input length\n"); return -EINVAL; } num_src_sgs = tegra_se_count_sgs(req->src, req->src_len); num_dst_sgs = tegra_se_count_sgs(req->dst, req->dst_len); if ((num_src_sgs > SE_MAX_SRC_SG_COUNT) || (num_dst_sgs > SE_MAX_DST_SG_COUNT)) { dev_err(se_dev->dev, "num of SG buffers are more\n"); return -EDOM; } mutex_lock(&se_dev->mtx); se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); if (req->src == req->dst) { se_dev->dst_ll = se_dev->src_ll; ret1 = tegra_map_sg(se_dev->dev, req->src, 1, DMA_BIDIRECTIONAL, se_dev->src_ll, req->src_len); if (!ret1) { mutex_unlock(&se_dev->mtx); return -EINVAL; } } else { se_dev->dst_ll = (struct tegra_se_ll *)(se_dev->dst_ll_buf); ret1 = tegra_map_sg(se_dev->dev, req->src, 1, DMA_TO_DEVICE, se_dev->src_ll, req->src_len); ret2 = tegra_map_sg(se_dev->dev, req->dst, 1, DMA_FROM_DEVICE, se_dev->dst_ll, req->dst_len); if (!ret1 || !ret2) { mutex_unlock(&se_dev->mtx); return -EINVAL; } } ret1 = tegra_se_send_rsa_data(se_dev, rsa_ctx); if (ret1) dev_err(se_dev->dev, "RSA send data failed err = %d\n", ret1); if (req->src == req->dst) { tegra_unmap_sg(se_dev->dev, req->src, DMA_BIDIRECTIONAL, req->src_len); } else { tegra_unmap_sg(se_dev->dev, req->src, DMA_TO_DEVICE, req->src_len); tegra_unmap_sg(se_dev->dev, req->dst, DMA_FROM_DEVICE, req->dst_len); } mutex_unlock(&se_dev->mtx); return ret1; } static void tegra_se_rsa_exit(struct crypto_akcipher *tfm) { struct tegra_se_aes_rsa_context *ctx = akcipher_tfm_ctx(tfm); tegra_se_rsa_free_key_slot(ctx->slot); ctx->slot = NULL; } static inline struct tegra_se_dh_context *tegra_se_dh_get_ctx( struct crypto_kpp *tfm) { return kpp_tfm_ctx(tfm); } static int tegra_se_dh_check_params_length(unsigned int p_len) { if (p_len < MIN_DH_SZ_BITS) { pr_err("DH Modulus length not in range\n"); return -EDOM; } return 0; } static int tegra_se_dh_set_params(struct tegra_se_dh_context *ctx, struct dh *params) { int ret = 0; ret = tegra_se_dh_check_params_length(params->p_size << 3); if (ret) return ret; ctx->key = (void *)params->key; ctx->key_size = params->key_size; if (!ctx->key) { dev_err(ctx->se_dev->dev, "Invalid DH Key\n"); return -ENODATA; } ctx->p = (void *)params->p; ctx->p_size = params->p_size; if (!ctx->p) { dev_err(ctx->se_dev->dev, "Invalid DH Modulus\n"); return -ENODATA; } ctx->g = (void *)params->g; ctx->g_size = params->g_size; if (!ctx->g) { dev_err(ctx->se_dev->dev, "Invalid DH generator\n"); return -ENODATA; } if (ctx->g_size > ctx->p_size) { dev_err(ctx->se_dev->dev, "Invalid DH generator size\n"); return -EDOM; } return 0; } static int tegra_se_dh_setkey(struct crypto_kpp *tfm) { struct tegra_se_dh_context *ctx = tegra_se_dh_get_ctx(tfm); struct tegra_se_dev *se_dev; struct tegra_se_cmdbuf *cmdbuf; u32 module_key_length = 0; u32 exponent_key_length = 0; u32 pkt, val; u32 key_size_words; u32 key_word_size = 4; u32 *pkeydata; struct tegra_se_rsa_slot *pslot; u32 cmdbuf_num_words = 0; int i = 0, err, j; if (!ctx) { pr_err("Invalid DH context\n"); return -EINVAL; } se_dev = ctx->se_dev; pkeydata = (u32 *)ctx->key; /* Allocate rsa key slot */ if (!ctx->slot) { pslot = tegra_se_alloc_rsa_key_slot(); if (!pslot) { dev_err(se_dev->dev, "no free key slot\n"); return -ENOMEM; } ctx->slot = pslot; } module_key_length = ctx->p_size; exponent_key_length = ctx->key_size; if (!(((module_key_length / 64) >= 1) && ((module_key_length / 64) <= 4))) { tegra_se_rsa_free_key_slot(ctx->slot); dev_err(se_dev->dev, "DH Modulus length not in range\n"); return -EDOM; } cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_64K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); return PTR_ERR(cmdbuf); } cmdbuf->size = SZ_64K; cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_OPERATION_OFFSET, 1); cmdbuf->addr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE); if (exponent_key_length) { key_size_words = (exponent_key_length / key_word_size); /* Write exponent */ for (j = (key_size_words - 1); j >= 0; j--) { pkt = RSA_KEY_NUM(ctx->slot->slot_num) | RSA_KEY_TYPE(RSA_KEY_TYPE_EXP) | RSA_KEY_PKT_WORD_ADDR(j); val = SE_RSA_KEYTABLE_PKT(pkt); cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_KEYTABLE_ADDR_OFFSET, 1); cmdbuf->addr[i++] = val; cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_KEYTABLE_DATA_OFFSET, 1); cmdbuf->addr[i++] = be32_to_cpu(*pkeydata++); } } if (module_key_length) { pkeydata = (u32 *)ctx->p; key_size_words = (module_key_length / key_word_size); /* Write modulus */ for (j = (key_size_words - 1); j >= 0; j--) { pkt = RSA_KEY_NUM(ctx->slot->slot_num) | RSA_KEY_TYPE(RSA_KEY_TYPE_MOD) | RSA_KEY_PKT_WORD_ADDR(j); val = SE_RSA_KEYTABLE_PKT(pkt); cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_KEYTABLE_ADDR_OFFSET, 1); cmdbuf->addr[i++] = val; cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_KEYTABLE_DATA_OFFSET, 1); cmdbuf->addr[i++] = be32_to_cpu(*pkeydata++); } } cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_OPERATION_OFFSET, 1); cmdbuf->addr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE) | SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(OP_DUMMY); cmdbuf_num_words = i; err = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, cmdbuf_num_words, NONE); if (err) { dev_err(se_dev->dev, "%s: channel_submit failed\n", __func__); tegra_se_rsa_free_key_slot(ctx->slot); } return err; } static void tegra_se_fix_endianness(struct tegra_se_dev *se_dev, struct scatterlist *sg, u32 num_sgs, unsigned int nbytes, bool be) { int j, k; sg_copy_to_buffer(sg, num_sgs, se_dev->dh_buf1, nbytes); for (j = (nbytes / 4 - 1), k = 0; j >= 0; j--, k++) { if (be) se_dev->dh_buf2[k] = be32_to_cpu(se_dev->dh_buf1[j]); else se_dev->dh_buf2[k] = cpu_to_be32(se_dev->dh_buf1[j]); } sg_copy_from_buffer(sg, num_sgs, se_dev->dh_buf2, nbytes); } static int tegra_se_dh_compute_value(struct kpp_request *req) { struct crypto_kpp *tfm = NULL; struct tegra_se_dh_context *dh_ctx = NULL; struct tegra_se_dev *se_dev; struct scatterlist *src_sg; struct tegra_se_cmdbuf *cmdbuf; u32 num_src_sgs, num_dst_sgs; u8 *base_buff = NULL; struct scatterlist src; u32 cmdbuf_num_words = 0, i = 0; int err, j; unsigned int total, zpad_sz; u32 val; if (!req) { pr_err("Invalid DH request\n"); return -EINVAL; } tfm = crypto_kpp_reqtfm(req); if (!tfm) { pr_err("Invalid DH transform\n"); return -EINVAL; } dh_ctx = tegra_se_dh_get_ctx(tfm); if (!dh_ctx || !dh_ctx->slot) { pr_err("Invalid DH context\n"); return -EINVAL; } se_dev = dh_ctx->se_dev; if (req->src) { src_sg = req->src; total = req->src_len; } else { if (dh_ctx->g_size < dh_ctx->p_size) { base_buff = (u8 *)devm_kzalloc(se_dev->dev, dh_ctx->p_size, GFP_KERNEL); if (!base_buff) return -ENOMEM; zpad_sz = dh_ctx->p_size - dh_ctx->g_size; for (j = 0; j < zpad_sz; j++) base_buff[j] = 0x0; for (j = zpad_sz; j < dh_ctx->p_size; j++) base_buff[j] = *(u8 *)(dh_ctx->g++); dh_ctx->g_size = dh_ctx->p_size; } else { base_buff = (u8 *)devm_kzalloc(se_dev->dev, dh_ctx->g_size, GFP_KERNEL); if (!base_buff) return -ENOMEM; memcpy(base_buff, (u8 *)(dh_ctx->g), dh_ctx->g_size); } sg_init_one(&src, base_buff, dh_ctx->g_size); src_sg = &src; total = dh_ctx->g_size; } num_src_sgs = tegra_se_count_sgs(src_sg, total); num_dst_sgs = tegra_se_count_sgs(req->dst, req->dst_len); if ((num_src_sgs > SE_MAX_SRC_SG_COUNT) || (num_dst_sgs > SE_MAX_DST_SG_COUNT)) { dev_err(se_dev->dev, "num of SG buffers are more\n"); err = -EDOM; goto free; } tegra_se_fix_endianness(se_dev, src_sg, num_src_sgs, total, true); se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); se_dev->dst_ll = (struct tegra_se_ll *)(se_dev->dst_ll_buf); err = tegra_map_sg(se_dev->dev, src_sg, 1, DMA_TO_DEVICE, se_dev->src_ll, total); if (!err) { err = -EINVAL; goto free; } err = tegra_map_sg(se_dev->dev, req->dst, 1, DMA_FROM_DEVICE, se_dev->dst_ll, req->dst_len); if (!err) { err = -EINVAL; goto unmap_src; } cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_4K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); err = PTR_ERR(cmdbuf); goto unmap_dst; } cmdbuf->size = SZ_4K; cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_OPERATION_OFFSET, 1); cmdbuf->addr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE); val = SE_CONFIG_ENC_ALG(ALG_RSA) | SE_CONFIG_DEC_ALG(ALG_NOP) | SE_CONFIG_DST(DST_MEMORY); cmdbuf->addr[i++] = __nvhost_opcode_incr(se_dev->opcode_addr, 8); cmdbuf->addr[i++] = val; cmdbuf->addr[i++] = RSA_KEY_SLOT(dh_ctx->slot->slot_num); cmdbuf->addr[i++] = (dh_ctx->p_size / 64) - 1; cmdbuf->addr[i++] = (dh_ctx->key_size / 4); cmdbuf->addr[i++] = (u32)(se_dev->src_ll->addr); cmdbuf->addr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(se_dev->src_ll->addr)) | SE_ADDR_HI_SZ(se_dev->src_ll->data_len)); cmdbuf->addr[i++] = (u32)(se_dev->dst_ll->addr); cmdbuf->addr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(se_dev->dst_ll->addr)) | SE_ADDR_HI_SZ(se_dev->dst_ll->data_len)); cmdbuf->addr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_RSA_OPERATION_OFFSET, 1); cmdbuf->addr[i++] = SE_OPERATION_WRSTALL(WRSTALL_TRUE) | SE_OPERATION_LASTBUF(LASTBUF_TRUE) | SE_OPERATION_OP(OP_START); cmdbuf_num_words = i; err = tegra_se_channel_submit_gather( se_dev, cmdbuf, 0, cmdbuf_num_words, NONE); if (err) { dev_err(se_dev->dev, "%s: channel_submit failed\n", __func__); goto unmap_dst; } tegra_se_fix_endianness(se_dev, req->dst, num_dst_sgs, req->dst_len, false); unmap_dst: tegra_unmap_sg(se_dev->dev, req->dst, DMA_FROM_DEVICE, req->dst_len); unmap_src: tegra_unmap_sg(se_dev->dev, src_sg, DMA_TO_DEVICE, total); free: if (!req->src) devm_kfree(se_dev->dev, base_buff); return err; } #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) static int tegra_se_dh_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) #else static int tegra_se_dh_set_secret(struct crypto_kpp *tfm, void *buf, unsigned int len) #endif { int ret = 0; struct tegra_se_dh_context *ctx = tegra_se_dh_get_ctx(tfm); struct dh params; ctx->se_dev = se_devices[SE_RSA]; ret = crypto_dh_decode_key(buf, len, ¶ms); if (ret) { dev_err(ctx->se_dev->dev, "failed to decode DH input\n"); return ret; } ret = tegra_se_dh_set_params(ctx, ¶ms); if (ret) return ret; ret = tegra_se_dh_setkey(tfm); if (ret) return ret; return 0; } #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) static unsigned int tegra_se_dh_max_size(struct crypto_kpp *tfm) #else static int tegra_se_dh_max_size(struct crypto_kpp *tfm) #endif { struct tegra_se_dh_context *ctx = tegra_se_dh_get_ctx(tfm); return ctx->p_size; } static void tegra_se_dh_exit_tfm(struct crypto_kpp *tfm) { struct tegra_se_dh_context *ctx = tegra_se_dh_get_ctx(tfm); tegra_se_rsa_free_key_slot(ctx->slot); ctx->key = NULL; ctx->p = NULL; ctx->g = NULL; } static int tegra_se_aes_ccm_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_dev *se_dev; struct tegra_se_slot *pslot; struct tegra_se_cmdbuf *cmdbuf; u32 *cpuvaddr = NULL; dma_addr_t iova = 0; int ret = 0; se_dev = se_devices[SE_AEAD]; mutex_lock(&se_dev->mtx); /* Check for valid arguments */ if (!ctx || !key) { dev_err(se_dev->dev, "invalid context or key"); ret = -EINVAL; goto free_ctx; } if ((keylen != TEGRA_SE_KEY_128_SIZE) && (keylen != TEGRA_SE_KEY_192_SIZE) && (keylen != TEGRA_SE_KEY_256_SIZE)) { dev_err(se_dev->dev, "invalid key size"); ret = -EINVAL; goto free_ctx; } /* Get free key slot */ pslot = tegra_se_alloc_key_slot(); if (!pslot) { dev_err(se_dev->dev, "no free key slot\n"); ret = -ENOMEM; goto free_ctx; } ctx->slot = pslot; ctx->keylen = keylen; /* Get free command buffer */ cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto free_ctx; } cpuvaddr = cmdbuf->addr; iova = cmdbuf->iova; ret = tegra_se_send_key_data( se_dev, (u8 *)key, keylen, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_KEY, se_dev->opcode_addr, cmdbuf, NONE); tegra_se_free_key_slot(ctx->slot); free_ctx: mutex_unlock(&se_dev->mtx); return ret; } static int tegra_se_aes_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); switch (authsize) { case 4: case 8: case 10: case 12: case 14: case 16: ctx->authsize = authsize; break; default: return -EINVAL; } return 0; } static inline int tegra_se_ccm_check_iv(const u8 *iv) { /* iv[0] gives value of q-1 * 2 <= q <= 8 as per NIST 800-38C notation * 2 <= L <= 8, so 1 <= L' <= 7. as per rfc 3610 notation */ if (iv[0] < 1 || iv[0] > 7) { pr_err("ccm_ccm_check_iv failed %d\n", iv[0]); return -EINVAL; } return 0; } static int tegra_se_ccm_init_crypt(struct aead_request *req) { u8 *iv = req->iv; int err; err = tegra_se_ccm_check_iv(iv); if (err) return err; /* Note: rfc 3610 and NIST 800-38C require counter (ctr_0) of * zero to encrypt auth tag. * req->iv has the formatted ctr_0 (i.e. Flags || N || 0). */ memset(iv + 15 - iv[0], 0, iv[0] + 1); return 0; } static int ccm_set_msg_len(u8 *block, unsigned int msglen, int csize) { __be32 data; memset(block, 0, csize); block += csize; if (csize >= 4) csize = 4; else if (msglen > (1 << (8 * csize))) return -EOVERFLOW; data = cpu_to_be32(msglen); memcpy(block - csize, (u8 *)&data + 4 - csize, csize); return 0; } /** * @brief Encode B0 block as per below format (16 bytes): * * +----------------------------------------------------+ * | Octet # | 0 | [1 .. (15-q)] | [(16-q) .. 15] | * +----------------------------------------------------+ * | Content | Flags | N | Q | * +----------------------------------------------------+ * * 1. "Flags" octet format (byte 0 of big endian B0): * * +-------------------------------------------------------------------------+ * | Bit no. | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | * +-------------------------------------------------------------------------+ * | Content |reserved| Adata | (t-2)/2 | q-1 | * +-------------------------------------------------------------------------+ * * Bit 7 => always zero * Bit 6 => flag for AAD presence * Bits [5..3] => one of the valid values of element t (mac length) * Bits [2..0] => one of the valid values of element q (bytes needs to encode * message length), already present in iv[0]. * * 2. N - nounce present in req->iv. * * 3. Q - contains the msg length(in bytes) in q octet/bytes space. * */ static int ccm_encode_b0(u8 *cinfo, struct aead_request *req, unsigned int cryptlen) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); unsigned int q, t; u8 *Q_ptr; int ret; memcpy(cinfo, req->iv, 16); /*** 1. Prepare Flags Octet ***/ /* Adata */ if (req->assoclen) *cinfo |= (1 << 6); /* Encode t (mac length) */ t = ctx->authsize; *cinfo |= (((t-2)/2) << 3); /* q already present in iv[0], do nothing */ /*** 2. N is already present in iv, do nothing ***/ /*** 3. Encode Q - message length ***/ q = req->iv[0] + 1; Q_ptr = cinfo + 16 - q; ret = ccm_set_msg_len(Q_ptr, cryptlen, q); return ret; } /** * @brief Encode AAD length * * Function returns the number of bytes used in encoding, * the specs set the AAD length encoding limits as follows: * * (0) < a < (2^16-2^8) => two bytes, no prefix * (2^16 - 2^8) <= a < (2^32) => six bytes, prefix: 0xFF || 0xFE * * Note that zero length AAD is not encoded at all; bit 6 * in B0 flags is zero if there is no AAD. */ static int ccm_encode_adata_len(u8 *adata, unsigned int a) { int len = 0; /* add control info for associated data * RFC 3610 and NIST Special Publication 800-38C */ if (a < 65280) { *(__be16 *)adata = cpu_to_be16(a); len = 2; } else { *(__be16 *)adata = cpu_to_be16(0xfffe); *(__be32 *)&adata[2] = cpu_to_be32(a); len = 6; } return len; } static int tegra_se_ccm_compute_auth(struct aead_request *req, bool encrypt) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_req_context *req_ctx = aead_request_ctx(req); unsigned int assoclen, cryptlen; struct scatterlist *sg, *sg_start; unsigned int ilen, pad_bytes_len, pt_bytes = 0, total = 0; u32 num_sgs, mapped_len = 0, *cpuvaddr = NULL; struct tegra_se_dev *se_dev; struct tegra_se_ll *src_ll; struct tegra_se_cmdbuf *cmdbuf; dma_addr_t pt_addr, adata_addr, iova = 0; u8 *buf, *adata = NULL; int ret, count; se_dev = se_devices[SE_AEAD]; /* 0 - Setup */ assoclen = req->assoclen; sg = req->src; if (encrypt) cryptlen = req->cryptlen; else cryptlen = req->cryptlen - ctx->authsize; /* 1 - Formatting of the Control Information and the Nonce */ buf = ctx->buf[0]; ret = ccm_encode_b0(buf, req, cryptlen); if (ret) return -EINVAL; /* 1.1 - Map B0 block */ se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); src_ll = se_dev->src_ll; src_ll->addr = ctx->buf_addr[0]; src_ll->data_len = 16; src_ll++; total += 16; /* 2 - Formatting of the Associated Data */ if (assoclen) { buf = ctx->buf[1]; /* 2.0 - Prepare alen encoding */ ilen = ccm_encode_adata_len(buf, assoclen); /* 2.1 - Map alen encoding */ src_ll->addr = ctx->buf_addr[1]; src_ll->data_len = ilen; src_ll++; total += ilen; /* 2.2 - Copy adata and map it */ adata = dma_alloc_coherent(se_dev->dev, assoclen, &adata_addr, GFP_KERNEL); num_sgs = tegra_se_count_sgs(sg, assoclen); sg_copy_to_buffer(sg, num_sgs, adata, assoclen); src_ll->addr = adata_addr; src_ll->data_len = assoclen; src_ll++; total += assoclen; /* 2.3 - Prepare and Map padding zero bytes data */ pad_bytes_len = 16 - (assoclen + ilen) % 16; if (pad_bytes_len) { memset(ctx->buf[2], 0, pad_bytes_len); src_ll->addr = ctx->buf_addr[2]; src_ll->data_len = pad_bytes_len; src_ll++; total += pad_bytes_len; } } /* 3 - Formatting of plain text */ /* In case of ccm decryption plain text is present in dst buffer */ sg = encrypt ? req->src : req->dst; sg_start = sg; /* 3.1 Iterative over sg, to skip associated data */ count = assoclen; while (count) { /* If a SG has both assoc data and PT data * make note of it and use for PT mac computation task. */ if (count < sg->length) { ret = dma_map_sg(se_dev->dev, sg, 1, DMA_TO_DEVICE); if (!ret) { dev_err(se_dev->dev, "dma_map_sg error\n"); goto out; } pt_addr = sg_dma_address(sg) + count; /* SG length can be more than cryptlen and associated * data, in this case encrypted length should be * cryptlen. */ if (cryptlen + count <= sg->length) pt_bytes = cryptlen; else pt_bytes = sg->length - count; sg_start = sg; mapped_len = sg->length; sg = sg_next(sg); break; } count -= min_t(size_t, (size_t)sg->length, (size_t)count); sg = sg_next(sg); } /* 3.2 Map the plain text buffer */ if (pt_bytes) { src_ll->addr = pt_addr; src_ll->data_len = pt_bytes; src_ll++; } count = cryptlen - pt_bytes; while (count) { ret = dma_map_sg(se_dev->dev, sg, 1, DMA_TO_DEVICE); if (!ret) { dev_err(se_dev->dev, "dma_map_sg error\n"); goto out; } src_ll->addr = sg_dma_address(sg); src_ll->data_len = min_t(size_t, (size_t)sg->length, (size_t)count); count -= min_t(size_t, (size_t)sg->length, (size_t)count); mapped_len += sg->length; sg = sg_next(sg); src_ll++; } total += cryptlen; /* 3.3 - Map padding data */ pad_bytes_len = 16 - (cryptlen % 16); if (pad_bytes_len) { memset(ctx->buf[3], 0, pad_bytes_len); src_ll->addr = ctx->buf_addr[3]; src_ll->data_len = pad_bytes_len; src_ll++; } total += pad_bytes_len; /* 4 - Compute CBC_MAC */ cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto out; } cpuvaddr = cmdbuf->addr; iova = cmdbuf->iova; se_dev->dst_ll = se_dev->src_ll; /* dummy */ req_ctx->op_mode = SE_AES_OP_MODE_CBC_MAC; req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, 0, ctx->keylen); req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, 0, ctx->slot->slot_num, 0, true); tegra_se_send_data(se_dev, req_ctx, NULL, total, se_dev->opcode_addr, cpuvaddr); ret = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); /* 4.1 - Read result */ ret = tegra_se_read_cmac_result(se_dev, ctx->mac, TEGRA_SE_AES_CBC_MAC_DIGEST_SIZE, false); if (ret) { dev_err(se_dev->dev, "failed to read cmac result\n"); goto out; } ret = tegra_se_clear_cmac_result(se_dev, TEGRA_SE_AES_CBC_MAC_DIGEST_SIZE); if (ret) { dev_err(se_dev->dev, "failed to clear cmac result\n"); goto out; } /* 5 - Clean up */ tegra_unmap_sg(se_dev->dev, sg_start, DMA_TO_DEVICE, mapped_len); out: if (assoclen) dma_free_coherent(se_dev->dev, assoclen, adata, adata_addr); return ret; } static int ccm_ctr_extract_encrypted_mac(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); u32 num_sgs; /* Extract cipher mac from buf */ memset(ctx->enc_mac, 0, 16); /* Padding zeros will be taken care */ num_sgs = tegra_se_count_sgs(req->src, req->cryptlen + req->assoclen); sg_pcopy_to_buffer(req->src, num_sgs, ctx->enc_mac, ctx->authsize, req->cryptlen + req->assoclen - ctx->authsize); return 0; } static int tegra_se_ccm_ctr(struct aead_request *req, bool encrypt) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_req_context *req_ctx = aead_request_ctx(req); struct tegra_se_ll *src_ll, *dst_ll; struct tegra_se_cmdbuf *cmdbuf; struct scatterlist *src_sg, *dst_sg, *src_sg_start; unsigned int assoclen, cryptlen, total, pad_bytes_len; unsigned int mapped_cryptlen, mapped_len; int ret = 0, count, index = 0, num_sgs; struct tegra_se_dev *se_dev; u8 *dst_buf; dma_addr_t dst_buf_dma_addr; se_dev = se_devices[SE_AEAD]; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K * SE_MAX_SUBMIT_CHAIN_SZ); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); return PTR_ERR(cmdbuf); } /* 0. Setup */ assoclen = req->assoclen; cryptlen = req->cryptlen; se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); se_dev->dst_ll = (struct tegra_se_ll *)(se_dev->dst_ll_buf); src_ll = se_dev->src_ll; dst_ll = se_dev->dst_ll; total = 0; mapped_cryptlen = 0; mapped_len = 0; if (encrypt) cryptlen = req->cryptlen; else cryptlen = req->cryptlen - ctx->authsize; /* If destination buffer is scattered over multiple sg, destination * buffer splits need not be same as src_sg so create * separate buffer for destination. Onebyte extra to handle zero * cryptlen case. */ dst_buf = dma_alloc_coherent(se_dev->dev, cryptlen+1, &dst_buf_dma_addr, GFP_KERNEL); if (!dst_buf) return -ENOMEM; /* 1. Add mac to src */ if (encrypt) { /* 1.1 Add mac to src_ll list */ src_ll->addr = ctx->mac_addr; src_ll->data_len = 16; src_ll++; total += 16; /* 1.2 Add corresponding entry to dst_ll list */ dst_ll->addr = ctx->enc_mac_addr; dst_ll->data_len = 16; dst_ll++; } else { /* 1.0 Extract encrypted mac to ctx->enc_mac */ ccm_ctr_extract_encrypted_mac(req); /* 1.1 Add encrypted mac to src_ll list */ src_ll->addr = ctx->enc_mac_addr; src_ll->data_len = 16; src_ll++; total += 16; /* 1.2 Add corresponding entry to dst_ll list */ memset(ctx->dec_mac, 0, 16); dst_ll->addr = ctx->dec_mac_addr; dst_ll->data_len = 16; dst_ll++; } /* 2. Add plain text to src */ src_sg = req->src; src_sg_start = src_sg; dst_sg = req->dst; /* 2.1 Iterative over assoclen, to skip it for encryption */ count = assoclen; while (count) { /* If a SG has both assoc data and PT/CT data */ if (count < src_sg->length) { ret = dma_map_sg(se_dev->dev, src_sg, 1, DMA_TO_DEVICE); if (!ret) { pr_err("dma_map_sg error\n"); goto out; } /* Fill PT/CT src list details from this sg */ src_ll->addr = sg_dma_address(src_sg) + count; /* SG length can be more than cryptlen and associated * data, in this case encrypted length should be * cryptlen. */ if (cryptlen + count <= src_sg->length) src_ll->data_len = cryptlen; else src_ll->data_len = src_sg->length - count; mapped_cryptlen = src_ll->data_len; mapped_len = src_sg->length; /* Fill dst list details */ dst_ll->addr = dst_buf_dma_addr; dst_ll->data_len = src_ll->data_len; index += src_ll->data_len; dst_ll++; src_ll++; /* Store from where src_sg is mapped, so it can be * unmapped. */ src_sg_start = src_sg; src_sg = sg_next(src_sg); break; } /* If whole SG is associated data iterate over it */ count -= min_t(size_t, (size_t)src_sg->length, (size_t)count); src_sg = sg_next(src_sg); } /* 2.2 Add plain text to src_ll and dst_ll list */ count = cryptlen - mapped_cryptlen; while (count) { ret = dma_map_sg(se_dev->dev, src_sg, 1, DMA_TO_DEVICE); if (!ret) { dev_err(se_dev->dev, "dma_map_sg error\n"); goto out; } src_ll->addr = sg_dma_address(src_sg); src_ll->data_len = min_t(size_t, (size_t)src_sg->length, (size_t)count); dst_ll->addr = dst_buf_dma_addr + index; dst_ll->data_len = src_ll->data_len; index += dst_ll->data_len; count -= min_t(size_t, (size_t)src_sg->length, (size_t)count); mapped_len += src_sg->length; src_sg = sg_next(src_sg); src_ll++; dst_ll++; } total += cryptlen; /* 3. Pad necessary zeros */ memset(ctx->buf[0], 0, 16); pad_bytes_len = 16 - (cryptlen % 16); if (pad_bytes_len) { src_ll->addr = ctx->buf_addr[0]; src_ll->data_len = pad_bytes_len; src_ll++; dst_ll->addr = ctx->buf_addr[0]; dst_ll->data_len = pad_bytes_len; dst_ll++; } total += pad_bytes_len; /* 4. Encrypt/Decrypt using CTR */ req_ctx->op_mode = SE_AES_OP_MODE_CTR; req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, 0, ctx->keylen); req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, 0, ctx->slot->slot_num, 0, true); tegra_se_send_ctr_seed(se_dev, (u32 *)req->iv, se_dev->opcode_addr, cmdbuf->addr); tegra_se_send_data(se_dev, req_ctx, NULL, total, se_dev->opcode_addr, cmdbuf->addr); ret = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); num_sgs = tegra_se_count_sgs(req->dst, assoclen + cryptlen); sg_pcopy_from_buffer(req->dst, num_sgs, dst_buf, cryptlen, assoclen); out: /* 5 Clean up */ tegra_unmap_sg(se_dev->dev, src_sg_start, DMA_TO_DEVICE, mapped_len); dma_free_coherent(se_dev->dev, cryptlen+1, dst_buf, dst_buf_dma_addr); return ret; } static void ccm_ctr_add_encrypted_mac_to_dest(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct scatterlist *dst_sg; u32 num_sgs; dst_sg = req->dst; num_sgs = tegra_se_count_sgs(dst_sg, req->assoclen + req->cryptlen + ctx->authsize); sg_pcopy_from_buffer(dst_sg, num_sgs, ctx->enc_mac, ctx->authsize, req->assoclen + req->cryptlen); } /* * CCM Generation-Encryption operation. * * Input: * valid nonce N of length Nlen bits or n octet length. * valid payload P of length Plen bits. * valid associated data A of length Alen bits or a octet length. * * Output: * ciphertext C. * authentication tag T of length Tlen bits or t octet length. * * Steps: * 1. Apply the formatting function to (N, A, P) to produce the blocks * B0, B1, B2, ..., Br. Each block is 16bytes and 128bit length. * 2. Set Y0 = CIPH_K(B0). * 3. For i = 1 to r, do Yi = CIPH_K(Bi ⊕ Yi-1). * 4. Set T = MSBTlen(Yr). * 5. Apply the counter generation function to generate the counter * blocks Ctr0, Ctr1,..., Ctrm, where m = ⎡Plen/128⎤. * 6. For j = 0 to m, do Sj = CIPH_K(Ctrj). * 7. Set S = S1 || S2 || ...|| Sm. * 8. Return C = (P ⊕ MSBPlen(S)) || (T ⊕ MSBTlen(S0)). * * req->iv has the formatted ctr0 (i.e. Flags || N || 0) as per * NIST 800-38C and rfc 3610 notation. */ static int tegra_se_aes_ccm_encrypt(struct aead_request *req) { struct tegra_se_dev *se_dev; int err; se_dev = se_devices[SE_AEAD]; mutex_lock(&se_dev->mtx); err = tegra_se_ccm_init_crypt(req); if (err) goto out; err = tegra_se_ccm_compute_auth(req, ENCRYPT); if (err) goto out; err = tegra_se_ccm_ctr(req, ENCRYPT); if (err) goto out; ccm_ctr_add_encrypted_mac_to_dest(req); out: mutex_unlock(&se_dev->mtx); return err; } /* * CCM Decryption-Verification operation. * * Input: * valid nonce N of length Nlen bits or n octet length. * valid associated data A of length Alen bits or a octet length. * purported ciphertext C (contains MAC of length Tlen) of length Clen bits. * * Output: * either the payload P or INVALID. * * Steps: * 1. If Clen ≤ Tlen, then return INVALID. * 2. Apply the counter generation function to generate the counter * blocks Ctr0, Ctr1,..., Ctrm, where m = ⎡(Clen-Tlen)/128⎤. * 3. For j = 0 to m, do Sj = CIPH_K(Ctrj). * 4. Set S = S1 || S2 || ...|| Sm. * 5. Set P = MSB(Clen-Tlen)(C) ⊕ MSB(Clen-Tlen)(S). * 6. Set T = LSB(Tlen)(C) ⊕ MSB(Tlen)(S0). * 7. If N, A, or P is not valid, then return INVALID, else apply the * formatting function to (N, A, P) to produce the blocks B0, B1,...,Br. * 8. Set Y0 = CIPH_K(B0). * 9. For i = 1 to r, do Yi = CIPH_K(Bi ⊕ Yi-1). * 10. If T ≠ MSBTlen(Yr), then return INVALID, else return P. * * req->iv has the formatted ctr0 (i.e. Flags || N || 0) as per * NIST 800-38C and rfc 3610 notation. */ static int tegra_se_aes_ccm_decrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_dev *se_dev; int err; se_dev = se_devices[SE_AEAD]; mutex_lock(&se_dev->mtx); err = tegra_se_ccm_init_crypt(req); if (err) goto out; err = tegra_se_ccm_ctr(req, DECRYPT); if (err) goto out; err = tegra_se_ccm_compute_auth(req, DECRYPT); if (err) goto out; if (crypto_memneq(ctx->mac, ctx->dec_mac, ctx->authsize)) err = -EBADMSG; out: mutex_unlock(&se_dev->mtx); return err; } static int tegra_se_aes_ccm_init(struct crypto_aead *tfm) { struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_dev *se_dev = se_devices[SE_AEAD]; int i = 0, err = 0; crypto_aead_set_reqsize(tfm, sizeof(struct tegra_se_req_context)); mutex_lock(&se_dev->mtx); for (i = 0; i < 4; i++) { ctx->buf[i] = dma_alloc_coherent( se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, &ctx->buf_addr[i], GFP_KERNEL); if (!ctx->buf[i]) { while (i > 0) { i--; dma_free_coherent( se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, ctx->buf[i], ctx->buf_addr[i]); ctx->buf[i] = NULL; } dev_err(se_dev->dev, "Cannot allocate memory to buf[0]\n"); mutex_unlock(&se_dev->mtx); return -ENOMEM; } } ctx->mac = dma_alloc_coherent(se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, &ctx->mac_addr, GFP_KERNEL); if (!ctx->mac) { err = -ENOMEM; goto out; } ctx->enc_mac = dma_alloc_coherent(se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, &ctx->enc_mac_addr, GFP_KERNEL); if (!ctx->enc_mac) { err = -ENOMEM; goto out; } ctx->dec_mac = dma_alloc_coherent(se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, &ctx->dec_mac_addr, GFP_KERNEL); if (!ctx->dec_mac) { err = -ENOMEM; goto out; } out: mutex_unlock(&se_dev->mtx); return err; } static void tegra_se_aes_ccm_exit(struct crypto_aead *tfm) { struct tegra_se_aes_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_dev *se_dev = se_devices[SE_AES]; int i = 0; mutex_lock(&se_dev->mtx); tegra_se_free_key_slot(ctx->slot); ctx->slot = NULL; for (i = 0; i < 4; i++) { dma_free_coherent( se_dev->dev, (TEGRA_SE_AES_BLOCK_SIZE), ctx->buf[i], ctx->buf_addr[i]); } dma_free_coherent(se_dev->dev, (TEGRA_SE_AES_BLOCK_SIZE), ctx->mac, ctx->mac_addr); dma_free_coherent(se_dev->dev, (TEGRA_SE_AES_BLOCK_SIZE), ctx->enc_mac, ctx->enc_mac_addr); dma_free_coherent(se_dev->dev, (TEGRA_SE_AES_BLOCK_SIZE), ctx->dec_mac, ctx->dec_mac_addr); mutex_unlock(&se_dev->mtx); } static int tegra_se_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_dev *se_dev; struct tegra_se_slot *pslot; struct tegra_se_cmdbuf *cmdbuf; u32 *cpuvaddr = NULL; dma_addr_t iova = 0; int ret = 0; se_dev = se_devices[SE_AEAD]; mutex_lock(&se_dev->mtx); /* Check for valid arguments */ if (!ctx || !key) { dev_err(se_dev->dev, "invalid context or key"); ret = -EINVAL; goto free_ctx; } if ((keylen != TEGRA_SE_KEY_128_SIZE) && (keylen != TEGRA_SE_KEY_192_SIZE) && (keylen != TEGRA_SE_KEY_256_SIZE)) { dev_err(se_dev->dev, "invalid key size"); ret = -EINVAL; goto free_ctx; } /* Get free key slot */ pslot = tegra_se_alloc_key_slot(); if (!pslot) { dev_err(se_dev->dev, "no free key slot\n"); ret = -ENOMEM; goto free_ctx; } ctx->slot = pslot; ctx->keylen = keylen; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto free_ctx; } cpuvaddr = cmdbuf->addr; iova = cmdbuf->iova; ret = tegra_se_send_key_data( se_dev, (u8 *)key, keylen, ctx->slot->slot_num, SE_KEY_TABLE_TYPE_GCM, se_dev->opcode_addr, cmdbuf, NONE); tegra_se_free_key_slot(ctx->slot); free_ctx: mutex_unlock(&se_dev->mtx); return ret; } static int tegra_se_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); switch (authsize) { case 4: case 8: case 10: case 12: case 14: case 16: ctx->authsize = authsize; break; default: return -EINVAL; } return 0; } static int tegra_se_gcm_gmac(struct aead_request *req, bool encrypt) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_req_context *req_ctx = aead_request_ctx(req); unsigned int assoclen; struct tegra_se_dev *se_dev; struct tegra_se_ll *src_ll; struct scatterlist *sg; struct tegra_se_cmdbuf *cmdbuf; int ret; u32 *cpuvaddr = NULL; dma_addr_t iova = 0; /* 0 - Setup */ se_dev = ctx->se_dev; assoclen = req->assoclen; sg = req->src; /* 1 - Map associated data */ se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); src_ll = se_dev->src_ll; ret = tegra_map_sg(se_dev->dev, sg, 1, DMA_TO_DEVICE, se_dev->src_ll, assoclen); if (!ret) return ret; /* 2 - Prepare command buffer and submit request to hardware */ cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto out; } cpuvaddr = cmdbuf->addr; iova = cmdbuf->iova; se_dev->dst_ll = se_dev->src_ll; /* dummy */ /* 2.2 Prepare and submit request */ req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, encrypt ? ENCRYPT : DECRYPT, SE_AES_GMAC); req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, 0, ctx->slot->slot_num, 0, true); tegra_se_send_gcm_data(se_dev, req_ctx, assoclen, se_dev->opcode_addr, cpuvaddr, SE_AES_GMAC); ret = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); out: /* 3 - Clean up */ tegra_unmap_sg(se_dev->dev, sg, DMA_TO_DEVICE, assoclen); return ret; } static int tegra_se_gcm_op(struct aead_request *req, bool encrypt) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_req_context *req_ctx = aead_request_ctx(req); unsigned int assoclen, cryptlen, mapped_cryptlen, mapped_len; struct tegra_se_ll *src_ll, *dst_ll; struct scatterlist *src_sg, *src_sg_start; struct tegra_se_dev *se_dev; struct tegra_se_cmdbuf *cmdbuf; u32 *cpuvaddr = NULL, num_sgs; dma_addr_t iova = 0; int ret, index, count; u32 iv[4]; u8 *dst_buf; dma_addr_t dst_buf_dma_addr; /* 0 - Setup */ se_dev = ctx->se_dev; se_dev->src_ll = (struct tegra_se_ll *)(se_dev->src_ll_buf); se_dev->dst_ll = (struct tegra_se_ll *)(se_dev->dst_ll_buf); src_ll = se_dev->src_ll; dst_ll = se_dev->dst_ll; mapped_cryptlen = 0; mapped_len = 0; ret = 0; index = 0; assoclen = req->assoclen; if (encrypt) cryptlen = req->cryptlen; else cryptlen = req->cryptlen - ctx->authsize; /* If destination buffer is scattered over multiple sg, destination * buffer splits need not be same as src_sg so create * separate buffer for destination. Onebyte extra to handle zero * cryptlen case. */ dst_buf = dma_alloc_coherent(se_dev->dev, cryptlen+1, &dst_buf_dma_addr, GFP_KERNEL); if (!dst_buf) return -ENOMEM; /* 1. Add plain text to src */ src_sg = req->src; src_sg_start = src_sg; /* 1.1 Iterative over assoclen, to skip it for encryption */ count = assoclen; while (count) { /* If a SG has both assoc data and PT/CT data */ if (count < src_sg->length) { ret = dma_map_sg(se_dev->dev, src_sg, 1, DMA_TO_DEVICE); if (!ret) { pr_err("%s:%d dma_map_sg error\n", __func__, __LINE__); goto free_dst_buf; } /* Fill PT/CT src list details from this sg */ src_ll->addr = sg_dma_address(src_sg) + count; /* SG length can be more than cryptlen and associated * data, in this case encrypted length should be * cryptlen */ if (cryptlen + count <= src_sg->length) src_ll->data_len = cryptlen; else src_ll->data_len = src_sg->length - count; mapped_cryptlen = src_ll->data_len; mapped_len = src_sg->length; /* Fill dst list details */ dst_ll->addr = dst_buf_dma_addr; dst_ll->data_len = src_ll->data_len; index += src_ll->data_len; dst_ll++; src_ll++; /* Store from where src_sg is mapped, so it can be * unmapped. */ src_sg_start = src_sg; src_sg = sg_next(src_sg); break; } /* If whole SG is associated data iterate over it */ count -= min_t(size_t, (size_t)src_sg->length, (size_t)count); src_sg = sg_next(src_sg); } /* 1.2 Add plain text to src_ll list */ ret = tegra_map_sg(se_dev->dev, src_sg, 1, DMA_TO_DEVICE, src_ll, cryptlen - mapped_cryptlen); if (ret < 0) goto free_dst_buf; mapped_len += (cryptlen - mapped_cryptlen); /* 1.3 Fill dst_ll list */ while (src_sg && cryptlen) { dst_ll->addr = dst_buf_dma_addr + index; dst_ll->data_len = src_ll->data_len; index += dst_ll->data_len; dst_ll++; src_ll++; src_sg = sg_next(src_sg); } /* 2. Encrypt/Decrypt using GCTR and perform GHASH on plain text */ cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); ret = PTR_ERR(cmdbuf); goto free_dst_buf; } cpuvaddr = cmdbuf->addr; iova = cmdbuf->iova; /* 2.2 Prepare and program J0 (i.e. iv || 0^31 || 1) using IV */ memset(iv, 0, 16); memcpy(iv, req->iv, 12); iv[3] = (1 << 24); tegra_se_send_ctr_seed(se_dev, iv, se_dev->opcode_addr, cpuvaddr); /* 2.3 Submit request */ req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, 0, encrypt ? SE_AES_GCM_ENC : SE_AES_GCM_DEC); req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, 0, ctx->slot->slot_num, 0, true); tegra_se_send_gcm_data(se_dev, req_ctx, cryptlen, se_dev->opcode_addr, cpuvaddr, encrypt ? SE_AES_GCM_ENC : SE_AES_GCM_DEC); ret = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); /* 3 clean up */ tegra_unmap_sg(se_dev->dev, src_sg_start, DMA_TO_DEVICE, mapped_len); num_sgs = tegra_se_count_sgs(req->dst, assoclen + cryptlen); sg_pcopy_from_buffer(req->dst, num_sgs, dst_buf, cryptlen, assoclen); free_dst_buf: dma_free_coherent(se_dev->dev, cryptlen+1, dst_buf, dst_buf_dma_addr); return ret; } static void gcm_program_aad_msg_len(struct tegra_se_dev *se_dev, unsigned int alen, unsigned int clen, unsigned int opcode_addr, u32 *cpuvaddr) { u32 i = 0; i = se_dev->cmdbuf_cnt; cpuvaddr[i++] = __nvhost_opcode_incr(opcode_addr + SE_AES_CRYPTO_AAD_LENGTH_0_OFFSET, 2); cpuvaddr[i++] = alen*8; cpuvaddr[i++] = 0; cpuvaddr[i++] = __nvhost_opcode_incr(opcode_addr + SE_AES_CRYPTO_MSG_LENGTH_0_OFFSET, 2); cpuvaddr[i++] = clen*8; cpuvaddr[i++] = 0; se_dev->cmdbuf_cnt = i; } static void gcm_add_encrypted_mac_to_dest(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct scatterlist *dst_sg; u32 num_sgs; dst_sg = req->dst; num_sgs = tegra_se_count_sgs(dst_sg, req->assoclen + req->cryptlen + ctx->authsize); sg_pcopy_from_buffer(dst_sg, num_sgs, ctx->mac, ctx->authsize, req->assoclen + req->cryptlen); } static int tegra_se_gcm_final(struct aead_request *req, bool encrypt) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_req_context *req_ctx = aead_request_ctx(req); struct tegra_se_cmdbuf *cmdbuf; u32 *cpuvaddr = NULL; unsigned int cryptlen; dma_addr_t iova = 0; int ret, i; struct tegra_se_dev *se_dev; u32 iv[4], val; /* 1 Get free command buffer */ se_dev = ctx->se_dev; cmdbuf = tegra_se_alloc_cmdbuf(se_dev->dev->parent, SZ_16K); if (IS_ERR(cmdbuf)) { dev_err(se_dev->dev, "cmdbuf allocation failed\n"); return PTR_ERR(cmdbuf); } cpuvaddr = cmdbuf->addr; iova = cmdbuf->iova; /* 2. Configure AAD and MSG length */ if (encrypt) cryptlen = req->cryptlen; else cryptlen = req->cryptlen - ctx->authsize; gcm_program_aad_msg_len(se_dev, req->assoclen, cryptlen, se_dev->opcode_addr, cpuvaddr); /* 3. Prepare and program J0 (i.e. iv || 0^31 || 1 ) using IV */ memset(iv, 0, 16); memcpy(iv, req->iv, 12); iv[3] = (1 << 24); tegra_se_send_ctr_seed(se_dev, iv, se_dev->opcode_addr, cpuvaddr); /* 4. Program command buffer */ req_ctx->op_mode = SE_AES_OP_MODE_GCM; req_ctx->config = tegra_se_get_config(se_dev, req_ctx->op_mode, encrypt ? ENCRYPT : DECRYPT, SE_AES_GCM_FINAL); req_ctx->crypto_config = tegra_se_get_crypto_config( se_dev, req_ctx->op_mode, 0, ctx->slot->slot_num, 0, true); i = se_dev->cmdbuf_cnt; /* 4.1 Config register */ cpuvaddr[i++] = __nvhost_opcode_incr(se_dev->opcode_addr, 2); cpuvaddr[i++] = req_ctx->config; cpuvaddr[i++] = req_ctx->crypto_config; /* 4.2 Destination address */ cpuvaddr[i++] = __nvhost_opcode_incr( se_dev->opcode_addr + SE_AES_OUT_ADDR_OFFSET, 2); cpuvaddr[i++] = (u32)(ctx->mac_addr); cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(MSB(ctx->mac_addr)) | SE_ADDR_HI_SZ(SE_AES_GCM_GMAC_SIZE)); /* 4.3 Handle zero size */ cpuvaddr[i++] = __nvhost_opcode_nonincr( se_dev->opcode_addr + SE_AES_OPERATION_OFFSET, 1); val = SE_OPERATION_WRSTALL(WRSTALL_TRUE); /* If GMAC or GCM_ENC/GCM_DEC is not operated before then * set SE_OPERATION.INIT to true. */ if (req_ctx->init != true) val |= SE_OPERATION_INIT(INIT_TRUE); val |= SE_OPERATION_FINAL(FINAL_TRUE); val |= SE_OPERATION_OP(OP_START); val |= SE_OPERATION_LASTBUF(LASTBUF_TRUE); cpuvaddr[i++] = val; se_dev->cmdbuf_cnt = i; /* 5 Submit request */ ret = tegra_se_channel_submit_gather(se_dev, cmdbuf, 0, se_dev->cmdbuf_cnt, NONE); return ret; } /* * GCM encrypt operation. * Input: * initialization vector IV (whose length is supported). * plaintext P (whose length is supported). * additional authenticated data A (whose length is supported). * Output: * ciphertext C. * authentication tag T. * Steps: * 1. Let H = CIPH_K(0^128). * 2. Define a block, J0, as follows: * If len(IV) = 96, then let J0 = IV || 0^31 || 1. * If len(IV) ≠ 96, then let s = 128 * ⎡len(IV)/128⎤-len(IV), * and let J0 = GHASH_H(IV || 0^(s+64) || [len(IV)]_64). * 3. Let C = GCTR_K(inc_32(J0), P). * 4. Let u = 128 * ⎡len(C)/128⎤-len(C) and v = 128 * ⎡len(A)/128⎤-len(A) * 5. Define a block, S, as follows: * S = GHASH_H (A || 0^v || C || 0^u || [len(A)]_64 || [len(C)]_64). * 6. Let T = MSB_t(GCTR_K(J0, S)) * 7. Return (C, T). * SE hardware provide below 3 operations for above steps. * Op1: GMAC - step 1 and partial of 5 (i.e. GHASH_H (A || 0^v)). * Op2: GCM_ENC - step 1 to 4 and parital of 5 (i.e. GHASH_H (C || 0^u)). * Op3: GCM_FINAL - step partial of 5 * (i.e. GHASH_H ([len(A)]_64 || [len(C)]_64) and 6. */ static int tegra_se_aes_gcm_encrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_req_context *req_ctx = aead_request_ctx(req); int ret; ctx->se_dev = se_devices[SE_AEAD]; req_ctx->init = false; req_ctx->op_mode = SE_AES_OP_MODE_GCM; mutex_lock(&ctx->se_dev->mtx); /* If there is associated date perform GMAC operation */ if (req->assoclen) { ret = tegra_se_gcm_gmac(req, ENCRYPT); if (ret) goto out; } /* GMAC_ENC operation */ if (req->cryptlen) { ret = tegra_se_gcm_op(req, ENCRYPT); if (ret) goto out; } /* GMAC_FINAL operation */ ret = tegra_se_gcm_final(req, ENCRYPT); if (ret) goto out; /* Copy the mac to request destination */ gcm_add_encrypted_mac_to_dest(req); out: mutex_unlock(&ctx->se_dev->mtx); return ret; } /* * GCM decrypt operation. * * Input: * initialization vector IV (whose length is supported). * ciphertext C (whose length is supported). * additional authenticated data A (whose length is supported). * authentication tag T. * * Output: * plaintext P. * verify tag. * * Steps: * 1. If the bit lengths of IV, A or C are not supported, or if len(T) ≠t, * then return FAIL. * 2. Let H = CIPH_K(0^128). * 3. Define a block, J0, as follows: * If len(IV) = 96, then J0 = IV || 0^31 || 1. * If len(IV) ≠ 96, then let s = 128 ⎡len(IV)/128⎤-len(IV), * and let J0 = GHASH_H(IV || 0^(s+64) || [len(IV)]_64). * 4. Let P = GCTR_K(inc_32(J0), C). * 5. Let u = 128 * ⎡len(C)/128⎤-len(C) and v = 128 * ⎡len(A)/128⎤-len(A) * 6. Define a block, S, as follows: * S = GHASH_H (A || 0^v || C || 0^u || [len(A)]_64 || [len(C)]_64) * 7. Let T′ = MSB_t(GCTR_K(J0, S)) * 8. If T = T′, then return P; else return FAIL. * * SE hardware provide below 3 operations. * Op1: GMAC - step 2 and partial of 6 (i.e. GHASH_H (A || 0^v)). * Op2: GCM_DEC - step 2 to 4 and parital of 6 (i.e. GHASH_H (C || 0^u)). * Op3: GCM_FINAL - step partial of 5 * (i.e. GHASH_H ([len(A)]_64 || [len(C)]_64) and 6. */ static int tegra_se_aes_gcm_decrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_req_context *req_ctx = aead_request_ctx(req); u8 mac[16]; struct scatterlist *sg; u32 num_sgs; int ret; ctx->se_dev = se_devices[SE_AEAD]; req_ctx->init = false; req_ctx->op_mode = SE_AES_OP_MODE_GCM; mutex_lock(&ctx->se_dev->mtx); if (req->assoclen) { ret = tegra_se_gcm_gmac(req, DECRYPT); if (ret) goto out; } if (req->cryptlen - req->assoclen - ctx->authsize) { ret = tegra_se_gcm_op(req, DECRYPT); if (ret) goto out; } ret = tegra_se_gcm_final(req, DECRYPT); if (ret) goto out; /* Verify the mac */ sg = req->src; num_sgs = tegra_se_count_sgs(sg, req->assoclen + req->cryptlen); sg_pcopy_to_buffer(sg, num_sgs, mac, ctx->authsize, req->assoclen + req->cryptlen - ctx->authsize); if (crypto_memneq(ctx->mac, mac, ctx->authsize)) ret = -EBADMSG; out: mutex_unlock(&ctx->se_dev->mtx); return ret; } static int tegra_se_aes_gcm_init(struct crypto_aead *tfm) { struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_dev *se_dev = se_devices[SE_AEAD]; int ret = 0; crypto_aead_set_reqsize(tfm, sizeof(struct tegra_se_req_context)); mutex_lock(&se_dev->mtx); ctx->mac = dma_alloc_coherent(se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, &ctx->mac_addr, GFP_KERNEL); if (!ctx->mac) ret = -ENOMEM; mutex_unlock(&se_dev->mtx); return ret; } static void tegra_se_aes_gcm_exit(struct crypto_aead *tfm) { struct tegra_se_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct tegra_se_dev *se_dev = se_devices[SE_AEAD]; mutex_lock(&se_dev->mtx); tegra_se_free_key_slot(ctx->slot); ctx->slot = NULL; dma_free_coherent(se_dev->dev, TEGRA_SE_AES_BLOCK_SIZE, ctx->mac, ctx->mac_addr); mutex_unlock(&se_dev->mtx); } static struct aead_alg aead_algs[] = { { .setkey = tegra_se_aes_ccm_setkey, .setauthsize = tegra_se_aes_ccm_setauthsize, .encrypt = tegra_se_aes_ccm_encrypt, .decrypt = tegra_se_aes_ccm_decrypt, .init = tegra_se_aes_ccm_init, .exit = tegra_se_aes_ccm_exit, .ivsize = AES_BLOCK_SIZE, .maxauthsize = AES_BLOCK_SIZE, .chunksize = AES_BLOCK_SIZE, .base = { .cra_name = "ccm(aes)", .cra_driver_name = "ccm-aes-tegra", .cra_priority = 1000, .cra_blocksize = TEGRA_SE_AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_aes_ccm_ctx), .cra_module = THIS_MODULE, } }, { .setkey = tegra_se_aes_gcm_setkey, .setauthsize = tegra_se_aes_gcm_setauthsize, .encrypt = tegra_se_aes_gcm_encrypt, .decrypt = tegra_se_aes_gcm_decrypt, .init = tegra_se_aes_gcm_init, .exit = tegra_se_aes_gcm_exit, .ivsize = GCM_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .chunksize = AES_BLOCK_SIZE, .base = { .cra_name = "gcm(aes)", .cra_driver_name = "gcm-aes-tegra", .cra_priority = 1000, .cra_blocksize = TEGRA_SE_AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_aes_gcm_ctx), .cra_module = THIS_MODULE, } } }; static struct kpp_alg dh_algs[] = { { .set_secret = tegra_se_dh_set_secret, .generate_public_key = tegra_se_dh_compute_value, .compute_shared_secret = tegra_se_dh_compute_value, .max_size = tegra_se_dh_max_size, .exit = tegra_se_dh_exit_tfm, .base = { .cra_name = "dh", .cra_driver_name = "tegra-se-dh", .cra_priority = 300, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct tegra_se_dh_context), } } }; static struct rng_alg rng_algs[] = { { .generate = tegra_se_rng_drbg_get_random, .seed = tegra_se_rng_drbg_reset, .seedsize = TEGRA_SE_RNG_SEED_SIZE, .base = { .cra_name = "rng_drbg", .cra_driver_name = "rng_drbg-aes-tegra", .cra_priority = 100, .cra_flags = CRYPTO_ALG_TYPE_RNG, .cra_ctxsize = sizeof(struct tegra_se_rng_context), .cra_module = THIS_MODULE, .cra_init = tegra_se_rng_drbg_init, .cra_exit = tegra_se_rng_drbg_exit, } } }; static struct skcipher_alg aes_algs[] = { { .base.cra_name = "xts(aes)", .base.cra_driver_name = "xts-aes-tegra", .base.cra_priority = 500, .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, .base.cra_blocksize = TEGRA_SE_AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct tegra_se_aes_context), .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = tegra_se_aes_cra_init, .exit = tegra_se_aes_cra_exit, .setkey = tegra_se_aes_setkey, .encrypt = tegra_se_aes_xts_encrypt, .decrypt = tegra_se_aes_xts_decrypt, .min_keysize = TEGRA_SE_AES_MIN_KEY_SIZE, .max_keysize = TEGRA_SE_AES_MAX_KEY_SIZE, .ivsize = TEGRA_SE_AES_IV_SIZE, }, { .base.cra_name = "cbc(aes)", .base.cra_driver_name = "cbc-aes-tegra", .base.cra_priority = 500, .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, .base.cra_blocksize = TEGRA_SE_AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct tegra_se_aes_context), .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = tegra_se_aes_cra_init, .exit = tegra_se_aes_cra_exit, .setkey = tegra_se_aes_setkey, .encrypt = tegra_se_aes_cbc_encrypt, .decrypt = tegra_se_aes_cbc_decrypt, .min_keysize = TEGRA_SE_AES_MIN_KEY_SIZE, .max_keysize = TEGRA_SE_AES_MAX_KEY_SIZE, .ivsize = TEGRA_SE_AES_IV_SIZE, }, { .base.cra_name = "ecb(aes)", .base.cra_driver_name = "ecb-aes-tegra", .base.cra_priority = 500, .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, .base.cra_blocksize = TEGRA_SE_AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct tegra_se_aes_context), .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = tegra_se_aes_cra_init, .exit = tegra_se_aes_cra_exit, .setkey = tegra_se_aes_setkey, .encrypt = tegra_se_aes_ecb_encrypt, .decrypt = tegra_se_aes_ecb_decrypt, .min_keysize = TEGRA_SE_AES_MIN_KEY_SIZE, .max_keysize = TEGRA_SE_AES_MAX_KEY_SIZE, .ivsize = TEGRA_SE_AES_IV_SIZE, }, { .base.cra_name = "ctr(aes)", .base.cra_driver_name = "ctr-aes-tegra", .base.cra_priority = 500, .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, .base.cra_blocksize = TEGRA_SE_AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct tegra_se_aes_context), .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = tegra_se_aes_cra_init, .exit = tegra_se_aes_cra_exit, .setkey = tegra_se_aes_setkey, .encrypt = tegra_se_aes_ctr_encrypt, .decrypt = tegra_se_aes_ctr_decrypt, .min_keysize = TEGRA_SE_AES_MIN_KEY_SIZE, .max_keysize = TEGRA_SE_AES_MAX_KEY_SIZE, .ivsize = TEGRA_SE_AES_IV_SIZE, }, { .base.cra_name = "ofb(aes)", .base.cra_driver_name = "ofb-aes-tegra", .base.cra_priority = 500, .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, .base.cra_blocksize = TEGRA_SE_AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct tegra_se_aes_context), .base.cra_alignmask = 0, .base.cra_module = THIS_MODULE, .init = tegra_se_aes_cra_init, .exit = tegra_se_aes_cra_exit, .setkey = tegra_se_aes_setkey, .encrypt = tegra_se_aes_ofb_encrypt, .decrypt = tegra_se_aes_ofb_decrypt, .min_keysize = TEGRA_SE_AES_MIN_KEY_SIZE, .max_keysize = TEGRA_SE_AES_MAX_KEY_SIZE, .ivsize = TEGRA_SE_AES_IV_SIZE, }, }; static struct ahash_alg hash_algs[] = { { .init = tegra_se_aes_cmac_init, .update = tegra_se_aes_cmac_update, .final = tegra_se_aes_cmac_final, .finup = tegra_se_aes_cmac_finup, .digest = tegra_se_aes_cmac_digest, .setkey = tegra_se_aes_cmac_setkey, .export = tegra_se_aes_cmac_export, .import = tegra_se_aes_cmac_import, .halg.digestsize = TEGRA_SE_AES_CMAC_DIGEST_SIZE, .halg.statesize = TEGRA_SE_AES_CMAC_STATE_SIZE, .halg.base = { .cra_name = "cmac(aes)", .cra_driver_name = "tegra-se-cmac(aes)", .cra_priority = 500, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = TEGRA_SE_AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_aes_cmac_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_aes_cmac_cra_init, .cra_exit = tegra_se_aes_cmac_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA1_DIGEST_SIZE, .halg.statesize = SHA1_STATE_SIZE, .halg.base = { .cra_name = "sha1", .cra_driver_name = "tegra-se-sha1", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA224_DIGEST_SIZE, .halg.statesize = SHA224_STATE_SIZE, .halg.base = { .cra_name = "sha224", .cra_driver_name = "tegra-se-sha224", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA256_DIGEST_SIZE, .halg.statesize = SHA256_STATE_SIZE, .halg.base = { .cra_name = "sha256", .cra_driver_name = "tegra-se-sha256", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA384_DIGEST_SIZE, .halg.statesize = SHA384_STATE_SIZE, .halg.base = { .cra_name = "sha384", .cra_driver_name = "tegra-se-sha384", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA512_DIGEST_SIZE, .halg.statesize = SHA512_STATE_SIZE, .halg.base = { .cra_name = "sha512", .cra_driver_name = "tegra-se-sha512", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA3_224_DIGEST_SIZE, .halg.statesize = SHA3_224_STATE_SIZE, .halg.base = { .cra_name = "sha3-224", .cra_driver_name = "tegra-se-sha3-224", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA3_224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA3_256_DIGEST_SIZE, .halg.statesize = SHA3_256_STATE_SIZE, .halg.base = { .cra_name = "sha3-256", .cra_driver_name = "tegra-se-sha3-256", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA3_256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA3_384_DIGEST_SIZE, .halg.statesize = SHA3_384_STATE_SIZE, .halg.base = { .cra_name = "sha3-384", .cra_driver_name = "tegra-se-sha3-384", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA3_384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA3_512_DIGEST_SIZE, .halg.statesize = SHA3_512_STATE_SIZE, .halg.base = { .cra_name = "sha3-512", .cra_driver_name = "tegra-se-sha3-512", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA3_512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA3_512_DIGEST_SIZE, .halg.statesize = SHA3_512_STATE_SIZE, .halg.base = { .cra_name = "shake128", .cra_driver_name = "tegra-se-shake128", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA3_512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .halg.digestsize = SHA3_512_DIGEST_SIZE, .halg.statesize = SHA3_512_STATE_SIZE, .halg.base = { .cra_name = "shake256", .cra_driver_name = "tegra-se-shake256", .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA3_512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .setkey = tegra_se_sha_hmac_setkey, .halg.digestsize = SHA224_DIGEST_SIZE, .halg.statesize = SHA224_STATE_SIZE, .halg.base = { .cra_name = "hmac(sha224)", .cra_driver_name = "tegra-se-hmac-sha224", .cra_priority = 500, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .setkey = tegra_se_sha_hmac_setkey, .halg.digestsize = SHA256_DIGEST_SIZE, .halg.statesize = SHA256_STATE_SIZE, .halg.base = { .cra_name = "hmac(sha256)", .cra_driver_name = "tegra-se-hmac-sha256", .cra_priority = 500, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .setkey = tegra_se_sha_hmac_setkey, .halg.digestsize = SHA384_DIGEST_SIZE, .halg.statesize = SHA384_STATE_SIZE, .halg.base = { .cra_name = "hmac(sha384)", .cra_driver_name = "tegra-se-hmac-sha384", .cra_priority = 500, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } }, { .init = tegra_se_sha_init, .update = tegra_se_sha_update, .final = tegra_se_sha_final, .finup = tegra_se_sha_finup, .digest = tegra_se_sha_digest, .export = tegra_se_sha_export, .import = tegra_se_sha_import, .setkey = tegra_se_sha_hmac_setkey, .halg.digestsize = SHA512_DIGEST_SIZE, .halg.statesize = SHA512_STATE_SIZE, .halg.base = { .cra_name = "hmac(sha512)", .cra_driver_name = "tegra-se-hmac-sha512", .cra_priority = 500, .cra_flags = CRYPTO_ALG_TYPE_AHASH, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct tegra_se_sha_context), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = tegra_se_sha_cra_init, .cra_exit = tegra_se_sha_cra_exit, } } }; static struct akcipher_alg rsa_alg = { .encrypt = tegra_se_rsa_op, .decrypt = tegra_se_rsa_op, .sign = tegra_se_rsa_op, .verify = tegra_se_rsa_op, .set_priv_key = tegra_se_rsa_setkey, .set_pub_key = tegra_se_rsa_setkey, .max_size = tegra_se_rsa_max_size, .exit = tegra_se_rsa_exit, .base = { .cra_name = "rsa-pka0", .cra_driver_name = "tegra-se-pka0-rsa", .cra_priority = 300, .cra_ctxsize = sizeof(struct tegra_se_aes_rsa_context), .cra_module = THIS_MODULE, } }; static int tegra_se_nvhost_prepare_poweroff(struct platform_device *pdev) { /* Runtime PM not supported by Host1x */ return 0; } static struct tegra_se_chipdata tegra18_se_chipdata = { .aes_freq = 600000000, .cpu_freq_mhz = 2400, .kac_type = SE_KAC_T18X, }; static struct nvhost_device_data nvhost_se1_info = { .clocks = {{"se", 600000000}, {"emc", UINT_MAX, NVHOST_MODULE_ID_EXTERNAL_MEMORY_CONTROLLER, 0, NULL /*TEGRA_BWMGR_SET_EMC_FLOOR*/}, {} }, .can_powergate = true, .autosuspend_delay = 500, .class = NV_SE1_CLASS_ID, .private_data = &tegra18_se_chipdata, .serialize = 1, .push_work_done = 1, .vm_regs = {{SE_STREAMID_REG_OFFSET, true} }, .kernel_only = true, // .bwmgr_client_id = TEGRA_BWMGR_CLIENT_SE1, .prepare_poweroff = tegra_se_nvhost_prepare_poweroff, }; static struct nvhost_device_data nvhost_se2_info = { .clocks = {{"se", 600000000}, {"emc", UINT_MAX, NVHOST_MODULE_ID_EXTERNAL_MEMORY_CONTROLLER, 0, NULL /*TEGRA_BWMGR_SET_EMC_FLOOR*/}, {} }, .can_powergate = true, .autosuspend_delay = 500, .class = NV_SE2_CLASS_ID, .private_data = &tegra18_se_chipdata, .serialize = 1, .push_work_done = 1, .vm_regs = {{SE_STREAMID_REG_OFFSET, true} }, .kernel_only = true, // .bwmgr_client_id = TEGRA_BWMGR_CLIENT_SE2, .prepare_poweroff = tegra_se_nvhost_prepare_poweroff, }; static struct nvhost_device_data nvhost_se3_info = { .clocks = {{"se", 600000000}, {"emc", UINT_MAX, NVHOST_MODULE_ID_EXTERNAL_MEMORY_CONTROLLER, 0, NULL /*TEGRA_BWMGR_SET_EMC_FLOOR*/}, {} }, .can_powergate = true, .autosuspend_delay = 500, .class = NV_SE3_CLASS_ID, .private_data = &tegra18_se_chipdata, .serialize = 1, .push_work_done = 1, .vm_regs = {{SE_STREAMID_REG_OFFSET, true} }, .kernel_only = true, // .bwmgr_client_id = TEGRA_BWMGR_CLIENT_SE3, .prepare_poweroff = tegra_se_nvhost_prepare_poweroff, }; static struct nvhost_device_data nvhost_se4_info = { .clocks = {{"se", 600000000}, {"emc", UINT_MAX, NVHOST_MODULE_ID_EXTERNAL_MEMORY_CONTROLLER, 0, NULL /*TEGRA_BWMGR_SET_EMC_FLOOR*/}, {} }, .can_powergate = true, .autosuspend_delay = 500, .class = NV_SE4_CLASS_ID, .private_data = &tegra18_se_chipdata, .serialize = 1, .push_work_done = 1, .vm_regs = {{SE_STREAMID_REG_OFFSET, true} }, .kernel_only = true, // .bwmgr_client_id = TEGRA_BWMGR_CLIENT_SE4, .prepare_poweroff = tegra_se_nvhost_prepare_poweroff, }; static struct tegra_se_chipdata tegra23_se_chipdata = { .aes_freq = 600000000, .cpu_freq_mhz = 2400, .kac_type = SE_KAC_T23X, }; static struct nvhost_device_data nvhost_t234_se1_info = { .clocks = {{"se", 600000000}, {"emc", UINT_MAX, NVHOST_MODULE_ID_EXTERNAL_MEMORY_CONTROLLER, TEGRA_SET_EMC_FLOOR}, {} }, .can_powergate = true, .autosuspend_delay = 500, .class = NV_SE1_CLASS_ID, .private_data = &tegra23_se_chipdata, .serialize = 1, .push_work_done = 1, .vm_regs = {{SE_STREAMID_REG_OFFSET, true} }, .kernel_only = true, .icc_id = TEGRA_ICC_SE, .prepare_poweroff = tegra_se_nvhost_prepare_poweroff, }; static struct nvhost_device_data nvhost_t234_se2_info = { .clocks = {{"se", 600000000}, {"emc", UINT_MAX, NVHOST_MODULE_ID_EXTERNAL_MEMORY_CONTROLLER, TEGRA_SET_EMC_FLOOR}, {} }, .can_powergate = true, .autosuspend_delay = 500, .class = NV_SE2_CLASS_ID, .private_data = &tegra23_se_chipdata, .serialize = 1, .push_work_done = 1, .vm_regs = {{SE_STREAMID_REG_OFFSET, true} }, .kernel_only = true, .icc_id = TEGRA_ICC_SE, .prepare_poweroff = tegra_se_nvhost_prepare_poweroff, }; static struct nvhost_device_data nvhost_t234_se4_info = { .clocks = {{"se", 600000000}, {"emc", UINT_MAX, NVHOST_MODULE_ID_EXTERNAL_MEMORY_CONTROLLER, TEGRA_SET_EMC_FLOOR}, {} }, .can_powergate = true, .autosuspend_delay = 500, .class = NV_SE4_CLASS_ID, .private_data = &tegra23_se_chipdata, .serialize = 1, .push_work_done = 1, .vm_regs = {{SE_STREAMID_REG_OFFSET, true} }, .kernel_only = true, .icc_id = TEGRA_ICC_SE, .prepare_poweroff = tegra_se_nvhost_prepare_poweroff, }; static const struct of_device_id tegra_se_of_match[] = { { .compatible = "nvidia,tegra186-se1-nvhost", .data = &nvhost_se1_info, }, { .compatible = "nvidia,tegra186-se2-nvhost", .data = &nvhost_se2_info, }, { .compatible = "nvidia,tegra186-se3-nvhost", .data = &nvhost_se3_info, }, { .compatible = "nvidia,tegra186-se4-nvhost", .data = &nvhost_se4_info, }, { .compatible = "nvidia,tegra234-se1-nvhost", .data = &nvhost_t234_se1_info, }, { .compatible = "nvidia,tegra234-se2-nvhost", .data = &nvhost_t234_se2_info, }, { .compatible = "nvidia,tegra234-se4-nvhost", .data = &nvhost_t234_se4_info, }, {} }; MODULE_DEVICE_TABLE(of, tegra_se_of_match); static bool is_algo_supported(struct device_node *node, char *algo) { if (of_property_match_string(node, "supported-algos", algo) >= 0) return true; else return false; } static void tegra_se_fill_se_dev_info(struct tegra_se_dev *se_dev) { struct device_node *node = of_node_get(se_dev->dev->of_node); if (is_algo_supported(node, "aes")) se_devices[SE_AES] = se_dev; if (is_algo_supported(node, "drbg")) se_devices[SE_DRBG] = se_dev; if (is_algo_supported(node, "sha")) se_devices[SE_SHA] = se_dev; if (is_algo_supported(node, "rsa")) se_devices[SE_RSA] = se_dev; if (is_algo_supported(node, "cmac")) se_devices[SE_CMAC] = se_dev; if (is_algo_supported(node, "aead")) se_devices[SE_AEAD] = se_dev; } static int tegra_se_crypto_register(struct tegra_se_dev *se_dev) { struct device_node *node = of_node_get(se_dev->dev->of_node); int i, err; if (is_algo_supported(node, "drbg")) { INIT_LIST_HEAD(&rng_algs[0].base.cra_list); err = crypto_register_rng(&rng_algs[0]); if (err) { dev_err(se_dev->dev, "crypto_register_rng failed\n"); goto reg_fail; } } if (is_algo_supported(node, "xts")) { INIT_LIST_HEAD(&aes_algs[0].base.cra_list); err = crypto_register_skcipher(&aes_algs[0]); if (err) { dev_err(se_dev->dev, "crypto_register_alg xts failed\n"); goto reg_fail; } } if (is_algo_supported(node, "aes")) { for (i = 1; i < ARRAY_SIZE(aes_algs); i++) { INIT_LIST_HEAD(&aes_algs[i].base.cra_list); err = crypto_register_skcipher(&aes_algs[i]); if (err) { dev_err(se_dev->dev, "crypto_register_alg %s failed\n", aes_algs[i].base.cra_name); goto reg_fail; } } } if (is_algo_supported(node, "cmac")) { err = crypto_register_ahash(&hash_algs[0]); if (err) { dev_err(se_dev->dev, "crypto_register_ahash cmac failed\n"); goto reg_fail; } } if (is_algo_supported(node, "sha")) { for (i = 1; i < 6; i++) { err = crypto_register_ahash(&hash_algs[i]); if (err) { dev_err(se_dev->dev, "crypto_register_ahash %s failed\n", hash_algs[i].halg.base.cra_name); goto reg_fail; } } } if (is_algo_supported(node, "sha3")) { for (i = 6; i < 12; i++) { err = crypto_register_ahash(&hash_algs[i]); if (err) { dev_err(se_dev->dev, "crypto_register_ahash %s failed\n", hash_algs[i].halg.base.cra_name); goto reg_fail; } } } if (is_algo_supported(node, "hmac")) { for (i = 12; i < 16; i++) { err = crypto_register_ahash(&hash_algs[i]); if (err) { dev_err(se_dev->dev, "crypto_register_ahash %s failed\n", hash_algs[i].halg.base.cra_name); goto reg_fail; } } } if (is_algo_supported(node, "aead")) { for (i = 0; i < 2; i++) { err = crypto_register_aead(&aead_algs[i]); if (err) { dev_err(se_dev->dev, "crypto_register_aead %s failed\n", aead_algs[i].base.cra_name); goto reg_fail; } } } if (is_algo_supported(node, "rsa")) { err = crypto_register_akcipher(&rsa_alg); if (err) { dev_err(se_dev->dev, "crypto_register_akcipher fail"); goto reg_fail; } err = crypto_register_kpp(&dh_algs[0]); if (err) { dev_err(se_dev->dev, "crypto_register_kpp fail"); goto reg_fail; } se_dev->dh_buf1 = (u32 *)devm_kzalloc( se_dev->dev, TEGRA_SE_RSA2048_INPUT_SIZE, GFP_KERNEL); se_dev->dh_buf2 = (u32 *)devm_kzalloc( se_dev->dev, TEGRA_SE_RSA2048_INPUT_SIZE, GFP_KERNEL); if (!se_dev->dh_buf1 || !se_dev->dh_buf2) goto reg_fail; } reg_fail: return 0; } static int tegra_se_clk_init(struct platform_device *pdev) { struct nvhost_device_data *pdata = platform_get_drvdata(pdev); unsigned int i; int err; err = devm_clk_bulk_get_all(&pdev->dev, &pdata->clks); if (err < 0) { dev_err(&pdev->dev, "failed to get clocks %d\n", err); return err; } pdata->num_clks = err; for (i = 0; i < pdata->num_clks; i++) { err = clk_set_rate(pdata->clks[i].clk, ULONG_MAX); if (err < 0) { dev_err(&pdev->dev, "failed to set clock rate!\n"); return err; } } err = clk_bulk_prepare_enable(pdata->num_clks, pdata->clks); if (err < 0) { dev_err(&pdev->dev, "failed to enabled clocks: %d\n", err); return err; } return 0; } static int tegra_se_client_init(struct host1x_client *client) { return 0; } static const struct host1x_client_ops tegra_se_client_ops = { .init = tegra_se_client_init, }; static int host1x_se_probe(struct host1x_device *dev) { return host1x_device_init(dev); } static int host1x_se_remove(struct host1x_device *dev) { host1x_device_exit(dev); return 0; } static int tegra_se_probe(struct platform_device *pdev) { struct tegra_se_dev *se_dev = NULL; struct nvhost_device_data *pdata = NULL; const struct of_device_id *match; int err = 0; unsigned int val; struct device_node *node = NULL; const char *rsa_name; struct iommu_fwspec *spec; int ret; se_dev = devm_kzalloc(&pdev->dev, sizeof(struct tegra_se_dev), GFP_KERNEL); if (!se_dev) return -ENOMEM; if (pdev->dev.of_node) { match = of_match_device(of_match_ptr(tegra_se_of_match), &pdev->dev); if (!match) { dev_err(&pdev->dev, "Error: No device match found\n"); return -ENODEV; } pdata = (struct nvhost_device_data *)match->data; } else { pdata = (struct nvhost_device_data *)pdev->id_entry->driver_data; } mutex_init(&se_dev->lock); crypto_init_queue(&se_dev->queue, TEGRA_SE_CRYPTO_QUEUE_LENGTH); se_dev->dev = &pdev->dev; se_dev->pdev = pdev; dma_set_mask_and_coherent(se_dev->dev, DMA_BIT_MASK(39)); mutex_init(&pdata->lock); pdata->pdev = pdev; /* store chipdata inside se_dev and store se_dev into private_data */ se_dev->chipdata = pdata->private_data; pdata->private_data = se_dev; /* store the pdata into drvdata */ platform_set_drvdata(pdev, pdata); INIT_LIST_HEAD(&se_dev->client.list); se_dev->client.dev = &pdev->dev; se_dev->client.ops = &tegra_se_client_ops; se_dev->client.class = pdata->class; ret = host1x_client_register(&se_dev->client); err = nvhost_client_device_get_resources(pdev); if (err) { dev_err(se_dev->dev, "nvhost_client_device_get_resources failed for SE(%s)\n", pdev->name); return err; } err = tegra_se_clk_init(pdev); if (err) { dev_err(se_dev->dev, "Enabling clocks failed for SE(%s)\n", pdev->name); return err; } #if 0 err = nvhost_client_device_init(pdev); if (err) { dev_err(se_dev->dev, "nvhost_client_device_init failed for SE(%s)\n", pdev->name); return err; } #endif se_dev->channel = host1x_channel_request(&se_dev->client); if (!se_dev->channel) { dev_err(se_dev->dev, "Nvhost Channel map failed\n"); return err; } se_dev->io_regs = pdata->aperture[0]; node = of_node_get(se_dev->dev->of_node); spec = dev_iommu_fwspec_get(se_dev->dev); if (spec) { se_dev->stream_id = spec->ids[0] & 0xffff; se_writel(se_dev, se_dev->stream_id, SE_STREAMID_REG_OFFSET); } se_dev->ioc = of_property_read_bool(node, "nvidia,io-coherent"); err = of_property_read_u32(node, "opcode_addr", &se_dev->opcode_addr); if (err) { dev_err(se_dev->dev, "Missing opcode_addr property\n"); return err; } if (!of_property_count_strings(node, "supported-algos")) return -ENOTSUPP; tegra_se_fill_se_dev_info(se_dev); if (is_algo_supported(node, "aes") || is_algo_supported(node, "drbg")) { err = tegra_init_key_slot(se_dev); if (err) { dev_err(se_dev->dev, "init_key_slot failed\n"); return err; } } if (is_algo_supported(node, "rsa")) { err = tegra_init_rsa_key_slot(se_dev); if (err) { dev_err(se_dev->dev, "init_rsa_key_slot failed\n"); return err; } } mutex_init(&se_dev->mtx); INIT_WORK(&se_dev->se_work, tegra_se_work_handler); se_dev->se_work_q = alloc_workqueue("se_work_q", WQ_HIGHPRI | WQ_UNBOUND, 1); if (!se_dev->se_work_q) { dev_err(se_dev->dev, "alloc_workqueue failed\n"); return -ENOMEM; } err = tegra_se_alloc_ll_buf(se_dev, SE_MAX_SRC_SG_COUNT, SE_MAX_DST_SG_COUNT); if (err) { dev_err(se_dev->dev, "can not allocate ll dma buffer\n"); goto ll_alloc_fail; } node = of_node_get(se_dev->dev->of_node); se_dev->syncpt_id = nvhost_get_syncpt_host_managed(se_dev->pdev, 0, pdev->name); if (!se_dev->syncpt_id) { err = -EINVAL; dev_err(se_dev->dev, "Cannot get syncpt_id for SE(%s)\n", pdev->name); goto reg_fail; } se_dev->aes_src_ll = devm_kzalloc(&pdev->dev, sizeof(struct tegra_se_ll), GFP_KERNEL); se_dev->aes_dst_ll = devm_kzalloc(&pdev->dev, sizeof(struct tegra_se_ll), GFP_KERNEL); if (!se_dev->aes_src_ll || !se_dev->aes_dst_ll) { dev_err(se_dev->dev, "Linked list memory allocation failed\n"); goto ll_alloc_fail; } if (se_dev->ioc) se_dev->total_aes_buf = dma_alloc_coherent( se_dev->dev, SE_MAX_MEM_ALLOC, &se_dev->total_aes_buf_addr, GFP_KERNEL); else se_dev->total_aes_buf = kzalloc(SE_MAX_MEM_ALLOC, GFP_KERNEL); if (!se_dev->total_aes_buf) { err = -ENOMEM; goto aes_buf_alloc_fail; } tegra_se_init_aesbuf(se_dev); tegra_se_boost_cpu_init(se_dev); err = of_property_read_u32(node, "pka0-rsa-priority", &val); if (!err) rsa_alg.base.cra_priority = val; err = of_property_read_string(node, "pka0-rsa-name", &rsa_name); if (!err) strncpy(rsa_alg.base.cra_name, rsa_name, sizeof(rsa_alg.base.cra_name) - 1); if (is_algo_supported(node, "drbg")) { se_writel(se_dev, SE_RNG_SRC_CONFIG_RO_ENT_SRC(DRBG_RO_ENT_SRC_ENABLE) | SE_RNG_SRC_CONFIG_RO_ENT_SRC_LOCK( DRBG_RO_ENT_SRC_LOCK_ENABLE), SE_RNG_SRC_CONFIG_REG_OFFSET); } tegra_se_crypto_register(se_dev); dev_info(se_dev->dev, "%s: complete", __func__); return 0; aes_buf_alloc_fail: nvhost_syncpt_put_ref_ext(se_dev->pdev, se_dev->syncpt_id); reg_fail: tegra_se_free_ll_buf(se_dev); ll_alloc_fail: if (se_dev->se_work_q) destroy_workqueue(se_dev->se_work_q); return err; } static int tegra_se_remove(struct platform_device *pdev) { struct nvhost_device_data *pdata = platform_get_drvdata(pdev); struct tegra_se_dev *se_dev = pdata->private_data; struct device_node *node; int i; if (!se_dev) { pr_err("Device is NULL\n"); return -ENODEV; } tegra_se_boost_cpu_deinit(se_dev); node = of_node_get(se_dev->dev->of_node); if (is_algo_supported(node, "drbg")) crypto_unregister_rng(&rng_algs[0]); if (is_algo_supported(node, "xts")) crypto_unregister_skcipher(&aes_algs[0]); if (is_algo_supported(node, "aes")) { crypto_unregister_skcipher(&aes_algs[1]); for (i = 2; i < ARRAY_SIZE(aes_algs); i++) crypto_unregister_skcipher(&aes_algs[i]); } if (is_algo_supported(node, "cmac")) crypto_unregister_ahash(&hash_algs[0]); if (is_algo_supported(node, "sha")) { for (i = 1; i < 6; i++) crypto_unregister_ahash(&hash_algs[i]); } if (is_algo_supported(node, "rsa")) { crypto_unregister_akcipher(&rsa_alg); crypto_unregister_kpp(&dh_algs[0]); } if (is_algo_supported(node, "aead")) { for (i = 0; i < 2; i++) crypto_unregister_aead(&aead_algs[i]); } tegra_se_free_ll_buf(se_dev); kfree(se_dev->total_aes_buf); cancel_work_sync(&se_dev->se_work); if (se_dev->se_work_q) destroy_workqueue(se_dev->se_work_q); mutex_destroy(&se_dev->mtx); nvhost_client_device_release(pdev); mutex_destroy(&pdata->lock); return 0; } static struct host1x_driver host1x_se_driver = { .driver = { .name = "tegra-se-host1x", }, .probe = host1x_se_probe, .remove = host1x_se_remove, .subdevs = tegra_se_of_match, }; static struct platform_driver tegra_se_driver = { .probe = tegra_se_probe, .remove = tegra_se_remove, .driver = { .name = "tegra-se-nvhost", .owner = THIS_MODULE, .of_match_table = of_match_ptr(tegra_se_of_match), .suppress_bind_attrs = true, }, }; static int __init tegra_se_module_init(void) { int err; err = host1x_driver_register(&host1x_se_driver); if (err < 0) return err; err = platform_driver_register(&tegra_se_driver); if (err < 0) { host1x_driver_unregister(&host1x_se_driver); return err; } return 0; } static void __exit tegra_se_module_exit(void) { platform_driver_unregister(&tegra_se_driver); } late_initcall(tegra_se_module_init); module_exit(tegra_se_module_exit); MODULE_DESCRIPTION("Tegra Crypto algorithm support using Host1x Interface"); MODULE_AUTHOR("NVIDIA Corporation"); MODULE_LICENSE("GPL"); MODULE_ALIAS("tegra-se-nvhost");