| /* |
| * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api. |
| * |
| * Copyright (C) 2014-2017 Axis Communications AB |
| */ |
| #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| |
| #include <linux/bitfield.h> |
| #include <linux/crypto.h> |
| #include <linux/debugfs.h> |
| #include <linux/delay.h> |
| #include <linux/dma-mapping.h> |
| #include <linux/fault-inject.h> |
| #include <linux/init.h> |
| #include <linux/interrupt.h> |
| #include <linux/kernel.h> |
| #include <linux/list.h> |
| #include <linux/module.h> |
| #include <linux/of.h> |
| #include <linux/platform_device.h> |
| #include <linux/scatterlist.h> |
| #include <linux/slab.h> |
| |
| #include <crypto/aes.h> |
| #include <crypto/gcm.h> |
| #include <crypto/internal/aead.h> |
| #include <crypto/internal/hash.h> |
| #include <crypto/internal/skcipher.h> |
| #include <crypto/scatterwalk.h> |
| #include <crypto/sha.h> |
| #include <crypto/xts.h> |
| |
| /* Max length of a line in all cache levels for Artpec SoCs. */ |
| #define ARTPEC_CACHE_LINE_MAX 32 |
| |
| #define PDMA_OUT_CFG 0x0000 |
| #define PDMA_OUT_BUF_CFG 0x0004 |
| #define PDMA_OUT_CMD 0x0008 |
| #define PDMA_OUT_DESCRQ_PUSH 0x0010 |
| #define PDMA_OUT_DESCRQ_STAT 0x0014 |
| |
| #define A6_PDMA_IN_CFG 0x0028 |
| #define A6_PDMA_IN_BUF_CFG 0x002c |
| #define A6_PDMA_IN_CMD 0x0030 |
| #define A6_PDMA_IN_STATQ_PUSH 0x0038 |
| #define A6_PDMA_IN_DESCRQ_PUSH 0x0044 |
| #define A6_PDMA_IN_DESCRQ_STAT 0x0048 |
| #define A6_PDMA_INTR_MASK 0x0068 |
| #define A6_PDMA_ACK_INTR 0x006c |
| #define A6_PDMA_MASKED_INTR 0x0074 |
| |
| #define A7_PDMA_IN_CFG 0x002c |
| #define A7_PDMA_IN_BUF_CFG 0x0030 |
| #define A7_PDMA_IN_CMD 0x0034 |
| #define A7_PDMA_IN_STATQ_PUSH 0x003c |
| #define A7_PDMA_IN_DESCRQ_PUSH 0x0048 |
| #define A7_PDMA_IN_DESCRQ_STAT 0x004C |
| #define A7_PDMA_INTR_MASK 0x006c |
| #define A7_PDMA_ACK_INTR 0x0070 |
| #define A7_PDMA_MASKED_INTR 0x0078 |
| |
| #define PDMA_OUT_CFG_EN BIT(0) |
| |
| #define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) |
| #define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) |
| |
| #define PDMA_OUT_CMD_START BIT(0) |
| #define A6_PDMA_OUT_CMD_STOP BIT(3) |
| #define A7_PDMA_OUT_CMD_STOP BIT(2) |
| |
| #define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0) |
| #define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6) |
| |
| #define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0) |
| #define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4) |
| |
| #define PDMA_IN_CFG_EN BIT(0) |
| |
| #define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) |
| #define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) |
| #define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10) |
| |
| #define PDMA_IN_CMD_START BIT(0) |
| #define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2) |
| #define A6_PDMA_IN_CMD_STOP BIT(3) |
| #define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1) |
| #define A7_PDMA_IN_CMD_STOP BIT(2) |
| |
| #define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0) |
| #define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6) |
| |
| #define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0) |
| #define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6) |
| |
| #define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0) |
| #define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4) |
| |
| #define A6_PDMA_INTR_MASK_IN_DATA BIT(2) |
| #define A6_PDMA_INTR_MASK_IN_EOP BIT(3) |
| #define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4) |
| |
| #define A7_PDMA_INTR_MASK_IN_DATA BIT(3) |
| #define A7_PDMA_INTR_MASK_IN_EOP BIT(4) |
| #define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5) |
| |
| #define A6_CRY_MD_OPER GENMASK(19, 16) |
| |
| #define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20) |
| #define A6_CRY_MD_HASH_HMAC_FIN BIT(23) |
| |
| #define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20) |
| #define A6_CRY_MD_CIPHER_DECR BIT(22) |
| #define A6_CRY_MD_CIPHER_TWEAK BIT(23) |
| #define A6_CRY_MD_CIPHER_DSEQ BIT(24) |
| |
| #define A7_CRY_MD_OPER GENMASK(11, 8) |
| |
| #define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12) |
| #define A7_CRY_MD_HASH_HMAC_FIN BIT(15) |
| |
| #define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12) |
| #define A7_CRY_MD_CIPHER_DECR BIT(14) |
| #define A7_CRY_MD_CIPHER_TWEAK BIT(15) |
| #define A7_CRY_MD_CIPHER_DSEQ BIT(16) |
| |
| /* DMA metadata constants */ |
| #define regk_crypto_aes_cbc 0x00000002 |
| #define regk_crypto_aes_ctr 0x00000003 |
| #define regk_crypto_aes_ecb 0x00000001 |
| #define regk_crypto_aes_gcm 0x00000004 |
| #define regk_crypto_aes_xts 0x00000005 |
| #define regk_crypto_cache 0x00000002 |
| #define a6_regk_crypto_dlkey 0x0000000a |
| #define a7_regk_crypto_dlkey 0x0000000e |
| #define regk_crypto_ext 0x00000001 |
| #define regk_crypto_hmac_sha1 0x00000007 |
| #define regk_crypto_hmac_sha256 0x00000009 |
| #define regk_crypto_hmac_sha384 0x0000000b |
| #define regk_crypto_hmac_sha512 0x0000000d |
| #define regk_crypto_init 0x00000000 |
| #define regk_crypto_key_128 0x00000000 |
| #define regk_crypto_key_192 0x00000001 |
| #define regk_crypto_key_256 0x00000002 |
| #define regk_crypto_null 0x00000000 |
| #define regk_crypto_sha1 0x00000006 |
| #define regk_crypto_sha256 0x00000008 |
| #define regk_crypto_sha384 0x0000000a |
| #define regk_crypto_sha512 0x0000000c |
| |
| /* DMA descriptor structures */ |
| struct pdma_descr_ctrl { |
| unsigned char short_descr : 1; |
| unsigned char pad1 : 1; |
| unsigned char eop : 1; |
| unsigned char intr : 1; |
| unsigned char short_len : 3; |
| unsigned char pad2 : 1; |
| } __packed; |
| |
| struct pdma_data_descr { |
| unsigned int len : 24; |
| unsigned int buf : 32; |
| } __packed; |
| |
| struct pdma_short_descr { |
| unsigned char data[7]; |
| } __packed; |
| |
| struct pdma_descr { |
| struct pdma_descr_ctrl ctrl; |
| union { |
| struct pdma_data_descr data; |
| struct pdma_short_descr shrt; |
| }; |
| }; |
| |
| struct pdma_stat_descr { |
| unsigned char pad1 : 1; |
| unsigned char pad2 : 1; |
| unsigned char eop : 1; |
| unsigned char pad3 : 5; |
| unsigned int len : 24; |
| }; |
| |
| /* Each descriptor array can hold max 64 entries */ |
| #define PDMA_DESCR_COUNT 64 |
| |
| #define MODULE_NAME "Artpec-6 CA" |
| |
| /* Hash modes (including HMAC variants) */ |
| #define ARTPEC6_CRYPTO_HASH_SHA1 1 |
| #define ARTPEC6_CRYPTO_HASH_SHA256 2 |
| #define ARTPEC6_CRYPTO_HASH_SHA384 3 |
| #define ARTPEC6_CRYPTO_HASH_SHA512 4 |
| |
| /* Crypto modes */ |
| #define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1 |
| #define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2 |
| #define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3 |
| #define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5 |
| |
| /* The PDMA is a DMA-engine tightly coupled with a ciphering engine. |
| * It operates on a descriptor array with up to 64 descriptor entries. |
| * The arrays must be 64 byte aligned in memory. |
| * |
| * The ciphering unit has no registers and is completely controlled by |
| * a 4-byte metadata that is inserted at the beginning of each dma packet. |
| * |
| * A dma packet is a sequence of descriptors terminated by setting the .eop |
| * field in the final descriptor of the packet. |
| * |
| * Multiple packets are used for providing context data, key data and |
| * the plain/ciphertext. |
| * |
| * PDMA Descriptors (Array) |
| * +------+------+------+~~+-------+------+---- |
| * | 0 | 1 | 2 |~~| 11 EOP| 12 | .... |
| * +--+---+--+---+----+-+~~+-------+----+-+---- |
| * | | | | | |
| * | | | | | |
| * __|__ +-------++-------++-------+ +----+ |
| * | MD | |Payload||Payload||Payload| | MD | |
| * +-----+ +-------++-------++-------+ +----+ |
| */ |
| |
| struct artpec6_crypto_bounce_buffer { |
| struct list_head list; |
| size_t length; |
| struct scatterlist *sg; |
| size_t offset; |
| /* buf is aligned to ARTPEC_CACHE_LINE_MAX and |
| * holds up to ARTPEC_CACHE_LINE_MAX bytes data. |
| */ |
| void *buf; |
| }; |
| |
| struct artpec6_crypto_dma_map { |
| dma_addr_t dma_addr; |
| size_t size; |
| enum dma_data_direction dir; |
| }; |
| |
| struct artpec6_crypto_dma_descriptors { |
| struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64); |
| struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64); |
| u32 stat[PDMA_DESCR_COUNT] __aligned(64); |
| struct list_head bounce_buffers; |
| /* Enough maps for all out/in buffers, and all three descr. arrays */ |
| struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2]; |
| dma_addr_t out_dma_addr; |
| dma_addr_t in_dma_addr; |
| dma_addr_t stat_dma_addr; |
| size_t out_cnt; |
| size_t in_cnt; |
| size_t map_count; |
| }; |
| |
| enum artpec6_crypto_variant { |
| ARTPEC6_CRYPTO, |
| ARTPEC7_CRYPTO, |
| }; |
| |
| struct artpec6_crypto { |
| void __iomem *base; |
| spinlock_t queue_lock; |
| struct list_head queue; /* waiting for pdma fifo space */ |
| struct list_head pending; /* submitted to pdma fifo */ |
| struct tasklet_struct task; |
| struct kmem_cache *dma_cache; |
| int pending_count; |
| struct timer_list timer; |
| enum artpec6_crypto_variant variant; |
| void *pad_buffer; /* cache-aligned block padding buffer */ |
| void *zero_buffer; |
| }; |
| |
| enum artpec6_crypto_hash_flags { |
| HASH_FLAG_INIT_CTX = 2, |
| HASH_FLAG_UPDATE = 4, |
| HASH_FLAG_FINALIZE = 8, |
| HASH_FLAG_HMAC = 16, |
| HASH_FLAG_UPDATE_KEY = 32, |
| }; |
| |
| struct artpec6_crypto_req_common { |
| struct list_head list; |
| struct list_head complete_in_progress; |
| struct artpec6_crypto_dma_descriptors *dma; |
| struct crypto_async_request *req; |
| void (*complete)(struct crypto_async_request *req); |
| gfp_t gfp_flags; |
| }; |
| |
| struct artpec6_hash_request_context { |
| char partial_buffer[SHA512_BLOCK_SIZE]; |
| char partial_buffer_out[SHA512_BLOCK_SIZE]; |
| char key_buffer[SHA512_BLOCK_SIZE]; |
| char pad_buffer[SHA512_BLOCK_SIZE + 32]; |
| unsigned char digeststate[SHA512_DIGEST_SIZE]; |
| size_t partial_bytes; |
| u64 digcnt; |
| u32 key_md; |
| u32 hash_md; |
| enum artpec6_crypto_hash_flags hash_flags; |
| struct artpec6_crypto_req_common common; |
| }; |
| |
| struct artpec6_hash_export_state { |
| char partial_buffer[SHA512_BLOCK_SIZE]; |
| unsigned char digeststate[SHA512_DIGEST_SIZE]; |
| size_t partial_bytes; |
| u64 digcnt; |
| int oper; |
| unsigned int hash_flags; |
| }; |
| |
| struct artpec6_hashalg_context { |
| char hmac_key[SHA512_BLOCK_SIZE]; |
| size_t hmac_key_length; |
| struct crypto_shash *child_hash; |
| }; |
| |
| struct artpec6_crypto_request_context { |
| u32 cipher_md; |
| bool decrypt; |
| struct artpec6_crypto_req_common common; |
| }; |
| |
| struct artpec6_cryptotfm_context { |
| unsigned char aes_key[2*AES_MAX_KEY_SIZE]; |
| size_t key_length; |
| u32 key_md; |
| int crypto_type; |
| struct crypto_skcipher *fallback; |
| }; |
| |
| struct artpec6_crypto_aead_hw_ctx { |
| __be64 aad_length_bits; |
| __be64 text_length_bits; |
| __u8 J0[AES_BLOCK_SIZE]; |
| }; |
| |
| struct artpec6_crypto_aead_req_ctx { |
| struct artpec6_crypto_aead_hw_ctx hw_ctx; |
| u32 cipher_md; |
| bool decrypt; |
| struct artpec6_crypto_req_common common; |
| __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned; |
| }; |
| |
| /* The crypto framework makes it hard to avoid this global. */ |
| static struct device *artpec6_crypto_dev; |
| |
| #ifdef CONFIG_FAULT_INJECTION |
| static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read); |
| static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full); |
| #endif |
| |
| enum { |
| ARTPEC6_CRYPTO_PREPARE_HASH_NO_START, |
| ARTPEC6_CRYPTO_PREPARE_HASH_START, |
| }; |
| |
| static int artpec6_crypto_prepare_aead(struct aead_request *areq); |
| static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq); |
| static int artpec6_crypto_prepare_hash(struct ahash_request *areq); |
| |
| static void |
| artpec6_crypto_complete_crypto(struct crypto_async_request *req); |
| static void |
| artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req); |
| static void |
| artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req); |
| static void |
| artpec6_crypto_complete_aead(struct crypto_async_request *req); |
| static void |
| artpec6_crypto_complete_hash(struct crypto_async_request *req); |
| |
| static int |
| artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common); |
| |
| static void |
| artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common); |
| |
| struct artpec6_crypto_walk { |
| struct scatterlist *sg; |
| size_t offset; |
| }; |
| |
| static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk, |
| struct scatterlist *sg) |
| { |
| awalk->sg = sg; |
| awalk->offset = 0; |
| } |
| |
| static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk, |
| size_t nbytes) |
| { |
| while (nbytes && awalk->sg) { |
| size_t piece; |
| |
| WARN_ON(awalk->offset > awalk->sg->length); |
| |
| piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); |
| nbytes -= piece; |
| awalk->offset += piece; |
| if (awalk->offset == awalk->sg->length) { |
| awalk->sg = sg_next(awalk->sg); |
| awalk->offset = 0; |
| } |
| |
| } |
| |
| return nbytes; |
| } |
| |
| static size_t |
| artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk) |
| { |
| WARN_ON(awalk->sg->length == awalk->offset); |
| |
| return awalk->sg->length - awalk->offset; |
| } |
| |
| static dma_addr_t |
| artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk) |
| { |
| return sg_phys(awalk->sg) + awalk->offset; |
| } |
| |
| static void |
| artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| struct artpec6_crypto_bounce_buffer *b; |
| struct artpec6_crypto_bounce_buffer *next; |
| |
| list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { |
| pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n", |
| b, b->length, b->offset, b->buf); |
| sg_pcopy_from_buffer(b->sg, |
| 1, |
| b->buf, |
| b->length, |
| b->offset); |
| |
| list_del(&b->list); |
| kfree(b); |
| } |
| } |
| |
| static inline bool artpec6_crypto_busy(void) |
| { |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| int fifo_count = ac->pending_count; |
| |
| return fifo_count > 6; |
| } |
| |
| static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req) |
| { |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| int ret = -EBUSY; |
| |
| spin_lock_bh(&ac->queue_lock); |
| |
| if (!artpec6_crypto_busy()) { |
| list_add_tail(&req->list, &ac->pending); |
| artpec6_crypto_start_dma(req); |
| ret = -EINPROGRESS; |
| } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { |
| list_add_tail(&req->list, &ac->queue); |
| } else { |
| artpec6_crypto_common_destroy(req); |
| } |
| |
| spin_unlock_bh(&ac->queue_lock); |
| |
| return ret; |
| } |
| |
| static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common) |
| { |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| enum artpec6_crypto_variant variant = ac->variant; |
| void __iomem *base = ac->base; |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| u32 ind, statd, outd; |
| |
| /* Make descriptor content visible to the DMA before starting it. */ |
| wmb(); |
| |
| ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | |
| FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); |
| |
| statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | |
| FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); |
| |
| outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | |
| FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); |
| |
| if (variant == ARTPEC6_CRYPTO) { |
| writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH); |
| writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH); |
| writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD); |
| } else { |
| writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH); |
| writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH); |
| writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD); |
| } |
| |
| writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH); |
| writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD); |
| |
| ac->pending_count++; |
| } |
| |
| static void |
| artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| |
| dma->out_cnt = 0; |
| dma->in_cnt = 0; |
| dma->map_count = 0; |
| INIT_LIST_HEAD(&dma->bounce_buffers); |
| } |
| |
| static bool fault_inject_dma_descr(void) |
| { |
| #ifdef CONFIG_FAULT_INJECTION |
| return should_fail(&artpec6_crypto_fail_dma_array_full, 1); |
| #else |
| return false; |
| #endif |
| } |
| |
| /** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a |
| * physical address |
| * |
| * @addr: The physical address of the data buffer |
| * @len: The length of the data buffer |
| * @eop: True if this is the last buffer in the packet |
| * |
| * @return 0 on success or -ENOSPC if there are no more descriptors available |
| */ |
| static int |
| artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common, |
| dma_addr_t addr, size_t len, bool eop) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| struct pdma_descr *d; |
| |
| if (dma->out_cnt >= PDMA_DESCR_COUNT || |
| fault_inject_dma_descr()) { |
| pr_err("No free OUT DMA descriptors available!\n"); |
| return -ENOSPC; |
| } |
| |
| d = &dma->out[dma->out_cnt++]; |
| memset(d, 0, sizeof(*d)); |
| |
| d->ctrl.short_descr = 0; |
| d->ctrl.eop = eop; |
| d->data.len = len; |
| d->data.buf = addr; |
| return 0; |
| } |
| |
| /** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor |
| * |
| * @dst: The virtual address of the data |
| * @len: The length of the data, must be between 1 to 7 bytes |
| * @eop: True if this is the last buffer in the packet |
| * |
| * @return 0 on success |
| * -ENOSPC if no more descriptors are available |
| * -EINVAL if the data length exceeds 7 bytes |
| */ |
| static int |
| artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common, |
| void *dst, unsigned int len, bool eop) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| struct pdma_descr *d; |
| |
| if (dma->out_cnt >= PDMA_DESCR_COUNT || |
| fault_inject_dma_descr()) { |
| pr_err("No free OUT DMA descriptors available!\n"); |
| return -ENOSPC; |
| } else if (len > 7 || len < 1) { |
| return -EINVAL; |
| } |
| d = &dma->out[dma->out_cnt++]; |
| memset(d, 0, sizeof(*d)); |
| |
| d->ctrl.short_descr = 1; |
| d->ctrl.short_len = len; |
| d->ctrl.eop = eop; |
| memcpy(d->shrt.data, dst, len); |
| return 0; |
| } |
| |
| static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common, |
| struct page *page, size_t offset, |
| size_t size, |
| enum dma_data_direction dir, |
| dma_addr_t *dma_addr_out) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| struct device *dev = artpec6_crypto_dev; |
| struct artpec6_crypto_dma_map *map; |
| dma_addr_t dma_addr; |
| |
| *dma_addr_out = 0; |
| |
| if (dma->map_count >= ARRAY_SIZE(dma->maps)) |
| return -ENOMEM; |
| |
| dma_addr = dma_map_page(dev, page, offset, size, dir); |
| if (dma_mapping_error(dev, dma_addr)) |
| return -ENOMEM; |
| |
| map = &dma->maps[dma->map_count++]; |
| map->size = size; |
| map->dma_addr = dma_addr; |
| map->dir = dir; |
| |
| *dma_addr_out = dma_addr; |
| |
| return 0; |
| } |
| |
| static int |
| artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common, |
| void *ptr, size_t size, |
| enum dma_data_direction dir, |
| dma_addr_t *dma_addr_out) |
| { |
| struct page *page = virt_to_page(ptr); |
| size_t offset = (uintptr_t)ptr & ~PAGE_MASK; |
| |
| return artpec6_crypto_dma_map_page(common, page, offset, size, dir, |
| dma_addr_out); |
| } |
| |
| static int |
| artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| int ret; |
| |
| ret = artpec6_crypto_dma_map_single(common, dma->in, |
| sizeof(dma->in[0]) * dma->in_cnt, |
| DMA_TO_DEVICE, &dma->in_dma_addr); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_dma_map_single(common, dma->out, |
| sizeof(dma->out[0]) * dma->out_cnt, |
| DMA_TO_DEVICE, &dma->out_dma_addr); |
| if (ret) |
| return ret; |
| |
| /* We only read one stat descriptor */ |
| dma->stat[dma->in_cnt - 1] = 0; |
| |
| /* |
| * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor |
| * to be written. |
| */ |
| return artpec6_crypto_dma_map_single(common, |
| dma->stat + dma->in_cnt - 1, |
| sizeof(dma->stat[0]), |
| DMA_BIDIRECTIONAL, |
| &dma->stat_dma_addr); |
| } |
| |
| static void |
| artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| struct device *dev = artpec6_crypto_dev; |
| int i; |
| |
| for (i = 0; i < dma->map_count; i++) { |
| struct artpec6_crypto_dma_map *map = &dma->maps[i]; |
| |
| dma_unmap_page(dev, map->dma_addr, map->size, map->dir); |
| } |
| |
| dma->map_count = 0; |
| } |
| |
| /** artpec6_crypto_setup_out_descr - Setup an out descriptor |
| * |
| * @dst: The virtual address of the data |
| * @len: The length of the data |
| * @eop: True if this is the last buffer in the packet |
| * @use_short: If this is true and the data length is 7 bytes or less then |
| * a short descriptor will be used |
| * |
| * @return 0 on success |
| * Any errors from artpec6_crypto_setup_out_descr_short() or |
| * setup_out_descr_phys() |
| */ |
| static int |
| artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, |
| void *dst, unsigned int len, bool eop, |
| bool use_short) |
| { |
| if (use_short && len < 7) { |
| return artpec6_crypto_setup_out_descr_short(common, dst, len, |
| eop); |
| } else { |
| int ret; |
| dma_addr_t dma_addr; |
| |
| ret = artpec6_crypto_dma_map_single(common, dst, len, |
| DMA_TO_DEVICE, |
| &dma_addr); |
| if (ret) |
| return ret; |
| |
| return artpec6_crypto_setup_out_descr_phys(common, dma_addr, |
| len, eop); |
| } |
| } |
| |
| /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a |
| * physical address |
| * |
| * @addr: The physical address of the data buffer |
| * @len: The length of the data buffer |
| * @intr: True if an interrupt should be fired after HW processing of this |
| * descriptor |
| * |
| */ |
| static int |
| artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common, |
| dma_addr_t addr, unsigned int len, bool intr) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| struct pdma_descr *d; |
| |
| if (dma->in_cnt >= PDMA_DESCR_COUNT || |
| fault_inject_dma_descr()) { |
| pr_err("No free IN DMA descriptors available!\n"); |
| return -ENOSPC; |
| } |
| d = &dma->in[dma->in_cnt++]; |
| memset(d, 0, sizeof(*d)); |
| |
| d->ctrl.intr = intr; |
| d->data.len = len; |
| d->data.buf = addr; |
| return 0; |
| } |
| |
| /** artpec6_crypto_setup_in_descr - Setup an in channel descriptor |
| * |
| * @buffer: The virtual address to of the data buffer |
| * @len: The length of the data buffer |
| * @last: If this is the last data buffer in the request (i.e. an interrupt |
| * is needed |
| * |
| * Short descriptors are not used for the in channel |
| */ |
| static int |
| artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common, |
| void *buffer, unsigned int len, bool last) |
| { |
| dma_addr_t dma_addr; |
| int ret; |
| |
| ret = artpec6_crypto_dma_map_single(common, buffer, len, |
| DMA_FROM_DEVICE, &dma_addr); |
| if (ret) |
| return ret; |
| |
| return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last); |
| } |
| |
| static struct artpec6_crypto_bounce_buffer * |
| artpec6_crypto_alloc_bounce(gfp_t flags) |
| { |
| void *base; |
| size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) + |
| 2 * ARTPEC_CACHE_LINE_MAX; |
| struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags); |
| |
| if (!bbuf) |
| return NULL; |
| |
| base = bbuf + 1; |
| bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); |
| return bbuf; |
| } |
| |
| static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common, |
| struct artpec6_crypto_walk *walk, size_t size) |
| { |
| struct artpec6_crypto_bounce_buffer *bbuf; |
| int ret; |
| |
| bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); |
| if (!bbuf) |
| return -ENOMEM; |
| |
| bbuf->length = size; |
| bbuf->sg = walk->sg; |
| bbuf->offset = walk->offset; |
| |
| ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); |
| if (ret) { |
| kfree(bbuf); |
| return ret; |
| } |
| |
| pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); |
| list_add_tail(&bbuf->list, &common->dma->bounce_buffers); |
| return 0; |
| } |
| |
| static int |
| artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common, |
| struct artpec6_crypto_walk *walk, |
| size_t count) |
| { |
| size_t chunk; |
| int ret; |
| dma_addr_t addr; |
| |
| while (walk->sg && count) { |
| chunk = min(count, artpec6_crypto_walk_chunklen(walk)); |
| addr = artpec6_crypto_walk_chunk_phys(walk); |
| |
| /* When destination buffers are not aligned to the cache line |
| * size we need bounce buffers. The DMA-API requires that the |
| * entire line is owned by the DMA buffer and this holds also |
| * for the case when coherent DMA is used. |
| */ |
| if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) { |
| chunk = min_t(dma_addr_t, chunk, |
| ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - |
| addr); |
| |
| pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); |
| ret = setup_bounce_buffer_in(common, walk, chunk); |
| } else if (chunk < ARTPEC_CACHE_LINE_MAX) { |
| pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); |
| ret = setup_bounce_buffer_in(common, walk, chunk); |
| } else { |
| dma_addr_t dma_addr; |
| |
| chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); |
| |
| pr_debug("CHUNK %pad:%zu\n", &addr, chunk); |
| |
| ret = artpec6_crypto_dma_map_page(common, |
| sg_page(walk->sg), |
| walk->sg->offset + |
| walk->offset, |
| chunk, |
| DMA_FROM_DEVICE, |
| &dma_addr); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_setup_in_descr_phys(common, |
| dma_addr, |
| chunk, false); |
| } |
| |
| if (ret) |
| return ret; |
| |
| count = count - chunk; |
| artpec6_crypto_walk_advance(walk, chunk); |
| } |
| |
| if (count) |
| pr_err("EOL unexpected %zu bytes left\n", count); |
| |
| return count ? -EINVAL : 0; |
| } |
| |
| static int |
| artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common, |
| struct artpec6_crypto_walk *walk, |
| size_t count) |
| { |
| size_t chunk; |
| int ret; |
| dma_addr_t addr; |
| |
| while (walk->sg && count) { |
| chunk = min(count, artpec6_crypto_walk_chunklen(walk)); |
| addr = artpec6_crypto_walk_chunk_phys(walk); |
| |
| pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); |
| |
| if (addr & 3) { |
| char buf[3]; |
| |
| chunk = min_t(size_t, chunk, (4-(addr&3))); |
| |
| sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, |
| walk->offset); |
| |
| ret = artpec6_crypto_setup_out_descr_short(common, buf, |
| chunk, |
| false); |
| } else { |
| dma_addr_t dma_addr; |
| |
| ret = artpec6_crypto_dma_map_page(common, |
| sg_page(walk->sg), |
| walk->sg->offset + |
| walk->offset, |
| chunk, |
| DMA_TO_DEVICE, |
| &dma_addr); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_setup_out_descr_phys(common, |
| dma_addr, |
| chunk, false); |
| } |
| |
| if (ret) |
| return ret; |
| |
| count = count - chunk; |
| artpec6_crypto_walk_advance(walk, chunk); |
| } |
| |
| if (count) |
| pr_err("EOL unexpected %zu bytes left\n", count); |
| |
| return count ? -EINVAL : 0; |
| } |
| |
| |
| /** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor |
| * |
| * If the out descriptor list is non-empty, then the eop flag on the |
| * last used out descriptor will be set. |
| * |
| * @return 0 on success |
| * -EINVAL if the out descriptor is empty or has overflown |
| */ |
| static int |
| artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| struct pdma_descr *d; |
| |
| if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { |
| pr_err("%s: OUT descriptor list is %s\n", |
| MODULE_NAME, dma->out_cnt ? "empty" : "full"); |
| return -EINVAL; |
| |
| } |
| |
| d = &dma->out[dma->out_cnt-1]; |
| d->ctrl.eop = 1; |
| |
| return 0; |
| } |
| |
| /** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last |
| * in descriptor |
| * |
| * See artpec6_crypto_terminate_out_descrs() for return values |
| */ |
| static int |
| artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common) |
| { |
| struct artpec6_crypto_dma_descriptors *dma = common->dma; |
| struct pdma_descr *d; |
| |
| if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { |
| pr_err("%s: IN descriptor list is %s\n", |
| MODULE_NAME, dma->in_cnt ? "empty" : "full"); |
| return -EINVAL; |
| } |
| |
| d = &dma->in[dma->in_cnt-1]; |
| d->ctrl.intr = 1; |
| return 0; |
| } |
| |
| /** create_hash_pad - Create a Secure Hash conformant pad |
| * |
| * @dst: The destination buffer to write the pad. Must be at least 64 bytes |
| * @dgstlen: The total length of the hash digest in bytes |
| * @bitcount: The total length of the digest in bits |
| * |
| * @return The total number of padding bytes written to @dst |
| */ |
| static size_t |
| create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount) |
| { |
| unsigned int mod, target, diff, pad_bytes, size_bytes; |
| __be64 bits = __cpu_to_be64(bitcount); |
| |
| switch (oper) { |
| case regk_crypto_sha1: |
| case regk_crypto_sha256: |
| case regk_crypto_hmac_sha1: |
| case regk_crypto_hmac_sha256: |
| target = 448 / 8; |
| mod = 512 / 8; |
| size_bytes = 8; |
| break; |
| default: |
| target = 896 / 8; |
| mod = 1024 / 8; |
| size_bytes = 16; |
| break; |
| } |
| |
| target -= 1; |
| diff = dgstlen & (mod - 1); |
| pad_bytes = diff > target ? target + mod - diff : target - diff; |
| |
| memset(dst + 1, 0, pad_bytes); |
| dst[0] = 0x80; |
| |
| if (size_bytes == 16) { |
| memset(dst + 1 + pad_bytes, 0, 8); |
| memcpy(dst + 1 + pad_bytes + 8, &bits, 8); |
| } else { |
| memcpy(dst + 1 + pad_bytes, &bits, 8); |
| } |
| |
| return pad_bytes + size_bytes + 1; |
| } |
| |
| static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common, |
| struct crypto_async_request *parent, |
| void (*complete)(struct crypto_async_request *req), |
| struct scatterlist *dstsg, unsigned int nbytes) |
| { |
| gfp_t flags; |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| |
| flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? |
| GFP_KERNEL : GFP_ATOMIC; |
| |
| common->gfp_flags = flags; |
| common->dma = kmem_cache_alloc(ac->dma_cache, flags); |
| if (!common->dma) |
| return -ENOMEM; |
| |
| common->req = parent; |
| common->complete = complete; |
| return 0; |
| } |
| |
| static void |
| artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma) |
| { |
| struct artpec6_crypto_bounce_buffer *b; |
| struct artpec6_crypto_bounce_buffer *next; |
| |
| list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { |
| kfree(b); |
| } |
| } |
| |
| static int |
| artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common) |
| { |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| |
| artpec6_crypto_dma_unmap_all(common); |
| artpec6_crypto_bounce_destroy(common->dma); |
| kmem_cache_free(ac->dma_cache, common->dma); |
| common->dma = NULL; |
| return 0; |
| } |
| |
| /* |
| * Ciphering functions. |
| */ |
| static int artpec6_crypto_encrypt(struct skcipher_request *req) |
| { |
| struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); |
| struct artpec6_crypto_request_context *req_ctx = NULL; |
| void (*complete)(struct crypto_async_request *req); |
| int ret; |
| |
| req_ctx = skcipher_request_ctx(req); |
| |
| switch (ctx->crypto_type) { |
| case ARTPEC6_CRYPTO_CIPHER_AES_CBC: |
| case ARTPEC6_CRYPTO_CIPHER_AES_ECB: |
| case ARTPEC6_CRYPTO_CIPHER_AES_XTS: |
| req_ctx->decrypt = 0; |
| break; |
| default: |
| break; |
| } |
| |
| switch (ctx->crypto_type) { |
| case ARTPEC6_CRYPTO_CIPHER_AES_CBC: |
| complete = artpec6_crypto_complete_cbc_encrypt; |
| break; |
| default: |
| complete = artpec6_crypto_complete_crypto; |
| break; |
| } |
| |
| ret = artpec6_crypto_common_init(&req_ctx->common, |
| &req->base, |
| complete, |
| req->dst, req->cryptlen); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_prepare_crypto(req); |
| if (ret) { |
| artpec6_crypto_common_destroy(&req_ctx->common); |
| return ret; |
| } |
| |
| return artpec6_crypto_submit(&req_ctx->common); |
| } |
| |
| static int artpec6_crypto_decrypt(struct skcipher_request *req) |
| { |
| int ret; |
| struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); |
| struct artpec6_crypto_request_context *req_ctx = NULL; |
| void (*complete)(struct crypto_async_request *req); |
| |
| req_ctx = skcipher_request_ctx(req); |
| |
| switch (ctx->crypto_type) { |
| case ARTPEC6_CRYPTO_CIPHER_AES_CBC: |
| case ARTPEC6_CRYPTO_CIPHER_AES_ECB: |
| case ARTPEC6_CRYPTO_CIPHER_AES_XTS: |
| req_ctx->decrypt = 1; |
| break; |
| default: |
| break; |
| } |
| |
| |
| switch (ctx->crypto_type) { |
| case ARTPEC6_CRYPTO_CIPHER_AES_CBC: |
| complete = artpec6_crypto_complete_cbc_decrypt; |
| break; |
| default: |
| complete = artpec6_crypto_complete_crypto; |
| break; |
| } |
| |
| ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, |
| complete, |
| req->dst, req->cryptlen); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_prepare_crypto(req); |
| if (ret) { |
| artpec6_crypto_common_destroy(&req_ctx->common); |
| return ret; |
| } |
| |
| return artpec6_crypto_submit(&req_ctx->common); |
| } |
| |
| static int |
| artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt) |
| { |
| struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); |
| size_t iv_len = crypto_skcipher_ivsize(cipher); |
| unsigned int counter = be32_to_cpup((__be32 *) |
| (req->iv + iv_len - 4)); |
| unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / |
| AES_BLOCK_SIZE; |
| |
| /* |
| * The hardware uses only the last 32-bits as the counter while the |
| * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that |
| * the whole IV is a counter. So fallback if the counter is going to |
| * overlow. |
| */ |
| if (counter + nblks < counter) { |
| int ret; |
| |
| pr_debug("counter %x will overflow (nblks %u), falling back\n", |
| counter, counter + nblks); |
| |
| ret = crypto_skcipher_setkey(ctx->fallback, ctx->aes_key, |
| ctx->key_length); |
| if (ret) |
| return ret; |
| |
| { |
| SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); |
| |
| skcipher_request_set_tfm(subreq, ctx->fallback); |
| skcipher_request_set_callback(subreq, req->base.flags, |
| NULL, NULL); |
| skcipher_request_set_crypt(subreq, req->src, req->dst, |
| req->cryptlen, req->iv); |
| ret = encrypt ? crypto_skcipher_encrypt(subreq) |
| : crypto_skcipher_decrypt(subreq); |
| skcipher_request_zero(subreq); |
| } |
| return ret; |
| } |
| |
| return encrypt ? artpec6_crypto_encrypt(req) |
| : artpec6_crypto_decrypt(req); |
| } |
| |
| static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req) |
| { |
| return artpec6_crypto_ctr_crypt(req, true); |
| } |
| |
| static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req) |
| { |
| return artpec6_crypto_ctr_crypt(req, false); |
| } |
| |
| /* |
| * AEAD functions |
| */ |
| static int artpec6_crypto_aead_init(struct crypto_aead *tfm) |
| { |
| struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm); |
| |
| memset(tfm_ctx, 0, sizeof(*tfm_ctx)); |
| |
| crypto_aead_set_reqsize(tfm, |
| sizeof(struct artpec6_crypto_aead_req_ctx)); |
| |
| return 0; |
| } |
| |
| static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, |
| unsigned int len) |
| { |
| struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); |
| |
| if (len != 16 && len != 24 && len != 32) { |
| crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); |
| return -EINVAL; |
| } |
| |
| ctx->key_length = len; |
| |
| memcpy(ctx->aes_key, key, len); |
| return 0; |
| } |
| |
| static int artpec6_crypto_aead_encrypt(struct aead_request *req) |
| { |
| int ret; |
| struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); |
| |
| req_ctx->decrypt = false; |
| ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, |
| artpec6_crypto_complete_aead, |
| NULL, 0); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_prepare_aead(req); |
| if (ret) { |
| artpec6_crypto_common_destroy(&req_ctx->common); |
| return ret; |
| } |
| |
| return artpec6_crypto_submit(&req_ctx->common); |
| } |
| |
| static int artpec6_crypto_aead_decrypt(struct aead_request *req) |
| { |
| int ret; |
| struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); |
| |
| req_ctx->decrypt = true; |
| if (req->cryptlen < AES_BLOCK_SIZE) |
| return -EINVAL; |
| |
| ret = artpec6_crypto_common_init(&req_ctx->common, |
| &req->base, |
| artpec6_crypto_complete_aead, |
| NULL, 0); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_prepare_aead(req); |
| if (ret) { |
| artpec6_crypto_common_destroy(&req_ctx->common); |
| return ret; |
| } |
| |
| return artpec6_crypto_submit(&req_ctx->common); |
| } |
| |
| static int artpec6_crypto_prepare_hash(struct ahash_request *areq) |
| { |
| struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq); |
| size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); |
| size_t contextsize = digestsize == SHA384_DIGEST_SIZE ? |
| SHA512_DIGEST_SIZE : digestsize; |
| size_t blocksize = crypto_tfm_alg_blocksize( |
| crypto_ahash_tfm(crypto_ahash_reqtfm(areq))); |
| struct artpec6_crypto_req_common *common = &req_ctx->common; |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| enum artpec6_crypto_variant variant = ac->variant; |
| u32 sel_ctx; |
| bool ext_ctx = false; |
| bool run_hw = false; |
| int error = 0; |
| |
| artpec6_crypto_init_dma_operation(common); |
| |
| /* Upload HMAC key, must be first the first packet */ |
| if (req_ctx->hash_flags & HASH_FLAG_HMAC) { |
| if (variant == ARTPEC6_CRYPTO) { |
| req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, |
| a6_regk_crypto_dlkey); |
| } else { |
| req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, |
| a7_regk_crypto_dlkey); |
| } |
| |
| /* Copy and pad up the key */ |
| memcpy(req_ctx->key_buffer, ctx->hmac_key, |
| ctx->hmac_key_length); |
| memset(req_ctx->key_buffer + ctx->hmac_key_length, 0, |
| blocksize - ctx->hmac_key_length); |
| |
| error = artpec6_crypto_setup_out_descr(common, |
| (void *)&req_ctx->key_md, |
| sizeof(req_ctx->key_md), false, false); |
| if (error) |
| return error; |
| |
| error = artpec6_crypto_setup_out_descr(common, |
| req_ctx->key_buffer, blocksize, |
| true, false); |
| if (error) |
| return error; |
| } |
| |
| if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { |
| /* Restore context */ |
| sel_ctx = regk_crypto_ext; |
| ext_ctx = true; |
| } else { |
| sel_ctx = regk_crypto_init; |
| } |
| |
| if (variant == ARTPEC6_CRYPTO) { |
| req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; |
| req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); |
| |
| /* If this is the final round, set the final flag */ |
| if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) |
| req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; |
| } else { |
| req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; |
| req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); |
| |
| /* If this is the final round, set the final flag */ |
| if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) |
| req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; |
| } |
| |
| /* Setup up metadata descriptors */ |
| error = artpec6_crypto_setup_out_descr(common, |
| (void *)&req_ctx->hash_md, |
| sizeof(req_ctx->hash_md), false, false); |
| if (error) |
| return error; |
| |
| error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); |
| if (error) |
| return error; |
| |
| if (ext_ctx) { |
| error = artpec6_crypto_setup_out_descr(common, |
| req_ctx->digeststate, |
| contextsize, false, false); |
| |
| if (error) |
| return error; |
| } |
| |
| if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { |
| size_t done_bytes = 0; |
| size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; |
| size_t ready_bytes = round_down(total_bytes, blocksize); |
| struct artpec6_crypto_walk walk; |
| |
| run_hw = ready_bytes > 0; |
| if (req_ctx->partial_bytes && ready_bytes) { |
| /* We have a partial buffer and will at least some bytes |
| * to the HW. Empty this partial buffer before tackling |
| * the SG lists |
| */ |
| memcpy(req_ctx->partial_buffer_out, |
| req_ctx->partial_buffer, |
| req_ctx->partial_bytes); |
| |
| error = artpec6_crypto_setup_out_descr(common, |
| req_ctx->partial_buffer_out, |
| req_ctx->partial_bytes, |
| false, true); |
| if (error) |
| return error; |
| |
| /* Reset partial buffer */ |
| done_bytes += req_ctx->partial_bytes; |
| req_ctx->partial_bytes = 0; |
| } |
| |
| artpec6_crypto_walk_init(&walk, areq->src); |
| |
| error = artpec6_crypto_setup_sg_descrs_out(common, &walk, |
| ready_bytes - |
| done_bytes); |
| if (error) |
| return error; |
| |
| if (walk.sg) { |
| size_t sg_skip = ready_bytes - done_bytes; |
| size_t sg_rem = areq->nbytes - sg_skip; |
| |
| sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), |
| req_ctx->partial_buffer + |
| req_ctx->partial_bytes, |
| sg_rem, sg_skip); |
| |
| req_ctx->partial_bytes += sg_rem; |
| } |
| |
| req_ctx->digcnt += ready_bytes; |
| req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); |
| } |
| |
| /* Finalize */ |
| if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { |
| bool needtrim = contextsize != digestsize; |
| size_t hash_pad_len; |
| u64 digest_bits; |
| u32 oper; |
| |
| if (variant == ARTPEC6_CRYPTO) |
| oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); |
| else |
| oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); |
| |
| /* Write out the partial buffer if present */ |
| if (req_ctx->partial_bytes) { |
| memcpy(req_ctx->partial_buffer_out, |
| req_ctx->partial_buffer, |
| req_ctx->partial_bytes); |
| error = artpec6_crypto_setup_out_descr(common, |
| req_ctx->partial_buffer_out, |
| req_ctx->partial_bytes, |
| false, true); |
| if (error) |
| return error; |
| |
| req_ctx->digcnt += req_ctx->partial_bytes; |
| req_ctx->partial_bytes = 0; |
| } |
| |
| if (req_ctx->hash_flags & HASH_FLAG_HMAC) |
| digest_bits = 8 * (req_ctx->digcnt + blocksize); |
| else |
| digest_bits = 8 * req_ctx->digcnt; |
| |
| /* Add the hash pad */ |
| hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, |
| req_ctx->digcnt, digest_bits); |
| error = artpec6_crypto_setup_out_descr(common, |
| req_ctx->pad_buffer, |
| hash_pad_len, false, |
| true); |
| req_ctx->digcnt = 0; |
| |
| if (error) |
| return error; |
| |
| /* Descriptor for the final result */ |
| error = artpec6_crypto_setup_in_descr(common, areq->result, |
| digestsize, |
| !needtrim); |
| if (error) |
| return error; |
| |
| if (needtrim) { |
| /* Discard the extra context bytes for SHA-384 */ |
| error = artpec6_crypto_setup_in_descr(common, |
| req_ctx->partial_buffer, |
| digestsize - contextsize, true); |
| if (error) |
| return error; |
| } |
| |
| } else { /* This is not the final operation for this request */ |
| if (!run_hw) |
| return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START; |
| |
| /* Save the result to the context */ |
| error = artpec6_crypto_setup_in_descr(common, |
| req_ctx->digeststate, |
| contextsize, false); |
| if (error) |
| return error; |
| /* fall through */ |
| } |
| |
| req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | |
| HASH_FLAG_FINALIZE); |
| |
| error = artpec6_crypto_terminate_in_descrs(common); |
| if (error) |
| return error; |
| |
| error = artpec6_crypto_terminate_out_descrs(common); |
| if (error) |
| return error; |
| |
| error = artpec6_crypto_dma_map_descs(common); |
| if (error) |
| return error; |
| |
| return ARTPEC6_CRYPTO_PREPARE_HASH_START; |
| } |
| |
| |
| static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) |
| { |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); |
| |
| tfm->reqsize = sizeof(struct artpec6_crypto_request_context); |
| ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; |
| |
| return 0; |
| } |
| |
| static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) |
| { |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); |
| |
| ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base), |
| 0, |
| CRYPTO_ALG_ASYNC | |
| CRYPTO_ALG_NEED_FALLBACK); |
| if (IS_ERR(ctx->fallback)) |
| return PTR_ERR(ctx->fallback); |
| |
| tfm->reqsize = sizeof(struct artpec6_crypto_request_context); |
| ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; |
| |
| return 0; |
| } |
| |
| static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) |
| { |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); |
| |
| tfm->reqsize = sizeof(struct artpec6_crypto_request_context); |
| ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; |
| |
| return 0; |
| } |
| |
| static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) |
| { |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); |
| |
| tfm->reqsize = sizeof(struct artpec6_crypto_request_context); |
| ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; |
| |
| return 0; |
| } |
| |
| static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm) |
| { |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); |
| |
| memset(ctx, 0, sizeof(*ctx)); |
| } |
| |
| static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm) |
| { |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); |
| |
| crypto_free_skcipher(ctx->fallback); |
| artpec6_crypto_aes_exit(tfm); |
| } |
| |
| static int |
| artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key, |
| unsigned int keylen) |
| { |
| struct artpec6_cryptotfm_context *ctx = |
| crypto_skcipher_ctx(cipher); |
| |
| switch (keylen) { |
| case 16: |
| case 24: |
| case 32: |
| break; |
| default: |
| crypto_skcipher_set_flags(cipher, |
| CRYPTO_TFM_RES_BAD_KEY_LEN); |
| return -EINVAL; |
| } |
| |
| memcpy(ctx->aes_key, key, keylen); |
| ctx->key_length = keylen; |
| return 0; |
| } |
| |
| static int |
| artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key, |
| unsigned int keylen) |
| { |
| struct artpec6_cryptotfm_context *ctx = |
| crypto_skcipher_ctx(cipher); |
| int ret; |
| |
| ret = xts_check_key(&cipher->base, key, keylen); |
| if (ret) |
| return ret; |
| |
| switch (keylen) { |
| case 32: |
| case 48: |
| case 64: |
| break; |
| default: |
| crypto_skcipher_set_flags(cipher, |
| CRYPTO_TFM_RES_BAD_KEY_LEN); |
| return -EINVAL; |
| } |
| |
| memcpy(ctx->aes_key, key, keylen); |
| ctx->key_length = keylen; |
| return 0; |
| } |
| |
| /** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request |
| * |
| * @req: The asynch request to process |
| * |
| * @return 0 if the dma job was successfully prepared |
| * <0 on error |
| * |
| * This function sets up the PDMA descriptors for a block cipher request. |
| * |
| * The required padding is added for AES-CTR using a statically defined |
| * buffer. |
| * |
| * The PDMA descriptor list will be as follows: |
| * |
| * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop> |
| * IN: <CIPHER_MD><data_0>...[data_n]<intr> |
| * |
| */ |
| static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) |
| { |
| int ret; |
| struct artpec6_crypto_walk walk; |
| struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); |
| struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); |
| struct artpec6_crypto_request_context *req_ctx = NULL; |
| size_t iv_len = crypto_skcipher_ivsize(cipher); |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| enum artpec6_crypto_variant variant = ac->variant; |
| struct artpec6_crypto_req_common *common; |
| bool cipher_decr = false; |
| size_t cipher_klen; |
| u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */ |
| u32 oper; |
| |
| req_ctx = skcipher_request_ctx(areq); |
| common = &req_ctx->common; |
| |
| artpec6_crypto_init_dma_operation(common); |
| |
| if (variant == ARTPEC6_CRYPTO) |
| ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); |
| else |
| ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); |
| |
| ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, |
| sizeof(ctx->key_md), false, false); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, |
| ctx->key_length, true, false); |
| if (ret) |
| return ret; |
| |
| req_ctx->cipher_md = 0; |
| |
| if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) |
| cipher_klen = ctx->key_length/2; |
| else |
| cipher_klen = ctx->key_length; |
| |
| /* Metadata */ |
| switch (cipher_klen) { |
| case 16: |
| cipher_len = regk_crypto_key_128; |
| break; |
| case 24: |
| cipher_len = regk_crypto_key_192; |
| break; |
| case 32: |
| cipher_len = regk_crypto_key_256; |
| break; |
| default: |
| pr_err("%s: Invalid key length %d!\n", |
| MODULE_NAME, ctx->key_length); |
| return -EINVAL; |
| } |
| |
| switch (ctx->crypto_type) { |
| case ARTPEC6_CRYPTO_CIPHER_AES_ECB: |
| oper = regk_crypto_aes_ecb; |
| cipher_decr = req_ctx->decrypt; |
| break; |
| |
| case ARTPEC6_CRYPTO_CIPHER_AES_CBC: |
| oper = regk_crypto_aes_cbc; |
| cipher_decr = req_ctx->decrypt; |
| break; |
| |
| case ARTPEC6_CRYPTO_CIPHER_AES_CTR: |
| oper = regk_crypto_aes_ctr; |
| cipher_decr = false; |
| break; |
| |
| case ARTPEC6_CRYPTO_CIPHER_AES_XTS: |
| oper = regk_crypto_aes_xts; |
| cipher_decr = req_ctx->decrypt; |
| |
| if (variant == ARTPEC6_CRYPTO) |
| req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; |
| else |
| req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; |
| break; |
| |
| default: |
| pr_err("%s: Invalid cipher mode %d!\n", |
| MODULE_NAME, ctx->crypto_type); |
| return -EINVAL; |
| } |
| |
| if (variant == ARTPEC6_CRYPTO) { |
| req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); |
| req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, |
| cipher_len); |
| if (cipher_decr) |
| req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; |
| } else { |
| req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); |
| req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, |
| cipher_len); |
| if (cipher_decr) |
| req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; |
| } |
| |
| ret = artpec6_crypto_setup_out_descr(common, |
| &req_ctx->cipher_md, |
| sizeof(req_ctx->cipher_md), |
| false, false); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); |
| if (ret) |
| return ret; |
| |
| if (iv_len) { |
| ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, |
| false, false); |
| if (ret) |
| return ret; |
| } |
| /* Data out */ |
| artpec6_crypto_walk_init(&walk, areq->src); |
| ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); |
| if (ret) |
| return ret; |
| |
| /* Data in */ |
| artpec6_crypto_walk_init(&walk, areq->dst); |
| ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); |
| if (ret) |
| return ret; |
| |
| /* CTR-mode padding required by the HW. */ |
| if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || |
| ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { |
| size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - |
| areq->cryptlen; |
| |
| if (pad) { |
| ret = artpec6_crypto_setup_out_descr(common, |
| ac->pad_buffer, |
| pad, false, false); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_setup_in_descr(common, |
| ac->pad_buffer, pad, |
| false); |
| if (ret) |
| return ret; |
| } |
| } |
| |
| ret = artpec6_crypto_terminate_out_descrs(common); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_terminate_in_descrs(common); |
| if (ret) |
| return ret; |
| |
| return artpec6_crypto_dma_map_descs(common); |
| } |
| |
| static int artpec6_crypto_prepare_aead(struct aead_request *areq) |
| { |
| size_t count; |
| int ret; |
| size_t input_length; |
| struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); |
| struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); |
| struct crypto_aead *cipher = crypto_aead_reqtfm(areq); |
| struct artpec6_crypto_req_common *common = &req_ctx->common; |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| enum artpec6_crypto_variant variant = ac->variant; |
| u32 md_cipher_len; |
| |
| artpec6_crypto_init_dma_operation(common); |
| |
| /* Key */ |
| if (variant == ARTPEC6_CRYPTO) { |
| ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, |
| a6_regk_crypto_dlkey); |
| } else { |
| ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, |
| a7_regk_crypto_dlkey); |
| } |
| ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, |
| sizeof(ctx->key_md), false, false); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, |
| ctx->key_length, true, false); |
| if (ret) |
| return ret; |
| |
| req_ctx->cipher_md = 0; |
| |
| switch (ctx->key_length) { |
| case 16: |
| md_cipher_len = regk_crypto_key_128; |
| break; |
| case 24: |
| md_cipher_len = regk_crypto_key_192; |
| break; |
| case 32: |
| md_cipher_len = regk_crypto_key_256; |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| if (variant == ARTPEC6_CRYPTO) { |
| req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, |
| regk_crypto_aes_gcm); |
| req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, |
| md_cipher_len); |
| if (req_ctx->decrypt) |
| req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; |
| } else { |
| req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, |
| regk_crypto_aes_gcm); |
| req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, |
| md_cipher_len); |
| if (req_ctx->decrypt) |
| req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; |
| } |
| |
| ret = artpec6_crypto_setup_out_descr(common, |
| (void *) &req_ctx->cipher_md, |
| sizeof(req_ctx->cipher_md), false, |
| false); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); |
| if (ret) |
| return ret; |
| |
| /* For the decryption, cryptlen includes the tag. */ |
| input_length = areq->cryptlen; |
| if (req_ctx->decrypt) |
| input_length -= AES_BLOCK_SIZE; |
| |
| /* Prepare the context buffer */ |
| req_ctx->hw_ctx.aad_length_bits = |
| __cpu_to_be64(8*areq->assoclen); |
| |
| req_ctx->hw_ctx.text_length_bits = |
| __cpu_to_be64(8*input_length); |
| |
| memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); |
| // The HW omits the initial increment of the counter field. |
| memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4); |
| |
| ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, |
| sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); |
| if (ret) |
| return ret; |
| |
| { |
| struct artpec6_crypto_walk walk; |
| |
| artpec6_crypto_walk_init(&walk, areq->src); |
| |
| /* Associated data */ |
| count = areq->assoclen; |
| ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); |
| if (ret) |
| return ret; |
| |
| if (!IS_ALIGNED(areq->assoclen, 16)) { |
| size_t assoc_pad = 16 - (areq->assoclen % 16); |
| /* The HW mandates zero padding here */ |
| ret = artpec6_crypto_setup_out_descr(common, |
| ac->zero_buffer, |
| assoc_pad, false, |
| false); |
| if (ret) |
| return ret; |
| } |
| |
| /* Data to crypto */ |
| count = input_length; |
| ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); |
| if (ret) |
| return ret; |
| |
| if (!IS_ALIGNED(input_length, 16)) { |
| size_t crypto_pad = 16 - (input_length % 16); |
| /* The HW mandates zero padding here */ |
| ret = artpec6_crypto_setup_out_descr(common, |
| ac->zero_buffer, |
| crypto_pad, |
| false, |
| false); |
| if (ret) |
| return ret; |
| } |
| } |
| |
| /* Data from crypto */ |
| { |
| struct artpec6_crypto_walk walk; |
| size_t output_len = areq->cryptlen; |
| |
| if (req_ctx->decrypt) |
| output_len -= AES_BLOCK_SIZE; |
| |
| artpec6_crypto_walk_init(&walk, areq->dst); |
| |
| /* skip associated data in the output */ |
| count = artpec6_crypto_walk_advance(&walk, areq->assoclen); |
| if (count) |
| return -EINVAL; |
| |
| count = output_len; |
| ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count); |
| if (ret) |
| return ret; |
| |
| /* Put padding between the cryptotext and the auth tag */ |
| if (!IS_ALIGNED(output_len, 16)) { |
| size_t crypto_pad = 16 - (output_len % 16); |
| |
| ret = artpec6_crypto_setup_in_descr(common, |
| ac->pad_buffer, |
| crypto_pad, false); |
| if (ret) |
| return ret; |
| } |
| |
| /* The authentication tag shall follow immediately after |
| * the output ciphertext. For decryption it is put in a context |
| * buffer for later compare against the input tag. |
| */ |
| count = AES_BLOCK_SIZE; |
| |
| if (req_ctx->decrypt) { |
| ret = artpec6_crypto_setup_in_descr(common, |
| req_ctx->decryption_tag, count, false); |
| if (ret) |
| return ret; |
| |
| } else { |
| ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, |
| count); |
| if (ret) |
| return ret; |
| } |
| |
| } |
| |
| ret = artpec6_crypto_terminate_in_descrs(common); |
| if (ret) |
| return ret; |
| |
| ret = artpec6_crypto_terminate_out_descrs(common); |
| if (ret) |
| return ret; |
| |
| return artpec6_crypto_dma_map_descs(common); |
| } |
| |
| static void artpec6_crypto_process_queue(struct artpec6_crypto *ac, |
| struct list_head *completions) |
| { |
| struct artpec6_crypto_req_common *req; |
| |
| while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { |
| req = list_first_entry(&ac->queue, |
| struct artpec6_crypto_req_common, |
| list); |
| list_move_tail(&req->list, &ac->pending); |
| artpec6_crypto_start_dma(req); |
| |
| list_add_tail(&req->complete_in_progress, completions); |
| } |
| |
| /* |
| * In some cases, the hardware can raise an in_eop_flush interrupt |
| * before actually updating the status, so we have an timer which will |
| * recheck the status on timeout. Since the cases are expected to be |
| * very rare, we use a relatively large timeout value. There should be |
| * no noticeable negative effect if we timeout spuriously. |
| */ |
| if (ac->pending_count) |
| mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); |
| else |
| del_timer(&ac->timer); |
| } |
| |
| static void artpec6_crypto_timeout(struct timer_list *t) |
| { |
| struct artpec6_crypto *ac = from_timer(ac, t, timer); |
| |
| dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); |
| |
| tasklet_schedule(&ac->task); |
| } |
| |
| static void artpec6_crypto_task(unsigned long data) |
| { |
| struct artpec6_crypto *ac = (struct artpec6_crypto *)data; |
| struct artpec6_crypto_req_common *req; |
| struct artpec6_crypto_req_common *n; |
| struct list_head complete_done; |
| struct list_head complete_in_progress; |
| |
| INIT_LIST_HEAD(&complete_done); |
| INIT_LIST_HEAD(&complete_in_progress); |
| |
| if (list_empty(&ac->pending)) { |
| pr_debug("Spurious IRQ\n"); |
| return; |
| } |
| |
| spin_lock_bh(&ac->queue_lock); |
| |
| list_for_each_entry_safe(req, n, &ac->pending, list) { |
| struct artpec6_crypto_dma_descriptors *dma = req->dma; |
| u32 stat; |
| |
| dma_sync_single_for_cpu(artpec6_crypto_dev, dma->stat_dma_addr, |
| sizeof(dma->stat[0]), |
| DMA_BIDIRECTIONAL); |
| |
| stat = req->dma->stat[req->dma->in_cnt-1]; |
| |
| /* A non-zero final status descriptor indicates |
| * this job has finished. |
| */ |
| pr_debug("Request %p status is %X\n", req, stat); |
| if (!stat) |
| break; |
| |
| /* Allow testing of timeout handling with fault injection */ |
| #ifdef CONFIG_FAULT_INJECTION |
| if (should_fail(&artpec6_crypto_fail_status_read, 1)) |
| continue; |
| #endif |
| |
| pr_debug("Completing request %p\n", req); |
| |
| list_move_tail(&req->list, &complete_done); |
| |
| artpec6_crypto_dma_unmap_all(req); |
| artpec6_crypto_copy_bounce_buffers(req); |
| |
| ac->pending_count--; |
| artpec6_crypto_common_destroy(req); |
| } |
| |
| artpec6_crypto_process_queue(ac, &complete_in_progress); |
| |
| spin_unlock_bh(&ac->queue_lock); |
| |
| /* Perform the completion callbacks without holding the queue lock |
| * to allow new request submissions from the callbacks. |
| */ |
| list_for_each_entry_safe(req, n, &complete_done, list) { |
| req->complete(req->req); |
| } |
| |
| list_for_each_entry_safe(req, n, &complete_in_progress, |
| complete_in_progress) { |
| req->req->complete(req->req, -EINPROGRESS); |
| } |
| } |
| |
| static void artpec6_crypto_complete_crypto(struct crypto_async_request *req) |
| { |
| req->complete(req, 0); |
| } |
| |
| static void |
| artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req) |
| { |
| struct skcipher_request *cipher_req = container_of(req, |
| struct skcipher_request, base); |
| |
| scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, |
| cipher_req->cryptlen - AES_BLOCK_SIZE, |
| AES_BLOCK_SIZE, 0); |
| req->complete(req, 0); |
| } |
| |
| static void |
| artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req) |
| { |
| struct skcipher_request *cipher_req = container_of(req, |
| struct skcipher_request, base); |
| |
| scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, |
| cipher_req->cryptlen - AES_BLOCK_SIZE, |
| AES_BLOCK_SIZE, 0); |
| req->complete(req, 0); |
| } |
| |
| static void artpec6_crypto_complete_aead(struct crypto_async_request *req) |
| { |
| int result = 0; |
| |
| /* Verify GCM hashtag. */ |
| struct aead_request *areq = container_of(req, |
| struct aead_request, base); |
| struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); |
| |
| if (req_ctx->decrypt) { |
| u8 input_tag[AES_BLOCK_SIZE]; |
| |
| sg_pcopy_to_buffer(areq->src, |
| sg_nents(areq->src), |
| input_tag, |
| AES_BLOCK_SIZE, |
| areq->assoclen + areq->cryptlen - |
| AES_BLOCK_SIZE); |
| |
| if (memcmp(req_ctx->decryption_tag, |
| input_tag, |
| AES_BLOCK_SIZE)) { |
| pr_debug("***EBADMSG:\n"); |
| print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1, |
| input_tag, AES_BLOCK_SIZE, true); |
| print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1, |
| req_ctx->decryption_tag, |
| AES_BLOCK_SIZE, true); |
| |
| result = -EBADMSG; |
| } |
| } |
| |
| req->complete(req, result); |
| } |
| |
| static void artpec6_crypto_complete_hash(struct crypto_async_request *req) |
| { |
| req->complete(req, 0); |
| } |
| |
| |
| /*------------------- Hash functions -----------------------------------------*/ |
| static int |
| artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, |
| const u8 *key, unsigned int keylen) |
| { |
| struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); |
| size_t blocksize; |
| int ret; |
| |
| if (!keylen) { |
| pr_err("Invalid length (%d) of HMAC key\n", |
| keylen); |
| return -EINVAL; |
| } |
| |
| memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); |
| |
| blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); |
| |
| if (keylen > blocksize) { |
| SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash); |
| |
| hdesc->tfm = tfm_ctx->child_hash; |
| hdesc->flags = crypto_ahash_get_flags(tfm) & |
| CRYPTO_TFM_REQ_MAY_SLEEP; |
| |
| tfm_ctx->hmac_key_length = blocksize; |
| ret = crypto_shash_digest(hdesc, key, keylen, |
| tfm_ctx->hmac_key); |
| if (ret) |
| return ret; |
| |
| } else { |
| memcpy(tfm_ctx->hmac_key, key, keylen); |
| tfm_ctx->hmac_key_length = keylen; |
| } |
| |
| return 0; |
| } |
| |
| static int |
| artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac) |
| { |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| enum artpec6_crypto_variant variant = ac->variant; |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| u32 oper; |
| |
| memset(req_ctx, 0, sizeof(*req_ctx)); |
| |
| req_ctx->hash_flags = HASH_FLAG_INIT_CTX; |
| if (hmac) |
| req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); |
| |
| switch (type) { |
| case ARTPEC6_CRYPTO_HASH_SHA1: |
| oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1; |
| break; |
| case ARTPEC6_CRYPTO_HASH_SHA256: |
| oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256; |
| break; |
| case ARTPEC6_CRYPTO_HASH_SHA384: |
| oper = hmac ? regk_crypto_hmac_sha384 : regk_crypto_sha384; |
| break; |
| case ARTPEC6_CRYPTO_HASH_SHA512: |
| oper = hmac ? regk_crypto_hmac_sha512 : regk_crypto_sha512; |
| break; |
| |
| default: |
| pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type); |
| return -EINVAL; |
| } |
| |
| if (variant == ARTPEC6_CRYPTO) |
| req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); |
| else |
| req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); |
| |
| return 0; |
| } |
| |
| static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| int ret; |
| |
| if (!req_ctx->common.dma) { |
| ret = artpec6_crypto_common_init(&req_ctx->common, |
| &req->base, |
| artpec6_crypto_complete_hash, |
| NULL, 0); |
| |
| if (ret) |
| return ret; |
| } |
| |
| ret = artpec6_crypto_prepare_hash(req); |
| switch (ret) { |
| case ARTPEC6_CRYPTO_PREPARE_HASH_START: |
| ret = artpec6_crypto_submit(&req_ctx->common); |
| break; |
| |
| case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START: |
| ret = 0; |
| /* Fallthrough */ |
| |
| default: |
| artpec6_crypto_common_destroy(&req_ctx->common); |
| break; |
| } |
| |
| return ret; |
| } |
| |
| static int artpec6_crypto_hash_final(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| req_ctx->hash_flags |= HASH_FLAG_FINALIZE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int artpec6_crypto_hash_update(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| req_ctx->hash_flags |= HASH_FLAG_UPDATE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int artpec6_crypto_sha1_init(struct ahash_request *req) |
| { |
| return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); |
| } |
| |
| static int artpec6_crypto_sha1_digest(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); |
| |
| req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int artpec6_crypto_sha256_init(struct ahash_request *req) |
| { |
| return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); |
| } |
| |
| static int artpec6_crypto_sha256_digest(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); |
| req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int __maybe_unused artpec6_crypto_sha384_init(struct ahash_request *req) |
| { |
| return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0); |
| } |
| |
| static int __maybe_unused |
| artpec6_crypto_sha384_digest(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0); |
| req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int artpec6_crypto_sha512_init(struct ahash_request *req) |
| { |
| return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0); |
| } |
| |
| static int artpec6_crypto_sha512_digest(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0); |
| req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req) |
| { |
| return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); |
| } |
| |
| static int __maybe_unused |
| artpec6_crypto_hmac_sha384_init(struct ahash_request *req) |
| { |
| return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1); |
| } |
| |
| static int artpec6_crypto_hmac_sha512_init(struct ahash_request *req) |
| { |
| return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1); |
| } |
| |
| static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); |
| req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int __maybe_unused |
| artpec6_crypto_hmac_sha384_digest(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1); |
| req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int artpec6_crypto_hmac_sha512_digest(struct ahash_request *req) |
| { |
| struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); |
| |
| artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1); |
| req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; |
| |
| return artpec6_crypto_prepare_submit_hash(req); |
| } |
| |
| static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm, |
| const char *base_hash_name) |
| { |
| struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); |
| |
| crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), |
| sizeof(struct artpec6_hash_request_context)); |
| memset(tfm_ctx, 0, sizeof(*tfm_ctx)); |
| |
| if (base_hash_name) { |
| struct crypto_shash *child; |
| |
| child = crypto_alloc_shash(base_hash_name, 0, |
| CRYPTO_ALG_NEED_FALLBACK); |
| |
| if (IS_ERR(child)) |
| return PTR_ERR(child); |
| |
| tfm_ctx->child_hash = child; |
| } |
| |
| return 0; |
| } |
| |
| static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm) |
| { |
| return artpec6_crypto_ahash_init_common(tfm, NULL); |
| } |
| |
| static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm) |
| { |
| return artpec6_crypto_ahash_init_common(tfm, "sha256"); |
| } |
| |
| static int __maybe_unused |
| artpec6_crypto_ahash_init_hmac_sha384(struct crypto_tfm *tfm) |
| { |
| return artpec6_crypto_ahash_init_common(tfm, "sha384"); |
| } |
| |
| static int artpec6_crypto_ahash_init_hmac_sha512(struct crypto_tfm *tfm) |
| { |
| return artpec6_crypto_ahash_init_common(tfm, "sha512"); |
| } |
| |
| static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm) |
| { |
| struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); |
| |
| if (tfm_ctx->child_hash) |
| crypto_free_shash(tfm_ctx->child_hash); |
| |
| memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); |
| tfm_ctx->hmac_key_length = 0; |
| } |
| |
| static int artpec6_crypto_hash_export(struct ahash_request *req, void *out) |
| { |
| const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); |
| struct artpec6_hash_export_state *state = out; |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| enum artpec6_crypto_variant variant = ac->variant; |
| |
| BUILD_BUG_ON(sizeof(state->partial_buffer) != |
| sizeof(ctx->partial_buffer)); |
| BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); |
| |
| state->digcnt = ctx->digcnt; |
| state->partial_bytes = ctx->partial_bytes; |
| state->hash_flags = ctx->hash_flags; |
| |
| if (variant == ARTPEC6_CRYPTO) |
| state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); |
| else |
| state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); |
| |
| memcpy(state->partial_buffer, ctx->partial_buffer, |
| sizeof(state->partial_buffer)); |
| memcpy(state->digeststate, ctx->digeststate, |
| sizeof(state->digeststate)); |
| |
| return 0; |
| } |
| |
| static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in) |
| { |
| struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); |
| const struct artpec6_hash_export_state *state = in; |
| struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); |
| enum artpec6_crypto_variant variant = ac->variant; |
| |
| memset(ctx, 0, sizeof(*ctx)); |
| |
| ctx->digcnt = state->digcnt; |
| ctx->partial_bytes = state->partial_bytes; |
| ctx->hash_flags = state->hash_flags; |
| |
| if (variant == ARTPEC6_CRYPTO) |
| ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); |
| else |
| ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); |
| |
| memcpy(ctx->partial_buffer, state->partial_buffer, |
| sizeof(state->partial_buffer)); |
| memcpy(ctx->digeststate, state->digeststate, |
| sizeof(state->digeststate)); |
| |
| return 0; |
| } |
| |
| static int init_crypto_hw(struct artpec6_crypto *ac) |
| { |
| enum artpec6_crypto_variant variant = ac->variant; |
| void __iomem *base = ac->base; |
| u32 out_descr_buf_size; |
| u32 out_data_buf_size; |
| u32 in_data_buf_size; |
| u32 in_descr_buf_size; |
| u32 in_stat_buf_size; |
| u32 in, out; |
| |
| /* |
| * The PDMA unit contains 1984 bytes of internal memory for the OUT |
| * channels and 1024 bytes for the IN channel. This is an elastic |
| * memory used to internally store the descriptors and data. The values |
| * ares specified in 64 byte incremements. Trustzone buffers are not |
| * used at this stage. |
| */ |
| out_data_buf_size = 16; /* 1024 bytes for data */ |
| out_descr_buf_size = 15; /* 960 bytes for descriptors */ |
| in_data_buf_size = 8; /* 512 bytes for data */ |
| in_descr_buf_size = 4; /* 256 bytes for descriptors */ |
| in_stat_buf_size = 4; /* 256 bytes for stat descrs */ |
| |
| BUILD_BUG_ON_MSG((out_data_buf_size |
| + out_descr_buf_size) * 64 > 1984, |
| "Invalid OUT configuration"); |
| |
| BUILD_BUG_ON_MSG((in_data_buf_size |
| + in_descr_buf_size |
| + in_stat_buf_size) * 64 > 1024, |
| "Invalid IN configuration"); |
| |
| in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) | |
| FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) | |
| FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size); |
| |
| out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) | |
| FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size); |
| |
| writel_relaxed(out, base + PDMA_OUT_BUF_CFG); |
| writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG); |
| |
| if (variant == ARTPEC6_CRYPTO) { |
| writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG); |
| writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG); |
| writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA | |
| A6_PDMA_INTR_MASK_IN_EOP_FLUSH, |
| base + A6_PDMA_INTR_MASK); |
| } else { |
| writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG); |
| writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG); |
| writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA | |
| A7_PDMA_INTR_MASK_IN_EOP_FLUSH, |
| base + A7_PDMA_INTR_MASK); |
| } |
| |
| return 0; |
| } |
| |
| static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac) |
| { |
| enum artpec6_crypto_variant variant = ac->variant; |
| void __iomem *base = ac->base; |
| |
| if (variant == ARTPEC6_CRYPTO) { |
| writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD); |
| writel_relaxed(0, base + A6_PDMA_IN_CFG); |
| writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); |
| } else { |
| writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD); |
| writel_relaxed(0, base + A7_PDMA_IN_CFG); |
| writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); |
| } |
| |
| writel_relaxed(0, base + PDMA_OUT_CFG); |
| |
| } |
| |
| static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id) |
| { |
| struct artpec6_crypto *ac = dev_id; |
| enum artpec6_crypto_variant variant = ac->variant; |
| void __iomem *base = ac->base; |
| u32 mask_in_data, mask_in_eop_flush; |
| u32 in_cmd_flush_stat, in_cmd_reg; |
| u32 ack_intr_reg; |
| u32 ack = 0; |
| u32 intr; |
| |
| if (variant == ARTPEC6_CRYPTO) { |
| intr = readl_relaxed(base + A6_PDMA_MASKED_INTR); |
| mask_in_data = A6_PDMA_INTR_MASK_IN_DATA; |
| mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH; |
| in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT; |
| in_cmd_reg = A6_PDMA_IN_CMD; |
| ack_intr_reg = A6_PDMA_ACK_INTR; |
| } else { |
| intr = readl_relaxed(base + A7_PDMA_MASKED_INTR); |
| mask_in_data = A7_PDMA_INTR_MASK_IN_DATA; |
| mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH; |
| in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT; |
| in_cmd_reg = A7_PDMA_IN_CMD; |
| ack_intr_reg = A7_PDMA_ACK_INTR; |
| } |
| |
| /* We get two interrupt notifications from each job. |
| * The in_data means all data was sent to memory and then |
| * we request a status flush command to write the per-job |
| * status to its status vector. This ensures that the |
| * tasklet can detect exactly how many submitted jobs |
| * that have finished. |
| */ |
| if (intr & mask_in_data) |
| ack |= mask_in_data; |
| |
| if (intr & mask_in_eop_flush) |
| ack |= mask_in_eop_flush; |
| else |
| writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg); |
| |
| writel_relaxed(ack, base + ack_intr_reg); |
| |
| if (intr & mask_in_eop_flush) |
| tasklet_schedule(&ac->task); |
| |
| return IRQ_HANDLED; |
| } |
| |
| /*------------------- Algorithm definitions ----------------------------------*/ |
| |
| /* Hashes */ |
| static struct ahash_alg hash_algos[] = { |
| /* SHA-1 */ |
| { |
| .init = artpec6_crypto_sha1_init, |
| .update = artpec6_crypto_hash_update, |
| .final = artpec6_crypto_hash_final, |
| .digest = artpec6_crypto_sha1_digest, |
| .import = artpec6_crypto_hash_import, |
| .export = artpec6_crypto_hash_export, |
| .halg.digestsize = SHA1_DIGEST_SIZE, |
| .halg.statesize = sizeof(struct artpec6_hash_export_state), |
| .halg.base = { |
| .cra_name = "sha1", |
| .cra_driver_name = "artpec-sha1", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = SHA1_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_hashalg_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| .cra_init = artpec6_crypto_ahash_init, |
| .cra_exit = artpec6_crypto_ahash_exit, |
| } |
| }, |
| /* SHA-256 */ |
| { |
| .init = artpec6_crypto_sha256_init, |
| .update = artpec6_crypto_hash_update, |
| .final = artpec6_crypto_hash_final, |
| .digest = artpec6_crypto_sha256_digest, |
| .import = artpec6_crypto_hash_import, |
| .export = artpec6_crypto_hash_export, |
| .halg.digestsize = SHA256_DIGEST_SIZE, |
| .halg.statesize = sizeof(struct artpec6_hash_export_state), |
| .halg.base = { |
| .cra_name = "sha256", |
| .cra_driver_name = "artpec-sha256", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = SHA256_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_hashalg_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| .cra_init = artpec6_crypto_ahash_init, |
| .cra_exit = artpec6_crypto_ahash_exit, |
| } |
| }, |
| /* HMAC SHA-256 */ |
| { |
| .init = artpec6_crypto_hmac_sha256_init, |
| .update = artpec6_crypto_hash_update, |
| .final = artpec6_crypto_hash_final, |
| .digest = artpec6_crypto_hmac_sha256_digest, |
| .import = artpec6_crypto_hash_import, |
| .export = artpec6_crypto_hash_export, |
| .setkey = artpec6_crypto_hash_set_key, |
| .halg.digestsize = SHA256_DIGEST_SIZE, |
| .halg.statesize = sizeof(struct artpec6_hash_export_state), |
| .halg.base = { |
| .cra_name = "hmac(sha256)", |
| .cra_driver_name = "artpec-hmac-sha256", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = SHA256_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_hashalg_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| .cra_init = artpec6_crypto_ahash_init_hmac_sha256, |
| .cra_exit = artpec6_crypto_ahash_exit, |
| } |
| }, |
| }; |
| |
| static struct ahash_alg artpec7_hash_algos[] = { |
| /* SHA-384 */ |
| { |
| .init = artpec6_crypto_sha384_init, |
| .update = artpec6_crypto_hash_update, |
| .final = artpec6_crypto_hash_final, |
| .digest = artpec6_crypto_sha384_digest, |
| .import = artpec6_crypto_hash_import, |
| .export = artpec6_crypto_hash_export, |
| .halg.digestsize = SHA384_DIGEST_SIZE, |
| .halg.statesize = sizeof(struct artpec6_hash_export_state), |
| .halg.base = { |
| .cra_name = "sha384", |
| .cra_driver_name = "artpec-sha384", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = SHA384_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_hashalg_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| .cra_init = artpec6_crypto_ahash_init, |
| .cra_exit = artpec6_crypto_ahash_exit, |
| } |
| }, |
| /* HMAC SHA-384 */ |
| { |
| .init = artpec6_crypto_hmac_sha384_init, |
| .update = artpec6_crypto_hash_update, |
| .final = artpec6_crypto_hash_final, |
| .digest = artpec6_crypto_hmac_sha384_digest, |
| .import = artpec6_crypto_hash_import, |
| .export = artpec6_crypto_hash_export, |
| .setkey = artpec6_crypto_hash_set_key, |
| .halg.digestsize = SHA384_DIGEST_SIZE, |
| .halg.statesize = sizeof(struct artpec6_hash_export_state), |
| .halg.base = { |
| .cra_name = "hmac(sha384)", |
| .cra_driver_name = "artpec-hmac-sha384", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = SHA384_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_hashalg_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| .cra_init = artpec6_crypto_ahash_init_hmac_sha384, |
| .cra_exit = artpec6_crypto_ahash_exit, |
| } |
| }, |
| /* SHA-512 */ |
| { |
| .init = artpec6_crypto_sha512_init, |
| .update = artpec6_crypto_hash_update, |
| .final = artpec6_crypto_hash_final, |
| .digest = artpec6_crypto_sha512_digest, |
| .import = artpec6_crypto_hash_import, |
| .export = artpec6_crypto_hash_export, |
| .halg.digestsize = SHA512_DIGEST_SIZE, |
| .halg.statesize = sizeof(struct artpec6_hash_export_state), |
| .halg.base = { |
| .cra_name = "sha512", |
| .cra_driver_name = "artpec-sha512", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = SHA512_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_hashalg_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| .cra_init = artpec6_crypto_ahash_init, |
| .cra_exit = artpec6_crypto_ahash_exit, |
| } |
| }, |
| /* HMAC SHA-512 */ |
| { |
| .init = artpec6_crypto_hmac_sha512_init, |
| .update = artpec6_crypto_hash_update, |
| .final = artpec6_crypto_hash_final, |
| .digest = artpec6_crypto_hmac_sha512_digest, |
| .import = artpec6_crypto_hash_import, |
| .export = artpec6_crypto_hash_export, |
| .setkey = artpec6_crypto_hash_set_key, |
| .halg.digestsize = SHA512_DIGEST_SIZE, |
| .halg.statesize = sizeof(struct artpec6_hash_export_state), |
| .halg.base = { |
| .cra_name = "hmac(sha512)", |
| .cra_driver_name = "artpec-hmac-sha512", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = SHA512_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_hashalg_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| .cra_init = artpec6_crypto_ahash_init_hmac_sha512, |
| .cra_exit = artpec6_crypto_ahash_exit, |
| } |
| }, |
| }; |
| |
| /* Crypto */ |
| static struct skcipher_alg crypto_algos[] = { |
| /* AES - ECB */ |
| { |
| .base = { |
| .cra_name = "ecb(aes)", |
| .cra_driver_name = "artpec6-ecb-aes", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = AES_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| }, |
| .min_keysize = AES_MIN_KEY_SIZE, |
| .max_keysize = AES_MAX_KEY_SIZE, |
| .setkey = artpec6_crypto_cipher_set_key, |
| .encrypt = artpec6_crypto_encrypt, |
| .decrypt = artpec6_crypto_decrypt, |
| .init = artpec6_crypto_aes_ecb_init, |
| .exit = artpec6_crypto_aes_exit, |
| }, |
| /* AES - CTR */ |
| { |
| .base = { |
| .cra_name = "ctr(aes)", |
| .cra_driver_name = "artpec6-ctr-aes", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC | |
| CRYPTO_ALG_NEED_FALLBACK, |
| .cra_blocksize = 1, |
| .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| }, |
| .min_keysize = AES_MIN_KEY_SIZE, |
| .max_keysize = AES_MAX_KEY_SIZE, |
| .ivsize = AES_BLOCK_SIZE, |
| .setkey = artpec6_crypto_cipher_set_key, |
| .encrypt = artpec6_crypto_ctr_encrypt, |
| .decrypt = artpec6_crypto_ctr_decrypt, |
| .init = artpec6_crypto_aes_ctr_init, |
| .exit = artpec6_crypto_aes_ctr_exit, |
| }, |
| /* AES - CBC */ |
| { |
| .base = { |
| .cra_name = "cbc(aes)", |
| .cra_driver_name = "artpec6-cbc-aes", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = AES_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| }, |
| .min_keysize = AES_MIN_KEY_SIZE, |
| .max_keysize = AES_MAX_KEY_SIZE, |
| .ivsize = AES_BLOCK_SIZE, |
| .setkey = artpec6_crypto_cipher_set_key, |
| .encrypt = artpec6_crypto_encrypt, |
| .decrypt = artpec6_crypto_decrypt, |
| .init = artpec6_crypto_aes_cbc_init, |
| .exit = artpec6_crypto_aes_exit |
| }, |
| /* AES - XTS */ |
| { |
| .base = { |
| .cra_name = "xts(aes)", |
| .cra_driver_name = "artpec6-xts-aes", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC, |
| .cra_blocksize = 1, |
| .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| }, |
| .min_keysize = 2*AES_MIN_KEY_SIZE, |
| .max_keysize = 2*AES_MAX_KEY_SIZE, |
| .ivsize = 16, |
| .setkey = artpec6_crypto_xts_set_key, |
| .encrypt = artpec6_crypto_encrypt, |
| .decrypt = artpec6_crypto_decrypt, |
| .init = artpec6_crypto_aes_xts_init, |
| .exit = artpec6_crypto_aes_exit, |
| }, |
| }; |
| |
| static struct aead_alg aead_algos[] = { |
| { |
| .init = artpec6_crypto_aead_init, |
| .setkey = artpec6_crypto_aead_set_key, |
| .encrypt = artpec6_crypto_aead_encrypt, |
| .decrypt = artpec6_crypto_aead_decrypt, |
| .ivsize = GCM_AES_IV_SIZE, |
| .maxauthsize = AES_BLOCK_SIZE, |
| |
| .base = { |
| .cra_name = "gcm(aes)", |
| .cra_driver_name = "artpec-gcm-aes", |
| .cra_priority = 300, |
| .cra_flags = CRYPTO_ALG_ASYNC | |
| CRYPTO_ALG_KERN_DRIVER_ONLY, |
| .cra_blocksize = 1, |
| .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), |
| .cra_alignmask = 3, |
| .cra_module = THIS_MODULE, |
| }, |
| } |
| }; |
| |
| #ifdef CONFIG_DEBUG_FS |
| |
| struct dbgfs_u32 { |
| char *name; |
| mode_t mode; |
| u32 *flag; |
| char *desc; |
| }; |
| |
| static struct dentry *dbgfs_root; |
| |
| static void artpec6_crypto_init_debugfs(void) |
| { |
| dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL); |
| |
| if (!dbgfs_root || IS_ERR(dbgfs_root)) { |
| dbgfs_root = NULL; |
| pr_err("%s: Could not initialise debugfs!\n", MODULE_NAME); |
| return; |
| } |
| |
| #ifdef CONFIG_FAULT_INJECTION |
| fault_create_debugfs_attr("fail_status_read", dbgfs_root, |
| &artpec6_crypto_fail_status_read); |
| |
| fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root, |
| &artpec6_crypto_fail_dma_array_full); |
| #endif |
| } |
| |
| static void artpec6_crypto_free_debugfs(void) |
| { |
| if (!dbgfs_root) |
| return; |
| |
| debugfs_remove_recursive(dbgfs_root); |
| dbgfs_root = NULL; |
| } |
| #endif |
| |
| static const struct of_device_id artpec6_crypto_of_match[] = { |
| { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO }, |
| { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO }, |
| {} |
| }; |
| MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); |
| |
| static int artpec6_crypto_probe(struct platform_device *pdev) |
| { |
| const struct of_device_id *match; |
| enum artpec6_crypto_variant variant; |
| struct artpec6_crypto *ac; |
| struct device *dev = &pdev->dev; |
| void __iomem *base; |
| struct resource *res; |
| int irq; |
| int err; |
| |
| if (artpec6_crypto_dev) |
| return -ENODEV; |
| |
| match = of_match_node(artpec6_crypto_of_match, dev->of_node); |
| if (!match) |
| return -EINVAL; |
| |
| variant = (enum artpec6_crypto_variant)match->data; |
| |
| res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| base = devm_ioremap_resource(&pdev->dev, res); |
| if (IS_ERR(base)) |
| return PTR_ERR(base); |
| |
| irq = platform_get_irq(pdev, 0); |
| if (irq < 0) |
| return -ENODEV; |
| |
| ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), |
| GFP_KERNEL); |
| if (!ac) |
| return -ENOMEM; |
| |
| platform_set_drvdata(pdev, ac); |
| ac->variant = variant; |
| |
| spin_lock_init(&ac->queue_lock); |
| INIT_LIST_HEAD(&ac->queue); |
| INIT_LIST_HEAD(&ac->pending); |
| timer_setup(&ac->timer, artpec6_crypto_timeout, 0); |
| |
| ac->base = base; |
| |
| ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", |
| sizeof(struct artpec6_crypto_dma_descriptors), |
| 64, |
| 0, |
| NULL); |
| if (!ac->dma_cache) |
| return -ENOMEM; |
| |
| #ifdef CONFIG_DEBUG_FS |
| artpec6_crypto_init_debugfs(); |
| #endif |
| |
| tasklet_init(&ac->task, artpec6_crypto_task, |
| (unsigned long)ac); |
| |
| ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, |
| GFP_KERNEL); |
| if (!ac->pad_buffer) |
| return -ENOMEM; |
| ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); |
| |
| ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, |
| GFP_KERNEL); |
| if (!ac->zero_buffer) |
| return -ENOMEM; |
| ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); |
| |
| err = init_crypto_hw(ac); |
| if (err) |
| goto free_cache; |
| |
| err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, |
| "artpec6-crypto", ac); |
| if (err) |
| goto disable_hw; |
| |
| artpec6_crypto_dev = &pdev->dev; |
| |
| err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); |
| if (err) { |
| dev_err(dev, "Failed to register ahashes\n"); |
| goto disable_hw; |
| } |
| |
| if (variant != ARTPEC6_CRYPTO) { |
| err = crypto_register_ahashes(artpec7_hash_algos, |
| ARRAY_SIZE(artpec7_hash_algos)); |
| if (err) { |
| dev_err(dev, "Failed to register ahashes\n"); |
| goto unregister_ahashes; |
| } |
| } |
| |
| err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); |
| if (err) { |
| dev_err(dev, "Failed to register ciphers\n"); |
| goto unregister_a7_ahashes; |
| } |
| |
| err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos)); |
| if (err) { |
| dev_err(dev, "Failed to register aeads\n"); |
| goto unregister_algs; |
| } |
| |
| return 0; |
| |
| unregister_algs: |
| crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); |
| unregister_a7_ahashes: |
| if (variant != ARTPEC6_CRYPTO) |
| crypto_unregister_ahashes(artpec7_hash_algos, |
| ARRAY_SIZE(artpec7_hash_algos)); |
| unregister_ahashes: |
| crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); |
| disable_hw: |
| artpec6_crypto_disable_hw( |