eip93-main.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2019 - 2021
  4 *
  5 * Richard van Schagen <vschagen@icloud.com>
  6 * Christian Marangi <ansuelsmth@gmail.com
  7 */
  8
  9#include <linux/atomic.h>
 10#include <linux/clk.h>
 11#include <linux/delay.h>
 12#include <linux/dma-mapping.h>
 13#include <linux/interrupt.h>
 14#include <linux/module.h>
 15#include <linux/of.h>
 16#include <linux/platform_device.h>
 17#include <linux/spinlock.h>
 18#include <crypto/aes.h>
 19#include <crypto/ctr.h>
 20
 21#include "eip93-main.h"
 22#include "eip93-regs.h"
 23#include "eip93-common.h"
 24#include "eip93-cipher.h"
 25#include "eip93-aes.h"
 26#include "eip93-des.h"
 27#include "eip93-aead.h"
 28#include "eip93-hash.h"
 29
 30static struct eip93_alg_template *eip93_algs[] = {
 31	&eip93_alg_ecb_des,
 32	&eip93_alg_cbc_des,
 33	&eip93_alg_ecb_des3_ede,
 34	&eip93_alg_cbc_des3_ede,
 35	&eip93_alg_ecb_aes,
 36	&eip93_alg_cbc_aes,
 37	&eip93_alg_ctr_aes,
 38	&eip93_alg_rfc3686_aes,
 39	&eip93_alg_authenc_hmac_md5_cbc_des,
 40	&eip93_alg_authenc_hmac_sha1_cbc_des,
 41	&eip93_alg_authenc_hmac_sha224_cbc_des,
 42	&eip93_alg_authenc_hmac_sha256_cbc_des,
 43	&eip93_alg_authenc_hmac_md5_cbc_des3_ede,
 44	&eip93_alg_authenc_hmac_sha1_cbc_des3_ede,
 45	&eip93_alg_authenc_hmac_sha224_cbc_des3_ede,
 46	&eip93_alg_authenc_hmac_sha256_cbc_des3_ede,
 47	&eip93_alg_authenc_hmac_md5_cbc_aes,
 48	&eip93_alg_authenc_hmac_sha1_cbc_aes,
 49	&eip93_alg_authenc_hmac_sha224_cbc_aes,
 50	&eip93_alg_authenc_hmac_sha256_cbc_aes,
 51	&eip93_alg_authenc_hmac_md5_rfc3686_aes,
 52	&eip93_alg_authenc_hmac_sha1_rfc3686_aes,
 53	&eip93_alg_authenc_hmac_sha224_rfc3686_aes,
 54	&eip93_alg_authenc_hmac_sha256_rfc3686_aes,
 55	&eip93_alg_md5,
 56	&eip93_alg_sha1,
 57	&eip93_alg_sha224,
 58	&eip93_alg_sha256,
 59	&eip93_alg_hmac_md5,
 60	&eip93_alg_hmac_sha1,
 61	&eip93_alg_hmac_sha224,
 62	&eip93_alg_hmac_sha256,
 63};
 64
 65inline void eip93_irq_disable(struct eip93_device *eip93, u32 mask)
 66{
 67	__raw_writel(mask, eip93->base + EIP93_REG_MASK_DISABLE);
 68}
 69
 70inline void eip93_irq_enable(struct eip93_device *eip93, u32 mask)
 71{
 72	__raw_writel(mask, eip93->base + EIP93_REG_MASK_ENABLE);
 73}
 74
 75inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask)
 76{
 77	__raw_writel(mask, eip93->base + EIP93_REG_INT_CLR);
 78}
 79
 80static int eip93_algo_is_supported(u32 alg_flags, u32 supported_algo_flags)
 81{
 82	if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) &&
 83	    !(supported_algo_flags & EIP93_PE_OPTION_TDES))
 84		return 0;
 85
 86	if (IS_AES(alg_flags) &&
 87	    !(supported_algo_flags & EIP93_PE_OPTION_AES))
 88		return 0;
 89
 90	if (IS_HASH_MD5(alg_flags) &&
 91	    !(supported_algo_flags & EIP93_PE_OPTION_MD5))
 92		return 0;
 93
 94	if (IS_HASH_SHA1(alg_flags) &&
 95	    !(supported_algo_flags & EIP93_PE_OPTION_SHA_1))
 96		return 0;
 97
 98	if (IS_HASH_SHA224(alg_flags) &&
 99	    !(supported_algo_flags & EIP93_PE_OPTION_SHA_224))
100		return 0;
101
102	if (IS_HASH_SHA256(alg_flags) &&
103	    !(supported_algo_flags & EIP93_PE_OPTION_SHA_256))
104		return 0;
105
106	return 1;
107}
108
109static void eip93_unregister_algs(u32 supported_algo_flags, unsigned int i)
110{
111	unsigned int j;
112
113	for (j = 0; j < i; j++) {
114		if (!eip93_algo_is_supported(eip93_algs[j]->flags,
115					     supported_algo_flags))
116			continue;
117
118		switch (eip93_algs[j]->type) {
119		case EIP93_ALG_TYPE_SKCIPHER:
120			crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher);
121			break;
122		case EIP93_ALG_TYPE_AEAD:
123			crypto_unregister_aead(&eip93_algs[j]->alg.aead);
124			break;
125		case EIP93_ALG_TYPE_HASH:
126			crypto_unregister_ahash(&eip93_algs[j]->alg.ahash);
127			break;
128		}
129	}
130}
131
132static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_flags)
133{
134	unsigned int i;
135	int ret = 0;
136
137	for (i = 0; i < ARRAY_SIZE(eip93_algs); i++) {
138		u32 alg_flags = eip93_algs[i]->flags;
139
140		eip93_algs[i]->eip93 = eip93;
141
142		if (!eip93_algo_is_supported(alg_flags, supported_algo_flags))
143			continue;
144
145		if (IS_AES(alg_flags) && !IS_HMAC(alg_flags)) {
146			if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128)
147				eip93_algs[i]->alg.skcipher.max_keysize =
148					AES_KEYSIZE_128;
149
150			if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192)
151				eip93_algs[i]->alg.skcipher.max_keysize =
152					AES_KEYSIZE_192;
153
154			if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256)
155				eip93_algs[i]->alg.skcipher.max_keysize =
156					AES_KEYSIZE_256;
157
158			if (IS_RFC3686(alg_flags))
159				eip93_algs[i]->alg.skcipher.max_keysize +=
160					CTR_RFC3686_NONCE_SIZE;
161		}
162
163		switch (eip93_algs[i]->type) {
164		case EIP93_ALG_TYPE_SKCIPHER:
165			ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher);
166			break;
167		case EIP93_ALG_TYPE_AEAD:
168			ret = crypto_register_aead(&eip93_algs[i]->alg.aead);
169			break;
170		case EIP93_ALG_TYPE_HASH:
171			ret = crypto_register_ahash(&eip93_algs[i]->alg.ahash);
172			break;
173		}
174		if (ret)
175			goto fail;
176	}
177
178	return 0;
179
180fail:
181	eip93_unregister_algs(supported_algo_flags, i);
182
183	return ret;
184}
185
186static void eip93_handle_result_descriptor(struct eip93_device *eip93)
187{
188	struct crypto_async_request *async;
189	struct eip93_descriptor *rdesc;
190	u16 desc_flags, crypto_idr;
191	bool last_entry;
192	int handled, left, err;
193	u32 pe_ctrl_stat;
194	u32 pe_length;
195
196get_more:
197	handled = 0;
198
199	left = readl(eip93->base + EIP93_REG_PE_RD_COUNT) & EIP93_PE_RD_COUNT;
200
201	if (!left) {
202		eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH);
203		eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH);
204		return;
205	}
206
207	last_entry = false;
208
209	while (left) {
210		scoped_guard(spinlock_irqsave, &eip93->ring->read_lock)
211			rdesc = eip93_get_descriptor(eip93);
212		if (IS_ERR(rdesc)) {
213			dev_err(eip93->dev, "Ndesc: %d nreq: %d\n",
214				handled, left);
215			err = -EIO;
216			break;
217		}
218		/* make sure DMA is finished writing */
219		do {
220			pe_ctrl_stat = READ_ONCE(rdesc->pe_ctrl_stat_word);
221			pe_length = READ_ONCE(rdesc->pe_length_word);
222		} while (FIELD_GET(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, pe_ctrl_stat) !=
223			 EIP93_PE_CTRL_PE_READY ||
224			 FIELD_GET(EIP93_PE_LENGTH_HOST_PE_READY, pe_length) !=
225			 EIP93_PE_LENGTH_PE_READY);
226
227		err = rdesc->pe_ctrl_stat_word & (EIP93_PE_CTRL_PE_EXT_ERR_CODE |
228						  EIP93_PE_CTRL_PE_EXT_ERR |
229						  EIP93_PE_CTRL_PE_SEQNUM_ERR |
230						  EIP93_PE_CTRL_PE_PAD_ERR |
231						  EIP93_PE_CTRL_PE_AUTH_ERR);
232
233		desc_flags = FIELD_GET(EIP93_PE_USER_ID_DESC_FLAGS, rdesc->user_id);
234		crypto_idr = FIELD_GET(EIP93_PE_USER_ID_CRYPTO_IDR, rdesc->user_id);
235
236		writel(1, eip93->base + EIP93_REG_PE_RD_COUNT);
237		eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH);
238
239		handled++;
240		left--;
241
242		if (desc_flags & EIP93_DESC_LAST) {
243			last_entry = true;
244			break;
245		}
246	}
247
248	if (!last_entry)
249		goto get_more;
250
251	/* Get crypto async ref only for last descriptor */
252	scoped_guard(spinlock_bh, &eip93->ring->idr_lock) {
253		async = idr_find(&eip93->ring->crypto_async_idr, crypto_idr);
254		idr_remove(&eip93->ring->crypto_async_idr, crypto_idr);
255	}
256
257	/* Parse error in ctrl stat word */
258	err = eip93_parse_ctrl_stat_err(eip93, err);
259
260	if (desc_flags & EIP93_DESC_SKCIPHER)
261		eip93_skcipher_handle_result(async, err);
262
263	if (desc_flags & EIP93_DESC_AEAD)
264		eip93_aead_handle_result(async, err);
265
266	if (desc_flags & EIP93_DESC_HASH)
267		eip93_hash_handle_result(async, err);
268
269	goto get_more;
270}
271
272static void eip93_done_task(unsigned long data)
273{
274	struct eip93_device *eip93 = (struct eip93_device *)data;
275
276	eip93_handle_result_descriptor(eip93);
277}
278
279static irqreturn_t eip93_irq_handler(int irq, void *data)
280{
281	struct eip93_device *eip93 = data;
282	u32 irq_status;
283
284	irq_status = readl(eip93->base + EIP93_REG_INT_MASK_STAT);
285	if (FIELD_GET(EIP93_INT_RDR_THRESH, irq_status)) {
286		eip93_irq_disable(eip93, EIP93_INT_RDR_THRESH);
287		tasklet_schedule(&eip93->ring->done_task);
288		return IRQ_HANDLED;
289	}
290
291	/* Ignore errors in AUTO mode, handled by the RDR */
292	eip93_irq_clear(eip93, irq_status);
293	if (irq_status)
294		eip93_irq_disable(eip93, irq_status);
295
296	return IRQ_NONE;
297}
298
299static void eip93_initialize(struct eip93_device *eip93, u32 supported_algo_flags)
300{
301	u32 val;
302
303	/* Reset PE and rings */
304	val = EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING;
305	val |= EIP93_PE_TARGET_AUTO_RING_MODE;
306	/* For Auto more, update the CDR ring owner after processing */
307	val |= EIP93_PE_CONFIG_EN_CDR_UPDATE;
308	writel(val, eip93->base + EIP93_REG_PE_CONFIG);
309
310	/* Wait for PE and ring to reset */
311	usleep_range(10, 20);
312
313	/* Release PE and ring reset */
314	val = readl(eip93->base + EIP93_REG_PE_CONFIG);
315	val &= ~(EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING);
316	writel(val, eip93->base + EIP93_REG_PE_CONFIG);
317
318	/* Config Clocks */
319	val = EIP93_PE_CLOCK_EN_PE_CLK;
320	if (supported_algo_flags & EIP93_PE_OPTION_TDES)
321		val |= EIP93_PE_CLOCK_EN_DES_CLK;
322	if (supported_algo_flags & EIP93_PE_OPTION_AES)
323		val |= EIP93_PE_CLOCK_EN_AES_CLK;
324	if (supported_algo_flags &
325	    (EIP93_PE_OPTION_MD5 | EIP93_PE_OPTION_SHA_1 | EIP93_PE_OPTION_SHA_224 |
326	     EIP93_PE_OPTION_SHA_256))
327		val |= EIP93_PE_CLOCK_EN_HASH_CLK;
328	writel(val, eip93->base + EIP93_REG_PE_CLOCK_CTRL);
329
330	/* Config DMA thresholds */
331	val = FIELD_PREP(EIP93_PE_OUTBUF_THRESH, 128) |
332	      FIELD_PREP(EIP93_PE_INBUF_THRESH, 128);
333	writel(val, eip93->base + EIP93_REG_PE_BUF_THRESH);
334
335	/* Clear/ack all interrupts before disable all */
336	eip93_irq_clear(eip93, EIP93_INT_ALL);
337	eip93_irq_disable(eip93, EIP93_INT_ALL);
338
339	/* Setup CRD threshold to trigger interrupt */
340	val = FIELD_PREP(EIPR93_PE_CDR_THRESH, EIP93_RING_NUM - EIP93_RING_BUSY);
341	/*
342	 * Configure RDR interrupt to be triggered if RD counter is not 0
343	 * for more than 2^(N+10) system clocks.
344	 */
345	val |= FIELD_PREP(EIPR93_PE_RD_TIMEOUT, 5) | EIPR93_PE_TIMEROUT_EN;
346	writel(val, eip93->base + EIP93_REG_PE_RING_THRESH);
347}
348
349static void eip93_desc_free(struct eip93_device *eip93)
350{
351	writel(0, eip93->base + EIP93_REG_PE_RING_CONFIG);
352	writel(0, eip93->base + EIP93_REG_PE_CDR_BASE);
353	writel(0, eip93->base + EIP93_REG_PE_RDR_BASE);
354}
355
356static int eip93_set_ring(struct eip93_device *eip93, struct eip93_desc_ring *ring)
357{
358	ring->offset = sizeof(struct eip93_descriptor);
359	ring->base = dmam_alloc_coherent(eip93->dev,
360					 sizeof(struct eip93_descriptor) * EIP93_RING_NUM,
361					 &ring->base_dma, GFP_KERNEL);
362	if (!ring->base)
363		return -ENOMEM;
364
365	ring->write = ring->base;
366	ring->base_end = ring->base + sizeof(struct eip93_descriptor) * (EIP93_RING_NUM - 1);
367	ring->read  = ring->base;
368
369	return 0;
370}
371
372static int eip93_desc_init(struct eip93_device *eip93)
373{
374	struct eip93_desc_ring *cdr = &eip93->ring->cdr;
375	struct eip93_desc_ring *rdr = &eip93->ring->rdr;
376	int ret;
377	u32 val;
378
379	ret = eip93_set_ring(eip93, cdr);
380	if (ret)
381		return ret;
382
383	ret = eip93_set_ring(eip93, rdr);
384	if (ret)
385		return ret;
386
387	writel((u32 __force)cdr->base_dma, eip93->base + EIP93_REG_PE_CDR_BASE);
388	writel((u32 __force)rdr->base_dma, eip93->base + EIP93_REG_PE_RDR_BASE);
389
390	val = FIELD_PREP(EIP93_PE_RING_SIZE, EIP93_RING_NUM - 1);
391	writel(val, eip93->base + EIP93_REG_PE_RING_CONFIG);
392
393	return 0;
394}
395
396static void eip93_cleanup(struct eip93_device *eip93)
397{
398	tasklet_kill(&eip93->ring->done_task);
399
400	/* Clear/ack all interrupts before disable all */
401	eip93_irq_clear(eip93, EIP93_INT_ALL);
402	eip93_irq_disable(eip93, EIP93_INT_ALL);
403
404	writel(0, eip93->base + EIP93_REG_PE_CLOCK_CTRL);
405
406	eip93_desc_free(eip93);
407
408	idr_destroy(&eip93->ring->crypto_async_idr);
409}
410
411static int eip93_crypto_probe(struct platform_device *pdev)
412{
413	struct device *dev = &pdev->dev;
414	struct eip93_device *eip93;
415	u32 ver, algo_flags;
416	int ret;
417
418	eip93 = devm_kzalloc(dev, sizeof(*eip93), GFP_KERNEL);
419	if (!eip93)
420		return -ENOMEM;
421
422	eip93->dev = dev;
423	platform_set_drvdata(pdev, eip93);
424
425	eip93->base = devm_platform_ioremap_resource(pdev, 0);
426	if (IS_ERR(eip93->base))
427		return PTR_ERR(eip93->base);
428
429	eip93->irq = platform_get_irq(pdev, 0);
430	if (eip93->irq < 0)
431		return eip93->irq;
432
433	ret = devm_request_threaded_irq(eip93->dev, eip93->irq, eip93_irq_handler,
434					NULL, IRQF_ONESHOT,
435					dev_name(eip93->dev), eip93);
436
437	eip93->ring = devm_kcalloc(eip93->dev, 1, sizeof(*eip93->ring), GFP_KERNEL);
438	if (!eip93->ring)
439		return -ENOMEM;
440
441	ret = eip93_desc_init(eip93);
442
443	if (ret)
444		return ret;
445
446	tasklet_init(&eip93->ring->done_task, eip93_done_task, (unsigned long)eip93);
447
448	spin_lock_init(&eip93->ring->read_lock);
449	spin_lock_init(&eip93->ring->write_lock);
450
451	spin_lock_init(&eip93->ring->idr_lock);
452	idr_init(&eip93->ring->crypto_async_idr);
453
454	algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1);
455
456	eip93_initialize(eip93, algo_flags);
457
458	/* Init finished, enable RDR interrupt */
459	eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH);
460
461	ret = eip93_register_algs(eip93, algo_flags);
462	if (ret) {
463		eip93_cleanup(eip93);
464		return ret;
465	}
466
467	ver = readl(eip93->base + EIP93_REG_PE_REVISION);
468	/* EIP_EIP_NO:MAJOR_HW_REV:MINOR_HW_REV:HW_PATCH,PE(ALGO_FLAGS) */
469	dev_info(eip93->dev, "EIP%lu:%lx:%lx:%lx,PE(0x%x:0x%x)\n",
470		 FIELD_GET(EIP93_PE_REVISION_EIP_NO, ver),
471		 FIELD_GET(EIP93_PE_REVISION_MAJ_HW_REV, ver),
472		 FIELD_GET(EIP93_PE_REVISION_MIN_HW_REV, ver),
473		 FIELD_GET(EIP93_PE_REVISION_HW_PATCH, ver),
474		 algo_flags,
475		 readl(eip93->base + EIP93_REG_PE_OPTION_0));
476
477	return 0;
478}
479
480static void eip93_crypto_remove(struct platform_device *pdev)
481{
482	struct eip93_device *eip93 = platform_get_drvdata(pdev);
483	u32 algo_flags;
484
485	algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1);
486
487	eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs));
488	eip93_cleanup(eip93);
489}
490
491static const struct of_device_id eip93_crypto_of_match[] = {
492	{ .compatible = "inside-secure,safexcel-eip93i", },
493	{ .compatible = "inside-secure,safexcel-eip93ie", },
494	{ .compatible = "inside-secure,safexcel-eip93is", },
495	{ .compatible = "inside-secure,safexcel-eip93ies", },
496	/* IW not supported currently, missing AES-XCB-MAC/AES-CCM */
497	/* { .compatible = "inside-secure,safexcel-eip93iw", }, */
498	{}
499};
500MODULE_DEVICE_TABLE(of, eip93_crypto_of_match);
501
502static struct platform_driver eip93_crypto_driver = {
503	.probe = eip93_crypto_probe,
504	.remove = eip93_crypto_remove,
505	.driver = {
506		.name = "inside-secure-eip93",
507		.of_match_table = eip93_crypto_of_match,
508	},
509};
510module_platform_driver(eip93_crypto_driver);
511
512MODULE_AUTHOR("Richard van Schagen <vschagen@cs.com>");
513MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>");
514MODULE_DESCRIPTION("Mediatek EIP-93 crypto engine driver");
515MODULE_LICENSE("GPL");