cptvf_algs.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2
  3/*
  4 * Copyright (C) 2016 Cavium, Inc.
  5 */
  6
  7#include <crypto/aes.h>
  8#include <crypto/algapi.h>
  9#include <crypto/authenc.h>
 10#include <crypto/internal/des.h>
 11#include <crypto/xts.h>
 12#include <linux/crypto.h>
 13#include <linux/err.h>
 14#include <linux/list.h>
 15#include <linux/scatterlist.h>
 16
 17#include "cptvf.h"
 18#include "cptvf_algs.h"
 19
 20struct cpt_device_handle {
 21	void *cdev[MAX_DEVICES];
 22	u32 dev_count;
 23};
 24
 25static struct cpt_device_handle dev_handle;
 26
 27static void cvm_callback(u32 status, void *arg)
 28{
 29	struct crypto_async_request *req = (struct crypto_async_request *)arg;
 30
 31	crypto_request_complete(req, !status);
 32}
 33
 34static inline void update_input_iv(struct cpt_request_info *req_info,
 35				   u8 *iv, u32 enc_iv_len,
 36				   u32 *argcnt)
 37{
 38	/* Setting the iv information */
 39	req_info->in[*argcnt].vptr = (void *)iv;
 40	req_info->in[*argcnt].size = enc_iv_len;
 41	req_info->req.dlen += enc_iv_len;
 42
 43	++(*argcnt);
 44}
 45
 46static inline void update_output_iv(struct cpt_request_info *req_info,
 47				    u8 *iv, u32 enc_iv_len,
 48				    u32 *argcnt)
 49{
 50	/* Setting the iv information */
 51	req_info->out[*argcnt].vptr = (void *)iv;
 52	req_info->out[*argcnt].size = enc_iv_len;
 53	req_info->rlen += enc_iv_len;
 54
 55	++(*argcnt);
 56}
 57
 58static inline void update_input_data(struct cpt_request_info *req_info,
 59				     struct scatterlist *inp_sg,
 60				     u32 nbytes, u32 *argcnt)
 61{
 62	req_info->req.dlen += nbytes;
 63
 64	while (nbytes) {
 65		u32 len = min(nbytes, inp_sg->length);
 66		u8 *ptr = sg_virt(inp_sg);
 67
 68		req_info->in[*argcnt].vptr = (void *)ptr;
 69		req_info->in[*argcnt].size = len;
 70		nbytes -= len;
 71
 72		++(*argcnt);
 73		++inp_sg;
 74	}
 75}
 76
 77static inline void update_output_data(struct cpt_request_info *req_info,
 78				      struct scatterlist *outp_sg,
 79				      u32 nbytes, u32 *argcnt)
 80{
 81	req_info->rlen += nbytes;
 82
 83	while (nbytes) {
 84		u32 len = min(nbytes, outp_sg->length);
 85		u8 *ptr = sg_virt(outp_sg);
 86
 87		req_info->out[*argcnt].vptr = (void *)ptr;
 88		req_info->out[*argcnt].size = len;
 89		nbytes -= len;
 90		++(*argcnt);
 91		++outp_sg;
 92	}
 93}
 94
 95static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
 96				 u32 *argcnt)
 97{
 98	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 99	struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
100	struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
101	struct fc_context *fctx = &rctx->fctx;
102	u32 enc_iv_len = crypto_skcipher_ivsize(tfm);
103	struct cpt_request_info *req_info = &rctx->cpt_req;
104	__be64 *ctrl_flags = NULL;
105	__be64 *offset_control;
106
107	req_info->ctrl.s.grp = 0;
108	req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
109	req_info->ctrl.s.se_req = SE_CORE_REQ;
110
111	req_info->req.opcode.s.major = MAJOR_OP_FC |
112					DMA_MODE_FLAG(DMA_GATHER_SCATTER);
113	if (enc)
114		req_info->req.opcode.s.minor = 2;
115	else
116		req_info->req.opcode.s.minor = 3;
117
118	req_info->req.param1 = req->cryptlen; /* Encryption Data length */
119	req_info->req.param2 = 0; /*Auth data length */
120
121	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
122	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
123	fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
124
125	if (ctx->cipher_type == AES_XTS)
126		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
127	else
128		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
129	ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags;
130	*ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags);
131
132	offset_control = (__be64 *)&rctx->control_word;
133	*offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
134	/* Storing  Packet Data Information in offset
135	 * Control Word First 8 bytes
136	 */
137	req_info->in[*argcnt].vptr = (u8 *)offset_control;
138	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
139	req_info->req.dlen += CONTROL_WORD_LEN;
140	++(*argcnt);
141
142	req_info->in[*argcnt].vptr = (u8 *)fctx;
143	req_info->in[*argcnt].size = sizeof(struct fc_context);
144	req_info->req.dlen += sizeof(struct fc_context);
145
146	++(*argcnt);
147
148	return 0;
149}
150
151static inline u32 create_input_list(struct skcipher_request  *req, u32 enc,
152				    u32 enc_iv_len)
153{
154	struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
155	struct cpt_request_info *req_info = &rctx->cpt_req;
156	u32 argcnt =  0;
157
158	create_ctx_hdr(req, enc, &argcnt);
159	update_input_iv(req_info, req->iv, enc_iv_len, &argcnt);
160	update_input_data(req_info, req->src, req->cryptlen, &argcnt);
161	req_info->incnt = argcnt;
162
163	return 0;
164}
165
166static inline void store_cb_info(struct skcipher_request *req,
167				 struct cpt_request_info *req_info)
168{
169	req_info->callback = (void *)cvm_callback;
170	req_info->callback_arg = (void *)&req->base;
171}
172
173static inline void create_output_list(struct skcipher_request *req,
174				      u32 enc_iv_len)
175{
176	struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
177	struct cpt_request_info *req_info = &rctx->cpt_req;
178	u32 argcnt = 0;
179
180	/* OUTPUT Buffer Processing
181	 * AES encryption/decryption output would be
182	 * received in the following format
183	 *
184	 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
185	 * [ 16 Bytes/     [   Request Enc/Dec/ DATA Len AES CBC ]
186	 */
187	/* Reading IV information */
188	update_output_iv(req_info, req->iv, enc_iv_len, &argcnt);
189	update_output_data(req_info, req->dst, req->cryptlen, &argcnt);
190	req_info->outcnt = argcnt;
191}
192
193static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc)
194{
195	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
196	struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
197	u32 enc_iv_len = crypto_skcipher_ivsize(tfm);
198	struct fc_context *fctx = &rctx->fctx;
199	struct cpt_request_info *req_info = &rctx->cpt_req;
200	void *cdev = NULL;
201	int status;
202
203	memset(req_info, 0, sizeof(struct cpt_request_info));
204	req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0;
205	memset(fctx, 0, sizeof(struct fc_context));
206	create_input_list(req, enc, enc_iv_len);
207	create_output_list(req, enc_iv_len);
208	store_cb_info(req, req_info);
209	cdev = dev_handle.cdev[smp_processor_id()];
210	status = cptvf_do_request(cdev, req_info);
211	/* We perform an asynchronous send and once
212	 * the request is completed the driver would
213	 * intimate through  registered call back functions
214	 */
215
216	if (status)
217		return status;
218	else
219		return -EINPROGRESS;
220}
221
222static int cvm_encrypt(struct skcipher_request *req)
223{
224	return cvm_enc_dec(req, true);
225}
226
227static int cvm_decrypt(struct skcipher_request *req)
228{
229	return cvm_enc_dec(req, false);
230}
231
232static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,
233		   u32 keylen)
234{
235	struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher);
236	int err;
237	const u8 *key1 = key;
238	const u8 *key2 = key + (keylen / 2);
239
240	err = xts_verify_key(cipher, key, keylen);
241	if (err)
242		return err;
243	ctx->key_len = keylen;
244	memcpy(ctx->enc_key, key1, keylen / 2);
245	memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
246	ctx->cipher_type = AES_XTS;
247	switch (ctx->key_len) {
248	case 32:
249		ctx->key_type = AES_128_BIT;
250		break;
251	case 64:
252		ctx->key_type = AES_256_BIT;
253		break;
254	default:
255		return -EINVAL;
256	}
257
258	return 0;
259}
260
261static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
262{
263	if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
264		ctx->key_len = keylen;
265		switch (ctx->key_len) {
266		case 16:
267			ctx->key_type = AES_128_BIT;
268			break;
269		case 24:
270			ctx->key_type = AES_192_BIT;
271			break;
272		case 32:
273			ctx->key_type = AES_256_BIT;
274			break;
275		default:
276			return -EINVAL;
277		}
278
279		if (ctx->cipher_type == DES3_CBC)
280			ctx->key_type = 0;
281
282		return 0;
283	}
284
285	return -EINVAL;
286}
287
288static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key,
289		      u32 keylen, u8 cipher_type)
290{
291	struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher);
292
293	ctx->cipher_type = cipher_type;
294	if (!cvm_validate_keylen(ctx, keylen)) {
295		memcpy(ctx->enc_key, key, keylen);
296		return 0;
297	} else {
298		return -EINVAL;
299	}
300}
301
302static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
303			      u32 keylen)
304{
305	return cvm_setkey(cipher, key, keylen, AES_CBC);
306}
307
308static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
309			      u32 keylen)
310{
311	return cvm_setkey(cipher, key, keylen, AES_ECB);
312}
313
314static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
315			       u32 keylen)
316{
317	return verify_skcipher_des3_key(cipher, key) ?:
318	       cvm_setkey(cipher, key, keylen, DES3_CBC);
319}
320
321static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
322			       u32 keylen)
323{
324	return verify_skcipher_des3_key(cipher, key) ?:
325	       cvm_setkey(cipher, key, keylen, DES3_ECB);
326}
327
328static int cvm_enc_dec_init(struct crypto_skcipher *tfm)
329{
330	crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct cvm_req_ctx));
331
332	return 0;
333}
334
335static struct skcipher_alg algs[] = { {
336	.base.cra_flags		= CRYPTO_ALG_ASYNC |
337				  CRYPTO_ALG_ALLOCATES_MEMORY,
338	.base.cra_blocksize	= AES_BLOCK_SIZE,
339	.base.cra_ctxsize	= sizeof(struct cvm_enc_ctx),
340	.base.cra_alignmask	= 7,
341	.base.cra_priority	= 4001,
342	.base.cra_name		= "xts(aes)",
343	.base.cra_driver_name	= "cavium-xts-aes",
344	.base.cra_module	= THIS_MODULE,
345
346	.ivsize			= AES_BLOCK_SIZE,
347	.min_keysize		= 2 * AES_MIN_KEY_SIZE,
348	.max_keysize		= 2 * AES_MAX_KEY_SIZE,
349	.setkey			= cvm_xts_setkey,
350	.encrypt		= cvm_encrypt,
351	.decrypt		= cvm_decrypt,
352	.init			= cvm_enc_dec_init,
353}, {
354	.base.cra_flags		= CRYPTO_ALG_ASYNC |
355				  CRYPTO_ALG_ALLOCATES_MEMORY,
356	.base.cra_blocksize	= AES_BLOCK_SIZE,
357	.base.cra_ctxsize	= sizeof(struct cvm_enc_ctx),
358	.base.cra_alignmask	= 7,
359	.base.cra_priority	= 4001,
360	.base.cra_name		= "cbc(aes)",
361	.base.cra_driver_name	= "cavium-cbc-aes",
362	.base.cra_module	= THIS_MODULE,
363
364	.ivsize			= AES_BLOCK_SIZE,
365	.min_keysize		= AES_MIN_KEY_SIZE,
366	.max_keysize		= AES_MAX_KEY_SIZE,
367	.setkey			= cvm_cbc_aes_setkey,
368	.encrypt		= cvm_encrypt,
369	.decrypt		= cvm_decrypt,
370	.init			= cvm_enc_dec_init,
371}, {
372	.base.cra_flags		= CRYPTO_ALG_ASYNC |
373				  CRYPTO_ALG_ALLOCATES_MEMORY,
374	.base.cra_blocksize	= AES_BLOCK_SIZE,
375	.base.cra_ctxsize	= sizeof(struct cvm_enc_ctx),
376	.base.cra_alignmask	= 7,
377	.base.cra_priority	= 4001,
378	.base.cra_name		= "ecb(aes)",
379	.base.cra_driver_name	= "cavium-ecb-aes",
380	.base.cra_module	= THIS_MODULE,
381
382	.min_keysize		= AES_MIN_KEY_SIZE,
383	.max_keysize		= AES_MAX_KEY_SIZE,
384	.setkey			= cvm_ecb_aes_setkey,
385	.encrypt		= cvm_encrypt,
386	.decrypt		= cvm_decrypt,
387	.init			= cvm_enc_dec_init,
388}, {
389	.base.cra_flags		= CRYPTO_ALG_ASYNC |
390				  CRYPTO_ALG_ALLOCATES_MEMORY,
391	.base.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
392	.base.cra_ctxsize	= sizeof(struct cvm_des3_ctx),
393	.base.cra_alignmask	= 7,
394	.base.cra_priority	= 4001,
395	.base.cra_name		= "cbc(des3_ede)",
396	.base.cra_driver_name	= "cavium-cbc-des3_ede",
397	.base.cra_module	= THIS_MODULE,
398
399	.min_keysize		= DES3_EDE_KEY_SIZE,
400	.max_keysize		= DES3_EDE_KEY_SIZE,
401	.ivsize			= DES_BLOCK_SIZE,
402	.setkey			= cvm_cbc_des3_setkey,
403	.encrypt		= cvm_encrypt,
404	.decrypt		= cvm_decrypt,
405	.init			= cvm_enc_dec_init,
406}, {
407	.base.cra_flags		= CRYPTO_ALG_ASYNC |
408				  CRYPTO_ALG_ALLOCATES_MEMORY,
409	.base.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
410	.base.cra_ctxsize	= sizeof(struct cvm_des3_ctx),
411	.base.cra_alignmask	= 7,
412	.base.cra_priority	= 4001,
413	.base.cra_name		= "ecb(des3_ede)",
414	.base.cra_driver_name	= "cavium-ecb-des3_ede",
415	.base.cra_module	= THIS_MODULE,
416
417	.min_keysize		= DES3_EDE_KEY_SIZE,
418	.max_keysize		= DES3_EDE_KEY_SIZE,
419	.ivsize			= DES_BLOCK_SIZE,
420	.setkey			= cvm_ecb_des3_setkey,
421	.encrypt		= cvm_encrypt,
422	.decrypt		= cvm_decrypt,
423	.init			= cvm_enc_dec_init,
424} };
425
426static inline int cav_register_algs(void)
427{
428	return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
429}
430
431static inline void cav_unregister_algs(void)
432{
433	crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
434}
435
436int cvm_crypto_init(struct cpt_vf *cptvf)
437{
438	struct pci_dev *pdev = cptvf->pdev;
439	u32 dev_count;
440
441	dev_count = dev_handle.dev_count;
442	dev_handle.cdev[dev_count] = cptvf;
443	dev_handle.dev_count++;
444
445	if (dev_count == 3) {
446		if (cav_register_algs()) {
447			dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
448			return -EINVAL;
449		}
450	}
451
452	return 0;
453}
454
455void cvm_crypto_exit(void)
456{
457	u32 dev_count;
458
459	dev_count = --dev_handle.dev_count;
460	if (!dev_count)
461		cav_unregister_algs();
462}