hifn_795x.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
   4 * All rights reserved.
   5 */
   6
   7#include <linux/kernel.h>
   8#include <linux/module.h>
   9#include <linux/moduleparam.h>
  10#include <linux/mod_devicetable.h>
  11#include <linux/interrupt.h>
  12#include <linux/pci.h>
  13#include <linux/slab.h>
  14#include <linux/delay.h>
  15#include <linux/mm.h>
  16#include <linux/dma-mapping.h>
  17#include <linux/scatterlist.h>
  18#include <linux/highmem.h>
  19#include <linux/crypto.h>
  20#include <linux/hw_random.h>
  21#include <linux/ktime.h>
  22
  23#include <crypto/algapi.h>
  24#include <crypto/internal/des.h>
  25#include <crypto/internal/skcipher.h>
  26
  27static char hifn_pll_ref[sizeof("extNNN")] = "ext";
  28module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
  29MODULE_PARM_DESC(hifn_pll_ref,
  30		 "PLL reference clock (pci[freq] or ext[freq], default ext)");
  31
  32static atomic_t hifn_dev_number;
  33
  34#define ACRYPTO_OP_DECRYPT	0
  35#define ACRYPTO_OP_ENCRYPT	1
  36#define ACRYPTO_OP_HMAC		2
  37#define ACRYPTO_OP_RNG		3
  38
  39#define ACRYPTO_MODE_ECB		0
  40#define ACRYPTO_MODE_CBC		1
  41#define ACRYPTO_MODE_CFB		2
  42#define ACRYPTO_MODE_OFB		3
  43
  44#define ACRYPTO_TYPE_AES_128	0
  45#define ACRYPTO_TYPE_AES_192	1
  46#define ACRYPTO_TYPE_AES_256	2
  47#define ACRYPTO_TYPE_3DES	3
  48#define ACRYPTO_TYPE_DES	4
  49
  50#define PCI_VENDOR_ID_HIFN		0x13A3
  51#define PCI_DEVICE_ID_HIFN_7955		0x0020
  52#define	PCI_DEVICE_ID_HIFN_7956		0x001d
  53
  54/* I/O region sizes */
  55
  56#define HIFN_BAR0_SIZE			0x1000
  57#define HIFN_BAR1_SIZE			0x2000
  58#define HIFN_BAR2_SIZE			0x8000
  59
  60/* DMA registres */
  61
  62#define HIFN_DMA_CRA			0x0C	/* DMA Command Ring Address */
  63#define HIFN_DMA_SDRA			0x1C	/* DMA Source Data Ring Address */
  64#define HIFN_DMA_RRA			0x2C	/* DMA Result Ring Address */
  65#define HIFN_DMA_DDRA			0x3C	/* DMA Destination Data Ring Address */
  66#define HIFN_DMA_STCTL			0x40	/* DMA Status and Control */
  67#define HIFN_DMA_INTREN			0x44	/* DMA Interrupt Enable */
  68#define HIFN_DMA_CFG1			0x48	/* DMA Configuration #1 */
  69#define HIFN_DMA_CFG2			0x6C	/* DMA Configuration #2 */
  70#define HIFN_CHIP_ID			0x98	/* Chip ID */
  71
  72/*
  73 * Processing Unit Registers (offset from BASEREG0)
  74 */
  75#define	HIFN_0_PUDATA		0x00	/* Processing Unit Data */
  76#define	HIFN_0_PUCTRL		0x04	/* Processing Unit Control */
  77#define	HIFN_0_PUISR		0x08	/* Processing Unit Interrupt Status */
  78#define	HIFN_0_PUCNFG		0x0c	/* Processing Unit Configuration */
  79#define	HIFN_0_PUIER		0x10	/* Processing Unit Interrupt Enable */
  80#define	HIFN_0_PUSTAT		0x14	/* Processing Unit Status/Chip ID */
  81#define	HIFN_0_FIFOSTAT		0x18	/* FIFO Status */
  82#define	HIFN_0_FIFOCNFG		0x1c	/* FIFO Configuration */
  83#define	HIFN_0_SPACESIZE	0x20	/* Register space size */
  84
  85/* Processing Unit Control Register (HIFN_0_PUCTRL) */
  86#define	HIFN_PUCTRL_CLRSRCFIFO	0x0010	/* clear source fifo */
  87#define	HIFN_PUCTRL_STOP	0x0008	/* stop pu */
  88#define	HIFN_PUCTRL_LOCKRAM	0x0004	/* lock ram */
  89#define	HIFN_PUCTRL_DMAENA	0x0002	/* enable dma */
  90#define	HIFN_PUCTRL_RESET	0x0001	/* Reset processing unit */
  91
  92/* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */
  93#define	HIFN_PUISR_CMDINVAL	0x8000	/* Invalid command interrupt */
  94#define	HIFN_PUISR_DATAERR	0x4000	/* Data error interrupt */
  95#define	HIFN_PUISR_SRCFIFO	0x2000	/* Source FIFO ready interrupt */
  96#define	HIFN_PUISR_DSTFIFO	0x1000	/* Destination FIFO ready interrupt */
  97#define	HIFN_PUISR_DSTOVER	0x0200	/* Destination overrun interrupt */
  98#define	HIFN_PUISR_SRCCMD	0x0080	/* Source command interrupt */
  99#define	HIFN_PUISR_SRCCTX	0x0040	/* Source context interrupt */
 100#define	HIFN_PUISR_SRCDATA	0x0020	/* Source data interrupt */
 101#define	HIFN_PUISR_DSTDATA	0x0010	/* Destination data interrupt */
 102#define	HIFN_PUISR_DSTRESULT	0x0004	/* Destination result interrupt */
 103
 104/* Processing Unit Configuration Register (HIFN_0_PUCNFG) */
 105#define	HIFN_PUCNFG_DRAMMASK	0xe000	/* DRAM size mask */
 106#define	HIFN_PUCNFG_DSZ_256K	0x0000	/* 256k dram */
 107#define	HIFN_PUCNFG_DSZ_512K	0x2000	/* 512k dram */
 108#define	HIFN_PUCNFG_DSZ_1M	0x4000	/* 1m dram */
 109#define	HIFN_PUCNFG_DSZ_2M	0x6000	/* 2m dram */
 110#define	HIFN_PUCNFG_DSZ_4M	0x8000	/* 4m dram */
 111#define	HIFN_PUCNFG_DSZ_8M	0xa000	/* 8m dram */
 112#define	HIFN_PUNCFG_DSZ_16M	0xc000	/* 16m dram */
 113#define	HIFN_PUCNFG_DSZ_32M	0xe000	/* 32m dram */
 114#define	HIFN_PUCNFG_DRAMREFRESH	0x1800	/* DRAM refresh rate mask */
 115#define	HIFN_PUCNFG_DRFR_512	0x0000	/* 512 divisor of ECLK */
 116#define	HIFN_PUCNFG_DRFR_256	0x0800	/* 256 divisor of ECLK */
 117#define	HIFN_PUCNFG_DRFR_128	0x1000	/* 128 divisor of ECLK */
 118#define	HIFN_PUCNFG_TCALLPHASES	0x0200	/* your guess is as good as mine... */
 119#define	HIFN_PUCNFG_TCDRVTOTEM	0x0100	/* your guess is as good as mine... */
 120#define	HIFN_PUCNFG_BIGENDIAN	0x0080	/* DMA big endian mode */
 121#define	HIFN_PUCNFG_BUS32	0x0040	/* Bus width 32bits */
 122#define	HIFN_PUCNFG_BUS16	0x0000	/* Bus width 16 bits */
 123#define	HIFN_PUCNFG_CHIPID	0x0020	/* Allow chipid from PUSTAT */
 124#define	HIFN_PUCNFG_DRAM	0x0010	/* Context RAM is DRAM */
 125#define	HIFN_PUCNFG_SRAM	0x0000	/* Context RAM is SRAM */
 126#define	HIFN_PUCNFG_COMPSING	0x0004	/* Enable single compression context */
 127#define	HIFN_PUCNFG_ENCCNFG	0x0002	/* Encryption configuration */
 128
 129/* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */
 130#define	HIFN_PUIER_CMDINVAL	0x8000	/* Invalid command interrupt */
 131#define	HIFN_PUIER_DATAERR	0x4000	/* Data error interrupt */
 132#define	HIFN_PUIER_SRCFIFO	0x2000	/* Source FIFO ready interrupt */
 133#define	HIFN_PUIER_DSTFIFO	0x1000	/* Destination FIFO ready interrupt */
 134#define	HIFN_PUIER_DSTOVER	0x0200	/* Destination overrun interrupt */
 135#define	HIFN_PUIER_SRCCMD	0x0080	/* Source command interrupt */
 136#define	HIFN_PUIER_SRCCTX	0x0040	/* Source context interrupt */
 137#define	HIFN_PUIER_SRCDATA	0x0020	/* Source data interrupt */
 138#define	HIFN_PUIER_DSTDATA	0x0010	/* Destination data interrupt */
 139#define	HIFN_PUIER_DSTRESULT	0x0004	/* Destination result interrupt */
 140
 141/* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */
 142#define	HIFN_PUSTAT_CMDINVAL	0x8000	/* Invalid command interrupt */
 143#define	HIFN_PUSTAT_DATAERR	0x4000	/* Data error interrupt */
 144#define	HIFN_PUSTAT_SRCFIFO	0x2000	/* Source FIFO ready interrupt */
 145#define	HIFN_PUSTAT_DSTFIFO	0x1000	/* Destination FIFO ready interrupt */
 146#define	HIFN_PUSTAT_DSTOVER	0x0200	/* Destination overrun interrupt */
 147#define	HIFN_PUSTAT_SRCCMD	0x0080	/* Source command interrupt */
 148#define	HIFN_PUSTAT_SRCCTX	0x0040	/* Source context interrupt */
 149#define	HIFN_PUSTAT_SRCDATA	0x0020	/* Source data interrupt */
 150#define	HIFN_PUSTAT_DSTDATA	0x0010	/* Destination data interrupt */
 151#define	HIFN_PUSTAT_DSTRESULT	0x0004	/* Destination result interrupt */
 152#define	HIFN_PUSTAT_CHIPREV	0x00ff	/* Chip revision mask */
 153#define	HIFN_PUSTAT_CHIPENA	0xff00	/* Chip enabled mask */
 154#define	HIFN_PUSTAT_ENA_2	0x1100	/* Level 2 enabled */
 155#define	HIFN_PUSTAT_ENA_1	0x1000	/* Level 1 enabled */
 156#define	HIFN_PUSTAT_ENA_0	0x3000	/* Level 0 enabled */
 157#define	HIFN_PUSTAT_REV_2	0x0020	/* 7751 PT6/2 */
 158#define	HIFN_PUSTAT_REV_3	0x0030	/* 7751 PT6/3 */
 159
 160/* FIFO Status Register (HIFN_0_FIFOSTAT) */
 161#define	HIFN_FIFOSTAT_SRC	0x7f00	/* Source FIFO available */
 162#define	HIFN_FIFOSTAT_DST	0x007f	/* Destination FIFO available */
 163
 164/* FIFO Configuration Register (HIFN_0_FIFOCNFG) */
 165#define	HIFN_FIFOCNFG_THRESHOLD	0x0400	/* must be written as 1 */
 166
 167/*
 168 * DMA Interface Registers (offset from BASEREG1)
 169 */
 170#define	HIFN_1_DMA_CRAR		0x0c	/* DMA Command Ring Address */
 171#define	HIFN_1_DMA_SRAR		0x1c	/* DMA Source Ring Address */
 172#define	HIFN_1_DMA_RRAR		0x2c	/* DMA Result Ring Address */
 173#define	HIFN_1_DMA_DRAR		0x3c	/* DMA Destination Ring Address */
 174#define	HIFN_1_DMA_CSR		0x40	/* DMA Status and Control */
 175#define	HIFN_1_DMA_IER		0x44	/* DMA Interrupt Enable */
 176#define	HIFN_1_DMA_CNFG		0x48	/* DMA Configuration */
 177#define	HIFN_1_PLL		0x4c	/* 795x: PLL config */
 178#define	HIFN_1_7811_RNGENA	0x60	/* 7811: rng enable */
 179#define	HIFN_1_7811_RNGCFG	0x64	/* 7811: rng config */
 180#define	HIFN_1_7811_RNGDAT	0x68	/* 7811: rng data */
 181#define	HIFN_1_7811_RNGSTS	0x6c	/* 7811: rng status */
 182#define	HIFN_1_7811_MIPSRST	0x94	/* 7811: MIPS reset */
 183#define	HIFN_1_REVID		0x98	/* Revision ID */
 184#define	HIFN_1_UNLOCK_SECRET1	0xf4
 185#define	HIFN_1_UNLOCK_SECRET2	0xfc
 186#define	HIFN_1_PUB_RESET	0x204	/* Public/RNG Reset */
 187#define	HIFN_1_PUB_BASE		0x300	/* Public Base Address */
 188#define	HIFN_1_PUB_OPLEN	0x304	/* Public Operand Length */
 189#define	HIFN_1_PUB_OP		0x308	/* Public Operand */
 190#define	HIFN_1_PUB_STATUS	0x30c	/* Public Status */
 191#define	HIFN_1_PUB_IEN		0x310	/* Public Interrupt enable */
 192#define	HIFN_1_RNG_CONFIG	0x314	/* RNG config */
 193#define	HIFN_1_RNG_DATA		0x318	/* RNG data */
 194#define	HIFN_1_PUB_MEM		0x400	/* start of Public key memory */
 195#define	HIFN_1_PUB_MEMEND	0xbff	/* end of Public key memory */
 196
 197/* DMA Status and Control Register (HIFN_1_DMA_CSR) */
 198#define	HIFN_DMACSR_D_CTRLMASK	0xc0000000	/* Destinition Ring Control */
 199#define	HIFN_DMACSR_D_CTRL_NOP	0x00000000	/* Dest. Control: no-op */
 200#define	HIFN_DMACSR_D_CTRL_DIS	0x40000000	/* Dest. Control: disable */
 201#define	HIFN_DMACSR_D_CTRL_ENA	0x80000000	/* Dest. Control: enable */
 202#define	HIFN_DMACSR_D_ABORT	0x20000000	/* Destinition Ring PCIAbort */
 203#define	HIFN_DMACSR_D_DONE	0x10000000	/* Destinition Ring Done */
 204#define	HIFN_DMACSR_D_LAST	0x08000000	/* Destinition Ring Last */
 205#define	HIFN_DMACSR_D_WAIT	0x04000000	/* Destinition Ring Waiting */
 206#define	HIFN_DMACSR_D_OVER	0x02000000	/* Destinition Ring Overflow */
 207#define	HIFN_DMACSR_R_CTRL	0x00c00000	/* Result Ring Control */
 208#define	HIFN_DMACSR_R_CTRL_NOP	0x00000000	/* Result Control: no-op */
 209#define	HIFN_DMACSR_R_CTRL_DIS	0x00400000	/* Result Control: disable */
 210#define	HIFN_DMACSR_R_CTRL_ENA	0x00800000	/* Result Control: enable */
 211#define	HIFN_DMACSR_R_ABORT	0x00200000	/* Result Ring PCI Abort */
 212#define	HIFN_DMACSR_R_DONE	0x00100000	/* Result Ring Done */
 213#define	HIFN_DMACSR_R_LAST	0x00080000	/* Result Ring Last */
 214#define	HIFN_DMACSR_R_WAIT	0x00040000	/* Result Ring Waiting */
 215#define	HIFN_DMACSR_R_OVER	0x00020000	/* Result Ring Overflow */
 216#define	HIFN_DMACSR_S_CTRL	0x0000c000	/* Source Ring Control */
 217#define	HIFN_DMACSR_S_CTRL_NOP	0x00000000	/* Source Control: no-op */
 218#define	HIFN_DMACSR_S_CTRL_DIS	0x00004000	/* Source Control: disable */
 219#define	HIFN_DMACSR_S_CTRL_ENA	0x00008000	/* Source Control: enable */
 220#define	HIFN_DMACSR_S_ABORT	0x00002000	/* Source Ring PCI Abort */
 221#define	HIFN_DMACSR_S_DONE	0x00001000	/* Source Ring Done */
 222#define	HIFN_DMACSR_S_LAST	0x00000800	/* Source Ring Last */
 223#define	HIFN_DMACSR_S_WAIT	0x00000400	/* Source Ring Waiting */
 224#define	HIFN_DMACSR_ILLW	0x00000200	/* Illegal write (7811 only) */
 225#define	HIFN_DMACSR_ILLR	0x00000100	/* Illegal read (7811 only) */
 226#define	HIFN_DMACSR_C_CTRL	0x000000c0	/* Command Ring Control */
 227#define	HIFN_DMACSR_C_CTRL_NOP	0x00000000	/* Command Control: no-op */
 228#define	HIFN_DMACSR_C_CTRL_DIS	0x00000040	/* Command Control: disable */
 229#define	HIFN_DMACSR_C_CTRL_ENA	0x00000080	/* Command Control: enable */
 230#define	HIFN_DMACSR_C_ABORT	0x00000020	/* Command Ring PCI Abort */
 231#define	HIFN_DMACSR_C_DONE	0x00000010	/* Command Ring Done */
 232#define	HIFN_DMACSR_C_LAST	0x00000008	/* Command Ring Last */
 233#define	HIFN_DMACSR_C_WAIT	0x00000004	/* Command Ring Waiting */
 234#define	HIFN_DMACSR_PUBDONE	0x00000002	/* Public op done (7951 only) */
 235#define	HIFN_DMACSR_ENGINE	0x00000001	/* Command Ring Engine IRQ */
 236
 237/* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */
 238#define	HIFN_DMAIER_D_ABORT	0x20000000	/* Destination Ring PCIAbort */
 239#define	HIFN_DMAIER_D_DONE	0x10000000	/* Destination Ring Done */
 240#define	HIFN_DMAIER_D_LAST	0x08000000	/* Destination Ring Last */
 241#define	HIFN_DMAIER_D_WAIT	0x04000000	/* Destination Ring Waiting */
 242#define	HIFN_DMAIER_D_OVER	0x02000000	/* Destination Ring Overflow */
 243#define	HIFN_DMAIER_R_ABORT	0x00200000	/* Result Ring PCI Abort */
 244#define	HIFN_DMAIER_R_DONE	0x00100000	/* Result Ring Done */
 245#define	HIFN_DMAIER_R_LAST	0x00080000	/* Result Ring Last */
 246#define	HIFN_DMAIER_R_WAIT	0x00040000	/* Result Ring Waiting */
 247#define	HIFN_DMAIER_R_OVER	0x00020000	/* Result Ring Overflow */
 248#define	HIFN_DMAIER_S_ABORT	0x00002000	/* Source Ring PCI Abort */
 249#define	HIFN_DMAIER_S_DONE	0x00001000	/* Source Ring Done */
 250#define	HIFN_DMAIER_S_LAST	0x00000800	/* Source Ring Last */
 251#define	HIFN_DMAIER_S_WAIT	0x00000400	/* Source Ring Waiting */
 252#define	HIFN_DMAIER_ILLW	0x00000200	/* Illegal write (7811 only) */
 253#define	HIFN_DMAIER_ILLR	0x00000100	/* Illegal read (7811 only) */
 254#define	HIFN_DMAIER_C_ABORT	0x00000020	/* Command Ring PCI Abort */
 255#define	HIFN_DMAIER_C_DONE	0x00000010	/* Command Ring Done */
 256#define	HIFN_DMAIER_C_LAST	0x00000008	/* Command Ring Last */
 257#define	HIFN_DMAIER_C_WAIT	0x00000004	/* Command Ring Waiting */
 258#define	HIFN_DMAIER_PUBDONE	0x00000002	/* public op done (7951 only) */
 259#define	HIFN_DMAIER_ENGINE	0x00000001	/* Engine IRQ */
 260
 261/* DMA Configuration Register (HIFN_1_DMA_CNFG) */
 262#define	HIFN_DMACNFG_BIGENDIAN	0x10000000	/* big endian mode */
 263#define	HIFN_DMACNFG_POLLFREQ	0x00ff0000	/* Poll frequency mask */
 264#define	HIFN_DMACNFG_UNLOCK	0x00000800
 265#define	HIFN_DMACNFG_POLLINVAL	0x00000700	/* Invalid Poll Scalar */
 266#define	HIFN_DMACNFG_LAST	0x00000010	/* Host control LAST bit */
 267#define	HIFN_DMACNFG_MODE	0x00000004	/* DMA mode */
 268#define	HIFN_DMACNFG_DMARESET	0x00000002	/* DMA Reset # */
 269#define	HIFN_DMACNFG_MSTRESET	0x00000001	/* Master Reset # */
 270
 271/* PLL configuration register */
 272#define HIFN_PLL_REF_CLK_HBI	0x00000000	/* HBI reference clock */
 273#define HIFN_PLL_REF_CLK_PLL	0x00000001	/* PLL reference clock */
 274#define HIFN_PLL_BP		0x00000002	/* Reference clock bypass */
 275#define HIFN_PLL_PK_CLK_HBI	0x00000000	/* PK engine HBI clock */
 276#define HIFN_PLL_PK_CLK_PLL	0x00000008	/* PK engine PLL clock */
 277#define HIFN_PLL_PE_CLK_HBI	0x00000000	/* PE engine HBI clock */
 278#define HIFN_PLL_PE_CLK_PLL	0x00000010	/* PE engine PLL clock */
 279#define HIFN_PLL_RESERVED_1	0x00000400	/* Reserved bit, must be 1 */
 280#define HIFN_PLL_ND_SHIFT	11		/* Clock multiplier shift */
 281#define HIFN_PLL_ND_MULT_2	0x00000000	/* PLL clock multiplier 2 */
 282#define HIFN_PLL_ND_MULT_4	0x00000800	/* PLL clock multiplier 4 */
 283#define HIFN_PLL_ND_MULT_6	0x00001000	/* PLL clock multiplier 6 */
 284#define HIFN_PLL_ND_MULT_8	0x00001800	/* PLL clock multiplier 8 */
 285#define HIFN_PLL_ND_MULT_10	0x00002000	/* PLL clock multiplier 10 */
 286#define HIFN_PLL_ND_MULT_12	0x00002800	/* PLL clock multiplier 12 */
 287#define HIFN_PLL_IS_1_8		0x00000000	/* charge pump (mult. 1-8) */
 288#define HIFN_PLL_IS_9_12	0x00010000	/* charge pump (mult. 9-12) */
 289
 290#define HIFN_PLL_FCK_MAX	266		/* Maximum PLL frequency */
 291
 292/* Public key reset register (HIFN_1_PUB_RESET) */
 293#define	HIFN_PUBRST_RESET	0x00000001	/* reset public/rng unit */
 294
 295/* Public base address register (HIFN_1_PUB_BASE) */
 296#define	HIFN_PUBBASE_ADDR	0x00003fff	/* base address */
 297
 298/* Public operand length register (HIFN_1_PUB_OPLEN) */
 299#define	HIFN_PUBOPLEN_MOD_M	0x0000007f	/* modulus length mask */
 300#define	HIFN_PUBOPLEN_MOD_S	0		/* modulus length shift */
 301#define	HIFN_PUBOPLEN_EXP_M	0x0003ff80	/* exponent length mask */
 302#define	HIFN_PUBOPLEN_EXP_S	7		/* exponent length shift */
 303#define	HIFN_PUBOPLEN_RED_M	0x003c0000	/* reducend length mask */
 304#define	HIFN_PUBOPLEN_RED_S	18		/* reducend length shift */
 305
 306/* Public operation register (HIFN_1_PUB_OP) */
 307#define	HIFN_PUBOP_AOFFSET_M	0x0000007f	/* A offset mask */
 308#define	HIFN_PUBOP_AOFFSET_S	0		/* A offset shift */
 309#define	HIFN_PUBOP_BOFFSET_M	0x00000f80	/* B offset mask */
 310#define	HIFN_PUBOP_BOFFSET_S	7		/* B offset shift */
 311#define	HIFN_PUBOP_MOFFSET_M	0x0003f000	/* M offset mask */
 312#define	HIFN_PUBOP_MOFFSET_S	12		/* M offset shift */
 313#define	HIFN_PUBOP_OP_MASK	0x003c0000	/* Opcode: */
 314#define	HIFN_PUBOP_OP_NOP	0x00000000	/*  NOP */
 315#define	HIFN_PUBOP_OP_ADD	0x00040000	/*  ADD */
 316#define	HIFN_PUBOP_OP_ADDC	0x00080000	/*  ADD w/carry */
 317#define	HIFN_PUBOP_OP_SUB	0x000c0000	/*  SUB */
 318#define	HIFN_PUBOP_OP_SUBC	0x00100000	/*  SUB w/carry */
 319#define	HIFN_PUBOP_OP_MODADD	0x00140000	/*  Modular ADD */
 320#define	HIFN_PUBOP_OP_MODSUB	0x00180000	/*  Modular SUB */
 321#define	HIFN_PUBOP_OP_INCA	0x001c0000	/*  INC A */
 322#define	HIFN_PUBOP_OP_DECA	0x00200000	/*  DEC A */
 323#define	HIFN_PUBOP_OP_MULT	0x00240000	/*  MULT */
 324#define	HIFN_PUBOP_OP_MODMULT	0x00280000	/*  Modular MULT */
 325#define	HIFN_PUBOP_OP_MODRED	0x002c0000	/*  Modular RED */
 326#define	HIFN_PUBOP_OP_MODEXP	0x00300000	/*  Modular EXP */
 327
 328/* Public status register (HIFN_1_PUB_STATUS) */
 329#define	HIFN_PUBSTS_DONE	0x00000001	/* operation done */
 330#define	HIFN_PUBSTS_CARRY	0x00000002	/* carry */
 331
 332/* Public interrupt enable register (HIFN_1_PUB_IEN) */
 333#define	HIFN_PUBIEN_DONE	0x00000001	/* operation done interrupt */
 334
 335/* Random number generator config register (HIFN_1_RNG_CONFIG) */
 336#define	HIFN_RNGCFG_ENA		0x00000001	/* enable rng */
 337
 338#define HIFN_NAMESIZE			32
 339#define HIFN_MAX_RESULT_ORDER		5
 340
 341#define	HIFN_D_CMD_RSIZE		(24 * 1)
 342#define	HIFN_D_SRC_RSIZE		(80 * 1)
 343#define	HIFN_D_DST_RSIZE		(80 * 1)
 344#define	HIFN_D_RES_RSIZE		(24 * 1)
 345
 346#define HIFN_D_DST_DALIGN		4
 347
 348#define HIFN_QUEUE_LENGTH		(HIFN_D_CMD_RSIZE - 1)
 349
 350#define AES_MIN_KEY_SIZE		16
 351#define AES_MAX_KEY_SIZE		32
 352
 353#define HIFN_DES_KEY_LENGTH		8
 354#define HIFN_3DES_KEY_LENGTH		24
 355#define HIFN_MAX_CRYPT_KEY_LENGTH	AES_MAX_KEY_SIZE
 356#define HIFN_IV_LENGTH			8
 357#define HIFN_AES_IV_LENGTH		16
 358#define	HIFN_MAX_IV_LENGTH		HIFN_AES_IV_LENGTH
 359
 360#define HIFN_MAC_KEY_LENGTH		64
 361#define HIFN_MD5_LENGTH			16
 362#define HIFN_SHA1_LENGTH		20
 363#define HIFN_MAC_TRUNC_LENGTH		12
 364
 365#define	HIFN_MAX_COMMAND		(8 + 8 + 8 + 64 + 260)
 366#define	HIFN_MAX_RESULT			(8 + 4 + 4 + 20 + 4)
 367#define HIFN_USED_RESULT		12
 368
 369struct hifn_desc {
 370	volatile __le32		l;
 371	volatile __le32		p;
 372};
 373
 374struct hifn_dma {
 375	struct hifn_desc	cmdr[HIFN_D_CMD_RSIZE + 1];
 376	struct hifn_desc	srcr[HIFN_D_SRC_RSIZE + 1];
 377	struct hifn_desc	dstr[HIFN_D_DST_RSIZE + 1];
 378	struct hifn_desc	resr[HIFN_D_RES_RSIZE + 1];
 379
 380	u8			command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
 381	u8			result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
 382
 383	/*
 384	 *  Our current positions for insertion and removal from the descriptor
 385	 *  rings.
 386	 */
 387	volatile int		cmdi, srci, dsti, resi;
 388	volatile int		cmdu, srcu, dstu, resu;
 389	int			cmdk, srck, dstk, resk;
 390};
 391
 392#define HIFN_FLAG_CMD_BUSY	(1 << 0)
 393#define HIFN_FLAG_SRC_BUSY	(1 << 1)
 394#define HIFN_FLAG_DST_BUSY	(1 << 2)
 395#define HIFN_FLAG_RES_BUSY	(1 << 3)
 396#define HIFN_FLAG_OLD_KEY	(1 << 4)
 397
 398#define HIFN_DEFAULT_ACTIVE_NUM	5
 399
 400struct hifn_device {
 401	char			name[HIFN_NAMESIZE];
 402
 403	int			irq;
 404
 405	struct pci_dev		*pdev;
 406	void __iomem		*bar[3];
 407
 408	void			*desc_virt;
 409	dma_addr_t		desc_dma;
 410
 411	u32			dmareg;
 412
 413	void			*sa[HIFN_D_RES_RSIZE];
 414
 415	spinlock_t		lock;
 416
 417	u32			flags;
 418	int			active, started;
 419	struct delayed_work	work;
 420	unsigned long		reset;
 421	unsigned long		success;
 422	unsigned long		prev_success;
 423
 424	u8			snum;
 425
 426	struct tasklet_struct	tasklet;
 427
 428	struct crypto_queue	queue;
 429	struct list_head	alg_list;
 430
 431	unsigned int		pk_clk_freq;
 432
 433#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 434	unsigned int		rng_wait_time;
 435	ktime_t			rngtime;
 436	struct hwrng		rng;
 437#endif
 438};
 439
 440#define	HIFN_D_LENGTH			0x0000ffff
 441#define	HIFN_D_NOINVALID		0x01000000
 442#define	HIFN_D_MASKDONEIRQ		0x02000000
 443#define	HIFN_D_DESTOVER			0x04000000
 444#define	HIFN_D_OVER			0x08000000
 445#define	HIFN_D_LAST			0x20000000
 446#define	HIFN_D_JUMP			0x40000000
 447#define	HIFN_D_VALID			0x80000000
 448
 449struct hifn_base_command {
 450	volatile __le16		masks;
 451	volatile __le16		session_num;
 452	volatile __le16		total_source_count;
 453	volatile __le16		total_dest_count;
 454};
 455
 456#define	HIFN_BASE_CMD_COMP		0x0100	/* enable compression engine */
 457#define	HIFN_BASE_CMD_PAD		0x0200	/* enable padding engine */
 458#define	HIFN_BASE_CMD_MAC		0x0400	/* enable MAC engine */
 459#define	HIFN_BASE_CMD_CRYPT		0x0800	/* enable crypt engine */
 460#define	HIFN_BASE_CMD_DECODE		0x2000
 461#define	HIFN_BASE_CMD_SRCLEN_M		0xc000
 462#define	HIFN_BASE_CMD_SRCLEN_S		14
 463#define	HIFN_BASE_CMD_DSTLEN_M		0x3000
 464#define	HIFN_BASE_CMD_DSTLEN_S		12
 465#define	HIFN_BASE_CMD_LENMASK_HI	0x30000
 466#define	HIFN_BASE_CMD_LENMASK_LO	0x0ffff
 467
 468/*
 469 * Structure to help build up the command data structure.
 470 */
 471struct hifn_crypt_command {
 472	volatile __le16		masks;
 473	volatile __le16		header_skip;
 474	volatile __le16		source_count;
 475	volatile __le16		reserved;
 476};
 477
 478#define	HIFN_CRYPT_CMD_ALG_MASK		0x0003		/* algorithm: */
 479#define	HIFN_CRYPT_CMD_ALG_DES		0x0000		/*   DES */
 480#define	HIFN_CRYPT_CMD_ALG_3DES		0x0001		/*   3DES */
 481#define	HIFN_CRYPT_CMD_ALG_RC4		0x0002		/*   RC4 */
 482#define	HIFN_CRYPT_CMD_ALG_AES		0x0003		/*   AES */
 483#define	HIFN_CRYPT_CMD_MODE_MASK	0x0018		/* Encrypt mode: */
 484#define	HIFN_CRYPT_CMD_MODE_ECB		0x0000		/*   ECB */
 485#define	HIFN_CRYPT_CMD_MODE_CBC		0x0008		/*   CBC */
 486#define	HIFN_CRYPT_CMD_MODE_CFB		0x0010		/*   CFB */
 487#define	HIFN_CRYPT_CMD_MODE_OFB		0x0018		/*   OFB */
 488#define	HIFN_CRYPT_CMD_CLR_CTX		0x0040		/* clear context */
 489#define	HIFN_CRYPT_CMD_KSZ_MASK		0x0600		/* AES key size: */
 490#define	HIFN_CRYPT_CMD_KSZ_128		0x0000		/*  128 bit */
 491#define	HIFN_CRYPT_CMD_KSZ_192		0x0200		/*  192 bit */
 492#define	HIFN_CRYPT_CMD_KSZ_256		0x0400		/*  256 bit */
 493#define	HIFN_CRYPT_CMD_NEW_KEY		0x0800		/* expect new key */
 494#define	HIFN_CRYPT_CMD_NEW_IV		0x1000		/* expect new iv */
 495#define	HIFN_CRYPT_CMD_SRCLEN_M		0xc000
 496#define	HIFN_CRYPT_CMD_SRCLEN_S		14
 497
 498#define	HIFN_MAC_CMD_ALG_MASK		0x0001
 499#define	HIFN_MAC_CMD_ALG_SHA1		0x0000
 500#define	HIFN_MAC_CMD_ALG_MD5		0x0001
 501#define	HIFN_MAC_CMD_MODE_MASK		0x000c
 502#define	HIFN_MAC_CMD_MODE_HMAC		0x0000
 503#define	HIFN_MAC_CMD_MODE_SSL_MAC	0x0004
 504#define	HIFN_MAC_CMD_MODE_HASH		0x0008
 505#define	HIFN_MAC_CMD_MODE_FULL		0x0004
 506#define	HIFN_MAC_CMD_TRUNC		0x0010
 507#define	HIFN_MAC_CMD_RESULT		0x0020
 508#define	HIFN_MAC_CMD_APPEND		0x0040
 509#define	HIFN_MAC_CMD_SRCLEN_M		0xc000
 510#define	HIFN_MAC_CMD_SRCLEN_S		14
 511
 512/*
 513 * MAC POS IPsec initiates authentication after encryption on encodes
 514 * and before decryption on decodes.
 515 */
 516#define	HIFN_MAC_CMD_POS_IPSEC		0x0200
 517#define	HIFN_MAC_CMD_NEW_KEY		0x0800
 518
 519#define	HIFN_COMP_CMD_SRCLEN_M		0xc000
 520#define	HIFN_COMP_CMD_SRCLEN_S		14
 521#define	HIFN_COMP_CMD_ONE		0x0100	/* must be one */
 522#define	HIFN_COMP_CMD_CLEARHIST		0x0010	/* clear history */
 523#define	HIFN_COMP_CMD_UPDATEHIST	0x0008	/* update history */
 524#define	HIFN_COMP_CMD_LZS_STRIP0	0x0004	/* LZS: strip zero */
 525#define	HIFN_COMP_CMD_MPPC_RESTART	0x0004	/* MPPC: restart */
 526#define	HIFN_COMP_CMD_ALG_MASK		0x0001	/* compression mode: */
 527#define	HIFN_COMP_CMD_ALG_MPPC		0x0001	/*   MPPC */
 528#define	HIFN_COMP_CMD_ALG_LZS		0x0000	/*   LZS */
 529
 530struct hifn_base_result {
 531	volatile __le16		flags;
 532	volatile __le16		session;
 533	volatile __le16		src_cnt;		/* 15:0 of source count */
 534	volatile __le16		dst_cnt;		/* 15:0 of dest count */
 535};
 536
 537#define	HIFN_BASE_RES_DSTOVERRUN	0x0200	/* destination overrun */
 538#define	HIFN_BASE_RES_SRCLEN_M		0xc000	/* 17:16 of source count */
 539#define	HIFN_BASE_RES_SRCLEN_S		14
 540#define	HIFN_BASE_RES_DSTLEN_M		0x3000	/* 17:16 of dest count */
 541#define	HIFN_BASE_RES_DSTLEN_S		12
 542
 543struct hifn_comp_result {
 544	volatile __le16		flags;
 545	volatile __le16		crc;
 546};
 547
 548#define	HIFN_COMP_RES_LCB_M		0xff00	/* longitudinal check byte */
 549#define	HIFN_COMP_RES_LCB_S		8
 550#define	HIFN_COMP_RES_RESTART		0x0004	/* MPPC: restart */
 551#define	HIFN_COMP_RES_ENDMARKER		0x0002	/* LZS: end marker seen */
 552#define	HIFN_COMP_RES_SRC_NOTZERO	0x0001	/* source expired */
 553
 554struct hifn_mac_result {
 555	volatile __le16		flags;
 556	volatile __le16		reserved;
 557	/* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
 558};
 559
 560#define	HIFN_MAC_RES_MISCOMPARE		0x0002	/* compare failed */
 561#define	HIFN_MAC_RES_SRC_NOTZERO	0x0001	/* source expired */
 562
 563struct hifn_crypt_result {
 564	volatile __le16		flags;
 565	volatile __le16		reserved;
 566};
 567
 568#define	HIFN_CRYPT_RES_SRC_NOTZERO	0x0001	/* source expired */
 569
 570#ifndef HIFN_POLL_FREQUENCY
 571#define	HIFN_POLL_FREQUENCY	0x1
 572#endif
 573
 574#ifndef HIFN_POLL_SCALAR
 575#define	HIFN_POLL_SCALAR	0x0
 576#endif
 577
 578#define	HIFN_MAX_SEGLEN		0xffff		/* maximum dma segment len */
 579#define	HIFN_MAX_DMALEN		0x3ffff		/* maximum dma length */
 580
 581struct hifn_crypto_alg {
 582	struct list_head	entry;
 583	struct skcipher_alg	alg;
 584	struct hifn_device	*dev;
 585};
 586
 587#define ASYNC_SCATTERLIST_CACHE	16
 588
 589#define ASYNC_FLAGS_MISALIGNED	(1 << 0)
 590
 591struct hifn_cipher_walk {
 592	struct scatterlist	cache[ASYNC_SCATTERLIST_CACHE];
 593	u32			flags;
 594	int			num;
 595};
 596
 597struct hifn_context {
 598	u8			key[HIFN_MAX_CRYPT_KEY_LENGTH];
 599	struct hifn_device	*dev;
 600	unsigned int		keysize;
 601};
 602
 603struct hifn_request_context {
 604	u8			*iv;
 605	unsigned int		ivsize;
 606	u8			op, type, mode, unused;
 607	struct hifn_cipher_walk	walk;
 608};
 609
 610#define crypto_alg_to_hifn(a)	container_of(a, struct hifn_crypto_alg, alg)
 611
 612static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
 613{
 614	return readl(dev->bar[0] + reg);
 615}
 616
 617static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
 618{
 619	return readl(dev->bar[1] + reg);
 620}
 621
 622static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
 623{
 624	writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
 625}
 626
 627static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
 628{
 629	writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
 630}
 631
 632static void hifn_wait_puc(struct hifn_device *dev)
 633{
 634	int i;
 635	u32 ret;
 636
 637	for (i = 10000; i > 0; --i) {
 638		ret = hifn_read_0(dev, HIFN_0_PUCTRL);
 639		if (!(ret & HIFN_PUCTRL_RESET))
 640			break;
 641
 642		udelay(1);
 643	}
 644
 645	if (!i)
 646		dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n");
 647}
 648
 649static void hifn_reset_puc(struct hifn_device *dev)
 650{
 651	hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
 652	hifn_wait_puc(dev);
 653}
 654
 655static void hifn_stop_device(struct hifn_device *dev)
 656{
 657	hifn_write_1(dev, HIFN_1_DMA_CSR,
 658		HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
 659		HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);
 660	hifn_write_0(dev, HIFN_0_PUIER, 0);
 661	hifn_write_1(dev, HIFN_1_DMA_IER, 0);
 662}
 663
 664static void hifn_reset_dma(struct hifn_device *dev, int full)
 665{
 666	hifn_stop_device(dev);
 667
 668	/*
 669	 * Setting poll frequency and others to 0.
 670	 */
 671	hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 672			HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 673	mdelay(1);
 674
 675	/*
 676	 * Reset DMA.
 677	 */
 678	if (full) {
 679		hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
 680		mdelay(1);
 681	} else {
 682		hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |
 683				HIFN_DMACNFG_MSTRESET);
 684		hifn_reset_puc(dev);
 685	}
 686
 687	hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 688			HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 689
 690	hifn_reset_puc(dev);
 691}
 692
 693static u32 hifn_next_signature(u32 a, u_int cnt)
 694{
 695	int i;
 696	u32 v;
 697
 698	for (i = 0; i < cnt; i++) {
 699		/* get the parity */
 700		v = a & 0x80080125;
 701		v ^= v >> 16;
 702		v ^= v >> 8;
 703		v ^= v >> 4;
 704		v ^= v >> 2;
 705		v ^= v >> 1;
 706
 707		a = (v & 1) ^ (a << 1);
 708	}
 709
 710	return a;
 711}
 712
 713static struct pci2id {
 714	u_short		pci_vendor;
 715	u_short		pci_prod;
 716	char		card_id[13];
 717} pci2id[] = {
 718	{
 719		PCI_VENDOR_ID_HIFN,
 720		PCI_DEVICE_ID_HIFN_7955,
 721		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 722		  0x00, 0x00, 0x00, 0x00, 0x00 }
 723	},
 724	{
 725		PCI_VENDOR_ID_HIFN,
 726		PCI_DEVICE_ID_HIFN_7956,
 727		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 728		  0x00, 0x00, 0x00, 0x00, 0x00 }
 729	}
 730};
 731
 732#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 733static int hifn_rng_data_present(struct hwrng *rng, int wait)
 734{
 735	struct hifn_device *dev = (struct hifn_device *)rng->priv;
 736	s64 nsec;
 737
 738	nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
 739	nsec -= dev->rng_wait_time;
 740	if (nsec <= 0)
 741		return 1;
 742	if (!wait)
 743		return 0;
 744	ndelay(nsec);
 745	return 1;
 746}
 747
 748static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
 749{
 750	struct hifn_device *dev = (struct hifn_device *)rng->priv;
 751
 752	*data = hifn_read_1(dev, HIFN_1_RNG_DATA);
 753	dev->rngtime = ktime_get();
 754	return 4;
 755}
 756
 757static int hifn_register_rng(struct hifn_device *dev)
 758{
 759	/*
 760	 * We must wait at least 256 Pk_clk cycles between two reads of the rng.
 761	 */
 762	dev->rng_wait_time	= DIV_ROUND_UP_ULL(NSEC_PER_SEC,
 763						   dev->pk_clk_freq) * 256;
 764
 765	dev->rng.name		= dev->name;
 766	dev->rng.data_present	= hifn_rng_data_present;
 767	dev->rng.data_read	= hifn_rng_data_read;
 768	dev->rng.priv		= (unsigned long)dev;
 769
 770	return hwrng_register(&dev->rng);
 771}
 772
 773static void hifn_unregister_rng(struct hifn_device *dev)
 774{
 775	hwrng_unregister(&dev->rng);
 776}
 777#else
 778#define hifn_register_rng(dev)		0
 779#define hifn_unregister_rng(dev)
 780#endif
 781
 782static int hifn_init_pubrng(struct hifn_device *dev)
 783{
 784	int i;
 785
 786	hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |
 787			HIFN_PUBRST_RESET);
 788
 789	for (i = 100; i > 0; --i) {
 790		mdelay(1);
 791
 792		if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)
 793			break;
 794	}
 795
 796	if (!i) {
 797		dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n");
 798	} else {
 799		hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
 800		dev->dmareg |= HIFN_DMAIER_PUBDONE;
 801		hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
 802
 803		dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n");
 804	}
 805
 806	/* Enable RNG engine. */
 807
 808	hifn_write_1(dev, HIFN_1_RNG_CONFIG,
 809			hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);
 810	dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n");
 811
 812#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 813	/* First value must be discarded */
 814	hifn_read_1(dev, HIFN_1_RNG_DATA);
 815	dev->rngtime = ktime_get();
 816#endif
 817	return 0;
 818}
 819
 820static int hifn_enable_crypto(struct hifn_device *dev)
 821{
 822	u32 dmacfg, addr;
 823	char *offtbl = NULL;
 824	int i;
 825
 826	for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
 827		if (pci2id[i].pci_vendor == dev->pdev->vendor &&
 828				pci2id[i].pci_prod == dev->pdev->device) {
 829			offtbl = pci2id[i].card_id;
 830			break;
 831		}
 832	}
 833
 834	if (!offtbl) {
 835		dev_err(&dev->pdev->dev, "Unknown card!\n");
 836		return -ENODEV;
 837	}
 838
 839	dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);
 840
 841	hifn_write_1(dev, HIFN_1_DMA_CNFG,
 842			HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |
 843			HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 844	mdelay(1);
 845	addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);
 846	mdelay(1);
 847	hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);
 848	mdelay(1);
 849
 850	for (i = 0; i < 12; ++i) {
 851		addr = hifn_next_signature(addr, offtbl[i] + 0x101);
 852		hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);
 853
 854		mdelay(1);
 855	}
 856	hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);
 857
 858	dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev));
 859
 860	return 0;
 861}
 862
 863static void hifn_init_dma(struct hifn_device *dev)
 864{
 865	struct hifn_dma *dma = dev->desc_virt;
 866	u32 dptr = dev->desc_dma;
 867	int i;
 868
 869	for (i = 0; i < HIFN_D_CMD_RSIZE; ++i)
 870		dma->cmdr[i].p = __cpu_to_le32(dptr +
 871				offsetof(struct hifn_dma, command_bufs[i][0]));
 872	for (i = 0; i < HIFN_D_RES_RSIZE; ++i)
 873		dma->resr[i].p = __cpu_to_le32(dptr +
 874				offsetof(struct hifn_dma, result_bufs[i][0]));
 875
 876	/* Setup LAST descriptors. */
 877	dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +
 878			offsetof(struct hifn_dma, cmdr[0]));
 879	dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +
 880			offsetof(struct hifn_dma, srcr[0]));
 881	dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +
 882			offsetof(struct hifn_dma, dstr[0]));
 883	dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +
 884			offsetof(struct hifn_dma, resr[0]));
 885
 886	dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
 887	dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
 888	dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
 889}
 890
 891/*
 892 * Initialize the PLL. We need to know the frequency of the reference clock
 893 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that
 894 * allows us to operate without the risk of overclocking the chip. If it
 895 * actually uses 33MHz, the chip will operate at half the speed, this can be
 896 * overridden by specifying the frequency as module parameter (pci33).
 897 *
 898 * Unfortunately the PCI clock is not very suitable since the HIFN needs a
 899 * stable clock and the PCI clock frequency may vary, so the default is the
 900 * external clock. There is no way to find out its frequency, we default to
 901 * 66MHz since according to Mike Ham of HiFn, almost every board in existence
 902 * has an external crystal populated at 66MHz.
 903 */
 904static void hifn_init_pll(struct hifn_device *dev)
 905{
 906	unsigned int freq, m;
 907	u32 pllcfg;
 908
 909	pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
 910
 911	if (strncmp(hifn_pll_ref, "ext", 3) == 0)
 912		pllcfg |= HIFN_PLL_REF_CLK_PLL;
 913	else
 914		pllcfg |= HIFN_PLL_REF_CLK_HBI;
 915
 916	if (hifn_pll_ref[3] == '\0' ||
 917	    kstrtouint(hifn_pll_ref + 3, 10, &freq)) {
 918		freq = 66;
 919		dev_info(&dev->pdev->dev, "assuming %u MHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n",
 920			 freq, hifn_pll_ref);
 921	}
 922
 923	m = HIFN_PLL_FCK_MAX / freq;
 924
 925	pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
 926	if (m <= 8)
 927		pllcfg |= HIFN_PLL_IS_1_8;
 928	else
 929		pllcfg |= HIFN_PLL_IS_9_12;
 930
 931	/* Select clock source and enable clock bypass */
 932	hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 933		     HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
 934
 935	/* Let the chip lock to the input clock */
 936	mdelay(10);
 937
 938	/* Disable clock bypass */
 939	hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 940		     HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
 941
 942	/* Switch the engines to the PLL */
 943	hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 944		     HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
 945
 946	/*
 947	 * The Fpk_clk runs at half the total speed. Its frequency is needed to
 948	 * calculate the minimum time between two reads of the rng. Since 33MHz
 949	 * is actually 33.333... we overestimate the frequency here, resulting
 950	 * in slightly larger intervals.
 951	 */
 952	dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
 953}
 954
 955static void hifn_init_registers(struct hifn_device *dev)
 956{
 957	u32 dptr = dev->desc_dma;
 958
 959	/* Initialization magic... */
 960	hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
 961	hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
 962	hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
 963
 964	/* write all 4 ring address registers */
 965	hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
 966				offsetof(struct hifn_dma, cmdr[0]));
 967	hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
 968				offsetof(struct hifn_dma, srcr[0]));
 969	hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
 970				offsetof(struct hifn_dma, dstr[0]));
 971	hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
 972				offsetof(struct hifn_dma, resr[0]));
 973
 974	mdelay(2);
 975#if 0
 976	hifn_write_1(dev, HIFN_1_DMA_CSR,
 977	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
 978	    HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
 979	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
 980	    HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
 981	    HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
 982	    HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
 983	    HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
 984	    HIFN_DMACSR_S_WAIT |
 985	    HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
 986	    HIFN_DMACSR_C_WAIT |
 987	    HIFN_DMACSR_ENGINE |
 988	    HIFN_DMACSR_PUBDONE);
 989#else
 990	hifn_write_1(dev, HIFN_1_DMA_CSR,
 991	    HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
 992	    HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |
 993	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
 994	    HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
 995	    HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
 996	    HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
 997	    HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
 998	    HIFN_DMACSR_S_WAIT |
 999	    HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1000	    HIFN_DMACSR_C_WAIT |
1001	    HIFN_DMACSR_ENGINE |
1002	    HIFN_DMACSR_PUBDONE);
1003#endif
1004	hifn_read_1(dev, HIFN_1_DMA_CSR);
1005
1006	dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1007	    HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1008	    HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1009	    HIFN_DMAIER_ENGINE;
1010	dev->dmareg &= ~HIFN_DMAIER_C_WAIT;
1011
1012	hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1013	hifn_read_1(dev, HIFN_1_DMA_IER);
1014#if 0
1015	hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |
1016		    HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1017		    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1018		    HIFN_PUCNFG_DRAM);
1019#else
1020	hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
1021#endif
1022	hifn_init_pll(dev);
1023
1024	hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1025	hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1026	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1027	    ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1028	    ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1029}
1030
1031static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,
1032		unsigned dlen, unsigned slen, u16 mask, u8 snum)
1033{
1034	struct hifn_base_command *base_cmd;
1035	u8 *buf_pos = buf;
1036
1037	base_cmd = (struct hifn_base_command *)buf_pos;
1038	base_cmd->masks = __cpu_to_le16(mask);
1039	base_cmd->total_source_count =
1040		__cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);
1041	base_cmd->total_dest_count =
1042		__cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1043
1044	dlen >>= 16;
1045	slen >>= 16;
1046	base_cmd->session_num = __cpu_to_le16(snum |
1047	    ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1048	    ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1049
1050	return sizeof(struct hifn_base_command);
1051}
1052
1053static int hifn_setup_crypto_command(struct hifn_device *dev,
1054		u8 *buf, unsigned dlen, unsigned slen,
1055		u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
1056{
1057	struct hifn_dma *dma = dev->desc_virt;
1058	struct hifn_crypt_command *cry_cmd;
1059	u8 *buf_pos = buf;
1060	u16 cmd_len;
1061
1062	cry_cmd = (struct hifn_crypt_command *)buf_pos;
1063
1064	cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);
1065	dlen >>= 16;
1066	cry_cmd->masks = __cpu_to_le16(mode |
1067			((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &
1068			 HIFN_CRYPT_CMD_SRCLEN_M));
1069	cry_cmd->header_skip = 0;
1070	cry_cmd->reserved = 0;
1071
1072	buf_pos += sizeof(struct hifn_crypt_command);
1073
1074	dma->cmdu++;
1075	if (dma->cmdu > 1) {
1076		dev->dmareg |= HIFN_DMAIER_C_WAIT;
1077		hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1078	}
1079
1080	if (keylen) {
1081		memcpy(buf_pos, key, keylen);
1082		buf_pos += keylen;
1083	}
1084	if (ivsize) {
1085		memcpy(buf_pos, iv, ivsize);
1086		buf_pos += ivsize;
1087	}
1088
1089	cmd_len = buf_pos - buf;
1090
1091	return cmd_len;
1092}
1093
1094static int hifn_setup_cmd_desc(struct hifn_device *dev,
1095		struct hifn_context *ctx, struct hifn_request_context *rctx,
1096		void *priv, unsigned int nbytes)
1097{
1098	struct hifn_dma *dma = dev->desc_virt;
1099	int cmd_len, sa_idx;
1100	u8 *buf, *buf_pos;
1101	u16 mask;
1102
1103	sa_idx = dma->cmdi;
1104	buf_pos = buf = dma->command_bufs[dma->cmdi];
1105
1106	mask = 0;
1107	switch (rctx->op) {
1108	case ACRYPTO_OP_DECRYPT:
1109		mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
1110		break;
1111	case ACRYPTO_OP_ENCRYPT:
1112		mask = HIFN_BASE_CMD_CRYPT;
1113		break;
1114	case ACRYPTO_OP_HMAC:
1115		mask = HIFN_BASE_CMD_MAC;
1116		break;
1117	default:
1118		goto err_out;
1119	}
1120
1121	buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
1122			nbytes, mask, dev->snum);
1123
1124	if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) {
1125		u16 md = 0;
1126
1127		if (ctx->keysize)
1128			md |= HIFN_CRYPT_CMD_NEW_KEY;
1129		if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB)
1130			md |= HIFN_CRYPT_CMD_NEW_IV;
1131
1132		switch (rctx->mode) {
1133		case ACRYPTO_MODE_ECB:
1134			md |= HIFN_CRYPT_CMD_MODE_ECB;
1135			break;
1136		case ACRYPTO_MODE_CBC:
1137			md |= HIFN_CRYPT_CMD_MODE_CBC;
1138			break;
1139		case ACRYPTO_MODE_CFB:
1140			md |= HIFN_CRYPT_CMD_MODE_CFB;
1141			break;
1142		case ACRYPTO_MODE_OFB:
1143			md |= HIFN_CRYPT_CMD_MODE_OFB;
1144			break;
1145		default:
1146			goto err_out;
1147		}
1148
1149		switch (rctx->type) {
1150		case ACRYPTO_TYPE_AES_128:
1151			if (ctx->keysize != 16)
1152				goto err_out;
1153			md |= HIFN_CRYPT_CMD_KSZ_128 |
1154				HIFN_CRYPT_CMD_ALG_AES;
1155			break;
1156		case ACRYPTO_TYPE_AES_192:
1157			if (ctx->keysize != 24)
1158				goto err_out;
1159			md |= HIFN_CRYPT_CMD_KSZ_192 |
1160				HIFN_CRYPT_CMD_ALG_AES;
1161			break;
1162		case ACRYPTO_TYPE_AES_256:
1163			if (ctx->keysize != 32)
1164				goto err_out;
1165			md |= HIFN_CRYPT_CMD_KSZ_256 |
1166				HIFN_CRYPT_CMD_ALG_AES;
1167			break;
1168		case ACRYPTO_TYPE_3DES:
1169			if (ctx->keysize != 24)
1170				goto err_out;
1171			md |= HIFN_CRYPT_CMD_ALG_3DES;
1172			break;
1173		case ACRYPTO_TYPE_DES:
1174			if (ctx->keysize != 8)
1175				goto err_out;
1176			md |= HIFN_CRYPT_CMD_ALG_DES;
1177			break;
1178		default:
1179			goto err_out;
1180		}
1181
1182		buf_pos += hifn_setup_crypto_command(dev, buf_pos,
1183				nbytes, nbytes, ctx->key, ctx->keysize,
1184				rctx->iv, rctx->ivsize, md);
1185	}
1186
1187	dev->sa[sa_idx] = priv;
1188	dev->started++;
1189
1190	cmd_len = buf_pos - buf;
1191	dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
1192			HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
1193
1194	if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
1195		dma->cmdr[dma->cmdi].l = __cpu_to_le32(
1196			HIFN_D_VALID | HIFN_D_LAST |
1197			HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
1198		dma->cmdi = 0;
1199	} else {
1200		dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID);
1201	}
1202
1203	if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {
1204		hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1205		dev->flags |= HIFN_FLAG_CMD_BUSY;
1206	}
1207	return 0;
1208
1209err_out:
1210	return -EINVAL;
1211}
1212
1213static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
1214		unsigned int offset, unsigned int size, int last)
1215{
1216	struct hifn_dma *dma = dev->desc_virt;
1217	int idx;
1218	dma_addr_t addr;
1219
1220	addr = dma_map_page(&dev->pdev->dev, page, offset, size,
1221			    DMA_TO_DEVICE);
1222
1223	idx = dma->srci;
1224
1225	dma->srcr[idx].p = __cpu_to_le32(addr);
1226	dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1227			HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
1228
1229	if (++idx == HIFN_D_SRC_RSIZE) {
1230		dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1231				HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1232				(last ? HIFN_D_LAST : 0));
1233		idx = 0;
1234	}
1235
1236	dma->srci = idx;
1237	dma->srcu++;
1238
1239	if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
1240		hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1241		dev->flags |= HIFN_FLAG_SRC_BUSY;
1242	}
1243
1244	return size;
1245}
1246
1247static void hifn_setup_res_desc(struct hifn_device *dev)
1248{
1249	struct hifn_dma *dma = dev->desc_virt;
1250
1251	dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
1252			HIFN_D_VALID | HIFN_D_LAST);
1253	/*
1254	 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
1255	 *					HIFN_D_LAST);
1256	 */
1257
1258	if (++dma->resi == HIFN_D_RES_RSIZE) {
1259		dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
1260				HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1261		dma->resi = 0;
1262	}
1263
1264	dma->resu++;
1265
1266	if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
1267		hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1268		dev->flags |= HIFN_FLAG_RES_BUSY;
1269	}
1270}
1271
1272static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
1273		unsigned offset, unsigned size, int last)
1274{
1275	struct hifn_dma *dma = dev->desc_virt;
1276	int idx;
1277	dma_addr_t addr;
1278
1279	addr = dma_map_page(&dev->pdev->dev, page, offset, size,
1280			    DMA_FROM_DEVICE);
1281
1282	idx = dma->dsti;
1283	dma->dstr[idx].p = __cpu_to_le32(addr);
1284	dma->dstr[idx].l = __cpu_to_le32(size |	HIFN_D_VALID |
1285			HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
1286
1287	if (++idx == HIFN_D_DST_RSIZE) {
1288		dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1289				HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1290				(last ? HIFN_D_LAST : 0));
1291		idx = 0;
1292	}
1293	dma->dsti = idx;
1294	dma->dstu++;
1295
1296	if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
1297		hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1298		dev->flags |= HIFN_FLAG_DST_BUSY;
1299	}
1300}
1301
1302static int hifn_setup_dma(struct hifn_device *dev,
1303		struct hifn_context *ctx, struct hifn_request_context *rctx,
1304		struct scatterlist *src, struct scatterlist *dst,
1305		unsigned int nbytes, void *priv)
1306{
1307	struct scatterlist *t;
1308	struct page *spage, *dpage;
1309	unsigned int soff, doff;
1310	unsigned int n, len;
1311
1312	n = nbytes;
1313	while (n) {
1314		spage = sg_page(src);
1315		soff = src->offset;
1316		len = min(src->length, n);
1317
1318		hifn_setup_src_desc(dev, spage, soff, len, n - len == 0);
1319
1320		src++;
1321		n -= len;
1322	}
1323
1324	t = &rctx->walk.cache[0];
1325	n = nbytes;
1326	while (n) {
1327		if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1328			BUG_ON(!sg_page(t));
1329			dpage = sg_page(t);
1330			doff = 0;
1331			len = t->length;
1332		} else {
1333			BUG_ON(!sg_page(dst));
1334			dpage = sg_page(dst);
1335			doff = dst->offset;
1336			len = dst->length;
1337		}
1338		len = min(len, n);
1339
1340		hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0);
1341
1342		dst++;
1343		t++;
1344		n -= len;
1345	}
1346
1347	hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes);
1348	hifn_setup_res_desc(dev);
1349	return 0;
1350}
1351
1352static int hifn_cipher_walk_init(struct hifn_cipher_walk *w,
1353		int num, gfp_t gfp_flags)
1354{
1355	int i;
1356
1357	num = min(ASYNC_SCATTERLIST_CACHE, num);
1358	sg_init_table(w->cache, num);
1359
1360	w->num = 0;
1361	for (i = 0; i < num; ++i) {
1362		struct page *page = alloc_page(gfp_flags);
1363		struct scatterlist *s;
1364
1365		if (!page)
1366			break;
1367
1368		s = &w->cache[i];
1369
1370		sg_set_page(s, page, PAGE_SIZE, 0);
1371		w->num++;
1372	}
1373
1374	return i;
1375}
1376
1377static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w)
1378{
1379	int i;
1380
1381	for (i = 0; i < w->num; ++i) {
1382		struct scatterlist *s = &w->cache[i];
1383
1384		__free_page(sg_page(s));
1385
1386		s->length = 0;
1387	}
1388
1389	w->num = 0;
1390}
1391
1392static int skcipher_add(unsigned int *drestp, struct scatterlist *dst,
1393		unsigned int size, unsigned int *nbytesp)
1394{
1395	unsigned int copy, drest = *drestp, nbytes = *nbytesp;
1396	int idx = 0;
1397
1398	if (drest < size || size > nbytes)
1399		return -EINVAL;
1400
1401	while (size) {
1402		copy = min3(drest, size, dst->length);
1403
1404		size -= copy;
1405		drest -= copy;
1406		nbytes -= copy;
1407
1408		pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
1409			 __func__, copy, size, drest, nbytes);
1410
1411		dst++;
1412		idx++;
1413	}
1414
1415	*nbytesp = nbytes;
1416	*drestp = drest;
1417
1418	return idx;
1419}
1420
1421static int hifn_cipher_walk(struct skcipher_request *req,
1422		struct hifn_cipher_walk *w)
1423{
1424	struct scatterlist *dst, *t;
1425	unsigned int nbytes = req->cryptlen, offset, copy, diff;
1426	int idx, tidx, err;
1427
1428	tidx = idx = 0;
1429	offset = 0;
1430	while (nbytes) {
1431		if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
1432			return -EINVAL;
1433
1434		dst = &req->dst[idx];
1435
1436		pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n",
1437			 __func__, dst->length, dst->offset, offset, nbytes);
1438
1439		if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1440		    !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
1441		    offset) {
1442			unsigned slen = min(dst->length - offset, nbytes);
1443			unsigned dlen = PAGE_SIZE;
1444
1445			t = &w->cache[idx];
1446
1447			err = skcipher_add(&dlen, dst, slen, &nbytes);
1448			if (err < 0)
1449				return err;
1450
1451			idx += err;
1452
1453			copy = slen & ~(HIFN_D_DST_DALIGN - 1);
1454			diff = slen & (HIFN_D_DST_DALIGN - 1);
1455
1456			if (dlen < nbytes) {
1457				/*
1458				 * Destination page does not have enough space
1459				 * to put there additional blocksized chunk,
1460				 * so we mark that page as containing only
1461				 * blocksize aligned chunks:
1462				 *	t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
1463				 * and increase number of bytes to be processed
1464				 * in next chunk:
1465				 *	nbytes += diff;
1466				 */
1467				nbytes += diff;
1468
1469				/*
1470				 * Temporary of course...
1471				 * Kick author if you will catch this one.
1472				 */
1473				pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n",
1474				       __func__, dlen, nbytes, slen, offset);
1475				pr_err("%s: please contact author to fix this "
1476				       "issue, generally you should not catch "
1477				       "this path under any condition but who "
1478				       "knows how did you use crypto code.\n"
1479				       "Thank you.\n",	__func__);
1480				BUG();
1481			} else {
1482				copy += diff + nbytes;
1483
1484				dst = &req->dst[idx];
1485
1486				err = skcipher_add(&dlen, dst, nbytes, &nbytes);
1487				if (err < 0)
1488					return err;
1489
1490				idx += err;
1491			}
1492
1493			t->length = copy;
1494			t->offset = offset;
1495		} else {
1496			nbytes -= min(dst->length, nbytes);
1497			idx++;
1498		}
1499
1500		tidx++;
1501	}
1502
1503	return tidx;
1504}
1505
1506static int hifn_setup_session(struct skcipher_request *req)
1507{
1508	struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1509	struct hifn_request_context *rctx = skcipher_request_ctx(req);
1510	struct hifn_device *dev = ctx->dev;
1511	unsigned long dlen, flags;
1512	unsigned int nbytes = req->cryptlen, idx = 0;
1513	int err = -EINVAL, sg_num;
1514	struct scatterlist *dst;
1515
1516	if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB)
1517		goto err_out_exit;
1518
1519	rctx->walk.flags = 0;
1520
1521	while (nbytes) {
1522		dst = &req->dst[idx];
1523		dlen = min(dst->length, nbytes);
1524
1525		if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1526		    !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
1527			rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
1528
1529		nbytes -= dlen;
1530		idx++;
1531	}
1532
1533	if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1534		err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);
1535		if (err < 0)
1536			return err;
1537	}
1538
1539	sg_num = hifn_cipher_walk(req, &rctx->walk);
1540	if (sg_num < 0) {
1541		err = sg_num;
1542		goto err_out_exit;
1543	}
1544
1545	spin_lock_irqsave(&dev->lock, flags);
1546	if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
1547		err = -EAGAIN;
1548		goto err_out;
1549	}
1550
1551	err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req);
1552	if (err)
1553		goto err_out;
1554
1555	dev->snum++;
1556
1557	dev->active = HIFN_DEFAULT_ACTIVE_NUM;
1558	spin_unlock_irqrestore(&dev->lock, flags);
1559
1560	return 0;
1561
1562err_out:
1563	spin_unlock_irqrestore(&dev->lock, flags);
1564err_out_exit:
1565	if (err) {
1566		dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
1567			 "type: %u, err: %d.\n",
1568			 rctx->iv, rctx->ivsize,
1569			 ctx->key, ctx->keysize,
1570			 rctx->mode, rctx->op, rctx->type, err);
1571	}
1572
1573	return err;
1574}
1575
1576static int hifn_start_device(struct hifn_device *dev)
1577{
1578	int err;
1579
1580	dev->started = dev->active = 0;
1581	hifn_reset_dma(dev, 1);
1582
1583	err = hifn_enable_crypto(dev);
1584	if (err)
1585		return err;
1586
1587	hifn_reset_puc(dev);
1588
1589	hifn_init_dma(dev);
1590
1591	hifn_init_registers(dev);
1592
1593	hifn_init_pubrng(dev);
1594
1595	return 0;
1596}
1597
1598static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,
1599		struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)
1600{
1601	unsigned int srest = *srestp, nbytes = *nbytesp, copy;
1602	void *daddr;
1603	int idx = 0;
1604
1605	if (srest < size || size > nbytes)
1606		return -EINVAL;
1607
1608	while (size) {
1609		copy = min3(srest, dst->length, size);
1610
1611		daddr = kmap_atomic(sg_page(dst));
1612		memcpy(daddr + dst->offset + offset, saddr, copy);
1613		kunmap_atomic(daddr);
1614
1615		nbytes -= copy;
1616		size -= copy;
1617		srest -= copy;
1618		saddr += copy;
1619		offset = 0;
1620
1621		pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",
1622			 __func__, copy, size, srest, nbytes);
1623
1624		dst++;
1625		idx++;
1626	}
1627
1628	*nbytesp = nbytes;
1629	*srestp = srest;
1630
1631	return idx;
1632}
1633
1634static inline void hifn_complete_sa(struct hifn_device *dev, int i)
1635{
1636	unsigned long flags;
1637
1638	spin_lock_irqsave(&dev->lock, flags);
1639	dev->sa[i] = NULL;
1640	dev->started--;
1641	if (dev->started < 0)
1642		dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__,
1643			 dev->started);
1644	spin_unlock_irqrestore(&dev->lock, flags);
1645	BUG_ON(dev->started < 0);
1646}
1647
1648static void hifn_process_ready(struct skcipher_request *req, int error)
1649{
1650	struct hifn_request_context *rctx = skcipher_request_ctx(req);
1651
1652	if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1653		unsigned int nbytes = req->cryptlen;
1654		int idx = 0, err;
1655		struct scatterlist *dst, *t;
1656		void *saddr;
1657
1658		while (nbytes) {
1659			t = &rctx->walk.cache[idx];
1660			dst = &req->dst[idx];
1661
1662			pr_debug("\n%s: sg_page(t): %p, t->length: %u, "
1663				"sg_page(dst): %p, dst->length: %u, "
1664				"nbytes: %u.\n",
1665				__func__, sg_page(t), t->length,
1666				sg_page(dst), dst->length, nbytes);
1667
1668			if (!t->length) {
1669				nbytes -= min(dst->length, nbytes);
1670				idx++;
1671				continue;
1672			}
1673
1674			saddr = kmap_atomic(sg_page(t));
1675
1676			err = skcipher_get(saddr, &t->length, t->offset,
1677					dst, nbytes, &nbytes);
1678			if (err < 0) {
1679				kunmap_atomic(saddr);
1680				break;
1681			}
1682
1683			idx += err;
1684			kunmap_atomic(saddr);
1685		}
1686
1687		hifn_cipher_walk_exit(&rctx->walk);
1688	}
1689
1690	skcipher_request_complete(req, error);
1691}
1692
1693static void hifn_clear_rings(struct hifn_device *dev, int error)
1694{
1695	struct hifn_dma *dma = dev->desc_virt;
1696	int i, u;
1697
1698	dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1699			"k: %d.%d.%d.%d.\n",
1700			dma->cmdi, dma->srci, dma->dsti, dma->resi,
1701			dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1702			dma->cmdk, dma->srck, dma->dstk, dma->resk);
1703
1704	i = dma->resk; u = dma->resu;
1705	while (u != 0) {
1706		if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
1707			break;
1708
1709		if (dev->sa[i]) {
1710			dev->success++;
1711			dev->reset = 0;
1712			hifn_process_ready(dev->sa[i], error);
1713			hifn_complete_sa(dev, i);
1714		}
1715
1716		if (++i == HIFN_D_RES_RSIZE)
1717			i = 0;
1718		u--;
1719	}
1720	dma->resk = i; dma->resu = u;
1721
1722	i = dma->srck; u = dma->srcu;
1723	while (u != 0) {
1724		if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
1725			break;
1726		if (++i == HIFN_D_SRC_RSIZE)
1727			i = 0;
1728		u--;
1729	}
1730	dma->srck = i; dma->srcu = u;
1731
1732	i = dma->cmdk; u = dma->cmdu;
1733	while (u != 0) {
1734		if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
1735			break;
1736		if (++i == HIFN_D_CMD_RSIZE)
1737			i = 0;
1738		u--;
1739	}
1740	dma->cmdk = i; dma->cmdu = u;
1741
1742	i = dma->dstk; u = dma->dstu;
1743	while (u != 0) {
1744		if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
1745			break;
1746		if (++i == HIFN_D_DST_RSIZE)
1747			i = 0;
1748		u--;
1749	}
1750	dma->dstk = i; dma->dstu = u;
1751
1752	dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1753			"k: %d.%d.%d.%d.\n",
1754			dma->cmdi, dma->srci, dma->dsti, dma->resi,
1755			dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1756			dma->cmdk, dma->srck, dma->dstk, dma->resk);
1757}
1758
1759static void hifn_work(struct work_struct *work)
1760{
1761	struct delayed_work *dw = to_delayed_work(work);
1762	struct hifn_device *dev = container_of(dw, struct hifn_device, work);
1763	unsigned long flags;
1764	int reset = 0;
1765	u32 r = 0;
1766
1767	spin_lock_irqsave(&dev->lock, flags);
1768	if (dev->active == 0) {
1769		struct hifn_dma *dma = dev->desc_virt;
1770
1771		if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
1772			dev->flags &= ~HIFN_FLAG_CMD_BUSY;
1773			r |= HIFN_DMACSR_C_CTRL_DIS;
1774		}
1775		if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {
1776			dev->flags &= ~HIFN_FLAG_SRC_BUSY;
1777			r |= HIFN_DMACSR_S_CTRL_DIS;
1778		}
1779		if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {
1780			dev->flags &= ~HIFN_FLAG_DST_BUSY;
1781			r |= HIFN_DMACSR_D_CTRL_DIS;
1782		}
1783		if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {
1784			dev->flags &= ~HIFN_FLAG_RES_BUSY;
1785			r |= HIFN_DMACSR_R_CTRL_DIS;
1786		}
1787		if (r)
1788			hifn_write_1(dev, HIFN_1_DMA_CSR, r);
1789	} else
1790		dev->active--;
1791
1792	if ((dev->prev_success == dev->success) && dev->started)
1793		reset = 1;
1794	dev->prev_success = dev->success;
1795	spin_unlock_irqrestore(&dev->lock, flags);
1796
1797	if (reset) {
1798		if (++dev->reset >= 5) {
1799			int i;
1800			struct hifn_dma *dma = dev->desc_virt;
1801
1802			dev_info(&dev->pdev->dev,
1803				 "r: %08x, active: %d, started: %d, "
1804				 "success: %lu: qlen: %u/%u, reset: %d.\n",
1805				 r, dev->active, dev->started,
1806				 dev->success, dev->queue.qlen, dev->queue.max_qlen,
1807				 reset);
1808
1809			dev_info(&dev->pdev->dev, "%s: res: ", __func__);
1810			for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
1811				pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]);
1812				if (dev->sa[i]) {
1813					hifn_process_ready(dev->sa[i], -ENODEV);
1814					hifn_complete_sa(dev, i);
1815				}
1816			}
1817			pr_info("\n");
1818
1819			hifn_reset_dma(dev, 1);
1820			hifn_stop_device(dev);
1821			hifn_start_device(dev);
1822			dev->reset = 0;
1823		}
1824
1825		tasklet_schedule(&dev->tasklet);
1826	}
1827
1828	schedule_delayed_work(&dev->work, HZ);
1829}
1830
1831static irqreturn_t hifn_interrupt(int irq, void *data)
1832{
1833	struct hifn_device *dev = data;
1834	struct hifn_dma *dma = dev->desc_virt;
1835	u32 dmacsr, restart;
1836
1837	dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
1838
1839	dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
1840			"i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
1841		dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
1842		dma->cmdi, dma->srci, dma->dsti, dma->resi,
1843		dma->cmdu, dma->srcu, dma->dstu, dma->resu);
1844
1845	if ((dmacsr & dev->dmareg) == 0)
1846		return IRQ_NONE;
1847
1848	hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);
1849
1850	if (dmacsr & HIFN_DMACSR_ENGINE)
1851		hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));
1852	if (dmacsr & HIFN_DMACSR_PUBDONE)
1853		hifn_write_1(dev, HIFN_1_PUB_STATUS,
1854			hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
1855
1856	restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
1857	if (restart) {
1858		u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
1859
1860		dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
1861			 !!(dmacsr & HIFN_DMACSR_R_OVER),
1862			 !!(dmacsr & HIFN_DMACSR_D_OVER),
1863			puisr, !!(puisr & HIFN_PUISR_DSTOVER));
1864		if (!!(puisr & HIFN_PUISR_DSTOVER))
1865			hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1866		hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
1867					HIFN_DMACSR_D_OVER));
1868	}
1869
1870	restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
1871			HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
1872	if (restart) {
1873		dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n",
1874			 !!(dmacsr & HIFN_DMACSR_C_ABORT),
1875			 !!(dmacsr & HIFN_DMACSR_S_ABORT),
1876			 !!(dmacsr & HIFN_DMACSR_D_ABORT),
1877			 !!(dmacsr & HIFN_DMACSR_R_ABORT));
1878		hifn_reset_dma(dev, 1);
1879		hifn_init_dma(dev);
1880		hifn_init_registers(dev);
1881	}
1882
1883	if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
1884		dev_dbg(&dev->pdev->dev, "wait on command.\n");
1885		dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);
1886		hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1887	}
1888
1889	tasklet_schedule(&dev->tasklet);
1890
1891	return IRQ_HANDLED;
1892}
1893
1894static void hifn_flush(struct hifn_device *dev)
1895{
1896	unsigned long flags;
1897	struct crypto_async_request *async_req;
1898	struct skcipher_request *req;
1899	struct hifn_dma *dma = dev->desc_virt;
1900	int i;
1901
1902	for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
1903		struct hifn_desc *d = &dma->resr[i];
1904
1905		if (dev->sa[i]) {
1906			hifn_process_ready(dev->sa[i],
1907				(d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0);
1908			hifn_complete_sa(dev, i);
1909		}
1910	}
1911
1912	spin_lock_irqsave(&dev->lock, flags);
1913	while ((async_req = crypto_dequeue_request(&dev->queue))) {
1914		req = skcipher_request_cast(async_req);
1915		spin_unlock_irqrestore(&dev->lock, flags);
1916
1917		hifn_process_ready(req, -ENODEV);
1918
1919		spin_lock_irqsave(&dev->lock, flags);
1920	}
1921	spin_unlock_irqrestore(&dev->lock, flags);
1922}
1923
1924static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key,
1925		unsigned int len)
1926{
1927	struct hifn_context *ctx = crypto_skcipher_ctx(cipher);
1928	struct hifn_device *dev = ctx->dev;
1929	int err;
1930
1931	err = verify_skcipher_des_key(cipher, key);
1932	if (err)
1933		return err;
1934
1935	dev->flags &= ~HIFN_FLAG_OLD_KEY;
1936
1937	memcpy(ctx->key, key, len);
1938	ctx->keysize = len;
1939
1940	return 0;
1941}
1942
1943static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
1944			    unsigned int len)
1945{
1946	struct hifn_context *ctx = crypto_skcipher_ctx(cipher);
1947	struct hifn_device *dev = ctx->dev;
1948	int err;
1949
1950	err = verify_skcipher_des3_key(cipher, key);
1951	if (err)
1952		return err;
1953
1954	dev->flags &= ~HIFN_FLAG_OLD_KEY;
1955
1956	memcpy(ctx->key, key, len);
1957	ctx->keysize = len;
1958
1959	return 0;
1960}
1961
1962static int hifn_handle_req(struct skcipher_request *req)
1963{
1964	struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1965	struct hifn_device *dev = ctx->dev;
1966	int err = -EAGAIN;
1967
1968	if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)
1969		err = hifn_setup_session(req);
1970
1971	if (err == -EAGAIN) {
1972		unsigned long flags;
1973
1974		spin_lock_irqsave(&dev->lock, flags);
1975		err = crypto_enqueue_request(&dev->queue, &req->base);
1976		spin_unlock_irqrestore(&dev->lock, flags);
1977	}
1978
1979	return err;
1980}
1981
1982static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op,
1983		u8 type, u8 mode)
1984{
1985	struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1986	struct hifn_request_context *rctx = skcipher_request_ctx(req);
1987	unsigned ivsize;
1988
1989	ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));
1990
1991	if (req->iv && mode != ACRYPTO_MODE_ECB) {
1992		if (type == ACRYPTO_TYPE_AES_128)
1993			ivsize = HIFN_AES_IV_LENGTH;
1994		else if (type == ACRYPTO_TYPE_DES)
1995			ivsize = HIFN_DES_KEY_LENGTH;
1996		else if (type == ACRYPTO_TYPE_3DES)
1997			ivsize = HIFN_3DES_KEY_LENGTH;
1998	}
1999
2000	if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {
2001		if (ctx->keysize == 24)
2002			type = ACRYPTO_TYPE_AES_192;
2003		else if (ctx->keysize == 32)
2004			type = ACRYPTO_TYPE_AES_256;
2005	}
2006
2007	rctx->op = op;
2008	rctx->mode = mode;
2009	rctx->type = type;
2010	rctx->iv = req->iv;
2011	rctx->ivsize = ivsize;
2012
2013	/*
2014	 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2015	 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2016	 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2017	 */
2018
2019	return hifn_handle_req(req);
2020}
2021
2022static int hifn_process_queue(struct hifn_device *dev)
2023{
2024	struct crypto_async_request *async_req, *backlog;
2025	struct skcipher_request *req;
2026	unsigned long flags;
2027	int err = 0;
2028
2029	while (dev->started < HIFN_QUEUE_LENGTH) {
2030		spin_lock_irqsave(&dev->lock, flags);
2031		backlog = crypto_get_backlog(&dev->queue);
2032		async_req = crypto_dequeue_request(&dev->queue);
2033		spin_unlock_irqrestore(&dev->lock, flags);
2034
2035		if (!async_req)
2036			break;
2037
2038		if (backlog)
2039			crypto_request_complete(backlog, -EINPROGRESS);
2040
2041		req = skcipher_request_cast(async_req);
2042
2043		err = hifn_handle_req(req);
2044		if (err)
2045			break;
2046	}
2047
2048	return err;
2049}
2050
2051static int hifn_setup_crypto(struct skcipher_request *req, u8 op,
2052		u8 type, u8 mode)
2053{
2054	int err;
2055	struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2056	struct hifn_device *dev = ctx->dev;
2057
2058	err = hifn_setup_crypto_req(req, op, type, mode);
2059	if (err)
2060		return err;
2061
2062	if (dev->started < HIFN_QUEUE_LENGTH &&	dev->queue.qlen)
2063		hifn_process_queue(dev);
2064
2065	return -EINPROGRESS;
2066}
2067
2068/*
2069 * AES ecryption functions.
2070 */
2071static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req)
2072{
2073	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2074			ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2075}
2076static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req)
2077{
2078	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2079			ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2080}
2081
2082/*
2083 * AES decryption functions.
2084 */
2085static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req)
2086{
2087	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2088			ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2089}
2090static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req)
2091{
2092	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2093			ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2094}
2095
2096/*
2097 * DES ecryption functions.
2098 */
2099static inline int hifn_encrypt_des_ecb(struct skcipher_request *req)
2100{
2101	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2102			ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2103}
2104static inline int hifn_encrypt_des_cbc(struct skcipher_request *req)
2105{
2106	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2107			ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2108}
2109
2110/*
2111 * DES decryption functions.
2112 */
2113static inline int hifn_decrypt_des_ecb(struct skcipher_request *req)
2114{
2115	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2116			ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2117}
2118static inline int hifn_decrypt_des_cbc(struct skcipher_request *req)
2119{
2120	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2121			ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2122}
2123
2124/*
2125 * 3DES ecryption functions.
2126 */
2127static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req)
2128{
2129	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2130			ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2131}
2132static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req)
2133{
2134	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2135			ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2136}
2137
2138/* 3DES decryption functions. */
2139static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req)
2140{
2141	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2142			ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2143}
2144static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req)
2145{
2146	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2147			ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2148}
2149
2150struct hifn_alg_template {
2151	char name[CRYPTO_MAX_ALG_NAME];
2152	char drv_name[CRYPTO_MAX_ALG_NAME];
2153	unsigned int bsize;
2154	struct skcipher_alg skcipher;
2155};
2156
2157static const struct hifn_alg_template hifn_alg_templates[] = {
2158	/*
2159	 * 3DES ECB and CBC modes.
2160	 */
2161	{
2162		.name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
2163		.skcipher = {
2164			.ivsize		=	HIFN_IV_LENGTH,
2165			.min_keysize	=	HIFN_3DES_KEY_LENGTH,
2166			.max_keysize	=	HIFN_3DES_KEY_LENGTH,
2167			.setkey		=	hifn_des3_setkey,
2168			.encrypt	=	hifn_encrypt_3des_cbc,
2169			.decrypt	=	hifn_decrypt_3des_cbc,
2170		},
2171	},
2172	{
2173		.name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
2174		.skcipher = {
2175			.min_keysize	=	HIFN_3DES_KEY_LENGTH,
2176			.max_keysize	=	HIFN_3DES_KEY_LENGTH,
2177			.setkey		=	hifn_des3_setkey,
2178			.encrypt	=	hifn_encrypt_3des_ecb,
2179			.decrypt	=	hifn_decrypt_3des_ecb,
2180		},
2181	},
2182
2183	/*
2184	 * DES ECB and CBC modes.
2185	 */
2186	{
2187		.name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
2188		.skcipher = {
2189			.ivsize		=	HIFN_IV_LENGTH,
2190			.min_keysize	=	HIFN_DES_KEY_LENGTH,
2191			.max_keysize	=	HIFN_DES_KEY_LENGTH,
2192			.setkey		=	hifn_setkey,
2193			.encrypt	=	hifn_encrypt_des_cbc,
2194			.decrypt	=	hifn_decrypt_des_cbc,
2195		},
2196	},
2197	{
2198		.name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
2199		.skcipher = {
2200			.min_keysize	=	HIFN_DES_KEY_LENGTH,
2201			.max_keysize	=	HIFN_DES_KEY_LENGTH,
2202			.setkey		=	hifn_setkey,
2203			.encrypt	=	hifn_encrypt_des_ecb,
2204			.decrypt	=	hifn_decrypt_des_ecb,
2205		},
2206	},
2207
2208	/*
2209	 * AES ECB and CBC modes.
2210	 */
2211	{
2212		.name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
2213		.skcipher = {
2214			.min_keysize	=	AES_MIN_KEY_SIZE,
2215			.max_keysize	=	AES_MAX_KEY_SIZE,
2216			.setkey		=	hifn_setkey,
2217			.encrypt	=	hifn_encrypt_aes_ecb,
2218			.decrypt	=	hifn_decrypt_aes_ecb,
2219		},
2220	},
2221	{
2222		.name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
2223		.skcipher = {
2224			.ivsize		=	HIFN_AES_IV_LENGTH,
2225			.min_keysize	=	AES_MIN_KEY_SIZE,
2226			.max_keysize	=	AES_MAX_KEY_SIZE,
2227			.setkey		=	hifn_setkey,
2228			.encrypt	=	hifn_encrypt_aes_cbc,
2229			.decrypt	=	hifn_decrypt_aes_cbc,
2230		},
2231	},
2232};
2233
2234static int hifn_init_tfm(struct crypto_skcipher *tfm)
2235{
2236	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
2237	struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);
2238	struct hifn_context *ctx = crypto_skcipher_ctx(tfm);
2239
2240	ctx->dev = ha->dev;
2241	crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context));
2242
2243	return 0;
2244}
2245
2246static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t)
2247{
2248	struct hifn_crypto_alg *alg;
2249	int err;
2250
2251	alg = kzalloc(sizeof(*alg), GFP_KERNEL);
2252	if (!alg)
2253		return -ENOMEM;
2254
2255	alg->alg = t->skcipher;
2256	alg->alg.init = hifn_init_tfm;
2257
2258	err = -EINVAL;
2259	if (snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
2260		     "%s", t->name) >= CRYPTO_MAX_ALG_NAME)
2261		goto out_free_alg;
2262	if (snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
2263		     "%s-%s", t->drv_name, dev->name) >= CRYPTO_MAX_ALG_NAME)
2264		goto out_free_alg;
2265
2266	alg->alg.base.cra_priority = 300;
2267	alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
2268	alg->alg.base.cra_blocksize = t->bsize;
2269	alg->alg.base.cra_ctxsize = sizeof(struct hifn_context);
2270	alg->alg.base.cra_alignmask = 0;
2271	alg->alg.base.cra_module = THIS_MODULE;
2272
2273	alg->dev = dev;
2274
2275	list_add_tail(&alg->entry, &dev->alg_list);
2276
2277	err = crypto_register_skcipher(&alg->alg);
2278	if (err) {
2279		list_del(&alg->entry);
2280out_free_alg:
2281		kfree(alg);
2282	}
2283
2284	return err;
2285}
2286
2287static void hifn_unregister_alg(struct hifn_device *dev)
2288{
2289	struct hifn_crypto_alg *a, *n;
2290
2291	list_for_each_entry_safe(a, n, &dev->alg_list, entry) {
2292		list_del(&a->entry);
2293		crypto_unregister_skcipher(&a->alg);
2294		kfree(a);
2295	}
2296}
2297
2298static int hifn_register_alg(struct hifn_device *dev)
2299{
2300	int i, err;
2301
2302	for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) {
2303		err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);
2304		if (err)
2305			goto err_out_exit;
2306	}
2307
2308	return 0;
2309
2310err_out_exit:
2311	hifn_unregister_alg(dev);
2312	return err;
2313}
2314
2315static void hifn_tasklet_callback(unsigned long data)
2316{
2317	struct hifn_device *dev = (struct hifn_device *)data;
2318
2319	/*
2320	 * This is ok to call this without lock being held,
2321	 * althogh it modifies some parameters used in parallel,
2322	 * (like dev->success), but they are used in process
2323	 * context or update is atomic (like setting dev->sa[i] to NULL).
2324	 */
2325	hifn_clear_rings(dev, 0);
2326
2327	if (dev->started < HIFN_QUEUE_LENGTH &&	dev->queue.qlen)
2328		hifn_process_queue(dev);
2329}
2330
2331static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2332{
2333	int err, i;
2334	struct hifn_device *dev;
2335	char name[8];
2336
2337	err = pci_enable_device(pdev);
2338	if (err)
2339		return err;
2340	pci_set_master(pdev);
2341
2342	err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2343	if (err)
2344		goto err_out_disable_pci_device;
2345
2346	snprintf(name, sizeof(name), "hifn%d",
2347			atomic_inc_return(&hifn_dev_number) - 1);
2348
2349	err = pci_request_regions(pdev, name);
2350	if (err)
2351		goto err_out_disable_pci_device;
2352
2353	if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||
2354	    pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||
2355	    pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {
2356		dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n");
2357		err = -ENODEV;
2358		goto err_out_free_regions;
2359	}
2360
2361	dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),
2362			GFP_KERNEL);
2363	if (!dev) {
2364		err = -ENOMEM;
2365		goto err_out_free_regions;
2366	}
2367
2368	INIT_LIST_HEAD(&dev->alg_list);
2369
2370	snprintf(dev->name, sizeof(dev->name), "%s", name);
2371	spin_lock_init(&dev->lock);
2372
2373	for (i = 0; i < 3; ++i) {
2374		unsigned long addr, size;
2375
2376		addr = pci_resource_start(pdev, i);
2377		size = pci_resource_len(pdev, i);
2378
2379		dev->bar[i] = ioremap(addr, size);
2380		if (!dev->bar[i]) {
2381			err = -ENOMEM;
2382			goto err_out_unmap_bars;
2383		}
2384	}
2385
2386	dev->desc_virt = dma_alloc_coherent(&pdev->dev,
2387					    sizeof(struct hifn_dma),
2388					    &dev->desc_dma, GFP_KERNEL);
2389	if (!dev->desc_virt) {
2390		dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n");
2391		err = -ENOMEM;
2392		goto err_out_unmap_bars;
2393	}
2394
2395	dev->pdev = pdev;
2396	dev->irq = pdev->irq;
2397
2398	for (i = 0; i < HIFN_D_RES_RSIZE; ++i)
2399		dev->sa[i] = NULL;
2400
2401	pci_set_drvdata(pdev, dev);
2402
2403	tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);
2404
2405	crypto_init_queue(&dev->queue, 1);
2406
2407	err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
2408	if (err) {
2409		dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n",
2410			dev->irq, err);
2411		dev->irq = 0;
2412		goto err_out_free_desc;
2413	}
2414
2415	err = hifn_start_device(dev);
2416	if (err)
2417		goto err_out_free_irq;
2418
2419	err = hifn_register_rng(dev);
2420	if (err)
2421		goto err_out_stop_device;
2422
2423	err = hifn_register_alg(dev);
2424	if (err)
2425		goto err_out_unregister_rng;
2426
2427	INIT_DELAYED_WORK(&dev->work, hifn_work);
2428	schedule_delayed_work(&dev->work, HZ);
2429
2430	dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been "
2431		"successfully registered as %s.\n",
2432		pci_name(pdev), dev->name);
2433
2434	return 0;
2435
2436err_out_unregister_rng:
2437	hifn_unregister_rng(dev);
2438err_out_stop_device:
2439	hifn_reset_dma(dev, 1);
2440	hifn_stop_device(dev);
2441err_out_free_irq:
2442	free_irq(dev->irq, dev);
2443	tasklet_kill(&dev->tasklet);
2444err_out_free_desc:
2445	dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt,
2446			  dev->desc_dma);
2447
2448err_out_unmap_bars:
2449	for (i = 0; i < 3; ++i)
2450		if (dev->bar[i])
2451			iounmap(dev->bar[i]);
2452	kfree(dev);
2453
2454err_out_free_regions:
2455	pci_release_regions(pdev);
2456
2457err_out_disable_pci_device:
2458	pci_disable_device(pdev);
2459
2460	return err;
2461}
2462
2463static void hifn_remove(struct pci_dev *pdev)
2464{
2465	int i;
2466	struct hifn_device *dev;
2467
2468	dev = pci_get_drvdata(pdev);
2469
2470	if (dev) {
2471		cancel_delayed_work_sync(&dev->work);
2472
2473		hifn_unregister_rng(dev);
2474		hifn_unregister_alg(dev);
2475		hifn_reset_dma(dev, 1);
2476		hifn_stop_device(dev);
2477
2478		free_irq(dev->irq, dev);
2479		tasklet_kill(&dev->tasklet);
2480
2481		hifn_flush(dev);
2482
2483		dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma),
2484				  dev->desc_virt, dev->desc_dma);
2485		for (i = 0; i < 3; ++i)
2486			if (dev->bar[i])
2487				iounmap(dev->bar[i]);
2488
2489		kfree(dev);
2490	}
2491
2492	pci_release_regions(pdev);
2493	pci_disable_device(pdev);
2494}
2495
2496static struct pci_device_id hifn_pci_tbl[] = {
2497	{ PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },
2498	{ PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },
2499	{ 0 }
2500};
2501MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);
2502
2503static struct pci_driver hifn_pci_driver = {
2504	.name     = "hifn795x",
2505	.id_table = hifn_pci_tbl,
2506	.probe    = hifn_probe,
2507	.remove   = hifn_remove,
2508};
2509
2510static int __init hifn_init(void)
2511{
2512	unsigned int freq;
2513	int err;
2514
2515	if (strncmp(hifn_pll_ref, "ext", 3) &&
2516	    strncmp(hifn_pll_ref, "pci", 3)) {
2517		pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext");
2518		return -EINVAL;
2519	}
2520
2521	/*
2522	 * For the 7955/7956 the reference clock frequency must be in the
2523	 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
2524	 * but this chip is currently not supported.
2525	 */
2526	if (hifn_pll_ref[3] != '\0') {
2527		freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
2528		if (freq < 20 || freq > 100) {
2529			pr_err("hifn795x: invalid hifn_pll_ref frequency, must"
2530			       "be in the range of 20-100");
2531			return -EINVAL;
2532		}
2533	}
2534
2535	err = pci_register_driver(&hifn_pci_driver);
2536	if (err < 0) {
2537		pr_err("Failed to register PCI driver for %s device.\n",
2538		       hifn_pci_driver.name);
2539		return -ENODEV;
2540	}
2541
2542	pr_info("Driver for HIFN 795x crypto accelerator chip "
2543		"has been successfully registered.\n");
2544
2545	return 0;
2546}
2547
2548static void __exit hifn_fini(void)
2549{
2550	pci_unregister_driver(&hifn_pci_driver);
2551
2552	pr_info("Driver for HIFN 795x crypto accelerator chip "
2553		"has been successfully unregistered.\n");
2554}
2555
2556module_init(hifn_init);
2557module_exit(hifn_fini);
2558
2559MODULE_LICENSE("GPL");
2560MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>");
2561MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");