iaa_crypto_main.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */
   3
   4#include <linux/init.h>
   5#include <linux/kernel.h>
   6#include <linux/module.h>
   7#include <linux/pci.h>
   8#include <linux/device.h>
   9#include <linux/iommu.h>
  10#include <uapi/linux/idxd.h>
  11#include <linux/highmem.h>
  12#include <linux/sched/smt.h>
  13#include <crypto/internal/acompress.h>
  14
  15#include "idxd.h"
  16#include "iaa_crypto.h"
  17#include "iaa_crypto_stats.h"
  18
  19#ifdef pr_fmt
  20#undef pr_fmt
  21#endif
  22
  23#define pr_fmt(fmt)			"idxd: " IDXD_SUBDRIVER_NAME ": " fmt
  24
  25#define IAA_ALG_PRIORITY               300
  26
  27/* number of iaa instances probed */
  28static unsigned int nr_iaa;
  29static unsigned int nr_cpus;
  30static unsigned int nr_nodes;
  31static unsigned int nr_cpus_per_node;
  32
  33/* Number of physical cpus sharing each iaa instance */
  34static unsigned int cpus_per_iaa;
  35
  36/* Per-cpu lookup table for balanced wqs */
  37static struct wq_table_entry __percpu *wq_table;
  38
  39static struct idxd_wq *wq_table_next_wq(int cpu)
  40{
  41	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
  42
  43	if (++entry->cur_wq >= entry->n_wqs)
  44		entry->cur_wq = 0;
  45
  46	if (!entry->wqs[entry->cur_wq])
  47		return NULL;
  48
  49	pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__,
  50		 entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id,
  51		 entry->wqs[entry->cur_wq]->id, cpu);
  52
  53	return entry->wqs[entry->cur_wq];
  54}
  55
  56static void wq_table_add(int cpu, struct idxd_wq *wq)
  57{
  58	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
  59
  60	if (WARN_ON(entry->n_wqs == entry->max_wqs))
  61		return;
  62
  63	entry->wqs[entry->n_wqs++] = wq;
  64
  65	pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__,
  66		 entry->wqs[entry->n_wqs - 1]->idxd->id,
  67		 entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu);
  68}
  69
  70static void wq_table_free_entry(int cpu)
  71{
  72	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
  73
  74	kfree(entry->wqs);
  75	memset(entry, 0, sizeof(*entry));
  76}
  77
  78static void wq_table_clear_entry(int cpu)
  79{
  80	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
  81
  82	entry->n_wqs = 0;
  83	entry->cur_wq = 0;
  84	memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *));
  85}
  86
  87LIST_HEAD(iaa_devices);
  88DEFINE_MUTEX(iaa_devices_lock);
  89
  90/* If enabled, IAA hw crypto algos are registered, unavailable otherwise */
  91static bool iaa_crypto_enabled;
  92static bool iaa_crypto_registered;
  93
  94/* Verify results of IAA compress or not */
  95static bool iaa_verify_compress = true;
  96
  97static ssize_t verify_compress_show(struct device_driver *driver, char *buf)
  98{
  99	return sprintf(buf, "%d\n", iaa_verify_compress);
 100}
 101
 102static ssize_t verify_compress_store(struct device_driver *driver,
 103				     const char *buf, size_t count)
 104{
 105	int ret = -EBUSY;
 106
 107	mutex_lock(&iaa_devices_lock);
 108
 109	if (iaa_crypto_enabled)
 110		goto out;
 111
 112	ret = kstrtobool(buf, &iaa_verify_compress);
 113	if (ret)
 114		goto out;
 115
 116	ret = count;
 117out:
 118	mutex_unlock(&iaa_devices_lock);
 119
 120	return ret;
 121}
 122static DRIVER_ATTR_RW(verify_compress);
 123
 124/*
 125 * The iaa crypto driver supports three 'sync' methods determining how
 126 * compressions and decompressions are performed:
 127 *
 128 * - sync:      the compression or decompression completes before
 129 *              returning.  This is the mode used by the async crypto
 130 *              interface when the sync mode is set to 'sync' and by
 131 *              the sync crypto interface regardless of setting.
 132 *
 133 * - async:     the compression or decompression is submitted and returns
 134 *              immediately.  Completion interrupts are not used so
 135 *              the caller is responsible for polling the descriptor
 136 *              for completion.  This mode is applicable to only the
 137 *              async crypto interface and is ignored for anything
 138 *              else.
 139 *
 140 * - async_irq: the compression or decompression is submitted and
 141 *              returns immediately.  Completion interrupts are
 142 *              enabled so the caller can wait for the completion and
 143 *              yield to other threads.  When the compression or
 144 *              decompression completes, the completion is signaled
 145 *              and the caller awakened.  This mode is applicable to
 146 *              only the async crypto interface and is ignored for
 147 *              anything else.
 148 *
 149 * These modes can be set using the iaa_crypto sync_mode driver
 150 * attribute.
 151 */
 152
 153/* Use async mode */
 154static bool async_mode;
 155/* Use interrupts */
 156static bool use_irq;
 157
 158/**
 159 * set_iaa_sync_mode - Set IAA sync mode
 160 * @name: The name of the sync mode
 161 *
 162 * Make the IAA sync mode named @name the current sync mode used by
 163 * compression/decompression.
 164 */
 165
 166static int set_iaa_sync_mode(const char *name)
 167{
 168	int ret = 0;
 169
 170	if (sysfs_streq(name, "sync")) {
 171		async_mode = false;
 172		use_irq = false;
 173	} else if (sysfs_streq(name, "async")) {
 174		async_mode = false;
 175		use_irq = false;
 176	} else if (sysfs_streq(name, "async_irq")) {
 177		async_mode = true;
 178		use_irq = true;
 179	} else {
 180		ret = -EINVAL;
 181	}
 182
 183	return ret;
 184}
 185
 186static ssize_t sync_mode_show(struct device_driver *driver, char *buf)
 187{
 188	int ret = 0;
 189
 190	if (!async_mode && !use_irq)
 191		ret = sprintf(buf, "%s\n", "sync");
 192	else if (async_mode && !use_irq)
 193		ret = sprintf(buf, "%s\n", "async");
 194	else if (async_mode && use_irq)
 195		ret = sprintf(buf, "%s\n", "async_irq");
 196
 197	return ret;
 198}
 199
 200static ssize_t sync_mode_store(struct device_driver *driver,
 201			       const char *buf, size_t count)
 202{
 203	int ret = -EBUSY;
 204
 205	mutex_lock(&iaa_devices_lock);
 206
 207	if (iaa_crypto_enabled)
 208		goto out;
 209
 210	ret = set_iaa_sync_mode(buf);
 211	if (ret == 0)
 212		ret = count;
 213out:
 214	mutex_unlock(&iaa_devices_lock);
 215
 216	return ret;
 217}
 218static DRIVER_ATTR_RW(sync_mode);
 219
 220static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX];
 221
 222static int find_empty_iaa_compression_mode(void)
 223{
 224	int i;
 225
 226	for (i = 0; i < IAA_COMP_MODES_MAX; i++)
 227		if (!iaa_compression_modes[i])
 228			return i;
 229
 230	return -EINVAL;
 231}
 232
 233static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx)
 234{
 235	struct iaa_compression_mode *mode;
 236	int i;
 237
 238	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
 239		mode = iaa_compression_modes[i];
 240		if (!mode)
 241			continue;
 242
 243		if (!strcmp(mode->name, name)) {
 244			*idx = i;
 245			return iaa_compression_modes[i];
 246		}
 247	}
 248
 249	return NULL;
 250}
 251
 252static void free_iaa_compression_mode(struct iaa_compression_mode *mode)
 253{
 254	kfree(mode->name);
 255	kfree(mode->ll_table);
 256	kfree(mode->d_table);
 257
 258	kfree(mode);
 259}
 260
 261/*
 262 * IAA Compression modes are defined by an ll_table and a d_table.
 263 * These tables are typically generated and captured using statistics
 264 * collected from running actual compress/decompress workloads.
 265 *
 266 * A module or other kernel code can add and remove compression modes
 267 * with a given name using the exported @add_iaa_compression_mode()
 268 * and @remove_iaa_compression_mode functions.
 269 *
 270 * When a new compression mode is added, the tables are saved in a
 271 * global compression mode list.  When IAA devices are added, a
 272 * per-IAA device dma mapping is created for each IAA device, for each
 273 * compression mode.  These are the tables used to do the actual
 274 * compression/deccompression and are unmapped if/when the devices are
 275 * removed.  Currently, compression modes must be added before any
 276 * device is added, and removed after all devices have been removed.
 277 */
 278
 279/**
 280 * remove_iaa_compression_mode - Remove an IAA compression mode
 281 * @name: The name the compression mode will be known as
 282 *
 283 * Remove the IAA compression mode named @name.
 284 */
 285void remove_iaa_compression_mode(const char *name)
 286{
 287	struct iaa_compression_mode *mode;
 288	int idx;
 289
 290	mutex_lock(&iaa_devices_lock);
 291
 292	if (!list_empty(&iaa_devices))
 293		goto out;
 294
 295	mode = find_iaa_compression_mode(name, &idx);
 296	if (mode) {
 297		free_iaa_compression_mode(mode);
 298		iaa_compression_modes[idx] = NULL;
 299	}
 300out:
 301	mutex_unlock(&iaa_devices_lock);
 302}
 303EXPORT_SYMBOL_GPL(remove_iaa_compression_mode);
 304
 305/**
 306 * add_iaa_compression_mode - Add an IAA compression mode
 307 * @name: The name the compression mode will be known as
 308 * @ll_table: The ll table
 309 * @ll_table_size: The ll table size in bytes
 310 * @d_table: The d table
 311 * @d_table_size: The d table size in bytes
 312 * @init: Optional callback function to init the compression mode data
 313 * @free: Optional callback function to free the compression mode data
 314 *
 315 * Add a new IAA compression mode named @name.
 316 *
 317 * Returns 0 if successful, errcode otherwise.
 318 */
 319int add_iaa_compression_mode(const char *name,
 320			     const u32 *ll_table,
 321			     int ll_table_size,
 322			     const u32 *d_table,
 323			     int d_table_size,
 324			     iaa_dev_comp_init_fn_t init,
 325			     iaa_dev_comp_free_fn_t free)
 326{
 327	struct iaa_compression_mode *mode;
 328	int idx, ret = -ENOMEM;
 329
 330	mutex_lock(&iaa_devices_lock);
 331
 332	if (!list_empty(&iaa_devices)) {
 333		ret = -EBUSY;
 334		goto out;
 335	}
 336
 337	mode = kzalloc(sizeof(*mode), GFP_KERNEL);
 338	if (!mode)
 339		goto out;
 340
 341	mode->name = kstrdup(name, GFP_KERNEL);
 342	if (!mode->name)
 343		goto free;
 344
 345	if (ll_table) {
 346		mode->ll_table = kmemdup(ll_table, ll_table_size, GFP_KERNEL);
 347		if (!mode->ll_table)
 348			goto free;
 349		mode->ll_table_size = ll_table_size;
 350	}
 351
 352	if (d_table) {
 353		mode->d_table = kmemdup(d_table, d_table_size, GFP_KERNEL);
 354		if (!mode->d_table)
 355			goto free;
 356		mode->d_table_size = d_table_size;
 357	}
 358
 359	mode->init = init;
 360	mode->free = free;
 361
 362	idx = find_empty_iaa_compression_mode();
 363	if (idx < 0)
 364		goto free;
 365
 366	pr_debug("IAA compression mode %s added at idx %d\n",
 367		 mode->name, idx);
 368
 369	iaa_compression_modes[idx] = mode;
 370
 371	ret = 0;
 372out:
 373	mutex_unlock(&iaa_devices_lock);
 374
 375	return ret;
 376free:
 377	free_iaa_compression_mode(mode);
 378	goto out;
 379}
 380EXPORT_SYMBOL_GPL(add_iaa_compression_mode);
 381
 382static struct iaa_device_compression_mode *
 383get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx)
 384{
 385	return iaa_device->compression_modes[idx];
 386}
 387
 388static void free_device_compression_mode(struct iaa_device *iaa_device,
 389					 struct iaa_device_compression_mode *device_mode)
 390{
 391	size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
 392	struct device *dev = &iaa_device->idxd->pdev->dev;
 393
 394	kfree(device_mode->name);
 395
 396	if (device_mode->aecs_comp_table)
 397		dma_free_coherent(dev, size, device_mode->aecs_comp_table,
 398				  device_mode->aecs_comp_table_dma_addr);
 399	kfree(device_mode);
 400}
 401
 402#define IDXD_OP_FLAG_AECS_RW_TGLS       0x400000
 403#define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC)
 404#define IAX_AECS_COMPRESS_FLAG	(IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
 405#define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
 406#define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \
 407						IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \
 408						IDXD_OP_FLAG_AECS_RW_TGLS)
 409
 410static int check_completion(struct device *dev,
 411			    struct iax_completion_record *comp,
 412			    bool compress,
 413			    bool only_once);
 414
 415static int init_device_compression_mode(struct iaa_device *iaa_device,
 416					struct iaa_compression_mode *mode,
 417					int idx, struct idxd_wq *wq)
 418{
 419	size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
 420	struct device *dev = &iaa_device->idxd->pdev->dev;
 421	struct iaa_device_compression_mode *device_mode;
 422	int ret = -ENOMEM;
 423
 424	device_mode = kzalloc(sizeof(*device_mode), GFP_KERNEL);
 425	if (!device_mode)
 426		return -ENOMEM;
 427
 428	device_mode->name = kstrdup(mode->name, GFP_KERNEL);
 429	if (!device_mode->name)
 430		goto free;
 431
 432	device_mode->aecs_comp_table = dma_alloc_coherent(dev, size,
 433							  &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL);
 434	if (!device_mode->aecs_comp_table)
 435		goto free;
 436
 437	/* Add Huffman table to aecs */
 438	memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table));
 439	memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size);
 440	memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size);
 441
 442	if (mode->init) {
 443		ret = mode->init(device_mode);
 444		if (ret)
 445			goto free;
 446	}
 447
 448	/* mode index should match iaa_compression_modes idx */
 449	iaa_device->compression_modes[idx] = device_mode;
 450
 451	pr_debug("IAA %s compression mode initialized for iaa device %d\n",
 452		 mode->name, iaa_device->idxd->id);
 453
 454	ret = 0;
 455out:
 456	return ret;
 457free:
 458	pr_debug("IAA %s compression mode initialization failed for iaa device %d\n",
 459		 mode->name, iaa_device->idxd->id);
 460
 461	free_device_compression_mode(iaa_device, device_mode);
 462	goto out;
 463}
 464
 465static int init_device_compression_modes(struct iaa_device *iaa_device,
 466					 struct idxd_wq *wq)
 467{
 468	struct iaa_compression_mode *mode;
 469	int i, ret = 0;
 470
 471	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
 472		mode = iaa_compression_modes[i];
 473		if (!mode)
 474			continue;
 475
 476		ret = init_device_compression_mode(iaa_device, mode, i, wq);
 477		if (ret)
 478			break;
 479	}
 480
 481	return ret;
 482}
 483
 484static void remove_device_compression_modes(struct iaa_device *iaa_device)
 485{
 486	struct iaa_device_compression_mode *device_mode;
 487	int i;
 488
 489	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
 490		device_mode = iaa_device->compression_modes[i];
 491		if (!device_mode)
 492			continue;
 493
 494		if (iaa_compression_modes[i]->free)
 495			iaa_compression_modes[i]->free(device_mode);
 496		free_device_compression_mode(iaa_device, device_mode);
 497		iaa_device->compression_modes[i] = NULL;
 498	}
 499}
 500
 501static struct iaa_device *iaa_device_alloc(void)
 502{
 503	struct iaa_device *iaa_device;
 504
 505	iaa_device = kzalloc(sizeof(*iaa_device), GFP_KERNEL);
 506	if (!iaa_device)
 507		return NULL;
 508
 509	INIT_LIST_HEAD(&iaa_device->wqs);
 510
 511	return iaa_device;
 512}
 513
 514static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
 515{
 516	struct iaa_wq *iaa_wq;
 517
 518	list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
 519		if (iaa_wq->wq == wq)
 520			return true;
 521	}
 522
 523	return false;
 524}
 525
 526static struct iaa_device *add_iaa_device(struct idxd_device *idxd)
 527{
 528	struct iaa_device *iaa_device;
 529
 530	iaa_device = iaa_device_alloc();
 531	if (!iaa_device)
 532		return NULL;
 533
 534	iaa_device->idxd = idxd;
 535
 536	list_add_tail(&iaa_device->list, &iaa_devices);
 537
 538	nr_iaa++;
 539
 540	return iaa_device;
 541}
 542
 543static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq)
 544{
 545	int ret = 0;
 546
 547	ret = init_device_compression_modes(iaa_device, iaa_wq->wq);
 548	if (ret)
 549		return ret;
 550
 551	return ret;
 552}
 553
 554static void del_iaa_device(struct iaa_device *iaa_device)
 555{
 556	list_del(&iaa_device->list);
 557
 558	nr_iaa--;
 559}
 560
 561static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq,
 562		      struct iaa_wq **new_wq)
 563{
 564	struct idxd_device *idxd = iaa_device->idxd;
 565	struct pci_dev *pdev = idxd->pdev;
 566	struct device *dev = &pdev->dev;
 567	struct iaa_wq *iaa_wq;
 568
 569	iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL);
 570	if (!iaa_wq)
 571		return -ENOMEM;
 572
 573	iaa_wq->wq = wq;
 574	iaa_wq->iaa_device = iaa_device;
 575	idxd_wq_set_private(wq, iaa_wq);
 576
 577	list_add_tail(&iaa_wq->list, &iaa_device->wqs);
 578
 579	iaa_device->n_wq++;
 580
 581	if (new_wq)
 582		*new_wq = iaa_wq;
 583
 584	dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n",
 585		wq->id, iaa_device->idxd->id, iaa_device->n_wq);
 586
 587	return 0;
 588}
 589
 590static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
 591{
 592	struct idxd_device *idxd = iaa_device->idxd;
 593	struct pci_dev *pdev = idxd->pdev;
 594	struct device *dev = &pdev->dev;
 595	struct iaa_wq *iaa_wq;
 596
 597	list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
 598		if (iaa_wq->wq == wq) {
 599			list_del(&iaa_wq->list);
 600			iaa_device->n_wq--;
 601
 602			dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n",
 603				wq->id, iaa_device->idxd->id,
 604				iaa_device->n_wq, nr_iaa);
 605
 606			if (iaa_device->n_wq == 0)
 607				del_iaa_device(iaa_device);
 608			break;
 609		}
 610	}
 611}
 612
 613static void clear_wq_table(void)
 614{
 615	int cpu;
 616
 617	for (cpu = 0; cpu < nr_cpus; cpu++)
 618		wq_table_clear_entry(cpu);
 619
 620	pr_debug("cleared wq table\n");
 621}
 622
 623static void free_iaa_device(struct iaa_device *iaa_device)
 624{
 625	if (!iaa_device)
 626		return;
 627
 628	remove_device_compression_modes(iaa_device);
 629	kfree(iaa_device);
 630}
 631
 632static void __free_iaa_wq(struct iaa_wq *iaa_wq)
 633{
 634	struct iaa_device *iaa_device;
 635
 636	if (!iaa_wq)
 637		return;
 638
 639	iaa_device = iaa_wq->iaa_device;
 640	if (iaa_device->n_wq == 0)
 641		free_iaa_device(iaa_wq->iaa_device);
 642}
 643
 644static void free_iaa_wq(struct iaa_wq *iaa_wq)
 645{
 646	struct idxd_wq *wq;
 647
 648	__free_iaa_wq(iaa_wq);
 649
 650	wq = iaa_wq->wq;
 651
 652	kfree(iaa_wq);
 653	idxd_wq_set_private(wq, NULL);
 654}
 655
 656static int iaa_wq_get(struct idxd_wq *wq)
 657{
 658	struct idxd_device *idxd = wq->idxd;
 659	struct iaa_wq *iaa_wq;
 660	int ret = 0;
 661
 662	spin_lock(&idxd->dev_lock);
 663	iaa_wq = idxd_wq_get_private(wq);
 664	if (iaa_wq && !iaa_wq->remove) {
 665		iaa_wq->ref++;
 666		idxd_wq_get(wq);
 667	} else {
 668		ret = -ENODEV;
 669	}
 670	spin_unlock(&idxd->dev_lock);
 671
 672	return ret;
 673}
 674
 675static int iaa_wq_put(struct idxd_wq *wq)
 676{
 677	struct idxd_device *idxd = wq->idxd;
 678	struct iaa_wq *iaa_wq;
 679	bool free = false;
 680	int ret = 0;
 681
 682	spin_lock(&idxd->dev_lock);
 683	iaa_wq = idxd_wq_get_private(wq);
 684	if (iaa_wq) {
 685		iaa_wq->ref--;
 686		if (iaa_wq->ref == 0 && iaa_wq->remove) {
 687			idxd_wq_set_private(wq, NULL);
 688			free = true;
 689		}
 690		idxd_wq_put(wq);
 691	} else {
 692		ret = -ENODEV;
 693	}
 694	spin_unlock(&idxd->dev_lock);
 695	if (free) {
 696		__free_iaa_wq(iaa_wq);
 697		kfree(iaa_wq);
 698	}
 699
 700	return ret;
 701}
 702
 703static void free_wq_table(void)
 704{
 705	int cpu;
 706
 707	for (cpu = 0; cpu < nr_cpus; cpu++)
 708		wq_table_free_entry(cpu);
 709
 710	free_percpu(wq_table);
 711
 712	pr_debug("freed wq table\n");
 713}
 714
 715static int alloc_wq_table(int max_wqs)
 716{
 717	struct wq_table_entry *entry;
 718	int cpu;
 719
 720	wq_table = alloc_percpu(struct wq_table_entry);
 721	if (!wq_table)
 722		return -ENOMEM;
 723
 724	for (cpu = 0; cpu < nr_cpus; cpu++) {
 725		entry = per_cpu_ptr(wq_table, cpu);
 726		entry->wqs = kcalloc(max_wqs, sizeof(*entry->wqs), GFP_KERNEL);
 727		if (!entry->wqs) {
 728			free_wq_table();
 729			return -ENOMEM;
 730		}
 731
 732		entry->max_wqs = max_wqs;
 733	}
 734
 735	pr_debug("initialized wq table\n");
 736
 737	return 0;
 738}
 739
 740static int save_iaa_wq(struct idxd_wq *wq)
 741{
 742	struct iaa_device *iaa_device, *found = NULL;
 743	struct idxd_device *idxd;
 744	struct pci_dev *pdev;
 745	struct device *dev;
 746	int ret = 0;
 747
 748	list_for_each_entry(iaa_device, &iaa_devices, list) {
 749		if (iaa_device->idxd == wq->idxd) {
 750			idxd = iaa_device->idxd;
 751			pdev = idxd->pdev;
 752			dev = &pdev->dev;
 753			/*
 754			 * Check to see that we don't already have this wq.
 755			 * Shouldn't happen but we don't control probing.
 756			 */
 757			if (iaa_has_wq(iaa_device, wq)) {
 758				dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n",
 759					iaa_device);
 760				goto out;
 761			}
 762
 763			found = iaa_device;
 764
 765			ret = add_iaa_wq(iaa_device, wq, NULL);
 766			if (ret)
 767				goto out;
 768
 769			break;
 770		}
 771	}
 772
 773	if (!found) {
 774		struct iaa_device *new_device;
 775		struct iaa_wq *new_wq;
 776
 777		new_device = add_iaa_device(wq->idxd);
 778		if (!new_device) {
 779			ret = -ENOMEM;
 780			goto out;
 781		}
 782
 783		ret = add_iaa_wq(new_device, wq, &new_wq);
 784		if (ret) {
 785			del_iaa_device(new_device);
 786			free_iaa_device(new_device);
 787			goto out;
 788		}
 789
 790		ret = init_iaa_device(new_device, new_wq);
 791		if (ret) {
 792			del_iaa_wq(new_device, new_wq->wq);
 793			del_iaa_device(new_device);
 794			free_iaa_wq(new_wq);
 795			goto out;
 796		}
 797	}
 798
 799	if (WARN_ON(nr_iaa == 0))
 800		return -EINVAL;
 801
 802	cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
 803	if (!cpus_per_iaa)
 804		cpus_per_iaa = 1;
 805out:
 806	return ret;
 807}
 808
 809static void remove_iaa_wq(struct idxd_wq *wq)
 810{
 811	struct iaa_device *iaa_device;
 812
 813	list_for_each_entry(iaa_device, &iaa_devices, list) {
 814		if (iaa_has_wq(iaa_device, wq)) {
 815			del_iaa_wq(iaa_device, wq);
 816			break;
 817		}
 818	}
 819
 820	if (nr_iaa) {
 821		cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
 822		if (!cpus_per_iaa)
 823			cpus_per_iaa = 1;
 824	} else
 825		cpus_per_iaa = 1;
 826}
 827
 828static int wq_table_add_wqs(int iaa, int cpu)
 829{
 830	struct iaa_device *iaa_device, *found_device = NULL;
 831	int ret = 0, cur_iaa = 0, n_wqs_added = 0;
 832	struct idxd_device *idxd;
 833	struct iaa_wq *iaa_wq;
 834	struct pci_dev *pdev;
 835	struct device *dev;
 836
 837	list_for_each_entry(iaa_device, &iaa_devices, list) {
 838		idxd = iaa_device->idxd;
 839		pdev = idxd->pdev;
 840		dev = &pdev->dev;
 841
 842		if (cur_iaa != iaa) {
 843			cur_iaa++;
 844			continue;
 845		}
 846
 847		found_device = iaa_device;
 848		dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n",
 849			found_device->idxd->id, cur_iaa);
 850		break;
 851	}
 852
 853	if (!found_device) {
 854		found_device = list_first_entry_or_null(&iaa_devices,
 855							struct iaa_device, list);
 856		if (!found_device) {
 857			pr_debug("couldn't find any iaa devices with wqs!\n");
 858			ret = -EINVAL;
 859			goto out;
 860		}
 861		cur_iaa = 0;
 862
 863		idxd = found_device->idxd;
 864		pdev = idxd->pdev;
 865		dev = &pdev->dev;
 866		dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n",
 867			found_device->idxd->id, cur_iaa);
 868	}
 869
 870	list_for_each_entry(iaa_wq, &found_device->wqs, list) {
 871		wq_table_add(cpu, iaa_wq->wq);
 872		pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n",
 873			 cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id);
 874		n_wqs_added++;
 875	}
 876
 877	if (!n_wqs_added) {
 878		pr_debug("couldn't find any iaa wqs!\n");
 879		ret = -EINVAL;
 880		goto out;
 881	}
 882out:
 883	return ret;
 884}
 885
 886/*
 887 * Rebalance the wq table so that given a cpu, it's easy to find the
 888 * closest IAA instance.  The idea is to try to choose the most
 889 * appropriate IAA instance for a caller and spread available
 890 * workqueues around to clients.
 891 */
 892static void rebalance_wq_table(void)
 893{
 894	const struct cpumask *node_cpus;
 895	int node_cpu, node, cpu, iaa = 0;
 896
 897	if (nr_iaa == 0)
 898		return;
 899
 900	pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n",
 901		 nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa);
 902
 903	clear_wq_table();
 904
 905	if (nr_iaa == 1) {
 906		for_each_possible_cpu(cpu) {
 907			if (WARN_ON(wq_table_add_wqs(0, cpu)))
 908				goto err;
 909		}
 910
 911		return;
 912	}
 913
 914	for_each_node_with_cpus(node) {
 915		cpu = 0;
 916		node_cpus = cpumask_of_node(node);
 917
 918		for_each_cpu(node_cpu, node_cpus) {
 919			iaa = cpu / cpus_per_iaa;
 920			if (WARN_ON(wq_table_add_wqs(iaa, node_cpu)))
 921				goto err;
 922			cpu++;
 923		}
 924	}
 925
 926	return;
 927err:
 928	pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu);
 929}
 930
 931static inline int check_completion(struct device *dev,
 932				   struct iax_completion_record *comp,
 933				   bool compress,
 934				   bool only_once)
 935{
 936	char *op_str = compress ? "compress" : "decompress";
 937	int status_checks = 0;
 938	int ret = 0;
 939
 940	while (!comp->status) {
 941		if (only_once)
 942			return -EAGAIN;
 943		cpu_relax();
 944		if (status_checks++ >= IAA_COMPLETION_TIMEOUT) {
 945			/* Something is wrong with the hw, disable it. */
 946			dev_err(dev, "%s completion timed out - "
 947				"assuming broken hw, iaa_crypto now DISABLED\n",
 948				op_str);
 949			iaa_crypto_enabled = false;
 950			ret = -ETIMEDOUT;
 951			goto out;
 952		}
 953	}
 954
 955	if (comp->status != IAX_COMP_SUCCESS) {
 956		if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) {
 957			ret = -ETIMEDOUT;
 958			dev_dbg(dev, "%s timed out, size=0x%x\n",
 959				op_str, comp->output_size);
 960			update_completion_timeout_errs();
 961			goto out;
 962		}
 963
 964		if (comp->status == IAA_ANALYTICS_ERROR &&
 965		    comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) {
 966			ret = -E2BIG;
 967			dev_dbg(dev, "compressed > uncompressed size,"
 968				" not compressing, size=0x%x\n",
 969				comp->output_size);
 970			update_completion_comp_buf_overflow_errs();
 971			goto out;
 972		}
 973
 974		if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) {
 975			ret = -EOVERFLOW;
 976			goto out;
 977		}
 978
 979		ret = -EINVAL;
 980		dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n",
 981			op_str, comp->status, comp->error_code, comp->output_size);
 982		print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, comp, 64, 0);
 983		update_completion_einval_errs();
 984
 985		goto out;
 986	}
 987out:
 988	return ret;
 989}
 990
 991static int deflate_generic_decompress(struct acomp_req *req)
 992{
 993	ACOMP_FBREQ_ON_STACK(fbreq, req);
 994	int ret;
 995
 996	ret = crypto_acomp_decompress(fbreq);
 997	req->dlen = fbreq->dlen;
 998
 999	update_total_sw_decomp_calls();
1000
1001	return ret;
1002}
1003
1004static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1005				struct acomp_req *req,
1006				dma_addr_t *src_addr, dma_addr_t *dst_addr);
1007
1008static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1009			       struct idxd_wq *wq,
1010			       dma_addr_t src_addr, unsigned int slen,
1011			       dma_addr_t dst_addr, unsigned int *dlen);
1012
1013static void iaa_desc_complete(struct idxd_desc *idxd_desc,
1014			      enum idxd_complete_type comp_type,
1015			      bool free_desc, void *__ctx,
1016			      u32 *status)
1017{
1018	struct iaa_device_compression_mode *active_compression_mode;
1019	struct iaa_compression_ctx *compression_ctx;
1020	struct crypto_ctx *ctx = __ctx;
1021	struct iaa_device *iaa_device;
1022	struct idxd_device *idxd;
1023	struct iaa_wq *iaa_wq;
1024	struct pci_dev *pdev;
1025	struct device *dev;
1026	int ret, err = 0;
1027
1028	compression_ctx = crypto_tfm_ctx(ctx->tfm);
1029
1030	iaa_wq = idxd_wq_get_private(idxd_desc->wq);
1031	iaa_device = iaa_wq->iaa_device;
1032	idxd = iaa_device->idxd;
1033	pdev = idxd->pdev;
1034	dev = &pdev->dev;
1035
1036	active_compression_mode = get_iaa_device_compression_mode(iaa_device,
1037								  compression_ctx->mode);
1038	dev_dbg(dev, "%s: compression mode %s,"
1039		" ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__,
1040		active_compression_mode->name,
1041		ctx->src_addr, ctx->dst_addr);
1042
1043	ret = check_completion(dev, idxd_desc->iax_completion,
1044			       ctx->compress, false);
1045	if (ret) {
1046		dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1047		if (!ctx->compress &&
1048		    idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1049			pr_warn("%s: falling back to deflate-generic decompress, "
1050				"analytics error code %x\n", __func__,
1051				idxd_desc->iax_completion->error_code);
1052			ret = deflate_generic_decompress(ctx->req);
1053			if (ret) {
1054				dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1055					__func__, ret);
1056				err = -EIO;
1057				goto err;
1058			}
1059		} else {
1060			err = -EIO;
1061			goto err;
1062		}
1063	} else {
1064		ctx->req->dlen = idxd_desc->iax_completion->output_size;
1065	}
1066
1067	/* Update stats */
1068	if (ctx->compress) {
1069		update_total_comp_bytes_out(ctx->req->dlen);
1070		update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen);
1071	} else {
1072		update_total_decomp_bytes_in(ctx->req->slen);
1073		update_wq_decomp_bytes(iaa_wq->wq, ctx->req->slen);
1074	}
1075
1076	if (ctx->compress && compression_ctx->verify_compress) {
1077		u32 *compression_crc = acomp_request_ctx(ctx->req);
1078		dma_addr_t src_addr, dst_addr;
1079
1080		*compression_crc = idxd_desc->iax_completion->crc;
1081
1082		ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);
1083		if (ret) {
1084			dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1085			err = -EIO;
1086			goto out;
1087		}
1088
1089		ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr,
1090					  ctx->req->slen, dst_addr, &ctx->req->dlen);
1091		if (ret) {
1092			dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret);
1093			err = -EIO;
1094		}
1095
1096		dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE);
1097		dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE);
1098
1099		goto out;
1100	}
1101err:
1102	dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
1103	dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
1104out:
1105	if (ret != 0)
1106		dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1107
1108	if (ctx->req->base.complete)
1109		acomp_request_complete(ctx->req, err);
1110
1111	if (free_desc)
1112		idxd_free_desc(idxd_desc->wq, idxd_desc);
1113	iaa_wq_put(idxd_desc->wq);
1114}
1115
1116static int iaa_compress(struct crypto_tfm *tfm,	struct acomp_req *req,
1117			struct idxd_wq *wq,
1118			dma_addr_t src_addr, unsigned int slen,
1119			dma_addr_t dst_addr, unsigned int *dlen)
1120{
1121	struct iaa_device_compression_mode *active_compression_mode;
1122	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1123	u32 *compression_crc = acomp_request_ctx(req);
1124	struct iaa_device *iaa_device;
1125	struct idxd_desc *idxd_desc;
1126	struct iax_hw_desc *desc;
1127	struct idxd_device *idxd;
1128	struct iaa_wq *iaa_wq;
1129	struct pci_dev *pdev;
1130	struct device *dev;
1131	int ret = 0;
1132
1133	iaa_wq = idxd_wq_get_private(wq);
1134	iaa_device = iaa_wq->iaa_device;
1135	idxd = iaa_device->idxd;
1136	pdev = idxd->pdev;
1137	dev = &pdev->dev;
1138
1139	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1140
1141	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1142	if (IS_ERR(idxd_desc)) {
1143		dev_dbg(dev, "idxd descriptor allocation failed\n");
1144		dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc));
1145		return PTR_ERR(idxd_desc);
1146	}
1147	desc = idxd_desc->iax_hw;
1148
1149	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR |
1150		IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC;
1151	desc->opcode = IAX_OPCODE_COMPRESS;
1152	desc->compr_flags = IAA_COMP_FLAGS;
1153	desc->priv = 0;
1154
1155	desc->src1_addr = (u64)src_addr;
1156	desc->src1_size = slen;
1157	desc->dst_addr = (u64)dst_addr;
1158	desc->max_dst_size = *dlen;
1159	desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr;
1160	desc->src2_size = sizeof(struct aecs_comp_table_record);
1161	desc->completion_addr = idxd_desc->compl_dma;
1162
1163	if (ctx->use_irq) {
1164		desc->flags |= IDXD_OP_FLAG_RCI;
1165
1166		idxd_desc->crypto.req = req;
1167		idxd_desc->crypto.tfm = tfm;
1168		idxd_desc->crypto.src_addr = src_addr;
1169		idxd_desc->crypto.dst_addr = dst_addr;
1170		idxd_desc->crypto.compress = true;
1171
1172		dev_dbg(dev, "%s use_async_irq: compression mode %s,"
1173			" src_addr %llx, dst_addr %llx\n", __func__,
1174			active_compression_mode->name,
1175			src_addr, dst_addr);
1176	}
1177
1178	dev_dbg(dev, "%s: compression mode %s,"
1179		" desc->src1_addr %llx, desc->src1_size %d,"
1180		" desc->dst_addr %llx, desc->max_dst_size %d,"
1181		" desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1182		active_compression_mode->name,
1183		desc->src1_addr, desc->src1_size, desc->dst_addr,
1184		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1185
1186	ret = idxd_submit_desc(wq, idxd_desc);
1187	if (ret) {
1188		dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1189		goto err;
1190	}
1191
1192	/* Update stats */
1193	update_total_comp_calls();
1194	update_wq_comp_calls(wq);
1195
1196	if (ctx->async_mode) {
1197		ret = -EINPROGRESS;
1198		dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1199		goto out;
1200	}
1201
1202	ret = check_completion(dev, idxd_desc->iax_completion, true, false);
1203	if (ret) {
1204		dev_dbg(dev, "check_completion failed ret=%d\n", ret);
1205		goto err;
1206	}
1207
1208	*dlen = idxd_desc->iax_completion->output_size;
1209
1210	/* Update stats */
1211	update_total_comp_bytes_out(*dlen);
1212	update_wq_comp_bytes(wq, *dlen);
1213
1214	*compression_crc = idxd_desc->iax_completion->crc;
1215
1216	if (!ctx->async_mode)
1217		idxd_free_desc(wq, idxd_desc);
1218out:
1219	return ret;
1220err:
1221	idxd_free_desc(wq, idxd_desc);
1222	dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1223
1224	goto out;
1225}
1226
1227static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1228				struct acomp_req *req,
1229				dma_addr_t *src_addr, dma_addr_t *dst_addr)
1230{
1231	int ret = 0;
1232	int nr_sgs;
1233
1234	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1235	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1236
1237	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1238	if (nr_sgs <= 0 || nr_sgs > 1) {
1239		dev_dbg(dev, "verify: couldn't map src sg for iaa device %d,"
1240			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1241			iaa_wq->wq->id, ret);
1242		ret = -EIO;
1243		goto out;
1244	}
1245	*src_addr = sg_dma_address(req->src);
1246	dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1247		" req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs,
1248		req->src, req->slen, sg_dma_len(req->src));
1249
1250	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1251	if (nr_sgs <= 0 || nr_sgs > 1) {
1252		dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d,"
1253			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1254			iaa_wq->wq->id, ret);
1255		ret = -EIO;
1256		dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1257		goto out;
1258	}
1259	*dst_addr = sg_dma_address(req->dst);
1260	dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1261		" req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs,
1262		req->dst, req->dlen, sg_dma_len(req->dst));
1263out:
1264	return ret;
1265}
1266
1267static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1268			       struct idxd_wq *wq,
1269			       dma_addr_t src_addr, unsigned int slen,
1270			       dma_addr_t dst_addr, unsigned int *dlen)
1271{
1272	struct iaa_device_compression_mode *active_compression_mode;
1273	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1274	u32 *compression_crc = acomp_request_ctx(req);
1275	struct iaa_device *iaa_device;
1276	struct idxd_desc *idxd_desc;
1277	struct iax_hw_desc *desc;
1278	struct idxd_device *idxd;
1279	struct iaa_wq *iaa_wq;
1280	struct pci_dev *pdev;
1281	struct device *dev;
1282	int ret = 0;
1283
1284	iaa_wq = idxd_wq_get_private(wq);
1285	iaa_device = iaa_wq->iaa_device;
1286	idxd = iaa_device->idxd;
1287	pdev = idxd->pdev;
1288	dev = &pdev->dev;
1289
1290	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1291
1292	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1293	if (IS_ERR(idxd_desc)) {
1294		dev_dbg(dev, "idxd descriptor allocation failed\n");
1295		dev_dbg(dev, "iaa compress failed: ret=%ld\n",
1296			PTR_ERR(idxd_desc));
1297		return PTR_ERR(idxd_desc);
1298	}
1299	desc = idxd_desc->iax_hw;
1300
1301	/* Verify (optional) - decompress and check crc, suppress dest write */
1302
1303	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1304	desc->opcode = IAX_OPCODE_DECOMPRESS;
1305	desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT;
1306	desc->priv = 0;
1307
1308	desc->src1_addr = (u64)dst_addr;
1309	desc->src1_size = *dlen;
1310	desc->dst_addr = (u64)src_addr;
1311	desc->max_dst_size = slen;
1312	desc->completion_addr = idxd_desc->compl_dma;
1313
1314	dev_dbg(dev, "(verify) compression mode %s,"
1315		" desc->src1_addr %llx, desc->src1_size %d,"
1316		" desc->dst_addr %llx, desc->max_dst_size %d,"
1317		" desc->src2_addr %llx, desc->src2_size %d\n",
1318		active_compression_mode->name,
1319		desc->src1_addr, desc->src1_size, desc->dst_addr,
1320		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1321
1322	ret = idxd_submit_desc(wq, idxd_desc);
1323	if (ret) {
1324		dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret);
1325		goto err;
1326	}
1327
1328	ret = check_completion(dev, idxd_desc->iax_completion, false, false);
1329	if (ret) {
1330		dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret);
1331		goto err;
1332	}
1333
1334	if (*compression_crc != idxd_desc->iax_completion->crc) {
1335		ret = -EINVAL;
1336		dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:"
1337			" comp=0x%x, decomp=0x%x\n", *compression_crc,
1338			idxd_desc->iax_completion->crc);
1339		print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET,
1340			       8, 1, idxd_desc->iax_completion, 64, 0);
1341		goto err;
1342	}
1343
1344	idxd_free_desc(wq, idxd_desc);
1345out:
1346	return ret;
1347err:
1348	idxd_free_desc(wq, idxd_desc);
1349	dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1350
1351	goto out;
1352}
1353
1354static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
1355			  struct idxd_wq *wq,
1356			  dma_addr_t src_addr, unsigned int slen,
1357			  dma_addr_t dst_addr, unsigned int *dlen)
1358{
1359	struct iaa_device_compression_mode *active_compression_mode;
1360	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1361	struct iaa_device *iaa_device;
1362	struct idxd_desc *idxd_desc;
1363	struct iax_hw_desc *desc;
1364	struct idxd_device *idxd;
1365	struct iaa_wq *iaa_wq;
1366	struct pci_dev *pdev;
1367	struct device *dev;
1368	int ret = 0;
1369
1370	iaa_wq = idxd_wq_get_private(wq);
1371	iaa_device = iaa_wq->iaa_device;
1372	idxd = iaa_device->idxd;
1373	pdev = idxd->pdev;
1374	dev = &pdev->dev;
1375
1376	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1377
1378	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1379	if (IS_ERR(idxd_desc)) {
1380		dev_dbg(dev, "idxd descriptor allocation failed\n");
1381		dev_dbg(dev, "iaa decompress failed: ret=%ld\n",
1382			PTR_ERR(idxd_desc));
1383		return PTR_ERR(idxd_desc);
1384	}
1385	desc = idxd_desc->iax_hw;
1386
1387	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1388	desc->opcode = IAX_OPCODE_DECOMPRESS;
1389	desc->max_dst_size = PAGE_SIZE;
1390	desc->decompr_flags = IAA_DECOMP_FLAGS;
1391	desc->priv = 0;
1392
1393	desc->src1_addr = (u64)src_addr;
1394	desc->dst_addr = (u64)dst_addr;
1395	desc->max_dst_size = *dlen;
1396	desc->src1_size = slen;
1397	desc->completion_addr = idxd_desc->compl_dma;
1398
1399	if (ctx->use_irq) {
1400		desc->flags |= IDXD_OP_FLAG_RCI;
1401
1402		idxd_desc->crypto.req = req;
1403		idxd_desc->crypto.tfm = tfm;
1404		idxd_desc->crypto.src_addr = src_addr;
1405		idxd_desc->crypto.dst_addr = dst_addr;
1406		idxd_desc->crypto.compress = false;
1407
1408		dev_dbg(dev, "%s: use_async_irq compression mode %s,"
1409			" src_addr %llx, dst_addr %llx\n", __func__,
1410			active_compression_mode->name,
1411			src_addr, dst_addr);
1412	}
1413
1414	dev_dbg(dev, "%s: decompression mode %s,"
1415		" desc->src1_addr %llx, desc->src1_size %d,"
1416		" desc->dst_addr %llx, desc->max_dst_size %d,"
1417		" desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1418		active_compression_mode->name,
1419		desc->src1_addr, desc->src1_size, desc->dst_addr,
1420		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1421
1422	ret = idxd_submit_desc(wq, idxd_desc);
1423	if (ret) {
1424		dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1425		goto err;
1426	}
1427
1428	/* Update stats */
1429	update_total_decomp_calls();
1430	update_wq_decomp_calls(wq);
1431
1432	if (ctx->async_mode) {
1433		ret = -EINPROGRESS;
1434		dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1435		goto out;
1436	}
1437
1438	ret = check_completion(dev, idxd_desc->iax_completion, false, false);
1439	if (ret) {
1440		dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1441		if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1442			pr_warn("%s: falling back to deflate-generic decompress, "
1443				"analytics error code %x\n", __func__,
1444				idxd_desc->iax_completion->error_code);
1445			ret = deflate_generic_decompress(req);
1446			if (ret) {
1447				dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1448					__func__, ret);
1449				goto err;
1450			}
1451		} else {
1452			goto err;
1453		}
1454	} else {
1455		req->dlen = idxd_desc->iax_completion->output_size;
1456	}
1457
1458	*dlen = req->dlen;
1459
1460	if (!ctx->async_mode)
1461		idxd_free_desc(wq, idxd_desc);
1462
1463	/* Update stats */
1464	update_total_decomp_bytes_in(slen);
1465	update_wq_decomp_bytes(wq, slen);
1466out:
1467	return ret;
1468err:
1469	idxd_free_desc(wq, idxd_desc);
1470	dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret);
1471
1472	goto out;
1473}
1474
1475static int iaa_comp_acompress(struct acomp_req *req)
1476{
1477	struct iaa_compression_ctx *compression_ctx;
1478	struct crypto_tfm *tfm = req->base.tfm;
1479	dma_addr_t src_addr, dst_addr;
1480	int nr_sgs, cpu, ret = 0;
1481	struct iaa_wq *iaa_wq;
1482	struct idxd_wq *wq;
1483	struct device *dev;
1484
1485	compression_ctx = crypto_tfm_ctx(tfm);
1486
1487	if (!iaa_crypto_enabled) {
1488		pr_debug("iaa_crypto disabled, not compressing\n");
1489		return -ENODEV;
1490	}
1491
1492	if (!req->src || !req->slen) {
1493		pr_debug("invalid src, not compressing\n");
1494		return -EINVAL;
1495	}
1496
1497	cpu = get_cpu();
1498	wq = wq_table_next_wq(cpu);
1499	put_cpu();
1500	if (!wq) {
1501		pr_debug("no wq configured for cpu=%d\n", cpu);
1502		return -ENODEV;
1503	}
1504
1505	ret = iaa_wq_get(wq);
1506	if (ret) {
1507		pr_debug("no wq available for cpu=%d\n", cpu);
1508		return -ENODEV;
1509	}
1510
1511	iaa_wq = idxd_wq_get_private(wq);
1512
1513	dev = &wq->idxd->pdev->dev;
1514
1515	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1516	if (nr_sgs <= 0 || nr_sgs > 1) {
1517		dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1518			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1519			iaa_wq->wq->id, ret);
1520		ret = -EIO;
1521		goto out;
1522	}
1523	src_addr = sg_dma_address(req->src);
1524	dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1525		" req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1526		req->src, req->slen, sg_dma_len(req->src));
1527
1528	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1529	if (nr_sgs <= 0 || nr_sgs > 1) {
1530		dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1531			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1532			iaa_wq->wq->id, ret);
1533		ret = -EIO;
1534		goto err_map_dst;
1535	}
1536	dst_addr = sg_dma_address(req->dst);
1537	dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1538		" req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1539		req->dst, req->dlen, sg_dma_len(req->dst));
1540
1541	ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr,
1542			   &req->dlen);
1543	if (ret == -EINPROGRESS)
1544		return ret;
1545
1546	if (!ret && compression_ctx->verify_compress) {
1547		ret = iaa_remap_for_verify(dev, iaa_wq, req, &src_addr, &dst_addr);
1548		if (ret) {
1549			dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1550			goto out;
1551		}
1552
1553		ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen,
1554					  dst_addr, &req->dlen);
1555		if (ret)
1556			dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret);
1557
1558		dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1559		dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1560
1561		goto out;
1562	}
1563
1564	if (ret)
1565		dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1566
1567	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1568err_map_dst:
1569	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1570out:
1571	iaa_wq_put(wq);
1572
1573	return ret;
1574}
1575
1576static int iaa_comp_adecompress(struct acomp_req *req)
1577{
1578	struct crypto_tfm *tfm = req->base.tfm;
1579	dma_addr_t src_addr, dst_addr;
1580	int nr_sgs, cpu, ret = 0;
1581	struct iaa_wq *iaa_wq;
1582	struct device *dev;
1583	struct idxd_wq *wq;
1584
1585	if (!iaa_crypto_enabled) {
1586		pr_debug("iaa_crypto disabled, not decompressing\n");
1587		return -ENODEV;
1588	}
1589
1590	if (!req->src || !req->slen) {
1591		pr_debug("invalid src, not decompressing\n");
1592		return -EINVAL;
1593	}
1594
1595	cpu = get_cpu();
1596	wq = wq_table_next_wq(cpu);
1597	put_cpu();
1598	if (!wq) {
1599		pr_debug("no wq configured for cpu=%d\n", cpu);
1600		return -ENODEV;
1601	}
1602
1603	ret = iaa_wq_get(wq);
1604	if (ret) {
1605		pr_debug("no wq available for cpu=%d\n", cpu);
1606		return -ENODEV;
1607	}
1608
1609	iaa_wq = idxd_wq_get_private(wq);
1610
1611	dev = &wq->idxd->pdev->dev;
1612
1613	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1614	if (nr_sgs <= 0 || nr_sgs > 1) {
1615		dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1616			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1617			iaa_wq->wq->id, ret);
1618		ret = -EIO;
1619		goto out;
1620	}
1621	src_addr = sg_dma_address(req->src);
1622	dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1623		" req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1624		req->src, req->slen, sg_dma_len(req->src));
1625
1626	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1627	if (nr_sgs <= 0 || nr_sgs > 1) {
1628		dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1629			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1630			iaa_wq->wq->id, ret);
1631		ret = -EIO;
1632		goto err_map_dst;
1633	}
1634	dst_addr = sg_dma_address(req->dst);
1635	dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1636		" req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1637		req->dst, req->dlen, sg_dma_len(req->dst));
1638
1639	ret = iaa_decompress(tfm, req, wq, src_addr, req->slen,
1640			     dst_addr, &req->dlen);
1641	if (ret == -EINPROGRESS)
1642		return ret;
1643
1644	if (ret != 0)
1645		dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
1646
1647	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1648err_map_dst:
1649	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1650out:
1651	iaa_wq_put(wq);
1652
1653	return ret;
1654}
1655
1656static void compression_ctx_init(struct iaa_compression_ctx *ctx)
1657{
1658	ctx->verify_compress = iaa_verify_compress;
1659	ctx->async_mode = async_mode;
1660	ctx->use_irq = use_irq;
1661}
1662
1663static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm)
1664{
1665	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
1666	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1667
1668	compression_ctx_init(ctx);
1669
1670	ctx->mode = IAA_MODE_FIXED;
1671
1672	return 0;
1673}
1674
1675static struct acomp_alg iaa_acomp_fixed_deflate = {
1676	.init			= iaa_comp_init_fixed,
1677	.compress		= iaa_comp_acompress,
1678	.decompress		= iaa_comp_adecompress,
1679	.base			= {
1680		.cra_name		= "deflate",
1681		.cra_driver_name	= "deflate-iaa",
1682		.cra_flags		= CRYPTO_ALG_ASYNC,
1683		.cra_ctxsize		= sizeof(struct iaa_compression_ctx),
1684		.cra_reqsize		= sizeof(u32),
1685		.cra_module		= THIS_MODULE,
1686		.cra_priority		= IAA_ALG_PRIORITY,
1687	}
1688};
1689
1690static int iaa_register_compression_device(void)
1691{
1692	int ret;
1693
1694	ret = crypto_register_acomp(&iaa_acomp_fixed_deflate);
1695	if (ret) {
1696		pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret);
1697		goto out;
1698	}
1699
1700	iaa_crypto_registered = true;
1701out:
1702	return ret;
1703}
1704
1705static int iaa_unregister_compression_device(void)
1706{
1707	if (iaa_crypto_registered)
1708		crypto_unregister_acomp(&iaa_acomp_fixed_deflate);
1709
1710	return 0;
1711}
1712
1713static int iaa_crypto_probe(struct idxd_dev *idxd_dev)
1714{
1715	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1716	struct idxd_device *idxd = wq->idxd;
1717	struct idxd_driver_data *data = idxd->data;
1718	struct device *dev = &idxd_dev->conf_dev;
1719	bool first_wq = false;
1720	int ret = 0;
1721
1722	if (idxd->state != IDXD_DEV_ENABLED)
1723		return -ENXIO;
1724
1725	if (data->type != IDXD_TYPE_IAX)
1726		return -ENODEV;
1727
1728	mutex_lock(&wq->wq_lock);
1729
1730	if (idxd_wq_get_private(wq)) {
1731		mutex_unlock(&wq->wq_lock);
1732		return -EBUSY;
1733	}
1734
1735	if (!idxd_wq_driver_name_match(wq, dev)) {
1736		dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n",
1737			idxd->id, wq->id, wq->driver_name, dev->driver->name);
1738		idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
1739		ret = -ENODEV;
1740		goto err;
1741	}
1742
1743	wq->type = IDXD_WQT_KERNEL;
1744
1745	ret = idxd_drv_enable_wq(wq);
1746	if (ret < 0) {
1747		dev_dbg(dev, "enable wq %d.%d failed: %d\n",
1748			idxd->id, wq->id, ret);
1749		ret = -ENXIO;
1750		goto err;
1751	}
1752
1753	mutex_lock(&iaa_devices_lock);
1754
1755	if (list_empty(&iaa_devices)) {
1756		ret = alloc_wq_table(wq->idxd->max_wqs);
1757		if (ret)
1758			goto err_alloc;
1759		first_wq = true;
1760	}
1761
1762	ret = save_iaa_wq(wq);
1763	if (ret)
1764		goto err_save;
1765
1766	rebalance_wq_table();
1767
1768	if (first_wq) {
1769		iaa_crypto_enabled = true;
1770		ret = iaa_register_compression_device();
1771		if (ret != 0) {
1772			iaa_crypto_enabled = false;
1773			dev_dbg(dev, "IAA compression device registration failed\n");
1774			goto err_register;
1775		}
1776		try_module_get(THIS_MODULE);
1777
1778		pr_info("iaa_crypto now ENABLED\n");
1779	}
1780
1781	mutex_unlock(&iaa_devices_lock);
1782out:
1783	mutex_unlock(&wq->wq_lock);
1784
1785	return ret;
1786
1787err_register:
1788	remove_iaa_wq(wq);
1789	free_iaa_wq(idxd_wq_get_private(wq));
1790err_save:
1791	if (first_wq)
1792		free_wq_table();
1793err_alloc:
1794	mutex_unlock(&iaa_devices_lock);
1795	idxd_drv_disable_wq(wq);
1796err:
1797	wq->type = IDXD_WQT_NONE;
1798
1799	goto out;
1800}
1801
1802static void iaa_crypto_remove(struct idxd_dev *idxd_dev)
1803{
1804	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1805	struct idxd_device *idxd = wq->idxd;
1806	struct iaa_wq *iaa_wq;
1807	bool free = false;
1808
1809	idxd_wq_quiesce(wq);
1810
1811	mutex_lock(&wq->wq_lock);
1812	mutex_lock(&iaa_devices_lock);
1813
1814	remove_iaa_wq(wq);
1815
1816	spin_lock(&idxd->dev_lock);
1817	iaa_wq = idxd_wq_get_private(wq);
1818	if (!iaa_wq) {
1819		spin_unlock(&idxd->dev_lock);
1820		pr_err("%s: no iaa_wq available to remove\n", __func__);
1821		goto out;
1822	}
1823
1824	if (iaa_wq->ref) {
1825		iaa_wq->remove = true;
1826	} else {
1827		wq = iaa_wq->wq;
1828		idxd_wq_set_private(wq, NULL);
1829		free = true;
1830	}
1831	spin_unlock(&idxd->dev_lock);
1832	if (free) {
1833		__free_iaa_wq(iaa_wq);
1834		kfree(iaa_wq);
1835	}
1836
1837	idxd_drv_disable_wq(wq);
1838	rebalance_wq_table();
1839
1840	if (nr_iaa == 0) {
1841		iaa_crypto_enabled = false;
1842		free_wq_table();
1843		module_put(THIS_MODULE);
1844
1845		pr_info("iaa_crypto now DISABLED\n");
1846	}
1847out:
1848	mutex_unlock(&iaa_devices_lock);
1849	mutex_unlock(&wq->wq_lock);
1850}
1851
1852static enum idxd_dev_type dev_types[] = {
1853	IDXD_DEV_WQ,
1854	IDXD_DEV_NONE,
1855};
1856
1857static struct idxd_device_driver iaa_crypto_driver = {
1858	.probe = iaa_crypto_probe,
1859	.remove = iaa_crypto_remove,
1860	.name = IDXD_SUBDRIVER_NAME,
1861	.type = dev_types,
1862	.desc_complete = iaa_desc_complete,
1863};
1864
1865static int __init iaa_crypto_init_module(void)
1866{
1867	int ret = 0;
1868	int node;
1869
1870	nr_cpus = num_possible_cpus();
1871	for_each_node_with_cpus(node)
1872		nr_nodes++;
1873	if (!nr_nodes) {
1874		pr_err("IAA couldn't find any nodes with cpus\n");
1875		return -ENODEV;
1876	}
1877	nr_cpus_per_node = nr_cpus / nr_nodes;
1878
1879	ret = iaa_aecs_init_fixed();
1880	if (ret < 0) {
1881		pr_debug("IAA fixed compression mode init failed\n");
1882		goto err_aecs_init;
1883	}
1884
1885	ret = idxd_driver_register(&iaa_crypto_driver);
1886	if (ret) {
1887		pr_debug("IAA wq sub-driver registration failed\n");
1888		goto err_driver_reg;
1889	}
1890
1891	ret = driver_create_file(&iaa_crypto_driver.drv,
1892				 &driver_attr_verify_compress);
1893	if (ret) {
1894		pr_debug("IAA verify_compress attr creation failed\n");
1895		goto err_verify_attr_create;
1896	}
1897
1898	ret = driver_create_file(&iaa_crypto_driver.drv,
1899				 &driver_attr_sync_mode);
1900	if (ret) {
1901		pr_debug("IAA sync mode attr creation failed\n");
1902		goto err_sync_attr_create;
1903	}
1904
1905	if (iaa_crypto_debugfs_init())
1906		pr_warn("debugfs init failed, stats not available\n");
1907
1908	pr_debug("initialized\n");
1909out:
1910	return ret;
1911
1912err_sync_attr_create:
1913	driver_remove_file(&iaa_crypto_driver.drv,
1914			   &driver_attr_verify_compress);
1915err_verify_attr_create:
1916	idxd_driver_unregister(&iaa_crypto_driver);
1917err_driver_reg:
1918	iaa_aecs_cleanup_fixed();
1919err_aecs_init:
1920
1921	goto out;
1922}
1923
1924static void __exit iaa_crypto_cleanup_module(void)
1925{
1926	if (iaa_unregister_compression_device())
1927		pr_debug("IAA compression device unregister failed\n");
1928
1929	iaa_crypto_debugfs_cleanup();
1930	driver_remove_file(&iaa_crypto_driver.drv,
1931			   &driver_attr_sync_mode);
1932	driver_remove_file(&iaa_crypto_driver.drv,
1933			   &driver_attr_verify_compress);
1934	idxd_driver_unregister(&iaa_crypto_driver);
1935	iaa_aecs_cleanup_fixed();
1936
1937	pr_debug("cleaned up\n");
1938}
1939
1940MODULE_IMPORT_NS("IDXD");
1941MODULE_LICENSE("GPL");
1942MODULE_ALIAS_IDXD_DEVICE(0);
1943MODULE_AUTHOR("Intel Corporation");
1944MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver");
1945
1946module_init(iaa_crypto_init_module);
1947module_exit(iaa_crypto_cleanup_module);