ap_bus.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Copyright IBM Corp. 2006, 2023
   4 * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
   5 *	      Martin Schwidefsky <schwidefsky@de.ibm.com>
   6 *	      Ralph Wuerthner <rwuerthn@de.ibm.com>
   7 *	      Felix Beck <felix.beck@de.ibm.com>
   8 *	      Holger Dengler <hd@linux.vnet.ibm.com>
   9 *	      Harald Freudenberger <freude@linux.ibm.com>
  10 *
  11 * Adjunct processor bus.
  12 */
  13
  14#define pr_fmt(fmt) "ap: " fmt
  15
  16#include <linux/kernel_stat.h>
  17#include <linux/moduleparam.h>
  18#include <linux/export.h>
  19#include <linux/init.h>
  20#include <linux/delay.h>
  21#include <linux/err.h>
  22#include <linux/freezer.h>
  23#include <linux/interrupt.h>
  24#include <linux/workqueue.h>
  25#include <linux/slab.h>
  26#include <linux/notifier.h>
  27#include <linux/kthread.h>
  28#include <linux/mutex.h>
  29#include <asm/machine.h>
  30#include <asm/airq.h>
  31#include <asm/tpi.h>
  32#include <linux/atomic.h>
  33#include <asm/isc.h>
  34#include <linux/hrtimer.h>
  35#include <linux/ktime.h>
  36#include <asm/facility.h>
  37#include <linux/crypto.h>
  38#include <linux/mod_devicetable.h>
  39#include <linux/debugfs.h>
  40#include <linux/ctype.h>
  41#include <linux/module.h>
  42#include <asm/uv.h>
  43#include <asm/chsc.h>
  44#include <linux/mempool.h>
  45
  46#include "ap_bus.h"
  47#include "ap_debug.h"
  48
  49MODULE_AUTHOR("IBM Corporation");
  50MODULE_DESCRIPTION("Adjunct Processor Bus driver");
  51MODULE_LICENSE("GPL");
  52
  53int ap_domain_index = -1;	/* Adjunct Processor Domain Index */
  54static DEFINE_SPINLOCK(ap_domain_lock);
  55module_param_named(domain, ap_domain_index, int, 0440);
  56MODULE_PARM_DESC(domain, "domain index for ap devices");
  57EXPORT_SYMBOL(ap_domain_index);
  58
  59static int ap_thread_flag;
  60module_param_named(poll_thread, ap_thread_flag, int, 0440);
  61MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 0 (off).");
  62
  63static char *apm_str;
  64module_param_named(apmask, apm_str, charp, 0440);
  65MODULE_PARM_DESC(apmask, "AP bus adapter mask.");
  66
  67static char *aqm_str;
  68module_param_named(aqmask, aqm_str, charp, 0440);
  69MODULE_PARM_DESC(aqmask, "AP bus domain mask.");
  70
  71static int ap_useirq = 1;
  72module_param_named(useirq, ap_useirq, int, 0440);
  73MODULE_PARM_DESC(useirq, "Use interrupt if available, default is 1 (on).");
  74
  75atomic_t ap_max_msg_size = ATOMIC_INIT(AP_DEFAULT_MAX_MSG_SIZE);
  76EXPORT_SYMBOL(ap_max_msg_size);
  77
  78static struct device *ap_root_device;
  79
  80/* Hashtable of all queue devices on the AP bus */
  81DEFINE_HASHTABLE(ap_queues, 8);
  82/* lock used for the ap_queues hashtable */
  83DEFINE_SPINLOCK(ap_queues_lock);
  84
  85/* Default permissions (ioctl, card and domain masking) */
  86struct ap_perms ap_perms;
  87EXPORT_SYMBOL(ap_perms);
  88/* true if apmask and/or aqmask are NOT default */
  89bool ap_apmask_aqmask_in_use;
  90/* counter for how many driver_overrides are currently active */
  91int ap_driver_override_ctr;
  92/*
  93 * Mutex for consistent read and write of the ap_perms struct,
  94 * ap_apmask_aqmask_in_use, ap_driver_override_ctr
  95 * and the ap bus sysfs attributes apmask and aqmask.
  96 */
  97DEFINE_MUTEX(ap_attr_mutex);
  98EXPORT_SYMBOL(ap_attr_mutex);
  99
 100/* # of bindings complete since init */
 101static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0);
 102
 103/* completion for APQN bindings complete */
 104static DECLARE_COMPLETION(ap_apqn_bindings_complete);
 105
 106static struct ap_config_info qci[2];
 107static struct ap_config_info *const ap_qci_info = &qci[0];
 108static struct ap_config_info *const ap_qci_info_old = &qci[1];
 109
 110/*
 111 * AP bus related debug feature things.
 112 */
 113debug_info_t *ap_dbf_info;
 114
 115/*
 116 * There is a need for a do-not-allocate-memory path through the AP bus
 117 * layer. The pkey layer may be triggered via the in-kernel interface from
 118 * a protected key crypto algorithm (namely PAES) to convert a secure key
 119 * into a protected key. This happens in a workqueue context, so sleeping
 120 * is allowed but memory allocations causing IO operations are not permitted.
 121 * To accomplish this, an AP message memory pool with pre-allocated space
 122 * is established. When ap_init_apmsg() with use_mempool set to true is
 123 * called, instead of kmalloc() the ap message buffer is allocated from
 124 * the ap_msg_pool. This pool only holds a limited amount of buffers:
 125 * ap_msg_pool_min_items with the item size AP_DEFAULT_MAX_MSG_SIZE and
 126 * exactly one of these items (if available) is returned if ap_init_apmsg()
 127 * with the use_mempool arg set to true is called. When this pool is exhausted
 128 * and use_mempool is set true, ap_init_apmsg() returns -ENOMEM without
 129 * any attempt to allocate memory and the caller has to deal with that.
 130 */
 131static mempool_t *ap_msg_pool;
 132static unsigned int ap_msg_pool_min_items = 8;
 133module_param_named(msgpool_min_items, ap_msg_pool_min_items, uint, 0440);
 134MODULE_PARM_DESC(msgpool_min_items, "AP message pool minimal items");
 135
 136/*
 137 * AP bus rescan related things.
 138 */
 139static bool ap_scan_bus(void);
 140static bool ap_scan_bus_result; /* result of last ap_scan_bus() */
 141static DEFINE_MUTEX(ap_scan_bus_mutex); /* mutex ap_scan_bus() invocations */
 142static struct task_struct *ap_scan_bus_task; /* thread holding the scan mutex */
 143static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */
 144static int ap_scan_bus_time = AP_CONFIG_TIME;
 145static struct timer_list ap_scan_bus_timer;
 146static void ap_scan_bus_wq_callback(struct work_struct *);
 147static DECLARE_WORK(ap_scan_bus_work, ap_scan_bus_wq_callback);
 148
 149/*
 150 * Tasklet & timer for AP request polling and interrupts
 151 */
 152static void ap_tasklet_fn(unsigned long);
 153static DECLARE_TASKLET_OLD(ap_tasklet, ap_tasklet_fn);
 154static DECLARE_WAIT_QUEUE_HEAD(ap_poll_wait);
 155static struct task_struct *ap_poll_kthread;
 156static DEFINE_MUTEX(ap_poll_thread_mutex);
 157static DEFINE_SPINLOCK(ap_poll_timer_lock);
 158static struct hrtimer ap_poll_timer;
 159/*
 160 * In LPAR poll with 4kHz frequency. Poll every 250000 nanoseconds.
 161 * If z/VM change to 1500000 nanoseconds to adjust to z/VM polling.
 162 */
 163static unsigned long poll_high_timeout = 250000UL;
 164
 165/*
 166 * Some state machine states only require a low frequency polling.
 167 * We use 25 Hz frequency for these.
 168 */
 169static unsigned long poll_low_timeout = 40000000UL;
 170
 171/* Maximum domain id, if not given via qci */
 172static int ap_max_domain_id = 15;
 173/* Maximum adapter id, if not given via qci */
 174static int ap_max_adapter_id = 63;
 175
 176static const struct bus_type ap_bus_type;
 177
 178/* Adapter interrupt definitions */
 179static void ap_interrupt_handler(struct airq_struct *airq,
 180				 struct tpi_info *tpi_info);
 181
 182static bool ap_irq_flag;
 183
 184static struct airq_struct ap_airq = {
 185	.handler = ap_interrupt_handler,
 186	.isc = AP_ISC,
 187};
 188
 189/**
 190 * ap_airq_ptr() - Get the address of the adapter interrupt indicator
 191 *
 192 * Returns the address of the local-summary-indicator of the adapter
 193 * interrupt handler for AP, or NULL if adapter interrupts are not
 194 * available.
 195 */
 196void *ap_airq_ptr(void)
 197{
 198	if (ap_irq_flag)
 199		return ap_airq.lsi_ptr;
 200	return NULL;
 201}
 202
 203/**
 204 * ap_interrupts_available(): Test if AP interrupts are available.
 205 *
 206 * Returns 1 if AP interrupts are available.
 207 */
 208static int ap_interrupts_available(void)
 209{
 210	return test_facility(65);
 211}
 212
 213/**
 214 * ap_qci_available(): Test if AP configuration
 215 * information can be queried via QCI subfunction.
 216 *
 217 * Returns 1 if subfunction PQAP(QCI) is available.
 218 */
 219static int ap_qci_available(void)
 220{
 221	return test_facility(12);
 222}
 223
 224/**
 225 * ap_apft_available(): Test if AP facilities test (APFT)
 226 * facility is available.
 227 *
 228 * Returns 1 if APFT is available.
 229 */
 230static int ap_apft_available(void)
 231{
 232	return test_facility(15);
 233}
 234
 235/*
 236 * ap_qact_available(): Test if the PQAP(QACT) subfunction is available.
 237 *
 238 * Returns 1 if the QACT subfunction is available.
 239 */
 240static inline int ap_qact_available(void)
 241{
 242	return ap_qci_info->qact;
 243}
 244
 245/*
 246 * ap_sb_available(): Test if the AP secure binding facility is available.
 247 *
 248 * Returns 1 if secure binding facility is available.
 249 */
 250int ap_sb_available(void)
 251{
 252	return ap_qci_info->apsb;
 253}
 254
 255/*
 256 * ap_is_se_guest(): Check for SE guest with AP pass-through support.
 257 */
 258bool ap_is_se_guest(void)
 259{
 260	return is_prot_virt_guest() && ap_sb_available();
 261}
 262EXPORT_SYMBOL(ap_is_se_guest);
 263
 264/**
 265 * ap_init_qci_info(): Allocate and query qci config info.
 266 * Does also update the static variables ap_max_domain_id
 267 * and ap_max_adapter_id if this info is available.
 268 */
 269static void __init ap_init_qci_info(void)
 270{
 271	if (!ap_qci_available() ||
 272	    ap_qci(ap_qci_info)) {
 273		AP_DBF_INFO("%s QCI not supported\n", __func__);
 274		return;
 275	}
 276	memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info));
 277	AP_DBF_INFO("%s successful fetched initial qci info\n", __func__);
 278
 279	if (ap_qci_info->apxa) {
 280		if (ap_qci_info->na) {
 281			ap_max_adapter_id = ap_qci_info->na;
 282			AP_DBF_INFO("%s new ap_max_adapter_id is %d\n",
 283				    __func__, ap_max_adapter_id);
 284		}
 285		if (ap_qci_info->nd) {
 286			ap_max_domain_id = ap_qci_info->nd;
 287			AP_DBF_INFO("%s new ap_max_domain_id is %d\n",
 288				    __func__, ap_max_domain_id);
 289		}
 290	}
 291}
 292
 293/*
 294 * ap_test_config(): helper function to extract the nrth bit
 295 *		     within the unsigned int array field.
 296 */
 297static inline int ap_test_config(unsigned int *field, unsigned int nr)
 298{
 299	return ap_test_bit((field + (nr >> 5)), (nr & 0x1f));
 300}
 301
 302/*
 303 * ap_test_config_card_id(): Test, whether an AP card ID is configured.
 304 *
 305 * Returns 0 if the card is not configured
 306 *	   1 if the card is configured or
 307 *	     if the configuration information is not available
 308 */
 309static inline int ap_test_config_card_id(unsigned int id)
 310{
 311	if (id > ap_max_adapter_id)
 312		return 0;
 313	if (ap_qci_info->flags)
 314		return ap_test_config(ap_qci_info->apm, id);
 315	return 1;
 316}
 317
 318/*
 319 * ap_test_config_usage_domain(): Test, whether an AP usage domain
 320 * is configured.
 321 *
 322 * Returns 0 if the usage domain is not configured
 323 *	   1 if the usage domain is configured or
 324 *	     if the configuration information is not available
 325 */
 326int ap_test_config_usage_domain(unsigned int domain)
 327{
 328	if (domain > ap_max_domain_id)
 329		return 0;
 330	if (ap_qci_info->flags)
 331		return ap_test_config(ap_qci_info->aqm, domain);
 332	return 1;
 333}
 334EXPORT_SYMBOL(ap_test_config_usage_domain);
 335
 336/*
 337 * ap_test_config_ctrl_domain(): Test, whether an AP control domain
 338 * is configured.
 339 * @domain AP control domain ID
 340 *
 341 * Returns 1 if the control domain is configured
 342 *	   0 in all other cases
 343 */
 344int ap_test_config_ctrl_domain(unsigned int domain)
 345{
 346	if (!ap_qci_info || domain > ap_max_domain_id)
 347		return 0;
 348	return ap_test_config(ap_qci_info->adm, domain);
 349}
 350EXPORT_SYMBOL(ap_test_config_ctrl_domain);
 351
 352/*
 353 * ap_queue_info(): Check and get AP queue info.
 354 * Returns: 1 if APQN exists and info is filled,
 355 *	    0 if APQN seems to exist but there is no info
 356 *	      available (eg. caused by an asynch pending error)
 357 *	   -1 invalid APQN, TAPQ error or AP queue status which
 358 *	      indicates there is no APQN.
 359 */
 360static int ap_queue_info(ap_qid_t qid, struct ap_tapq_hwinfo *hwinfo,
 361			 bool *decfg, bool *cstop)
 362{
 363	struct ap_queue_status status;
 364
 365	hwinfo->value = 0;
 366
 367	/* make sure we don't run into a specifiation exception */
 368	if (AP_QID_CARD(qid) > ap_max_adapter_id ||
 369	    AP_QID_QUEUE(qid) > ap_max_domain_id)
 370		return -1;
 371
 372	/* call TAPQ on this APQN */
 373	status = ap_test_queue(qid, ap_apft_available(), hwinfo);
 374
 375	switch (status.response_code) {
 376	case AP_RESPONSE_NORMAL:
 377	case AP_RESPONSE_RESET_IN_PROGRESS:
 378	case AP_RESPONSE_DECONFIGURED:
 379	case AP_RESPONSE_CHECKSTOPPED:
 380	case AP_RESPONSE_BUSY:
 381		/* For all these RCs the tapq info should be available */
 382		break;
 383	default:
 384		/* On a pending async error the info should be available */
 385		if (!status.async)
 386			return -1;
 387		break;
 388	}
 389
 390	/* There should be at least one of the mode bits set */
 391	if (WARN_ON_ONCE(!hwinfo->value))
 392		return 0;
 393
 394	*decfg = status.response_code == AP_RESPONSE_DECONFIGURED;
 395	*cstop = status.response_code == AP_RESPONSE_CHECKSTOPPED;
 396
 397	return 1;
 398}
 399
 400void ap_wait(enum ap_sm_wait wait)
 401{
 402	ktime_t hr_time;
 403
 404	switch (wait) {
 405	case AP_SM_WAIT_AGAIN:
 406	case AP_SM_WAIT_INTERRUPT:
 407		if (ap_irq_flag)
 408			break;
 409		if (ap_poll_kthread) {
 410			wake_up(&ap_poll_wait);
 411			break;
 412		}
 413		fallthrough;
 414	case AP_SM_WAIT_LOW_TIMEOUT:
 415	case AP_SM_WAIT_HIGH_TIMEOUT:
 416		spin_lock_bh(&ap_poll_timer_lock);
 417		if (!hrtimer_is_queued(&ap_poll_timer)) {
 418			hr_time =
 419				wait == AP_SM_WAIT_LOW_TIMEOUT ?
 420				poll_low_timeout : poll_high_timeout;
 421			hrtimer_forward_now(&ap_poll_timer, hr_time);
 422			hrtimer_restart(&ap_poll_timer);
 423		}
 424		spin_unlock_bh(&ap_poll_timer_lock);
 425		break;
 426	case AP_SM_WAIT_NONE:
 427	default:
 428		break;
 429	}
 430}
 431
 432/**
 433 * ap_request_timeout(): Handling of request timeouts
 434 * @t: timer making this callback
 435 *
 436 * Handles request timeouts.
 437 */
 438void ap_request_timeout(struct timer_list *t)
 439{
 440	struct ap_queue *aq = timer_container_of(aq, t, timeout);
 441
 442	spin_lock_bh(&aq->lock);
 443	ap_wait(ap_sm_event(aq, AP_SM_EVENT_TIMEOUT));
 444	spin_unlock_bh(&aq->lock);
 445}
 446
 447/**
 448 * ap_poll_timeout(): AP receive polling for finished AP requests.
 449 * @unused: Unused pointer.
 450 *
 451 * Schedules the AP tasklet using a high resolution timer.
 452 */
 453static enum hrtimer_restart ap_poll_timeout(struct hrtimer *unused)
 454{
 455	tasklet_schedule(&ap_tasklet);
 456	return HRTIMER_NORESTART;
 457}
 458
 459/**
 460 * ap_interrupt_handler() - Schedule ap_tasklet on interrupt
 461 * @airq: pointer to adapter interrupt descriptor
 462 * @tpi_info: ignored
 463 */
 464static void ap_interrupt_handler(struct airq_struct *airq,
 465				 struct tpi_info *tpi_info)
 466{
 467	inc_irq_stat(IRQIO_APB);
 468	tasklet_schedule(&ap_tasklet);
 469}
 470
 471/**
 472 * ap_tasklet_fn(): Tasklet to poll all AP devices.
 473 * @dummy: Unused variable
 474 *
 475 * Poll all AP devices on the bus.
 476 */
 477static void ap_tasklet_fn(unsigned long dummy)
 478{
 479	int bkt;
 480	struct ap_queue *aq;
 481	enum ap_sm_wait wait = AP_SM_WAIT_NONE;
 482
 483	/* Reset the indicator if interrupts are used. Thus new interrupts can
 484	 * be received. Doing it in the beginning of the tasklet is therefore
 485	 * important that no requests on any AP get lost.
 486	 */
 487	if (ap_irq_flag)
 488		WRITE_ONCE(*ap_airq.lsi_ptr, 0);
 489
 490	spin_lock_bh(&ap_queues_lock);
 491	hash_for_each(ap_queues, bkt, aq, hnode) {
 492		spin_lock_bh(&aq->lock);
 493		wait = min(wait, ap_sm_event_loop(aq, AP_SM_EVENT_POLL));
 494		spin_unlock_bh(&aq->lock);
 495	}
 496	spin_unlock_bh(&ap_queues_lock);
 497
 498	ap_wait(wait);
 499}
 500
 501static int ap_pending_requests(void)
 502{
 503	int bkt;
 504	struct ap_queue *aq;
 505
 506	spin_lock_bh(&ap_queues_lock);
 507	hash_for_each(ap_queues, bkt, aq, hnode) {
 508		if (aq->queue_count == 0)
 509			continue;
 510		spin_unlock_bh(&ap_queues_lock);
 511		return 1;
 512	}
 513	spin_unlock_bh(&ap_queues_lock);
 514	return 0;
 515}
 516
 517/**
 518 * ap_poll_thread(): Thread that polls for finished requests.
 519 * @data: Unused pointer
 520 *
 521 * AP bus poll thread. The purpose of this thread is to poll for
 522 * finished requests in a loop if there is a "free" cpu - that is
 523 * a cpu that doesn't have anything better to do. The polling stops
 524 * as soon as there is another task or if all messages have been
 525 * delivered.
 526 */
 527static int ap_poll_thread(void *data)
 528{
 529	DECLARE_WAITQUEUE(wait, current);
 530
 531	set_user_nice(current, MAX_NICE);
 532	set_freezable();
 533	while (!kthread_should_stop()) {
 534		add_wait_queue(&ap_poll_wait, &wait);
 535		set_current_state(TASK_INTERRUPTIBLE);
 536		if (!ap_pending_requests()) {
 537			schedule();
 538			try_to_freeze();
 539		}
 540		set_current_state(TASK_RUNNING);
 541		remove_wait_queue(&ap_poll_wait, &wait);
 542		if (need_resched()) {
 543			schedule();
 544			try_to_freeze();
 545			continue;
 546		}
 547		ap_tasklet_fn(0);
 548	}
 549
 550	return 0;
 551}
 552
 553static int ap_poll_thread_start(void)
 554{
 555	int rc;
 556
 557	if (ap_irq_flag || ap_poll_kthread)
 558		return 0;
 559	mutex_lock(&ap_poll_thread_mutex);
 560	ap_poll_kthread = kthread_run(ap_poll_thread, NULL, "appoll");
 561	rc = PTR_ERR_OR_ZERO(ap_poll_kthread);
 562	if (rc)
 563		ap_poll_kthread = NULL;
 564	mutex_unlock(&ap_poll_thread_mutex);
 565	return rc;
 566}
 567
 568static void ap_poll_thread_stop(void)
 569{
 570	if (!ap_poll_kthread)
 571		return;
 572	mutex_lock(&ap_poll_thread_mutex);
 573	kthread_stop(ap_poll_kthread);
 574	ap_poll_kthread = NULL;
 575	mutex_unlock(&ap_poll_thread_mutex);
 576}
 577
 578#define is_card_dev(x) ((x)->parent == ap_root_device)
 579#define is_queue_dev(x) ((x)->parent != ap_root_device)
 580
 581/*
 582 * ap_init_apmsg() - Initialize ap_message.
 583 */
 584int ap_init_apmsg(struct ap_message *ap_msg, u32 flags)
 585{
 586	unsigned int maxmsgsize;
 587
 588	memset(ap_msg, 0, sizeof(*ap_msg));
 589	ap_msg->flags = flags;
 590
 591	if (flags & AP_MSG_FLAG_MEMPOOL) {
 592		ap_msg->msg = mempool_alloc_preallocated(ap_msg_pool);
 593		if (!ap_msg->msg)
 594			return -ENOMEM;
 595		ap_msg->bufsize = AP_DEFAULT_MAX_MSG_SIZE;
 596		return 0;
 597	}
 598
 599	maxmsgsize = atomic_read(&ap_max_msg_size);
 600	ap_msg->msg = kmalloc(maxmsgsize, GFP_KERNEL);
 601	if (!ap_msg->msg)
 602		return -ENOMEM;
 603	ap_msg->bufsize = maxmsgsize;
 604
 605	return 0;
 606}
 607EXPORT_SYMBOL(ap_init_apmsg);
 608
 609/*
 610 * ap_release_apmsg() - Release ap_message.
 611 */
 612void ap_release_apmsg(struct ap_message *ap_msg)
 613{
 614	if (ap_msg->flags & AP_MSG_FLAG_MEMPOOL) {
 615		memzero_explicit(ap_msg->msg, ap_msg->bufsize);
 616		mempool_free(ap_msg->msg, ap_msg_pool);
 617	} else {
 618		kfree_sensitive(ap_msg->msg);
 619	}
 620}
 621EXPORT_SYMBOL(ap_release_apmsg);
 622
 623/**
 624 * ap_bus_match()
 625 * @dev: Pointer to device
 626 * @drv: Pointer to device_driver
 627 *
 628 * AP bus driver registration/unregistration.
 629 */
 630static int ap_bus_match(struct device *dev, const struct device_driver *drv)
 631{
 632	const struct ap_driver *ap_drv = to_ap_drv(drv);
 633	struct ap_device_id *id;
 634
 635	/*
 636	 * Compare device type of the device with the list of
 637	 * supported types of the device_driver.
 638	 */
 639	for (id = ap_drv->ids; id->match_flags; id++) {
 640		if (is_card_dev(dev) &&
 641		    id->match_flags & AP_DEVICE_ID_MATCH_CARD_TYPE &&
 642		    id->dev_type == to_ap_dev(dev)->device_type)
 643			return 1;
 644		if (is_queue_dev(dev) &&
 645		    id->match_flags & AP_DEVICE_ID_MATCH_QUEUE_TYPE &&
 646		    id->dev_type == to_ap_dev(dev)->device_type)
 647			return 1;
 648	}
 649	return 0;
 650}
 651
 652/**
 653 * ap_uevent(): Uevent function for AP devices.
 654 * @dev: Pointer to device
 655 * @env: Pointer to kobj_uevent_env
 656 *
 657 * It sets up a single environment variable DEV_TYPE which contains the
 658 * hardware device type.
 659 */
 660static int ap_uevent(const struct device *dev, struct kobj_uevent_env *env)
 661{
 662	int rc = 0;
 663	const struct ap_device *ap_dev = to_ap_dev(dev);
 664
 665	/* Uevents from ap bus core don't need extensions to the env */
 666	if (dev == ap_root_device)
 667		return 0;
 668
 669	if (is_card_dev(dev)) {
 670		struct ap_card *ac = to_ap_card(&ap_dev->device);
 671
 672		/* Set up DEV_TYPE environment variable. */
 673		rc = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
 674		if (rc)
 675			return rc;
 676		/* Add MODALIAS= */
 677		rc = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);
 678		if (rc)
 679			return rc;
 680
 681		/* Add MODE=<accel|cca|ep11> */
 682		if (ac->hwinfo.accel)
 683			rc = add_uevent_var(env, "MODE=accel");
 684		else if (ac->hwinfo.cca)
 685			rc = add_uevent_var(env, "MODE=cca");
 686		else if (ac->hwinfo.ep11)
 687			rc = add_uevent_var(env, "MODE=ep11");
 688		if (rc)
 689			return rc;
 690	} else {
 691		struct ap_queue *aq = to_ap_queue(&ap_dev->device);
 692
 693		/* Add MODE=<accel|cca|ep11> */
 694		if (aq->card->hwinfo.accel)
 695			rc = add_uevent_var(env, "MODE=accel");
 696		else if (aq->card->hwinfo.cca)
 697			rc = add_uevent_var(env, "MODE=cca");
 698		else if (aq->card->hwinfo.ep11)
 699			rc = add_uevent_var(env, "MODE=ep11");
 700		if (rc)
 701			return rc;
 702	}
 703
 704	return 0;
 705}
 706
 707static void ap_send_init_scan_done_uevent(void)
 708{
 709	char *envp[] = { "INITSCAN=done", NULL };
 710
 711	kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp);
 712}
 713
 714static void ap_send_bindings_complete_uevent(void)
 715{
 716	char buf[32];
 717	char *envp[] = { "BINDINGS=complete", buf, NULL };
 718
 719	snprintf(buf, sizeof(buf), "COMPLETECOUNT=%llu",
 720		 atomic64_inc_return(&ap_bindings_complete_count));
 721	kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp);
 722}
 723
 724void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg)
 725{
 726	char buf[16];
 727	char *envp[] = { buf, NULL };
 728
 729	snprintf(buf, sizeof(buf), "CONFIG=%d", cfg ? 1 : 0);
 730
 731	kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp);
 732}
 733EXPORT_SYMBOL(ap_send_config_uevent);
 734
 735void ap_send_online_uevent(struct ap_device *ap_dev, int online)
 736{
 737	char buf[16];
 738	char *envp[] = { buf, NULL };
 739
 740	snprintf(buf, sizeof(buf), "ONLINE=%d", online ? 1 : 0);
 741
 742	kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp);
 743}
 744EXPORT_SYMBOL(ap_send_online_uevent);
 745
 746static void ap_send_mask_changed_uevent(unsigned long *newapm,
 747					unsigned long *newaqm)
 748{
 749	char buf[100];
 750	char *envp[] = { buf, NULL };
 751
 752	if (newapm)
 753		snprintf(buf, sizeof(buf),
 754			 "APMASK=0x%016lx%016lx%016lx%016lx\n",
 755			 newapm[0], newapm[1], newapm[2], newapm[3]);
 756	else
 757		snprintf(buf, sizeof(buf),
 758			 "AQMASK=0x%016lx%016lx%016lx%016lx\n",
 759			 newaqm[0], newaqm[1], newaqm[2], newaqm[3]);
 760
 761	kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp);
 762}
 763
 764/*
 765 * calc # of bound APQNs
 766 */
 767
 768struct __ap_calc_ctrs {
 769	unsigned int apqns;
 770	unsigned int bound;
 771};
 772
 773static int __ap_calc_helper(struct device *dev, void *arg)
 774{
 775	struct __ap_calc_ctrs *pctrs = (struct __ap_calc_ctrs *)arg;
 776
 777	if (is_queue_dev(dev)) {
 778		pctrs->apqns++;
 779		if (dev->driver)
 780			pctrs->bound++;
 781	}
 782
 783	return 0;
 784}
 785
 786static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound)
 787{
 788	struct __ap_calc_ctrs ctrs;
 789
 790	memset(&ctrs, 0, sizeof(ctrs));
 791	bus_for_each_dev(&ap_bus_type, NULL, (void *)&ctrs, __ap_calc_helper);
 792
 793	*apqns = ctrs.apqns;
 794	*bound = ctrs.bound;
 795}
 796
 797/*
 798 * After ap bus scan do check if all existing APQNs are
 799 * bound to device drivers.
 800 */
 801static void ap_check_bindings_complete(void)
 802{
 803	unsigned int apqns, bound;
 804
 805	if (atomic64_read(&ap_scan_bus_count) >= 1) {
 806		ap_calc_bound_apqns(&apqns, &bound);
 807		if (bound == apqns) {
 808			if (!completion_done(&ap_apqn_bindings_complete)) {
 809				complete_all(&ap_apqn_bindings_complete);
 810				ap_send_bindings_complete_uevent();
 811				pr_debug("all apqn bindings complete\n");
 812			}
 813		}
 814	}
 815}
 816
 817/*
 818 * Interface to wait for the AP bus to have done one initial ap bus
 819 * scan and all detected APQNs have been bound to device drivers.
 820 * If these both conditions are not fulfilled, this function blocks
 821 * on a condition with wait_for_completion_interruptible_timeout().
 822 * If these both conditions are fulfilled (before the timeout hits)
 823 * the return value is 0. If the timeout (in jiffies) hits instead
 824 * -ETIME is returned. On failures negative return values are
 825 * returned to the caller.
 826 */
 827int ap_wait_apqn_bindings_complete(unsigned long timeout)
 828{
 829	int rc = 0;
 830	long l;
 831
 832	if (completion_done(&ap_apqn_bindings_complete))
 833		return 0;
 834
 835	if (timeout)
 836		l = wait_for_completion_interruptible_timeout(
 837			&ap_apqn_bindings_complete, timeout);
 838	else
 839		l = wait_for_completion_interruptible(
 840			&ap_apqn_bindings_complete);
 841	if (l < 0)
 842		rc = l == -ERESTARTSYS ? -EINTR : l;
 843	else if (l == 0 && timeout)
 844		rc = -ETIME;
 845
 846	pr_debug("rc=%d\n", rc);
 847	return rc;
 848}
 849EXPORT_SYMBOL(ap_wait_apqn_bindings_complete);
 850
 851static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data)
 852{
 853	if (is_queue_dev(dev) &&
 854	    AP_QID_CARD(to_ap_queue(dev)->qid) == (int)(long)data)
 855		device_unregister(dev);
 856	return 0;
 857}
 858
 859static int __ap_revise_reserved(struct device *dev, void *dummy)
 860{
 861	int rc, card, queue, devres, drvres;
 862
 863	if (is_queue_dev(dev)) {
 864		struct ap_driver *ap_drv = to_ap_drv(dev->driver);
 865		struct ap_queue *aq = to_ap_queue(dev);
 866		struct ap_device *ap_dev = &aq->ap_dev;
 867
 868		card = AP_QID_CARD(aq->qid);
 869		queue = AP_QID_QUEUE(aq->qid);
 870
 871		if (ap_dev->driver_override) {
 872			if (strcmp(ap_dev->driver_override,
 873				   ap_drv->driver.name)) {
 874				pr_debug("reprobing queue=%02x.%04x\n", card, queue);
 875				rc = device_reprobe(dev);
 876				if (rc) {
 877					AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n",
 878						    __func__, card, queue);
 879				}
 880			}
 881		} else {
 882			mutex_lock(&ap_attr_mutex);
 883			devres = test_bit_inv(card, ap_perms.apm) &&
 884				test_bit_inv(queue, ap_perms.aqm);
 885			mutex_unlock(&ap_attr_mutex);
 886			drvres = to_ap_drv(dev->driver)->flags
 887				& AP_DRIVER_FLAG_DEFAULT;
 888			if (!!devres != !!drvres) {
 889				pr_debug("reprobing queue=%02x.%04x\n", card, queue);
 890				rc = device_reprobe(dev);
 891				if (rc) {
 892					AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n",
 893						    __func__, card, queue);
 894				}
 895			}
 896		}
 897	}
 898
 899	return 0;
 900}
 901
 902static void ap_bus_revise_bindings(void)
 903{
 904	bus_for_each_dev(&ap_bus_type, NULL, NULL, __ap_revise_reserved);
 905}
 906
 907/**
 908 * ap_owned_by_def_drv: indicates whether an AP adapter is reserved for the
 909 *			default host driver or not.
 910 * @card: the APID of the adapter card to check
 911 * @queue: the APQI of the queue to check
 912 *
 913 * Note: the ap_attr_mutex must be locked by the caller of this function.
 914 *
 915 * Return: an int specifying whether the AP adapter is reserved for the host (1)
 916 *	   or not (0).
 917 */
 918int ap_owned_by_def_drv(int card, int queue)
 919{
 920	struct ap_queue *aq;
 921	int rc = 0;
 922
 923	if (card < 0 || card >= AP_DEVICES || queue < 0 || queue >= AP_DOMAINS)
 924		return -EINVAL;
 925
 926	aq = ap_get_qdev(AP_MKQID(card, queue));
 927	if (aq) {
 928		const struct device_driver *drv = aq->ap_dev.device.driver;
 929		const struct ap_driver *ap_drv = to_ap_drv(drv);
 930		bool override = !!aq->ap_dev.driver_override;
 931
 932		if (override && drv && ap_drv->flags & AP_DRIVER_FLAG_DEFAULT)
 933			rc = 1;
 934		put_device(&aq->ap_dev.device);
 935		if (override)
 936			goto out;
 937	}
 938
 939	if (test_bit_inv(card, ap_perms.apm) &&
 940	    test_bit_inv(queue, ap_perms.aqm))
 941		rc = 1;
 942
 943out:
 944	return rc;
 945}
 946EXPORT_SYMBOL(ap_owned_by_def_drv);
 947
 948/**
 949 * ap_apqn_in_matrix_owned_by_def_drv: indicates whether every APQN contained in
 950 *				       a set is reserved for the host drivers
 951 *				       or not.
 952 * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check
 953 * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check
 954 *
 955 * Note: the ap_attr_mutex must be locked by the caller of this function.
 956 *
 957 * Return: an int specifying whether each APQN is reserved for the host (1) or
 958 *	   not (0)
 959 */
 960int ap_apqn_in_matrix_owned_by_def_drv(unsigned long *apm,
 961				       unsigned long *aqm)
 962{
 963	int card, queue, rc = 0;
 964
 965	for (card = 0; !rc && card < AP_DEVICES; card++)
 966		if (test_bit_inv(card, apm))
 967			for (queue = 0; !rc && queue < AP_DOMAINS; queue++)
 968				if (test_bit_inv(queue, aqm))
 969					rc = ap_owned_by_def_drv(card, queue);
 970
 971	return rc;
 972}
 973EXPORT_SYMBOL(ap_apqn_in_matrix_owned_by_def_drv);
 974
 975static int ap_device_probe(struct device *dev)
 976{
 977	struct ap_device *ap_dev = to_ap_dev(dev);
 978	struct ap_driver *ap_drv = to_ap_drv(dev->driver);
 979	int card, queue, devres, drvres, rc = -ENODEV;
 980
 981	if (!get_device(dev))
 982		return rc;
 983
 984	if (is_queue_dev(dev)) {
 985		/*
 986		 * If the apqn is marked as reserved/used by ap bus and
 987		 * default drivers, only probe with drivers with the default
 988		 * flag set. If it is not marked, only probe with drivers
 989		 * with the default flag not set.
 990		 */
 991		card = AP_QID_CARD(to_ap_queue(dev)->qid);
 992		queue = AP_QID_QUEUE(to_ap_queue(dev)->qid);
 993		if (ap_dev->driver_override) {
 994			if (strcmp(ap_dev->driver_override,
 995				   ap_drv->driver.name))
 996				goto out;
 997		} else {
 998			mutex_lock(&ap_attr_mutex);
 999			devres = test_bit_inv(card, ap_perms.apm) &&
1000				test_bit_inv(queue, ap_perms.aqm);
1001			mutex_unlock(&ap_attr_mutex);
1002			drvres = ap_drv->flags & AP_DRIVER_FLAG_DEFAULT;
1003			if (!!devres != !!drvres)
1004				goto out;
1005		}
1006	}
1007
1008	/*
1009	 * Rearm the bindings complete completion to trigger
1010	 * bindings complete when all devices are bound again
1011	 */
1012	reinit_completion(&ap_apqn_bindings_complete);
1013
1014	/* Add queue/card to list of active queues/cards */
1015	spin_lock_bh(&ap_queues_lock);
1016	if (is_queue_dev(dev))
1017		hash_add(ap_queues, &to_ap_queue(dev)->hnode,
1018			 to_ap_queue(dev)->qid);
1019	spin_unlock_bh(&ap_queues_lock);
1020
1021	rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
1022
1023	if (rc) {
1024		spin_lock_bh(&ap_queues_lock);
1025		if (is_queue_dev(dev))
1026			hash_del(&to_ap_queue(dev)->hnode);
1027		spin_unlock_bh(&ap_queues_lock);
1028	}
1029
1030out:
1031	if (rc) {
1032		put_device(dev);
1033	} else {
1034		if (is_queue_dev(dev)) {
1035			pr_debug("queue=%02x.%04x new driver=%s\n",
1036				 card, queue, ap_drv->driver.name);
1037		} else {
1038			pr_debug("card=%02x new driver=%s\n",
1039				 to_ap_card(dev)->id, ap_drv->driver.name);
1040		}
1041	}
1042	return rc;
1043}
1044
1045static void ap_device_remove(struct device *dev)
1046{
1047	struct ap_device *ap_dev = to_ap_dev(dev);
1048	struct ap_driver *ap_drv = to_ap_drv(dev->driver);
1049
1050	/* prepare ap queue device removal */
1051	if (is_queue_dev(dev))
1052		ap_queue_prepare_remove(to_ap_queue(dev));
1053
1054	/* driver's chance to clean up gracefully */
1055	if (ap_drv->remove)
1056		ap_drv->remove(ap_dev);
1057
1058	/* now do the ap queue device remove */
1059	if (is_queue_dev(dev))
1060		ap_queue_remove(to_ap_queue(dev));
1061
1062	/* Remove queue/card from list of active queues/cards */
1063	spin_lock_bh(&ap_queues_lock);
1064	if (is_queue_dev(dev))
1065		hash_del(&to_ap_queue(dev)->hnode);
1066	spin_unlock_bh(&ap_queues_lock);
1067
1068	put_device(dev);
1069}
1070
1071struct ap_queue *ap_get_qdev(ap_qid_t qid)
1072{
1073	int bkt;
1074	struct ap_queue *aq;
1075
1076	spin_lock_bh(&ap_queues_lock);
1077	hash_for_each(ap_queues, bkt, aq, hnode) {
1078		if (aq->qid == qid) {
1079			get_device(&aq->ap_dev.device);
1080			spin_unlock_bh(&ap_queues_lock);
1081			return aq;
1082		}
1083	}
1084	spin_unlock_bh(&ap_queues_lock);
1085
1086	return NULL;
1087}
1088EXPORT_SYMBOL(ap_get_qdev);
1089
1090int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
1091		       char *name)
1092{
1093	struct device_driver *drv = &ap_drv->driver;
1094	int rc;
1095
1096	drv->bus = &ap_bus_type;
1097	drv->owner = owner;
1098	drv->name = name;
1099	rc = driver_register(drv);
1100
1101	ap_check_bindings_complete();
1102
1103	return rc;
1104}
1105EXPORT_SYMBOL(ap_driver_register);
1106
1107void ap_driver_unregister(struct ap_driver *ap_drv)
1108{
1109	driver_unregister(&ap_drv->driver);
1110}
1111EXPORT_SYMBOL(ap_driver_unregister);
1112
1113/*
1114 * Enforce a synchronous AP bus rescan.
1115 * Returns true if the bus scan finds a change in the AP configuration
1116 * and AP devices have been added or deleted when this function returns.
1117 */
1118bool ap_bus_force_rescan(void)
1119{
1120	unsigned long scan_counter = atomic64_read(&ap_scan_bus_count);
1121	bool rc = false;
1122
1123	pr_debug("> scan counter=%lu\n", scan_counter);
1124
1125	/* Only trigger AP bus scans after the initial scan is done */
1126	if (scan_counter <= 0)
1127		goto out;
1128
1129	/*
1130	 * There is one unlikely but nevertheless valid scenario where the
1131	 * thread holding the mutex may try to send some crypto load but
1132	 * all cards are offline so a rescan is triggered which causes
1133	 * a recursive call of ap_bus_force_rescan(). A simple return if
1134	 * the mutex is already locked by this thread solves this.
1135	 */
1136	if (mutex_is_locked(&ap_scan_bus_mutex)) {
1137		if (ap_scan_bus_task == current)
1138			goto out;
1139	}
1140
1141	/* Try to acquire the AP scan bus mutex */
1142	if (mutex_trylock(&ap_scan_bus_mutex)) {
1143		/* mutex acquired, run the AP bus scan */
1144		ap_scan_bus_task = current;
1145		ap_scan_bus_result = ap_scan_bus();
1146		rc = ap_scan_bus_result;
1147		ap_scan_bus_task = NULL;
1148		mutex_unlock(&ap_scan_bus_mutex);
1149		goto out;
1150	}
1151
1152	/*
1153	 * Mutex acquire failed. So there is currently another task
1154	 * already running the AP bus scan. Then let's simple wait
1155	 * for the lock which means the other task has finished and
1156	 * stored the result in ap_scan_bus_result.
1157	 */
1158	if (mutex_lock_interruptible(&ap_scan_bus_mutex)) {
1159		/* some error occurred, ignore and go out */
1160		goto out;
1161	}
1162	rc = ap_scan_bus_result;
1163	mutex_unlock(&ap_scan_bus_mutex);
1164
1165out:
1166	pr_debug("rc=%d\n", rc);
1167	return rc;
1168}
1169EXPORT_SYMBOL(ap_bus_force_rescan);
1170
1171/*
1172 * A config change has happened, force an ap bus rescan.
1173 */
1174static int ap_bus_cfg_chg(struct notifier_block *nb,
1175			  unsigned long action, void *data)
1176{
1177	if (action != CHSC_NOTIFY_AP_CFG)
1178		return NOTIFY_DONE;
1179
1180	pr_debug("config change, forcing bus rescan\n");
1181
1182	ap_bus_force_rescan();
1183
1184	return NOTIFY_OK;
1185}
1186
1187static struct notifier_block ap_bus_nb = {
1188	.notifier_call = ap_bus_cfg_chg,
1189};
1190
1191int ap_hex2bitmap(const char *str, unsigned long *bitmap, int bits)
1192{
1193	int i, n, b;
1194
1195	/* bits needs to be a multiple of 8 */
1196	if (bits & 0x07)
1197		return -EINVAL;
1198
1199	if (str[0] == '0' && str[1] == 'x')
1200		str++;
1201	if (*str == 'x')
1202		str++;
1203
1204	for (i = 0; isxdigit(*str) && i < bits; str++) {
1205		b = hex_to_bin(*str);
1206		for (n = 0; n < 4; n++)
1207			if (b & (0x08 >> n))
1208				set_bit_inv(i + n, bitmap);
1209		i += 4;
1210	}
1211
1212	if (*str == '\n')
1213		str++;
1214	if (*str)
1215		return -EINVAL;
1216	return 0;
1217}
1218EXPORT_SYMBOL(ap_hex2bitmap);
1219
1220/*
1221 * modify_bitmap() - parse bitmask argument and modify an existing
1222 * bit mask accordingly. A concatenation (done with ',') of these
1223 * terms is recognized:
1224 *   +<bitnr>[-<bitnr>] or -<bitnr>[-<bitnr>]
1225 * <bitnr> may be any valid number (hex, decimal or octal) in the range
1226 * 0...bits-1; the leading + or - is required. Here are some examples:
1227 *   +0-15,+32,-128,-0xFF
1228 *   -0-255,+1-16,+0x128
1229 *   +1,+2,+3,+4,-5,-7-10
1230 * Returns the new bitmap after all changes have been applied. Every
1231 * positive value in the string will set a bit and every negative value
1232 * in the string will clear a bit. As a bit may be touched more than once,
1233 * the last 'operation' wins:
1234 * +0-255,-128 = first bits 0-255 will be set, then bit 128 will be
1235 * cleared again. All other bits are unmodified.
1236 */
1237static int modify_bitmap(const char *str, unsigned long *bitmap, int bits)
1238{
1239	unsigned long a, i, z;
1240	char *np, sign;
1241
1242	/* bits needs to be a multiple of 8 */
1243	if (bits & 0x07)
1244		return -EINVAL;
1245
1246	while (*str) {
1247		sign = *str++;
1248		if (sign != '+' && sign != '-')
1249			return -EINVAL;
1250		a = z = simple_strtoul(str, &np, 0);
1251		if (str == np || a >= bits)
1252			return -EINVAL;
1253		str = np;
1254		if (*str == '-') {
1255			z = simple_strtoul(++str, &np, 0);
1256			if (str == np || a > z || z >= bits)
1257				return -EINVAL;
1258			str = np;
1259		}
1260		for (i = a; i <= z; i++)
1261			if (sign == '+')
1262				set_bit_inv(i, bitmap);
1263			else
1264				clear_bit_inv(i, bitmap);
1265		while (*str == ',' || *str == '\n')
1266			str++;
1267	}
1268
1269	return 0;
1270}
1271
1272static int ap_parse_bitmap_str(const char *str, unsigned long *bitmap, int bits,
1273			       unsigned long *newmap)
1274{
1275	unsigned long size;
1276	int rc;
1277
1278	size = BITS_TO_LONGS(bits) * sizeof(unsigned long);
1279	if (*str == '+' || *str == '-') {
1280		memcpy(newmap, bitmap, size);
1281		rc = modify_bitmap(str, newmap, bits);
1282	} else {
1283		memset(newmap, 0, size);
1284		rc = ap_hex2bitmap(str, newmap, bits);
1285	}
1286	return rc;
1287}
1288
1289int ap_parse_mask_str(const char *str,
1290		      unsigned long *bitmap, int bits,
1291		      struct mutex *lock)
1292{
1293	unsigned long *newmap, size;
1294	int rc;
1295
1296	/* bits needs to be a multiple of 8 */
1297	if (bits & 0x07)
1298		return -EINVAL;
1299
1300	size = BITS_TO_LONGS(bits) * sizeof(unsigned long);
1301	newmap = kmalloc(size, GFP_KERNEL);
1302	if (!newmap)
1303		return -ENOMEM;
1304	if (mutex_lock_interruptible(lock)) {
1305		kfree(newmap);
1306		return -ERESTARTSYS;
1307	}
1308	rc = ap_parse_bitmap_str(str, bitmap, bits, newmap);
1309	if (rc == 0)
1310		memcpy(bitmap, newmap, size);
1311	mutex_unlock(lock);
1312	kfree(newmap);
1313	return rc;
1314}
1315EXPORT_SYMBOL(ap_parse_mask_str);
1316
1317/*
1318 * AP bus attributes.
1319 */
1320
1321static ssize_t ap_domain_show(const struct bus_type *bus, char *buf)
1322{
1323	return sysfs_emit(buf, "%d\n", ap_domain_index);
1324}
1325
1326static ssize_t ap_domain_store(const struct bus_type *bus,
1327			       const char *buf, size_t count)
1328{
1329	int domain;
1330
1331	if (sscanf(buf, "%i\n", &domain) != 1 ||
1332	    domain < 0 || domain > ap_max_domain_id ||
1333	    !test_bit_inv(domain, ap_perms.aqm))
1334		return -EINVAL;
1335
1336	spin_lock_bh(&ap_domain_lock);
1337	ap_domain_index = domain;
1338	spin_unlock_bh(&ap_domain_lock);
1339
1340	AP_DBF_INFO("%s stored new default domain=%d\n",
1341		    __func__, domain);
1342
1343	return count;
1344}
1345
1346static BUS_ATTR_RW(ap_domain);
1347
1348static ssize_t ap_control_domain_mask_show(const struct bus_type *bus, char *buf)
1349{
1350	if (!ap_qci_info->flags)	/* QCI not supported */
1351		return sysfs_emit(buf, "not supported\n");
1352
1353	return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
1354			  ap_qci_info->adm[0], ap_qci_info->adm[1],
1355			  ap_qci_info->adm[2], ap_qci_info->adm[3],
1356			  ap_qci_info->adm[4], ap_qci_info->adm[5],
1357			  ap_qci_info->adm[6], ap_qci_info->adm[7]);
1358}
1359
1360static BUS_ATTR_RO(ap_control_domain_mask);
1361
1362static ssize_t ap_usage_domain_mask_show(const struct bus_type *bus, char *buf)
1363{
1364	if (!ap_qci_info->flags)	/* QCI not supported */
1365		return sysfs_emit(buf, "not supported\n");
1366
1367	return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
1368			  ap_qci_info->aqm[0], ap_qci_info->aqm[1],
1369			  ap_qci_info->aqm[2], ap_qci_info->aqm[3],
1370			  ap_qci_info->aqm[4], ap_qci_info->aqm[5],
1371			  ap_qci_info->aqm[6], ap_qci_info->aqm[7]);
1372}
1373
1374static BUS_ATTR_RO(ap_usage_domain_mask);
1375
1376static ssize_t ap_adapter_mask_show(const struct bus_type *bus, char *buf)
1377{
1378	if (!ap_qci_info->flags)	/* QCI not supported */
1379		return sysfs_emit(buf, "not supported\n");
1380
1381	return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
1382			  ap_qci_info->apm[0], ap_qci_info->apm[1],
1383			  ap_qci_info->apm[2], ap_qci_info->apm[3],
1384			  ap_qci_info->apm[4], ap_qci_info->apm[5],
1385			  ap_qci_info->apm[6], ap_qci_info->apm[7]);
1386}
1387
1388static BUS_ATTR_RO(ap_adapter_mask);
1389
1390static ssize_t ap_interrupts_show(const struct bus_type *bus, char *buf)
1391{
1392	return sysfs_emit(buf, "%d\n", ap_irq_flag ? 1 : 0);
1393}
1394
1395static BUS_ATTR_RO(ap_interrupts);
1396
1397static ssize_t config_time_show(const struct bus_type *bus, char *buf)
1398{
1399	return sysfs_emit(buf, "%d\n", ap_scan_bus_time);
1400}
1401
1402static ssize_t config_time_store(const struct bus_type *bus,
1403				 const char *buf, size_t count)
1404{
1405	int time;
1406
1407	if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120)
1408		return -EINVAL;
1409	ap_scan_bus_time = time;
1410	mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ);
1411	return count;
1412}
1413
1414static BUS_ATTR_RW(config_time);
1415
1416static ssize_t poll_thread_show(const struct bus_type *bus, char *buf)
1417{
1418	return sysfs_emit(buf, "%d\n", ap_poll_kthread ? 1 : 0);
1419}
1420
1421static ssize_t poll_thread_store(const struct bus_type *bus,
1422				 const char *buf, size_t count)
1423{
1424	bool value;
1425	int rc;
1426
1427	rc = kstrtobool(buf, &value);
1428	if (rc)
1429		return rc;
1430
1431	if (value) {
1432		rc = ap_poll_thread_start();
1433		if (rc)
1434			count = rc;
1435	} else {
1436		ap_poll_thread_stop();
1437	}
1438	return count;
1439}
1440
1441static BUS_ATTR_RW(poll_thread);
1442
1443static ssize_t poll_timeout_show(const struct bus_type *bus, char *buf)
1444{
1445	return sysfs_emit(buf, "%lu\n", poll_high_timeout);
1446}
1447
1448static ssize_t poll_timeout_store(const struct bus_type *bus, const char *buf,
1449				  size_t count)
1450{
1451	unsigned long value;
1452	ktime_t hr_time;
1453	int rc;
1454
1455	rc = kstrtoul(buf, 0, &value);
1456	if (rc)
1457		return rc;
1458
1459	/* 120 seconds = maximum poll interval */
1460	if (value > 120000000000UL)
1461		return -EINVAL;
1462	poll_high_timeout = value;
1463	hr_time = poll_high_timeout;
1464
1465	spin_lock_bh(&ap_poll_timer_lock);
1466	hrtimer_cancel(&ap_poll_timer);
1467	hrtimer_set_expires(&ap_poll_timer, hr_time);
1468	hrtimer_start_expires(&ap_poll_timer, HRTIMER_MODE_ABS);
1469	spin_unlock_bh(&ap_poll_timer_lock);
1470
1471	return count;
1472}
1473
1474static BUS_ATTR_RW(poll_timeout);
1475
1476static ssize_t ap_max_domain_id_show(const struct bus_type *bus, char *buf)
1477{
1478	return sysfs_emit(buf, "%d\n", ap_max_domain_id);
1479}
1480
1481static BUS_ATTR_RO(ap_max_domain_id);
1482
1483static ssize_t ap_max_adapter_id_show(const struct bus_type *bus, char *buf)
1484{
1485	return sysfs_emit(buf, "%d\n", ap_max_adapter_id);
1486}
1487
1488static BUS_ATTR_RO(ap_max_adapter_id);
1489
1490static ssize_t apmask_show(const struct bus_type *bus, char *buf)
1491{
1492	int rc;
1493
1494	if (mutex_lock_interruptible(&ap_attr_mutex))
1495		return -ERESTARTSYS;
1496	rc = sysfs_emit(buf, "0x%016lx%016lx%016lx%016lx\n",
1497			ap_perms.apm[0], ap_perms.apm[1],
1498			ap_perms.apm[2], ap_perms.apm[3]);
1499	mutex_unlock(&ap_attr_mutex);
1500
1501	return rc;
1502}
1503
1504static int __verify_card_reservations(struct device_driver *drv, void *data)
1505{
1506	int rc = 0;
1507	struct ap_driver *ap_drv = to_ap_drv(drv);
1508	unsigned long *newapm = (unsigned long *)data;
1509	unsigned long aqm_any[BITS_TO_LONGS(AP_DOMAINS)];
1510
1511	/*
1512	 * increase the driver's module refcounter to be sure it is not
1513	 * going away when we invoke the callback function.
1514	 */
1515	if (!try_module_get(drv->owner))
1516		return 0;
1517
1518	if (ap_drv->in_use) {
1519		bitmap_fill(aqm_any, AP_DOMAINS);
1520		rc = ap_drv->in_use(newapm, aqm_any);
1521		if (rc)
1522			rc = -EBUSY;
1523	}
1524
1525	/* release the driver's module */
1526	module_put(drv->owner);
1527
1528	return rc;
1529}
1530
1531static int apmask_commit(unsigned long *newapm)
1532{
1533	int rc;
1534	unsigned long reserved[BITS_TO_LONGS(AP_DEVICES)];
1535
1536	/*
1537	 * Check if any bits in the apmask have been set which will
1538	 * result in queues being removed from non-default drivers
1539	 */
1540	if (bitmap_andnot(reserved, newapm, ap_perms.apm, AP_DEVICES)) {
1541		rc = bus_for_each_drv(&ap_bus_type, NULL, reserved,
1542				      __verify_card_reservations);
1543		if (rc)
1544			return rc;
1545	}
1546
1547	memcpy(ap_perms.apm, newapm, APMASKSIZE);
1548
1549	/*
1550	 * Update ap_apmask_aqmask_in_use. Note that the
1551	 * ap_attr_mutex has to be obtained here.
1552	 */
1553	ap_apmask_aqmask_in_use =
1554		bitmap_full(ap_perms.apm, AP_DEVICES) &&
1555		bitmap_full(ap_perms.aqm, AP_DOMAINS) ?
1556		false : true;
1557
1558	return 0;
1559}
1560
1561static ssize_t apmask_store(const struct bus_type *bus, const char *buf,
1562			    size_t count)
1563{
1564	DECLARE_BITMAP(newapm, AP_DEVICES);
1565	int rc = -EINVAL, changes = 0;
1566
1567	if (mutex_lock_interruptible(&ap_attr_mutex))
1568		return -ERESTARTSYS;
1569
1570	/* Do not allow apmask/aqmask if driver override is active */
1571	if (ap_driver_override_ctr)
1572		goto done;
1573
1574	rc = ap_parse_bitmap_str(buf, ap_perms.apm, AP_DEVICES, newapm);
1575	if (rc)
1576		goto done;
1577
1578	changes = memcmp(ap_perms.apm, newapm, APMASKSIZE);
1579	if (changes)
1580		rc = apmask_commit(newapm);
1581
1582done:
1583	mutex_unlock(&ap_attr_mutex);
1584	if (rc)
1585		return rc;
1586
1587	if (changes) {
1588		ap_bus_revise_bindings();
1589		ap_send_mask_changed_uevent(newapm, NULL);
1590	}
1591
1592	return count;
1593}
1594
1595static BUS_ATTR_RW(apmask);
1596
1597static ssize_t aqmask_show(const struct bus_type *bus, char *buf)
1598{
1599	int rc;
1600
1601	if (mutex_lock_interruptible(&ap_attr_mutex))
1602		return -ERESTARTSYS;
1603	rc = sysfs_emit(buf, "0x%016lx%016lx%016lx%016lx\n",
1604			ap_perms.aqm[0], ap_perms.aqm[1],
1605			ap_perms.aqm[2], ap_perms.aqm[3]);
1606	mutex_unlock(&ap_attr_mutex);
1607
1608	return rc;
1609}
1610
1611static int __verify_queue_reservations(struct device_driver *drv, void *data)
1612{
1613	int rc = 0;
1614	struct ap_driver *ap_drv = to_ap_drv(drv);
1615	unsigned long *newaqm = (unsigned long *)data;
1616	unsigned long apm_any[BITS_TO_LONGS(AP_DEVICES)];
1617
1618	/*
1619	 * increase the driver's module refcounter to be sure it is not
1620	 * going away when we invoke the callback function.
1621	 */
1622	if (!try_module_get(drv->owner))
1623		return 0;
1624
1625	if (ap_drv->in_use) {
1626		bitmap_fill(apm_any, AP_DEVICES);
1627		rc = ap_drv->in_use(apm_any, newaqm);
1628		if (rc)
1629			rc = -EBUSY;
1630	}
1631
1632	/* release the driver's module */
1633	module_put(drv->owner);
1634
1635	return rc;
1636}
1637
1638static int aqmask_commit(unsigned long *newaqm)
1639{
1640	int rc;
1641	unsigned long reserved[BITS_TO_LONGS(AP_DOMAINS)];
1642
1643	/*
1644	 * Check if any bits in the aqmask have been set which will
1645	 * result in queues being removed from non-default drivers
1646	 */
1647	if (bitmap_andnot(reserved, newaqm, ap_perms.aqm, AP_DOMAINS)) {
1648		rc = bus_for_each_drv(&ap_bus_type, NULL, reserved,
1649				      __verify_queue_reservations);
1650		if (rc)
1651			return rc;
1652	}
1653
1654	memcpy(ap_perms.aqm, newaqm, AQMASKSIZE);
1655
1656	/*
1657	 * Update ap_apmask_aqmask_in_use. Note that the
1658	 * ap_attr_mutex has to be obtained here.
1659	 */
1660	ap_apmask_aqmask_in_use =
1661		bitmap_full(ap_perms.apm, AP_DEVICES) &&
1662		bitmap_full(ap_perms.aqm, AP_DOMAINS) ?
1663		false : true;
1664
1665	return 0;
1666}
1667
1668static ssize_t aqmask_store(const struct bus_type *bus, const char *buf,
1669			    size_t count)
1670{
1671	DECLARE_BITMAP(newaqm, AP_DOMAINS);
1672	int rc = -EINVAL, changes = 0;
1673
1674	if (mutex_lock_interruptible(&ap_attr_mutex))
1675		return -ERESTARTSYS;
1676
1677	/* Do not allow apmask/aqmask if driver override is active */
1678	if (ap_driver_override_ctr)
1679		goto done;
1680
1681	rc = ap_parse_bitmap_str(buf, ap_perms.aqm, AP_DOMAINS, newaqm);
1682	if (rc)
1683		goto done;
1684
1685	changes = memcmp(ap_perms.aqm, newaqm, APMASKSIZE);
1686	if (changes)
1687		rc = aqmask_commit(newaqm);
1688
1689done:
1690	mutex_unlock(&ap_attr_mutex);
1691	if (rc)
1692		return rc;
1693
1694	if (changes) {
1695		ap_bus_revise_bindings();
1696		ap_send_mask_changed_uevent(NULL, newaqm);
1697	}
1698
1699	return count;
1700}
1701
1702static BUS_ATTR_RW(aqmask);
1703
1704static ssize_t scans_show(const struct bus_type *bus, char *buf)
1705{
1706	return sysfs_emit(buf, "%llu\n", atomic64_read(&ap_scan_bus_count));
1707}
1708
1709static ssize_t scans_store(const struct bus_type *bus, const char *buf,
1710			   size_t count)
1711{
1712	AP_DBF_INFO("%s force AP bus rescan\n", __func__);
1713
1714	ap_bus_force_rescan();
1715
1716	return count;
1717}
1718
1719static BUS_ATTR_RW(scans);
1720
1721static ssize_t bindings_show(const struct bus_type *bus, char *buf)
1722{
1723	int rc;
1724	unsigned int apqns, n;
1725
1726	ap_calc_bound_apqns(&apqns, &n);
1727	if (atomic64_read(&ap_scan_bus_count) >= 1 && n == apqns)
1728		rc = sysfs_emit(buf, "%u/%u (complete)\n", n, apqns);
1729	else
1730		rc = sysfs_emit(buf, "%u/%u\n", n, apqns);
1731
1732	return rc;
1733}
1734
1735static BUS_ATTR_RO(bindings);
1736
1737static ssize_t bindings_complete_count_show(const struct bus_type *bus,
1738					    char *buf)
1739{
1740	return sysfs_emit(buf, "%llu\n",
1741			  atomic64_read(&ap_bindings_complete_count));
1742}
1743
1744static BUS_ATTR_RO(bindings_complete_count);
1745
1746static ssize_t features_show(const struct bus_type *bus, char *buf)
1747{
1748	int n = 0;
1749
1750	if (!ap_qci_info->flags)	/* QCI not supported */
1751		return sysfs_emit(buf, "-\n");
1752
1753	if (ap_qci_info->apsc)
1754		n += sysfs_emit_at(buf, n, "APSC ");
1755	if (ap_qci_info->apxa)
1756		n += sysfs_emit_at(buf, n, "APXA ");
1757	if (ap_qci_info->qact)
1758		n += sysfs_emit_at(buf, n, "QACT ");
1759	if (ap_qci_info->rc8a)
1760		n += sysfs_emit_at(buf, n, "RC8A ");
1761	if (ap_qci_info->apsb)
1762		n += sysfs_emit_at(buf, n, "APSB ");
1763
1764	sysfs_emit_at(buf, n == 0 ? 0 : n - 1, "\n");
1765
1766	return n;
1767}
1768
1769static BUS_ATTR_RO(features);
1770
1771static struct attribute *ap_bus_attrs[] = {
1772	&bus_attr_ap_domain.attr,
1773	&bus_attr_ap_control_domain_mask.attr,
1774	&bus_attr_ap_usage_domain_mask.attr,
1775	&bus_attr_ap_adapter_mask.attr,
1776	&bus_attr_config_time.attr,
1777	&bus_attr_poll_thread.attr,
1778	&bus_attr_ap_interrupts.attr,
1779	&bus_attr_poll_timeout.attr,
1780	&bus_attr_ap_max_domain_id.attr,
1781	&bus_attr_ap_max_adapter_id.attr,
1782	&bus_attr_apmask.attr,
1783	&bus_attr_aqmask.attr,
1784	&bus_attr_scans.attr,
1785	&bus_attr_bindings.attr,
1786	&bus_attr_bindings_complete_count.attr,
1787	&bus_attr_features.attr,
1788	NULL,
1789};
1790ATTRIBUTE_GROUPS(ap_bus);
1791
1792static const struct bus_type ap_bus_type = {
1793	.name = "ap",
1794	.bus_groups = ap_bus_groups,
1795	.match = &ap_bus_match,
1796	.uevent = &ap_uevent,
1797	.probe = ap_device_probe,
1798	.remove = ap_device_remove,
1799};
1800
1801/**
1802 * ap_select_domain(): Select an AP domain if possible and we haven't
1803 * already done so before.
1804 */
1805static void ap_select_domain(void)
1806{
1807	struct ap_queue_status status;
1808	int card, dom;
1809
1810	/*
1811	 * Choose the default domain. Either the one specified with
1812	 * the "domain=" parameter or the first domain with at least
1813	 * one valid APQN.
1814	 */
1815	spin_lock_bh(&ap_domain_lock);
1816	if (ap_domain_index >= 0) {
1817		/* Domain has already been selected. */
1818		goto out;
1819	}
1820	for (dom = 0; dom <= ap_max_domain_id; dom++) {
1821		if (!ap_test_config_usage_domain(dom) ||
1822		    !test_bit_inv(dom, ap_perms.aqm))
1823			continue;
1824		for (card = 0; card <= ap_max_adapter_id; card++) {
1825			if (!ap_test_config_card_id(card) ||
1826			    !test_bit_inv(card, ap_perms.apm))
1827				continue;
1828			status = ap_test_queue(AP_MKQID(card, dom),
1829					       ap_apft_available(),
1830					       NULL);
1831			if (status.response_code == AP_RESPONSE_NORMAL)
1832				break;
1833		}
1834		if (card <= ap_max_adapter_id)
1835			break;
1836	}
1837	if (dom <= ap_max_domain_id) {
1838		ap_domain_index = dom;
1839		AP_DBF_INFO("%s new default domain is %d\n",
1840			    __func__, ap_domain_index);
1841	}
1842out:
1843	spin_unlock_bh(&ap_domain_lock);
1844}
1845
1846/*
1847 * This function checks the type and returns either 0 for not
1848 * supported or the highest compatible type value (which may
1849 * include the input type value).
1850 */
1851static int ap_get_compatible_type(ap_qid_t qid, int rawtype, unsigned int func)
1852{
1853	int comp_type = 0;
1854
1855	/* < CEX4 is not supported */
1856	if (rawtype < AP_DEVICE_TYPE_CEX4) {
1857		AP_DBF_WARN("%s queue=%02x.%04x unsupported type %d\n",
1858			    __func__, AP_QID_CARD(qid),
1859			    AP_QID_QUEUE(qid), rawtype);
1860		return 0;
1861	}
1862	/* up to CEX8 known and fully supported */
1863	if (rawtype <= AP_DEVICE_TYPE_CEX8)
1864		return rawtype;
1865	/*
1866	 * unknown new type > CEX8, check for compatibility
1867	 * to the highest known and supported type which is
1868	 * currently CEX8 with the help of the QACT function.
1869	 */
1870	if (ap_qact_available()) {
1871		struct ap_queue_status status;
1872		union ap_qact_ap_info apinfo = {0};
1873
1874		apinfo.mode = (func >> 26) & 0x07;
1875		apinfo.cat = AP_DEVICE_TYPE_CEX8;
1876		status = ap_qact(qid, 0, &apinfo);
1877		if (status.response_code == AP_RESPONSE_NORMAL &&
1878		    apinfo.cat >= AP_DEVICE_TYPE_CEX4 &&
1879		    apinfo.cat <= AP_DEVICE_TYPE_CEX8)
1880			comp_type = apinfo.cat;
1881	}
1882	if (!comp_type)
1883		AP_DBF_WARN("%s queue=%02x.%04x unable to map type %d\n",
1884			    __func__, AP_QID_CARD(qid),
1885			    AP_QID_QUEUE(qid), rawtype);
1886	else if (comp_type != rawtype)
1887		AP_DBF_INFO("%s queue=%02x.%04x map type %d to %d\n",
1888			    __func__, AP_QID_CARD(qid), AP_QID_QUEUE(qid),
1889			    rawtype, comp_type);
1890	return comp_type;
1891}
1892
1893/*
1894 * Helper function to be used with bus_find_dev
1895 * matches for the card device with the given id
1896 */
1897static int __match_card_device_with_id(struct device *dev, const void *data)
1898{
1899	return is_card_dev(dev) && to_ap_card(dev)->id == (int)(long)(void *)data;
1900}
1901
1902/*
1903 * Helper function to be used with bus_find_dev
1904 * matches for the queue device with a given qid
1905 */
1906static int __match_queue_device_with_qid(struct device *dev, const void *data)
1907{
1908	return is_queue_dev(dev) && to_ap_queue(dev)->qid == (int)(long)data;
1909}
1910
1911/*
1912 * Helper function to be used with bus_find_dev
1913 * matches any queue device with given queue id
1914 */
1915static int __match_queue_device_with_queue_id(struct device *dev, const void *data)
1916{
1917	return is_queue_dev(dev) &&
1918		AP_QID_QUEUE(to_ap_queue(dev)->qid) == (int)(long)data;
1919}
1920
1921/* Helper function for notify_config_changed */
1922static int __drv_notify_config_changed(struct device_driver *drv, void *data)
1923{
1924	struct ap_driver *ap_drv = to_ap_drv(drv);
1925
1926	if (try_module_get(drv->owner)) {
1927		if (ap_drv->on_config_changed)
1928			ap_drv->on_config_changed(ap_qci_info, ap_qci_info_old);
1929		module_put(drv->owner);
1930	}
1931
1932	return 0;
1933}
1934
1935/* Notify all drivers about an qci config change */
1936static inline void notify_config_changed(void)
1937{
1938	bus_for_each_drv(&ap_bus_type, NULL, NULL,
1939			 __drv_notify_config_changed);
1940}
1941
1942/* Helper function for notify_scan_complete */
1943static int __drv_notify_scan_complete(struct device_driver *drv, void *data)
1944{
1945	struct ap_driver *ap_drv = to_ap_drv(drv);
1946
1947	if (try_module_get(drv->owner)) {
1948		if (ap_drv->on_scan_complete)
1949			ap_drv->on_scan_complete(ap_qci_info,
1950						 ap_qci_info_old);
1951		module_put(drv->owner);
1952	}
1953
1954	return 0;
1955}
1956
1957/* Notify all drivers about bus scan complete */
1958static inline void notify_scan_complete(void)
1959{
1960	bus_for_each_drv(&ap_bus_type, NULL, NULL,
1961			 __drv_notify_scan_complete);
1962}
1963
1964/*
1965 * Helper function for ap_scan_bus().
1966 * Remove card device and associated queue devices.
1967 */
1968static inline void ap_scan_rm_card_dev_and_queue_devs(struct ap_card *ac)
1969{
1970	bus_for_each_dev(&ap_bus_type, NULL,
1971			 (void *)(long)ac->id,
1972			 __ap_queue_devices_with_id_unregister);
1973	device_unregister(&ac->ap_dev.device);
1974}
1975
1976/*
1977 * Helper function for ap_scan_bus().
1978 * Does the scan bus job for all the domains within
1979 * a valid adapter given by an ap_card ptr.
1980 */
1981static inline void ap_scan_domains(struct ap_card *ac)
1982{
1983	struct ap_tapq_hwinfo hwinfo;
1984	bool decfg, chkstop;
1985	struct ap_queue *aq;
1986	struct device *dev;
1987	ap_qid_t qid;
1988	int rc, dom;
1989
1990	/*
1991	 * Go through the configuration for the domains and compare them
1992	 * to the existing queue devices. Also take care of the config
1993	 * and error state for the queue devices.
1994	 */
1995
1996	for (dom = 0; dom <= ap_max_domain_id; dom++) {
1997		qid = AP_MKQID(ac->id, dom);
1998		dev = bus_find_device(&ap_bus_type, NULL,
1999				      (void *)(long)qid,
2000				      __match_queue_device_with_qid);
2001		aq = dev ? to_ap_queue(dev) : NULL;
2002		if (!ap_test_config_usage_domain(dom)) {
2003			if (dev) {
2004				AP_DBF_INFO("%s(%d,%d) not in config anymore, rm queue dev\n",
2005					    __func__, ac->id, dom);
2006				device_unregister(dev);
2007			}
2008			goto put_dev_and_continue;
2009		}
2010		/* domain is valid, get info from this APQN */
2011		rc = ap_queue_info(qid, &hwinfo, &decfg, &chkstop);
2012		switch (rc) {
2013		case -1:
2014			if (dev) {
2015				AP_DBF_INFO("%s(%d,%d) queue_info() failed, rm queue dev\n",
2016					    __func__, ac->id, dom);
2017				device_unregister(dev);
2018			}
2019			fallthrough;
2020		case 0:
2021			goto put_dev_and_continue;
2022		default:
2023			break;
2024		}
2025		/* if no queue device exists, create a new one */
2026		if (!aq) {
2027			aq = ap_queue_create(qid, ac);
2028			if (!aq) {
2029				AP_DBF_WARN("%s(%d,%d) ap_queue_create() failed\n",
2030					    __func__, ac->id, dom);
2031				continue;
2032			}
2033			aq->config = !decfg;
2034			aq->chkstop = chkstop;
2035			aq->se_bstate = hwinfo.bs;
2036			dev = &aq->ap_dev.device;
2037			dev->bus = &ap_bus_type;
2038			dev->parent = &ac->ap_dev.device;
2039			dev_set_name(dev, "%02x.%04x", ac->id, dom);
2040			/* register queue device */
2041			rc = device_register(dev);
2042			if (rc) {
2043				AP_DBF_WARN("%s(%d,%d) device_register() failed\n",
2044					    __func__, ac->id, dom);
2045				goto put_dev_and_continue;
2046			}
2047			/* get it and thus adjust reference counter */
2048			get_device(dev);
2049			if (decfg) {
2050				AP_DBF_INFO("%s(%d,%d) new (decfg) queue dev created\n",
2051					    __func__, ac->id, dom);
2052			} else if (chkstop) {
2053				AP_DBF_INFO("%s(%d,%d) new (chkstop) queue dev created\n",
2054					    __func__, ac->id, dom);
2055			} else {
2056				/* nudge the queue's state machine */
2057				ap_queue_init_state(aq);
2058				AP_DBF_INFO("%s(%d,%d) new queue dev created\n",
2059					    __func__, ac->id, dom);
2060			}
2061			goto put_dev_and_continue;
2062		}
2063		/* handle state changes on already existing queue device */
2064		spin_lock_bh(&aq->lock);
2065		/* SE bind state */
2066		aq->se_bstate = hwinfo.bs;
2067		/* checkstop state */
2068		if (chkstop && !aq->chkstop) {
2069			/* checkstop on */
2070			aq->chkstop = true;
2071			if (aq->dev_state > AP_DEV_STATE_UNINITIATED) {
2072				aq->dev_state = AP_DEV_STATE_ERROR;
2073				aq->last_err_rc = AP_RESPONSE_CHECKSTOPPED;
2074			}
2075			spin_unlock_bh(&aq->lock);
2076			pr_debug("(%d,%d) queue dev checkstop on\n",
2077				 ac->id, dom);
2078			/* 'receive' pending messages with -EAGAIN */
2079			ap_flush_queue(aq);
2080			goto put_dev_and_continue;
2081		} else if (!chkstop && aq->chkstop) {
2082			/* checkstop off */
2083			aq->chkstop = false;
2084			if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
2085				_ap_queue_init_state(aq);
2086			spin_unlock_bh(&aq->lock);
2087			pr_debug("(%d,%d) queue dev checkstop off\n",
2088				 ac->id, dom);
2089			goto put_dev_and_continue;
2090		}
2091		/* config state change */
2092		if (decfg && aq->config) {
2093			/* config off this queue device */
2094			aq->config = false;
2095			if (aq->dev_state > AP_DEV_STATE_UNINITIATED) {
2096				aq->dev_state = AP_DEV_STATE_ERROR;
2097				aq->last_err_rc = AP_RESPONSE_DECONFIGURED;
2098			}
2099			spin_unlock_bh(&aq->lock);
2100			pr_debug("(%d,%d) queue dev config off\n",
2101				 ac->id, dom);
2102			ap_send_config_uevent(&aq->ap_dev, aq->config);
2103			/* 'receive' pending messages with -EAGAIN */
2104			ap_flush_queue(aq);
2105			goto put_dev_and_continue;
2106		} else if (!decfg && !aq->config) {
2107			/* config on this queue device */
2108			aq->config = true;
2109			if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
2110				_ap_queue_init_state(aq);
2111			spin_unlock_bh(&aq->lock);
2112			pr_debug("(%d,%d) queue dev config on\n",
2113				 ac->id, dom);
2114			ap_send_config_uevent(&aq->ap_dev, aq->config);
2115			goto put_dev_and_continue;
2116		}
2117		/* handle other error states */
2118		if (!decfg && aq->dev_state == AP_DEV_STATE_ERROR) {
2119			spin_unlock_bh(&aq->lock);
2120			/* 'receive' pending messages with -EAGAIN */
2121			ap_flush_queue(aq);
2122			/* re-init (with reset) the queue device */
2123			ap_queue_init_state(aq);
2124			AP_DBF_INFO("%s(%d,%d) queue dev reinit enforced\n",
2125				    __func__, ac->id, dom);
2126			goto put_dev_and_continue;
2127		}
2128		spin_unlock_bh(&aq->lock);
2129put_dev_and_continue:
2130		put_device(dev);
2131	}
2132}
2133
2134/*
2135 * Helper function for ap_scan_bus().
2136 * Does the scan bus job for the given adapter id.
2137 */
2138static inline void ap_scan_adapter(int ap)
2139{
2140	struct ap_tapq_hwinfo hwinfo;
2141	int rc, dom, comp_type;
2142	bool decfg, chkstop;
2143	struct ap_card *ac;
2144	struct device *dev;
2145	ap_qid_t qid;
2146
2147	/* Is there currently a card device for this adapter ? */
2148	dev = bus_find_device(&ap_bus_type, NULL,
2149			      (void *)(long)ap,
2150			      __match_card_device_with_id);
2151	ac = dev ? to_ap_card(dev) : NULL;
2152
2153	/* Adapter not in configuration ? */
2154	if (!ap_test_config_card_id(ap)) {
2155		if (ac) {
2156			AP_DBF_INFO("%s(%d) ap not in config any more, rm card and queue devs\n",
2157				    __func__, ap);
2158			ap_scan_rm_card_dev_and_queue_devs(ac);
2159			put_device(dev);
2160		}
2161		return;
2162	}
2163
2164	/*
2165	 * Adapter ap is valid in the current configuration. So do some checks:
2166	 * If no card device exists, build one. If a card device exists, check
2167	 * for type and functions changed. For all this we need to find a valid
2168	 * APQN first.
2169	 */
2170
2171	for (dom = 0; dom <= ap_max_domain_id; dom++)
2172		if (ap_test_config_usage_domain(dom)) {
2173			qid = AP_MKQID(ap, dom);
2174			if (ap_queue_info(qid, &hwinfo, &decfg, &chkstop) > 0)
2175				break;
2176		}
2177	if (dom > ap_max_domain_id) {
2178		/* Could not find one valid APQN for this adapter */
2179		if (ac) {
2180			AP_DBF_INFO("%s(%d) no type info (no APQN found), rm card and queue devs\n",
2181				    __func__, ap);
2182			ap_scan_rm_card_dev_and_queue_devs(ac);
2183			put_device(dev);
2184		} else {
2185			pr_debug("(%d) no type info (no APQN found), ignored\n",
2186				 ap);
2187		}
2188		return;
2189	}
2190	if (!hwinfo.at) {
2191		/* No apdater type info available, an unusable adapter */
2192		if (ac) {
2193			AP_DBF_INFO("%s(%d) no valid type (0) info, rm card and queue devs\n",
2194				    __func__, ap);
2195			ap_scan_rm_card_dev_and_queue_devs(ac);
2196			put_device(dev);
2197		} else {
2198			pr_debug("(%d) no valid type (0) info, ignored\n", ap);
2199		}
2200		return;
2201	}
2202	hwinfo.value &= TAPQ_CARD_HWINFO_MASK; /* filter card specific hwinfo */
2203	if (ac) {
2204		/* Check APQN against existing card device for changes */
2205		if (ac->hwinfo.at != hwinfo.at) {
2206			AP_DBF_INFO("%s(%d) hwtype %d changed, rm card and queue devs\n",
2207				    __func__, ap, hwinfo.at);
2208			ap_scan_rm_card_dev_and_queue_devs(ac);
2209			put_device(dev);
2210			ac = NULL;
2211		} else if (ac->hwinfo.fac != hwinfo.fac) {
2212			AP_DBF_INFO("%s(%d) functions 0x%08x changed, rm card and queue devs\n",
2213				    __func__, ap, hwinfo.fac);
2214			ap_scan_rm_card_dev_and_queue_devs(ac);
2215			put_device(dev);
2216			ac = NULL;
2217		} else {
2218			/* handle checkstop state change */
2219			if (chkstop && !ac->chkstop) {
2220				/* checkstop on */
2221				ac->chkstop = true;
2222				AP_DBF_INFO("%s(%d) card dev checkstop on\n",
2223					    __func__, ap);
2224			} else if (!chkstop && ac->chkstop) {
2225				/* checkstop off */
2226				ac->chkstop = false;
2227				AP_DBF_INFO("%s(%d) card dev checkstop off\n",
2228					    __func__, ap);
2229			}
2230			/* handle config state change */
2231			if (decfg && ac->config) {
2232				ac->config = false;
2233				AP_DBF_INFO("%s(%d) card dev config off\n",
2234					    __func__, ap);
2235				ap_send_config_uevent(&ac->ap_dev, ac->config);
2236			} else if (!decfg && !ac->config) {
2237				ac->config = true;
2238				AP_DBF_INFO("%s(%d) card dev config on\n",
2239					    __func__, ap);
2240				ap_send_config_uevent(&ac->ap_dev, ac->config);
2241			}
2242		}
2243	}
2244
2245	if (!ac) {
2246		/* Build a new card device */
2247		comp_type = ap_get_compatible_type(qid, hwinfo.at, hwinfo.fac);
2248		if (!comp_type) {
2249			AP_DBF_WARN("%s(%d) type %d, can't get compatibility type\n",
2250				    __func__, ap, hwinfo.at);
2251			return;
2252		}
2253		ac = ap_card_create(ap, hwinfo, comp_type);
2254		if (!ac) {
2255			AP_DBF_WARN("%s(%d) ap_card_create() failed\n",
2256				    __func__, ap);
2257			return;
2258		}
2259		ac->config = !decfg;
2260		ac->chkstop = chkstop;
2261		dev = &ac->ap_dev.device;
2262		dev->bus = &ap_bus_type;
2263		dev->parent = ap_root_device;
2264		dev_set_name(dev, "card%02x", ap);
2265		/* maybe enlarge ap_max_msg_size to support this card */
2266		if (ac->maxmsgsize > atomic_read(&ap_max_msg_size)) {
2267			atomic_set(&ap_max_msg_size, ac->maxmsgsize);
2268			AP_DBF_INFO("%s(%d) ap_max_msg_size update to %d byte\n",
2269				    __func__, ap,
2270				    atomic_read(&ap_max_msg_size));
2271		}
2272		/* Register the new card device with AP bus */
2273		rc = device_register(dev);
2274		if (rc) {
2275			AP_DBF_WARN("%s(%d) device_register() failed\n",
2276				    __func__, ap);
2277			put_device(dev);
2278			return;
2279		}
2280		/* get it and thus adjust reference counter */
2281		get_device(dev);
2282		if (decfg)
2283			AP_DBF_INFO("%s(%d) new (decfg) card dev type=%d func=0x%08x created\n",
2284				    __func__, ap, hwinfo.at, hwinfo.fac);
2285		else if (chkstop)
2286			AP_DBF_INFO("%s(%d) new (chkstop) card dev type=%d func=0x%08x created\n",
2287				    __func__, ap, hwinfo.at, hwinfo.fac);
2288		else
2289			AP_DBF_INFO("%s(%d) new card dev type=%d func=0x%08x created\n",
2290				    __func__, ap, hwinfo.at, hwinfo.fac);
2291	}
2292
2293	/* Verify the domains and the queue devices for this card */
2294	ap_scan_domains(ac);
2295
2296	/* release the card device */
2297	put_device(&ac->ap_dev.device);
2298}
2299
2300/**
2301 * ap_get_configuration - get the host AP configuration
2302 *
2303 * Stores the host AP configuration information returned from the previous call
2304 * to Query Configuration Information (QCI), then retrieves and stores the
2305 * current AP configuration returned from QCI.
2306 *
2307 * Return: true if the host AP configuration changed between calls to QCI;
2308 * otherwise, return false.
2309 */
2310static bool ap_get_configuration(void)
2311{
2312	if (!ap_qci_info->flags)	/* QCI not supported */
2313		return false;
2314
2315	memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info));
2316	ap_qci(ap_qci_info);
2317
2318	return memcmp(ap_qci_info, ap_qci_info_old,
2319		      sizeof(struct ap_config_info)) != 0;
2320}
2321
2322/*
2323 * ap_config_has_new_aps - Check current against old qci info if
2324 * new adapters have appeared. Returns true if at least one new
2325 * adapter in the apm mask is showing up. Existing adapters or
2326 * receding adapters are not counted.
2327 */
2328static bool ap_config_has_new_aps(void)
2329{
2330
2331	unsigned long m[BITS_TO_LONGS(AP_DEVICES)];
2332
2333	if (!ap_qci_info->flags)
2334		return false;
2335
2336	bitmap_andnot(m, (unsigned long *)ap_qci_info->apm,
2337		      (unsigned long *)ap_qci_info_old->apm, AP_DEVICES);
2338	if (!bitmap_empty(m, AP_DEVICES))
2339		return true;
2340
2341	return false;
2342}
2343
2344/*
2345 * ap_config_has_new_doms - Check current against old qci info if
2346 * new (usage) domains have appeared. Returns true if at least one
2347 * new domain in the aqm mask is showing up. Existing domains or
2348 * receding domains are not counted.
2349 */
2350static bool ap_config_has_new_doms(void)
2351{
2352	unsigned long m[BITS_TO_LONGS(AP_DOMAINS)];
2353
2354	if (!ap_qci_info->flags)
2355		return false;
2356
2357	bitmap_andnot(m, (unsigned long *)ap_qci_info->aqm,
2358		      (unsigned long *)ap_qci_info_old->aqm, AP_DOMAINS);
2359	if (!bitmap_empty(m, AP_DOMAINS))
2360		return true;
2361
2362	return false;
2363}
2364
2365/**
2366 * ap_scan_bus(): Scan the AP bus for new devices
2367 * Always run under mutex ap_scan_bus_mutex protection
2368 * which needs to get locked/unlocked by the caller!
2369 * Returns true if any config change has been detected
2370 * during the scan, otherwise false.
2371 */
2372static bool ap_scan_bus(void)
2373{
2374	bool config_changed;
2375	int ap;
2376
2377	pr_debug(">\n");
2378
2379	/* (re-)fetch configuration via QCI */
2380	config_changed = ap_get_configuration();
2381	if (config_changed) {
2382		if (ap_config_has_new_aps() || ap_config_has_new_doms()) {
2383			/*
2384			 * Appearance of new adapters and/or domains need to
2385			 * build new ap devices which need to get bound to an
2386			 * device driver. Thus reset the APQN bindings complete
2387			 * completion.
2388			 */
2389			reinit_completion(&ap_apqn_bindings_complete);
2390		}
2391		/* post a config change notify */
2392		notify_config_changed();
2393	}
2394	ap_select_domain();
2395
2396	/* loop over all possible adapters */
2397	for (ap = 0; ap <= ap_max_adapter_id; ap++)
2398		ap_scan_adapter(ap);
2399
2400	/* scan complete notify */
2401	if (config_changed)
2402		notify_scan_complete();
2403
2404	/* check if there is at least one queue available with default domain */
2405	if (ap_domain_index >= 0) {
2406		struct device *dev =
2407			bus_find_device(&ap_bus_type, NULL,
2408					(void *)(long)ap_domain_index,
2409					__match_queue_device_with_queue_id);
2410		if (dev)
2411			put_device(dev);
2412		else
2413			AP_DBF_INFO("%s no queue device with default domain %d available\n",
2414				    __func__, ap_domain_index);
2415	}
2416
2417	if (atomic64_inc_return(&ap_scan_bus_count) == 1) {
2418		pr_debug("init scan complete\n");
2419		ap_send_init_scan_done_uevent();
2420	}
2421
2422	ap_check_bindings_complete();
2423
2424	mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ);
2425
2426	pr_debug("< config_changed=%d\n", config_changed);
2427
2428	return config_changed;
2429}
2430
2431/*
2432 * Callback for the ap_scan_bus_timer
2433 * Runs periodically, workqueue timer (ap_scan_bus_time)
2434 */
2435static void ap_scan_bus_timer_callback(struct timer_list *unused)
2436{
2437	/*
2438	 * schedule work into the system long wq which when
2439	 * the work is finally executed, calls the AP bus scan.
2440	 */
2441	queue_work(system_long_wq, &ap_scan_bus_work);
2442}
2443
2444/*
2445 * Callback for the ap_scan_bus_work
2446 */
2447static void ap_scan_bus_wq_callback(struct work_struct *unused)
2448{
2449	/*
2450	 * Try to invoke an ap_scan_bus(). If the mutex acquisition
2451	 * fails there is currently another task already running the
2452	 * AP scan bus and there is no need to wait and re-trigger the
2453	 * scan again. Please note at the end of the scan bus function
2454	 * the AP scan bus timer is re-armed which triggers then the
2455	 * ap_scan_bus_timer_callback which enqueues a work into the
2456	 * system_long_wq which invokes this function here again.
2457	 */
2458	if (mutex_trylock(&ap_scan_bus_mutex)) {
2459		ap_scan_bus_task = current;
2460		ap_scan_bus_result = ap_scan_bus();
2461		ap_scan_bus_task = NULL;
2462		mutex_unlock(&ap_scan_bus_mutex);
2463	}
2464}
2465
2466static inline void __exit ap_async_exit(void)
2467{
2468	if (ap_thread_flag)
2469		ap_poll_thread_stop();
2470	chsc_notifier_unregister(&ap_bus_nb);
2471	cancel_work(&ap_scan_bus_work);
2472	hrtimer_cancel(&ap_poll_timer);
2473	timer_delete(&ap_scan_bus_timer);
2474}
2475
2476static inline int __init ap_async_init(void)
2477{
2478	int rc;
2479
2480	/* Setup the AP bus rescan timer. */
2481	timer_setup(&ap_scan_bus_timer, ap_scan_bus_timer_callback, 0);
2482
2483	/*
2484	 * Setup the high resolution poll timer.
2485	 * If we are running under z/VM adjust polling to z/VM polling rate.
2486	 */
2487	if (machine_is_vm())
2488		poll_high_timeout = 1500000;
2489	hrtimer_setup(&ap_poll_timer, ap_poll_timeout, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2490
2491	queue_work(system_long_wq, &ap_scan_bus_work);
2492
2493	rc = chsc_notifier_register(&ap_bus_nb);
2494	if (rc)
2495		goto out;
2496
2497	/* Start the low priority AP bus poll thread. */
2498	if (!ap_thread_flag)
2499		return 0;
2500
2501	rc = ap_poll_thread_start();
2502	if (rc)
2503		goto out_notifier;
2504
2505	return 0;
2506
2507out_notifier:
2508	chsc_notifier_unregister(&ap_bus_nb);
2509out:
2510	cancel_work(&ap_scan_bus_work);
2511	hrtimer_cancel(&ap_poll_timer);
2512	timer_delete(&ap_scan_bus_timer);
2513	return rc;
2514}
2515
2516static inline void ap_irq_exit(void)
2517{
2518	if (ap_irq_flag)
2519		unregister_adapter_interrupt(&ap_airq);
2520}
2521
2522static inline int __init ap_irq_init(void)
2523{
2524	int rc;
2525
2526	if (!ap_interrupts_available() || !ap_useirq)
2527		return 0;
2528
2529	rc = register_adapter_interrupt(&ap_airq);
2530	ap_irq_flag = (rc == 0);
2531
2532	return rc;
2533}
2534
2535static inline void ap_debug_exit(void)
2536{
2537	debug_unregister(ap_dbf_info);
2538}
2539
2540static inline int __init ap_debug_init(void)
2541{
2542	ap_dbf_info = debug_register("ap", 2, 1,
2543				     AP_DBF_MAX_SPRINTF_ARGS * sizeof(long));
2544	debug_register_view(ap_dbf_info, &debug_sprintf_view);
2545	debug_set_level(ap_dbf_info, DBF_ERR);
2546
2547	return 0;
2548}
2549
2550static void __init ap_perms_init(void)
2551{
2552	/* all resources usable if no kernel parameter string given */
2553	memset(&ap_perms.ioctlm, 0xFF, sizeof(ap_perms.ioctlm));
2554	memset(&ap_perms.apm, 0xFF, sizeof(ap_perms.apm));
2555	memset(&ap_perms.aqm, 0xFF, sizeof(ap_perms.aqm));
2556
2557	/* apm kernel parameter string */
2558	if (apm_str) {
2559		memset(&ap_perms.apm, 0, sizeof(ap_perms.apm));
2560		ap_parse_mask_str(apm_str, ap_perms.apm, AP_DEVICES,
2561				  &ap_attr_mutex);
2562	}
2563
2564	/* aqm kernel parameter string */
2565	if (aqm_str) {
2566		memset(&ap_perms.aqm, 0, sizeof(ap_perms.aqm));
2567		ap_parse_mask_str(aqm_str, ap_perms.aqm, AP_DOMAINS,
2568				  &ap_attr_mutex);
2569	}
2570}
2571
2572/**
2573 * ap_module_init(): The module initialization code.
2574 *
2575 * Initializes the module.
2576 */
2577static int __init ap_module_init(void)
2578{
2579	int rc;
2580
2581	if (!ap_instructions_available()) {
2582		pr_warn("The hardware system does not support AP instructions\n");
2583		return -ENODEV;
2584	}
2585
2586	rc = ap_debug_init();
2587	if (rc)
2588		return rc;
2589
2590	/* init ap_queue hashtable */
2591	hash_init(ap_queues);
2592
2593	/* create ap msg buffer memory pool */
2594	ap_msg_pool = mempool_create_kmalloc_pool(ap_msg_pool_min_items,
2595						  AP_DEFAULT_MAX_MSG_SIZE);
2596	if (!ap_msg_pool) {
2597		rc = -ENOMEM;
2598		goto out;
2599	}
2600
2601	/* set up the AP permissions (ioctls, ap and aq masks) */
2602	ap_perms_init();
2603
2604	/* Get AP configuration data if available */
2605	ap_init_qci_info();
2606
2607	/* check default domain setting */
2608	if (ap_domain_index < -1 || ap_domain_index > ap_max_domain_id ||
2609	    (ap_domain_index >= 0 &&
2610	     !test_bit_inv(ap_domain_index, ap_perms.aqm))) {
2611		pr_warn("%d is not a valid cryptographic domain\n",
2612			ap_domain_index);
2613		ap_domain_index = -1;
2614	}
2615
2616	/* Create /sys/bus/ap. */
2617	rc = bus_register(&ap_bus_type);
2618	if (rc)
2619		goto out;
2620
2621	/* Create /sys/devices/ap. */
2622	ap_root_device = root_device_register("ap");
2623	rc = PTR_ERR_OR_ZERO(ap_root_device);
2624	if (rc)
2625		goto out_bus;
2626	ap_root_device->bus = &ap_bus_type;
2627
2628	/* enable interrupts if available */
2629	rc = ap_irq_init();
2630	if (rc)
2631		goto out_device;
2632
2633	/* Setup asynchronous work (timers, workqueue, etc). */
2634	rc = ap_async_init();
2635	if (rc)
2636		goto out_irq;
2637
2638	return 0;
2639
2640out_irq:
2641	ap_irq_exit();
2642out_device:
2643	root_device_unregister(ap_root_device);
2644out_bus:
2645	bus_unregister(&ap_bus_type);
2646out:
2647	mempool_destroy(ap_msg_pool);
2648	ap_debug_exit();
2649	return rc;
2650}
2651
2652static void __exit ap_module_exit(void)
2653{
2654	ap_async_exit();
2655	ap_irq_exit();
2656	root_device_unregister(ap_root_device);
2657	bus_unregister(&ap_bus_type);
2658	mempool_destroy(ap_msg_pool);
2659	ap_debug_exit();
2660}
2661
2662module_init(ap_module_init);
2663module_exit(ap_module_exit);