priv.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * handling privileged instructions
   4 *
   5 * Copyright IBM Corp. 2008, 2020
   6 *
   7 *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8 *               Christian Borntraeger <borntraeger@de.ibm.com>
   9 */
  10
  11#include <linux/kvm.h>
  12#include <linux/gfp.h>
  13#include <linux/errno.h>
  14#include <linux/mm_types.h>
  15#include <linux/pgtable.h>
  16#include <linux/io.h>
  17#include <asm/asm-offsets.h>
  18#include <asm/facility.h>
  19#include <asm/current.h>
  20#include <asm/debug.h>
  21#include <asm/ebcdic.h>
  22#include <asm/sysinfo.h>
  23#include <asm/page-states.h>
  24#include <asm/gmap.h>
  25#include <asm/ptrace.h>
  26#include <asm/sclp.h>
  27#include <asm/ap.h>
  28#include "gaccess.h"
  29#include "kvm-s390.h"
  30#include "trace.h"
  31
  32static int handle_ri(struct kvm_vcpu *vcpu)
  33{
  34	vcpu->stat.instruction_ri++;
  35
  36	if (test_kvm_facility(vcpu->kvm, 64)) {
  37		VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)");
  38		vcpu->arch.sie_block->ecb3 |= ECB3_RI;
  39		kvm_s390_retry_instr(vcpu);
  40		return 0;
  41	} else
  42		return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
  43}
  44
  45int kvm_s390_handle_aa(struct kvm_vcpu *vcpu)
  46{
  47	if ((vcpu->arch.sie_block->ipa & 0xf) <= 4)
  48		return handle_ri(vcpu);
  49	else
  50		return -EOPNOTSUPP;
  51}
  52
  53static int handle_gs(struct kvm_vcpu *vcpu)
  54{
  55	vcpu->stat.instruction_gs++;
  56
  57	if (test_kvm_facility(vcpu->kvm, 133)) {
  58		VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)");
  59		preempt_disable();
  60		local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
  61		current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb;
  62		restore_gs_cb(current->thread.gs_cb);
  63		preempt_enable();
  64		vcpu->arch.sie_block->ecb |= ECB_GS;
  65		vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
  66		vcpu->arch.gs_enabled = 1;
  67		kvm_s390_retry_instr(vcpu);
  68		return 0;
  69	} else
  70		return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
  71}
  72
  73int kvm_s390_handle_e3(struct kvm_vcpu *vcpu)
  74{
  75	int code = vcpu->arch.sie_block->ipb & 0xff;
  76
  77	if (code == 0x49 || code == 0x4d)
  78		return handle_gs(vcpu);
  79	else
  80		return -EOPNOTSUPP;
  81}
  82/* Handle SCK (SET CLOCK) interception */
  83static int handle_set_clock(struct kvm_vcpu *vcpu)
  84{
  85	struct kvm_s390_vm_tod_clock gtod = { 0 };
  86	int rc;
  87	u8 ar;
  88	u64 op2;
  89
  90	vcpu->stat.instruction_sck++;
  91
  92	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
  93		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
  94
  95	op2 = kvm_s390_get_base_disp_s(vcpu, &ar);
  96	if (op2 & 7)	/* Operand must be on a doubleword boundary */
  97		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
  98	rc = read_guest(vcpu, op2, ar, &gtod.tod, sizeof(gtod.tod));
  99	if (rc)
 100		return kvm_s390_inject_prog_cond(vcpu, rc);
 101
 102	VCPU_EVENT(vcpu, 3, "SCK: setting guest TOD to 0x%llx", gtod.tod);
 103	/*
 104	 * To set the TOD clock the kvm lock must be taken, but the vcpu lock
 105	 * is already held in handle_set_clock. The usual lock order is the
 106	 * opposite.  As SCK is deprecated and should not be used in several
 107	 * cases, for example when the multiple epoch facility or TOD clock
 108	 * steering facility is installed (see Principles of Operation),  a
 109	 * slow path can be used.  If the lock can not be taken via try_lock,
 110	 * the instruction will be retried via -EAGAIN at a later point in
 111	 * time.
 112	 */
 113	if (!kvm_s390_try_set_tod_clock(vcpu->kvm, &gtod)) {
 114		kvm_s390_retry_instr(vcpu);
 115		return -EAGAIN;
 116	}
 117
 118	kvm_s390_set_psw_cc(vcpu, 0);
 119	return 0;
 120}
 121
 122static int handle_set_prefix(struct kvm_vcpu *vcpu)
 123{
 124	u64 operand2;
 125	u32 address;
 126	int rc;
 127	u8 ar;
 128
 129	vcpu->stat.instruction_spx++;
 130
 131	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 132		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 133
 134	operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
 135
 136	/* must be word boundary */
 137	if (operand2 & 3)
 138		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 139
 140	/* get the value */
 141	rc = read_guest(vcpu, operand2, ar, &address, sizeof(address));
 142	if (rc)
 143		return kvm_s390_inject_prog_cond(vcpu, rc);
 144
 145	address &= 0x7fffe000u;
 146
 147	/*
 148	 * Make sure the new value is valid memory. We only need to check the
 149	 * first page, since address is 8k aligned and memory pieces are always
 150	 * at least 1MB aligned and have at least a size of 1MB.
 151	 */
 152	if (!kvm_is_gpa_in_memslot(vcpu->kvm, address))
 153		return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 154
 155	kvm_s390_set_prefix(vcpu, address);
 156	trace_kvm_s390_handle_prefix(vcpu, 1, address);
 157	return 0;
 158}
 159
 160static int handle_store_prefix(struct kvm_vcpu *vcpu)
 161{
 162	u64 operand2;
 163	u32 address;
 164	int rc;
 165	u8 ar;
 166
 167	vcpu->stat.instruction_stpx++;
 168
 169	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 170		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 171
 172	operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
 173
 174	/* must be word boundary */
 175	if (operand2 & 3)
 176		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 177
 178	address = kvm_s390_get_prefix(vcpu);
 179
 180	/* get the value */
 181	rc = write_guest(vcpu, operand2, ar, &address, sizeof(address));
 182	if (rc)
 183		return kvm_s390_inject_prog_cond(vcpu, rc);
 184
 185	VCPU_EVENT(vcpu, 3, "STPX: storing prefix 0x%x into 0x%llx", address, operand2);
 186	trace_kvm_s390_handle_prefix(vcpu, 0, address);
 187	return 0;
 188}
 189
 190static int handle_store_cpu_address(struct kvm_vcpu *vcpu)
 191{
 192	u16 vcpu_id = vcpu->vcpu_id;
 193	u64 ga;
 194	int rc;
 195	u8 ar;
 196
 197	vcpu->stat.instruction_stap++;
 198
 199	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 200		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 201
 202	ga = kvm_s390_get_base_disp_s(vcpu, &ar);
 203
 204	if (ga & 1)
 205		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 206
 207	rc = write_guest(vcpu, ga, ar, &vcpu_id, sizeof(vcpu_id));
 208	if (rc)
 209		return kvm_s390_inject_prog_cond(vcpu, rc);
 210
 211	VCPU_EVENT(vcpu, 3, "STAP: storing cpu address (%u) to 0x%llx", vcpu_id, ga);
 212	trace_kvm_s390_handle_stap(vcpu, ga);
 213	return 0;
 214}
 215
 216int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
 217{
 218	int rc;
 219
 220	trace_kvm_s390_skey_related_inst(vcpu);
 221	/* Already enabled? */
 222	if (vcpu->arch.skey_enabled)
 223		return 0;
 224
 225	rc = s390_enable_skey();
 226	VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc);
 227	if (rc)
 228		return rc;
 229
 230	if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
 231		kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
 232	if (!vcpu->kvm->arch.use_skf)
 233		vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
 234	else
 235		vcpu->arch.sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | ICTL_RRBE);
 236	vcpu->arch.skey_enabled = true;
 237	return 0;
 238}
 239
 240static int try_handle_skey(struct kvm_vcpu *vcpu)
 241{
 242	int rc;
 243
 244	rc = kvm_s390_skey_check_enable(vcpu);
 245	if (rc)
 246		return rc;
 247	if (vcpu->kvm->arch.use_skf) {
 248		/* with storage-key facility, SIE interprets it for us */
 249		kvm_s390_retry_instr(vcpu);
 250		VCPU_EVENT(vcpu, 4, "%s", "retrying storage key operation");
 251		return -EAGAIN;
 252	}
 253	return 0;
 254}
 255
 256static int handle_iske(struct kvm_vcpu *vcpu)
 257{
 258	unsigned long gaddr, vmaddr;
 259	unsigned char key;
 260	int reg1, reg2;
 261	bool unlocked;
 262	int rc;
 263
 264	vcpu->stat.instruction_iske++;
 265
 266	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 267		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 268
 269	rc = try_handle_skey(vcpu);
 270	if (rc)
 271		return rc != -EAGAIN ? rc : 0;
 272
 273	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
 274
 275	gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
 276	gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
 277	gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
 278	vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr));
 279	if (kvm_is_error_hva(vmaddr))
 280		return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 281retry:
 282	unlocked = false;
 283	mmap_read_lock(current->mm);
 284	rc = get_guest_storage_key(current->mm, vmaddr, &key);
 285
 286	if (rc) {
 287		rc = fixup_user_fault(current->mm, vmaddr,
 288				      FAULT_FLAG_WRITE, &unlocked);
 289		if (!rc) {
 290			mmap_read_unlock(current->mm);
 291			goto retry;
 292		}
 293	}
 294	mmap_read_unlock(current->mm);
 295	if (rc == -EFAULT)
 296		return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 297	if (rc < 0)
 298		return rc;
 299	vcpu->run->s.regs.gprs[reg1] &= ~0xff;
 300	vcpu->run->s.regs.gprs[reg1] |= key;
 301	return 0;
 302}
 303
 304static int handle_rrbe(struct kvm_vcpu *vcpu)
 305{
 306	unsigned long vmaddr, gaddr;
 307	int reg1, reg2;
 308	bool unlocked;
 309	int rc;
 310
 311	vcpu->stat.instruction_rrbe++;
 312
 313	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 314		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 315
 316	rc = try_handle_skey(vcpu);
 317	if (rc)
 318		return rc != -EAGAIN ? rc : 0;
 319
 320	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
 321
 322	gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
 323	gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
 324	gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
 325	vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr));
 326	if (kvm_is_error_hva(vmaddr))
 327		return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 328retry:
 329	unlocked = false;
 330	mmap_read_lock(current->mm);
 331	rc = reset_guest_reference_bit(current->mm, vmaddr);
 332	if (rc < 0) {
 333		rc = fixup_user_fault(current->mm, vmaddr,
 334				      FAULT_FLAG_WRITE, &unlocked);
 335		if (!rc) {
 336			mmap_read_unlock(current->mm);
 337			goto retry;
 338		}
 339	}
 340	mmap_read_unlock(current->mm);
 341	if (rc == -EFAULT)
 342		return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 343	if (rc < 0)
 344		return rc;
 345	kvm_s390_set_psw_cc(vcpu, rc);
 346	return 0;
 347}
 348
 349#define SSKE_NQ 0x8
 350#define SSKE_MR 0x4
 351#define SSKE_MC 0x2
 352#define SSKE_MB 0x1
 353static int handle_sske(struct kvm_vcpu *vcpu)
 354{
 355	unsigned char m3 = vcpu->arch.sie_block->ipb >> 28;
 356	unsigned long start, end;
 357	unsigned char key, oldkey;
 358	int reg1, reg2;
 359	bool unlocked;
 360	int rc;
 361
 362	vcpu->stat.instruction_sske++;
 363
 364	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 365		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 366
 367	rc = try_handle_skey(vcpu);
 368	if (rc)
 369		return rc != -EAGAIN ? rc : 0;
 370
 371	if (!test_kvm_facility(vcpu->kvm, 8))
 372		m3 &= ~SSKE_MB;
 373	if (!test_kvm_facility(vcpu->kvm, 10))
 374		m3 &= ~(SSKE_MC | SSKE_MR);
 375	if (!test_kvm_facility(vcpu->kvm, 14))
 376		m3 &= ~SSKE_NQ;
 377
 378	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
 379
 380	key = vcpu->run->s.regs.gprs[reg1] & 0xfe;
 381	start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
 382	start = kvm_s390_logical_to_effective(vcpu, start);
 383	if (m3 & SSKE_MB) {
 384		/* start already designates an absolute address */
 385		end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1);
 386	} else {
 387		start = kvm_s390_real_to_abs(vcpu, start);
 388		end = start + PAGE_SIZE;
 389	}
 390
 391	while (start != end) {
 392		unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
 393		unlocked = false;
 394
 395		if (kvm_is_error_hva(vmaddr))
 396			return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 397
 398		mmap_read_lock(current->mm);
 399		rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey,
 400						m3 & SSKE_NQ, m3 & SSKE_MR,
 401						m3 & SSKE_MC);
 402
 403		if (rc < 0) {
 404			rc = fixup_user_fault(current->mm, vmaddr,
 405					      FAULT_FLAG_WRITE, &unlocked);
 406			rc = !rc ? -EAGAIN : rc;
 407		}
 408		mmap_read_unlock(current->mm);
 409		if (rc == -EFAULT)
 410			return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 411		if (rc == -EAGAIN)
 412			continue;
 413		if (rc < 0)
 414			return rc;
 415		start += PAGE_SIZE;
 416	}
 417
 418	if (m3 & (SSKE_MC | SSKE_MR)) {
 419		if (m3 & SSKE_MB) {
 420			/* skey in reg1 is unpredictable */
 421			kvm_s390_set_psw_cc(vcpu, 3);
 422		} else {
 423			kvm_s390_set_psw_cc(vcpu, rc);
 424			vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL;
 425			vcpu->run->s.regs.gprs[reg1] |= (u64) oldkey << 8;
 426		}
 427	}
 428	if (m3 & SSKE_MB) {
 429		if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT)
 430			vcpu->run->s.regs.gprs[reg2] &= ~PAGE_MASK;
 431		else
 432			vcpu->run->s.regs.gprs[reg2] &= ~0xfffff000UL;
 433		end = kvm_s390_logical_to_effective(vcpu, end);
 434		vcpu->run->s.regs.gprs[reg2] |= end;
 435	}
 436	return 0;
 437}
 438
 439static int handle_ipte_interlock(struct kvm_vcpu *vcpu)
 440{
 441	vcpu->stat.instruction_ipte_interlock++;
 442	if (psw_bits(vcpu->arch.sie_block->gpsw).pstate)
 443		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 444	wait_event(vcpu->kvm->arch.ipte_wq, !ipte_lock_held(vcpu->kvm));
 445	kvm_s390_retry_instr(vcpu);
 446	VCPU_EVENT(vcpu, 4, "%s", "retrying ipte interlock operation");
 447	return 0;
 448}
 449
 450static int handle_test_block(struct kvm_vcpu *vcpu)
 451{
 452	gpa_t addr;
 453	int reg2;
 454
 455	vcpu->stat.instruction_tb++;
 456
 457	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 458		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 459
 460	kvm_s390_get_regs_rre(vcpu, NULL, &reg2);
 461	addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
 462	addr = kvm_s390_logical_to_effective(vcpu, addr);
 463	if (kvm_s390_check_low_addr_prot_real(vcpu, addr))
 464		return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
 465	addr = kvm_s390_real_to_abs(vcpu, addr);
 466
 467	if (!kvm_is_gpa_in_memslot(vcpu->kvm, addr))
 468		return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 469	/*
 470	 * We don't expect errors on modern systems, and do not care
 471	 * about storage keys (yet), so let's just clear the page.
 472	 */
 473	if (kvm_clear_guest(vcpu->kvm, addr, PAGE_SIZE))
 474		return -EFAULT;
 475	kvm_s390_set_psw_cc(vcpu, 0);
 476	vcpu->run->s.regs.gprs[0] = 0;
 477	return 0;
 478}
 479
 480static int handle_tpi(struct kvm_vcpu *vcpu)
 481{
 482	struct kvm_s390_interrupt_info *inti;
 483	unsigned long len;
 484	u32 tpi_data[3];
 485	int rc;
 486	u64 addr;
 487	u8 ar;
 488
 489	vcpu->stat.instruction_tpi++;
 490
 491	addr = kvm_s390_get_base_disp_s(vcpu, &ar);
 492	if (addr & 3)
 493		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 494
 495	inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0);
 496	if (!inti) {
 497		kvm_s390_set_psw_cc(vcpu, 0);
 498		return 0;
 499	}
 500
 501	tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr;
 502	tpi_data[1] = inti->io.io_int_parm;
 503	tpi_data[2] = inti->io.io_int_word;
 504	if (addr) {
 505		/*
 506		 * Store the two-word I/O interruption code into the
 507		 * provided area.
 508		 */
 509		len = sizeof(tpi_data) - 4;
 510		rc = write_guest(vcpu, addr, ar, &tpi_data, len);
 511		if (rc) {
 512			rc = kvm_s390_inject_prog_cond(vcpu, rc);
 513			goto reinject_interrupt;
 514		}
 515	} else {
 516		/*
 517		 * Store the three-word I/O interruption code into
 518		 * the appropriate lowcore area.
 519		 */
 520		len = sizeof(tpi_data);
 521		if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) {
 522			/* failed writes to the low core are not recoverable */
 523			rc = -EFAULT;
 524			goto reinject_interrupt;
 525		}
 526	}
 527
 528	/* irq was successfully handed to the guest */
 529	kfree(inti);
 530	kvm_s390_set_psw_cc(vcpu, 1);
 531	return 0;
 532reinject_interrupt:
 533	/*
 534	 * If we encounter a problem storing the interruption code, the
 535	 * instruction is suppressed from the guest's view: reinject the
 536	 * interrupt.
 537	 */
 538	if (kvm_s390_reinject_io_int(vcpu->kvm, inti)) {
 539		kfree(inti);
 540		rc = -EFAULT;
 541	}
 542	/* don't set the cc, a pgm irq was injected or we drop to user space */
 543	return rc ? -EFAULT : 0;
 544}
 545
 546static int handle_tsch(struct kvm_vcpu *vcpu)
 547{
 548	struct kvm_s390_interrupt_info *inti = NULL;
 549	const u64 isc_mask = 0xffUL << 24; /* all iscs set */
 550
 551	vcpu->stat.instruction_tsch++;
 552
 553	/* a valid schid has at least one bit set */
 554	if (vcpu->run->s.regs.gprs[1])
 555		inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask,
 556					   vcpu->run->s.regs.gprs[1]);
 557
 558	/*
 559	 * Prepare exit to userspace.
 560	 * We indicate whether we dequeued a pending I/O interrupt
 561	 * so that userspace can re-inject it if the instruction gets
 562	 * a program check. While this may re-order the pending I/O
 563	 * interrupts, this is no problem since the priority is kept
 564	 * intact.
 565	 */
 566	vcpu->run->exit_reason = KVM_EXIT_S390_TSCH;
 567	vcpu->run->s390_tsch.dequeued = !!inti;
 568	if (inti) {
 569		vcpu->run->s390_tsch.subchannel_id = inti->io.subchannel_id;
 570		vcpu->run->s390_tsch.subchannel_nr = inti->io.subchannel_nr;
 571		vcpu->run->s390_tsch.io_int_parm = inti->io.io_int_parm;
 572		vcpu->run->s390_tsch.io_int_word = inti->io.io_int_word;
 573	}
 574	vcpu->run->s390_tsch.ipb = vcpu->arch.sie_block->ipb;
 575	kfree(inti);
 576	return -EREMOTE;
 577}
 578
 579static int handle_io_inst(struct kvm_vcpu *vcpu)
 580{
 581	VCPU_EVENT(vcpu, 4, "%s", "I/O instruction");
 582
 583	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 584		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 585
 586	if (vcpu->kvm->arch.css_support) {
 587		/*
 588		 * Most I/O instructions will be handled by userspace.
 589		 * Exceptions are tpi and the interrupt portion of tsch.
 590		 */
 591		if (vcpu->arch.sie_block->ipa == 0xb236)
 592			return handle_tpi(vcpu);
 593		if (vcpu->arch.sie_block->ipa == 0xb235)
 594			return handle_tsch(vcpu);
 595		/* Handle in userspace. */
 596		vcpu->stat.instruction_io_other++;
 597		return -EOPNOTSUPP;
 598	} else {
 599		/*
 600		 * Set condition code 3 to stop the guest from issuing channel
 601		 * I/O instructions.
 602		 */
 603		kvm_s390_set_psw_cc(vcpu, 3);
 604		return 0;
 605	}
 606}
 607
 608#if IS_ENABLED(CONFIG_VFIO_AP)
 609bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa)
 610{
 611	return kvm_is_gpa_in_memslot(kvm, gpa);
 612}
 613EXPORT_SYMBOL_FOR_MODULES(kvm_s390_is_gpa_in_memslot, "vfio_ap");
 614#endif
 615
 616/*
 617 * handle_pqap: Handling pqap interception
 618 * @vcpu: the vcpu having issue the pqap instruction
 619 *
 620 * We now support PQAP/AQIC instructions and we need to correctly
 621 * answer the guest even if no dedicated driver's hook is available.
 622 *
 623 * The intercepting code calls a dedicated callback for this instruction
 624 * if a driver did register one in the CRYPTO satellite of the
 625 * SIE block.
 626 *
 627 * If no callback is available, the queues are not available, return this
 628 * response code to the caller and set CC to 3.
 629 * Else return the response code returned by the callback.
 630 */
 631static int handle_pqap(struct kvm_vcpu *vcpu)
 632{
 633	struct ap_queue_status status = {};
 634	crypto_hook pqap_hook;
 635	unsigned long reg0;
 636	int ret;
 637	uint8_t fc;
 638
 639	/* Verify that the AP instruction are available */
 640	if (!ap_instructions_available())
 641		return -EOPNOTSUPP;
 642	/* Verify that the guest is allowed to use AP instructions */
 643	if (!(vcpu->arch.sie_block->eca & ECA_APIE))
 644		return -EOPNOTSUPP;
 645	/*
 646	 * The only possibly intercepted functions when AP instructions are
 647	 * available for the guest are AQIC and TAPQ with the t bit set
 648	 * since we do not set IC.3 (FIII) we currently will only intercept
 649	 * the AQIC function code.
 650	 * Note: running nested under z/VM can result in intercepts for other
 651	 * function codes, e.g. PQAP(QCI). We do not support this and bail out.
 652	 */
 653	reg0 = vcpu->run->s.regs.gprs[0];
 654	fc = (reg0 >> 24) & 0xff;
 655	if (fc != 0x03)
 656		return -EOPNOTSUPP;
 657
 658	/* PQAP instruction is allowed for guest kernel only */
 659	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 660		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 661
 662	/* Common PQAP instruction specification exceptions */
 663	/* bits 41-47 must all be zeros */
 664	if (reg0 & 0x007f0000UL)
 665		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 666	/* APFT not install and T bit set */
 667	if (!test_kvm_facility(vcpu->kvm, 15) && (reg0 & 0x00800000UL))
 668		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 669	/* APXA not installed and APID greater 64 or APQI greater 16 */
 670	if (!(vcpu->kvm->arch.crypto.crycbd & 0x02) && (reg0 & 0x0000c0f0UL))
 671		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 672
 673	/* AQIC function code specific exception */
 674	/* facility 65 not present for AQIC function code */
 675	if (!test_kvm_facility(vcpu->kvm, 65))
 676		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 677
 678	/*
 679	 * If the hook callback is registered, there will be a pointer to the
 680	 * hook function pointer in the kvm_s390_crypto structure. Lock the
 681	 * owner, retrieve the hook function pointer and call the hook.
 682	 */
 683	down_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
 684	if (vcpu->kvm->arch.crypto.pqap_hook) {
 685		pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook;
 686		ret = pqap_hook(vcpu);
 687		if (!ret) {
 688			if (vcpu->run->s.regs.gprs[1] & 0x00ff0000)
 689				kvm_s390_set_psw_cc(vcpu, 3);
 690			else
 691				kvm_s390_set_psw_cc(vcpu, 0);
 692		}
 693		up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
 694		return ret;
 695	}
 696	up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
 697	/*
 698	 * A vfio_driver must register a hook.
 699	 * No hook means no driver to enable the SIE CRYCB and no queues.
 700	 * We send this response to the guest.
 701	 */
 702	status.response_code = 0x01;
 703	memcpy(&vcpu->run->s.regs.gprs[1], &status, sizeof(status));
 704	kvm_s390_set_psw_cc(vcpu, 3);
 705	return 0;
 706}
 707
 708static int handle_stfl(struct kvm_vcpu *vcpu)
 709{
 710	int rc;
 711	unsigned int fac;
 712
 713	vcpu->stat.instruction_stfl++;
 714
 715	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 716		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 717
 718	/*
 719	 * We need to shift the lower 32 facility bits (bit 0-31) from a u64
 720	 * into a u32 memory representation. They will remain bits 0-31.
 721	 */
 722	fac = *vcpu->kvm->arch.model.fac_list >> 32;
 723	rc = write_guest_lc(vcpu, offsetof(struct lowcore, stfl_fac_list),
 724			    &fac, sizeof(fac));
 725	if (rc)
 726		return rc;
 727	VCPU_EVENT(vcpu, 3, "STFL: store facility list 0x%x", fac);
 728	trace_kvm_s390_handle_stfl(vcpu, fac);
 729	return 0;
 730}
 731
 732#define PSW_MASK_ADDR_MODE (PSW_MASK_EA | PSW_MASK_BA)
 733#define PSW_MASK_UNASSIGNED 0xb80800fe7fffffffUL
 734#define PSW_ADDR_24 0x0000000000ffffffUL
 735#define PSW_ADDR_31 0x000000007fffffffUL
 736
 737int is_valid_psw(psw_t *psw)
 738{
 739	if (psw->mask & PSW_MASK_UNASSIGNED)
 740		return 0;
 741	if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_BA) {
 742		if (psw->addr & ~PSW_ADDR_31)
 743			return 0;
 744	}
 745	if (!(psw->mask & PSW_MASK_ADDR_MODE) && (psw->addr & ~PSW_ADDR_24))
 746		return 0;
 747	if ((psw->mask & PSW_MASK_ADDR_MODE) ==  PSW_MASK_EA)
 748		return 0;
 749	if (psw->addr & 1)
 750		return 0;
 751	return 1;
 752}
 753
 754int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu)
 755{
 756	psw_t *gpsw = &vcpu->arch.sie_block->gpsw;
 757	psw32_t new_psw;
 758	u64 addr;
 759	int rc;
 760	u8 ar;
 761
 762	vcpu->stat.instruction_lpsw++;
 763
 764	if (gpsw->mask & PSW_MASK_PSTATE)
 765		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 766
 767	addr = kvm_s390_get_base_disp_s(vcpu, &ar);
 768	if (addr & 7)
 769		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 770
 771	rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
 772	if (rc)
 773		return kvm_s390_inject_prog_cond(vcpu, rc);
 774	if (!(new_psw.mask & PSW32_MASK_BASE))
 775		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 776	gpsw->mask = (new_psw.mask & ~PSW32_MASK_BASE) << 32;
 777	gpsw->mask |= new_psw.addr & PSW32_ADDR_AMODE;
 778	gpsw->addr = new_psw.addr & ~PSW32_ADDR_AMODE;
 779	if (!is_valid_psw(gpsw))
 780		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 781	return 0;
 782}
 783
 784static int handle_lpswe(struct kvm_vcpu *vcpu)
 785{
 786	psw_t new_psw;
 787	u64 addr;
 788	int rc;
 789	u8 ar;
 790
 791	vcpu->stat.instruction_lpswe++;
 792
 793	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 794		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 795
 796	addr = kvm_s390_get_base_disp_s(vcpu, &ar);
 797	if (addr & 7)
 798		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 799	rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
 800	if (rc)
 801		return kvm_s390_inject_prog_cond(vcpu, rc);
 802	vcpu->arch.sie_block->gpsw = new_psw;
 803	if (!is_valid_psw(&vcpu->arch.sie_block->gpsw))
 804		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 805	return 0;
 806}
 807
 808static int handle_lpswey(struct kvm_vcpu *vcpu)
 809{
 810	psw_t new_psw;
 811	u64 addr;
 812	int rc;
 813	u8 ar;
 814
 815	vcpu->stat.instruction_lpswey++;
 816
 817	if (!test_kvm_facility(vcpu->kvm, 193))
 818		return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
 819
 820	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 821		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 822
 823	addr = kvm_s390_get_base_disp_siy(vcpu, &ar);
 824	if (addr & 7)
 825		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 826
 827	rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
 828	if (rc)
 829		return kvm_s390_inject_prog_cond(vcpu, rc);
 830
 831	vcpu->arch.sie_block->gpsw = new_psw;
 832	if (!is_valid_psw(&vcpu->arch.sie_block->gpsw))
 833		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 834
 835	return 0;
 836}
 837
 838static int handle_stidp(struct kvm_vcpu *vcpu)
 839{
 840	u64 stidp_data = vcpu->kvm->arch.model.cpuid;
 841	u64 operand2;
 842	int rc;
 843	u8 ar;
 844
 845	vcpu->stat.instruction_stidp++;
 846
 847	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 848		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 849
 850	operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
 851
 852	if (operand2 & 7)
 853		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 854
 855	rc = write_guest(vcpu, operand2, ar, &stidp_data, sizeof(stidp_data));
 856	if (rc)
 857		return kvm_s390_inject_prog_cond(vcpu, rc);
 858
 859	VCPU_EVENT(vcpu, 3, "STIDP: store cpu id 0x%llx", stidp_data);
 860	return 0;
 861}
 862
 863static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem)
 864{
 865	int cpus = 0;
 866	int n;
 867
 868	cpus = atomic_read(&vcpu->kvm->online_vcpus);
 869
 870	/* deal with other level 3 hypervisors */
 871	if (stsi(mem, 3, 2, 2))
 872		mem->count = 0;
 873	if (mem->count < 8)
 874		mem->count++;
 875	for (n = mem->count - 1; n > 0 ; n--)
 876		memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0]));
 877
 878	memset(&mem->vm[0], 0, sizeof(mem->vm[0]));
 879	mem->vm[0].cpus_total = cpus;
 880	mem->vm[0].cpus_configured = cpus;
 881	mem->vm[0].cpus_standby = 0;
 882	mem->vm[0].cpus_reserved = 0;
 883	mem->vm[0].caf = 1000;
 884	memcpy(mem->vm[0].name, "KVMguest", 8);
 885	ASCEBC(mem->vm[0].name, 8);
 886	memcpy(mem->vm[0].cpi, "KVM/Linux       ", 16);
 887	ASCEBC(mem->vm[0].cpi, 16);
 888}
 889
 890static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, u8 ar,
 891				 u8 fc, u8 sel1, u16 sel2)
 892{
 893	vcpu->run->exit_reason = KVM_EXIT_S390_STSI;
 894	vcpu->run->s390_stsi.addr = addr;
 895	vcpu->run->s390_stsi.ar = ar;
 896	vcpu->run->s390_stsi.fc = fc;
 897	vcpu->run->s390_stsi.sel1 = sel1;
 898	vcpu->run->s390_stsi.sel2 = sel2;
 899}
 900
 901static int handle_stsi(struct kvm_vcpu *vcpu)
 902{
 903	int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28;
 904	int sel1 = vcpu->run->s.regs.gprs[0] & 0xff;
 905	int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff;
 906	unsigned long mem = 0;
 907	u64 operand2;
 908	int rc = 0;
 909	u8 ar;
 910
 911	vcpu->stat.instruction_stsi++;
 912	VCPU_EVENT(vcpu, 3, "STSI: fc: %u sel1: %u sel2: %u", fc, sel1, sel2);
 913
 914	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 915		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 916
 917	/* Bailout forbidden function codes */
 918	if (fc > 3 && fc != 15)
 919		goto out_no_data;
 920
 921	/*
 922	 * fc 15 is provided only with
 923	 *   - PTF/CPU topology support through facility 15
 924	 *   - KVM_CAP_S390_USER_STSI
 925	 */
 926	if (fc == 15 && (!test_kvm_facility(vcpu->kvm, 11) ||
 927			 !vcpu->kvm->arch.user_stsi))
 928		goto out_no_data;
 929
 930	if (vcpu->run->s.regs.gprs[0] & 0x0fffff00
 931	    || vcpu->run->s.regs.gprs[1] & 0xffff0000)
 932		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 933
 934	if (fc == 0) {
 935		vcpu->run->s.regs.gprs[0] = 3 << 28;
 936		kvm_s390_set_psw_cc(vcpu, 0);
 937		return 0;
 938	}
 939
 940	operand2 = kvm_s390_get_base_disp_s(vcpu, &ar);
 941
 942	if (!kvm_s390_pv_cpu_is_protected(vcpu) && (operand2 & 0xfff))
 943		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 944
 945	switch (fc) {
 946	case 1: /* same handling for 1 and 2 */
 947	case 2:
 948		mem = get_zeroed_page(GFP_KERNEL_ACCOUNT);
 949		if (!mem)
 950			goto out_no_data;
 951		if (stsi((void *) mem, fc, sel1, sel2))
 952			goto out_no_data;
 953		break;
 954	case 3:
 955		if (sel1 != 2 || sel2 != 2)
 956			goto out_no_data;
 957		mem = get_zeroed_page(GFP_KERNEL_ACCOUNT);
 958		if (!mem)
 959			goto out_no_data;
 960		handle_stsi_3_2_2(vcpu, (void *) mem);
 961		break;
 962	case 15: /* fc 15 is fully handled in userspace */
 963		insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2);
 964		trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2);
 965		return -EREMOTE;
 966	}
 967	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 968		memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE);
 969		rc = 0;
 970	} else {
 971		rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE);
 972	}
 973	if (rc) {
 974		rc = kvm_s390_inject_prog_cond(vcpu, rc);
 975		goto out;
 976	}
 977	if (vcpu->kvm->arch.user_stsi) {
 978		insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2);
 979		rc = -EREMOTE;
 980	}
 981	trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2);
 982	free_page(mem);
 983	kvm_s390_set_psw_cc(vcpu, 0);
 984	vcpu->run->s.regs.gprs[0] = 0;
 985	return rc;
 986out_no_data:
 987	kvm_s390_set_psw_cc(vcpu, 3);
 988out:
 989	free_page(mem);
 990	return rc;
 991}
 992
 993int kvm_s390_handle_b2(struct kvm_vcpu *vcpu)
 994{
 995	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
 996	case 0x02:
 997		return handle_stidp(vcpu);
 998	case 0x04:
 999		return handle_set_clock(vcpu);
1000	case 0x10:
1001		return handle_set_prefix(vcpu);
1002	case 0x11:
1003		return handle_store_prefix(vcpu);
1004	case 0x12:
1005		return handle_store_cpu_address(vcpu);
1006	case 0x14:
1007		return kvm_s390_handle_vsie(vcpu);
1008	case 0x21:
1009	case 0x50:
1010		return handle_ipte_interlock(vcpu);
1011	case 0x29:
1012		return handle_iske(vcpu);
1013	case 0x2a:
1014		return handle_rrbe(vcpu);
1015	case 0x2b:
1016		return handle_sske(vcpu);
1017	case 0x2c:
1018		return handle_test_block(vcpu);
1019	case 0x30:
1020	case 0x31:
1021	case 0x32:
1022	case 0x33:
1023	case 0x34:
1024	case 0x35:
1025	case 0x36:
1026	case 0x37:
1027	case 0x38:
1028	case 0x39:
1029	case 0x3a:
1030	case 0x3b:
1031	case 0x3c:
1032	case 0x5f:
1033	case 0x74:
1034	case 0x76:
1035		return handle_io_inst(vcpu);
1036	case 0x56:
1037		return handle_sthyi(vcpu);
1038	case 0x7d:
1039		return handle_stsi(vcpu);
1040	case 0xaf:
1041		return handle_pqap(vcpu);
1042	case 0xb1:
1043		return handle_stfl(vcpu);
1044	case 0xb2:
1045		return handle_lpswe(vcpu);
1046	default:
1047		return -EOPNOTSUPP;
1048	}
1049}
1050
1051static int handle_epsw(struct kvm_vcpu *vcpu)
1052{
1053	int reg1, reg2;
1054
1055	vcpu->stat.instruction_epsw++;
1056
1057	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
1058
1059	/* This basically extracts the mask half of the psw. */
1060	vcpu->run->s.regs.gprs[reg1] &= 0xffffffff00000000UL;
1061	vcpu->run->s.regs.gprs[reg1] |= vcpu->arch.sie_block->gpsw.mask >> 32;
1062	if (reg2) {
1063		vcpu->run->s.regs.gprs[reg2] &= 0xffffffff00000000UL;
1064		vcpu->run->s.regs.gprs[reg2] |=
1065			vcpu->arch.sie_block->gpsw.mask & 0x00000000ffffffffUL;
1066	}
1067	return 0;
1068}
1069
1070#define PFMF_RESERVED   0xfffc0101UL
1071#define PFMF_SK         0x00020000UL
1072#define PFMF_CF         0x00010000UL
1073#define PFMF_UI         0x00008000UL
1074#define PFMF_FSC        0x00007000UL
1075#define PFMF_NQ         0x00000800UL
1076#define PFMF_MR         0x00000400UL
1077#define PFMF_MC         0x00000200UL
1078#define PFMF_KEY        0x000000feUL
1079
1080static int handle_pfmf(struct kvm_vcpu *vcpu)
1081{
1082	bool mr = false, mc = false, nq;
1083	int reg1, reg2;
1084	unsigned long start, end;
1085	unsigned char key;
1086
1087	vcpu->stat.instruction_pfmf++;
1088
1089	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
1090
1091	if (!test_kvm_facility(vcpu->kvm, 8))
1092		return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
1093
1094	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1095		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1096
1097	if (vcpu->run->s.regs.gprs[reg1] & PFMF_RESERVED)
1098		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1099
1100	/* Only provide non-quiescing support if enabled for the guest */
1101	if (vcpu->run->s.regs.gprs[reg1] & PFMF_NQ &&
1102	    !test_kvm_facility(vcpu->kvm, 14))
1103		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1104
1105	/* Only provide conditional-SSKE support if enabled for the guest */
1106	if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK &&
1107	    test_kvm_facility(vcpu->kvm, 10)) {
1108		mr = vcpu->run->s.regs.gprs[reg1] & PFMF_MR;
1109		mc = vcpu->run->s.regs.gprs[reg1] & PFMF_MC;
1110	}
1111
1112	nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ;
1113	key = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY;
1114	start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
1115	start = kvm_s390_logical_to_effective(vcpu, start);
1116
1117	if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
1118		if (kvm_s390_check_low_addr_prot_real(vcpu, start))
1119			return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
1120	}
1121
1122	switch (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) {
1123	case 0x00000000:
1124		/* only 4k frames specify a real address */
1125		start = kvm_s390_real_to_abs(vcpu, start);
1126		end = (start + PAGE_SIZE) & ~(PAGE_SIZE - 1);
1127		break;
1128	case 0x00001000:
1129		end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1);
1130		break;
1131	case 0x00002000:
1132		/* only support 2G frame size if EDAT2 is available and we are
1133		   not in 24-bit addressing mode */
1134		if (!test_kvm_facility(vcpu->kvm, 78) ||
1135		    psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_24BIT)
1136			return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1137		end = (start + _REGION3_SIZE) & ~(_REGION3_SIZE - 1);
1138		break;
1139	default:
1140		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1141	}
1142
1143	while (start != end) {
1144		unsigned long vmaddr;
1145		bool unlocked = false;
1146
1147		/* Translate guest address to host address */
1148		vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
1149		if (kvm_is_error_hva(vmaddr))
1150			return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1151
1152		if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
1153			if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE))
1154				return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1155		}
1156
1157		if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK) {
1158			int rc = kvm_s390_skey_check_enable(vcpu);
1159
1160			if (rc)
1161				return rc;
1162			mmap_read_lock(current->mm);
1163			rc = cond_set_guest_storage_key(current->mm, vmaddr,
1164							key, NULL, nq, mr, mc);
1165			if (rc < 0) {
1166				rc = fixup_user_fault(current->mm, vmaddr,
1167						      FAULT_FLAG_WRITE, &unlocked);
1168				rc = !rc ? -EAGAIN : rc;
1169			}
1170			mmap_read_unlock(current->mm);
1171			if (rc == -EFAULT)
1172				return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1173			if (rc == -EAGAIN)
1174				continue;
1175			if (rc < 0)
1176				return rc;
1177		}
1178		start += PAGE_SIZE;
1179	}
1180	if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) {
1181		if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) {
1182			vcpu->run->s.regs.gprs[reg2] = end;
1183		} else {
1184			vcpu->run->s.regs.gprs[reg2] &= ~0xffffffffUL;
1185			end = kvm_s390_logical_to_effective(vcpu, end);
1186			vcpu->run->s.regs.gprs[reg2] |= end;
1187		}
1188	}
1189	return 0;
1190}
1191
1192/*
1193 * Must be called with relevant read locks held (kvm->mm->mmap_lock, kvm->srcu)
1194 */
1195static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc)
1196{
1197	int r1, r2, nappended, entries;
1198	unsigned long gfn, hva, res, pgstev, ptev;
1199	unsigned long *cbrlo;
1200
1201	/*
1202	 * We don't need to set SD.FPF.SK to 1 here, because if we have a
1203	 * machine check here we either handle it or crash
1204	 */
1205
1206	kvm_s390_get_regs_rre(vcpu, &r1, &r2);
1207	gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT;
1208	hva = gfn_to_hva(vcpu->kvm, gfn);
1209	entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3;
1210
1211	if (kvm_is_error_hva(hva))
1212		return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1213
1214	nappended = pgste_perform_essa(vcpu->kvm->mm, hva, orc, &ptev, &pgstev);
1215	if (nappended < 0) {
1216		res = orc ? 0x10 : 0;
1217		vcpu->run->s.regs.gprs[r1] = res; /* Exception Indication */
1218		return 0;
1219	}
1220	res = (pgstev & _PGSTE_GPS_USAGE_MASK) >> 22;
1221	/*
1222	 * Set the block-content state part of the result. 0 means resident, so
1223	 * nothing to do if the page is valid. 2 is for preserved pages
1224	 * (non-present and non-zero), and 3 for zero pages (non-present and
1225	 * zero).
1226	 */
1227	if (ptev & _PAGE_INVALID) {
1228		res |= 2;
1229		if (pgstev & _PGSTE_GPS_ZERO)
1230			res |= 1;
1231	}
1232	if (pgstev & _PGSTE_GPS_NODAT)
1233		res |= 0x20;
1234	vcpu->run->s.regs.gprs[r1] = res;
1235	/*
1236	 * It is possible that all the normal 511 slots were full, in which case
1237	 * we will now write in the 512th slot, which is reserved for host use.
1238	 * In both cases we let the normal essa handling code process all the
1239	 * slots, including the reserved one, if needed.
1240	 */
1241	if (nappended > 0) {
1242		cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK);
1243		cbrlo[entries] = gfn << PAGE_SHIFT;
1244	}
1245
1246	if (orc) {
1247		struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn);
1248
1249		/* Increment only if we are really flipping the bit */
1250		if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
1251			atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages);
1252	}
1253
1254	return nappended;
1255}
1256
1257static int handle_essa(struct kvm_vcpu *vcpu)
1258{
1259	lockdep_assert_held(&vcpu->kvm->srcu);
1260
1261	/* entries expected to be 1FF */
1262	int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3;
1263	unsigned long *cbrlo;
1264	struct gmap *gmap;
1265	int i, orc;
1266
1267	VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries);
1268	gmap = vcpu->arch.gmap;
1269	vcpu->stat.instruction_essa++;
1270	if (!vcpu->kvm->arch.use_cmma)
1271		return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
1272
1273	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1274		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1275	/* Check for invalid operation request code */
1276	orc = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28;
1277	/* ORCs 0-6 are always valid */
1278	if (orc > (test_kvm_facility(vcpu->kvm, 147) ? ESSA_SET_STABLE_NODAT
1279						: ESSA_SET_STABLE_IF_RESIDENT))
1280		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1281
1282	if (!vcpu->kvm->arch.migration_mode) {
1283		/*
1284		 * CMMA is enabled in the KVM settings, but is disabled in
1285		 * the SIE block and in the mm_context, and we are not doing
1286		 * a migration. Enable CMMA in the mm_context.
1287		 * Since we need to take a write lock to write to the context
1288		 * to avoid races with storage keys handling, we check if the
1289		 * value really needs to be written to; if the value is
1290		 * already correct, we do nothing and avoid the lock.
1291		 */
1292		if (vcpu->kvm->mm->context.uses_cmm == 0) {
1293			mmap_write_lock(vcpu->kvm->mm);
1294			vcpu->kvm->mm->context.uses_cmm = 1;
1295			mmap_write_unlock(vcpu->kvm->mm);
1296		}
1297		/*
1298		 * If we are here, we are supposed to have CMMA enabled in
1299		 * the SIE block. Enabling CMMA works on a per-CPU basis,
1300		 * while the context use_cmma flag is per process.
1301		 * It's possible that the context flag is enabled and the
1302		 * SIE flag is not, so we set the flag always; if it was
1303		 * already set, nothing changes, otherwise we enable it
1304		 * on this CPU too.
1305		 */
1306		vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
1307		/* Retry the ESSA instruction */
1308		kvm_s390_retry_instr(vcpu);
1309	} else {
1310		mmap_read_lock(vcpu->kvm->mm);
1311		i = __do_essa(vcpu, orc);
1312		mmap_read_unlock(vcpu->kvm->mm);
1313		if (i < 0)
1314			return i;
1315		/* Account for the possible extra cbrl entry */
1316		entries += i;
1317	}
1318	vcpu->arch.sie_block->cbrlo &= PAGE_MASK;	/* reset nceo */
1319	cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo);
1320	mmap_read_lock(gmap->mm);
1321	for (i = 0; i < entries; ++i)
1322		__gmap_zap(gmap, cbrlo[i]);
1323	mmap_read_unlock(gmap->mm);
1324	return 0;
1325}
1326
1327int kvm_s390_handle_b9(struct kvm_vcpu *vcpu)
1328{
1329	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
1330	case 0x8a:
1331	case 0x8e:
1332	case 0x8f:
1333		return handle_ipte_interlock(vcpu);
1334	case 0x8d:
1335		return handle_epsw(vcpu);
1336	case 0xab:
1337		return handle_essa(vcpu);
1338	case 0xaf:
1339		return handle_pfmf(vcpu);
1340	default:
1341		return -EOPNOTSUPP;
1342	}
1343}
1344
1345int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
1346{
1347	int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
1348	int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
1349	int reg, rc, nr_regs;
1350	u32 ctl_array[16];
1351	u64 ga;
1352	u8 ar;
1353
1354	vcpu->stat.instruction_lctl++;
1355
1356	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1357		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1358
1359	ga = kvm_s390_get_base_disp_rs(vcpu, &ar);
1360
1361	if (ga & 3)
1362		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1363
1364	VCPU_EVENT(vcpu, 4, "LCTL: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga);
1365	trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga);
1366
1367	nr_regs = ((reg3 - reg1) & 0xf) + 1;
1368	rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32));
1369	if (rc)
1370		return kvm_s390_inject_prog_cond(vcpu, rc);
1371	reg = reg1;
1372	nr_regs = 0;
1373	do {
1374		vcpu->arch.sie_block->gcr[reg] &= 0xffffffff00000000ul;
1375		vcpu->arch.sie_block->gcr[reg] |= ctl_array[nr_regs++];
1376		if (reg == reg3)
1377			break;
1378		reg = (reg + 1) % 16;
1379	} while (1);
1380	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1381	return 0;
1382}
1383
1384int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu)
1385{
1386	int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
1387	int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
1388	int reg, rc, nr_regs;
1389	u32 ctl_array[16];
1390	u64 ga;
1391	u8 ar;
1392
1393	vcpu->stat.instruction_stctl++;
1394
1395	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1396		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1397
1398	ga = kvm_s390_get_base_disp_rs(vcpu, &ar);
1399
1400	if (ga & 3)
1401		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1402
1403	VCPU_EVENT(vcpu, 4, "STCTL r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga);
1404	trace_kvm_s390_handle_stctl(vcpu, 0, reg1, reg3, ga);
1405
1406	reg = reg1;
1407	nr_regs = 0;
1408	do {
1409		ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg];
1410		if (reg == reg3)
1411			break;
1412		reg = (reg + 1) % 16;
1413	} while (1);
1414	rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32));
1415	return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
1416}
1417
1418static int handle_lctlg(struct kvm_vcpu *vcpu)
1419{
1420	int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
1421	int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
1422	int reg, rc, nr_regs;
1423	u64 ctl_array[16];
1424	u64 ga;
1425	u8 ar;
1426
1427	vcpu->stat.instruction_lctlg++;
1428
1429	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1430		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1431
1432	ga = kvm_s390_get_base_disp_rsy(vcpu, &ar);
1433
1434	if (ga & 7)
1435		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1436
1437	VCPU_EVENT(vcpu, 4, "LCTLG: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga);
1438	trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga);
1439
1440	nr_regs = ((reg3 - reg1) & 0xf) + 1;
1441	rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64));
1442	if (rc)
1443		return kvm_s390_inject_prog_cond(vcpu, rc);
1444	reg = reg1;
1445	nr_regs = 0;
1446	do {
1447		vcpu->arch.sie_block->gcr[reg] = ctl_array[nr_regs++];
1448		if (reg == reg3)
1449			break;
1450		reg = (reg + 1) % 16;
1451	} while (1);
1452	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1453	return 0;
1454}
1455
1456static int handle_stctg(struct kvm_vcpu *vcpu)
1457{
1458	int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
1459	int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
1460	int reg, rc, nr_regs;
1461	u64 ctl_array[16];
1462	u64 ga;
1463	u8 ar;
1464
1465	vcpu->stat.instruction_stctg++;
1466
1467	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1468		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1469
1470	ga = kvm_s390_get_base_disp_rsy(vcpu, &ar);
1471
1472	if (ga & 7)
1473		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1474
1475	VCPU_EVENT(vcpu, 4, "STCTG r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga);
1476	trace_kvm_s390_handle_stctl(vcpu, 1, reg1, reg3, ga);
1477
1478	reg = reg1;
1479	nr_regs = 0;
1480	do {
1481		ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg];
1482		if (reg == reg3)
1483			break;
1484		reg = (reg + 1) % 16;
1485	} while (1);
1486	rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64));
1487	return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
1488}
1489
1490int kvm_s390_handle_eb(struct kvm_vcpu *vcpu)
1491{
1492	switch (vcpu->arch.sie_block->ipb & 0x000000ff) {
1493	case 0x25:
1494		return handle_stctg(vcpu);
1495	case 0x2f:
1496		return handle_lctlg(vcpu);
1497	case 0x60:
1498	case 0x61:
1499	case 0x62:
1500		return handle_ri(vcpu);
1501	case 0x71:
1502		return handle_lpswey(vcpu);
1503	default:
1504		return -EOPNOTSUPP;
1505	}
1506}
1507
1508static int handle_tprot(struct kvm_vcpu *vcpu)
1509{
1510	u64 address, operand2;
1511	unsigned long gpa;
1512	u8 access_key;
1513	bool writable;
1514	int ret, cc;
1515	u8 ar;
1516
1517	vcpu->stat.instruction_tprot++;
1518
1519	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1520		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1521
1522	kvm_s390_get_base_disp_sse(vcpu, &address, &operand2, &ar, NULL);
1523	access_key = (operand2 & 0xf0) >> 4;
1524
1525	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT)
1526		ipte_lock(vcpu->kvm);
1527
1528	ret = guest_translate_address_with_key(vcpu, address, ar, &gpa,
1529					       GACC_STORE, access_key);
1530	if (ret == 0) {
1531		gfn_to_hva_prot(vcpu->kvm, gpa_to_gfn(gpa), &writable);
1532	} else if (ret == PGM_PROTECTION) {
1533		writable = false;
1534		/* Write protected? Try again with read-only... */
1535		ret = guest_translate_address_with_key(vcpu, address, ar, &gpa,
1536						       GACC_FETCH, access_key);
1537	}
1538	if (ret >= 0) {
1539		cc = -1;
1540
1541		/* Fetching permitted; storing permitted */
1542		if (ret == 0 && writable)
1543			cc = 0;
1544		/* Fetching permitted; storing not permitted */
1545		else if (ret == 0 && !writable)
1546			cc = 1;
1547		/* Fetching not permitted; storing not permitted */
1548		else if (ret == PGM_PROTECTION)
1549			cc = 2;
1550		/* Translation not available */
1551		else if (ret != PGM_ADDRESSING && ret != PGM_TRANSLATION_SPEC)
1552			cc = 3;
1553
1554		if (cc != -1) {
1555			kvm_s390_set_psw_cc(vcpu, cc);
1556			ret = 0;
1557		} else {
1558			ret = kvm_s390_inject_program_int(vcpu, ret);
1559		}
1560	}
1561
1562	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT)
1563		ipte_unlock(vcpu->kvm);
1564	return ret;
1565}
1566
1567int kvm_s390_handle_e5(struct kvm_vcpu *vcpu)
1568{
1569	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
1570	case 0x01:
1571		return handle_tprot(vcpu);
1572	default:
1573		return -EOPNOTSUPP;
1574	}
1575}
1576
1577static int handle_sckpf(struct kvm_vcpu *vcpu)
1578{
1579	u32 value;
1580
1581	vcpu->stat.instruction_sckpf++;
1582
1583	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1584		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1585
1586	if (vcpu->run->s.regs.gprs[0] & 0x00000000ffff0000)
1587		return kvm_s390_inject_program_int(vcpu,
1588						   PGM_SPECIFICATION);
1589
1590	value = vcpu->run->s.regs.gprs[0] & 0x000000000000ffff;
1591	vcpu->arch.sie_block->todpr = value;
1592
1593	return 0;
1594}
1595
1596static int handle_ptff(struct kvm_vcpu *vcpu)
1597{
1598	vcpu->stat.instruction_ptff++;
1599
1600	/* we don't emulate any control instructions yet */
1601	kvm_s390_set_psw_cc(vcpu, 3);
1602	return 0;
1603}
1604
1605int kvm_s390_handle_01(struct kvm_vcpu *vcpu)
1606{
1607	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
1608	case 0x04:
1609		return handle_ptff(vcpu);
1610	case 0x07:
1611		return handle_sckpf(vcpu);
1612	default:
1613		return -EOPNOTSUPP;
1614	}
1615}