copy_user_64.S

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
  4 * Copyright 2002 Andi Kleen, SuSE Labs.
  5 *
  6 * Functions to copy from and to user space.
  7 */
  8
  9#include <linux/export.h>
 10#include <linux/linkage.h>
 11#include <linux/cfi_types.h>
 12#include <linux/objtool.h>
 13#include <asm/cpufeatures.h>
 14#include <asm/alternative.h>
 15#include <asm/asm.h>
 16
 17/*
 18 * rep_movs_alternative - memory copy with exception handling.
 19 * This version is for CPUs that don't have FSRM (Fast Short Rep Movs)
 20 *
 21 * Input:
 22 * rdi destination
 23 * rsi source
 24 * rcx count
 25 *
 26 * Output:
 27 * rcx uncopied bytes or 0 if successful.
 28 *
 29 * NOTE! The calling convention is very intentionally the same as
 30 * for 'rep movs', so that we can rewrite the function call with
 31 * just a plain 'rep movs' on machines that have FSRM.  But to make
 32 * it simpler for us, we can clobber rsi/rdi and rax freely.
 33 */
 34SYM_FUNC_START(rep_movs_alternative)
 35	ANNOTATE_NOENDBR
 36	cmpq $64,%rcx
 37	jae .Llarge
 38
 39	cmp $8,%ecx
 40	jae .Lword
 41
 42	testl %ecx,%ecx
 43	je .Lexit
 44
 45.Lcopy_user_tail:
 460:	movb (%rsi),%al
 471:	movb %al,(%rdi)
 48	inc %rdi
 49	inc %rsi
 50	dec %rcx
 51	jne .Lcopy_user_tail
 52.Lexit:
 53	RET
 54
 55	_ASM_EXTABLE_UA( 0b, .Lexit)
 56	_ASM_EXTABLE_UA( 1b, .Lexit)
 57
 58	.p2align 4
 59.Lword:
 602:	movq (%rsi),%rax
 613:	movq %rax,(%rdi)
 62	addq $8,%rsi
 63	addq $8,%rdi
 64	sub $8,%ecx
 65	je .Lexit
 66	cmp $8,%ecx
 67	jae .Lword
 68	jmp .Lcopy_user_tail
 69
 70	_ASM_EXTABLE_UA( 2b, .Lcopy_user_tail)
 71	_ASM_EXTABLE_UA( 3b, .Lcopy_user_tail)
 72
 73.Llarge:
 740:	ALTERNATIVE "jmp .Llarge_movsq", "rep movsb", X86_FEATURE_ERMS
 751:	RET
 76
 77	_ASM_EXTABLE_UA( 0b, 1b)
 78
 79.Llarge_movsq:
 80	/* Do the first possibly unaligned word */
 810:	movq (%rsi),%rax
 821:	movq %rax,(%rdi)
 83
 84	_ASM_EXTABLE_UA( 0b, .Lcopy_user_tail)
 85	_ASM_EXTABLE_UA( 1b, .Lcopy_user_tail)
 86
 87	/* What would be the offset to the aligned destination? */
 88	leaq 8(%rdi),%rax
 89	andq $-8,%rax
 90	subq %rdi,%rax
 91
 92	/* .. and update pointers and count to match */
 93	addq %rax,%rdi
 94	addq %rax,%rsi
 95	subq %rax,%rcx
 96
 97	/* make %rcx contain the number of words, %rax the remainder */
 98	movq %rcx,%rax
 99	shrq $3,%rcx
100	andl $7,%eax
1010:	rep movsq
102	movl %eax,%ecx
103	testl %ecx,%ecx
104	jne .Lcopy_user_tail
105	RET
106
1071:	leaq (%rax,%rcx,8),%rcx
108	jmp .Lcopy_user_tail
109
110	_ASM_EXTABLE_UA( 0b, 1b)
111SYM_FUNC_END(rep_movs_alternative)
112EXPORT_SYMBOL(rep_movs_alternative)