string.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *  linux/lib/string.c
  4 *
  5 *  Copyright (C) 1991, 1992  Linus Torvalds
  6 */
  7
  8/*
  9 * This file should be used only for "library" routines that may have
 10 * alternative implementations on specific architectures (generally
 11 * found in <asm-xx/string.h>), or get overloaded by FORTIFY_SOURCE.
 12 * (Specifically, this file is built with __NO_FORTIFY.)
 13 *
 14 * Other helper functions should live in string_helpers.c.
 15 */
 16
 17#define __NO_FORTIFY
 18#include <linux/bits.h>
 19#include <linux/bug.h>
 20#include <linux/ctype.h>
 21#include <linux/errno.h>
 22#include <linux/limits.h>
 23#include <linux/linkage.h>
 24#include <linux/stddef.h>
 25#include <linux/string.h>
 26#include <linux/types.h>
 27
 28#include <asm/page.h>
 29#include <asm/rwonce.h>
 30#include <linux/unaligned.h>
 31#include <asm/word-at-a-time.h>
 32
 33#ifndef __HAVE_ARCH_STRNCASECMP
 34/**
 35 * strncasecmp - Case insensitive, length-limited string comparison
 36 * @s1: One string
 37 * @s2: The other string
 38 * @len: the maximum number of characters to compare
 39 */
 40int strncasecmp(const char *s1, const char *s2, size_t len)
 41{
 42	/* Yes, Virginia, it had better be unsigned */
 43	unsigned char c1, c2;
 44
 45	if (!len)
 46		return 0;
 47
 48	do {
 49		c1 = *s1++;
 50		c2 = *s2++;
 51		if (!c1 || !c2)
 52			break;
 53		if (c1 == c2)
 54			continue;
 55		c1 = tolower(c1);
 56		c2 = tolower(c2);
 57		if (c1 != c2)
 58			break;
 59	} while (--len);
 60	return (int)c1 - (int)c2;
 61}
 62EXPORT_SYMBOL(strncasecmp);
 63#endif
 64
 65#ifndef __HAVE_ARCH_STRCASECMP
 66int strcasecmp(const char *s1, const char *s2)
 67{
 68	int c1, c2;
 69
 70	do {
 71		c1 = tolower(*s1++);
 72		c2 = tolower(*s2++);
 73	} while (c1 == c2 && c1 != 0);
 74	return c1 - c2;
 75}
 76EXPORT_SYMBOL(strcasecmp);
 77#endif
 78
 79#ifndef __HAVE_ARCH_STRCPY
 80char *strcpy(char *dest, const char *src)
 81{
 82	char *tmp = dest;
 83
 84	while ((*dest++ = *src++) != '\0')
 85		/* nothing */;
 86	return tmp;
 87}
 88EXPORT_SYMBOL(strcpy);
 89#endif
 90
 91#ifndef __HAVE_ARCH_STRNCPY
 92char *strncpy(char *dest, const char *src, size_t count)
 93{
 94	char *tmp = dest;
 95
 96	while (count) {
 97		if ((*tmp = *src) != 0)
 98			src++;
 99		tmp++;
100		count--;
101	}
102	return dest;
103}
104EXPORT_SYMBOL(strncpy);
105#endif
106
107#ifdef __BIG_ENDIAN
108# define ALLBUTLAST_BYTE_MASK (~255ul)
109#else
110# define ALLBUTLAST_BYTE_MASK (~0ul >> 8)
111#endif
112
113ssize_t sized_strscpy(char *dest, const char *src, size_t count)
114{
115	const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
116	size_t max = count;
117	long res = 0;
118
119	if (count == 0 || WARN_ON_ONCE(count > INT_MAX))
120		return -E2BIG;
121
122#ifndef CONFIG_DCACHE_WORD_ACCESS
123#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
124	/*
125	 * If src is unaligned, don't cross a page boundary,
126	 * since we don't know if the next page is mapped.
127	 */
128	if ((long)src & (sizeof(long) - 1)) {
129		size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1));
130		if (limit < max)
131			max = limit;
132	}
133#else
134	/* If src or dest is unaligned, don't do word-at-a-time. */
135	if (((long) dest | (long) src) & (sizeof(long) - 1))
136		max = 0;
137#endif
138#endif
139
140	/*
141	 * load_unaligned_zeropad() or read_word_at_a_time() below may read
142	 * uninitialized bytes after the trailing zero and use them in
143	 * comparisons. Disable this optimization under KMSAN to prevent
144	 * false positive reports.
145	 */
146	if (IS_ENABLED(CONFIG_KMSAN))
147		max = 0;
148
149	while (max >= sizeof(unsigned long)) {
150		unsigned long c, data;
151
152#ifdef CONFIG_DCACHE_WORD_ACCESS
153		c = load_unaligned_zeropad(src+res);
154#else
155		c = read_word_at_a_time(src+res);
156#endif
157		if (has_zero(c, &data, &constants)) {
158			data = prep_zero_mask(c, data, &constants);
159			data = create_zero_mask(data);
160			*(unsigned long *)(dest+res) = c & zero_bytemask(data);
161			return res + find_zero(data);
162		}
163		count -= sizeof(unsigned long);
164		if (unlikely(!count)) {
165			c &= ALLBUTLAST_BYTE_MASK;
166			*(unsigned long *)(dest+res) = c;
167			return -E2BIG;
168		}
169		*(unsigned long *)(dest+res) = c;
170		res += sizeof(unsigned long);
171		max -= sizeof(unsigned long);
172	}
173
174	while (count > 1) {
175		char c;
176
177		c = src[res];
178		dest[res] = c;
179		if (!c)
180			return res;
181		res++;
182		count--;
183	}
184
185	/* Force NUL-termination. */
186	dest[res] = '\0';
187
188	/* Return E2BIG if the source didn't stop */
189	return src[res] ? -E2BIG : res;
190}
191EXPORT_SYMBOL(sized_strscpy);
192
193/**
194 * stpcpy - copy a string from src to dest returning a pointer to the new end
195 *          of dest, including src's %NUL-terminator. May overrun dest.
196 * @dest: pointer to end of string being copied into. Must be large enough
197 *        to receive copy.
198 * @src: pointer to the beginning of string being copied from. Must not overlap
199 *       dest.
200 *
201 * stpcpy differs from strcpy in a key way: the return value is a pointer
202 * to the new %NUL-terminating character in @dest. (For strcpy, the return
203 * value is a pointer to the start of @dest). This interface is considered
204 * unsafe as it doesn't perform bounds checking of the inputs. As such it's
205 * not recommended for usage. Instead, its definition is provided in case
206 * the compiler lowers other libcalls to stpcpy.
207 */
208char *stpcpy(char *__restrict__ dest, const char *__restrict__ src);
209char *stpcpy(char *__restrict__ dest, const char *__restrict__ src)
210{
211	while ((*dest++ = *src++) != '\0')
212		/* nothing */;
213	return --dest;
214}
215EXPORT_SYMBOL(stpcpy);
216
217#ifndef __HAVE_ARCH_STRCAT
218char *strcat(char *dest, const char *src)
219{
220	char *tmp = dest;
221
222	while (*dest)
223		dest++;
224	while ((*dest++ = *src++) != '\0')
225		;
226	return tmp;
227}
228EXPORT_SYMBOL(strcat);
229#endif
230
231#ifndef __HAVE_ARCH_STRNCAT
232char *strncat(char *dest, const char *src, size_t count)
233{
234	char *tmp = dest;
235
236	if (count) {
237		while (*dest)
238			dest++;
239		while ((*dest++ = *src++) != 0) {
240			if (--count == 0) {
241				*dest = '\0';
242				break;
243			}
244		}
245	}
246	return tmp;
247}
248EXPORT_SYMBOL(strncat);
249#endif
250
251#ifndef __HAVE_ARCH_STRLCAT
252size_t strlcat(char *dest, const char *src, size_t count)
253{
254	size_t dsize = strlen(dest);
255	size_t len = strlen(src);
256	size_t res = dsize + len;
257
258	/* This would be a bug */
259	BUG_ON(dsize >= count);
260
261	dest += dsize;
262	count -= dsize;
263	if (len >= count)
264		len = count-1;
265	__builtin_memcpy(dest, src, len);
266	dest[len] = 0;
267	return res;
268}
269EXPORT_SYMBOL(strlcat);
270#endif
271
272#ifndef __HAVE_ARCH_STRCMP
273/**
274 * strcmp - Compare two strings
275 * @cs: One string
276 * @ct: Another string
277 */
278int strcmp(const char *cs, const char *ct)
279{
280	unsigned char c1, c2;
281
282	while (1) {
283		c1 = *cs++;
284		c2 = *ct++;
285		if (c1 != c2)
286			return c1 < c2 ? -1 : 1;
287		if (!c1)
288			break;
289	}
290	return 0;
291}
292EXPORT_SYMBOL(strcmp);
293#endif
294
295#ifndef __HAVE_ARCH_STRNCMP
296/**
297 * strncmp - Compare two length-limited strings
298 * @cs: One string
299 * @ct: Another string
300 * @count: The maximum number of bytes to compare
301 */
302int strncmp(const char *cs, const char *ct, size_t count)
303{
304	unsigned char c1, c2;
305
306	while (count) {
307		c1 = *cs++;
308		c2 = *ct++;
309		if (c1 != c2)
310			return c1 < c2 ? -1 : 1;
311		if (!c1)
312			break;
313		count--;
314	}
315	return 0;
316}
317EXPORT_SYMBOL(strncmp);
318#endif
319
320#ifndef __HAVE_ARCH_STRCHR
321/**
322 * strchr - Find the first occurrence of a character in a string
323 * @s: The string to be searched
324 * @c: The character to search for
325 *
326 * Note that the %NUL-terminator is considered part of the string, and can
327 * be searched for.
328 */
329char *strchr(const char *s, int c)
330{
331	for (; *s != (char)c; ++s)
332		if (*s == '\0')
333			return NULL;
334	return (char *)s;
335}
336EXPORT_SYMBOL(strchr);
337#endif
338
339#ifndef __HAVE_ARCH_STRCHRNUL
340/**
341 * strchrnul - Find and return a character in a string, or end of string
342 * @s: The string to be searched
343 * @c: The character to search for
344 *
345 * Returns pointer to first occurrence of 'c' in s. If c is not found, then
346 * return a pointer to the null byte at the end of s.
347 */
348char *strchrnul(const char *s, int c)
349{
350	while (*s && *s != (char)c)
351		s++;
352	return (char *)s;
353}
354EXPORT_SYMBOL(strchrnul);
355#endif
356
357/**
358 * strnchrnul - Find and return a character in a length limited string,
359 * or end of string
360 * @s: The string to be searched
361 * @count: The number of characters to be searched
362 * @c: The character to search for
363 *
364 * Returns pointer to the first occurrence of 'c' in s. If c is not found,
365 * then return a pointer to the last character of the string.
366 */
367char *strnchrnul(const char *s, size_t count, int c)
368{
369	while (count-- && *s && *s != (char)c)
370		s++;
371	return (char *)s;
372}
373
374#ifndef __HAVE_ARCH_STRRCHR
375/**
376 * strrchr - Find the last occurrence of a character in a string
377 * @s: The string to be searched
378 * @c: The character to search for
379 */
380char *strrchr(const char *s, int c)
381{
382	const char *last = NULL;
383	do {
384		if (*s == (char)c)
385			last = s;
386	} while (*s++);
387	return (char *)last;
388}
389EXPORT_SYMBOL(strrchr);
390#endif
391
392#ifndef __HAVE_ARCH_STRNCHR
393/**
394 * strnchr - Find a character in a length limited string
395 * @s: The string to be searched
396 * @count: The number of characters to be searched
397 * @c: The character to search for
398 *
399 * Note that the %NUL-terminator is considered part of the string, and can
400 * be searched for.
401 */
402char *strnchr(const char *s, size_t count, int c)
403{
404	while (count--) {
405		if (*s == (char)c)
406			return (char *)s;
407		if (*s++ == '\0')
408			break;
409	}
410	return NULL;
411}
412EXPORT_SYMBOL(strnchr);
413#endif
414
415#ifndef __HAVE_ARCH_STRLEN
416size_t strlen(const char *s)
417{
418	const char *sc;
419
420	for (sc = s; *sc != '\0'; ++sc)
421		/* nothing */;
422	return sc - s;
423}
424EXPORT_SYMBOL(strlen);
425#endif
426
427#ifndef __HAVE_ARCH_STRNLEN
428size_t strnlen(const char *s, size_t count)
429{
430	const char *sc;
431
432	for (sc = s; count-- && *sc != '\0'; ++sc)
433		/* nothing */;
434	return sc - s;
435}
436EXPORT_SYMBOL(strnlen);
437#endif
438
439#ifndef __HAVE_ARCH_STRSPN
440/**
441 * strspn - Calculate the length of the initial substring of @s which only contain letters in @accept
442 * @s: The string to be searched
443 * @accept: The string to search for
444 */
445size_t strspn(const char *s, const char *accept)
446{
447	const char *p;
448
449	for (p = s; *p != '\0'; ++p) {
450		if (!strchr(accept, *p))
451			break;
452	}
453	return p - s;
454}
455EXPORT_SYMBOL(strspn);
456#endif
457
458#ifndef __HAVE_ARCH_STRCSPN
459/**
460 * strcspn - Calculate the length of the initial substring of @s which does not contain letters in @reject
461 * @s: The string to be searched
462 * @reject: The string to avoid
463 */
464size_t strcspn(const char *s, const char *reject)
465{
466	const char *p;
467
468	for (p = s; *p != '\0'; ++p) {
469		if (strchr(reject, *p))
470			break;
471	}
472	return p - s;
473}
474EXPORT_SYMBOL(strcspn);
475#endif
476
477#ifndef __HAVE_ARCH_STRPBRK
478/**
479 * strpbrk - Find the first occurrence of a set of characters
480 * @cs: The string to be searched
481 * @ct: The characters to search for
482 */
483char *strpbrk(const char *cs, const char *ct)
484{
485	const char *sc;
486
487	for (sc = cs; *sc != '\0'; ++sc) {
488		if (strchr(ct, *sc))
489			return (char *)sc;
490	}
491	return NULL;
492}
493EXPORT_SYMBOL(strpbrk);
494#endif
495
496#ifndef __HAVE_ARCH_STRSEP
497/**
498 * strsep - Split a string into tokens
499 * @s: The string to be searched
500 * @ct: The characters to search for
501 *
502 * strsep() updates @s to point after the token, ready for the next call.
503 *
504 * It returns empty tokens, too, behaving exactly like the libc function
505 * of that name. In fact, it was stolen from glibc2 and de-fancy-fied.
506 * Same semantics, slimmer shape. ;)
507 */
508char *strsep(char **s, const char *ct)
509{
510	char *sbegin = *s;
511	char *end;
512
513	if (sbegin == NULL)
514		return NULL;
515
516	end = strpbrk(sbegin, ct);
517	if (end)
518		*end++ = '\0';
519	*s = end;
520	return sbegin;
521}
522EXPORT_SYMBOL(strsep);
523#endif
524
525#ifndef __HAVE_ARCH_MEMSET
526/**
527 * memset - Fill a region of memory with the given value
528 * @s: Pointer to the start of the area.
529 * @c: The byte to fill the area with
530 * @count: The size of the area.
531 *
532 * Do not use memset() to access IO space, use memset_io() instead.
533 */
534void *memset(void *s, int c, size_t count)
535{
536	char *xs = s;
537
538	while (count--)
539		*xs++ = c;
540	return s;
541}
542EXPORT_SYMBOL(memset);
543#endif
544
545#ifndef __HAVE_ARCH_MEMSET16
546/**
547 * memset16() - Fill a memory area with a uint16_t
548 * @s: Pointer to the start of the area.
549 * @v: The value to fill the area with
550 * @count: The number of values to store
551 *
552 * Differs from memset() in that it fills with a uint16_t instead
553 * of a byte.  Remember that @count is the number of uint16_ts to
554 * store, not the number of bytes.
555 */
556void *memset16(uint16_t *s, uint16_t v, size_t count)
557{
558	uint16_t *xs = s;
559
560	while (count--)
561		*xs++ = v;
562	return s;
563}
564EXPORT_SYMBOL(memset16);
565#endif
566
567#ifndef __HAVE_ARCH_MEMSET32
568/**
569 * memset32() - Fill a memory area with a uint32_t
570 * @s: Pointer to the start of the area.
571 * @v: The value to fill the area with
572 * @count: The number of values to store
573 *
574 * Differs from memset() in that it fills with a uint32_t instead
575 * of a byte.  Remember that @count is the number of uint32_ts to
576 * store, not the number of bytes.
577 */
578void *memset32(uint32_t *s, uint32_t v, size_t count)
579{
580	uint32_t *xs = s;
581
582	while (count--)
583		*xs++ = v;
584	return s;
585}
586EXPORT_SYMBOL(memset32);
587#endif
588
589#ifndef __HAVE_ARCH_MEMSET64
590/**
591 * memset64() - Fill a memory area with a uint64_t
592 * @s: Pointer to the start of the area.
593 * @v: The value to fill the area with
594 * @count: The number of values to store
595 *
596 * Differs from memset() in that it fills with a uint64_t instead
597 * of a byte.  Remember that @count is the number of uint64_ts to
598 * store, not the number of bytes.
599 */
600void *memset64(uint64_t *s, uint64_t v, size_t count)
601{
602	uint64_t *xs = s;
603
604	while (count--)
605		*xs++ = v;
606	return s;
607}
608EXPORT_SYMBOL(memset64);
609#endif
610
611#ifndef __HAVE_ARCH_MEMCPY
612/**
613 * memcpy - Copy one area of memory to another
614 * @dest: Where to copy to
615 * @src: Where to copy from
616 * @count: The size of the area.
617 *
618 * You should not use this function to access IO space, use memcpy_toio()
619 * or memcpy_fromio() instead.
620 */
621void *memcpy(void *dest, const void *src, size_t count)
622{
623	char *tmp = dest;
624	const char *s = src;
625
626	while (count--)
627		*tmp++ = *s++;
628	return dest;
629}
630EXPORT_SYMBOL(memcpy);
631#endif
632
633#ifndef __HAVE_ARCH_MEMMOVE
634/**
635 * memmove - Copy one area of memory to another
636 * @dest: Where to copy to
637 * @src: Where to copy from
638 * @count: The size of the area.
639 *
640 * Unlike memcpy(), memmove() copes with overlapping areas.
641 */
642void *memmove(void *dest, const void *src, size_t count)
643{
644	char *tmp;
645	const char *s;
646
647	if (dest <= src) {
648		tmp = dest;
649		s = src;
650		while (count--)
651			*tmp++ = *s++;
652	} else {
653		tmp = dest;
654		tmp += count;
655		s = src;
656		s += count;
657		while (count--)
658			*--tmp = *--s;
659	}
660	return dest;
661}
662EXPORT_SYMBOL(memmove);
663#endif
664
665#ifndef __HAVE_ARCH_MEMCMP
666/**
667 * memcmp - Compare two areas of memory
668 * @cs: One area of memory
669 * @ct: Another area of memory
670 * @count: The size of the area.
671 */
672#undef memcmp
673__visible int memcmp(const void *cs, const void *ct, size_t count)
674{
675	const unsigned char *su1, *su2;
676	int res = 0;
677
678#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
679	if (count >= sizeof(unsigned long)) {
680		const unsigned long *u1 = cs;
681		const unsigned long *u2 = ct;
682		do {
683			if (get_unaligned(u1) != get_unaligned(u2))
684				break;
685			u1++;
686			u2++;
687			count -= sizeof(unsigned long);
688		} while (count >= sizeof(unsigned long));
689		cs = u1;
690		ct = u2;
691	}
692#endif
693	for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--)
694		if ((res = *su1 - *su2) != 0)
695			break;
696	return res;
697}
698EXPORT_SYMBOL(memcmp);
699#endif
700
701#ifndef __HAVE_ARCH_BCMP
702/**
703 * bcmp - returns 0 if and only if the buffers have identical contents.
704 * @a: pointer to first buffer.
705 * @b: pointer to second buffer.
706 * @len: size of buffers.
707 *
708 * The sign or magnitude of a non-zero return value has no particular
709 * meaning, and architectures may implement their own more efficient bcmp(). So
710 * while this particular implementation is a simple (tail) call to memcmp, do
711 * not rely on anything but whether the return value is zero or non-zero.
712 */
713int bcmp(const void *a, const void *b, size_t len)
714{
715	return memcmp(a, b, len);
716}
717EXPORT_SYMBOL(bcmp);
718#endif
719
720#ifndef __HAVE_ARCH_MEMSCAN
721/**
722 * memscan - Find a character in an area of memory.
723 * @addr: The memory area
724 * @c: The byte to search for
725 * @size: The size of the area.
726 *
727 * returns the address of the first occurrence of @c, or 1 byte past
728 * the area if @c is not found
729 */
730void *memscan(void *addr, int c, size_t size)
731{
732	unsigned char *p = addr;
733
734	while (size) {
735		if (*p == (unsigned char)c)
736			return (void *)p;
737		p++;
738		size--;
739	}
740  	return (void *)p;
741}
742EXPORT_SYMBOL(memscan);
743#endif
744
745#ifndef __HAVE_ARCH_STRSTR
746/**
747 * strstr - Find the first substring in a %NUL terminated string
748 * @s1: The string to be searched
749 * @s2: The string to search for
750 */
751char *strstr(const char *s1, const char *s2)
752{
753	size_t l1, l2;
754
755	l2 = strlen(s2);
756	if (!l2)
757		return (char *)s1;
758	l1 = strlen(s1);
759	while (l1 >= l2) {
760		l1--;
761		if (!memcmp(s1, s2, l2))
762			return (char *)s1;
763		s1++;
764	}
765	return NULL;
766}
767EXPORT_SYMBOL(strstr);
768#endif
769
770#ifndef __HAVE_ARCH_STRNSTR
771/**
772 * strnstr - Find the first substring in a length-limited string
773 * @s1: The string to be searched
774 * @s2: The string to search for
775 * @len: the maximum number of characters to search
776 */
777char *strnstr(const char *s1, const char *s2, size_t len)
778{
779	size_t l2;
780
781	l2 = strlen(s2);
782	if (!l2)
783		return (char *)s1;
784	while (len >= l2) {
785		len--;
786		if (!memcmp(s1, s2, l2))
787			return (char *)s1;
788		s1++;
789	}
790	return NULL;
791}
792EXPORT_SYMBOL(strnstr);
793#endif
794
795#ifndef __HAVE_ARCH_MEMCHR
796/**
797 * memchr - Find a character in an area of memory.
798 * @s: The memory area
799 * @c: The byte to search for
800 * @n: The size of the area.
801 *
802 * returns the address of the first occurrence of @c, or %NULL
803 * if @c is not found
804 */
805void *memchr(const void *s, int c, size_t n)
806{
807	const unsigned char *p = s;
808	while (n-- != 0) {
809        	if ((unsigned char)c == *p++) {
810			return (void *)(p - 1);
811		}
812	}
813	return NULL;
814}
815EXPORT_SYMBOL(memchr);
816#endif
817
818static void *check_bytes8(const u8 *start, u8 value, unsigned int bytes)
819{
820	while (bytes) {
821		if (*start != value)
822			return (void *)start;
823		start++;
824		bytes--;
825	}
826	return NULL;
827}
828
829/**
830 * memchr_inv - Find an unmatching character in an area of memory.
831 * @start: The memory area
832 * @c: Find a character other than c
833 * @bytes: The size of the area.
834 *
835 * returns the address of the first character other than @c, or %NULL
836 * if the whole buffer contains just @c.
837 */
838void *memchr_inv(const void *start, int c, size_t bytes)
839{
840	u8 value = c;
841	u64 value64;
842	unsigned int words, prefix;
843
844	if (bytes <= 16)
845		return check_bytes8(start, value, bytes);
846
847	value64 = value;
848#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64
849	value64 *= 0x0101010101010101ULL;
850#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER)
851	value64 *= 0x01010101;
852	value64 |= value64 << 32;
853#else
854	value64 |= value64 << 8;
855	value64 |= value64 << 16;
856	value64 |= value64 << 32;
857#endif
858
859	prefix = (unsigned long)start % 8;
860	if (prefix) {
861		u8 *r;
862
863		prefix = 8 - prefix;
864		r = check_bytes8(start, value, prefix);
865		if (r)
866			return r;
867		start += prefix;
868		bytes -= prefix;
869	}
870
871	words = bytes / 8;
872
873	while (words) {
874		if (*(u64 *)start != value64)
875			return check_bytes8(start, value, 8);
876		start += 8;
877		words--;
878	}
879
880	return check_bytes8(start, value, bytes % 8);
881}
882EXPORT_SYMBOL(memchr_inv);