dso.c

   1// SPDX-License-Identifier: GPL-2.0
   2#include <asm/bug.h>
   3#include <linux/kernel.h>
   4#include <linux/string.h>
   5#include <linux/zalloc.h>
   6#include <sys/time.h>
   7#include <sys/resource.h>
   8#include <sys/types.h>
   9#include <sys/stat.h>
  10#include <unistd.h>
  11#include <errno.h>
  12#include <fcntl.h>
  13#include <stdlib.h>
  14#ifdef HAVE_LIBBPF_SUPPORT
  15#include <bpf/libbpf.h>
  16#include "bpf-event.h"
  17#include "bpf-utils.h"
  18#endif
  19#include "compress.h"
  20#include "env.h"
  21#include "namespaces.h"
  22#include "path.h"
  23#include "map.h"
  24#include "symbol.h"
  25#include "srcline.h"
  26#include "dso.h"
  27#include "dsos.h"
  28#include "machine.h"
  29#include "auxtrace.h"
  30#include "util.h" /* O_CLOEXEC for older systems */
  31#include "debug.h"
  32#include "string2.h"
  33#include "vdso.h"
  34#include "annotate-data.h"
  35
  36static const char * const debuglink_paths[] = {
  37	"%.0s%s",
  38	"%s/%s",
  39	"%s/.debug/%s",
  40	"/usr/lib/debug%s/%s"
  41};
  42
  43void dso__set_nsinfo(struct dso *dso, struct nsinfo *nsi)
  44{
  45	nsinfo__put(RC_CHK_ACCESS(dso)->nsinfo);
  46	RC_CHK_ACCESS(dso)->nsinfo = nsi;
  47}
  48
  49char dso__symtab_origin(const struct dso *dso)
  50{
  51	static const char origin[] = {
  52		[DSO_BINARY_TYPE__KALLSYMS]			= 'k',
  53		[DSO_BINARY_TYPE__VMLINUX]			= 'v',
  54		[DSO_BINARY_TYPE__JAVA_JIT]			= 'j',
  55		[DSO_BINARY_TYPE__DEBUGLINK]			= 'l',
  56		[DSO_BINARY_TYPE__BUILD_ID_CACHE]		= 'B',
  57		[DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO]	= 'D',
  58		[DSO_BINARY_TYPE__FEDORA_DEBUGINFO]		= 'f',
  59		[DSO_BINARY_TYPE__UBUNTU_DEBUGINFO]		= 'u',
  60		[DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO]	= 'x',
  61		[DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO]	= 'o',
  62		[DSO_BINARY_TYPE__BUILDID_DEBUGINFO]		= 'b',
  63		[DSO_BINARY_TYPE__SYSTEM_PATH_DSO]		= 'd',
  64		[DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE]		= 'K',
  65		[DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP]	= 'm',
  66		[DSO_BINARY_TYPE__GUEST_KALLSYMS]		= 'g',
  67		[DSO_BINARY_TYPE__GUEST_KMODULE]		= 'G',
  68		[DSO_BINARY_TYPE__GUEST_KMODULE_COMP]		= 'M',
  69		[DSO_BINARY_TYPE__GUEST_VMLINUX]		= 'V',
  70		[DSO_BINARY_TYPE__GNU_DEBUGDATA]		= 'n',
  71	};
  72
  73	if (dso == NULL || dso__symtab_type(dso) == DSO_BINARY_TYPE__NOT_FOUND)
  74		return '!';
  75	return origin[dso__symtab_type(dso)];
  76}
  77
  78bool dso__is_object_file(const struct dso *dso)
  79{
  80	switch (dso__binary_type(dso)) {
  81	case DSO_BINARY_TYPE__KALLSYMS:
  82	case DSO_BINARY_TYPE__GUEST_KALLSYMS:
  83	case DSO_BINARY_TYPE__JAVA_JIT:
  84	case DSO_BINARY_TYPE__BPF_PROG_INFO:
  85	case DSO_BINARY_TYPE__BPF_IMAGE:
  86	case DSO_BINARY_TYPE__OOL:
  87		return false;
  88	case DSO_BINARY_TYPE__VMLINUX:
  89	case DSO_BINARY_TYPE__GUEST_VMLINUX:
  90	case DSO_BINARY_TYPE__DEBUGLINK:
  91	case DSO_BINARY_TYPE__BUILD_ID_CACHE:
  92	case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO:
  93	case DSO_BINARY_TYPE__FEDORA_DEBUGINFO:
  94	case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO:
  95	case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO:
  96	case DSO_BINARY_TYPE__BUILDID_DEBUGINFO:
  97	case DSO_BINARY_TYPE__GNU_DEBUGDATA:
  98	case DSO_BINARY_TYPE__SYSTEM_PATH_DSO:
  99	case DSO_BINARY_TYPE__GUEST_KMODULE:
 100	case DSO_BINARY_TYPE__GUEST_KMODULE_COMP:
 101	case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE:
 102	case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP:
 103	case DSO_BINARY_TYPE__KCORE:
 104	case DSO_BINARY_TYPE__GUEST_KCORE:
 105	case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO:
 106	case DSO_BINARY_TYPE__NOT_FOUND:
 107	default:
 108		return true;
 109	}
 110}
 111
 112int dso__read_binary_type_filename(const struct dso *dso,
 113				   enum dso_binary_type type,
 114				   const char *root_dir, char *filename, size_t size)
 115{
 116	char build_id_hex[SBUILD_ID_SIZE];
 117	int ret = 0;
 118	size_t len;
 119
 120	switch (type) {
 121	case DSO_BINARY_TYPE__DEBUGLINK:
 122	{
 123		const char *last_slash;
 124		char dso_dir[PATH_MAX];
 125		char symfile[PATH_MAX];
 126		unsigned int i;
 127
 128		len = __symbol__join_symfs(filename, size, dso__long_name(dso));
 129		last_slash = filename + len;
 130		while (last_slash != filename && *last_slash != '/')
 131			last_slash--;
 132
 133		strncpy(dso_dir, filename, last_slash - filename);
 134		dso_dir[last_slash-filename] = '\0';
 135
 136		if (!is_regular_file(filename)) {
 137			ret = -1;
 138			break;
 139		}
 140
 141		ret = filename__read_debuglink(filename, symfile, PATH_MAX);
 142		if (ret)
 143			break;
 144
 145		/* Check predefined locations where debug file might reside */
 146		ret = -1;
 147		for (i = 0; i < ARRAY_SIZE(debuglink_paths); i++) {
 148			snprintf(filename, size,
 149					debuglink_paths[i], dso_dir, symfile);
 150			if (is_regular_file(filename)) {
 151				ret = 0;
 152				break;
 153			}
 154		}
 155
 156		break;
 157	}
 158	case DSO_BINARY_TYPE__BUILD_ID_CACHE:
 159		if (dso__build_id_filename(dso, filename, size, false) == NULL)
 160			ret = -1;
 161		break;
 162
 163	case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO:
 164		if (dso__build_id_filename(dso, filename, size, true) == NULL)
 165			ret = -1;
 166		break;
 167
 168	case DSO_BINARY_TYPE__FEDORA_DEBUGINFO:
 169		len = __symbol__join_symfs(filename, size, "/usr/lib/debug");
 170		snprintf(filename + len, size - len, "%s.debug", dso__long_name(dso));
 171		break;
 172
 173	case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO:
 174		len = __symbol__join_symfs(filename, size, "/usr/lib/debug");
 175		snprintf(filename + len, size - len, "%s", dso__long_name(dso));
 176		break;
 177
 178	case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO:
 179		/*
 180		 * Ubuntu can mixup /usr/lib with /lib, putting debuginfo in
 181		 * /usr/lib/debug/lib when it is expected to be in
 182		 * /usr/lib/debug/usr/lib
 183		 */
 184		if (strlen(dso__long_name(dso)) < 9 ||
 185		    strncmp(dso__long_name(dso), "/usr/lib/", 9)) {
 186			ret = -1;
 187			break;
 188		}
 189		len = __symbol__join_symfs(filename, size, "/usr/lib/debug");
 190		snprintf(filename + len, size - len, "%s", dso__long_name(dso) + 4);
 191		break;
 192
 193	case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO:
 194	{
 195		const char *last_slash;
 196		size_t dir_size;
 197
 198		last_slash = dso__long_name(dso) + dso__long_name_len(dso);
 199		while (last_slash != dso__long_name(dso) && *last_slash != '/')
 200			last_slash--;
 201
 202		len = __symbol__join_symfs(filename, size, "");
 203		dir_size = last_slash - dso__long_name(dso) + 2;
 204		if (dir_size > (size - len)) {
 205			ret = -1;
 206			break;
 207		}
 208		len += scnprintf(filename + len, dir_size, "%s",  dso__long_name(dso));
 209		len += scnprintf(filename + len , size - len, ".debug%s",
 210								last_slash);
 211		break;
 212	}
 213
 214	case DSO_BINARY_TYPE__BUILDID_DEBUGINFO:
 215		if (!dso__has_build_id(dso)) {
 216			ret = -1;
 217			break;
 218		}
 219
 220		build_id__snprintf(dso__bid(dso), build_id_hex, sizeof(build_id_hex));
 221		len = __symbol__join_symfs(filename, size, "/usr/lib/debug/.build-id/");
 222		snprintf(filename + len, size - len, "%.2s/%s.debug",
 223			 build_id_hex, build_id_hex + 2);
 224		break;
 225
 226	case DSO_BINARY_TYPE__VMLINUX:
 227	case DSO_BINARY_TYPE__GUEST_VMLINUX:
 228	case DSO_BINARY_TYPE__SYSTEM_PATH_DSO:
 229	case DSO_BINARY_TYPE__GNU_DEBUGDATA:
 230		__symbol__join_symfs(filename, size, dso__long_name(dso));
 231		break;
 232
 233	case DSO_BINARY_TYPE__GUEST_KMODULE:
 234	case DSO_BINARY_TYPE__GUEST_KMODULE_COMP:
 235		path__join3(filename, size, symbol_conf.symfs,
 236			    root_dir, dso__long_name(dso));
 237		break;
 238
 239	case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE:
 240	case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP:
 241		__symbol__join_symfs(filename, size, dso__long_name(dso));
 242		break;
 243
 244	case DSO_BINARY_TYPE__KCORE:
 245	case DSO_BINARY_TYPE__GUEST_KCORE:
 246		snprintf(filename, size, "%s", dso__long_name(dso));
 247		break;
 248
 249	default:
 250	case DSO_BINARY_TYPE__KALLSYMS:
 251	case DSO_BINARY_TYPE__GUEST_KALLSYMS:
 252	case DSO_BINARY_TYPE__JAVA_JIT:
 253	case DSO_BINARY_TYPE__BPF_PROG_INFO:
 254	case DSO_BINARY_TYPE__BPF_IMAGE:
 255	case DSO_BINARY_TYPE__OOL:
 256	case DSO_BINARY_TYPE__NOT_FOUND:
 257		ret = -1;
 258		break;
 259	}
 260
 261	return ret;
 262}
 263
 264enum {
 265	COMP_ID__NONE = 0,
 266};
 267
 268static const struct {
 269	const char *fmt;
 270	int (*decompress)(const char *input, int output);
 271	bool (*is_compressed)(const char *input);
 272} compressions[] = {
 273	[COMP_ID__NONE] = { .fmt = NULL, },
 274#ifdef HAVE_ZLIB_SUPPORT
 275	{ "gz", gzip_decompress_to_file, gzip_is_compressed },
 276#endif
 277#ifdef HAVE_LZMA_SUPPORT
 278	{ "xz", lzma_decompress_to_file, lzma_is_compressed },
 279#endif
 280	{ NULL, NULL, NULL },
 281};
 282
 283static int is_supported_compression(const char *ext)
 284{
 285	unsigned i;
 286
 287	for (i = 1; compressions[i].fmt; i++) {
 288		if (!strcmp(ext, compressions[i].fmt))
 289			return i;
 290	}
 291	return COMP_ID__NONE;
 292}
 293
 294bool is_kernel_module(const char *pathname, int cpumode)
 295{
 296	struct kmod_path m;
 297	int mode = cpumode & PERF_RECORD_MISC_CPUMODE_MASK;
 298
 299	WARN_ONCE(mode != cpumode,
 300		  "Internal error: passing unmasked cpumode (%x) to is_kernel_module",
 301		  cpumode);
 302
 303	switch (mode) {
 304	case PERF_RECORD_MISC_USER:
 305	case PERF_RECORD_MISC_HYPERVISOR:
 306	case PERF_RECORD_MISC_GUEST_USER:
 307		return false;
 308	/* Treat PERF_RECORD_MISC_CPUMODE_UNKNOWN as kernel */
 309	default:
 310		if (kmod_path__parse(&m, pathname)) {
 311			pr_err("Failed to check whether %s is a kernel module or not. Assume it is.",
 312					pathname);
 313			return true;
 314		}
 315	}
 316
 317	return m.kmod;
 318}
 319
 320bool dso__needs_decompress(struct dso *dso)
 321{
 322	return dso__symtab_type(dso) == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP ||
 323		dso__symtab_type(dso) == DSO_BINARY_TYPE__GUEST_KMODULE_COMP;
 324}
 325
 326int filename__decompress(const char *name, char *pathname,
 327			 size_t len, int comp, int *err)
 328{
 329	char tmpbuf[] = KMOD_DECOMP_NAME;
 330	int fd = -1;
 331
 332	/*
 333	 * We have proper compression id for DSO and yet the file
 334	 * behind the 'name' can still be plain uncompressed object.
 335	 *
 336	 * The reason is behind the logic we open the DSO object files,
 337	 * when we try all possible 'debug' objects until we find the
 338	 * data. So even if the DSO is represented by 'krava.xz' module,
 339	 * we can end up here opening ~/.debug/....23432432/debug' file
 340	 * which is not compressed.
 341	 *
 342	 * To keep this transparent, we detect this and return the file
 343	 * descriptor to the uncompressed file.
 344	 */
 345	if (!compressions[comp].is_compressed(name))
 346		return open(name, O_RDONLY);
 347
 348	fd = mkstemp(tmpbuf);
 349	if (fd < 0) {
 350		*err = errno;
 351		return -1;
 352	}
 353
 354	if (compressions[comp].decompress(name, fd)) {
 355		*err = DSO_LOAD_ERRNO__DECOMPRESSION_FAILURE;
 356		close(fd);
 357		fd = -1;
 358	}
 359
 360	if (!pathname || (fd < 0))
 361		unlink(tmpbuf);
 362
 363	if (pathname && (fd >= 0))
 364		strlcpy(pathname, tmpbuf, len);
 365
 366	return fd;
 367}
 368
 369static int decompress_kmodule(struct dso *dso, const char *name,
 370			      char *pathname, size_t len)
 371{
 372	if (!dso__needs_decompress(dso))
 373		return -1;
 374
 375	if (dso__comp(dso) == COMP_ID__NONE)
 376		return -1;
 377
 378	return filename__decompress(name, pathname, len, dso__comp(dso), dso__load_errno(dso));
 379}
 380
 381int dso__decompress_kmodule_fd(struct dso *dso, const char *name)
 382{
 383	return decompress_kmodule(dso, name, NULL, 0);
 384}
 385
 386int dso__decompress_kmodule_path(struct dso *dso, const char *name,
 387				 char *pathname, size_t len)
 388{
 389	int fd = decompress_kmodule(dso, name, pathname, len);
 390
 391	close(fd);
 392	return fd >= 0 ? 0 : -1;
 393}
 394
 395/*
 396 * Parses kernel module specified in @path and updates
 397 * @m argument like:
 398 *
 399 *    @comp - true if @path contains supported compression suffix,
 400 *            false otherwise
 401 *    @kmod - true if @path contains '.ko' suffix in right position,
 402 *            false otherwise
 403 *    @name - if (@alloc_name && @kmod) is true, it contains strdup-ed base name
 404 *            of the kernel module without suffixes, otherwise strudup-ed
 405 *            base name of @path
 406 *    @ext  - if (@alloc_ext && @comp) is true, it contains strdup-ed string
 407 *            the compression suffix
 408 *
 409 * Returns 0 if there's no strdup error, -ENOMEM otherwise.
 410 */
 411int __kmod_path__parse(struct kmod_path *m, const char *path,
 412		       bool alloc_name)
 413{
 414	const char *name = strrchr(path, '/');
 415	const char *ext  = strrchr(path, '.');
 416	bool is_simple_name = false;
 417
 418	memset(m, 0x0, sizeof(*m));
 419	name = name ? name + 1 : path;
 420
 421	/*
 422	 * '.' is also a valid character for module name. For example:
 423	 * [aaa.bbb] is a valid module name. '[' should have higher
 424	 * priority than '.ko' suffix.
 425	 *
 426	 * The kernel names are from machine__mmap_name. Such
 427	 * name should belong to kernel itself, not kernel module.
 428	 */
 429	if (name[0] == '[') {
 430		is_simple_name = true;
 431		if ((strncmp(name, "[kernel.kallsyms]", 17) == 0) ||
 432		    (strncmp(name, "[guest.kernel.kallsyms", 22) == 0) ||
 433		    (strncmp(name, "[vdso]", 6) == 0) ||
 434		    (strncmp(name, "[vdso32]", 8) == 0) ||
 435		    (strncmp(name, "[vdsox32]", 9) == 0) ||
 436		    (strncmp(name, "[vsyscall]", 10) == 0)) {
 437			m->kmod = false;
 438
 439		} else
 440			m->kmod = true;
 441	}
 442
 443	/* No extension, just return name. */
 444	if ((ext == NULL) || is_simple_name) {
 445		if (alloc_name) {
 446			m->name = strdup(name);
 447			return m->name ? 0 : -ENOMEM;
 448		}
 449		return 0;
 450	}
 451
 452	m->comp = is_supported_compression(ext + 1);
 453	if (m->comp > COMP_ID__NONE)
 454		ext -= 3;
 455
 456	/* Check .ko extension only if there's enough name left. */
 457	if (ext > name)
 458		m->kmod = !strncmp(ext, ".ko", 3);
 459
 460	if (alloc_name) {
 461		if (m->kmod) {
 462			if (asprintf(&m->name, "[%.*s]", (int) (ext - name), name) == -1)
 463				return -ENOMEM;
 464		} else {
 465			if (asprintf(&m->name, "%s", name) == -1)
 466				return -ENOMEM;
 467		}
 468
 469		strreplace(m->name, '-', '_');
 470	}
 471
 472	return 0;
 473}
 474
 475void dso__set_module_info(struct dso *dso, struct kmod_path *m,
 476			  struct machine *machine)
 477{
 478	if (machine__is_host(machine))
 479		dso__set_symtab_type(dso, DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE);
 480	else
 481		dso__set_symtab_type(dso, DSO_BINARY_TYPE__GUEST_KMODULE);
 482
 483	/* _KMODULE_COMP should be next to _KMODULE */
 484	if (m->kmod && m->comp) {
 485		dso__set_symtab_type(dso, dso__symtab_type(dso) + 1);
 486		dso__set_comp(dso, m->comp);
 487	}
 488
 489	dso__set_is_kmod(dso);
 490	dso__set_short_name(dso, strdup(m->name), true);
 491}
 492
 493/*
 494 * Global list of open DSOs and the counter.
 495 */
 496struct mutex _dso__data_open_lock;
 497static LIST_HEAD(dso__data_open);
 498static long dso__data_open_cnt GUARDED_BY(_dso__data_open_lock);
 499
 500static void dso__data_open_lock_init(void)
 501{
 502	mutex_init(&_dso__data_open_lock);
 503}
 504
 505static struct mutex *dso__data_open_lock(void) LOCK_RETURNED(_dso__data_open_lock)
 506{
 507	static pthread_once_t data_open_lock_once = PTHREAD_ONCE_INIT;
 508
 509	pthread_once(&data_open_lock_once, dso__data_open_lock_init);
 510
 511	return &_dso__data_open_lock;
 512}
 513
 514static void dso__list_add(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 515{
 516	list_add_tail(&dso__data(dso)->open_entry, &dso__data_open);
 517#ifdef REFCNT_CHECKING
 518	dso__data(dso)->dso = dso__get(dso);
 519#endif
 520	/* Assume the dso is part of dsos, hence the optional reference count above. */
 521	assert(dso__dsos(dso));
 522	dso__data_open_cnt++;
 523}
 524
 525static void dso__list_del(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 526{
 527	list_del_init(&dso__data(dso)->open_entry);
 528#ifdef REFCNT_CHECKING
 529	mutex_unlock(dso__data_open_lock());
 530	dso__put(dso__data(dso)->dso);
 531	mutex_lock(dso__data_open_lock());
 532#endif
 533	WARN_ONCE(dso__data_open_cnt <= 0,
 534		  "DSO data fd counter out of bounds.");
 535	dso__data_open_cnt--;
 536}
 537
 538static void close_first_dso(void);
 539
 540static int do_open(char *name) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 541{
 542	int fd;
 543	char sbuf[STRERR_BUFSIZE];
 544
 545	do {
 546		fd = open(name, O_RDONLY|O_CLOEXEC);
 547		if (fd >= 0)
 548			return fd;
 549
 550		pr_debug("dso open failed: %s\n",
 551			 str_error_r(errno, sbuf, sizeof(sbuf)));
 552		if (!dso__data_open_cnt || errno != EMFILE)
 553			break;
 554
 555		close_first_dso();
 556	} while (1);
 557
 558	return -1;
 559}
 560
 561char *dso__filename_with_chroot(const struct dso *dso, const char *filename)
 562{
 563	return filename_with_chroot(nsinfo__pid(dso__nsinfo_const(dso)), filename);
 564}
 565
 566static char *dso__get_filename(struct dso *dso, const char *root_dir,
 567			       bool *decomp)
 568{
 569	char *name = malloc(PATH_MAX);
 570
 571	*decomp = false;
 572
 573	if (name == NULL)
 574		return NULL;
 575
 576	if (dso__read_binary_type_filename(dso, dso__binary_type(dso),
 577					    root_dir, name, PATH_MAX))
 578		goto out;
 579
 580	if (!is_regular_file(name)) {
 581		char *new_name;
 582
 583		if (errno != ENOENT || dso__nsinfo(dso) == NULL)
 584			goto out;
 585
 586		new_name = dso__filename_with_chroot(dso, name);
 587		if (!new_name)
 588			goto out;
 589
 590		free(name);
 591		name = new_name;
 592	}
 593
 594	if (dso__needs_decompress(dso)) {
 595		char newpath[KMOD_DECOMP_LEN];
 596		size_t len = sizeof(newpath);
 597
 598		if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) {
 599			errno = *dso__load_errno(dso);
 600			goto out;
 601		}
 602
 603		*decomp = true;
 604		strcpy(name, newpath);
 605	}
 606	return name;
 607
 608out:
 609	free(name);
 610	return NULL;
 611}
 612
 613static int __open_dso(struct dso *dso, struct machine *machine)
 614	EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 615{
 616	int fd = -EINVAL;
 617	char *name;
 618	bool decomp = false;
 619
 620	mutex_lock(dso__lock(dso));
 621
 622	name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp);
 623	if (name)
 624		fd = do_open(name);
 625	else
 626		fd = -errno;
 627
 628	if (decomp)
 629		unlink(name);
 630
 631	mutex_unlock(dso__lock(dso));
 632	free(name);
 633	return fd;
 634}
 635
 636static void check_data_close(void);
 637
 638/**
 639 * dso_close - Open DSO data file
 640 * @dso: dso object
 641 *
 642 * Open @dso's data file descriptor and updates
 643 * list/count of open DSO objects.
 644 */
 645static int open_dso(struct dso *dso, struct machine *machine)
 646	EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 647{
 648	int fd;
 649	struct nscookie nsc;
 650
 651	if (dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE) {
 652		mutex_lock(dso__lock(dso));
 653		nsinfo__mountns_enter(dso__nsinfo(dso), &nsc);
 654		mutex_unlock(dso__lock(dso));
 655	}
 656	fd = __open_dso(dso, machine);
 657	if (dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE)
 658		nsinfo__mountns_exit(&nsc);
 659
 660	if (fd >= 0) {
 661		dso__list_add(dso);
 662		/*
 663		 * Check if we crossed the allowed number
 664		 * of opened DSOs and close one if needed.
 665		 */
 666		check_data_close();
 667	}
 668
 669	return fd;
 670}
 671
 672static void close_data_fd(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 673{
 674	if (dso__data(dso)->fd >= 0) {
 675		close(dso__data(dso)->fd);
 676		dso__data(dso)->fd = -1;
 677		dso__data(dso)->file_size = 0;
 678		dso__list_del(dso);
 679	}
 680}
 681
 682/**
 683 * dso_close - Close DSO data file
 684 * @dso: dso object
 685 *
 686 * Close @dso's data file descriptor and updates
 687 * list/count of open DSO objects.
 688 */
 689static void close_dso(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 690{
 691	close_data_fd(dso);
 692}
 693
 694static void close_first_dso(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 695{
 696	struct dso_data *dso_data;
 697	struct dso *dso;
 698
 699	dso_data = list_first_entry(&dso__data_open, struct dso_data, open_entry);
 700#ifdef REFCNT_CHECKING
 701	dso = dso_data->dso;
 702#else
 703	dso = container_of(dso_data, struct dso, data);
 704#endif
 705	close_dso(dso);
 706}
 707
 708static rlim_t get_fd_limit(void)
 709{
 710	struct rlimit l;
 711	rlim_t limit = 0;
 712
 713	/* Allow half of the current open fd limit. */
 714	if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
 715		if (l.rlim_cur == RLIM_INFINITY)
 716			limit = l.rlim_cur;
 717		else
 718			limit = l.rlim_cur / 2;
 719	} else {
 720		pr_err("failed to get fd limit\n");
 721		limit = 1;
 722	}
 723
 724	return limit;
 725}
 726
 727static rlim_t fd_limit;
 728
 729/*
 730 * Used only by tests/dso-data.c to reset the environment
 731 * for tests. I dont expect we should change this during
 732 * standard runtime.
 733 */
 734void reset_fd_limit(void)
 735{
 736	fd_limit = 0;
 737}
 738
 739static bool may_cache_fd(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 740{
 741	if (!fd_limit)
 742		fd_limit = get_fd_limit();
 743
 744	if (fd_limit == RLIM_INFINITY)
 745		return true;
 746
 747	return fd_limit > (rlim_t) dso__data_open_cnt;
 748}
 749
 750/*
 751 * Check and close LRU dso if we crossed allowed limit
 752 * for opened dso file descriptors. The limit is half
 753 * of the RLIMIT_NOFILE files opened.
 754*/
 755static void check_data_close(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 756{
 757	bool cache_fd = may_cache_fd();
 758
 759	if (!cache_fd)
 760		close_first_dso();
 761}
 762
 763/**
 764 * dso__data_close - Close DSO data file
 765 * @dso: dso object
 766 *
 767 * External interface to close @dso's data file descriptor.
 768 */
 769void dso__data_close(struct dso *dso)
 770{
 771	mutex_lock(dso__data_open_lock());
 772	close_dso(dso);
 773	mutex_unlock(dso__data_open_lock());
 774}
 775
 776static void try_to_open_dso(struct dso *dso, struct machine *machine)
 777	EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock)
 778{
 779	enum dso_binary_type binary_type_data[] = {
 780		DSO_BINARY_TYPE__BUILD_ID_CACHE,
 781		DSO_BINARY_TYPE__SYSTEM_PATH_DSO,
 782		DSO_BINARY_TYPE__NOT_FOUND,
 783	};
 784	int i = 0;
 785	struct dso_data *dso_data = dso__data(dso);
 786
 787	if (dso_data->fd >= 0)
 788		return;
 789
 790	if (dso__binary_type(dso) != DSO_BINARY_TYPE__NOT_FOUND) {
 791		dso_data->fd = open_dso(dso, machine);
 792		goto out;
 793	}
 794
 795	do {
 796		dso__set_binary_type(dso, binary_type_data[i++]);
 797
 798		dso_data->fd = open_dso(dso, machine);
 799		if (dso_data->fd >= 0)
 800			goto out;
 801
 802	} while (dso__binary_type(dso) != DSO_BINARY_TYPE__NOT_FOUND);
 803out:
 804	if (dso_data->fd >= 0)
 805		dso_data->status = DSO_DATA_STATUS_OK;
 806	else
 807		dso_data->status = DSO_DATA_STATUS_ERROR;
 808}
 809
 810/**
 811 * dso__data_get_fd - Get dso's data file descriptor
 812 * @dso: dso object
 813 * @machine: machine object
 814 *
 815 * External interface to find dso's file, open it and
 816 * returns file descriptor.  It should be paired with
 817 * dso__data_put_fd() if it returns non-negative value.
 818 */
 819bool dso__data_get_fd(struct dso *dso, struct machine *machine, int *fd)
 820{
 821	*fd = -1;
 822	if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR)
 823		return false;
 824
 825	mutex_lock(dso__data_open_lock());
 826
 827	try_to_open_dso(dso, machine);
 828
 829	*fd = dso__data(dso)->fd;
 830	if (*fd >= 0)
 831		return true;
 832
 833	mutex_unlock(dso__data_open_lock());
 834	return false;
 835}
 836
 837void dso__data_put_fd(struct dso *dso __maybe_unused)
 838{
 839	mutex_unlock(dso__data_open_lock());
 840}
 841
 842bool dso__data_status_seen(struct dso *dso, enum dso_data_status_seen by)
 843{
 844	u32 flag = 1 << by;
 845
 846	if (dso__data(dso)->status_seen & flag)
 847		return true;
 848
 849	dso__data(dso)->status_seen |= flag;
 850
 851	return false;
 852}
 853
 854#ifdef HAVE_LIBBPF_SUPPORT
 855static ssize_t bpf_read(struct dso *dso, u64 offset, char *data)
 856{
 857	struct bpf_prog_info_node *node;
 858	ssize_t size = DSO__DATA_CACHE_SIZE;
 859	struct dso_bpf_prog *dso_bpf_prog = dso__bpf_prog(dso);
 860	u64 len;
 861	u8 *buf;
 862
 863	node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id);
 864	if (!node || !node->info_linear) {
 865		dso__data(dso)->status = DSO_DATA_STATUS_ERROR;
 866		return -1;
 867	}
 868
 869	len = node->info_linear->info.jited_prog_len;
 870	buf = (u8 *)(uintptr_t)node->info_linear->info.jited_prog_insns;
 871
 872	if (offset >= len)
 873		return -1;
 874
 875	size = (ssize_t)min(len - offset, (u64)size);
 876	memcpy(data, buf + offset, size);
 877	return size;
 878}
 879
 880static int bpf_size(struct dso *dso)
 881{
 882	struct bpf_prog_info_node *node;
 883	struct dso_bpf_prog *dso_bpf_prog = dso__bpf_prog(dso);
 884
 885	node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id);
 886	if (!node || !node->info_linear) {
 887		dso__data(dso)->status = DSO_DATA_STATUS_ERROR;
 888		return -1;
 889	}
 890
 891	dso__data(dso)->file_size = node->info_linear->info.jited_prog_len;
 892	return 0;
 893}
 894#endif // HAVE_LIBBPF_SUPPORT
 895
 896static void
 897dso_cache__free(struct dso *dso)
 898{
 899	struct rb_root *root = &dso__data(dso)->cache;
 900	struct rb_node *next = rb_first(root);
 901
 902	mutex_lock(dso__lock(dso));
 903	while (next) {
 904		struct dso_cache *cache;
 905
 906		cache = rb_entry(next, struct dso_cache, rb_node);
 907		next = rb_next(&cache->rb_node);
 908		rb_erase(&cache->rb_node, root);
 909		free(cache);
 910	}
 911	mutex_unlock(dso__lock(dso));
 912}
 913
 914static struct dso_cache *__dso_cache__find(struct dso *dso, u64 offset)
 915{
 916	const struct rb_root *root = &dso__data(dso)->cache;
 917	struct rb_node * const *p = &root->rb_node;
 918	const struct rb_node *parent = NULL;
 919	struct dso_cache *cache;
 920
 921	while (*p != NULL) {
 922		u64 end;
 923
 924		parent = *p;
 925		cache = rb_entry(parent, struct dso_cache, rb_node);
 926		end = cache->offset + DSO__DATA_CACHE_SIZE;
 927
 928		if (offset < cache->offset)
 929			p = &(*p)->rb_left;
 930		else if (offset >= end)
 931			p = &(*p)->rb_right;
 932		else
 933			return cache;
 934	}
 935
 936	return NULL;
 937}
 938
 939static struct dso_cache *
 940dso_cache__insert(struct dso *dso, struct dso_cache *new)
 941{
 942	struct rb_root *root = &dso__data(dso)->cache;
 943	struct rb_node **p = &root->rb_node;
 944	struct rb_node *parent = NULL;
 945	struct dso_cache *cache;
 946	u64 offset = new->offset;
 947
 948	mutex_lock(dso__lock(dso));
 949	while (*p != NULL) {
 950		u64 end;
 951
 952		parent = *p;
 953		cache = rb_entry(parent, struct dso_cache, rb_node);
 954		end = cache->offset + DSO__DATA_CACHE_SIZE;
 955
 956		if (offset < cache->offset)
 957			p = &(*p)->rb_left;
 958		else if (offset >= end)
 959			p = &(*p)->rb_right;
 960		else
 961			goto out;
 962	}
 963
 964	rb_link_node(&new->rb_node, parent, p);
 965	rb_insert_color(&new->rb_node, root);
 966
 967	cache = NULL;
 968out:
 969	mutex_unlock(dso__lock(dso));
 970	return cache;
 971}
 972
 973static ssize_t dso_cache__memcpy(struct dso_cache *cache, u64 offset, u8 *data,
 974				 u64 size, bool out)
 975{
 976	u64 cache_offset = offset - cache->offset;
 977	u64 cache_size   = min(cache->size - cache_offset, size);
 978
 979	if (out)
 980		memcpy(data, cache->data + cache_offset, cache_size);
 981	else
 982		memcpy(cache->data + cache_offset, data, cache_size);
 983	return cache_size;
 984}
 985
 986static ssize_t file_read(struct dso *dso, struct machine *machine,
 987			 u64 offset, char *data)
 988{
 989	ssize_t ret;
 990
 991	mutex_lock(dso__data_open_lock());
 992
 993	/*
 994	 * dso__data(dso)->fd might be closed if other thread opened another
 995	 * file (dso) due to open file limit (RLIMIT_NOFILE).
 996	 */
 997	try_to_open_dso(dso, machine);
 998
 999	if (dso__data(dso)->fd < 0) {
1000		dso__data(dso)->status = DSO_DATA_STATUS_ERROR;
1001		ret = -errno;
1002		goto out;
1003	}
1004
1005	ret = pread(dso__data(dso)->fd, data, DSO__DATA_CACHE_SIZE, offset);
1006out:
1007	mutex_unlock(dso__data_open_lock());
1008	return ret;
1009}
1010
1011static struct dso_cache *dso_cache__populate(struct dso *dso,
1012					     struct machine *machine,
1013					     u64 offset, ssize_t *ret)
1014{
1015	u64 cache_offset = offset & DSO__DATA_CACHE_MASK;
1016	struct dso_cache *cache;
1017	struct dso_cache *old;
1018
1019	cache = zalloc(sizeof(*cache) + DSO__DATA_CACHE_SIZE);
1020	if (!cache) {
1021		*ret = -ENOMEM;
1022		return NULL;
1023	}
1024#ifdef HAVE_LIBBPF_SUPPORT
1025	if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO)
1026		*ret = bpf_read(dso, cache_offset, cache->data);
1027	else
1028#endif
1029	if (dso__binary_type(dso) == DSO_BINARY_TYPE__OOL)
1030		*ret = DSO__DATA_CACHE_SIZE;
1031	else
1032		*ret = file_read(dso, machine, cache_offset, cache->data);
1033
1034	if (*ret <= 0) {
1035		free(cache);
1036		return NULL;
1037	}
1038
1039	cache->offset = cache_offset;
1040	cache->size   = *ret;
1041
1042	old = dso_cache__insert(dso, cache);
1043	if (old) {
1044		/* we lose the race */
1045		free(cache);
1046		cache = old;
1047	}
1048
1049	return cache;
1050}
1051
1052static struct dso_cache *dso_cache__find(struct dso *dso,
1053					 struct machine *machine,
1054					 u64 offset,
1055					 ssize_t *ret)
1056{
1057	struct dso_cache *cache = __dso_cache__find(dso, offset);
1058
1059	return cache ? cache : dso_cache__populate(dso, machine, offset, ret);
1060}
1061
1062static ssize_t dso_cache_io(struct dso *dso, struct machine *machine,
1063			    u64 offset, u8 *data, ssize_t size, bool out)
1064{
1065	struct dso_cache *cache;
1066	ssize_t ret = 0;
1067
1068	cache = dso_cache__find(dso, machine, offset, &ret);
1069	if (!cache)
1070		return ret;
1071
1072	return dso_cache__memcpy(cache, offset, data, size, out);
1073}
1074
1075/*
1076 * Reads and caches dso data DSO__DATA_CACHE_SIZE size chunks
1077 * in the rb_tree. Any read to already cached data is served
1078 * by cached data. Writes update the cache only, not the backing file.
1079 */
1080static ssize_t cached_io(struct dso *dso, struct machine *machine,
1081			 u64 offset, u8 *data, ssize_t size, bool out)
1082{
1083	ssize_t r = 0;
1084	u8 *p = data;
1085
1086	do {
1087		ssize_t ret;
1088
1089		ret = dso_cache_io(dso, machine, offset, p, size, out);
1090		if (ret < 0)
1091			return ret;
1092
1093		/* Reached EOF, return what we have. */
1094		if (!ret)
1095			break;
1096
1097		BUG_ON(ret > size);
1098
1099		r      += ret;
1100		p      += ret;
1101		offset += ret;
1102		size   -= ret;
1103
1104	} while (size);
1105
1106	return r;
1107}
1108
1109static int file_size(struct dso *dso, struct machine *machine)
1110{
1111	int ret = 0;
1112	struct stat st;
1113	char sbuf[STRERR_BUFSIZE];
1114
1115	mutex_lock(dso__data_open_lock());
1116
1117	/*
1118	 * dso__data(dso)->fd might be closed if other thread opened another
1119	 * file (dso) due to open file limit (RLIMIT_NOFILE).
1120	 */
1121	try_to_open_dso(dso, machine);
1122
1123	if (dso__data(dso)->fd < 0) {
1124		ret = -errno;
1125		dso__data(dso)->status = DSO_DATA_STATUS_ERROR;
1126		goto out;
1127	}
1128
1129	if (fstat(dso__data(dso)->fd, &st) < 0) {
1130		ret = -errno;
1131		pr_err("dso cache fstat failed: %s\n",
1132		       str_error_r(errno, sbuf, sizeof(sbuf)));
1133		dso__data(dso)->status = DSO_DATA_STATUS_ERROR;
1134		goto out;
1135	}
1136	dso__data(dso)->file_size = st.st_size;
1137
1138out:
1139	mutex_unlock(dso__data_open_lock());
1140	return ret;
1141}
1142
1143int dso__data_file_size(struct dso *dso, struct machine *machine)
1144{
1145	if (dso__data(dso)->file_size)
1146		return 0;
1147
1148	if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR)
1149		return -1;
1150#ifdef HAVE_LIBBPF_SUPPORT
1151	if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO)
1152		return bpf_size(dso);
1153#endif
1154	return file_size(dso, machine);
1155}
1156
1157/**
1158 * dso__data_size - Return dso data size
1159 * @dso: dso object
1160 * @machine: machine object
1161 *
1162 * Return: dso data size
1163 */
1164off_t dso__data_size(struct dso *dso, struct machine *machine)
1165{
1166	if (dso__data_file_size(dso, machine))
1167		return -1;
1168
1169	/* For now just estimate dso data size is close to file size */
1170	return dso__data(dso)->file_size;
1171}
1172
1173static ssize_t data_read_write_offset(struct dso *dso, struct machine *machine,
1174				      u64 offset, u8 *data, ssize_t size,
1175				      bool out)
1176{
1177	if (dso__data_file_size(dso, machine))
1178		return -1;
1179
1180	/* Check the offset sanity. */
1181	if (offset > dso__data(dso)->file_size)
1182		return -1;
1183
1184	if (offset + size < offset)
1185		return -1;
1186
1187	return cached_io(dso, machine, offset, data, size, out);
1188}
1189
1190/**
1191 * dso__data_read_offset - Read data from dso file offset
1192 * @dso: dso object
1193 * @machine: machine object
1194 * @offset: file offset
1195 * @data: buffer to store data
1196 * @size: size of the @data buffer
1197 *
1198 * External interface to read data from dso file offset. Open
1199 * dso data file and use cached_read to get the data.
1200 */
1201ssize_t dso__data_read_offset(struct dso *dso, struct machine *machine,
1202			      u64 offset, u8 *data, ssize_t size)
1203{
1204	if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR)
1205		return -1;
1206
1207	return data_read_write_offset(dso, machine, offset, data, size, true);
1208}
1209
1210uint16_t dso__e_machine(struct dso *dso, struct machine *machine)
1211{
1212	uint16_t e_machine = EM_NONE;
1213	int fd;
1214
1215	switch (dso__binary_type(dso)) {
1216	case DSO_BINARY_TYPE__KALLSYMS:
1217	case DSO_BINARY_TYPE__GUEST_KALLSYMS:
1218	case DSO_BINARY_TYPE__VMLINUX:
1219	case DSO_BINARY_TYPE__GUEST_VMLINUX:
1220	case DSO_BINARY_TYPE__GUEST_KMODULE:
1221	case DSO_BINARY_TYPE__GUEST_KMODULE_COMP:
1222	case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE:
1223	case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP:
1224	case DSO_BINARY_TYPE__KCORE:
1225	case DSO_BINARY_TYPE__GUEST_KCORE:
1226	case DSO_BINARY_TYPE__BPF_PROG_INFO:
1227	case DSO_BINARY_TYPE__BPF_IMAGE:
1228	case DSO_BINARY_TYPE__OOL:
1229	case DSO_BINARY_TYPE__JAVA_JIT:
1230		return EM_HOST;
1231	case DSO_BINARY_TYPE__DEBUGLINK:
1232	case DSO_BINARY_TYPE__BUILD_ID_CACHE:
1233	case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO:
1234	case DSO_BINARY_TYPE__GNU_DEBUGDATA:
1235	case DSO_BINARY_TYPE__SYSTEM_PATH_DSO:
1236	case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO:
1237	case DSO_BINARY_TYPE__FEDORA_DEBUGINFO:
1238	case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO:
1239	case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO:
1240	case DSO_BINARY_TYPE__BUILDID_DEBUGINFO:
1241		break;
1242	case DSO_BINARY_TYPE__NOT_FOUND:
1243	default:
1244		return EM_NONE;
1245	}
1246
1247	mutex_lock(dso__data_open_lock());
1248
1249	/*
1250	 * dso__data(dso)->fd might be closed if other thread opened another
1251	 * file (dso) due to open file limit (RLIMIT_NOFILE).
1252	 */
1253	try_to_open_dso(dso, machine);
1254	fd = dso__data(dso)->fd;
1255	if (fd >= 0) {
1256		_Static_assert(offsetof(Elf32_Ehdr, e_machine) == 18, "Unexpected offset");
1257		_Static_assert(offsetof(Elf64_Ehdr, e_machine) == 18, "Unexpected offset");
1258		if (dso__needs_swap(dso) == DSO_SWAP__UNSET) {
1259			unsigned char eidata;
1260
1261			if (pread(fd, &eidata, sizeof(eidata), EI_DATA) == sizeof(eidata))
1262				dso__swap_init(dso, eidata);
1263		}
1264		if (dso__needs_swap(dso) != DSO_SWAP__UNSET &&
1265		    pread(fd, &e_machine, sizeof(e_machine), 18) == sizeof(e_machine))
1266			e_machine = DSO__SWAP(dso, uint16_t, e_machine);
1267	}
1268	mutex_unlock(dso__data_open_lock());
1269	return e_machine;
1270}
1271
1272/**
1273 * dso__data_read_addr - Read data from dso address
1274 * @dso: dso object
1275 * @machine: machine object
1276 * @add: virtual memory address
1277 * @data: buffer to store data
1278 * @size: size of the @data buffer
1279 *
1280 * External interface to read data from dso address.
1281 */
1282ssize_t dso__data_read_addr(struct dso *dso, struct map *map,
1283			    struct machine *machine, u64 addr,
1284			    u8 *data, ssize_t size)
1285{
1286	u64 offset = map__map_ip(map, addr);
1287
1288	return dso__data_read_offset(dso, machine, offset, data, size);
1289}
1290
1291/**
1292 * dso__data_write_cache_offs - Write data to dso data cache at file offset
1293 * @dso: dso object
1294 * @machine: machine object
1295 * @offset: file offset
1296 * @data: buffer to write
1297 * @size: size of the @data buffer
1298 *
1299 * Write into the dso file data cache, but do not change the file itself.
1300 */
1301ssize_t dso__data_write_cache_offs(struct dso *dso, struct machine *machine,
1302				   u64 offset, const u8 *data_in, ssize_t size)
1303{
1304	u8 *data = (u8 *)data_in; /* cast away const to use same fns for r/w */
1305
1306	if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR)
1307		return -1;
1308
1309	return data_read_write_offset(dso, machine, offset, data, size, false);
1310}
1311
1312/**
1313 * dso__data_write_cache_addr - Write data to dso data cache at dso address
1314 * @dso: dso object
1315 * @machine: machine object
1316 * @add: virtual memory address
1317 * @data: buffer to write
1318 * @size: size of the @data buffer
1319 *
1320 * External interface to write into the dso file data cache, but do not change
1321 * the file itself.
1322 */
1323ssize_t dso__data_write_cache_addr(struct dso *dso, struct map *map,
1324				   struct machine *machine, u64 addr,
1325				   const u8 *data, ssize_t size)
1326{
1327	u64 offset = map__map_ip(map, addr);
1328
1329	return dso__data_write_cache_offs(dso, machine, offset, data, size);
1330}
1331
1332struct map *dso__new_map(const char *name)
1333{
1334	struct map *map = NULL;
1335	struct dso *dso = dso__new(name);
1336
1337	if (dso) {
1338		map = map__new2(0, dso);
1339		dso__put(dso);
1340	}
1341
1342	return map;
1343}
1344
1345struct dso *machine__findnew_kernel(struct machine *machine, const char *name,
1346				    const char *short_name, int dso_type)
1347{
1348	/*
1349	 * The kernel dso could be created by build_id processing.
1350	 */
1351	struct dso *dso = machine__findnew_dso(machine, name);
1352
1353	/*
1354	 * We need to run this in all cases, since during the build_id
1355	 * processing we had no idea this was the kernel dso.
1356	 */
1357	if (dso != NULL) {
1358		dso__set_short_name(dso, short_name, false);
1359		dso__set_kernel(dso, dso_type);
1360	}
1361
1362	return dso;
1363}
1364
1365static void __dso__set_long_name_id(struct dso *dso, const char *name, bool name_allocated)
1366{
1367	if (dso__long_name_allocated(dso))
1368		free((char *)dso__long_name(dso));
1369
1370	RC_CHK_ACCESS(dso)->long_name = name;
1371	RC_CHK_ACCESS(dso)->long_name_len = strlen(name);
1372	dso__set_long_name_allocated(dso, name_allocated);
1373}
1374
1375static void dso__set_long_name_id(struct dso *dso, const char *name, bool name_allocated)
1376{
1377	struct dsos *dsos = dso__dsos(dso);
1378
1379	if (name == NULL)
1380		return;
1381
1382	if (dsos) {
1383		/*
1384		 * Need to avoid re-sorting the dsos breaking by non-atomically
1385		 * renaming the dso.
1386		 */
1387		down_write(&dsos->lock);
1388		__dso__set_long_name_id(dso, name, name_allocated);
1389		dsos->sorted = false;
1390		up_write(&dsos->lock);
1391	} else {
1392		__dso__set_long_name_id(dso, name, name_allocated);
1393	}
1394}
1395
1396static int __dso_id__cmp(const struct dso_id *a, const struct dso_id *b)
1397{
1398	if (a->mmap2_valid && b->mmap2_valid) {
1399		if (a->maj > b->maj) return -1;
1400		if (a->maj < b->maj) return 1;
1401
1402		if (a->min > b->min) return -1;
1403		if (a->min < b->min) return 1;
1404
1405		if (a->ino > b->ino) return -1;
1406		if (a->ino < b->ino) return 1;
1407	}
1408	if (a->mmap2_ino_generation_valid && b->mmap2_ino_generation_valid) {
1409		if (a->ino_generation > b->ino_generation) return -1;
1410		if (a->ino_generation < b->ino_generation) return 1;
1411	}
1412	if (build_id__is_defined(&a->build_id) && build_id__is_defined(&b->build_id)) {
1413		if (a->build_id.size != b->build_id.size)
1414			return a->build_id.size < b->build_id.size ? -1 : 1;
1415		return memcmp(a->build_id.data, b->build_id.data, a->build_id.size);
1416	}
1417	return 0;
1418}
1419
1420const struct dso_id dso_id_empty = {
1421	{
1422		.maj = 0,
1423		.min = 0,
1424		.ino = 0,
1425		.ino_generation = 0,
1426	},
1427	.mmap2_valid = false,
1428	.mmap2_ino_generation_valid = false,
1429	{
1430		.size = 0,
1431	}
1432};
1433
1434void __dso__improve_id(struct dso *dso, const struct dso_id *id)
1435{
1436	struct dsos *dsos = dso__dsos(dso);
1437	struct dso_id *dso_id = dso__id(dso);
1438	bool changed = false;
1439
1440	/* dsos write lock held by caller. */
1441
1442	if (id->mmap2_valid && !dso_id->mmap2_valid) {
1443		dso_id->maj = id->maj;
1444		dso_id->min = id->min;
1445		dso_id->ino = id->ino;
1446		dso_id->mmap2_valid = true;
1447		changed = true;
1448	}
1449	if (id->mmap2_ino_generation_valid && !dso_id->mmap2_ino_generation_valid) {
1450		dso_id->ino_generation = id->ino_generation;
1451		dso_id->mmap2_ino_generation_valid = true;
1452		changed = true;
1453	}
1454	if (build_id__is_defined(&id->build_id) && !build_id__is_defined(&dso_id->build_id)) {
1455		dso_id->build_id = id->build_id;
1456		changed = true;
1457	}
1458	if (changed && dsos)
1459		dsos->sorted = false;
1460}
1461
1462int dso_id__cmp(const struct dso_id *a, const struct dso_id *b)
1463{
1464	if (a == &dso_id_empty || b == &dso_id_empty) {
1465		/* There is no valid data to compare so the comparison always returns identical. */
1466		return 0;
1467	}
1468
1469	return __dso_id__cmp(a, b);
1470}
1471
1472int dso__cmp_id(struct dso *a, struct dso *b)
1473{
1474	return __dso_id__cmp(dso__id(a), dso__id(b));
1475}
1476
1477void dso__set_long_name(struct dso *dso, const char *name, bool name_allocated)
1478{
1479	dso__set_long_name_id(dso, name, name_allocated);
1480}
1481
1482static void __dso__set_short_name(struct dso *dso, const char *name, bool name_allocated)
1483{
1484	if (dso__short_name_allocated(dso))
1485		free((char *)dso__short_name(dso));
1486
1487	RC_CHK_ACCESS(dso)->short_name		  = name;
1488	RC_CHK_ACCESS(dso)->short_name_len	  = strlen(name);
1489	dso__set_short_name_allocated(dso, name_allocated);
1490}
1491
1492void dso__set_short_name(struct dso *dso, const char *name, bool name_allocated)
1493{
1494	struct dsos *dsos = dso__dsos(dso);
1495
1496	if (name == NULL)
1497		return;
1498
1499	if (dsos) {
1500		/*
1501		 * Need to avoid re-sorting the dsos breaking by non-atomically
1502		 * renaming the dso.
1503		 */
1504		down_write(&dsos->lock);
1505		__dso__set_short_name(dso, name, name_allocated);
1506		dsos->sorted = false;
1507		up_write(&dsos->lock);
1508	} else {
1509		__dso__set_short_name(dso, name, name_allocated);
1510	}
1511}
1512
1513int dso__name_len(const struct dso *dso)
1514{
1515	if (!dso)
1516		return strlen("[unknown]");
1517	if (verbose > 0)
1518		return dso__long_name_len(dso);
1519
1520	return dso__short_name_len(dso);
1521}
1522
1523bool dso__loaded(const struct dso *dso)
1524{
1525	return RC_CHK_ACCESS(dso)->loaded;
1526}
1527
1528bool dso__sorted_by_name(const struct dso *dso)
1529{
1530	return RC_CHK_ACCESS(dso)->sorted_by_name;
1531}
1532
1533void dso__set_sorted_by_name(struct dso *dso)
1534{
1535	RC_CHK_ACCESS(dso)->sorted_by_name = true;
1536}
1537
1538struct dso *dso__new_id(const char *name, const struct dso_id *id)
1539{
1540	RC_STRUCT(dso) *dso = zalloc(sizeof(*dso) + strlen(name) + 1);
1541	struct dso *res;
1542	struct dso_data *data;
1543
1544	if (!dso)
1545		return NULL;
1546
1547	if (ADD_RC_CHK(res, dso)) {
1548		strcpy(dso->name, name);
1549		if (id)
1550			dso->id = *id;
1551		dso__set_long_name_id(res, dso->name, false);
1552		dso__set_short_name(res, dso->name, false);
1553		dso->symbols = RB_ROOT_CACHED;
1554		dso->symbol_names = NULL;
1555		dso->symbol_names_len = 0;
1556		dso->inlined_nodes = RB_ROOT_CACHED;
1557		dso->srclines = RB_ROOT_CACHED;
1558		dso->data_types = RB_ROOT;
1559		dso->global_vars = RB_ROOT;
1560		dso->data.fd = -1;
1561		dso->data.status = DSO_DATA_STATUS_UNKNOWN;
1562		dso->symtab_type = DSO_BINARY_TYPE__NOT_FOUND;
1563		dso->binary_type = DSO_BINARY_TYPE__NOT_FOUND;
1564		dso->is_64_bit = (sizeof(void *) == 8);
1565		dso->loaded = 0;
1566		dso->rel = 0;
1567		dso->sorted_by_name = 0;
1568		dso->has_srcline = 1;
1569		dso->a2l_fails = 1;
1570		dso->kernel = DSO_SPACE__USER;
1571		dso->is_kmod = 0;
1572		dso->needs_swap = DSO_SWAP__UNSET;
1573		dso->comp = COMP_ID__NONE;
1574		mutex_init(&dso->lock);
1575		refcount_set(&dso->refcnt, 1);
1576		data = &dso->data;
1577		data->cache = RB_ROOT;
1578		data->fd = -1;
1579		data->status = DSO_DATA_STATUS_UNKNOWN;
1580		INIT_LIST_HEAD(&data->open_entry);
1581#ifdef REFCNT_CHECKING
1582		data->dso = NULL; /* Set when on the open_entry list. */
1583#endif
1584	}
1585	return res;
1586}
1587
1588struct dso *dso__new(const char *name)
1589{
1590	return dso__new_id(name, NULL);
1591}
1592
1593void dso__delete(struct dso *dso)
1594{
1595	if (dso__dsos(dso))
1596		pr_err("DSO %s is still in rbtree when being deleted!\n", dso__long_name(dso));
1597
1598	/* free inlines first, as they reference symbols */
1599	inlines__tree_delete(&RC_CHK_ACCESS(dso)->inlined_nodes);
1600	srcline__tree_delete(&RC_CHK_ACCESS(dso)->srclines);
1601	symbols__delete(&RC_CHK_ACCESS(dso)->symbols);
1602	RC_CHK_ACCESS(dso)->symbol_names_len = 0;
1603	zfree(&RC_CHK_ACCESS(dso)->symbol_names);
1604	annotated_data_type__tree_delete(dso__data_types(dso));
1605	global_var_type__tree_delete(dso__global_vars(dso));
1606
1607	if (RC_CHK_ACCESS(dso)->short_name_allocated) {
1608		zfree((char **)&RC_CHK_ACCESS(dso)->short_name);
1609		RC_CHK_ACCESS(dso)->short_name_allocated = false;
1610	}
1611
1612	if (RC_CHK_ACCESS(dso)->long_name_allocated) {
1613		zfree((char **)&RC_CHK_ACCESS(dso)->long_name);
1614		RC_CHK_ACCESS(dso)->long_name_allocated = false;
1615	}
1616
1617	dso__data_close(dso);
1618	auxtrace_cache__free(RC_CHK_ACCESS(dso)->auxtrace_cache);
1619	dso_cache__free(dso);
1620	dso__free_a2l(dso);
1621	dso__free_symsrc_filename(dso);
1622	nsinfo__zput(RC_CHK_ACCESS(dso)->nsinfo);
1623	mutex_destroy(dso__lock(dso));
1624	RC_CHK_FREE(dso);
1625}
1626
1627struct dso *dso__get(struct dso *dso)
1628{
1629	struct dso *result;
1630
1631	if (RC_CHK_GET(result, dso))
1632		refcount_inc(&RC_CHK_ACCESS(dso)->refcnt);
1633
1634	return result;
1635}
1636
1637void dso__put(struct dso *dso)
1638{
1639#ifdef REFCNT_CHECKING
1640	if (dso && dso__data(dso) && refcount_read(&RC_CHK_ACCESS(dso)->refcnt) == 2)
1641		dso__data_close(dso);
1642#endif
1643	if (dso && refcount_dec_and_test(&RC_CHK_ACCESS(dso)->refcnt))
1644		dso__delete(dso);
1645	else
1646		RC_CHK_PUT(dso);
1647}
1648
1649int dso__swap_init(struct dso *dso, unsigned char eidata)
1650{
1651	static unsigned int const endian = 1;
1652
1653	dso__set_needs_swap(dso, DSO_SWAP__NO);
1654
1655	switch (eidata) {
1656	case ELFDATA2LSB:
1657		/* We are big endian, DSO is little endian. */
1658		if (*(unsigned char const *)&endian != 1)
1659			dso__set_needs_swap(dso, DSO_SWAP__YES);
1660		break;
1661
1662	case ELFDATA2MSB:
1663		/* We are little endian, DSO is big endian. */
1664		if (*(unsigned char const *)&endian != 0)
1665			dso__set_needs_swap(dso, DSO_SWAP__YES);
1666		break;
1667
1668	default:
1669		pr_err("unrecognized DSO data encoding %d\n", eidata);
1670		return -EINVAL;
1671	}
1672
1673	return 0;
1674}
1675
1676void dso__set_build_id(struct dso *dso, const struct build_id *bid)
1677{
1678	dso__id(dso)->build_id = *bid;
1679}
1680
1681bool dso__build_id_equal(const struct dso *dso, const struct build_id *bid)
1682{
1683	const struct build_id *dso_bid = dso__bid(dso);
1684
1685	if (dso_bid->size > bid->size && dso_bid->size == BUILD_ID_SIZE) {
1686		/*
1687		 * For the backward compatibility, it allows a build-id has
1688		 * trailing zeros.
1689		 */
1690		return !memcmp(dso_bid->data, bid->data, bid->size) &&
1691			!memchr_inv(&dso_bid->data[bid->size], 0,
1692				    dso_bid->size - bid->size);
1693	}
1694
1695	return dso_bid->size == bid->size &&
1696	       memcmp(dso_bid->data, bid->data, dso_bid->size) == 0;
1697}
1698
1699void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine)
1700{
1701	char path[PATH_MAX];
1702	struct build_id bid = { .size = 0, };
1703
1704	if (machine__is_default_guest(machine))
1705		return;
1706	sprintf(path, "%s/sys/kernel/notes", machine->root_dir);
1707	sysfs__read_build_id(path, &bid);
1708	dso__set_build_id(dso, &bid);
1709}
1710
1711int dso__kernel_module_get_build_id(struct dso *dso,
1712				    const char *root_dir)
1713{
1714	char filename[PATH_MAX];
1715	struct build_id bid = { .size = 0, };
1716	/*
1717	 * kernel module short names are of the form "[module]" and
1718	 * we need just "module" here.
1719	 */
1720	const char *name = dso__short_name(dso) + 1;
1721
1722	snprintf(filename, sizeof(filename),
1723		 "%s/sys/module/%.*s/notes/.note.gnu.build-id",
1724		 root_dir, (int)strlen(name) - 1, name);
1725
1726	sysfs__read_build_id(filename, &bid);
1727	dso__set_build_id(dso, &bid);
1728	return 0;
1729}
1730
1731static size_t dso__fprintf_buildid(struct dso *dso, FILE *fp)
1732{
1733	char sbuild_id[SBUILD_ID_SIZE];
1734
1735	build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id));
1736	return fprintf(fp, "%s", sbuild_id);
1737}
1738
1739size_t dso__fprintf(struct dso *dso, FILE *fp)
1740{
1741	struct rb_node *nd;
1742	size_t ret = fprintf(fp, "dso: %s (", dso__short_name(dso));
1743
1744	if (dso__short_name(dso) != dso__long_name(dso))
1745		ret += fprintf(fp, "%s, ", dso__long_name(dso));
1746	ret += fprintf(fp, "%sloaded, ", dso__loaded(dso) ? "" : "NOT ");
1747	ret += dso__fprintf_buildid(dso, fp);
1748	ret += fprintf(fp, ")\n");
1749	for (nd = rb_first_cached(dso__symbols(dso)); nd; nd = rb_next(nd)) {
1750		struct symbol *pos = rb_entry(nd, struct symbol, rb_node);
1751		ret += symbol__fprintf(pos, fp);
1752	}
1753
1754	return ret;
1755}
1756
1757enum dso_type dso__type(struct dso *dso, struct machine *machine)
1758{
1759	int fd = -1;
1760	enum dso_type type = DSO__TYPE_UNKNOWN;
1761
1762	if (dso__data_get_fd(dso, machine, &fd)) {
1763		type = dso__type_fd(fd);
1764		dso__data_put_fd(dso);
1765	}
1766
1767	return type;
1768}
1769
1770int dso__strerror_load(struct dso *dso, char *buf, size_t buflen)
1771{
1772	int idx, errnum = *dso__load_errno(dso);
1773	/*
1774	 * This must have a same ordering as the enum dso_load_errno.
1775	 */
1776	static const char *dso_load__error_str[] = {
1777	"Internal tools/perf/ library error",
1778	"Invalid ELF file",
1779	"Can not read build id",
1780	"Mismatching build id",
1781	"Decompression failure",
1782	};
1783
1784	BUG_ON(buflen == 0);
1785
1786	if (errnum >= 0) {
1787		const char *err = str_error_r(errnum, buf, buflen);
1788
1789		if (err != buf)
1790			scnprintf(buf, buflen, "%s", err);
1791
1792		return 0;
1793	}
1794
1795	if (errnum <  __DSO_LOAD_ERRNO__START || errnum >= __DSO_LOAD_ERRNO__END)
1796		return -1;
1797
1798	idx = errnum - __DSO_LOAD_ERRNO__START;
1799	scnprintf(buf, buflen, "%s", dso_load__error_str[idx]);
1800	return 0;
1801}
1802
1803bool perf_pid_map_tid(const char *dso_name, int *tid)
1804{
1805	return sscanf(dso_name, "/tmp/perf-%d.map", tid) == 1;
1806}
1807
1808bool is_perf_pid_map_name(const char *dso_name)
1809{
1810	int tid;
1811
1812	return perf_pid_map_tid(dso_name, &tid);
1813}
1814
1815struct find_file_offset_data {
1816	u64 ip;
1817	u64 offset;
1818};
1819
1820/* This will be called for each PHDR in an ELF binary */
1821static int find_file_offset(u64 start, u64 len, u64 pgoff, void *arg)
1822{
1823	struct find_file_offset_data *data = arg;
1824
1825	if (start <= data->ip && data->ip < start + len) {
1826		data->offset = pgoff + data->ip - start;
1827		return 1;
1828	}
1829	return 0;
1830}
1831
1832static const u8 *__dso__read_symbol(struct dso *dso, const char *symfs_filename,
1833				    u64 start, size_t len,
1834				    u8 **out_buf, u64 *out_buf_len, bool *is_64bit)
1835{
1836	struct nscookie nsc;
1837	int fd;
1838	ssize_t count;
1839	struct find_file_offset_data data = {
1840		.ip = start,
1841	};
1842	u8 *code_buf = NULL;
1843	int saved_errno;
1844
1845	nsinfo__mountns_enter(dso__nsinfo(dso), &nsc);
1846	fd = open(symfs_filename, O_RDONLY);
1847	saved_errno = errno;
1848	nsinfo__mountns_exit(&nsc);
1849	if (fd < 0) {
1850		errno = saved_errno;
1851		return NULL;
1852	}
1853	if (file__read_maps(fd, /*exe=*/true, find_file_offset, &data, is_64bit) <= 0) {
1854		close(fd);
1855		errno = ENOENT;
1856		return NULL;
1857	}
1858	code_buf = malloc(len);
1859	if (code_buf == NULL) {
1860		close(fd);
1861		errno = ENOMEM;
1862		return NULL;
1863	}
1864	count = pread(fd, code_buf, len, data.offset);
1865	saved_errno = errno;
1866	close(fd);
1867	if ((u64)count != len) {
1868		free(code_buf);
1869		errno = saved_errno;
1870		return NULL;
1871	}
1872	*out_buf = code_buf;
1873	*out_buf_len = len;
1874	return code_buf;
1875}
1876
1877/*
1878 * Read a symbol into memory for disassembly by a library like capstone of
1879 * libLLVM. If memory is allocated out_buf holds it.
1880 */
1881const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename,
1882			   const struct map *map, const struct symbol *sym,
1883			   u8 **out_buf, u64 *out_buf_len, bool *is_64bit)
1884{
1885	u64 start = map__rip_2objdump(map, sym->start);
1886	u64 end = map__rip_2objdump(map, sym->end);
1887	size_t len = end - start;
1888
1889	*out_buf = NULL;
1890	*out_buf_len = 0;
1891	*is_64bit = false;
1892
1893	if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_IMAGE) {
1894		/*
1895		 * Note, there is fallback BPF image disassembly in the objdump
1896		 * version but it currently does nothing.
1897		 */
1898		errno = EOPNOTSUPP;
1899		return NULL;
1900	}
1901	if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) {
1902#ifdef HAVE_LIBBPF_SUPPORT
1903		struct bpf_prog_info_node *info_node;
1904		struct perf_bpil *info_linear;
1905
1906		*is_64bit = sizeof(void *) == sizeof(u64);
1907		info_node = perf_env__find_bpf_prog_info(dso__bpf_prog(dso)->env,
1908							 dso__bpf_prog(dso)->id);
1909		if (!info_node) {
1910			errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF;
1911			return NULL;
1912		}
1913		info_linear = info_node->info_linear;
1914		assert(len <= info_linear->info.jited_prog_len);
1915		*out_buf_len = len;
1916		return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns);
1917#else
1918		pr_debug("No BPF program disassembly support\n");
1919		errno = EOPNOTSUPP;
1920		return NULL;
1921#endif
1922	}
1923	return __dso__read_symbol(dso, symfs_filename, start, len,
1924				  out_buf, out_buf_len, is_64bit);
1925}
1926
1927struct debuginfo *dso__debuginfo(struct dso *dso)
1928{
1929	char *name;
1930	bool decomp = false;
1931	struct debuginfo *dinfo = NULL;
1932
1933	mutex_lock(dso__lock(dso));
1934
1935	name = dso__get_filename(dso, "", &decomp);
1936	if (name)
1937		dinfo = debuginfo__new(name);
1938
1939	if (decomp)
1940		unlink(name);
1941
1942	mutex_unlock(dso__lock(dso));
1943	free(name);
1944	return dinfo;
1945}