module.c 110 KB
Newer Older
1
/*
Linus Torvalds's avatar
Linus Torvalds committed
2
   Copyright (C) 2002 Richard Henderson
3
   Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
Linus Torvalds's avatar
Linus Torvalds committed
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/
19
#include <linux/export.h>
20
#include <linux/extable.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
#include <linux/moduleloader.h>
22
#include <linux/trace_events.h>
Linus Torvalds's avatar
Linus Torvalds committed
23
#include <linux/init.h>
24
#include <linux/kallsyms.h>
25
#include <linux/file.h>
26
#include <linux/fs.h>
Roland McGrath's avatar
Roland McGrath committed
27
#include <linux/sysfs.h>
28
#include <linux/kernel.h>
Linus Torvalds's avatar
Linus Torvalds committed
29 30 31
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/elf.h>
32
#include <linux/proc_fs.h>
33
#include <linux/security.h>
Linus Torvalds's avatar
Linus Torvalds committed
34 35 36 37
#include <linux/seq_file.h>
#include <linux/syscalls.h>
#include <linux/fcntl.h>
#include <linux/rcupdate.h>
38
#include <linux/capability.h>
Linus Torvalds's avatar
Linus Torvalds committed
39 40 41 42 43 44
#include <linux/cpu.h>
#include <linux/moduleparam.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/vermagic.h>
#include <linux/notifier.h>
Al Viro's avatar
Al Viro committed
45
#include <linux/sched.h>
Linus Torvalds's avatar
Linus Torvalds committed
46
#include <linux/device.h>
47
#include <linux/string.h>
48
#include <linux/mutex.h>
49
#include <linux/rculist.h>
50
#include <linux/uaccess.h>
Linus Torvalds's avatar
Linus Torvalds committed
51
#include <asm/cacheflush.h>
52
#include <linux/set_memory.h>
53
#include <asm/mmu_context.h>
54
#include <linux/license.h>
55
#include <asm/sections.h>
56
#include <linux/tracepoint.h>
57
#include <linux/ftrace.h>
58
#include <linux/livepatch.h>
59
#include <linux/async.h>
60
#include <linux/percpu.h>
61
#include <linux/kmemleak.h>
62
#include <linux/jump_label.h>
63
#include <linux/pfn.h>
64
#include <linux/bsearch.h>
65
#include <linux/dynamic_debug.h>
66
#include <linux/audit.h>
67
#include <uapi/linux/module.h>
68
#include "module-internal.h"
Linus Torvalds's avatar
Linus Torvalds committed
69

70 71 72
#define CREATE_TRACE_POINTS
#include <trace/events/module.h>

Linus Torvalds's avatar
Linus Torvalds committed
73 74 75 76
#ifndef ARCH_SHF_SMALL
#define ARCH_SHF_SMALL 0
#endif

77 78 79
/*
 * Modules' sections will be aligned on page boundaries
 * to ensure complete separation of code and data, but
80
 * only when CONFIG_STRICT_MODULE_RWX=y
81
 */
82
#ifdef CONFIG_STRICT_MODULE_RWX
83 84 85 86 87
# define debug_align(X) ALIGN(X, PAGE_SIZE)
#else
# define debug_align(X) (X)
#endif

Linus Torvalds's avatar
Linus Torvalds committed
88 89 90
/* If this is set, the section belongs in the init part of the module */
#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))

91 92 93 94 95
/*
 * Mutex protects:
 * 1) List of modules (also safely readable with preempt_disable),
 * 2) module_use links,
 * 3) module_addr_min/module_addr_max.
96
 * (delete and add uses RCU list operations). */
97 98
DEFINE_MUTEX(module_mutex);
EXPORT_SYMBOL_GPL(module_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
99
static LIST_HEAD(modules);
100

101
#ifdef CONFIG_MODULES_TREE_LOOKUP
102

103 104 105 106
/*
 * Use a latched RB-tree for __module_address(); this allows us to use
 * RCU-sched lookups of the address from any context.
 *
107 108 109
 * This is conditional on PERF_EVENTS || TRACING because those can really hit
 * __module_address() hard by doing a lot of stack unwinding; potentially from
 * NMI context.
110 111 112
 */

static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)
113
{
114
	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
115

116
	return (unsigned long)layout->base;
117 118 119 120
}

static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)
{
121
	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
122

123
	return (unsigned long)layout->size;
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
}

static __always_inline bool
mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b)
{
	return __mod_tree_val(a) < __mod_tree_val(b);
}

static __always_inline int
mod_tree_comp(void *key, struct latch_tree_node *n)
{
	unsigned long val = (unsigned long)key;
	unsigned long start, end;

	start = __mod_tree_val(n);
	if (val < start)
		return -1;

	end = start + __mod_tree_size(n);
	if (val >= end)
		return 1;
145 146 147 148

	return 0;
}

149 150 151 152 153
static const struct latch_tree_ops mod_tree_ops = {
	.less = mod_tree_less,
	.comp = mod_tree_comp,
};

154 155 156 157 158 159
static struct mod_tree_root {
	struct latch_tree_root root;
	unsigned long addr_min;
	unsigned long addr_max;
} mod_tree __cacheline_aligned = {
	.addr_min = -1UL,
160 161
};

162 163 164 165 166 167 168 169 170 171 172 173
#define module_addr_min mod_tree.addr_min
#define module_addr_max mod_tree.addr_max

static noinline void __mod_tree_insert(struct mod_tree_node *node)
{
	latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops);
}

static void __mod_tree_remove(struct mod_tree_node *node)
{
	latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops);
}
174 175 176 177 178 179 180

/*
 * These modifications: insert, remove_init and remove; are serialized by the
 * module_mutex.
 */
static void mod_tree_insert(struct module *mod)
{
181 182
	mod->core_layout.mtn.mod = mod;
	mod->init_layout.mtn.mod = mod;
183

184 185 186
	__mod_tree_insert(&mod->core_layout.mtn);
	if (mod->init_layout.size)
		__mod_tree_insert(&mod->init_layout.mtn);
187 188 189 190
}

static void mod_tree_remove_init(struct module *mod)
{
191 192
	if (mod->init_layout.size)
		__mod_tree_remove(&mod->init_layout.mtn);
193 194 195 196
}

static void mod_tree_remove(struct module *mod)
{
197
	__mod_tree_remove(&mod->core_layout.mtn);
198 199 200
	mod_tree_remove_init(mod);
}

201
static struct module *mod_find(unsigned long addr)
202 203 204
{
	struct latch_tree_node *ltn;

205
	ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops);
206 207 208 209 210 211
	if (!ltn)
		return NULL;

	return container_of(ltn, struct mod_tree_node, node)->mod;
}

212 213
#else /* MODULES_TREE_LOOKUP */

214 215
static unsigned long module_addr_min = -1UL, module_addr_max = 0;

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
static void mod_tree_insert(struct module *mod) { }
static void mod_tree_remove_init(struct module *mod) { }
static void mod_tree_remove(struct module *mod) { }

static struct module *mod_find(unsigned long addr)
{
	struct module *mod;

	list_for_each_entry_rcu(mod, &modules, list) {
		if (within_module(addr, mod))
			return mod;
	}

	return NULL;
}

#endif /* MODULES_TREE_LOOKUP */

234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
/*
 * Bounds of module text, for speeding up __module_address.
 * Protected by module_mutex.
 */
static void __mod_update_bounds(void *base, unsigned int size)
{
	unsigned long min = (unsigned long)base;
	unsigned long max = min + size;

	if (min < module_addr_min)
		module_addr_min = min;
	if (max > module_addr_max)
		module_addr_max = max;
}

static void mod_update_bounds(struct module *mod)
{
251 252 253
	__mod_update_bounds(mod->core_layout.base, mod->core_layout.size);
	if (mod->init_layout.size)
		__mod_update_bounds(mod->init_layout.base, mod->init_layout.size);
254 255
}

256 257 258 259
#ifdef CONFIG_KGDB_KDB
struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
#endif /* CONFIG_KGDB_KDB */

260 261 262 263 264 265 266 267 268 269 270
static void module_assert_mutex(void)
{
	lockdep_assert_held(&module_mutex);
}

static void module_assert_mutex_or_preempt(void)
{
#ifdef CONFIG_LOCKDEP
	if (unlikely(!debug_locks))
		return;

271
	WARN_ON_ONCE(!rcu_read_lock_sched_held() &&
272 273 274 275
		!lockdep_is_held(&module_mutex));
#endif
}

276 277
static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);
#ifndef CONFIG_MODULE_SIG_FORCE
278 279
module_param(sig_enforce, bool_enable_only, 0644);
#endif /* !CONFIG_MODULE_SIG_FORCE */
Linus Torvalds's avatar
Linus Torvalds committed
280

281 282
/* Block module loading/unloading? */
int modules_disabled = 0;
283
core_param(nomodule, modules_disabled, bint, 0);
284

285 286 287
/* Waiting for a module to finish initializing? */
static DECLARE_WAIT_QUEUE_HEAD(module_wq);

288
static BLOCKING_NOTIFIER_HEAD(module_notify_list);
Linus Torvalds's avatar
Linus Torvalds committed
289

290
int register_module_notifier(struct notifier_block *nb)
Linus Torvalds's avatar
Linus Torvalds committed
291
{
292
	return blocking_notifier_chain_register(&module_notify_list, nb);
Linus Torvalds's avatar
Linus Torvalds committed
293 294 295
}
EXPORT_SYMBOL(register_module_notifier);

296
int unregister_module_notifier(struct notifier_block *nb)
Linus Torvalds's avatar
Linus Torvalds committed
297
{
298
	return blocking_notifier_chain_unregister(&module_notify_list, nb);
Linus Torvalds's avatar
Linus Torvalds committed
299 300 301
}
EXPORT_SYMBOL(unregister_module_notifier);

302
struct load_info {
303
	const char *name;
304 305 306
	Elf_Ehdr *hdr;
	unsigned long len;
	Elf_Shdr *sechdrs;
307
	char *secstrings, *strtab;
Rusty Russell's avatar
Rusty Russell committed
308
	unsigned long symoffs, stroffs;
309 310
	struct _ddebug *debug;
	unsigned int num_debug;
311
	bool sig_ok;
312 313 314
#ifdef CONFIG_KALLSYMS
	unsigned long mod_kallsyms_init_off;
#endif
315 316 317 318 319
	struct {
		unsigned int sym, str, mod, vers, info, pcpu;
	} index;
};

320 321 322 323 324
/*
 * We require a truly strong try_module_get(): 0 means success.
 * Otherwise an error is returned due to ongoing or failed
 * initialization etc.
 */
Linus Torvalds's avatar
Linus Torvalds committed
325 326
static inline int strong_try_module_get(struct module *mod)
{
327
	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
Linus Torvalds's avatar
Linus Torvalds committed
328
	if (mod && mod->state == MODULE_STATE_COMING)
329 330
		return -EBUSY;
	if (try_module_get(mod))
Linus Torvalds's avatar
Linus Torvalds committed
331
		return 0;
332 333
	else
		return -ENOENT;
Linus Torvalds's avatar
Linus Torvalds committed
334 335
}

336 337
static inline void add_taint_module(struct module *mod, unsigned flag,
				    enum lockdep_ok lockdep_ok)
338
{
339
	add_taint(flag, lockdep_ok);
340
	set_bit(flag, &mod->taints);
341 342
}

343 344 345
/*
 * A thread that wants to hold a reference to a module only while it
 * is running can call this to safely exit.  nfsd and lockd use this.
Linus Torvalds's avatar
Linus Torvalds committed
346
 */
347
void __noreturn __module_put_and_exit(struct module *mod, long code)
Linus Torvalds's avatar
Linus Torvalds committed
348 349 350 351 352
{
	module_put(mod);
	do_exit(code);
}
EXPORT_SYMBOL(__module_put_and_exit);
353

Linus Torvalds's avatar
Linus Torvalds committed
354
/* Find a module section: 0 means not found. */
355
static unsigned int find_sec(const struct load_info *info, const char *name)
Linus Torvalds's avatar
Linus Torvalds committed
356 357 358
{
	unsigned int i;

359 360
	for (i = 1; i < info->hdr->e_shnum; i++) {
		Elf_Shdr *shdr = &info->sechdrs[i];
Linus Torvalds's avatar
Linus Torvalds committed
361
		/* Alloc bit cleared means "ignore it." */
362 363
		if ((shdr->sh_flags & SHF_ALLOC)
		    && strcmp(info->secstrings + shdr->sh_name, name) == 0)
Linus Torvalds's avatar
Linus Torvalds committed
364
			return i;
365
	}
Linus Torvalds's avatar
Linus Torvalds committed
366 367 368
	return 0;
}

369
/* Find a module section, or NULL. */
370
static void *section_addr(const struct load_info *info, const char *name)
371 372
{
	/* Section 0 has sh_addr 0. */
373
	return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
374 375 376
}

/* Find a module section, or NULL.  Fill in number of "objects" in section. */
377
static void *section_objs(const struct load_info *info,
378 379 380 381
			  const char *name,
			  size_t object_size,
			  unsigned int *num)
{
382
	unsigned int sec = find_sec(info, name);
383 384

	/* Section 0 has sh_addr 0 and sh_size 0. */
385 386
	*num = info->sechdrs[sec].sh_size / object_size;
	return (void *)info->sechdrs[sec].sh_addr;
387 388
}

Linus Torvalds's avatar
Linus Torvalds committed
389 390 391 392 393
/* Provided by the linker */
extern const struct kernel_symbol __start___ksymtab[];
extern const struct kernel_symbol __stop___ksymtab[];
extern const struct kernel_symbol __start___ksymtab_gpl[];
extern const struct kernel_symbol __stop___ksymtab_gpl[];
394 395
extern const struct kernel_symbol __start___ksymtab_gpl_future[];
extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
396 397 398
extern const s32 __start___kcrctab[];
extern const s32 __start___kcrctab_gpl[];
extern const s32 __start___kcrctab_gpl_future[];
399 400 401 402 403
#ifdef CONFIG_UNUSED_SYMBOLS
extern const struct kernel_symbol __start___ksymtab_unused[];
extern const struct kernel_symbol __stop___ksymtab_unused[];
extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
404 405
extern const s32 __start___kcrctab_unused[];
extern const s32 __start___kcrctab_unused_gpl[];
406
#endif
Linus Torvalds's avatar
Linus Torvalds committed
407 408 409 410

#ifndef CONFIG_MODVERSIONS
#define symversion(base, idx) NULL
#else
411
#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
Linus Torvalds's avatar
Linus Torvalds committed
412 413
#endif

414 415 416 417 418
static bool each_symbol_in_section(const struct symsearch *arr,
				   unsigned int arrsize,
				   struct module *owner,
				   bool (*fn)(const struct symsearch *syms,
					      struct module *owner,
419
					      void *data),
420
				   void *data)
421
{
422
	unsigned int j;
423

424
	for (j = 0; j < arrsize; j++) {
425 426
		if (fn(&arr[j], owner, data))
			return true;
427
	}
428 429

	return false;
430 431
}

432
/* Returns true as soon as fn returns true, otherwise false. */
433 434 435 436
bool each_symbol_section(bool (*fn)(const struct symsearch *arr,
				    struct module *owner,
				    void *data),
			 void *data)
437 438
{
	struct module *mod;
439
	static const struct symsearch arr[] = {
440
		{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
441
		  NOT_GPL_ONLY, false },
442
		{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
443 444
		  __start___kcrctab_gpl,
		  GPL_ONLY, false },
445
		{ __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future,
446 447
		  __start___kcrctab_gpl_future,
		  WILL_BE_GPL_ONLY, false },
448
#ifdef CONFIG_UNUSED_SYMBOLS
449
		{ __start___ksymtab_unused, __stop___ksymtab_unused,
450 451
		  __start___kcrctab_unused,
		  NOT_GPL_ONLY, true },
452
		{ __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl,
453 454
		  __start___kcrctab_unused_gpl,
		  GPL_ONLY, true },
455
#endif
456
	};
457

458 459
	module_assert_mutex_or_preempt();

460 461
	if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
		return true;
462

463
	list_for_each_entry_rcu(mod, &modules, list) {
464 465
		struct symsearch arr[] = {
			{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
466
			  NOT_GPL_ONLY, false },
467
			{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
468 469
			  mod->gpl_crcs,
			  GPL_ONLY, false },
470 471
			{ mod->gpl_future_syms,
			  mod->gpl_future_syms + mod->num_gpl_future_syms,
472 473
			  mod->gpl_future_crcs,
			  WILL_BE_GPL_ONLY, false },
474
#ifdef CONFIG_UNUSED_SYMBOLS
475 476
			{ mod->unused_syms,
			  mod->unused_syms + mod->num_unused_syms,
477 478
			  mod->unused_crcs,
			  NOT_GPL_ONLY, true },
479 480
			{ mod->unused_gpl_syms,
			  mod->unused_gpl_syms + mod->num_unused_gpl_syms,
481 482
			  mod->unused_gpl_crcs,
			  GPL_ONLY, true },
483
#endif
484 485
		};

486 487 488
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;

489 490 491 492 493
		if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
			return true;
	}
	return false;
}
494
EXPORT_SYMBOL_GPL(each_symbol_section);
495 496 497 498 499 500 501 502 503

struct find_symbol_arg {
	/* Input */
	const char *name;
	bool gplok;
	bool warn;

	/* Output */
	struct module *owner;
504
	const s32 *crc;
505
	const struct kernel_symbol *sym;
506 507
};

508 509 510
static bool check_symbol(const struct symsearch *syms,
				 struct module *owner,
				 unsigned int symnum, void *data)
511 512 513 514 515 516 517
{
	struct find_symbol_arg *fsa = data;

	if (!fsa->gplok) {
		if (syms->licence == GPL_ONLY)
			return false;
		if (syms->licence == WILL_BE_GPL_ONLY && fsa->warn) {
518 519 520
			pr_warn("Symbol %s is being used by a non-GPL module, "
				"which will not be allowed in the future\n",
				fsa->name);
521
		}
Linus Torvalds's avatar
Linus Torvalds committed
522
	}
523

524
#ifdef CONFIG_UNUSED_SYMBOLS
525
	if (syms->unused && fsa->warn) {
526 527 528
		pr_warn("Symbol %s is marked as UNUSED, however this module is "
			"using it.\n", fsa->name);
		pr_warn("This symbol will go away in the future.\n");
529 530 531
		pr_warn("Please evaluate if this is the right api to use and "
			"if it really is, submit a report to the linux kernel "
			"mailing list together with submitting your code for "
532
			"inclusion.\n");
533
	}
534
#endif
535 536 537

	fsa->owner = owner;
	fsa->crc = symversion(syms->crcs, symnum);
538
	fsa->sym = &syms->start[symnum];
539 540 541
	return true;
}

542 543 544 545 546 547 548 549
static int cmp_name(const void *va, const void *vb)
{
	const char *a;
	const struct kernel_symbol *b;
	a = va; b = vb;
	return strcmp(a, b->name);
}

550 551 552 553 554
static bool find_symbol_in_section(const struct symsearch *syms,
				   struct module *owner,
				   void *data)
{
	struct find_symbol_arg *fsa = data;
555 556 557 558 559 560 561
	struct kernel_symbol *sym;

	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
			sizeof(struct kernel_symbol), cmp_name);

	if (sym != NULL && check_symbol(syms, owner, sym - syms->start, data))
		return true;
562 563 564 565

	return false;
}

566
/* Find a symbol and return it, along with, (optional) crc and
567
 * (optional) module which owns it.  Needs preempt disabled or module_mutex. */
568 569
const struct kernel_symbol *find_symbol(const char *name,
					struct module **owner,
570
					const s32 **crc,
571 572
					bool gplok,
					bool warn)
573 574 575 576 577 578 579
{
	struct find_symbol_arg fsa;

	fsa.name = name;
	fsa.gplok = gplok;
	fsa.warn = warn;

580
	if (each_symbol_section(find_symbol_in_section, &fsa)) {
581 582 583 584
		if (owner)
			*owner = fsa.owner;
		if (crc)
			*crc = fsa.crc;
585
		return fsa.sym;
586 587
	}

588
	pr_debug("Failed to find symbol %s\n", name);
589
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
590
}
591
EXPORT_SYMBOL_GPL(find_symbol);
Linus Torvalds's avatar
Linus Torvalds committed
592

593 594 595 596
/*
 * Search for module by name: must hold module_mutex (or preempt disabled
 * for read-only access).
 */
597
static struct module *find_module_all(const char *name, size_t len,
598
				      bool even_unformed)
Linus Torvalds's avatar
Linus Torvalds committed
599 600 601
{
	struct module *mod;

602
	module_assert_mutex_or_preempt();
603

604
	list_for_each_entry_rcu(mod, &modules, list) {
605 606
		if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
			continue;
607
		if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
Linus Torvalds's avatar
Linus Torvalds committed
608 609 610 611
			return mod;
	}
	return NULL;
}
612 613 614

struct module *find_module(const char *name)
{
615
	module_assert_mutex();
616
	return find_module_all(name, strlen(name), false);
617
}
618
EXPORT_SYMBOL_GPL(find_module);
Linus Torvalds's avatar
Linus Torvalds committed
619 620

#ifdef CONFIG_SMP
621

622
static inline void __percpu *mod_percpu(struct module *mod)
623
{
624 625
	return mod->percpu;
}
626

Rusty Russell's avatar
Rusty Russell committed
627
static int percpu_modalloc(struct module *mod, struct load_info *info)
628
{
Rusty Russell's avatar
Rusty Russell committed
629 630 631 632 633 634
	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
	unsigned long align = pcpusec->sh_addralign;

	if (!pcpusec->sh_size)
		return 0;

635
	if (align > PAGE_SIZE) {
636 637
		pr_warn("%s: per-cpu alignment %li > %li\n",
			mod->name, align, PAGE_SIZE);
638 639 640
		align = PAGE_SIZE;
	}

Rusty Russell's avatar
Rusty Russell committed
641
	mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
642
	if (!mod->percpu) {
643 644
		pr_warn("%s: Could not allocate %lu bytes percpu data\n",
			mod->name, (unsigned long)pcpusec->sh_size);
645 646
		return -ENOMEM;
	}
Rusty Russell's avatar
Rusty Russell committed
647
	mod->percpu_size = pcpusec->sh_size;
648
	return 0;
649 650
}

651
static void percpu_modfree(struct module *mod)
652
{
653
	free_percpu(mod->percpu);
654 655
}

656
static unsigned int find_pcpusec(struct load_info *info)
657
{
658
	return find_sec(info, ".data..percpu");
659 660
}

661 662
static void percpu_modcopy(struct module *mod,
			   const void *from, unsigned long size)
663 664 665 666
{
	int cpu;

	for_each_possible_cpu(cpu)
667
		memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
668 669
}

670
bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
671 672 673 674 675 676 677
{
	struct module *mod;
	unsigned int cpu;

	preempt_disable();

	list_for_each_entry_rcu(mod, &modules, list) {
678 679
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
680 681 682 683
		if (!mod->percpu_size)
			continue;
		for_each_possible_cpu(cpu) {
			void *start = per_cpu_ptr(mod->percpu, cpu);
684
			void *va = (void *)addr;
685

686
			if (va >= start && va < start + mod->percpu_size) {
687
				if (can_addr) {
688
					*can_addr = (unsigned long) (va - start);
689 690 691 692
					*can_addr += (unsigned long)
						per_cpu_ptr(mod->percpu,
							    get_boot_cpu_id());
				}
693 694 695 696 697 698 699 700
				preempt_enable();
				return true;
			}
		}
	}

	preempt_enable();
	return false;
701 702
}

703 704 705 706 707 708 709 710 711 712 713 714 715 716
/**
 * is_module_percpu_address - test whether address is from module static percpu
 * @addr: address to test
 *
 * Test whether @addr belongs to module static percpu area.
 *
 * RETURNS:
 * %true if @addr is from module static percpu area
 */
bool is_module_percpu_address(unsigned long addr)
{
	return __is_module_percpu_address(addr, NULL);
}

Linus Torvalds's avatar
Linus Torvalds committed
717
#else /* ... !CONFIG_SMP */
718

719
static inline void __percpu *mod_percpu(struct module *mod)
Linus Torvalds's avatar
Linus Torvalds committed
720 721 722
{
	return NULL;
}
Rusty Russell's avatar
Rusty Russell committed
723
static int percpu_modalloc(struct module *mod, struct load_info *info)
724
{
Rusty Russell's avatar
Rusty Russell committed
725 726 727 728
	/* UP modules shouldn't have this section: ENOMEM isn't quite right */
	if (info->sechdrs[info->index.pcpu].sh_size != 0)
		return -ENOMEM;
	return 0;
729 730
}
static inline void percpu_modfree(struct module *mod)
Linus Torvalds's avatar
Linus Torvalds committed
731 732
{
}
733
static unsigned int find_pcpusec(struct load_info *info)
Linus Torvalds's avatar
Linus Torvalds committed
734 735 736
{
	return 0;
}
737 738
static inline void percpu_modcopy(struct module *mod,
				  const void *from, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
739 740 741 742
{
	/* pcpusec should be 0, and size of that section should be 0. */
	BUG_ON(size != 0);
}
743 744 745 746
bool is_module_percpu_address(unsigned long addr)
{
	return false;
}
747

748 749 750 751 752
bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
{
	return false;
}

Linus Torvalds's avatar
Linus Torvalds committed
753 754
#endif /* CONFIG_SMP */

755 756 757 758 759 760
#define MODINFO_ATTR(field)	\
static void setup_modinfo_##field(struct module *mod, const char *s)  \
{                                                                     \
	mod->field = kstrdup(s, GFP_KERNEL);                          \
}                                                                     \
static ssize_t show_modinfo_##field(struct module_attribute *mattr,   \
761
			struct module_kobject *mk, char *buffer)      \
762
{                                                                     \
763
	return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field);  \
764 765 766 767 768 769 770
}                                                                     \
static int modinfo_##field##_exists(struct module *mod)               \
{                                                                     \
	return mod->field != NULL;                                    \
}                                                                     \
static void free_modinfo_##field(struct module *mod)                  \
{                                                                     \
771 772
	kfree(mod->field);                                            \
	mod->field = NULL;                                            \
773 774
}                                                                     \
static struct module_attribute modinfo_##field = {                    \
775
	.attr = { .name = __stringify(field), .mode = 0444 },         \
776 777 778 779 780 781 782 783 784
	.show = show_modinfo_##field,                                 \
	.setup = setup_modinfo_##field,                               \
	.test = modinfo_##field##_exists,                             \
	.free = free_modinfo_##field,                                 \
};

MODINFO_ATTR(version);
MODINFO_ATTR(srcversion);

785 786
static char last_unloaded_module[MODULE_NAME_LEN+1];

787
#ifdef CONFIG_MODULE_UNLOAD
788 789 790

EXPORT_TRACEPOINT_SYMBOL(module_get);

791 792 793
/* MODULE_REF_BASE is the base reference count by kmodule loader. */
#define MODULE_REF_BASE	1

Linus Torvalds's avatar
Linus Torvalds committed
794
/* Init the unload section of the module. */
795
static int module_unload_init(struct module *mod)
Linus Torvalds's avatar
Linus Torvalds committed
796
{
797 798 799 800 801
	/*
	 * Initialize reference counter to MODULE_REF_BASE.
	 * refcnt == 0 means module is going.
	 */
	atomic_set(&mod->refcnt, MODULE_REF_BASE);
802

803 804
	INIT_LIST_HEAD(&mod->source_list);
	INIT_LIST_HEAD(&mod->target_list);
805

Linus Torvalds's avatar
Linus Torvalds committed
806
	/* Hold reference count during initialization. */
807
	atomic_inc(&mod->refcnt);
808 809

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
810 811 812 813 814 815 816
}

/* Does a already use b? */
static int already_uses(struct module *a, struct module *b)
{
	struct module_use *use;

817 818
	list_for_each_entry(use, &b->source_list, source_list) {
		if (use->source == a) {
819
			pr_debug("%s uses %s!\n", a->name, b->name);
Linus Torvalds's avatar
Linus Torvalds committed
820 821 822
			return 1;
		}
	}
823
	pr_debug("%s does not use %s!\n", a->name, b->name);
Linus Torvalds's avatar
Linus Torvalds committed
824 825 826
	return 0;
}

827 828 829 830 831 832 833 834 835 836 837
/*
 * Module a uses b
 *  - we add 'a' as a "source", 'b' as a "target" of module use
 *  - the module_use is added to the list of 'b' sources (so
 *    'b' can walk the list to see who sourced them), and of 'a'
 *    targets (so 'a' can see what modules it targets).
 */
static int add_module_usage(struct module *a, struct module *b)
{
	struct module_use *use;

838
	pr_debug("Allocating new usage for %s.\n", a->name);
839 840
	use = kmalloc(sizeof(*use), GFP_ATOMIC);
	if (!use) {
841
		pr_warn("%s: out of memory loading\n", a->name);
842 843 844 845 846 847 848 849 850 851
		return -ENOMEM;
	}

	use->source = a;
	use->target = b;
	list_add(&use->source_list, &b->source_list);
	list_add(&use->target_list, &a->target_list);
	return 0;
}

852
/* Module a uses b: caller needs module_mutex() */
853
int ref_module(struct module *a, struct module *b)
Linus Torvalds's avatar
Linus Torvalds committed
854
{
855
	int err;
Kay Sievers's avatar
Kay Sievers committed
856

857
	if (b == NULL || already_uses(a, b))
858 859
		return 0;

860 861
	/* If module isn't available, we fail. */
	err = strong_try_module_get(b);
862
	if (err)
863
		return err;
Linus Torvalds's avatar
Linus Torvalds committed
864

865 866
	err = add_module_usage(a, b);
	if (err) {
Linus Torvalds's avatar
Linus Torvalds committed
867
		module_put(b);
868
		return err;
Linus Torvalds's avatar
Linus Torvalds committed
869
	}
870
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
871
}
872
EXPORT_SYMBOL_GPL(ref_module);
Linus Torvalds's avatar
Linus Torvalds committed
873 874 875 876

/* Clear the unload stuff of the module. */
static void module_unload_free(struct module *mod)
{
877
	struct module_use *use, *tmp;
Linus Torvalds's avatar
Linus Torvalds committed
878

879
	mutex_lock(&module_mutex);
880 881
	list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
		struct module *i = use->target;
882
		pr_debug("%s unusing %s\n", mod->name, i->name);
883 884 885 886
		module_put(i);
		list_del(&use->source_list);
		list_del(&use->target_list);
		kfree(use);
Linus Torvalds's avatar
Linus Torvalds committed
887
	}
888
	mutex_unlock(&module_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
889 890 891
}

#ifdef CONFIG_MODULE_FORCE_UNLOAD
892
static inline int try_force_unload(unsigned int flags)
Linus Torvalds's avatar
Linus Torvalds committed
893 894 895
{
	int ret = (flags & O_TRUNC);
	if (ret)
896
		add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds's avatar
Linus Torvalds committed
897 898 899
	return ret;
}
#else
900
static inline int try_force_unload(unsigned int flags)
Linus Torvalds's avatar
Linus Torvalds committed
901 902 903 904 905
{
	return 0;
}
#endif /* CONFIG_MODULE_FORCE_UNLOAD */

906 907
/* Try to release refcount of module, 0 means success. */
static int try_release_module_ref(struct module *mod)
Linus Torvalds's avatar
Linus Torvalds committed
908
{
909
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
910

911 912 913 914 915 916
	/* Try to decrement refcnt which we set at loading */
	ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
	BUG_ON(ret < 0);
	if (ret)
		/* Someone can put this right now, recover with checking */
		ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
Linus Torvalds's avatar
Linus Torvalds committed
917

918 919
	return ret;
}
Linus Torvalds's avatar
Linus Torvalds committed
920

921 922
static int try_stop_module(struct module *mod, int flags, int *forced)
{
923
	/* If it's not unused, quit unless we're forcing. */
924 925 926
	if (try_release_module_ref(mod) != 0) {
		*forced = try_force_unload(flags);
		if (!(*forced))
Linus Torvalds's avatar
Linus Torvalds committed
927 928 929 930
			return -EWOULDBLOCK;
	}

	/* Mark it as dying. */
931
	mod->state = MODULE_STATE_GOING;
Linus Torvalds's avatar
Linus Torvalds committed
932

933
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
934 935
}

936 937 938 939 940 941 942 943 944 945
/**
 * module_refcount - return the refcount or -1 if unloading
 *
 * @mod:	the module we're checking
 *
 * Returns:
 *	-1 if the module is in the process of unloading
 *	otherwise the number of references in the kernel to the module
 */
int module_refcount(struct module *mod)
Linus Torvalds's avatar
Linus Torvalds committed
946
{
947
	return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
Linus Torvalds's avatar
Linus Torvalds committed
948 949 950 951 952 953
}
EXPORT_SYMBOL(module_refcount);

/* This exists whether we can unload or not */
static void free_module(struct module *mod);

954 955
SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
		unsigned int, flags)
Linus Torvalds's avatar
Linus Torvalds committed
956 957
{
	struct module *mod;
958
	char name[MODULE_NAME_LEN];
Linus Torvalds's avatar
Linus Torvalds committed
959 960
	int ret, forced = 0;

961
	if (!capable(CAP_SYS_MODULE) || modules_disabled)
962 963 964 965 966 967
		return -EPERM;

	if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
		return -EFAULT;
	name[MODULE_NAME_LEN-1] = '\0';

968 969
	audit_log_kern_module(name);

970 971
	if (mutex_lock_interruptible(&module_mutex) != 0)
		return -EINTR;
Linus Torvalds's avatar
Linus Torvalds committed
972 973 974 975 976 977 978

	mod = find_module(name);
	if (!mod) {
		ret = -ENOENT;
		goto out;
	}

979
	if (!list_empty(&mod->source_list)) {
Linus Torvalds's avatar
Linus Torvalds committed
980 981 982 983 984 985 986
		/* Other modules depend on us: get rid of them first. */
		ret = -EWOULDBLOCK;
		goto out;
	}

	/* Doing init or already dying? */
	if (mod->state != MODULE_STATE_LIVE) {
987
		/* FIXME: if (force), slam module count damn the torpedoes */
988
		pr_debug("%s already dying\n", mod->name);
Linus Torvalds's avatar
Linus Torvalds committed
989 990 991 992 993
		ret = -EBUSY;
		goto out;
	}

	/* If it has an init func, it must have an exit func to unload */
994
	if (mod->init && !mod->exit) {
995
		forced = try_force_unload(flags);
Linus Torvalds's avatar
Linus Torvalds committed
996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
		if (!forced) {
			/* This module can't be removed */
			ret = -EBUSY;
			goto out;
		}
	}

	/* Stop the machine so refcounts can't move and disable module. */
	ret = try_stop_module(mod, flags, &forced);
	if (ret != 0)
		goto out;

1008
	mutex_unlock(&module_mutex);
Lucas De Marchi's avatar
Lucas De Marchi committed
1009
	/* Final destruction now no one is using it. */
1010
	if (mod->exit != NULL)
Linus Torvalds's avatar
Linus Torvalds committed
1011
		mod->exit();
1012 1013
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_GOING, mod);
1014
	klp_module_going(mod);
1015 1016
	ftrace_release_mod(mod);

1017
	async_synchronize_full();
1018

1019
	/* Store the name of the last unloaded module for diagnostic purposes */
1020
	strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
Linus Torvalds's avatar
Linus Torvalds committed
1021

1022 1023 1024
	free_module(mod);
	return 0;
out:
1025
	mutex_unlock(&module_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
1026 1027 1028
	return ret;
}

1029
static inline void print_unload_info(struct seq_file *m, struct module *mod)
Linus Torvalds's avatar
Linus Torvalds committed
1030 1031 1032 1033
{
	struct module_use *use;
	int printed_something = 0;

1034
	seq_printf(m, " %i ", module_refcount(mod));
Linus Torvalds's avatar
Linus Torvalds committed
1035

1036 1037 1038 1039
	/*
	 * Always include a trailing , so userspace can differentiate
	 * between this and the old multi-field proc format.
	 */
1040
	list_for_each_entry(use, &mod->source_list, source_list) {
Linus Torvalds's avatar
Linus Torvalds committed
1041
		printed_something = 1;
1042
		seq_printf(m, "%s,", use->source->name);
Linus Torvalds's avatar
Linus Torvalds committed
1043 1044 1045 1046
	}

	if (mod->init != NULL && mod->exit == NULL) {
		printed_something = 1;
1047
		seq_puts(m, "[permanent],");
Linus Torvalds's avatar
Linus Torvalds committed
1048 1049 1050
	}

	if (!printed_something)
1051
		seq_puts(m, "-");
Linus Torvalds's avatar
Linus Torvalds committed
1052 1053 1054 1055 1056 1057
}

void __symbol_put(const char *symbol)
{
	struct module *owner;

Rusty Russell's avatar
Rusty Russell committed
1058
	preempt_disable();
1059
	if (!find_symbol(symbol, &owner, NULL, true, false))
Linus Torvalds's avatar
Linus Torvalds committed
1060 1061
		BUG();
	module_put(owner);
Rusty Russell's avatar
Rusty Russell committed
1062
	preempt_enable();
Linus Torvalds's avatar
Linus Torvalds committed
1063 1064 1065
}
EXPORT_SYMBOL(__symbol_put);

1066
/* Note this assumes addr is a function, which it currently always is. */
Linus Torvalds's avatar
Linus Torvalds committed
1067 1068
void symbol_put_addr(void *addr)
{
1069
	struct module *modaddr;
1070
	unsigned long a = (unsigned long)dereference_function_descriptor(addr);
Linus Torvalds's avatar
Linus Torvalds committed
1071

1072
	if (core_kernel_text(a))
1073
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1074

1075 1076 1077 1078 1079
	/*
	 * Even though we hold a reference on the module; we still need to
	 * disable preemption in order to safely traverse the data structure.
	 */
	preempt_disable();
1080
	modaddr = __module_text_address(a);
1081
	BUG_ON(!modaddr);
1082
	module_put(modaddr);
1083
	preempt_enable();
Linus Torvalds's avatar
Linus Torvalds committed
1084 1085 1086 1087
}
EXPORT_SYMBOL_GPL(symbol_put_addr);

static ssize_t show_refcnt(struct module_attribute *mattr,
1088
			   struct module_kobject *mk, char *buffer)
Linus Torvalds's avatar
Linus Torvalds committed
1089
{
1090
	return sprintf(buffer, "%i\n", module_refcount(mk->mod));
Linus Torvalds's avatar
Linus Torvalds committed
1091 1092
}

1093 1094
static struct module_attribute modinfo_refcnt =
	__ATTR(refcnt, 0444, show_refcnt, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
1095

1096 1097 1098 1099
void __module_get(struct module *module)
{
	if (module) {
		preempt_disable();
1100
		atomic_inc(&module->refcnt);