process.c 39.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
 * Copyright (C) 2001-2014 Philippe Gerum <rpm@xenomai.org>.
 * Copyright (C) 2001-2014 The Xenomai project <http://www.xenomai.org>
 * Copyright (C) 2006 Gilles Chanteperdrix <gilles.chanteperdrix@xenomai.org>
 *
 * SMP support Copyright (C) 2004 The HYADES project <http://www.hyades-itea.org>
 * RTAI/fusion Copyright (C) 2004 The RTAI project <http://www.rtai.org>
 *
 * Xenomai is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published
 * by the Free Software Foundation; either version 2 of the License,
 * or (at your option) any later version.
 *
 * Xenomai is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Xenomai; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
 * 02111-1307, USA.
 */
#include <stdarg.h>
#include <linux/unistd.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/fs.h>
29
#include <linux/anon_inodes.h>
30 31 32 33 34 35
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/cred.h>
#include <linux/file.h>
#include <linux/ptrace.h>
36
#include <linux/sched.h>
37
#include <linux/signal.h>
38
#include <linux/kallsyms.h>
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
#include <linux/ipipe.h>
#include <linux/ipipe_tickdev.h>
#include <cobalt/kernel/sched.h>
#include <cobalt/kernel/heap.h>
#include <cobalt/kernel/synch.h>
#include <cobalt/kernel/clock.h>
#include <cobalt/kernel/ppd.h>
#include <cobalt/kernel/trace.h>
#include <cobalt/kernel/stat.h>
#include <cobalt/kernel/ppd.h>
#include <cobalt/kernel/vdso.h>
#include <cobalt/kernel/thread.h>
#include <cobalt/uapi/signal.h>
#include <cobalt/uapi/syscall.h>
#include <trace/events/cobalt-core.h>
54
#include <rtdm/driver.h>
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
#include <asm/xenomai/features.h>
#include <asm/xenomai/syscall.h>
#include <asm-generic/xenomai/mayday.h>
#include "../debug.h"
#include "internal.h"
#include "thread.h"
#include "sched.h"
#include "mutex.h"
#include "cond.h"
#include "mqueue.h"
#include "sem.h"
#include "signal.h"
#include "timer.h"
#include "monitor.h"
#include "clock.h"
#include "event.h"
#include "timerfd.h"
#include "io.h"

static int gid_arg = -1;
module_param_named(allowed_group, gid_arg, int, 0644);

static DEFINE_MUTEX(personality_lock);

static struct hlist_head *process_hash;
DEFINE_PRIVATE_XNLOCK(process_hash_lock);
#define PROCESS_HASH_SIZE 13

struct xnthread_personality *cobalt_personalities[NR_PERSONALITIES];

static struct xnsynch yield_sync;

87 88 89 90 91 92 93 94 95 96 97
LIST_HEAD(cobalt_thread_list);

struct cobalt_resources cobalt_global_resources = {
	.condq = LIST_HEAD_INIT(cobalt_global_resources.condq),
	.mutexq = LIST_HEAD_INIT(cobalt_global_resources.mutexq),
	.semq = LIST_HEAD_INIT(cobalt_global_resources.semq),
	.monitorq = LIST_HEAD_INIT(cobalt_global_resources.monitorq),
	.eventq = LIST_HEAD_INIT(cobalt_global_resources.eventq),
	.schedq = LIST_HEAD_INIT(cobalt_global_resources.schedq),
};

98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
static unsigned __attribute__((pure)) process_hash_crunch(struct mm_struct *mm)
{
	unsigned long hash = ((unsigned long)mm - PAGE_OFFSET) / sizeof(*mm);
	return hash % PROCESS_HASH_SIZE;
}

static struct cobalt_process *__process_hash_search(struct mm_struct *mm)
{
	unsigned int bucket = process_hash_crunch(mm);
	struct cobalt_process *p;

	hlist_for_each_entry(p, &process_hash[bucket], hlink)
		if (p->mm == mm)
			return p;
	
	return NULL;
}

static int process_hash_enter(struct cobalt_process *p)
{
	struct mm_struct *mm = current->mm;
	unsigned int bucket = process_hash_crunch(mm);
	int err;
	spl_t s;

	xnlock_get_irqsave(&process_hash_lock, s);
	if (__process_hash_search(mm)) {
		err = -EBUSY;
		goto out;
	}

	p->mm = mm;
	hlist_add_head(&p->hlink, &process_hash[bucket]);
	err = 0;
  out:
	xnlock_put_irqrestore(&process_hash_lock, s);
	return err;
}

static void process_hash_remove(struct cobalt_process *p)
{
	spl_t s;

	xnlock_get_irqsave(&process_hash_lock, s);
	if (p->mm)
		hlist_del(&p->hlink);
	xnlock_put_irqrestore(&process_hash_lock, s);
}

struct cobalt_process *cobalt_search_process(struct mm_struct *mm)
{
	struct cobalt_process *process;
	spl_t s;
	
	xnlock_get_irqsave(&process_hash_lock, s);
	process = __process_hash_search(mm);
	xnlock_put_irqrestore(&process_hash_lock, s);
	
	return process;
}

static void *lookup_context(int xid)
{
	struct cobalt_process *process = cobalt_current_process();
	void *priv = NULL;
	spl_t s;

	xnlock_get_irqsave(&process_hash_lock, s);
	/*
	 * First try matching the process context attached to the
	 * (usually main) thread which issued sc_cobalt_bind. If not
	 * found, try matching by mm context, which should point us
	 * back to the latter. If none match, then the current process
	 * is unbound.
	 */
	if (process == NULL && current->mm)
		process = __process_hash_search(current->mm);
	if (process)
		priv = process->priv[xid];

	xnlock_put_irqrestore(&process_hash_lock, s);

	return priv;
}

183
static void remove_process(struct cobalt_process *process)
184 185 186 187 188 189 190 191
{
	struct xnthread_personality *personality;
	void *priv;
	int xid;

	mutex_lock(&personality_lock);

	for (xid = NR_PERSONALITIES - 1; xid >= 0; xid--) {
192
		if (!__test_and_clear_bit(xid, &process->permap))
193 194
			continue;
		personality = cobalt_personalities[xid];
195
		priv = process->priv[xid];
196 197
		if (priv == NULL)
			continue;
198 199 200 201 202
		/*
		 * CAUTION: process potentially refers to stale memory
		 * upon return from detach_process() for the Cobalt
		 * personality, so don't dereference it afterwards.
		 */
203 204
		if (xid)
			process->priv[xid] = NULL;
205 206 207
		__clear_bit(personality->xid, &process->permap);
		personality->ops.detach_process(priv);
		atomic_dec(&personality->refcnt);
208
		XENO_WARN_ON(COBALT, atomic_read(&personality->refcnt) < 0);
209 210
		if (personality->module)
			module_put(personality->module);
211 212 213
	}

	cobalt_set_process(NULL);
214 215

	mutex_unlock(&personality_lock);
216 217
}

218
static void post_ppd_release(struct cobalt_umm *umm)
219 220 221
{
	struct cobalt_process *process;

222
	process = container_of(umm, struct cobalt_process, sys_ppd.umm);
223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
	kfree(process);
}

static inline char *get_exe_path(struct task_struct *p)
{
	struct file *exe_file;
	char *pathname, *buf;
	struct mm_struct *mm;
	struct path path;

	/*
	 * PATH_MAX is fairly large, and in any case won't fit on the
	 * caller's stack happily; since we are mapping a shadow,
	 * which is a heavyweight operation anyway, let's pick the
	 * memory from the page allocator.
	 */
239
	buf = (char *)__get_free_page(GFP_KERNEL);
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
	if (buf == NULL)
		return ERR_PTR(-ENOMEM);

	mm = get_task_mm(p);
	if (mm == NULL) {
		pathname = "vmlinux";
		goto copy;	/* kernel thread */
	}

	exe_file = get_mm_exe_file(mm);
	mmput(mm);
	if (exe_file == NULL) {
		pathname = ERR_PTR(-ENOENT);
		goto out;	/* no luck. */
	}

	path = exe_file->f_path;
	path_get(&exe_file->f_path);
	fput(exe_file);
	pathname = d_path(&path, buf, PATH_MAX);
	path_put(&path);
	if (IS_ERR(pathname))
		goto out;	/* mmmh... */
copy:
	/* caution: d_path() may start writing anywhere in the buffer. */
	pathname = kstrdup(pathname, GFP_KERNEL);
out:
	free_page((unsigned long)buf);

	return pathname;
}

static inline int raise_cap(int cap)
{
	struct cred *new;

	new = prepare_creds();
	if (new == NULL)
		return -ENOMEM;

	cap_raise(new->cap_effective, cap);

	return commit_creds(new);
}

static int bind_personality(struct xnthread_personality *personality)
{
	struct cobalt_process *process;
	void *priv;

	/*
	 * We also check capabilities for stacking a Cobalt extension,
	 * in case the process dropped the supervisor privileges after
	 * a successful initial binding to the Cobalt interface.
	 */
	if (!capable(CAP_SYS_NICE) &&
	    (gid_arg == -1 || !in_group_p(KGIDT_INIT(gid_arg))))
		return -EPERM;
	/*
	 * Protect from the same process binding to the same interface
	 * several times.
	 */
	priv = lookup_context(personality->xid);
	if (priv)
		return 0;

	priv = personality->ops.attach_process();
	if (IS_ERR(priv))
		return PTR_ERR(priv);

	process = cobalt_current_process();
311 312 313 314 315
	/*
	 * We are still covered by the personality_lock, so we may
	 * safely bump the module refcount after the attach handler
	 * has returned.
	 */
316
	if (personality->module && !try_module_get(personality->module)) {
317
		personality->ops.detach_process(priv);
318
		return -EAGAIN;
319 320
	}

321 322
	__set_bit(personality->xid, &process->permap);
	atomic_inc(&personality->refcnt);
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
	process->priv[personality->xid] = priv;

	raise_cap(CAP_SYS_NICE);
	raise_cap(CAP_IPC_LOCK);
	raise_cap(CAP_SYS_RAWIO);

	return 0;
}

int cobalt_bind_personality(unsigned int magic)
{
	struct xnthread_personality *personality;
	int xid, ret = -ESRCH;

	mutex_lock(&personality_lock);

	for (xid = 1; xid < NR_PERSONALITIES; xid++) {
		personality = cobalt_personalities[xid];
		if (personality && personality->magic == magic) {
			ret = bind_personality(personality);
			break;
		}
	}

	mutex_unlock(&personality_lock);

	return ret ?: xid;
}

352
int cobalt_bind_core(int ufeatures)
353
{
354
	struct cobalt_process *process;
355 356 357 358 359
	int ret;

	mutex_lock(&personality_lock);
	ret = bind_personality(&cobalt_personality);
	mutex_unlock(&personality_lock);
360 361
	if (ret)
		return ret;
362

363 364 365 366 367
	process = cobalt_current_process();
	/* Feature set userland knows about. */
	process->ufeatures = ufeatures;

	return 0;
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
}

/**
 * @fn int cobalt_register_personality(struct xnthread_personality *personality)
 * @internal
 * @brief Register a new interface personality.
 *
 * - personality->ops.attach_process() is called when a user-space
 *   process binds to the personality, on behalf of one of its
 *   threads. The attach_process() handler may return:
 *
 *   . an opaque pointer, representing the context of the calling
 *   process for this personality;
 *
 *   . a NULL pointer, meaning that no per-process structure should be
 *   attached to this process for this personality;
 *
 *   . ERR_PTR(negative value) indicating an error, the binding
 *   process will then abort.
 *
388 389
 * - personality->ops.detach_process() is called on behalf of an
 *   exiting user-space process which has previously attached to the
390 391 392
 *   personality. This handler is passed a pointer to the per-process
 *   data received earlier from the ops->attach_process() handler.
 *
393 394 395 396 397
 * @return the personality (extension) identifier.
 *
 * @note cobalt_get_context() is NULL when ops.detach_process() is
 * invoked for the personality the caller detaches from.
 *
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
 * @coretags{secondary-only}
 */
int cobalt_register_personality(struct xnthread_personality *personality)
{
	int xid;

	mutex_lock(&personality_lock);

	for (xid = 0; xid < NR_PERSONALITIES; xid++) {
		if (cobalt_personalities[xid] == NULL) {
			personality->xid = xid;
			atomic_set(&personality->refcnt, 0);
			cobalt_personalities[xid] = personality;
			goto out;
		}
	}

	xid = -EAGAIN;
out:
	mutex_unlock(&personality_lock);

	return xid;
}
EXPORT_SYMBOL_GPL(cobalt_register_personality);

/*
 * @brief Unregister an interface personality.
 *
 * @coretags{secondary-only}
 */
int cobalt_unregister_personality(int xid)
{
	struct xnthread_personality *personality;
	int ret = 0;

	if (xid < 0 || xid >= NR_PERSONALITIES)
		return -EINVAL;

	mutex_lock(&personality_lock);

	personality = cobalt_personalities[xid];
	if (atomic_read(&personality->refcnt) > 0)
		ret = -EBUSY;
	else
		cobalt_personalities[xid] = NULL;

	mutex_unlock(&personality_lock);

	return ret;
}
EXPORT_SYMBOL_GPL(cobalt_unregister_personality);

/**
 * Stack a new personality over Cobalt for the current thread.
 *
 * This service registers the current thread as a member of the
 * additional personality identified by @a xid. If the current thread
 * is already assigned this personality, the call returns successfully
 * with no effect.
 *
 * @param xid the identifier of the additional personality.
 *
 * @return A handle to the previous personality. The caller should
 * save this handle for unstacking @a xid when applicable via a call
 * to cobalt_pop_personality().
 *
 * @coretags{secondary-only}
 */
struct xnthread_personality *
cobalt_push_personality(int xid)
{
	struct ipipe_threadinfo *p = ipipe_current_threadinfo();
	struct xnthread_personality *prev, *next;
	struct xnthread *thread = p->thread;

	secondary_mode_only();

	mutex_lock(&personality_lock);

	if (xid < 0 || xid >= NR_PERSONALITIES ||
	    p->process == NULL || !test_bit(xid, &p->process->permap)) {
		mutex_unlock(&personality_lock);
		return NULL;
	}

	next = cobalt_personalities[xid];
	prev = thread->personality;
	if (next == prev) {
		mutex_unlock(&personality_lock);
		return prev;
	}

	thread->personality = next;
	mutex_unlock(&personality_lock);
	xnthread_run_handler(thread, map_thread);

	return prev;
}
EXPORT_SYMBOL_GPL(cobalt_push_personality);

/**
 * Pop the topmost personality from the current thread.
 *
 * This service pops the topmost personality off the current thread.
 *
 * @param prev the previous personality which was returned by the
 * latest call to cobalt_push_personality() for the current thread.
 *
 * @coretags{secondary-only}
 */
void cobalt_pop_personality(struct xnthread_personality *prev)
{
	struct ipipe_threadinfo *p = ipipe_current_threadinfo();
	struct xnthread *thread = p->thread;

	secondary_mode_only();
	thread->personality = prev;
}
EXPORT_SYMBOL_GPL(cobalt_pop_personality);

/**
 * Return the per-process data attached to the calling user process.
 *
 * This service returns the per-process data attached to the calling
 * user process for the personality whose xid is @a xid.
 *
 * The per-process data was obtained from the ->attach_process()
 * handler defined for the personality @a xid refers to.
 *
 * See cobalt_register_personality() documentation for information on
 * the way to attach a per-process data to a process.
 *
 * @param xid the personality identifier.
 *
 * @return the per-process data if the current context is a user-space
 * process; @return NULL otherwise. As a special case,
 * cobalt_get_context(0) returns the current Cobalt process
 * descriptor, which is strictly identical to calling
 * cobalt_current_process().
 *
 * @coretags{task-unrestricted}
 */
void *cobalt_get_context(int xid)
{
	return lookup_context(xid);
}
EXPORT_SYMBOL_GPL(cobalt_get_context);

int cobalt_yield(xnticks_t min, xnticks_t max)
{
	xnticks_t start;
	int ret;

	start = xnclock_read_monotonic(&nkclock);
	max += start;
	min += start;

	do {
		ret = xnsynch_sleep_on(&yield_sync, max, XN_ABSOLUTE);
		if (ret & XNBREAK)
			return -EINTR;
	} while (ret == 0 && xnclock_read_monotonic(&nkclock) < min);

	return 0;
}
EXPORT_SYMBOL_GPL(cobalt_yield);

static inline void init_uthread_info(struct xnthread *thread)
{
	struct ipipe_threadinfo *p;

	p = ipipe_current_threadinfo();
	p->thread = thread;
	p->process = cobalt_search_process(current->mm);
}

static inline void clear_threadinfo(void)
{
	struct ipipe_threadinfo *p = ipipe_current_threadinfo();
	p->thread = NULL;
	p->process = NULL;
}

#ifdef CONFIG_MMU

static inline int disable_ondemand_memory(void)
{
	struct task_struct *p = current;
	siginfo_t si;

	if ((p->mm->def_flags & VM_LOCKED) == 0) {
		memset(&si, 0, sizeof(si));
		si.si_signo = SIGDEBUG;
		si.si_code = SI_QUEUE;
		si.si_int = SIGDEBUG_NOMLOCK | sigdebug_marker;
		send_sig_info(SIGDEBUG, &si, p);
		return 0;
	}

	return __ipipe_disable_ondemand_mappings(p);
}

600 601 602 603 604
static inline int get_mayday_prot(void)
{
	return PROT_READ|PROT_EXEC;
}

605 606 607 608 609 610 611
#else /* !CONFIG_MMU */

static inline int disable_ondemand_memory(void)
{
	return 0;
}

612 613 614 615 616 617 618 619 620 621 622 623
static inline int get_mayday_prot(void)
{
	/*
	 * Until we stop backing /dev/mem with the mayday page, we
	 * can't ask for PROT_EXEC since the former does not define
	 * mmap capabilities, and default ones won't allow an
	 * executable mapping with MAP_SHARED. In the NOMMU case, this
	 * is (currently) not an issue.
	 */
	return PROT_READ;
}

624 625 626
#endif /* !CONFIG_MMU */

/**
627
 * @fn int cobalt_map_user(struct xnthread *thread, __u32 __user *u_winoff)
628 629 630 631 632 633 634 635 636 637 638 639 640
 * @internal
 * @brief Create a shadow thread context over a user task.
 *
 * This call maps a Xenomai thread to the current regular Linux task
 * running in userland.  The priority and scheduling class of the
 * underlying Linux task are not affected; it is assumed that the
 * interface library did set them appropriately before issuing the
 * shadow mapping request.
 *
 * @param thread The descriptor address of the new shadow thread to be
 * mapped to current. This descriptor must have been previously
 * initialized by a call to xnthread_init().
 *
641
 * @param u_winoff will receive the offset of the per-thread
642
 * "u_window" structure in the global heap associated to @a
643 644 645 646 647 648 649 650 651 652 653 654 655
 * thread. This structure reflects thread state information visible
 * from userland through a shared memory window.
 *
 * @return 0 is returned on success. Otherwise:
 *
 * - -EINVAL is returned if the thread control block does not bear the
 * XNUSER bit.
 *
 * - -EBUSY is returned if either the current Linux task or the
 * associated shadow thread is already involved in a shadow mapping.
 *
 * @coretags{secondary-only}
 */
656
int cobalt_map_user(struct xnthread *thread, __u32 __user *u_winoff)
657 658 659
{
	struct xnthread_user_window *u_window;
	struct xnthread_start_attr attr;
660
	struct cobalt_ppd *sys_ppd;
661
	struct cobalt_umm *umm;
662 663 664 665 666 667 668 669
	int ret;

	if (!xnthread_test_state(thread, XNUSER))
		return -EINVAL;

	if (xnthread_current() || xnthread_test_state(thread, XNMAPPED))
		return -EBUSY;

670
	if (!access_wok(u_winoff, sizeof(*u_winoff)))
671 672 673 674 675 676
		return -EFAULT;

	ret = disable_ondemand_memory();
	if (ret)
		return ret;

677
	umm = &cobalt_kernel_ppd.umm;
678
	u_window = cobalt_umm_zalloc(umm, sizeof(*u_window));
679 680 681 682
	if (u_window == NULL)
		return -ENOMEM;

	thread->u_window = u_window;
683
	__xn_put_user(cobalt_umm_offset(umm, u_window), u_winoff);
684 685 686 687 688 689 690 691
	xnthread_pin_initial(thread);

	/*
	 * CAUTION: we enable the pipeline notifier only when our
	 * shadow TCB is consistent, so that we won't trigger false
	 * positive in debug code from handle_schedule_event() and
	 * friends.
	 */
692
	xnthread_init_shadow_tcb(thread);
693 694 695 696
	xnthread_suspend(thread, XNRELAX, XN_INFINITE, XN_RELATIVE, NULL);
	init_uthread_info(thread);
	xnthread_set_state(thread, XNMAPPED);
	xndebug_shadow_init(thread);
697
	sys_ppd = cobalt_ppd_get(0);
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734
	atomic_inc(&sys_ppd->refcnt);
	/*
	 * ->map_thread() handler is invoked after the TCB is fully
	 * built, and when we know for sure that current will go
	 * through our task-exit handler, because it has a shadow
	 * extension and I-pipe notifications will soon be enabled for
	 * it.
	 */
	xnthread_run_handler(thread, map_thread);
	ipipe_enable_notifier(current);

	attr.mode = 0;
	attr.entry = NULL;
	attr.cookie = NULL;
	ret = xnthread_start(thread, &attr);
	if (ret)
		return ret;

	xnthread_sync_window(thread);

	xntrace_pid(xnthread_host_pid(thread),
		    xnthread_current_priority(thread));

	return 0;
}

static inline int handle_exception(struct ipipe_trap_data *d)
{
	struct xnthread *thread;
	struct xnsched *sched;

	sched = xnsched_current();
	thread = sched->curr;

	if (xnthread_test_state(thread, XNROOT))
		return 0;

735
	trace_cobalt_thread_fault(d);
736 737

	if (xnarch_fault_fpu_p(d)) {
738
#ifdef CONFIG_XENO_ARCH_FPU
739 740
		spl_t s;

741
		/* FPU exception received in primary mode. */
742
		splhigh(s);
743 744
		if (xnarch_handle_fpu_fault(sched->fpuholder, thread, d)) {
			sched->fpuholder = thread;
745
			splexit(s);
746 747
			return 1;
		}
748
		splexit(s);
749
#endif /* CONFIG_XENO_ARCH_FPU */
750 751 752 753 754 755 756 757 758
		print_symbol("invalid use of FPU in Xenomai context at %s\n",
			     xnarch_fault_pc(d));
	}

	/*
	 * If we experienced a trap on behalf of a shadow thread
	 * running in primary mode, move it to the Linux domain,
	 * leaving the kernel process the exception.
	 */
759
#if defined(CONFIG_XENO_OPT_DEBUG_COBALT) || defined(CONFIG_XENO_OPT_DEBUG_USER)
760 761
	if (!user_mode(d->regs)) {
		xntrace_panic_freeze();
762
		printk(XENO_WARNING
763 764 765 766 767 768 769
		       "switching %s to secondary mode after exception #%u in "
		       "kernel-space at 0x%lx (pid %d)\n", thread->name,
		       xnarch_fault_trap(d),
		       xnarch_fault_pc(d),
		       xnthread_host_pid(thread));
		xntrace_panic_dump();
	} else if (xnarch_fault_notify(d)) /* Don't report debug traps */
770
		printk(XENO_WARNING
771 772 773 774 775
		       "switching %s to secondary mode after exception #%u from "
		       "user-space at 0x%lx (pid %d)\n", thread->name,
		       xnarch_fault_trap(d),
		       xnarch_fault_pc(d),
		       xnthread_host_pid(thread));
776
#endif
777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794

	if (xnarch_fault_pf_p(d))
		/*
		 * The page fault counter is not SMP-safe, but it's a
		 * simple indicator that something went wrong wrt
		 * memory locking anyway.
		 */
		xnstat_counter_inc(&thread->stat.pf);

	xnthread_relax(xnarch_fault_notify(d), SIGDEBUG_MIGRATE_FAULT);

	return 0;
}

static int handle_mayday_event(struct pt_regs *regs)
{
	struct xnthread *thread = xnthread_current();
	struct xnarchtcb *tcb = xnthread_archtcb(thread);
795
	struct cobalt_ppd *sys_ppd;
796

797
	XENO_BUG_ON(COBALT, !xnthread_test_state(thread, XNUSER));
798 799 800 801

	/* We enter the mayday handler with hw IRQs off. */
	sys_ppd = cobalt_ppd_get(0);

802
	xnarch_handle_mayday(tcb, regs, sys_ppd->mayday_tramp);
803 804 805 806 807 808 809 810 811 812 813 814 815

	return KEVENT_PROPAGATE;
}

int ipipe_trap_hook(struct ipipe_trap_data *data)
{
	if (data->exception == IPIPE_TRAP_MAYDAY)
		return handle_mayday_event(data->regs);

	/*
	 * No migration is possible on behalf of the head domain, so
	 * the following access is safe.
	 */
816
	raw_cpu_ptr(&cobalt_machine_cpudata)->faults[data->exception]++;
817 818 819 820 821 822 823 824 825 826 827 828 829

	if (handle_exception(data))
		return KEVENT_STOP;

	/*
	 * CAUTION: access faults must be propagated downstream
	 * whichever domain caused them, so that we don't spuriously
	 * raise a fatal error when some Linux fixup code is available
	 * to recover from the fault.
	 */
	return KEVENT_PROPAGATE;
}

830 831 832 833
/*
 * Legacy idle hook, unconditionally allow entering the idle state.
 */
bool ipipe_enter_idle_hook(void)
834
{
835
	return true;
836 837
}

838 839 840 841 842 843 844 845 846 847 848 849 850
#ifdef CONFIG_SMP

static int handle_setaffinity_event(struct ipipe_cpu_migration_data *d)
{
	struct task_struct *p = d->task;
	struct xnthread *thread;
	spl_t s;

	thread = xnthread_from_task(p);
	if (thread == NULL)
		return KEVENT_PROPAGATE;

	/*
851 852 853 854 855 856 857 858
	 * Detect a Cobalt thread sleeping in primary mode which is
	 * required to migrate to another CPU by the host kernel.
	 *
	 * We may NOT fix up thread->sched immediately using the
	 * passive migration call, because that latter always has to
	 * take place on behalf of the target thread itself while
	 * running in secondary mode. Therefore, that thread needs to
	 * go through secondary mode first, then move back to primary
859
	 * mode, so that affinity_ok() does the fixup work.
860 861 862 863 864
	 *
	 * We force this by sending a SIGSHADOW signal to the migrated
	 * thread, asking it to switch back to primary mode from the
	 * handler, at which point the interrupted syscall may be
	 * restarted.
865 866 867
	 */
	xnlock_get_irqsave(&nklock, s);

868 869 870 871
	if (xnthread_test_state(thread, XNTHREAD_BLOCK_BITS & ~XNRELAX))
		xnthread_signal(thread, SIGSHADOW, SIGSHADOW_ACTION_HARDEN);

	xnlock_put_irqrestore(&nklock, s);
872 873 874 875

	return KEVENT_PROPAGATE;
}

876
static inline bool affinity_ok(struct task_struct *p) /* nklocked, IRQs off */
877 878 879 880 881 882
{
	struct xnthread *thread = xnthread_from_task(p);
	struct xnsched *sched;
	int cpu = task_cpu(p);

	/*
883 884 885 886 887 888 889 890 891 892 893
	 * To maintain consistency between both Cobalt and host
	 * schedulers, reflecting a thread migration to another CPU
	 * into the Cobalt scheduler state must happen from secondary
	 * mode only, on behalf of the migrated thread itself once it
	 * runs on the target CPU.
	 *
	 * This means that the Cobalt scheduler state regarding the
	 * CPU information lags behind the host scheduler state until
	 * the migrated thread switches back to primary mode
	 * (i.e. task_cpu(p) != xnsched_cpu(xnthread_from_task(p)->sched)).
	 * This is ok since Cobalt does not schedule such thread until then.
894
	 *
895 896 897 898
	 * check_affinity() detects when a Cobalt thread switching
	 * back to primary mode did move to another CPU earlier while
	 * in secondary mode. If so, do the fixups to reflect the
	 * change.
899 900
	 */
	if (!xnsched_supported_cpu(cpu)) {
901 902 903 904 905
		/*
		 * The thread is about to switch to primary mode on a
		 * non-rt CPU, which is damn wrong and hopeless.
		 * Whine and cancel that thread.
		 */
906
		printk(XENO_WARNING "thread %s[%d] switched to non-rt CPU%d, aborted.\n",
907 908 909 910 911 912 913 914
		       thread->name, xnthread_host_pid(thread), cpu);
		/*
		 * Can't call xnthread_cancel() from a migration
		 * point, that would break. Since we are on the wakeup
		 * path to hardening, just raise XNCANCELD to catch it
		 * in xnthread_harden().
		 */
		xnthread_set_info(thread, XNCANCELD);
915
		return false;
916 917 918 919
	}

	sched = xnsched_struct(cpu);
	if (sched == thread->sched)
920
		return true;
921 922 923 924 925 926

	/*
	 * The current thread moved to a supported real-time CPU,
	 * which is not part of its original affinity mask
	 * though. Assume user wants to extend this mask.
	 */
927 928
	if (!cpumask_test_cpu(cpu, &thread->affinity))
		cpumask_set_cpu(cpu, &thread->affinity);
929

930
	xnthread_run_handler_stack(thread, move_thread, cpu);
931
	xnthread_migrate_passive(thread, sched);
932 933

	return true;
934 935 936 937 938 939 940 941 942 943 944
}

#else /* !CONFIG_SMP */

struct ipipe_cpu_migration_data;

static int handle_setaffinity_event(struct ipipe_cpu_migration_data *d)
{
	return KEVENT_PROPAGATE;
}

945 946 947 948
static inline bool affinity_ok(struct task_struct *p)
{
	return true;
}
949 950 951 952 953 954 955 956 957 958 959 960 961 962

#endif /* CONFIG_SMP */

void ipipe_migration_hook(struct task_struct *p) /* hw IRQs off */
{
	struct xnthread *thread = xnthread_from_task(p);

	/*
	 * We fire the handler before the thread is migrated, so that
	 * thread->sched does not change between paired invocations of
	 * relax_thread/harden_thread handlers.
	 */
	xnlock_get(&nklock);
	xnthread_run_handler_stack(thread, harden_thread);
963 964
	if (affinity_ok(p))
		xnthread_resume(thread, XNRELAX);
965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984
	xnlock_put(&nklock);

	xnsched_run();
}

#ifdef CONFIG_XENO_OPT_HOSTRT

static IPIPE_DEFINE_SPINLOCK(__hostrtlock);

static int handle_hostrt_event(struct ipipe_hostrt_data *hostrt)
{
	unsigned long flags;
	urwstate_t tmp;

	/*
	 * The locking strategy is twofold:
	 * - The spinlock protects against concurrent updates from within the
	 *   Linux kernel and against preemption by Xenomai
	 * - The unsynced R/W block is for lockless read-only access.
	 */
985
	raw_spin_lock_irqsave(&__hostrtlock, flags);
986 987 988 989 990 991 992

	unsynced_write_block(&tmp, &nkvdso->hostrt_data.lock) {
		nkvdso->hostrt_data.live = 1;
		nkvdso->hostrt_data.cycle_last = hostrt->cycle_last;
		nkvdso->hostrt_data.mask = hostrt->mask;
		nkvdso->hostrt_data.mult = hostrt->mult;
		nkvdso->hostrt_data.shift = hostrt->shift;
993 994 995 996
		nkvdso->hostrt_data.wall_sec = hostrt->wall_time_sec;
		nkvdso->hostrt_data.wall_nsec = hostrt->wall_time_nsec;
		nkvdso->hostrt_data.wtom_sec = hostrt->wall_to_monotonic.tv_sec;
		nkvdso->hostrt_data.wtom_nsec = hostrt->wall_to_monotonic.tv_nsec;
997 998
	}

999
	raw_spin_unlock_irqrestore(&__hostrtlock, flags);
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022

	return KEVENT_PROPAGATE;
}

static inline void init_hostrt(void)
{
	unsynced_rw_init(&nkvdso->hostrt_data.lock);
	nkvdso->hostrt_data.live = 0;
}

#else /* !CONFIG_XENO_OPT_HOSTRT */

struct ipipe_hostrt_data;

static inline int handle_hostrt_event(struct ipipe_hostrt_data *hostrt)
{
	return KEVENT_PROPAGATE;
}

static inline void init_hostrt(void) { }

#endif /* !CONFIG_XENO_OPT_HOSTRT */

1023
static void __handle_taskexit_event(struct task_struct *p)
1024
{
1025
	struct cobalt_ppd *sys_ppd;
1026 1027 1028 1029 1030 1031 1032
	struct xnthread *thread;

	/*
	 * We are called for both kernel and user shadows over the
	 * root thread.
	 */
	secondary_mode_only();
1033

1034
	thread = xnthread_current();
1035
	XENO_BUG_ON(COBALT, thread == NULL);
1036 1037 1038 1039 1040 1041
	trace_cobalt_shadow_unmap(thread);

	xnthread_run_handler_stack(thread, exit_thread);
	xnsched_run();

	if (xnthread_test_state(thread, XNUSER)) {
1042
		cobalt_umm_free(&cobalt_kernel_ppd.umm, thread->u_window);
1043
		thread->u_window = NULL;
1044
		sys_ppd = cobalt_ppd_get(0);
1045 1046 1047
		if (atomic_dec_and_test(&sys_ppd->refcnt))
			remove_process(cobalt_current_process());
	}
1048 1049 1050 1051 1052
}

static int handle_taskexit_event(struct task_struct *p) /* p == current */
{
	__handle_taskexit_event(p);
1053 1054 1055 1056 1057 1058

	/*
	 * __xnthread_cleanup() -> ... -> finalize_thread
	 * handler. From that point, the TCB is dropped. Be careful of
	 * not treading on stale memory within @thread.
	 */
1059
	__xnthread_cleanup(xnthread_current());
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083

	clear_threadinfo();

	return KEVENT_PROPAGATE;
}

static inline void signal_yield(void)
{
	spl_t s;

	if (!xnsynch_pended_p(&yield_sync))
		return;

	xnlock_get_irqsave(&nklock, s);
	if (xnsynch_pended_p(&yield_sync)) {
		xnsynch_flush(&yield_sync, 0);
		xnsched_run();
	}
	xnlock_put_irqrestore(&nklock, s);
}

static int handle_schedule_event(struct task_struct *next_task)
{
	struct task_struct *prev_task;
1084
	struct xnthread *next;
1085
	sigset_t pending;
1086
	spl_t s;
1087 1088 1089 1090 1091 1092 1093 1094 1095

	signal_yield();

	prev_task = current;
	next = xnthread_from_task(next_task);
	if (next == NULL)
		goto out;

	/*
1096 1097 1098
	 * Track tasks leaving the ptraced state.  Check both SIGSTOP
	 * (NPTL) and SIGINT (LinuxThreads) to detect ptrace
	 * continuation.
1099
	 */
1100
	if (xnthread_test_state(next, XNSSTEP)) {
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
		if (signal_pending(next_task)) {
			/*
			 * Do not grab the sighand lock here: it's
			 * useless, and we already own the runqueue
			 * lock, so this would expose us to deadlock
			 * situations on SMP.
			 */
			sigorsets(&pending,
				  &next_task->pending.signal,
				  &next_task->signal->shared_pending.signal);
			if (sigismember(&pending, SIGSTOP) ||
			    sigismember(&pending, SIGINT))
1113
				goto check;
1114
		}
1115 1116 1117 1118
		xnlock_get_irqsave(&nklock, s);
		xnthread_clear_state(next, XNSSTEP);
		xnlock_put_irqrestore(&nklock, s);
		xnthread_set_localinfo(next, XNHICCUP);
1119 1120
	}

1121
check:
1122 1123 1124 1125 1126 1127 1128 1129
	/*
	 * Do basic sanity checks on the incoming thread state.
	 * NOTE: we allow ptraced threads to run shortly in order to
	 * properly recover from a stopped state.
	 */
	if (!XENO_WARN(COBALT, !xnthread_test_state(next, XNRELAX),
		       "hardened thread %s[%d] running in Linux domain?! "
		       "(status=0x%x, sig=%d, prev=%s[%d])",
1130 1131 1132 1133
		       next->name, task_pid_nr(next_task),
		       xnthread_get_state(next),
		       signal_pending(next_task),
		       prev_task->comm, task_pid_nr(prev_task)))
1134 1135
		XENO_WARN(COBALT,
			  !(next_task->ptrace & PT_PTRACED) &&
1136
			   !xnthread_test_state(next, XNDORMANT)
1137 1138 1139
			  && xnthread_test_state(next, XNPEND),
			  "blocked thread %s[%d] rescheduled?! "
			  "(status=0x%x, sig=%d, prev=%s[%d])",
1140 1141
			  next->name, task_pid_nr(next_task),
			  xnthread_get_state(next),
1142
			  signal_pending(next_task), prev_task->comm,
1143
			  task_pid_nr(prev_task));
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
out:
	return KEVENT_PROPAGATE;
}

static int handle_sigwake_event(struct task_struct *p)
{
	struct xnthread *thread;
	sigset_t pending;
	spl_t s;

	thread = xnthread_from_task(p);
	if (thread == NULL)
		return KEVENT_PROPAGATE;

	xnlock_get_irqsave(&nklock, s);

	/*
	 * CAUTION: __TASK_TRACED is not set in p->state yet. This
	 * state bit will be set right after we return, when the task
	 * is woken up.
	 */
1165
	if ((p->ptrace & PT_PTRACED) && !xnthread_test_state(thread, XNSSTEP)) {
1166 1167 1168 1169 1170 1171 1172
		/* We already own the siglock. */
		sigorsets(&pending,
			  &p->pending.signal,
			  &p->signal->shared_pending.signal);

		if (sigismember(&pending, SIGTRAP) ||
		    sigismember(&pending, SIGSTOP)
1173
		    || sigismember(&pending, SIGINT))
1174
			xnthread_set_state(thread, XNSSTEP);
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
	}

	if (xnthread_test_state(thread, XNRELAX)) {
		xnlock_put_irqrestore(&nklock, s);
		return KEVENT_PROPAGATE;
	}

	/*
	 * If kicking a shadow thread in primary mode, make sure Linux
	 * won't schedule in its mate under our feet as a result of
	 * running signal_wake_up(). The Xenomai scheduler must remain
	 * in control for now, until we explicitly relax the shadow
	 * thread to allow for processing the pending signals. Make
	 * sure we keep the additional state flags unmodified so that
	 * we don't break any undergoing ptrace.
	 */
	if (p->state & (TASK_INTERRUPTIBLE|TASK_UNINTERRUPTIBLE))
1192
		cobalt_set_task_state(p, p->state | TASK_NOWAKEUP);
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205

	__xnthread_kick(thread);

	xnsched_run();

	xnlock_put_irqrestore(&nklock, s);

	return KEVENT_PROPAGATE;
}

static int handle_cleanup_event(struct mm_struct *mm)
{
	struct cobalt_process *old, *process;
1206
	struct cobalt_ppd *sys_ppd;
1207
	struct xnthread *curr;
1208

1209 1210 1211 1212 1213 1214
	/*
	 * We are NOT called for exiting kernel shadows.
	 * cobalt_current_process() is cleared if we get there after
	 * handle_task_exit(), so we need to restore this context
	 * pointer temporarily.
	 */
1215 1216 1217
	process = cobalt_search_process(mm);
	old = cobalt_set_process(process);
	sys_ppd = cobalt_ppd_get(0);
1218
	if (sys_ppd != &cobalt_kernel_ppd) {
1219 1220
		bool running_exec;

1221 1222 1223 1224 1225 1226 1227 1228 1229
		/*
		 * Detect a userland shadow running exec(), i.e. still
		 * attached to the current linux task (no prior
		 * clear_threadinfo). In this case, we emulate a task
		 * exit, since the Xenomai binding shall not survive
		 * the exec() syscall. Since the process will keep on
		 * running though, we have to disable the event
		 * notifier manually for it.
		 */
1230 1231
		curr = xnthread_current();
		running_exec = curr && (current->flags & PF_EXITING) == 0;
1232 1233
		if (running_exec) {
			__handle_taskexit_event(current);
1234 1235 1236 1237
			ipipe_disable_notifier(current);
		}
		if (atomic_dec_and_test(&sys_ppd->refcnt))
			remove_process(process);
1238
		if (running_exec) {
1239
			__xnthread_cleanup(curr);
1240 1241
			clear_threadinfo();
		}
1242 1243
	}

1244 1245 1246 1247 1248 1249
	/*
	 * CAUTION: Do not override a state change caused by
	 * remove_process().
	 */
	if (cobalt_current_process() == process)
		cobalt_set_process(old);
1250 1251 1252 1253

	return KEVENT_PROPAGATE;
}

1254
static inline int handle_clockfreq_event(unsigned int *p)
1255 1256 1257 1258 1259 1260 1261 1262
{
	unsigned int newfreq = *p;

	xnclock_update_freq(newfreq);

	return KEVENT_PROPAGATE;
}

1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
int ipipe_kevent_hook(int kevent, void *data)
{
	int ret;

	switch (kevent) {
	case IPIPE_KEVT_SCHEDULE:
		ret = handle_schedule_event(data);
		break;
	case IPIPE_KEVT_SIGWAKE:
		ret = handle_sigwake_event(data);
		break;
	case IPIPE_KEVT_EXIT:
		ret = handle_taskexit_event(data);
		break;
	case IPIPE_KEVT_CLEANUP:
		ret = handle_cleanup_event(data);
		break;
	case IPIPE_KEVT_HOSTRT:
		ret = handle_hostrt_event(data);
		break;
	case IPIPE_KEVT_SETAFFINITY:
		ret = handle_setaffinity_event(data);
		break;
1286 1287 1288 1289 1290
#ifdef IPIPE_KEVT_CLOCKFREQ
	case IPIPE_KEVT_CLOCKFREQ:
		ret = handle_clockfreq_event(data);
		break;
#endif
1291 1292 1293 1294 1295 1296 1297
	default:
		ret = KEVENT_PROPAGATE;
	}

	return ret;
}

1298 1299 1300 1301 1302 1303 1304 1305
static inline unsigned long map_mayday_page(void)
{
	void __user *u_addr = NULL;
	void *mayday_page;
	int ret;

	mayday_page = xnarch_get_mayday_page();
	ret = rtdm_mmap_to_user(NULL, mayday_page, PAGE_SIZE,
1306
				get_mayday_prot(), &u_addr, NULL, NULL);
1307 1308 1309 1310 1311 1312
	if (ret)
		return 0UL;

	return (unsigned long)u_addr;
}

1313 1314
static int attach_process(struct cobalt_process *process)
{
1315
	struct cobalt_ppd *p = &process->sys_ppd;
1316 1317 1318
	char *exe_path;
	int ret;

1319 1320
	ret = cobalt_umm_init(&p->umm, CONFIG_XENO_OPT_PRIVATE_HEAPSZ * 1024,
			      post_ppd_release);
1321 1322 1323
	if (ret)
		return ret;

1324
	cobalt_umm_set_name(&p->umm, "private heap[%d]", task_pid_nr(current));
1325

1326 1327
	p->mayday_tramp = map_mayday_page();
	if (p->mayday_tramp == 0) {
1328
		printk(XENO_WARNING
1329
		       "%s[%d] cannot map MAYDAY page\n",
1330
		       current->comm, task_pid_nr(current));
1331 1332 1333 1334 1335 1336
		ret = -ENOMEM;
		goto fail_mayday;
	}

	exe_path = get_exe_path(current);
	if (IS_ERR(exe_path)) {
1337
		printk(XENO_WARNING
1338
		       "%s[%d] can't find exe path\n",
1339
		       current->comm, task_pid_nr(current));
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354
		exe_path = NULL; /* Not lethal, but weird. */
	}
	p->exe_path = exe_path;
	xntree_init(&p->fds);
	atomic_set(&p->refcnt, 1);

	ret = process_hash_enter(process);
	if (ret)
		goto fail_hash;

	return 0;
fail_hash:
	if (p->exe_path)
		kfree(p->exe_path);
fail_mayday:
1355
	cobalt_umm_destroy(&p->umm);
1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374

	return ret;
}

static void *cobalt_process_attach(void)
{
	struct cobalt_process *process;
	int ret;

	process = kzalloc(sizeof(*process), GFP_KERNEL);
	if (process == NULL)
		return ERR_PTR(-ENOMEM);

	ret = attach_process(process);
	if (ret) {
		kfree(process);
		return ERR_PTR(ret);
	}

1375 1376 1377 1378 1379 1380
	INIT_LIST_HEAD(&process->resources.condq);
	INIT_LIST_HEAD(&process->resources.mutexq);
	INIT_LIST_HEAD(&process->resources.semq);
	INIT_LIST_HEAD(&process->resources.monitorq);
	INIT_LIST_HEAD(&process->resources.eventq);
	INIT_LIST_HEAD(&process->resources.schedq);
1381 1382 1383 1384 1385 1386 1387 1388 1389 1390
	INIT_LIST_HEAD(&process->sigwaiters);
	xntree_init(&process->usems);
	bitmap_fill(process->timers_map, CONFIG_XENO_OPT_NRTIMERS);
	cobalt_set_process(process);

	return process;
}

static void detach_process(struct cobalt_process *process)
{
1391
	struct cobalt_ppd *p = &process->sys_ppd;
1392 1393 1394 1395 1396 1397

	if (p->exe_path)
		kfree(p->exe_path);

	rtdm_fd_cleanup(p);
	process_hash_remove(process);
1398 1399
	/*
	 * CAUTION: the process descriptor might be immediately
1400 1401
	 * released as a result of calling cobalt_umm_destroy(), so we
	 * must do this last, not to tread on stale memory.
1402
	 */
1403
	cobalt_umm_destroy(&p->umm);
1404 1405
}

1406 1407 1408 1409 1410 1411
static void __reclaim_resource(struct cobalt_process *process,
			       void (*reclaim)(struct cobalt_resnode *node, spl_t s),
			       struct list_head *local,
			       struct list_head *global)
{
	struct cobalt_resnode *node, *tmp;
1412
	LIST_HEAD(stash);
1413
	spl_t s;
1414

1415 1416 1417 1418 1419 1420 1421
	xnlock_get_irqsave(&nklock, s);

	if (list_empty(global))
		goto flush_local;

	list_for_each_entry_safe(node, tmp, global, next) {
		if (node->owner == process) {
1422 1423
			list_del(&node->next);
			list_add(&node->next, &stash);
1424 1425 1426
		}
	}
		
1427 1428 1429 1430 1431 1432 1433
	list_for_each_entry_safe(node, tmp, &stash, next) {
		reclaim(node, s);
		xnlock_get_irqsave(&nklock, s);
	}

	XENO_BUG_ON(COBALT, !list_empty(&stash));

1434 1435 1436 1437 1438
flush_local:
	if (list_empty(local))
		goto out;

	list_for_each_entry_safe(node, tmp, local, next) {
1439
		reclaim(node, s);
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
		xnlock_get_irqsave(&nklock, s);
	}
out:
	xnsched_run();
	xnlock_put_irqrestore(&nklock, s);
}

#define cobalt_reclaim_resource(__process, __reclaim, __type)		\
	__reclaim_resource(__process, __reclaim,			\
			   &(__process)->resources.__type ## q,		\
			   &cobalt_global_resources.__type ## q)

1452 1453 1454 1455
static void cobalt_process_detach(void *arg)
{
	struct cobalt_process *process = arg;

1456 1457 1458
	cobalt_nsem_reclaim(process);
 	cobalt_timer_reclaim(process);
 	cobalt_sched_reclaim(process);
1459
	cobalt_reclaim_resource(process, cobalt_cond_reclaim, cond);
1460
	cobalt_reclaim_resource(process, cobalt_mutex_reclaim, mutex);
1461
	cobalt_reclaim_resource(process, cobalt_event_reclaim, event);
1462
	cobalt_reclaim_resource(process, cobalt_monitor_reclaim, monitor);
1463
	cobalt_reclaim_resource(process, cobalt_sem_reclaim, sem);
1464
 	detach_process(process);
1465
	/*
1466 1467 1468
	 * The cobalt_process descriptor release may be deferred until
	 * the last mapping on the private heap is gone. However, this
	 * is potentially stale memory already.
1469 1470 1471
	 */
}

1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485
struct xnthread_personality cobalt_personality = {
	.name = "cobalt",
	.magic = 0,
	.ops = {
		.attach_process = cobalt_process_attach,
		.detach_process = cobalt_process_detach,
		.map_thread = cobalt_thread_map,
		.exit_thread = cobalt_thread_exit,
		.finalize_thread = cobalt_thread_finalize,
	},
};
EXPORT_SYMBOL_GPL(cobalt_personality);

__init int cobalt_init(void)
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
{
	unsigned int i, size;
	int ret;

	size = sizeof(*process_hash) * PROCESS_HASH_SIZE;
	process_hash = kmalloc(size, GFP_KERNEL);
	if (process_hash == NULL) {
		printk(XENO_ERR "cannot allocate processes hash table\n");
		return -ENOMEM;
	}

	ret = xndebug_init();
	if (ret)
		goto fail_debug;

	/*
1502
	 * Setup the mayday stuff early, before userland can mess with
1503 1504
	 * real-time ops.
	 */
1505
	ret = xnarch_init_mayday();
1506 1507 1508 1509 1510 1511 1512 1513
	if (ret)
		goto fail_mayday;

	for (i = 0; i < PROCESS_HASH_SIZE; i++)
		INIT_HLIST_HEAD(&process_hash[i]);

	xnsynch_init(&yield_sync, XNSYNCH_FIFO, NULL);

1514 1515 1516 1517
	ret = cobalt_memdev_init();
	if (ret)
		goto fail_memdev;

1518 1519 1520 1521
	ret = cobalt_register_personality(&cobalt_personality);
	if (ret)
		goto fail_register;

1522 1523 1524 1525
	ret = cobalt_signal_init();
	if (ret)
		goto fail_siginit;

1526 1527 1528 1529
	init_hostrt();
	ipipe_set_hooks(ipipe_root_domain, IPIPE_SYSCALL|IPIPE_KEVENT);
	ipipe_set_hooks(&xnsched_realtime_domain, IPIPE_SYSCALL|IPIPE_TRAP);

1530 1531 1532
	if (gid_arg != -1)
		printk(XENO_INFO "allowing access to group %d\n", gid_arg);

1533
	return 0;
1534 1535
fail_siginit:
	cobalt_unregister_personality(0);
1536
fail_register:
1537 1538
	cobalt_memdev_cleanup();
fail_memdev:
1539
	xnsynch_destroy(&yield_sync);
1540
	xnarch_cleanup_mayday();
1541 1542 1543 1544 1545 1546 1547
fail_mayday:
	xndebug_cleanup();
fail_debug:
	kfree(process_hash);

	return ret;
}