select.c 24.2 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * This file contains the procedures for the handling of select and poll
 *
 * Created for Linux based loosely upon Mathius Lattner's minix
 * patches by Peter MacDonald. Heavily edited by Linus.
 *
 *  4 February 1994
 *     COFF/ELF binary emulation. If the process has the STICKY_TIMEOUTS
 *     flag set in its personality we do *not* modify the given timeout
 *     parameter to reflect time remaining.
 *
 *  24 January 2000
 *     Changed sys_poll()/do_poll() to use PAGE_SIZE chunk-based allocation 
 *     of fds to overcome nfds < 16390 descriptors limit (Tigran Aivazian).
 */

17
#include <linux/kernel.h>
18
#include <linux/sched.h>
Linus Torvalds's avatar
Linus Torvalds committed
19
#include <linux/syscalls.h>
20
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
21 22 23 24
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/personality.h> /* for STICKY_TIMEOUTS */
#include <linux/file.h>
Al Viro's avatar
Al Viro committed
25
#include <linux/fdtable.h>
Linus Torvalds's avatar
Linus Torvalds committed
26
#include <linux/fs.h>
27
#include <linux/rcupdate.h>
28
#include <linux/hrtimer.h>
Linus Torvalds's avatar
Linus Torvalds committed
29 30 31

#include <asm/uaccess.h>

32 33 34 35 36 37 38 39 40 41 42 43 44

/*
 * Estimate expected accuracy in ns from a timeval.
 *
 * After quite a bit of churning around, we've settled on
 * a simple thing of taking 0.1% of the timeout as the
 * slack, with a cap of 100 msec.
 * "nice" tasks get a 0.5% slack instead.
 *
 * Consider this comment an open invitation to come up with even
 * better solutions..
 */

45 46
#define MAX_SLACK	(100 * NSEC_PER_MSEC)

47
static long __estimate_accuracy(struct timespec *tv)
48
{
49
	long slack;
50 51
	int divfactor = 1000;

52 53 54
	if (tv->tv_sec < 0)
		return 0;

55
	if (task_nice(current) > 0)
56 57
		divfactor = divfactor / 5;

58 59 60
	if (tv->tv_sec > MAX_SLACK / (NSEC_PER_SEC/divfactor))
		return MAX_SLACK;

61 62 63
	slack = tv->tv_nsec / divfactor;
	slack += tv->tv_sec * (NSEC_PER_SEC/divfactor);

64 65
	if (slack > MAX_SLACK)
		return MAX_SLACK;
66

67 68 69
	return slack;
}

70
long select_estimate_accuracy(struct timespec *tv)
71 72 73 74 75 76 77 78
{
	unsigned long ret;
	struct timespec now;

	/*
	 * Realtime tasks get a slack of 0 for obvious reasons.
	 */

79
	if (rt_task(current))
80 81 82 83 84 85 86 87 88 89 90 91
		return 0;

	ktime_get_ts(&now);
	now = timespec_sub(*tv, now);
	ret = __estimate_accuracy(&now);
	if (ret < current->timer_slack_ns)
		return current->timer_slack_ns;
	return ret;
}



Linus Torvalds's avatar
Linus Torvalds committed
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
struct poll_table_page {
	struct poll_table_page * next;
	struct poll_table_entry * entry;
	struct poll_table_entry entries[0];
};

#define POLL_TABLE_FULL(table) \
	((unsigned long)((table)->entry+1) > PAGE_SIZE + (unsigned long)(table))

/*
 * Ok, Peter made a complicated, but straightforward multiple_wait() function.
 * I have rewritten this, taking some shortcuts: This code may not be easy to
 * follow, but it should be free of race-conditions, and it's practical. If you
 * understand what I'm doing here, then you understand how the linux
 * sleep/wakeup mechanism works.
 *
 * Two very simple procedures, poll_wait() and poll_freewait() make all the
 * work.  poll_wait() is an inline-function defined in <linux/poll.h>,
 * as all select/poll functions have to call it to add an entry to the
 * poll table.
 */
Adrian Bunk's avatar
Adrian Bunk committed
113 114
static void __pollwait(struct file *filp, wait_queue_head_t *wait_address,
		       poll_table *p);
Linus Torvalds's avatar
Linus Torvalds committed
115 116 117 118

void poll_initwait(struct poll_wqueues *pwq)
{
	init_poll_funcptr(&pwq->pt, __pollwait);
Tejun Heo's avatar
Tejun Heo committed
119
	pwq->polling_task = current;
120
	pwq->triggered = 0;
Linus Torvalds's avatar
Linus Torvalds committed
121 122
	pwq->error = 0;
	pwq->table = NULL;
123
	pwq->inline_index = 0;
Linus Torvalds's avatar
Linus Torvalds committed
124 125 126
}
EXPORT_SYMBOL(poll_initwait);

127 128
static void free_poll_entry(struct poll_table_entry *entry)
{
WANG Cong's avatar
WANG Cong committed
129
	remove_wait_queue(entry->wait_address, &entry->wait);
130 131 132
	fput(entry->filp);
}

Linus Torvalds's avatar
Linus Torvalds committed
133 134 135
void poll_freewait(struct poll_wqueues *pwq)
{
	struct poll_table_page * p = pwq->table;
136 137 138
	int i;
	for (i = 0; i < pwq->inline_index; i++)
		free_poll_entry(pwq->inline_entries + i);
Linus Torvalds's avatar
Linus Torvalds committed
139 140 141 142 143 144 145
	while (p) {
		struct poll_table_entry * entry;
		struct poll_table_page *old;

		entry = p->entry;
		do {
			entry--;
146
			free_poll_entry(entry);
Linus Torvalds's avatar
Linus Torvalds committed
147 148 149 150 151 152 153 154
		} while (entry > p->entries);
		old = p;
		p = p->next;
		free_page((unsigned long) old);
	}
}
EXPORT_SYMBOL(poll_freewait);

Tejun Heo's avatar
Tejun Heo committed
155
static struct poll_table_entry *poll_get_entry(struct poll_wqueues *p)
Linus Torvalds's avatar
Linus Torvalds committed
156 157 158
{
	struct poll_table_page *table = p->table;

159 160 161
	if (p->inline_index < N_INLINE_POLL_ENTRIES)
		return p->inline_entries + p->inline_index++;

Linus Torvalds's avatar
Linus Torvalds committed
162 163 164 165 166 167
	if (!table || POLL_TABLE_FULL(table)) {
		struct poll_table_page *new_table;

		new_table = (struct poll_table_page *) __get_free_page(GFP_KERNEL);
		if (!new_table) {
			p->error = -ENOMEM;
168
			return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
169 170 171 172 173 174 175
		}
		new_table->entry = new_table->entries;
		new_table->next = table;
		p->table = new_table;
		table = new_table;
	}

176 177 178
	return table->entry++;
}

179
static int __pollwake(wait_queue_t *wait, unsigned mode, int sync, void *key)
Tejun Heo's avatar
Tejun Heo committed
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
{
	struct poll_wqueues *pwq = wait->private;
	DECLARE_WAITQUEUE(dummy_wait, pwq->polling_task);

	/*
	 * Although this function is called under waitqueue lock, LOCK
	 * doesn't imply write barrier and the users expect write
	 * barrier semantics on wakeup functions.  The following
	 * smp_wmb() is equivalent to smp_wmb() in try_to_wake_up()
	 * and is paired with set_mb() in poll_schedule_timeout.
	 */
	smp_wmb();
	pwq->triggered = 1;

	/*
	 * Perform the default wake up operation using a dummy
	 * waitqueue.
	 *
	 * TODO: This is hacky but there currently is no interface to
	 * pass in @sync.  @sync is scheduled to be removed and once
	 * that happens, wake_up_process() can be used directly.
	 */
	return default_wake_function(&dummy_wait, mode, sync, key);
}

205 206 207 208 209 210 211 212 213 214
static int pollwake(wait_queue_t *wait, unsigned mode, int sync, void *key)
{
	struct poll_table_entry *entry;

	entry = container_of(wait, struct poll_table_entry, wait);
	if (key && !((unsigned long)key & entry->key))
		return 0;
	return __pollwake(wait, mode, sync, key);
}

215 216 217 218
/* Add a new entry */
static void __pollwait(struct file *filp, wait_queue_head_t *wait_address,
				poll_table *p)
{
Tejun Heo's avatar
Tejun Heo committed
219 220
	struct poll_wqueues *pwq = container_of(p, struct poll_wqueues, pt);
	struct poll_table_entry *entry = poll_get_entry(pwq);
221 222 223 224 225
	if (!entry)
		return;
	get_file(filp);
	entry->filp = filp;
	entry->wait_address = wait_address;
226
	entry->key = p->key;
Tejun Heo's avatar
Tejun Heo committed
227 228
	init_waitqueue_func_entry(&entry->wait, pollwake);
	entry->wait.private = pwq;
WANG Cong's avatar
WANG Cong committed
229
	add_wait_queue(wait_address, &entry->wait);
Linus Torvalds's avatar
Linus Torvalds committed
230 231
}

Tejun Heo's avatar
Tejun Heo committed
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
int poll_schedule_timeout(struct poll_wqueues *pwq, int state,
			  ktime_t *expires, unsigned long slack)
{
	int rc = -EINTR;

	set_current_state(state);
	if (!pwq->triggered)
		rc = schedule_hrtimeout_range(expires, slack, HRTIMER_MODE_ABS);
	__set_current_state(TASK_RUNNING);

	/*
	 * Prepare for the next iteration.
	 *
	 * The following set_mb() serves two purposes.  First, it's
	 * the counterpart rmb of the wmb in pollwake() such that data
	 * written before wake up is always visible after wake up.
	 * Second, the full barrier guarantees that triggered clearing
	 * doesn't pass event check of the next iteration.  Note that
	 * this problem doesn't exist for the first iteration as
	 * add_wait_queue() has full barrier semantics.
	 */
	set_mb(pwq->triggered, 0);

	return rc;
}
EXPORT_SYMBOL(poll_schedule_timeout);

259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
/**
 * poll_select_set_timeout - helper function to setup the timeout value
 * @to:		pointer to timespec variable for the final timeout
 * @sec:	seconds (from user space)
 * @nsec:	nanoseconds (from user space)
 *
 * Note, we do not use a timespec for the user space value here, That
 * way we can use the function for timeval and compat interfaces as well.
 *
 * Returns -EINVAL if sec/nsec are not normalized. Otherwise 0.
 */
int poll_select_set_timeout(struct timespec *to, long sec, long nsec)
{
	struct timespec ts = {.tv_sec = sec, .tv_nsec = nsec};

	if (!timespec_valid(&ts))
		return -EINVAL;

	/* Optimize for the zero timeout value here */
	if (!sec && !nsec) {
		to->tv_sec = to->tv_nsec = 0;
	} else {
		ktime_get_ts(to);
		*to = timespec_add_safe(*to, ts);
	}
	return 0;
}

static int poll_select_copy_remaining(struct timespec *end_time, void __user *p,
				      int timeval, int ret)
{
	struct timespec rts;
	struct timeval rtv;

	if (!p)
		return ret;

	if (current->personality & STICKY_TIMEOUTS)
		goto sticky;

	/* No update for zero timeout */
	if (!end_time->tv_sec && !end_time->tv_nsec)
		return ret;

	ktime_get_ts(&rts);
	rts = timespec_sub(*end_time, rts);
	if (rts.tv_sec < 0)
		rts.tv_sec = rts.tv_nsec = 0;

	if (timeval) {
309 310
		if (sizeof(rtv) > sizeof(rtv.tv_sec) + sizeof(rtv.tv_usec))
			memset(&rtv, 0, sizeof(rtv));
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333
		rtv.tv_sec = rts.tv_sec;
		rtv.tv_usec = rts.tv_nsec / NSEC_PER_USEC;

		if (!copy_to_user(p, &rtv, sizeof(rtv)))
			return ret;

	} else if (!copy_to_user(p, &rts, sizeof(rts)))
		return ret;

	/*
	 * If an application puts its timeval in read-only memory, we
	 * don't want the Linux-specific update to the timeval to
	 * cause a fault after the select has completed
	 * successfully. However, because we're not updating the
	 * timeval, we can't restart the system call.
	 */

sticky:
	if (ret == -ERESTARTNOHAND)
		ret = -EINTR;
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
334 335 336 337 338 339 340 341 342 343 344
#define FDS_IN(fds, n)		(fds->in + n)
#define FDS_OUT(fds, n)		(fds->out + n)
#define FDS_EX(fds, n)		(fds->ex + n)

#define BITS(fds, n)	(*FDS_IN(fds, n)|*FDS_OUT(fds, n)|*FDS_EX(fds, n))

static int max_select_fd(unsigned long n, fd_set_bits *fds)
{
	unsigned long *open_fds;
	unsigned long set;
	int max;
345
	struct fdtable *fdt;
Linus Torvalds's avatar
Linus Torvalds committed
346 347 348 349

	/* handle last in-complete long-word first */
	set = ~(~0UL << (n & (__NFDBITS-1)));
	n /= __NFDBITS;
350 351
	fdt = files_fdtable(current->files);
	open_fds = fdt->open_fds->fds_bits+n;
Linus Torvalds's avatar
Linus Torvalds committed
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
	max = 0;
	if (set) {
		set &= BITS(fds, n);
		if (set) {
			if (!(set & ~*open_fds))
				goto get_max;
			return -EBADF;
		}
	}
	while (n) {
		open_fds--;
		n--;
		set = BITS(fds, n);
		if (!set)
			continue;
		if (set & ~*open_fds)
			return -EBADF;
		if (max)
			continue;
get_max:
		do {
			max++;
			set >>= 1;
		} while (set);
		max += n * __NFDBITS;
	}

	return max;
}

#define POLLIN_SET (POLLRDNORM | POLLRDBAND | POLLIN | POLLHUP | POLLERR)
#define POLLOUT_SET (POLLWRBAND | POLLWRNORM | POLLOUT | POLLERR)
#define POLLEX_SET (POLLPRI)

386 387 388 389 390 391 392 393 394 395 396 397
static inline void wait_key_set(poll_table *wait, unsigned long in,
				unsigned long out, unsigned long bit)
{
	if (wait) {
		wait->key = POLLEX_SET;
		if (in & bit)
			wait->key |= POLLIN_SET;
		if (out & bit)
			wait->key |= POLLOUT_SET;
	}
}

398
int do_select(int n, fd_set_bits *fds, struct timespec *end_time)
Linus Torvalds's avatar
Linus Torvalds committed
399
{
400
	ktime_t expire, *to = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
401 402
	struct poll_wqueues table;
	poll_table *wait;
403
	int retval, i, timed_out = 0;
404
	unsigned long slack = 0;
Linus Torvalds's avatar
Linus Torvalds committed
405

406
	rcu_read_lock();
Linus Torvalds's avatar
Linus Torvalds committed
407
	retval = max_select_fd(n, fds);
408
	rcu_read_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
409 410 411 412 413 414 415

	if (retval < 0)
		return retval;
	n = retval;

	poll_initwait(&table);
	wait = &table.pt;
416
	if (end_time && !end_time->tv_sec && !end_time->tv_nsec) {
Linus Torvalds's avatar
Linus Torvalds committed
417
		wait = NULL;
418 419 420
		timed_out = 1;
	}

421
	if (end_time && !timed_out)
422
		slack = select_estimate_accuracy(end_time);
423

Linus Torvalds's avatar
Linus Torvalds committed
424 425 426 427 428 429 430 431 432 433
	retval = 0;
	for (;;) {
		unsigned long *rinp, *routp, *rexp, *inp, *outp, *exp;

		inp = fds->in; outp = fds->out; exp = fds->ex;
		rinp = fds->res_in; routp = fds->res_out; rexp = fds->res_ex;

		for (i = 0; i < n; ++rinp, ++routp, ++rexp) {
			unsigned long in, out, ex, all_bits, bit = 1, mask, j;
			unsigned long res_in = 0, res_out = 0, res_ex = 0;
434
			const struct file_operations *f_op = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
435 436 437 438 439 440 441 442 443 444
			struct file *file = NULL;

			in = *inp++; out = *outp++; ex = *exp++;
			all_bits = in | out | ex;
			if (all_bits == 0) {
				i += __NFDBITS;
				continue;
			}

			for (j = 0; j < __NFDBITS; ++j, ++i, bit <<= 1) {
445
				int fput_needed;
Linus Torvalds's avatar
Linus Torvalds committed
446 447 448 449
				if (i >= n)
					break;
				if (!(bit & all_bits))
					continue;
450
				file = fget_light(i, &fput_needed);
Linus Torvalds's avatar
Linus Torvalds committed
451 452 453
				if (file) {
					f_op = file->f_op;
					mask = DEFAULT_POLLMASK;
454 455 456 457
					if (f_op && f_op->poll) {
						wait_key_set(wait, in, out, bit);
						mask = (*f_op->poll)(file, wait);
					}
458
					fput_light(file, fput_needed);
Linus Torvalds's avatar
Linus Torvalds committed
459 460 461
					if ((mask & POLLIN_SET) && (in & bit)) {
						res_in |= bit;
						retval++;
462
						wait = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
463 464 465 466
					}
					if ((mask & POLLOUT_SET) && (out & bit)) {
						res_out |= bit;
						retval++;
467
						wait = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
468 469 470 471
					}
					if ((mask & POLLEX_SET) && (ex & bit)) {
						res_ex |= bit;
						retval++;
472
						wait = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
473 474 475 476 477 478 479 480 481
					}
				}
			}
			if (res_in)
				*rinp = res_in;
			if (res_out)
				*routp = res_out;
			if (res_ex)
				*rexp = res_ex;
482
			cond_resched();
Linus Torvalds's avatar
Linus Torvalds committed
483 484
		}
		wait = NULL;
485
		if (retval || timed_out || signal_pending(current))
Linus Torvalds's avatar
Linus Torvalds committed
486
			break;
Pavel Machek's avatar
Pavel Machek committed
487
		if (table.error) {
Linus Torvalds's avatar
Linus Torvalds committed
488 489 490
			retval = table.error;
			break;
		}
491

492 493 494 495 496 497 498 499
		/*
		 * If this is the first loop and we have a timeout
		 * given, then we convert to ktime_t and set the to
		 * pointer to the expiry value.
		 */
		if (end_time && !to) {
			expire = timespec_to_ktime(*end_time);
			to = &expire;
500
		}
501

Tejun Heo's avatar
Tejun Heo committed
502 503
		if (!poll_schedule_timeout(&table, TASK_INTERRUPTIBLE,
					   to, slack))
504
			timed_out = 1;
Linus Torvalds's avatar
Linus Torvalds committed
505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
	}

	poll_freewait(&table);

	return retval;
}

/*
 * We can actually return ERESTARTSYS instead of EINTR, but I'd
 * like to be certain this leads to no problems. So I return
 * EINTR just for safety.
 *
 * Update: ERESTARTSYS breaks at least the xview clock binary, so
 * I'm trying ERESTARTNOHAND which restart only when you want to.
 */
520
int core_sys_select(int n, fd_set __user *inp, fd_set __user *outp,
521
			   fd_set __user *exp, struct timespec *end_time)
Linus Torvalds's avatar
Linus Torvalds committed
522 523
{
	fd_set_bits fds;
Andrew Morton's avatar
Andrew Morton committed
524
	void *bits;
525
	int ret, max_fds;
526
	unsigned int size;
527
	struct fdtable *fdt;
528
	/* Allocate small arguments on the stack to save memory and be faster */
529
	long stack_fds[SELECT_STACK_ALLOC/sizeof(long)];
Linus Torvalds's avatar
Linus Torvalds committed
530 531 532 533 534

	ret = -EINVAL;
	if (n < 0)
		goto out_nofds;

535
	/* max_fds can increase, so grab it once to avoid race */
536
	rcu_read_lock();
537
	fdt = files_fdtable(current->files);
538
	max_fds = fdt->max_fds;
539
	rcu_read_unlock();
540 541
	if (n > max_fds)
		n = max_fds;
Linus Torvalds's avatar
Linus Torvalds committed
542 543 544 545 546 547 548

	/*
	 * We need 6 bitmaps (in/out/ex for both incoming and outgoing),
	 * since we used fdset we need to allocate memory in units of
	 * long-words. 
	 */
	size = FDS_BYTES(n);
549 550 551 552
	bits = stack_fds;
	if (size > sizeof(stack_fds) / 6) {
		/* Not enough space in on-stack array; must use kmalloc */
		ret = -ENOMEM;
553
		bits = kmalloc(6 * size, GFP_KERNEL);
554 555 556
		if (!bits)
			goto out_nofds;
	}
Andrew Morton's avatar
Andrew Morton committed
557 558 559 560 561 562
	fds.in      = bits;
	fds.out     = bits +   size;
	fds.ex      = bits + 2*size;
	fds.res_in  = bits + 3*size;
	fds.res_out = bits + 4*size;
	fds.res_ex  = bits + 5*size;
Linus Torvalds's avatar
Linus Torvalds committed
563 564 565 566 567 568 569 570 571

	if ((ret = get_fd_set(n, inp, fds.in)) ||
	    (ret = get_fd_set(n, outp, fds.out)) ||
	    (ret = get_fd_set(n, exp, fds.ex)))
		goto out;
	zero_fd_set(n, fds.res_in);
	zero_fd_set(n, fds.res_out);
	zero_fd_set(n, fds.res_ex);

572
	ret = do_select(n, &fds, end_time);
Linus Torvalds's avatar
Linus Torvalds committed
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588

	if (ret < 0)
		goto out;
	if (!ret) {
		ret = -ERESTARTNOHAND;
		if (signal_pending(current))
			goto out;
		ret = 0;
	}

	if (set_fd_set(n, inp, fds.res_in) ||
	    set_fd_set(n, outp, fds.res_out) ||
	    set_fd_set(n, exp, fds.res_ex))
		ret = -EFAULT;

out:
589 590
	if (bits != stack_fds)
		kfree(bits);
Linus Torvalds's avatar
Linus Torvalds committed
591 592 593 594
out_nofds:
	return ret;
}

595 596
SYSCALL_DEFINE5(select, int, n, fd_set __user *, inp, fd_set __user *, outp,
		fd_set __user *, exp, struct timeval __user *, tvp)
597
{
598
	struct timespec end_time, *to = NULL;
599 600 601 602 603 604 605
	struct timeval tv;
	int ret;

	if (tvp) {
		if (copy_from_user(&tv, tvp, sizeof(tv)))
			return -EFAULT;

606
		to = &end_time;
607 608 609
		if (poll_select_set_timeout(to,
				tv.tv_sec + (tv.tv_usec / USEC_PER_SEC),
				(tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC))
610 611 612
			return -EINVAL;
	}

613 614
	ret = core_sys_select(n, inp, outp, exp, to);
	ret = poll_select_copy_remaining(&end_time, tvp, 1, ret);
615 616 617 618

	return ret;
}

619
#ifdef HAVE_SET_RESTORE_SIGMASK
620 621 622
static long do_pselect(int n, fd_set __user *inp, fd_set __user *outp,
		       fd_set __user *exp, struct timespec __user *tsp,
		       const sigset_t __user *sigmask, size_t sigsetsize)
623 624
{
	sigset_t ksigmask, sigsaved;
625
	struct timespec ts, end_time, *to = NULL;
626 627 628 629 630 631
	int ret;

	if (tsp) {
		if (copy_from_user(&ts, tsp, sizeof(ts)))
			return -EFAULT;

632 633
		to = &end_time;
		if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))
634 635 636 637 638 639 640 641 642 643 644 645 646 647
			return -EINVAL;
	}

	if (sigmask) {
		/* XXX: Don't preclude handling different sized sigset_t's.  */
		if (sigsetsize != sizeof(sigset_t))
			return -EINVAL;
		if (copy_from_user(&ksigmask, sigmask, sizeof(ksigmask)))
			return -EFAULT;

		sigdelsetmask(&ksigmask, sigmask(SIGKILL)|sigmask(SIGSTOP));
		sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved);
	}

Bernd Schmidt's avatar
Bernd Schmidt committed
648
	ret = core_sys_select(n, inp, outp, exp, to);
649
	ret = poll_select_copy_remaining(&end_time, tsp, 0, ret);
650 651 652 653 654 655 656 657 658 659

	if (ret == -ERESTARTNOHAND) {
		/*
		 * Don't restore the signal mask yet. Let do_signal() deliver
		 * the signal on the way back to userspace, before the signal
		 * mask is restored.
		 */
		if (sigmask) {
			memcpy(&current->saved_sigmask, &sigsaved,
					sizeof(sigsaved));
660
			set_restore_sigmask();
661 662 663 664 665 666 667 668 669 670 671 672 673
		}
	} else if (sigmask)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

	return ret;
}

/*
 * Most architectures can't handle 7-argument syscalls. So we provide a
 * 6-argument version where the sixth argument is a pointer to a structure
 * which has a pointer to the sigset_t itself followed by a size_t containing
 * the sigset size.
 */
674 675 676
SYSCALL_DEFINE6(pselect6, int, n, fd_set __user *, inp, fd_set __user *, outp,
		fd_set __user *, exp, struct timespec __user *, tsp,
		void __user *, sig)
677 678 679 680 681 682
{
	size_t sigsetsize = 0;
	sigset_t __user *up = NULL;

	if (sig) {
		if (!access_ok(VERIFY_READ, sig, sizeof(void *)+sizeof(size_t))
683
		    || __get_user(up, (sigset_t __user * __user *)sig)
684
		    || __get_user(sigsetsize,
685
				(size_t __user *)(sig+sizeof(void *))))
686 687 688
			return -EFAULT;
	}

689
	return do_pselect(n, inp, outp, exp, tsp, up, sigsetsize);
690
}
691
#endif /* HAVE_SET_RESTORE_SIGMASK */
692

693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
#ifdef __ARCH_WANT_SYS_OLD_SELECT
struct sel_arg_struct {
	unsigned long n;
	fd_set __user *inp, *outp, *exp;
	struct timeval __user *tvp;
};

SYSCALL_DEFINE1(old_select, struct sel_arg_struct __user *, arg)
{
	struct sel_arg_struct a;

	if (copy_from_user(&a, arg, sizeof(a)))
		return -EFAULT;
	return sys_select(a.n, a.inp, a.outp, a.exp, a.tvp);
}
#endif

Linus Torvalds's avatar
Linus Torvalds committed
710 711 712 713 714 715 716 717
struct poll_list {
	struct poll_list *next;
	int len;
	struct pollfd entries[0];
};

#define POLLFD_PER_PAGE  ((PAGE_SIZE-sizeof(struct poll_list)) / sizeof(struct pollfd))

718 719 720 721 722 723 724 725
/*
 * Fish for pollable events on the pollfd->fd file descriptor. We're only
 * interested in events matching the pollfd->events mask, and the result
 * matching that mask is both recorded in pollfd->revents and returned. The
 * pwait poll_table will be used by the fd-provided poll handler for waiting,
 * if non-NULL.
 */
static inline unsigned int do_pollfd(struct pollfd *pollfd, poll_table *pwait)
Linus Torvalds's avatar
Linus Torvalds committed
726
{
727 728 729 730 731 732 733 734 735 736 737 738 739
	unsigned int mask;
	int fd;

	mask = 0;
	fd = pollfd->fd;
	if (fd >= 0) {
		int fput_needed;
		struct file * file;

		file = fget_light(fd, &fput_needed);
		mask = POLLNVAL;
		if (file != NULL) {
			mask = DEFAULT_POLLMASK;
740 741 742 743
			if (file->f_op && file->f_op->poll) {
				if (pwait)
					pwait->key = pollfd->events |
							POLLERR | POLLHUP;
744
				mask = file->f_op->poll(file, pwait);
745
			}
746 747 748
			/* Mask out unneeded events. */
			mask &= pollfd->events | POLLERR | POLLHUP;
			fput_light(file, fput_needed);
Linus Torvalds's avatar
Linus Torvalds committed
749 750
		}
	}
751 752 753
	pollfd->revents = mask;

	return mask;
Linus Torvalds's avatar
Linus Torvalds committed
754 755 756
}

static int do_poll(unsigned int nfds,  struct poll_list *list,
757
		   struct poll_wqueues *wait, struct timespec *end_time)
Linus Torvalds's avatar
Linus Torvalds committed
758 759
{
	poll_table* pt = &wait->pt;
760 761
	ktime_t expire, *to = NULL;
	int timed_out = 0, count = 0;
762
	unsigned long slack = 0;
Linus Torvalds's avatar
Linus Torvalds committed
763

764
	/* Optimise the no-wait case */
765
	if (end_time && !end_time->tv_sec && !end_time->tv_nsec) {
Linus Torvalds's avatar
Linus Torvalds committed
766
		pt = NULL;
767 768
		timed_out = 1;
	}
769

770
	if (end_time && !timed_out)
771
		slack = select_estimate_accuracy(end_time);
772

Linus Torvalds's avatar
Linus Torvalds committed
773 774
	for (;;) {
		struct poll_list *walk;
775

776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
		for (walk = list; walk != NULL; walk = walk->next) {
			struct pollfd * pfd, * pfd_end;

			pfd = walk->entries;
			pfd_end = pfd + walk->len;
			for (; pfd != pfd_end; pfd++) {
				/*
				 * Fish for events. If we found one, record it
				 * and kill the poll_table, so we don't
				 * needlessly register any other waiters after
				 * this. They'll get immediately deregistered
				 * when we break out and return.
				 */
				if (do_pollfd(pfd, pt)) {
					count++;
					pt = NULL;
				}
			}
Linus Torvalds's avatar
Linus Torvalds committed
794
		}
795 796 797 798
		/*
		 * All waiters have already been registered, so don't provide
		 * a poll_table to them on the next loop iteration.
		 */
Linus Torvalds's avatar
Linus Torvalds committed
799
		pt = NULL;
800 801 802 803 804
		if (!count) {
			count = wait->error;
			if (signal_pending(current))
				count = -EINTR;
		}
805
		if (count || timed_out)
Linus Torvalds's avatar
Linus Torvalds committed
806
			break;
807

808 809 810 811 812 813 814 815
		/*
		 * If this is the first loop and we have a timeout
		 * given, then we convert to ktime_t and set the to
		 * pointer to the expiry value.
		 */
		if (end_time && !to) {
			expire = timespec_to_ktime(*end_time);
			to = &expire;
816 817
		}

Tejun Heo's avatar
Tejun Heo committed
818
		if (!poll_schedule_timeout(wait, TASK_INTERRUPTIBLE, to, slack))
819
			timed_out = 1;
Linus Torvalds's avatar
Linus Torvalds committed
820 821 822 823
	}
	return count;
}

824 825 826
#define N_STACK_PPS ((sizeof(stack_pps) - sizeof(struct poll_list))  / \
			sizeof(struct pollfd))

827 828
int do_sys_poll(struct pollfd __user *ufds, unsigned int nfds,
		struct timespec *end_time)
Linus Torvalds's avatar
Linus Torvalds committed
829 830
{
	struct poll_wqueues table;
831
 	int err = -EFAULT, fdcount, len, size;
832 833 834 835
	/* Allocate small arguments on the stack to save memory and be
	   faster - use long to make sure the buffer is aligned properly
	   on 64 bit archs to avoid unaligned access */
	long stack_pps[POLL_STACK_ALLOC/sizeof(long)];
836 837 838
	struct poll_list *const head = (struct poll_list *)stack_pps;
 	struct poll_list *walk = head;
 	unsigned long todo = nfds;
Linus Torvalds's avatar
Linus Torvalds committed
839

Jiri Slaby's avatar
Jiri Slaby committed
840
	if (nfds > rlimit(RLIMIT_NOFILE))
Linus Torvalds's avatar
Linus Torvalds committed
841 842
		return -EINVAL;

843 844 845 846 847 848
	len = min_t(unsigned int, nfds, N_STACK_PPS);
	for (;;) {
		walk->next = NULL;
		walk->len = len;
		if (!len)
			break;
Linus Torvalds's avatar
Linus Torvalds committed
849

850 851 852 853 854 855 856
		if (copy_from_user(walk->entries, ufds + nfds-todo,
					sizeof(struct pollfd) * walk->len))
			goto out_fds;

		todo -= walk->len;
		if (!todo)
			break;
Linus Torvalds's avatar
Linus Torvalds committed
857

858 859 860 861 862
		len = min(todo, POLLFD_PER_PAGE);
		size = sizeof(struct poll_list) + sizeof(struct pollfd) * len;
		walk = walk->next = kmalloc(size, GFP_KERNEL);
		if (!walk) {
			err = -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
863 864 865
			goto out_fds;
		}
	}
866

867
	poll_initwait(&table);
868
	fdcount = do_poll(nfds, head, &table, end_time);
869
	poll_freewait(&table);
Linus Torvalds's avatar
Linus Torvalds committed
870

871
	for (walk = head; walk; walk = walk->next) {
Linus Torvalds's avatar
Linus Torvalds committed
872 873 874
		struct pollfd *fds = walk->entries;
		int j;

875 876
		for (j = 0; j < walk->len; j++, ufds++)
			if (__put_user(fds[j].revents, &ufds->revents))
Linus Torvalds's avatar
Linus Torvalds committed
877 878
				goto out_fds;
  	}
879

Linus Torvalds's avatar
Linus Torvalds committed
880 881
	err = fdcount;
out_fds:
882 883 884 885 886
	walk = head->next;
	while (walk) {
		struct poll_list *pos = walk;
		walk = walk->next;
		kfree(pos);
Linus Torvalds's avatar
Linus Torvalds committed
887
	}
888

Linus Torvalds's avatar
Linus Torvalds committed
889 890
	return err;
}
891

892 893
static long do_restart_poll(struct restart_block *restart_block)
{
894 895 896
	struct pollfd __user *ufds = restart_block->poll.ufds;
	int nfds = restart_block->poll.nfds;
	struct timespec *to = NULL, end_time;
897 898
	int ret;

899 900 901 902 903 904 905 906
	if (restart_block->poll.has_timeout) {
		end_time.tv_sec = restart_block->poll.tv_sec;
		end_time.tv_nsec = restart_block->poll.tv_nsec;
		to = &end_time;
	}

	ret = do_sys_poll(ufds, nfds, to);

907 908 909 910 911 912 913
	if (ret == -EINTR) {
		restart_block->fn = do_restart_poll;
		ret = -ERESTART_RESTARTBLOCK;
	}
	return ret;
}

914
SYSCALL_DEFINE3(poll, struct pollfd __user *, ufds, unsigned int, nfds,
915
		int, timeout_msecs)
916
{
917
	struct timespec end_time, *to = NULL;
918
	int ret;
919

920 921 922 923
	if (timeout_msecs >= 0) {
		to = &end_time;
		poll_select_set_timeout(to, timeout_msecs / MSEC_PER_SEC,
			NSEC_PER_MSEC * (timeout_msecs % MSEC_PER_SEC));
924 925
	}

926 927
	ret = do_sys_poll(ufds, nfds, to);

928 929
	if (ret == -EINTR) {
		struct restart_block *restart_block;
930

931 932
		restart_block = &current_thread_info()->restart_block;
		restart_block->fn = do_restart_poll;
933 934 935 936 937 938 939 940 941 942
		restart_block->poll.ufds = ufds;
		restart_block->poll.nfds = nfds;

		if (timeout_msecs >= 0) {
			restart_block->poll.tv_sec = end_time.tv_sec;
			restart_block->poll.tv_nsec = end_time.tv_nsec;
			restart_block->poll.has_timeout = 1;
		} else
			restart_block->poll.has_timeout = 0;

943 944 945
		ret = -ERESTART_RESTARTBLOCK;
	}
	return ret;
946 947
}

948
#ifdef HAVE_SET_RESTORE_SIGMASK
949 950 951
SYSCALL_DEFINE5(ppoll, struct pollfd __user *, ufds, unsigned int, nfds,
		struct timespec __user *, tsp, const sigset_t __user *, sigmask,
		size_t, sigsetsize)
952 953
{
	sigset_t ksigmask, sigsaved;
954
	struct timespec ts, end_time, *to = NULL;
955 956 957 958 959 960
	int ret;

	if (tsp) {
		if (copy_from_user(&ts, tsp, sizeof(ts)))
			return -EFAULT;

961 962 963
		to = &end_time;
		if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))
			return -EINVAL;
964 965 966 967 968 969 970 971 972 973 974 975 976
	}

	if (sigmask) {
		/* XXX: Don't preclude handling different sized sigset_t's.  */
		if (sigsetsize != sizeof(sigset_t))
			return -EINVAL;
		if (copy_from_user(&ksigmask, sigmask, sizeof(ksigmask)))
			return -EFAULT;

		sigdelsetmask(&ksigmask, sigmask(SIGKILL)|sigmask(SIGSTOP));
		sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved);
	}

977
	ret = do_sys_poll(ufds, nfds, to);
978 979 980 981 982 983 984 985 986 987 988

	/* We can restart this syscall, usually */
	if (ret == -EINTR) {
		/*
		 * Don't restore the signal mask yet. Let do_signal() deliver
		 * the signal on the way back to userspace, before the signal
		 * mask is restored.
		 */
		if (sigmask) {
			memcpy(&current->saved_sigmask, &sigsaved,
					sizeof(sigsaved));
989
			set_restore_sigmask();
990 991 992 993 994
		}
		ret = -ERESTARTNOHAND;
	} else if (sigmask)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

995
	ret = poll_select_copy_remaining(&end_time, tsp, 0, ret);
996 997 998

	return ret;
}
999
#endif /* HAVE_SET_RESTORE_SIGMASK */