select.c 25.7 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * This file contains the procedures for the handling of select and poll
 *
 * Created for Linux based loosely upon Mathius Lattner's minix
 * patches by Peter MacDonald. Heavily edited by Linus.
 *
 *  4 February 1994
 *     COFF/ELF binary emulation. If the process has the STICKY_TIMEOUTS
 *     flag set in its personality we do *not* modify the given timeout
 *     parameter to reflect time remaining.
 *
 *  24 January 2000
 *     Changed sys_poll()/do_poll() to use PAGE_SIZE chunk-based allocation 
 *     of fds to overcome nfds < 16390 descriptors limit (Tigran Aivazian).
 */

17
#include <linux/kernel.h>
18
#include <linux/sched.h>
Linus Torvalds's avatar
Linus Torvalds committed
19
#include <linux/syscalls.h>
20
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
21 22 23 24
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/personality.h> /* for STICKY_TIMEOUTS */
#include <linux/file.h>
Al Viro's avatar
Al Viro committed
25
#include <linux/fdtable.h>
Linus Torvalds's avatar
Linus Torvalds committed
26
#include <linux/fs.h>
27
#include <linux/rcupdate.h>
28
#include <linux/hrtimer.h>
29
#include <linux/sched/rt.h>
30
#include <linux/freezer.h>
31
#include <net/busy_poll.h>
32
#include <linux/vmalloc.h>
Linus Torvalds's avatar
Linus Torvalds committed
33 34 35

#include <asm/uaccess.h>

36 37 38 39 40 41 42 43 44 45 46 47 48

/*
 * Estimate expected accuracy in ns from a timeval.
 *
 * After quite a bit of churning around, we've settled on
 * a simple thing of taking 0.1% of the timeout as the
 * slack, with a cap of 100 msec.
 * "nice" tasks get a 0.5% slack instead.
 *
 * Consider this comment an open invitation to come up with even
 * better solutions..
 */

49 50
#define MAX_SLACK	(100 * NSEC_PER_MSEC)

51
static long __estimate_accuracy(struct timespec64 *tv)
52
{
53
	long slack;
54 55
	int divfactor = 1000;

56 57 58
	if (tv->tv_sec < 0)
		return 0;

59
	if (task_nice(current) > 0)
60 61
		divfactor = divfactor / 5;

62 63 64
	if (tv->tv_sec > MAX_SLACK / (NSEC_PER_SEC/divfactor))
		return MAX_SLACK;

65 66 67
	slack = tv->tv_nsec / divfactor;
	slack += tv->tv_sec * (NSEC_PER_SEC/divfactor);

68 69
	if (slack > MAX_SLACK)
		return MAX_SLACK;
70

71 72 73
	return slack;
}

74
u64 select_estimate_accuracy(struct timespec64 *tv)
75
{
76
	u64 ret;
77
	struct timespec64 now;
78 79 80 81 82

	/*
	 * Realtime tasks get a slack of 0 for obvious reasons.
	 */

83
	if (rt_task(current))
84 85
		return 0;

86 87
	ktime_get_ts64(&now);
	now = timespec64_sub(*tv, now);
88 89 90 91 92 93 94 95
	ret = __estimate_accuracy(&now);
	if (ret < current->timer_slack_ns)
		return current->timer_slack_ns;
	return ret;
}



Linus Torvalds's avatar
Linus Torvalds committed
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
struct poll_table_page {
	struct poll_table_page * next;
	struct poll_table_entry * entry;
	struct poll_table_entry entries[0];
};

#define POLL_TABLE_FULL(table) \
	((unsigned long)((table)->entry+1) > PAGE_SIZE + (unsigned long)(table))

/*
 * Ok, Peter made a complicated, but straightforward multiple_wait() function.
 * I have rewritten this, taking some shortcuts: This code may not be easy to
 * follow, but it should be free of race-conditions, and it's practical. If you
 * understand what I'm doing here, then you understand how the linux
 * sleep/wakeup mechanism works.
 *
 * Two very simple procedures, poll_wait() and poll_freewait() make all the
 * work.  poll_wait() is an inline-function defined in <linux/poll.h>,
 * as all select/poll functions have to call it to add an entry to the
 * poll table.
 */
117 118
static void __pollwait(struct file *filp, wait_queue_head_t *wait_address,
		       poll_table *p);
Linus Torvalds's avatar
Linus Torvalds committed
119 120 121 122

void poll_initwait(struct poll_wqueues *pwq)
{
	init_poll_funcptr(&pwq->pt, __pollwait);
123
	pwq->polling_task = current;
124
	pwq->triggered = 0;
Linus Torvalds's avatar
Linus Torvalds committed
125 126
	pwq->error = 0;
	pwq->table = NULL;
127
	pwq->inline_index = 0;
Linus Torvalds's avatar
Linus Torvalds committed
128 129 130
}
EXPORT_SYMBOL(poll_initwait);

131 132
static void free_poll_entry(struct poll_table_entry *entry)
{
WANG Cong's avatar
WANG Cong committed
133
	remove_wait_queue(entry->wait_address, &entry->wait);
134 135 136
	fput(entry->filp);
}

Linus Torvalds's avatar
Linus Torvalds committed
137 138 139
void poll_freewait(struct poll_wqueues *pwq)
{
	struct poll_table_page * p = pwq->table;
140 141 142
	int i;
	for (i = 0; i < pwq->inline_index; i++)
		free_poll_entry(pwq->inline_entries + i);
Linus Torvalds's avatar
Linus Torvalds committed
143 144 145 146 147 148 149
	while (p) {
		struct poll_table_entry * entry;
		struct poll_table_page *old;

		entry = p->entry;
		do {
			entry--;
150
			free_poll_entry(entry);
Linus Torvalds's avatar
Linus Torvalds committed
151 152 153 154 155 156 157 158
		} while (entry > p->entries);
		old = p;
		p = p->next;
		free_page((unsigned long) old);
	}
}
EXPORT_SYMBOL(poll_freewait);

159
static struct poll_table_entry *poll_get_entry(struct poll_wqueues *p)
Linus Torvalds's avatar
Linus Torvalds committed
160 161 162
{
	struct poll_table_page *table = p->table;

163 164 165
	if (p->inline_index < N_INLINE_POLL_ENTRIES)
		return p->inline_entries + p->inline_index++;

Linus Torvalds's avatar
Linus Torvalds committed
166 167 168 169 170 171
	if (!table || POLL_TABLE_FULL(table)) {
		struct poll_table_page *new_table;

		new_table = (struct poll_table_page *) __get_free_page(GFP_KERNEL);
		if (!new_table) {
			p->error = -ENOMEM;
172
			return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
173 174 175 176 177 178 179
		}
		new_table->entry = new_table->entries;
		new_table->next = table;
		p->table = new_table;
		table = new_table;
	}

180 181 182
	return table->entry++;
}

183
static int __pollwake(wait_queue_t *wait, unsigned mode, int sync, void *key)
184 185 186 187 188 189 190 191 192
{
	struct poll_wqueues *pwq = wait->private;
	DECLARE_WAITQUEUE(dummy_wait, pwq->polling_task);

	/*
	 * Although this function is called under waitqueue lock, LOCK
	 * doesn't imply write barrier and the users expect write
	 * barrier semantics on wakeup functions.  The following
	 * smp_wmb() is equivalent to smp_wmb() in try_to_wake_up()
193
	 * and is paired with smp_store_mb() in poll_schedule_timeout.
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
	 */
	smp_wmb();
	pwq->triggered = 1;

	/*
	 * Perform the default wake up operation using a dummy
	 * waitqueue.
	 *
	 * TODO: This is hacky but there currently is no interface to
	 * pass in @sync.  @sync is scheduled to be removed and once
	 * that happens, wake_up_process() can be used directly.
	 */
	return default_wake_function(&dummy_wait, mode, sync, key);
}

209 210 211 212 213 214 215 216 217 218
static int pollwake(wait_queue_t *wait, unsigned mode, int sync, void *key)
{
	struct poll_table_entry *entry;

	entry = container_of(wait, struct poll_table_entry, wait);
	if (key && !((unsigned long)key & entry->key))
		return 0;
	return __pollwake(wait, mode, sync, key);
}

219 220 221 222
/* Add a new entry */
static void __pollwait(struct file *filp, wait_queue_head_t *wait_address,
				poll_table *p)
{
223 224
	struct poll_wqueues *pwq = container_of(p, struct poll_wqueues, pt);
	struct poll_table_entry *entry = poll_get_entry(pwq);
225 226
	if (!entry)
		return;
227
	entry->filp = get_file(filp);
228
	entry->wait_address = wait_address;
229
	entry->key = p->_key;
230 231
	init_waitqueue_func_entry(&entry->wait, pollwake);
	entry->wait.private = pwq;
WANG Cong's avatar
WANG Cong committed
232
	add_wait_queue(wait_address, &entry->wait);
Linus Torvalds's avatar
Linus Torvalds committed
233 234
}

235 236 237 238 239 240 241
int poll_schedule_timeout(struct poll_wqueues *pwq, int state,
			  ktime_t *expires, unsigned long slack)
{
	int rc = -EINTR;

	set_current_state(state);
	if (!pwq->triggered)
242
		rc = schedule_hrtimeout_range(expires, slack, HRTIMER_MODE_ABS);
243 244 245 246 247
	__set_current_state(TASK_RUNNING);

	/*
	 * Prepare for the next iteration.
	 *
248
	 * The following smp_store_mb() serves two purposes.  First, it's
249 250 251 252 253 254 255
	 * the counterpart rmb of the wmb in pollwake() such that data
	 * written before wake up is always visible after wake up.
	 * Second, the full barrier guarantees that triggered clearing
	 * doesn't pass event check of the next iteration.  Note that
	 * this problem doesn't exist for the first iteration as
	 * add_wait_queue() has full barrier semantics.
	 */
256
	smp_store_mb(pwq->triggered, 0);
257 258 259 260 261

	return rc;
}
EXPORT_SYMBOL(poll_schedule_timeout);

262 263
/**
 * poll_select_set_timeout - helper function to setup the timeout value
264
 * @to:		pointer to timespec64 variable for the final timeout
265 266 267 268 269 270 271 272
 * @sec:	seconds (from user space)
 * @nsec:	nanoseconds (from user space)
 *
 * Note, we do not use a timespec for the user space value here, That
 * way we can use the function for timeval and compat interfaces as well.
 *
 * Returns -EINVAL if sec/nsec are not normalized. Otherwise 0.
 */
273
int poll_select_set_timeout(struct timespec64 *to, time64_t sec, long nsec)
274
{
275
	struct timespec64 ts = {.tv_sec = sec, .tv_nsec = nsec};
276

277
	if (!timespec64_valid(&ts))
278 279 280 281 282 283
		return -EINVAL;

	/* Optimize for the zero timeout value here */
	if (!sec && !nsec) {
		to->tv_sec = to->tv_nsec = 0;
	} else {
284 285
		ktime_get_ts64(to);
		*to = timespec64_add_safe(*to, ts);
286 287 288 289
	}
	return 0;
}

290 291
static int poll_select_copy_remaining(struct timespec64 *end_time,
				      void __user *p,
292 293
				      int timeval, int ret)
{
294
	struct timespec64 rts64;
295 296 297 298 299 300 301 302 303 304 305 306 307
	struct timespec rts;
	struct timeval rtv;

	if (!p)
		return ret;

	if (current->personality & STICKY_TIMEOUTS)
		goto sticky;

	/* No update for zero timeout */
	if (!end_time->tv_sec && !end_time->tv_nsec)
		return ret;

308 309 310 311 312 313
	ktime_get_ts64(&rts64);
	rts64 = timespec64_sub(*end_time, rts64);
	if (rts64.tv_sec < 0)
		rts64.tv_sec = rts64.tv_nsec = 0;

	rts = timespec64_to_timespec(rts64);
314 315

	if (timeval) {
316 317
		if (sizeof(rtv) > sizeof(rtv.tv_sec) + sizeof(rtv.tv_usec))
			memset(&rtv, 0, sizeof(rtv));
318 319
		rtv.tv_sec = rts64.tv_sec;
		rtv.tv_usec = rts64.tv_nsec / NSEC_PER_USEC;
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340

		if (!copy_to_user(p, &rtv, sizeof(rtv)))
			return ret;

	} else if (!copy_to_user(p, &rts, sizeof(rts)))
		return ret;

	/*
	 * If an application puts its timeval in read-only memory, we
	 * don't want the Linux-specific update to the timeval to
	 * cause a fault after the select has completed
	 * successfully. However, because we're not updating the
	 * timeval, we can't restart the system call.
	 */

sticky:
	if (ret == -ERESTARTNOHAND)
		ret = -EINTR;
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
341 342 343 344 345 346 347 348 349 350 351
#define FDS_IN(fds, n)		(fds->in + n)
#define FDS_OUT(fds, n)		(fds->out + n)
#define FDS_EX(fds, n)		(fds->ex + n)

#define BITS(fds, n)	(*FDS_IN(fds, n)|*FDS_OUT(fds, n)|*FDS_EX(fds, n))

static int max_select_fd(unsigned long n, fd_set_bits *fds)
{
	unsigned long *open_fds;
	unsigned long set;
	int max;
352
	struct fdtable *fdt;
Linus Torvalds's avatar
Linus Torvalds committed
353 354

	/* handle last in-complete long-word first */
355 356
	set = ~(~0UL << (n & (BITS_PER_LONG-1)));
	n /= BITS_PER_LONG;
357
	fdt = files_fdtable(current->files);
358
	open_fds = fdt->open_fds + n;
Linus Torvalds's avatar
Linus Torvalds committed
359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
	max = 0;
	if (set) {
		set &= BITS(fds, n);
		if (set) {
			if (!(set & ~*open_fds))
				goto get_max;
			return -EBADF;
		}
	}
	while (n) {
		open_fds--;
		n--;
		set = BITS(fds, n);
		if (!set)
			continue;
		if (set & ~*open_fds)
			return -EBADF;
		if (max)
			continue;
get_max:
		do {
			max++;
			set >>= 1;
		} while (set);
383
		max += n * BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
384 385 386 387 388 389 390 391 392
	}

	return max;
}

#define POLLIN_SET (POLLRDNORM | POLLRDBAND | POLLIN | POLLHUP | POLLERR)
#define POLLOUT_SET (POLLWRBAND | POLLWRNORM | POLLOUT | POLLERR)
#define POLLEX_SET (POLLPRI)

393
static inline void wait_key_set(poll_table *wait, unsigned long in,
394 395
				unsigned long out, unsigned long bit,
				unsigned int ll_flag)
396
{
397
	wait->_key = POLLEX_SET | ll_flag;
398 399 400 401
	if (in & bit)
		wait->_key |= POLLIN_SET;
	if (out & bit)
		wait->_key |= POLLOUT_SET;
402 403
}

404
int do_select(int n, fd_set_bits *fds, struct timespec64 *end_time)
Linus Torvalds's avatar
Linus Torvalds committed
405
{
406
	ktime_t expire, *to = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
407 408
	struct poll_wqueues table;
	poll_table *wait;
409
	int retval, i, timed_out = 0;
410
	u64 slack = 0;
411
	unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;
412
	unsigned long busy_end = 0;
Linus Torvalds's avatar
Linus Torvalds committed
413

414
	rcu_read_lock();
Linus Torvalds's avatar
Linus Torvalds committed
415
	retval = max_select_fd(n, fds);
416
	rcu_read_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
417 418 419 420 421 422 423

	if (retval < 0)
		return retval;
	n = retval;

	poll_initwait(&table);
	wait = &table.pt;
424
	if (end_time && !end_time->tv_sec && !end_time->tv_nsec) {
425
		wait->_qproc = NULL;
426 427 428
		timed_out = 1;
	}

429
	if (end_time && !timed_out)
430
		slack = select_estimate_accuracy(end_time);
431

Linus Torvalds's avatar
Linus Torvalds committed
432 433 434
	retval = 0;
	for (;;) {
		unsigned long *rinp, *routp, *rexp, *inp, *outp, *exp;
435
		bool can_busy_loop = false;
Linus Torvalds's avatar
Linus Torvalds committed
436 437 438 439 440 441 442 443 444 445 446

		inp = fds->in; outp = fds->out; exp = fds->ex;
		rinp = fds->res_in; routp = fds->res_out; rexp = fds->res_ex;

		for (i = 0; i < n; ++rinp, ++routp, ++rexp) {
			unsigned long in, out, ex, all_bits, bit = 1, mask, j;
			unsigned long res_in = 0, res_out = 0, res_ex = 0;

			in = *inp++; out = *outp++; ex = *exp++;
			all_bits = in | out | ex;
			if (all_bits == 0) {
447
				i += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
448 449 450
				continue;
			}

451
			for (j = 0; j < BITS_PER_LONG; ++j, ++i, bit <<= 1) {
452
				struct fd f;
Linus Torvalds's avatar
Linus Torvalds committed
453 454 455 456
				if (i >= n)
					break;
				if (!(bit & all_bits))
					continue;
457 458 459 460
				f = fdget(i);
				if (f.file) {
					const struct file_operations *f_op;
					f_op = f.file->f_op;
Linus Torvalds's avatar
Linus Torvalds committed
461
					mask = DEFAULT_POLLMASK;
Al Viro's avatar
Al Viro committed
462
					if (f_op->poll) {
463
						wait_key_set(wait, in, out,
464
							     bit, busy_flag);
465
						mask = (*f_op->poll)(f.file, wait);
466
					}
467
					fdput(f);
Linus Torvalds's avatar
Linus Torvalds committed
468 469 470
					if ((mask & POLLIN_SET) && (in & bit)) {
						res_in |= bit;
						retval++;
471
						wait->_qproc = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
472 473 474 475
					}
					if ((mask & POLLOUT_SET) && (out & bit)) {
						res_out |= bit;
						retval++;
476
						wait->_qproc = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
477 478 479 480
					}
					if ((mask & POLLEX_SET) && (ex & bit)) {
						res_ex |= bit;
						retval++;
481
						wait->_qproc = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
482
					}
483
					/* got something, stop busy polling */
484 485 486 487 488 489 490 491 492 493 494
					if (retval) {
						can_busy_loop = false;
						busy_flag = 0;

					/*
					 * only remember a returned
					 * POLL_BUSY_LOOP if we asked for it
					 */
					} else if (busy_flag & mask)
						can_busy_loop = true;

Linus Torvalds's avatar
Linus Torvalds committed
495 496 497 498 499 500 501 502
				}
			}
			if (res_in)
				*rinp = res_in;
			if (res_out)
				*routp = res_out;
			if (res_ex)
				*rexp = res_ex;
503
			cond_resched();
Linus Torvalds's avatar
Linus Torvalds committed
504
		}
505
		wait->_qproc = NULL;
506
		if (retval || timed_out || signal_pending(current))
Linus Torvalds's avatar
Linus Torvalds committed
507
			break;
Pavel Machek's avatar
Pavel Machek committed
508
		if (table.error) {
Linus Torvalds's avatar
Linus Torvalds committed
509 510 511
			retval = table.error;
			break;
		}
512

513
		/* only if found POLL_BUSY_LOOP sockets && not out of time */
514 515 516 517 518 519 520 521 522
		if (can_busy_loop && !need_resched()) {
			if (!busy_end) {
				busy_end = busy_loop_end_time();
				continue;
			}
			if (!busy_loop_timeout(busy_end))
				continue;
		}
		busy_flag = 0;
523

524 525 526 527 528 529
		/*
		 * If this is the first loop and we have a timeout
		 * given, then we convert to ktime_t and set the to
		 * pointer to the expiry value.
		 */
		if (end_time && !to) {
530
			expire = timespec64_to_ktime(*end_time);
531
			to = &expire;
532
		}
533

534 535
		if (!poll_schedule_timeout(&table, TASK_INTERRUPTIBLE,
					   to, slack))
536
			timed_out = 1;
Linus Torvalds's avatar
Linus Torvalds committed
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
	}

	poll_freewait(&table);

	return retval;
}

/*
 * We can actually return ERESTARTSYS instead of EINTR, but I'd
 * like to be certain this leads to no problems. So I return
 * EINTR just for safety.
 *
 * Update: ERESTARTSYS breaks at least the xview clock binary, so
 * I'm trying ERESTARTNOHAND which restart only when you want to.
 */
552
int core_sys_select(int n, fd_set __user *inp, fd_set __user *outp,
553
			   fd_set __user *exp, struct timespec64 *end_time)
Linus Torvalds's avatar
Linus Torvalds committed
554 555
{
	fd_set_bits fds;
556
	void *bits;
557
	int ret, max_fds;
558
	size_t size, alloc_size;
559
	struct fdtable *fdt;
560
	/* Allocate small arguments on the stack to save memory and be faster */
561
	long stack_fds[SELECT_STACK_ALLOC/sizeof(long)];
Linus Torvalds's avatar
Linus Torvalds committed
562 563 564 565 566

	ret = -EINVAL;
	if (n < 0)
		goto out_nofds;

567
	/* max_fds can increase, so grab it once to avoid race */
568
	rcu_read_lock();
569
	fdt = files_fdtable(current->files);
570
	max_fds = fdt->max_fds;
571
	rcu_read_unlock();
572 573
	if (n > max_fds)
		n = max_fds;
Linus Torvalds's avatar
Linus Torvalds committed
574 575 576 577 578 579 580

	/*
	 * We need 6 bitmaps (in/out/ex for both incoming and outgoing),
	 * since we used fdset we need to allocate memory in units of
	 * long-words. 
	 */
	size = FDS_BYTES(n);
581 582 583 584
	bits = stack_fds;
	if (size > sizeof(stack_fds) / 6) {
		/* Not enough space in on-stack array; must use kmalloc */
		ret = -ENOMEM;
585 586 587 588 589 590 591 592
		if (size > (SIZE_MAX / 6))
			goto out_nofds;

		alloc_size = 6 * size;
		bits = kmalloc(alloc_size, GFP_KERNEL|__GFP_NOWARN);
		if (!bits && alloc_size > PAGE_SIZE)
			bits = vmalloc(alloc_size);

593 594 595
		if (!bits)
			goto out_nofds;
	}
596 597 598 599 600 601
	fds.in      = bits;
	fds.out     = bits +   size;
	fds.ex      = bits + 2*size;
	fds.res_in  = bits + 3*size;
	fds.res_out = bits + 4*size;
	fds.res_ex  = bits + 5*size;
Linus Torvalds's avatar
Linus Torvalds committed
602 603 604 605 606 607 608 609 610

	if ((ret = get_fd_set(n, inp, fds.in)) ||
	    (ret = get_fd_set(n, outp, fds.out)) ||
	    (ret = get_fd_set(n, exp, fds.ex)))
		goto out;
	zero_fd_set(n, fds.res_in);
	zero_fd_set(n, fds.res_out);
	zero_fd_set(n, fds.res_ex);

611
	ret = do_select(n, &fds, end_time);
Linus Torvalds's avatar
Linus Torvalds committed
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627

	if (ret < 0)
		goto out;
	if (!ret) {
		ret = -ERESTARTNOHAND;
		if (signal_pending(current))
			goto out;
		ret = 0;
	}

	if (set_fd_set(n, inp, fds.res_in) ||
	    set_fd_set(n, outp, fds.res_out) ||
	    set_fd_set(n, exp, fds.res_ex))
		ret = -EFAULT;

out:
628
	if (bits != stack_fds)
629
		kvfree(bits);
Linus Torvalds's avatar
Linus Torvalds committed
630 631 632 633
out_nofds:
	return ret;
}

634 635
SYSCALL_DEFINE5(select, int, n, fd_set __user *, inp, fd_set __user *, outp,
		fd_set __user *, exp, struct timeval __user *, tvp)
636
{
637
	struct timespec64 end_time, *to = NULL;
638 639 640 641 642 643 644
	struct timeval tv;
	int ret;

	if (tvp) {
		if (copy_from_user(&tv, tvp, sizeof(tv)))
			return -EFAULT;

645
		to = &end_time;
646 647 648
		if (poll_select_set_timeout(to,
				tv.tv_sec + (tv.tv_usec / USEC_PER_SEC),
				(tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC))
649 650 651
			return -EINVAL;
	}

652 653
	ret = core_sys_select(n, inp, outp, exp, to);
	ret = poll_select_copy_remaining(&end_time, tvp, 1, ret);
654 655 656 657

	return ret;
}

658 659 660
static long do_pselect(int n, fd_set __user *inp, fd_set __user *outp,
		       fd_set __user *exp, struct timespec __user *tsp,
		       const sigset_t __user *sigmask, size_t sigsetsize)
661 662
{
	sigset_t ksigmask, sigsaved;
663 664
	struct timespec ts;
	struct timespec64 ts64, end_time, *to = NULL;
665 666 667 668 669
	int ret;

	if (tsp) {
		if (copy_from_user(&ts, tsp, sizeof(ts)))
			return -EFAULT;
670
		ts64 = timespec_to_timespec64(ts);
671

672
		to = &end_time;
673
		if (poll_select_set_timeout(to, ts64.tv_sec, ts64.tv_nsec))
674 675 676 677 678 679 680 681 682 683 684 685 686 687
			return -EINVAL;
	}

	if (sigmask) {
		/* XXX: Don't preclude handling different sized sigset_t's.  */
		if (sigsetsize != sizeof(sigset_t))
			return -EINVAL;
		if (copy_from_user(&ksigmask, sigmask, sizeof(ksigmask)))
			return -EFAULT;

		sigdelsetmask(&ksigmask, sigmask(SIGKILL)|sigmask(SIGSTOP));
		sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved);
	}

688
	ret = core_sys_select(n, inp, outp, exp, to);
689
	ret = poll_select_copy_remaining(&end_time, tsp, 0, ret);
690 691 692 693 694 695 696 697 698 699

	if (ret == -ERESTARTNOHAND) {
		/*
		 * Don't restore the signal mask yet. Let do_signal() deliver
		 * the signal on the way back to userspace, before the signal
		 * mask is restored.
		 */
		if (sigmask) {
			memcpy(&current->saved_sigmask, &sigsaved,
					sizeof(sigsaved));
700
			set_restore_sigmask();
701 702 703 704 705 706 707 708 709 710 711 712 713
		}
	} else if (sigmask)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

	return ret;
}

/*
 * Most architectures can't handle 7-argument syscalls. So we provide a
 * 6-argument version where the sixth argument is a pointer to a structure
 * which has a pointer to the sigset_t itself followed by a size_t containing
 * the sigset size.
 */
714 715 716
SYSCALL_DEFINE6(pselect6, int, n, fd_set __user *, inp, fd_set __user *, outp,
		fd_set __user *, exp, struct timespec __user *, tsp,
		void __user *, sig)
717 718 719 720 721 722
{
	size_t sigsetsize = 0;
	sigset_t __user *up = NULL;

	if (sig) {
		if (!access_ok(VERIFY_READ, sig, sizeof(void *)+sizeof(size_t))
723
		    || __get_user(up, (sigset_t __user * __user *)sig)
724
		    || __get_user(sigsetsize,
725
				(size_t __user *)(sig+sizeof(void *))))
726 727 728
			return -EFAULT;
	}

729
	return do_pselect(n, inp, outp, exp, tsp, up, sigsetsize);
730 731
}

732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
#ifdef __ARCH_WANT_SYS_OLD_SELECT
struct sel_arg_struct {
	unsigned long n;
	fd_set __user *inp, *outp, *exp;
	struct timeval __user *tvp;
};

SYSCALL_DEFINE1(old_select, struct sel_arg_struct __user *, arg)
{
	struct sel_arg_struct a;

	if (copy_from_user(&a, arg, sizeof(a)))
		return -EFAULT;
	return sys_select(a.n, a.inp, a.outp, a.exp, a.tvp);
}
#endif

Linus Torvalds's avatar
Linus Torvalds committed
749 750 751 752 753 754 755 756
struct poll_list {
	struct poll_list *next;
	int len;
	struct pollfd entries[0];
};

#define POLLFD_PER_PAGE  ((PAGE_SIZE-sizeof(struct poll_list)) / sizeof(struct pollfd))

757 758 759 760 761
/*
 * Fish for pollable events on the pollfd->fd file descriptor. We're only
 * interested in events matching the pollfd->events mask, and the result
 * matching that mask is both recorded in pollfd->revents and returned. The
 * pwait poll_table will be used by the fd-provided poll handler for waiting,
762
 * if pwait->_qproc is non-NULL.
763
 */
764
static inline unsigned int do_pollfd(struct pollfd *pollfd, poll_table *pwait,
765 766
				     bool *can_busy_poll,
				     unsigned int busy_flag)
Linus Torvalds's avatar
Linus Torvalds committed
767
{
768 769 770 771 772 773
	unsigned int mask;
	int fd;

	mask = 0;
	fd = pollfd->fd;
	if (fd >= 0) {
774
		struct fd f = fdget(fd);
775
		mask = POLLNVAL;
776
		if (f.file) {
777
			mask = DEFAULT_POLLMASK;
Al Viro's avatar
Al Viro committed
778
			if (f.file->f_op->poll) {
779
				pwait->_key = pollfd->events|POLLERR|POLLHUP;
780
				pwait->_key |= busy_flag;
781
				mask = f.file->f_op->poll(f.file, pwait);
782 783
				if (mask & busy_flag)
					*can_busy_poll = true;
784
			}
785 786
			/* Mask out unneeded events. */
			mask &= pollfd->events | POLLERR | POLLHUP;
787
			fdput(f);
Linus Torvalds's avatar
Linus Torvalds committed
788 789
		}
	}
790 791 792
	pollfd->revents = mask;

	return mask;
Linus Torvalds's avatar
Linus Torvalds committed
793 794
}

795
static int do_poll(struct poll_list *list, struct poll_wqueues *wait,
796
		   struct timespec64 *end_time)
Linus Torvalds's avatar
Linus Torvalds committed
797 798
{
	poll_table* pt = &wait->pt;
799 800
	ktime_t expire, *to = NULL;
	int timed_out = 0, count = 0;
801
	u64 slack = 0;
802
	unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;
803
	unsigned long busy_end = 0;
Linus Torvalds's avatar
Linus Torvalds committed
804

805
	/* Optimise the no-wait case */
806
	if (end_time && !end_time->tv_sec && !end_time->tv_nsec) {
807
		pt->_qproc = NULL;
808 809
		timed_out = 1;
	}
810

811
	if (end_time && !timed_out)
812
		slack = select_estimate_accuracy(end_time);
813

Linus Torvalds's avatar
Linus Torvalds committed
814 815
	for (;;) {
		struct poll_list *walk;
816
		bool can_busy_loop = false;
817

818 819 820 821 822 823 824 825
		for (walk = list; walk != NULL; walk = walk->next) {
			struct pollfd * pfd, * pfd_end;

			pfd = walk->entries;
			pfd_end = pfd + walk->len;
			for (; pfd != pfd_end; pfd++) {
				/*
				 * Fish for events. If we found one, record it
826
				 * and kill poll_table->_qproc, so we don't
827 828 829 830
				 * needlessly register any other waiters after
				 * this. They'll get immediately deregistered
				 * when we break out and return.
				 */
831 832
				if (do_pollfd(pfd, pt, &can_busy_loop,
					      busy_flag)) {
833
					count++;
834
					pt->_qproc = NULL;
835 836 837
					/* found something, stop busy polling */
					busy_flag = 0;
					can_busy_loop = false;
838 839
				}
			}
Linus Torvalds's avatar
Linus Torvalds committed
840
		}
841 842
		/*
		 * All waiters have already been registered, so don't provide
843
		 * a poll_table->_qproc to them on the next loop iteration.
844
		 */
845
		pt->_qproc = NULL;
846 847 848 849 850
		if (!count) {
			count = wait->error;
			if (signal_pending(current))
				count = -EINTR;
		}
851
		if (count || timed_out)
Linus Torvalds's avatar
Linus Torvalds committed
852
			break;
853

854
		/* only if found POLL_BUSY_LOOP sockets && not out of time */
855 856 857 858 859 860 861 862 863
		if (can_busy_loop && !need_resched()) {
			if (!busy_end) {
				busy_end = busy_loop_end_time();
				continue;
			}
			if (!busy_loop_timeout(busy_end))
				continue;
		}
		busy_flag = 0;
864

865 866 867 868 869 870
		/*
		 * If this is the first loop and we have a timeout
		 * given, then we convert to ktime_t and set the to
		 * pointer to the expiry value.
		 */
		if (end_time && !to) {
871
			expire = timespec64_to_ktime(*end_time);
872
			to = &expire;
873 874
		}

875
		if (!poll_schedule_timeout(wait, TASK_INTERRUPTIBLE, to, slack))
876
			timed_out = 1;
Linus Torvalds's avatar
Linus Torvalds committed
877 878 879 880
	}
	return count;
}

881 882 883
#define N_STACK_PPS ((sizeof(stack_pps) - sizeof(struct poll_list))  / \
			sizeof(struct pollfd))

884
int do_sys_poll(struct pollfd __user *ufds, unsigned int nfds,
885
		struct timespec64 *end_time)
Linus Torvalds's avatar
Linus Torvalds committed
886 887
{
	struct poll_wqueues table;
888
 	int err = -EFAULT, fdcount, len, size;
889 890 891 892
	/* Allocate small arguments on the stack to save memory and be
	   faster - use long to make sure the buffer is aligned properly
	   on 64 bit archs to avoid unaligned access */
	long stack_pps[POLL_STACK_ALLOC/sizeof(long)];
893 894 895
	struct poll_list *const head = (struct poll_list *)stack_pps;
 	struct poll_list *walk = head;
 	unsigned long todo = nfds;
Linus Torvalds's avatar
Linus Torvalds committed
896

Jiri Slaby's avatar
Jiri Slaby committed
897
	if (nfds > rlimit(RLIMIT_NOFILE))
Linus Torvalds's avatar
Linus Torvalds committed
898 899
		return -EINVAL;

900 901 902 903 904 905
	len = min_t(unsigned int, nfds, N_STACK_PPS);
	for (;;) {
		walk->next = NULL;
		walk->len = len;
		if (!len)
			break;
Linus Torvalds's avatar
Linus Torvalds committed
906

907 908 909 910 911 912 913
		if (copy_from_user(walk->entries, ufds + nfds-todo,
					sizeof(struct pollfd) * walk->len))
			goto out_fds;

		todo -= walk->len;
		if (!todo)
			break;
Linus Torvalds's avatar
Linus Torvalds committed
914

915 916 917 918 919
		len = min(todo, POLLFD_PER_PAGE);
		size = sizeof(struct poll_list) + sizeof(struct pollfd) * len;
		walk = walk->next = kmalloc(size, GFP_KERNEL);
		if (!walk) {
			err = -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
920 921 922
			goto out_fds;
		}
	}
923

924
	poll_initwait(&table);
925
	fdcount = do_poll(head, &table, end_time);
926
	poll_freewait(&table);
Linus Torvalds's avatar
Linus Torvalds committed
927

928
	for (walk = head; walk; walk = walk->next) {
Linus Torvalds's avatar
Linus Torvalds committed
929 930 931
		struct pollfd *fds = walk->entries;
		int j;

932 933
		for (j = 0; j < walk->len; j++, ufds++)
			if (__put_user(fds[j].revents, &ufds->revents))
Linus Torvalds's avatar
Linus Torvalds committed
934 935
				goto out_fds;
  	}
936

Linus Torvalds's avatar
Linus Torvalds committed
937 938
	err = fdcount;
out_fds:
939 940 941 942 943
	walk = head->next;
	while (walk) {
		struct poll_list *pos = walk;
		walk = walk->next;
		kfree(pos);
Linus Torvalds's avatar
Linus Torvalds committed
944
	}
945

Linus Torvalds's avatar
Linus Torvalds committed
946 947
	return err;
}
948

949 950
static long do_restart_poll(struct restart_block *restart_block)
{
951 952
	struct pollfd __user *ufds = restart_block->poll.ufds;
	int nfds = restart_block->poll.nfds;
953
	struct timespec64 *to = NULL, end_time;
954 955
	int ret;

956 957 958 959 960 961 962 963
	if (restart_block->poll.has_timeout) {
		end_time.tv_sec = restart_block->poll.tv_sec;
		end_time.tv_nsec = restart_block->poll.tv_nsec;
		to = &end_time;
	}

	ret = do_sys_poll(ufds, nfds, to);

964 965 966 967 968 969 970
	if (ret == -EINTR) {
		restart_block->fn = do_restart_poll;
		ret = -ERESTART_RESTARTBLOCK;
	}
	return ret;
}

971
SYSCALL_DEFINE3(poll, struct pollfd __user *, ufds, unsigned int, nfds,
972
		int, timeout_msecs)
973
{
974
	struct timespec64 end_time, *to = NULL;
975
	int ret;
976

977 978 979 980
	if (timeout_msecs >= 0) {
		to = &end_time;
		poll_select_set_timeout(to, timeout_msecs / MSEC_PER_SEC,
			NSEC_PER_MSEC * (timeout_msecs % MSEC_PER_SEC));
981 982
	}

983 984
	ret = do_sys_poll(ufds, nfds, to);

985 986
	if (ret == -EINTR) {
		struct restart_block *restart_block;
987

988
		restart_block = &current->restart_block;
989
		restart_block->fn = do_restart_poll;
990 991 992 993 994 995 996 997 998 999
		restart_block->poll.ufds = ufds;
		restart_block->poll.nfds = nfds;

		if (timeout_msecs >= 0) {
			restart_block->poll.tv_sec = end_time.tv_sec;
			restart_block->poll.tv_nsec = end_time.tv_nsec;
			restart_block->poll.has_timeout = 1;
		} else
			restart_block->poll.has_timeout = 0;

1000 1001 1002
		ret = -ERESTART_RESTARTBLOCK;
	}
	return ret;
1003 1004
}

1005 1006 1007
SYSCALL_DEFINE5(ppoll, struct pollfd __user *, ufds, unsigned int, nfds,
		struct timespec __user *, tsp, const sigset_t __user *, sigmask,
		size_t, sigsetsize)
1008 1009
{
	sigset_t ksigmask, sigsaved;
1010 1011
	struct timespec ts;
	struct timespec64 end_time, *to = NULL;
1012 1013 1014 1015 1016 1017
	int ret;

	if (tsp) {
		if (copy_from_user(&ts, tsp, sizeof(ts)))
			return -EFAULT;

1018 1019 1020
		to = &end_time;
		if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))
			return -EINVAL;
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
	}

	if (sigmask) {
		/* XXX: Don't preclude handling different sized sigset_t's.  */
		if (sigsetsize != sizeof(sigset_t))
			return -EINVAL;
		if (copy_from_user(&ksigmask, sigmask, sizeof(ksigmask)))
			return -EFAULT;

		sigdelsetmask(&ksigmask, sigmask(SIGKILL)|sigmask(SIGSTOP));
		sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved);
	}

1034
	ret = do_sys_poll(ufds, nfds, to);
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045

	/* We can restart this syscall, usually */
	if (ret == -EINTR) {
		/*
		 * Don't restore the signal mask yet. Let do_signal() deliver
		 * the signal on the way back to userspace, before the signal
		 * mask is restored.
		 */
		if (sigmask) {
			memcpy(&current->saved_sigmask, &sigsaved,
					sizeof(sigsaved));
1046
			set_restore_sigmask();
1047 1048 1049 1050 1051
		}
		ret = -ERESTARTNOHAND;
	} else if (sigmask)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

1052
	ret = poll_select_copy_remaining(&end_time, tsp, 0, ret);
1053 1054 1055

	return ret;
}