blk-sysfs.c 23.4 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5
/*
 * Functions related to sysfs handling
 */
#include <linux/kernel.h>
6
#include <linux/slab.h>
7 8 9
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
10
#include <linux/backing-dev.h>
11
#include <linux/blktrace_api.h>
12
#include <linux/blk-mq.h>
13
#include <linux/blk-cgroup.h>
14 15

#include "blk.h"
16
#include "blk-mq.h"
17
#include "blk-mq-debugfs.h"
18
#include "blk-wbt.h"
19 20 21 22 23 24 25 26

struct queue_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct request_queue *, char *);
	ssize_t (*store)(struct request_queue *, const char *, size_t);
};

static ssize_t
27
queue_var_show(unsigned long var, char *page)
28
{
29
	return sprintf(page, "%lu\n", var);
30 31 32 33 34
}

static ssize_t
queue_var_store(unsigned long *var, const char *page, size_t count)
{
35 36 37
	int err;
	unsigned long v;

38
	err = kstrtoul(page, 10, &v);
39 40 41 42
	if (err || v > UINT_MAX)
		return -EINVAL;

	*var = v;
43 44 45 46

	return count;
}

47
static ssize_t queue_var_store64(s64 *var, const char *page)
48 49
{
	int err;
50
	s64 v;
51

52
	err = kstrtos64(page, 10, &v);
53 54 55 56 57 58 59
	if (err < 0)
		return err;

	*var = v;
	return 0;
}

60 61 62 63 64 65 66 67 68
static ssize_t queue_requests_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->nr_requests, (page));
}

static ssize_t
queue_requests_store(struct request_queue *q, const char *page, size_t count)
{
	unsigned long nr;
69
	int ret, err;
70

71
	if (!q->request_fn && !q->mq_ops)
72 73 74
		return -EINVAL;

	ret = queue_var_store(&nr, page, count);
75 76 77
	if (ret < 0)
		return ret;

78 79 80
	if (nr < BLKDEV_MIN_RQ)
		nr = BLKDEV_MIN_RQ;

81 82 83 84 85 86 87 88
	if (q->request_fn)
		err = blk_update_nr_requests(q, nr);
	else
		err = blk_mq_update_nr_requests(q, nr);

	if (err)
		return err;

89 90 91 92 93
	return ret;
}

static ssize_t queue_ra_show(struct request_queue *q, char *page)
{
94
	unsigned long ra_kb = q->backing_dev_info->ra_pages <<
95
					(PAGE_SHIFT - 10);
96 97 98 99 100 101 102 103 104 105

	return queue_var_show(ra_kb, (page));
}

static ssize_t
queue_ra_store(struct request_queue *q, const char *page, size_t count)
{
	unsigned long ra_kb;
	ssize_t ret = queue_var_store(&ra_kb, page, count);

106 107 108
	if (ret < 0)
		return ret;

109
	q->backing_dev_info->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
110 111 112 113 114 115

	return ret;
}

static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
{
116
	int max_sectors_kb = queue_max_sectors(q) >> 1;
117 118 119 120

	return queue_var_show(max_sectors_kb, (page));
}

121 122 123 124 125
static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
{
	return queue_var_show(queue_max_segments(q), (page));
}

126 127 128 129 130 131
static ssize_t queue_max_discard_segments_show(struct request_queue *q,
		char *page)
{
	return queue_var_show(queue_max_discard_segments(q), (page));
}

132 133 134 135 136
static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->limits.max_integrity_segments, (page));
}

137 138
static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)
{
139
	if (blk_queue_cluster(q))
140 141
		return queue_var_show(queue_max_segment_size(q), (page));

142
	return queue_var_show(PAGE_SIZE, (page));
143 144
}

145
static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)
146
{
147
	return queue_var_show(queue_logical_block_size(q), page);
148 149
}

150 151 152 153 154
static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)
{
	return queue_var_show(queue_physical_block_size(q), page);
}

155 156 157 158 159
static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->limits.chunk_sectors, page);
}

160 161 162 163 164 165 166 167
static ssize_t queue_io_min_show(struct request_queue *q, char *page)
{
	return queue_var_show(queue_io_min(q), page);
}

static ssize_t queue_io_opt_show(struct request_queue *q, char *page)
{
	return queue_var_show(queue_io_opt(q), page);
168 169
}

170 171 172 173 174
static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->limits.discard_granularity, page);
}

175 176 177
static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
{

178 179
	return sprintf(page, "%llu\n",
		(unsigned long long)q->limits.max_hw_discard_sectors << 9);
180 181
}

182 183
static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
{
184 185
	return sprintf(page, "%llu\n",
		       (unsigned long long)q->limits.max_discard_sectors << 9);
186 187
}

188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
static ssize_t queue_discard_max_store(struct request_queue *q,
				       const char *page, size_t count)
{
	unsigned long max_discard;
	ssize_t ret = queue_var_store(&max_discard, page, count);

	if (ret < 0)
		return ret;

	if (max_discard & (q->limits.discard_granularity - 1))
		return -EINVAL;

	max_discard >>= 9;
	if (max_discard > UINT_MAX)
		return -EINVAL;

	if (max_discard > q->limits.max_hw_discard_sectors)
		max_discard = q->limits.max_hw_discard_sectors;

	q->limits.max_discard_sectors = max_discard;
	return ret;
}

211 212
static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
{
213
	return queue_var_show(0, page);
214 215
}

216 217 218 219 220 221
static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
{
	return sprintf(page, "%llu\n",
		(unsigned long long)q->limits.max_write_same_sectors << 9);
}

222 223 224 225 226
static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)
{
	return sprintf(page, "%llu\n",
		(unsigned long long)q->limits.max_write_zeroes_sectors << 9);
}
227

228 229 230 231
static ssize_t
queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
{
	unsigned long max_sectors_kb,
232
		max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1,
233
			page_kb = 1 << (PAGE_SHIFT - 10);
234 235
	ssize_t ret = queue_var_store(&max_sectors_kb, page, count);

236 237 238
	if (ret < 0)
		return ret;

239 240 241
	max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long)
					 q->limits.max_dev_sectors >> 1);

242 243
	if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
		return -EINVAL;
244

245
	spin_lock_irq(q->queue_lock);
246
	q->limits.max_sectors = max_sectors_kb << 1;
247
	q->backing_dev_info->io_pages = max_sectors_kb >> (PAGE_SHIFT - 10);
248 249 250 251 252 253 254
	spin_unlock_irq(q->queue_lock);

	return ret;
}

static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
{
255
	int max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1;
256 257 258 259

	return queue_var_show(max_hw_sectors_kb, (page));
}

260 261 262 263 264 265 266 267 268 269 270 271 272 273
#define QUEUE_SYSFS_BIT_FNS(name, flag, neg)				\
static ssize_t								\
queue_show_##name(struct request_queue *q, char *page)			\
{									\
	int bit;							\
	bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags);		\
	return queue_var_show(neg ? !bit : bit, page);			\
}									\
static ssize_t								\
queue_store_##name(struct request_queue *q, const char *page, size_t count) \
{									\
	unsigned long val;						\
	ssize_t ret;							\
	ret = queue_var_store(&val, page, count);			\
274 275
	if (ret < 0)							\
		 return ret;						\
276 277 278 279 280 281 282 283 284 285
	if (neg)							\
		val = !val;						\
									\
	spin_lock_irq(q->queue_lock);					\
	if (val)							\
		queue_flag_set(QUEUE_FLAG_##flag, q);			\
	else								\
		queue_flag_clear(QUEUE_FLAG_##flag, q);			\
	spin_unlock_irq(q->queue_lock);					\
	return ret;							\
286 287
}

288 289 290 291
QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
#undef QUEUE_SYSFS_BIT_FNS
292

293 294 295 296 297 298 299 300 301 302 303 304
static ssize_t queue_zoned_show(struct request_queue *q, char *page)
{
	switch (blk_queue_zoned_model(q)) {
	case BLK_ZONED_HA:
		return sprintf(page, "host-aware\n");
	case BLK_ZONED_HM:
		return sprintf(page, "host-managed\n");
	default:
		return sprintf(page, "none\n");
	}
}

305 306
static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
{
307 308
	return queue_var_show((blk_queue_nomerges(q) << 1) |
			       blk_queue_noxmerges(q), page);
309 310 311 312 313 314 315 316
}

static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,
				    size_t count)
{
	unsigned long nm;
	ssize_t ret = queue_var_store(&nm, page, count);

317 318 319
	if (ret < 0)
		return ret;

320
	spin_lock_irq(q->queue_lock);
321 322 323
	queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
	queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
	if (nm == 2)
324
		queue_flag_set(QUEUE_FLAG_NOMERGES, q);
325 326
	else if (nm)
		queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
327
	spin_unlock_irq(q->queue_lock);
328

329 330 331
	return ret;
}

332 333
static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)
{
334
	bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);
Dan Williams's avatar
Dan Williams committed
335
	bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);
336

Dan Williams's avatar
Dan Williams committed
337
	return queue_var_show(set << force, page);
338 339 340 341 342 343
}

static ssize_t
queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
{
	ssize_t ret = -EINVAL;
344
#ifdef CONFIG_SMP
345 346 347
	unsigned long val;

	ret = queue_var_store(&val, page, count);
348 349 350
	if (ret < 0)
		return ret;

351
	spin_lock_irq(q->queue_lock);
352
	if (val == 2) {
353
		queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
354 355 356 357 358
		queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
	} else if (val == 1) {
		queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
		queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
	} else if (val == 0) {
Dan Williams's avatar
Dan Williams committed
359 360 361
		queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
		queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
	}
362 363 364 365
	spin_unlock_irq(q->queue_lock);
#endif
	return ret;
}
366

367 368
static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)
{
369 370 371 372 373 374 375 376
	int val;

	if (q->poll_nsec == -1)
		val = -1;
	else
		val = q->poll_nsec / 1000;

	return sprintf(page, "%d\n", val);
377 378 379 380 381
}

static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,
				size_t count)
{
382
	int err, val;
383 384 385 386

	if (!q->mq_ops || !q->mq_ops->poll)
		return -EINVAL;

387 388 389
	err = kstrtoint(page, 10, &val);
	if (err < 0)
		return err;
390

391 392 393 394 395 396
	if (val == -1)
		q->poll_nsec = -1;
	else
		q->poll_nsec = val * 1000;

	return count;
397 398
}

399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
static ssize_t queue_poll_show(struct request_queue *q, char *page)
{
	return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
}

static ssize_t queue_poll_store(struct request_queue *q, const char *page,
				size_t count)
{
	unsigned long poll_on;
	ssize_t ret;

	if (!q->mq_ops || !q->mq_ops->poll)
		return -EINVAL;

	ret = queue_var_store(&poll_on, page, count);
	if (ret < 0)
		return ret;

	spin_lock_irq(q->queue_lock);
	if (poll_on)
		queue_flag_set(QUEUE_FLAG_POLL, q);
	else
		queue_flag_clear(QUEUE_FLAG_POLL, q);
	spin_unlock_irq(q->queue_lock);

	return ret;
}

427 428 429 430 431 432 433 434 435 436 437
static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
{
	if (!q->rq_wb)
		return -EINVAL;

	return sprintf(page, "%llu\n", div_u64(q->rq_wb->min_lat_nsec, 1000));
}

static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
				  size_t count)
{
438
	struct rq_wb *rwb;
439
	ssize_t ret;
440
	s64 val;
441 442 443 444

	ret = queue_var_store64(&val, page);
	if (ret < 0)
		return ret;
445 446 447 448 449 450 451 452 453 454 455 456 457
	if (val < -1)
		return -EINVAL;

	rwb = q->rq_wb;
	if (!rwb) {
		ret = wbt_init(q);
		if (ret)
			return ret;

		rwb = q->rq_wb;
		if (!rwb)
			return -EINVAL;
	}
458

459 460 461 462
	if (val == -1)
		rwb->min_lat_nsec = wbt_default_latency_nsec(q);
	else if (val >= 0)
		rwb->min_lat_nsec = val * 1000ULL;
463 464 465

	if (rwb->enable_state == WBT_STATE_ON_DEFAULT)
		rwb->enable_state = WBT_STATE_ON_MANUAL;
466 467

	wbt_update_limits(rwb);
468 469 470
	return count;
}

471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
static ssize_t queue_wc_show(struct request_queue *q, char *page)
{
	if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
		return sprintf(page, "write back\n");

	return sprintf(page, "write through\n");
}

static ssize_t queue_wc_store(struct request_queue *q, const char *page,
			      size_t count)
{
	int set = -1;

	if (!strncmp(page, "write back", 10))
		set = 1;
	else if (!strncmp(page, "write through", 13) ||
		 !strncmp(page, "none", 4))
		set = 0;

	if (set == -1)
		return -EINVAL;

	spin_lock_irq(q->queue_lock);
	if (set)
		queue_flag_set(QUEUE_FLAG_WC, q);
	else
		queue_flag_clear(QUEUE_FLAG_WC, q);
	spin_unlock_irq(q->queue_lock);

	return count;
}

503 504 505 506 507
static ssize_t queue_dax_show(struct request_queue *q, char *page)
{
	return queue_var_show(blk_queue_dax(q), page);
}

508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
static struct queue_sysfs_entry queue_requests_entry = {
	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
	.show = queue_requests_show,
	.store = queue_requests_store,
};

static struct queue_sysfs_entry queue_ra_entry = {
	.attr = {.name = "read_ahead_kb", .mode = S_IRUGO | S_IWUSR },
	.show = queue_ra_show,
	.store = queue_ra_store,
};

static struct queue_sysfs_entry queue_max_sectors_entry = {
	.attr = {.name = "max_sectors_kb", .mode = S_IRUGO | S_IWUSR },
	.show = queue_max_sectors_show,
	.store = queue_max_sectors_store,
};

static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
	.attr = {.name = "max_hw_sectors_kb", .mode = S_IRUGO },
	.show = queue_max_hw_sectors_show,
};

531 532 533 534 535
static struct queue_sysfs_entry queue_max_segments_entry = {
	.attr = {.name = "max_segments", .mode = S_IRUGO },
	.show = queue_max_segments_show,
};

536 537 538 539 540
static struct queue_sysfs_entry queue_max_discard_segments_entry = {
	.attr = {.name = "max_discard_segments", .mode = S_IRUGO },
	.show = queue_max_discard_segments_show,
};

541 542 543 544 545
static struct queue_sysfs_entry queue_max_integrity_segments_entry = {
	.attr = {.name = "max_integrity_segments", .mode = S_IRUGO },
	.show = queue_max_integrity_segments_show,
};

546 547 548 549 550
static struct queue_sysfs_entry queue_max_segment_size_entry = {
	.attr = {.name = "max_segment_size", .mode = S_IRUGO },
	.show = queue_max_segment_size_show,
};

551 552 553 554 555 556
static struct queue_sysfs_entry queue_iosched_entry = {
	.attr = {.name = "scheduler", .mode = S_IRUGO | S_IWUSR },
	.show = elv_iosched_show,
	.store = elv_iosched_store,
};

557 558
static struct queue_sysfs_entry queue_hw_sector_size_entry = {
	.attr = {.name = "hw_sector_size", .mode = S_IRUGO },
559 560 561 562 563 564
	.show = queue_logical_block_size_show,
};

static struct queue_sysfs_entry queue_logical_block_size_entry = {
	.attr = {.name = "logical_block_size", .mode = S_IRUGO },
	.show = queue_logical_block_size_show,
565 566
};

567 568 569 570 571
static struct queue_sysfs_entry queue_physical_block_size_entry = {
	.attr = {.name = "physical_block_size", .mode = S_IRUGO },
	.show = queue_physical_block_size_show,
};

572 573 574 575 576
static struct queue_sysfs_entry queue_chunk_sectors_entry = {
	.attr = {.name = "chunk_sectors", .mode = S_IRUGO },
	.show = queue_chunk_sectors_show,
};

577 578 579 580 581 582 583 584
static struct queue_sysfs_entry queue_io_min_entry = {
	.attr = {.name = "minimum_io_size", .mode = S_IRUGO },
	.show = queue_io_min_show,
};

static struct queue_sysfs_entry queue_io_opt_entry = {
	.attr = {.name = "optimal_io_size", .mode = S_IRUGO },
	.show = queue_io_opt_show,
585 586
};

587 588 589 590 591
static struct queue_sysfs_entry queue_discard_granularity_entry = {
	.attr = {.name = "discard_granularity", .mode = S_IRUGO },
	.show = queue_discard_granularity_show,
};

592 593 594 595 596
static struct queue_sysfs_entry queue_discard_max_hw_entry = {
	.attr = {.name = "discard_max_hw_bytes", .mode = S_IRUGO },
	.show = queue_discard_max_hw_show,
};

597
static struct queue_sysfs_entry queue_discard_max_entry = {
598
	.attr = {.name = "discard_max_bytes", .mode = S_IRUGO | S_IWUSR },
599
	.show = queue_discard_max_show,
600
	.store = queue_discard_max_store,
601 602
};

603 604 605 606 607
static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
	.attr = {.name = "discard_zeroes_data", .mode = S_IRUGO },
	.show = queue_discard_zeroes_data_show,
};

608 609 610 611 612
static struct queue_sysfs_entry queue_write_same_max_entry = {
	.attr = {.name = "write_same_max_bytes", .mode = S_IRUGO },
	.show = queue_write_same_max_show,
};

613 614 615 616 617
static struct queue_sysfs_entry queue_write_zeroes_max_entry = {
	.attr = {.name = "write_zeroes_max_bytes", .mode = S_IRUGO },
	.show = queue_write_zeroes_max_show,
};

618 619
static struct queue_sysfs_entry queue_nonrot_entry = {
	.attr = {.name = "rotational", .mode = S_IRUGO | S_IWUSR },
620 621
	.show = queue_show_nonrot,
	.store = queue_store_nonrot,
622 623
};

624 625 626 627 628
static struct queue_sysfs_entry queue_zoned_entry = {
	.attr = {.name = "zoned", .mode = S_IRUGO },
	.show = queue_zoned_show,
};

629 630 631 632 633 634
static struct queue_sysfs_entry queue_nomerges_entry = {
	.attr = {.name = "nomerges", .mode = S_IRUGO | S_IWUSR },
	.show = queue_nomerges_show,
	.store = queue_nomerges_store,
};

635 636 637 638 639 640
static struct queue_sysfs_entry queue_rq_affinity_entry = {
	.attr = {.name = "rq_affinity", .mode = S_IRUGO | S_IWUSR },
	.show = queue_rq_affinity_show,
	.store = queue_rq_affinity_store,
};

641 642
static struct queue_sysfs_entry queue_iostats_entry = {
	.attr = {.name = "iostats", .mode = S_IRUGO | S_IWUSR },
643 644
	.show = queue_show_iostats,
	.store = queue_store_iostats,
645 646
};

647 648
static struct queue_sysfs_entry queue_random_entry = {
	.attr = {.name = "add_random", .mode = S_IRUGO | S_IWUSR },
649 650
	.show = queue_show_random,
	.store = queue_store_random,
651 652
};

653 654 655 656 657 658
static struct queue_sysfs_entry queue_poll_entry = {
	.attr = {.name = "io_poll", .mode = S_IRUGO | S_IWUSR },
	.show = queue_poll_show,
	.store = queue_poll_store,
};

659 660 661 662 663 664
static struct queue_sysfs_entry queue_poll_delay_entry = {
	.attr = {.name = "io_poll_delay", .mode = S_IRUGO | S_IWUSR },
	.show = queue_poll_delay_show,
	.store = queue_poll_delay_store,
};

665 666 667 668 669 670
static struct queue_sysfs_entry queue_wc_entry = {
	.attr = {.name = "write_cache", .mode = S_IRUGO | S_IWUSR },
	.show = queue_wc_show,
	.store = queue_wc_store,
};

671 672 673 674 675
static struct queue_sysfs_entry queue_dax_entry = {
	.attr = {.name = "dax", .mode = S_IRUGO },
	.show = queue_dax_show,
};

676 677 678 679 680 681
static struct queue_sysfs_entry queue_wb_lat_entry = {
	.attr = {.name = "wbt_lat_usec", .mode = S_IRUGO | S_IWUSR },
	.show = queue_wb_lat_show,
	.store = queue_wb_lat_store,
};

682 683 684 685 686 687 688 689
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
static struct queue_sysfs_entry throtl_sample_time_entry = {
	.attr = {.name = "throttle_sample_time", .mode = S_IRUGO | S_IWUSR },
	.show = blk_throtl_sample_time_show,
	.store = blk_throtl_sample_time_store,
};
#endif

690 691 692 693 694
static struct attribute *default_attrs[] = {
	&queue_requests_entry.attr,
	&queue_ra_entry.attr,
	&queue_max_hw_sectors_entry.attr,
	&queue_max_sectors_entry.attr,
695
	&queue_max_segments_entry.attr,
696
	&queue_max_discard_segments_entry.attr,
697
	&queue_max_integrity_segments_entry.attr,
698
	&queue_max_segment_size_entry.attr,
699
	&queue_iosched_entry.attr,
700
	&queue_hw_sector_size_entry.attr,
701
	&queue_logical_block_size_entry.attr,
702
	&queue_physical_block_size_entry.attr,
703
	&queue_chunk_sectors_entry.attr,
704 705
	&queue_io_min_entry.attr,
	&queue_io_opt_entry.attr,
706 707
	&queue_discard_granularity_entry.attr,
	&queue_discard_max_entry.attr,
708
	&queue_discard_max_hw_entry.attr,
709
	&queue_discard_zeroes_data_entry.attr,
710
	&queue_write_same_max_entry.attr,
711
	&queue_write_zeroes_max_entry.attr,
712
	&queue_nonrot_entry.attr,
713
	&queue_zoned_entry.attr,
714
	&queue_nomerges_entry.attr,
715
	&queue_rq_affinity_entry.attr,
716
	&queue_iostats_entry.attr,
717
	&queue_random_entry.attr,
718
	&queue_poll_entry.attr,
719
	&queue_wc_entry.attr,
720
	&queue_dax_entry.attr,
721
	&queue_wb_lat_entry.attr,
722
	&queue_poll_delay_entry.attr,
723 724 725
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
	&throtl_sample_time_entry.attr,
#endif
726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741
	NULL,
};

#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)

static ssize_t
queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{
	struct queue_sysfs_entry *entry = to_queue(attr);
	struct request_queue *q =
		container_of(kobj, struct request_queue, kobj);
	ssize_t res;

	if (!entry->show)
		return -EIO;
	mutex_lock(&q->sysfs_lock);
742
	if (blk_queue_dying(q)) {
743 744 745 746 747 748 749 750 751 752 753 754 755
		mutex_unlock(&q->sysfs_lock);
		return -ENOENT;
	}
	res = entry->show(q, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t
queue_attr_store(struct kobject *kobj, struct attribute *attr,
		    const char *page, size_t length)
{
	struct queue_sysfs_entry *entry = to_queue(attr);
756
	struct request_queue *q;
757 758 759 760
	ssize_t res;

	if (!entry->store)
		return -EIO;
761 762

	q = container_of(kobj, struct request_queue, kobj);
763
	mutex_lock(&q->sysfs_lock);
764
	if (blk_queue_dying(q)) {
765 766 767 768 769 770 771 772
		mutex_unlock(&q->sysfs_lock);
		return -ENOENT;
	}
	res = entry->store(q, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

773 774 775 776 777 778 779
static void blk_free_queue_rcu(struct rcu_head *rcu_head)
{
	struct request_queue *q = container_of(rcu_head, struct request_queue,
					       rcu_head);
	kmem_cache_free(blk_requestq_cachep, q);
}

780
/**
781 782
 * __blk_release_queue - release a request queue when it is no longer needed
 * @work: pointer to the release_work member of the request queue to be released
783 784
 *
 * Description:
785 786 787 788
 *     blk_release_queue is the counterpart of blk_init_queue(). It should be
 *     called when a request queue is being released; typically when a block
 *     device is being de-registered. Its primary task it to free the queue
 *     itself.
789
 *
790
 * Notes:
791 792
 *     The low level driver must have finished any outstanding requests first
 *     via blk_cleanup_queue().
793 794 795 796 797
 *
 *     Although blk_release_queue() may be called with preemption disabled,
 *     __blk_release_queue() may sleep.
 */
static void __blk_release_queue(struct work_struct *work)
798
{
799
	struct request_queue *q = container_of(work, typeof(*q), release_work);
800

801 802 803
	if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
		blk_stat_remove_callback(q, q->poll_cb);
	blk_stat_free_callback(q->poll_cb);
804
	bdi_put(q->backing_dev_info);
805 806
	blkcg_exit_queue(q);

807 808
	if (q->elevator) {
		ioc_clear_queue(q);
809
		elevator_exit(q, q->elevator);
810
	}
811

812 813
	blk_free_queue_stats(q->stats);

814
	blk_exit_rl(q, &q->root_rl);
815 816 817 818

	if (q->queue_tags)
		__blk_queue_free_tags(q);

819 820 821
	if (!q->mq_ops) {
		if (q->exit_rq_fn)
			q->exit_rq_fn(q, q->fq->flush_rq);
822
		blk_free_flush_queue(q->fq);
823
	} else {
824
		blk_mq_release(q);
825
	}
826

827 828
	blk_trace_shutdown(q);

829 830 831
	if (q->mq_ops)
		blk_mq_debugfs_unregister(q);

832 833 834
	if (q->bio_split)
		bioset_free(q->bio_split);

835
	ida_simple_remove(&blk_queue_ida, q->id);
836
	call_rcu(&q->rcu_head, blk_free_queue_rcu);
837 838
}

839 840 841 842 843 844 845 846 847
static void blk_release_queue(struct kobject *kobj)
{
	struct request_queue *q =
		container_of(kobj, struct request_queue, kobj);

	INIT_WORK(&q->release_work, __blk_release_queue);
	schedule_work(&q->release_work);
}

848
static const struct sysfs_ops queue_sysfs_ops = {
849 850 851 852 853 854 855 856 857 858 859 860 861
	.show	= queue_attr_show,
	.store	= queue_attr_store,
};

struct kobj_type blk_queue_ktype = {
	.sysfs_ops	= &queue_sysfs_ops,
	.default_attrs	= default_attrs,
	.release	= blk_release_queue,
};

int blk_register_queue(struct gendisk *disk)
{
	int ret;
862
	struct device *dev = disk_to_dev(disk);
863 864
	struct request_queue *q = disk->queue;

865
	if (WARN_ON(!q))
866 867
		return -ENXIO;

868 869 870 871 872
	WARN_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags),
		  "%s is registering an already registered queue\n",
		  kobject_name(&dev->kobj));
	queue_flag_set_unlocked(QUEUE_FLAG_REGISTERED, q);

873
	/*
874 875 876 877 878 879 880
	 * SCSI probing may synchronously create and destroy a lot of
	 * request_queues for non-existent devices.  Shutting down a fully
	 * functional queue takes measureable wallclock time as RCU grace
	 * periods are involved.  To avoid excessive latency in these
	 * cases, a request_queue starts out in a degraded mode which is
	 * faster to shut down and is made fully functional here as
	 * request_queues for non-existent devices never get registered.
881
	 */
882 883
	if (!blk_queue_init_done(q)) {
		queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q);
884
		percpu_ref_switch_to_percpu(&q->q_usage_counter);
885 886
		blk_queue_bypass_end(q);
	}
887

888 889 890 891
	ret = blk_trace_init_sysfs(dev);
	if (ret)
		return ret;

892 893 894
	/* Prevent changes through sysfs until registration is completed. */
	mutex_lock(&q->sysfs_lock);

895
	ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
896 897
	if (ret < 0) {
		blk_trace_remove_sysfs(dev);
898
		goto unlock;
899
	}
900

901
	if (q->mq_ops) {
902
		__blk_mq_register_dev(dev, q);
903 904
		blk_mq_debugfs_register(q);
	}
905

906 907
	kobject_uevent(&q->kobj, KOBJ_ADD);

908
	wbt_enable_default(q);
909

910 911
	blk_throtl_register_queue(q);

912 913 914 915 916 917 918
	if (q->request_fn || (q->mq_ops && q->elevator)) {
		ret = elv_register_queue(q);
		if (ret) {
			kobject_uevent(&q->kobj, KOBJ_REMOVE);
			kobject_del(&q->kobj);
			blk_trace_remove_sysfs(dev);
			kobject_put(&dev->kobj);
919
			goto unlock;
920
		}
921
	}
922 923 924 925
	ret = 0;
unlock:
	mutex_unlock(&q->sysfs_lock);
	return ret;
926 927 928 929 930 931
}

void blk_unregister_queue(struct gendisk *disk)
{
	struct request_queue *q = disk->queue;

932 933 934
	if (WARN_ON(!q))
		return;

935
	mutex_lock(&q->sysfs_lock);
936
	queue_flag_clear_unlocked(QUEUE_FLAG_REGISTERED, q);
937
	mutex_unlock(&q->sysfs_lock);
938

939 940 941
	wbt_exit(q);


942
	if (q->mq_ops)
943
		blk_mq_unregister_dev(disk_to_dev(disk), q);
944

945
	if (q->request_fn || (q->mq_ops && q->elevator))
946 947
		elv_unregister_queue(q);

948 949 950 951
	kobject_uevent(&q->kobj, KOBJ_REMOVE);
	kobject_del(&q->kobj);
	blk_trace_remove_sysfs(disk_to_dev(disk));
	kobject_put(&disk_to_dev(disk)->kobj);
952
}