blk.h 9.87 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3 4
#ifndef BLK_INTERNAL_H
#define BLK_INTERNAL_H

5
#include <linux/idr.h>
6 7
#include <linux/blk-mq.h>
#include "blk-mq.h"
8

9 10 11 12 13 14
/* Amount of time in which a process may batch requests */
#define BLK_BATCH_TIME	(HZ/50UL)

/* Number of requests a "batching" process may submit */
#define BLK_BATCH_REQ	32

15 16 17
/* Max future timer expiry for timeouts */
#define BLK_MAX_TIMEOUT		(5 * HZ)

18 19 20 21
#ifdef CONFIG_DEBUG_FS
extern struct dentry *blk_debugfs_root;
#endif

22 23 24 25 26 27 28 29
struct blk_flush_queue {
	unsigned int		flush_queue_delayed:1;
	unsigned int		flush_pending_idx:1;
	unsigned int		flush_running_idx:1;
	unsigned long		flush_pending_since;
	struct list_head	flush_queue[2];
	struct list_head	flush_data_in_flight;
	struct request		*flush_rq;
30 31 32 33 34 35

	/*
	 * flush_rq shares tag with this rq, both can't be active
	 * at the same time
	 */
	struct request		*orig_rq;
36 37 38
	spinlock_t		mq_flush_lock;
};

39
extern struct kmem_cache *blk_requestq_cachep;
40
extern struct kmem_cache *request_cachep;
41
extern struct kobj_type blk_queue_ktype;
42
extern struct ida blk_queue_ida;
43

44
static inline struct blk_flush_queue *blk_get_flush_queue(
45
		struct request_queue *q, struct blk_mq_ctx *ctx)
46
{
47 48 49
	if (q->mq_ops)
		return blk_mq_map_queue(q, ctx->cpu)->fq;
	return q->fq;
50 51
}

52 53 54 55 56
static inline void __blk_get_queue(struct request_queue *q)
{
	kobject_get(&q->kobj);
}

57 58 59
struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q,
		int node, int cmd_size);
void blk_free_flush_queue(struct blk_flush_queue *q);
60

61 62
int blk_init_rl(struct request_list *rl, struct request_queue *q,
		gfp_t gfp_mask);
63
void blk_exit_rl(struct request_queue *q, struct request_list *rl);
64 65
void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
			struct bio *bio);
66 67
void blk_queue_bypass_start(struct request_queue *q);
void blk_queue_bypass_end(struct request_queue *q);
68
void __blk_queue_free_tags(struct request_queue *q);
69 70 71 72 73 74 75 76 77 78 79 80
void blk_freeze_queue(struct request_queue *q);

static inline void blk_queue_enter_live(struct request_queue *q)
{
	/*
	 * Given that running in generic_make_request() context
	 * guarantees that a live reference against q_usage_counter has
	 * been established, further references under that same context
	 * need not check that the queue has been frozen (marked dead).
	 */
	percpu_ref_get(&q->q_usage_counter);
}
81

82 83
#ifdef CONFIG_BLK_DEV_INTEGRITY
void blk_flush_integrity(void);
84 85 86 87 88 89 90
bool __bio_integrity_endio(struct bio *);
static inline bool bio_integrity_endio(struct bio *bio)
{
	if (bio_integrity(bio))
		return __bio_integrity_endio(bio);
	return true;
}
91 92 93 94
#else
static inline void blk_flush_integrity(void)
{
}
95 96 97 98
static inline bool bio_integrity_endio(struct bio *bio)
{
	return true;
}
99
#endif
100

101
void blk_timeout_work(struct work_struct *work);
102
unsigned long blk_rq_timeout(unsigned long timeout);
103
void blk_add_timer(struct request *req);
104 105
void blk_delete_timer(struct request *);

106 107 108 109 110

bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
			     struct bio *bio);
bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
			    struct bio *bio);
111 112
bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
		struct bio *bio);
113
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
114 115
			    unsigned int *request_count,
			    struct request **same_queue_rq);
116
unsigned int blk_plug_queued_count(struct request_queue *q);
117 118 119 120 121

void blk_account_io_start(struct request *req, bool new_io);
void blk_account_io_completion(struct request *req, unsigned int bytes);
void blk_account_io_done(struct request *req);

122 123 124 125
/*
 * Internal atomic flags for request handling
 */
enum rq_atomic_flags {
126 127 128 129 130 131
	/*
	 * Keep these two bits first - not because we depend on the
	 * value of them, but we do depend on them being in the same
	 * byte of storage to ensure ordering on writes. Keeping them
	 * first will achieve that nicely.
	 */
132
	REQ_ATOM_COMPLETE = 0,
133
	REQ_ATOM_STARTED,
134

135
	REQ_ATOM_POLL_SLEPT,
136 137 138 139
};

/*
 * EH timer and IO completion will both attempt to 'grab' the request, make
140
 * sure that only one of them succeeds
141 142 143 144 145 146 147 148 149 150
 */
static inline int blk_mark_rq_complete(struct request *rq)
{
	return test_and_set_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
}

static inline void blk_clear_rq_complete(struct request *rq)
{
	clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
}
151

152 153 154
/*
 * Internal elevator interface
 */
155
#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED)
156

157
void blk_insert_flush(struct request *rq);
158

159 160 161 162
static inline void elv_activate_rq(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

163 164
	if (e->type->ops.sq.elevator_activate_req_fn)
		e->type->ops.sq.elevator_activate_req_fn(q, rq);
165 166 167 168 169 170
}

static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

171 172
	if (e->type->ops.sq.elevator_deactivate_req_fn)
		e->type->ops.sq.elevator_deactivate_req_fn(q, rq);
173 174
}

175 176
struct hd_struct *__disk_get_part(struct gendisk *disk, int partno);

177 178 179 180 181 182 183 184 185 186 187 188
#ifdef CONFIG_FAIL_IO_TIMEOUT
int blk_should_fake_timeout(struct request_queue *);
ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
ssize_t part_timeout_store(struct device *, struct device_attribute *,
				const char *, size_t);
#else
static inline int blk_should_fake_timeout(struct request_queue *q)
{
	return 0;
}
#endif

189 190 191 192
int ll_back_merge_fn(struct request_queue *q, struct request *req,
		     struct bio *bio);
int ll_front_merge_fn(struct request_queue *q, struct request *req, 
		      struct bio *bio);
193 194
struct request *attempt_back_merge(struct request_queue *q, struct request *rq);
struct request *attempt_front_merge(struct request_queue *q, struct request *rq);
195 196
int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
				struct request *next);
197
void blk_recalc_rq_segments(struct request *rq);
198
void blk_rq_set_mixed_merge(struct request *rq);
199
bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
200
enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
201

202 203
void blk_queue_congestion_threshold(struct request_queue *q);

204 205
int blk_dev_init(void);

206

207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
/*
 * Return the threshold (number of used requests) at which the queue is
 * considered to be congested.  It include a little hysteresis to keep the
 * context switch rate down.
 */
static inline int queue_congestion_on_threshold(struct request_queue *q)
{
	return q->nr_congestion_on;
}

/*
 * The threshold at which a queue is considered to be uncongested
 */
static inline int queue_congestion_off_threshold(struct request_queue *q)
{
	return q->nr_congestion_off;
}

225 226
extern int blk_update_nr_requests(struct request_queue *, unsigned int);

227 228 229 230 231
/*
 * Contribute to IO statistics IFF:
 *
 *	a) it's attached to a gendisk, and
 *	b) the queue had IO stats enabled when this request was started, and
232
 *	c) it's a file system request
233
 */
234
static inline int blk_do_io_stat(struct request *rq)
235
{
236
	return rq->rq_disk &&
237
	       (rq->rq_flags & RQF_IO_STAT) &&
238
		!blk_rq_is_passthrough(rq);
239 240
}

241 242 243 244 245 246 247
static inline void req_set_nomerge(struct request_queue *q, struct request *req)
{
	req->cmd_flags |= REQ_NOMERGE;
	if (req == q->last_merge)
		q->last_merge = NULL;
}

248 249 250 251
/*
 * Internal io_context interface
 */
void get_io_context(struct io_context *ioc);
252
struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q);
253 254
struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,
			     gfp_t gfp_mask);
255
void ioc_clear_queue(struct request_queue *q);
256

257
int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node);
258

259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
/**
 * rq_ioc - determine io_context for request allocation
 * @bio: request being allocated is for this bio (can be %NULL)
 *
 * Determine io_context to use for request allocation for @bio.  May return
 * %NULL if %current->io_context doesn't exist.
 */
static inline struct io_context *rq_ioc(struct bio *bio)
{
#ifdef CONFIG_BLK_CGROUP
	if (bio && bio->bi_ioc)
		return bio->bi_ioc;
#endif
	return current->io_context;
}

275 276 277 278 279
/**
 * create_io_context - try to create task->io_context
 * @gfp_mask: allocation mask
 * @node: allocation node
 *
280 281 282
 * If %current->io_context is %NULL, allocate a new io_context and install
 * it.  Returns the current %current->io_context which may be %NULL if
 * allocation failed.
283 284
 *
 * Note that this function can't be called with IRQ disabled because
285
 * task_lock which protects %current->io_context is IRQ-unsafe.
286
 */
287
static inline struct io_context *create_io_context(gfp_t gfp_mask, int node)
288 289
{
	WARN_ON_ONCE(irqs_disabled());
290 291 292
	if (unlikely(!current->io_context))
		create_task_io_context(current, gfp_mask, node);
	return current->io_context;
293 294 295 296 297
}

/*
 * Internal throttling interface
 */
298
#ifdef CONFIG_BLK_DEV_THROTTLING
299
extern void blk_throtl_drain(struct request_queue *q);
300 301
extern int blk_throtl_init(struct request_queue *q);
extern void blk_throtl_exit(struct request_queue *q);
302
extern void blk_throtl_register_queue(struct request_queue *q);
303
#else /* CONFIG_BLK_DEV_THROTTLING */
304
static inline void blk_throtl_drain(struct request_queue *q) { }
305 306
static inline int blk_throtl_init(struct request_queue *q) { return 0; }
static inline void blk_throtl_exit(struct request_queue *q) { }
307
static inline void blk_throtl_register_queue(struct request_queue *q) { }
308
#endif /* CONFIG_BLK_DEV_THROTTLING */
309 310 311 312
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page);
extern ssize_t blk_throtl_sample_time_store(struct request_queue *q,
	const char *page, size_t count);
313
extern void blk_throtl_bio_endio(struct bio *bio);
314
extern void blk_throtl_stat_add(struct request *rq, u64 time);
315 316
#else
static inline void blk_throtl_bio_endio(struct bio *bio) { }
317
static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }
318
#endif
319

320 321 322 323 324 325 326 327 328 329 330 331 332
#ifdef CONFIG_BOUNCE
extern int init_emergency_isa_pool(void);
extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
#else
static inline int init_emergency_isa_pool(void)
{
	return 0;
}
static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
{
}
#endif /* CONFIG_BOUNCE */

333
#endif /* BLK_INTERNAL_H */