blk-ioc.c 11 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7 8 9
/*
 * Functions related to io context handling
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
10
#include <linux/slab.h>
11
#include <linux/sched/task.h>
12 13 14 15 16 17 18 19

#include "blk.h"

/*
 * For io context allocations
 */
static struct kmem_cache *iocontext_cachep;

20 21 22 23 24 25 26 27 28 29 30 31 32
/**
 * get_io_context - increment reference count to io_context
 * @ioc: io_context to get
 *
 * Increment reference count to @ioc.
 */
void get_io_context(struct io_context *ioc)
{
	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
	atomic_long_inc(&ioc->refcount);
}
EXPORT_SYMBOL(get_io_context);

33 34 35 36 37 38 39
static void icq_free_icq_rcu(struct rcu_head *head)
{
	struct io_cq *icq = container_of(head, struct io_cq, __rcu_head);

	kmem_cache_free(icq->__rcu_icq_cache, icq);
}

40
/*
41 42
 * Exit an icq. Called with ioc locked for blk-mq, and with both ioc
 * and queue locked for legacy.
43
 */
44
static void ioc_exit_icq(struct io_cq *icq)
45 46 47 48 49 50
{
	struct elevator_type *et = icq->q->elevator->type;

	if (icq->flags & ICQ_EXITED)
		return;

51 52 53
	if (et->uses_mq && et->ops.mq.exit_icq)
		et->ops.mq.exit_icq(icq);
	else if (!et->uses_mq && et->ops.sq.elevator_exit_icq_fn)
54
		et->ops.sq.elevator_exit_icq_fn(icq);
55 56 57 58

	icq->flags |= ICQ_EXITED;
}

59 60 61 62
/*
 * Release an icq. Called with ioc locked for blk-mq, and with both ioc
 * and queue locked for legacy.
 */
63
static void ioc_destroy_icq(struct io_cq *icq)
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
{
	struct io_context *ioc = icq->ioc;
	struct request_queue *q = icq->q;
	struct elevator_type *et = q->elevator->type;

	lockdep_assert_held(&ioc->lock);

	radix_tree_delete(&ioc->icq_tree, icq->q->id);
	hlist_del_init(&icq->ioc_node);
	list_del_init(&icq->q_node);

	/*
	 * Both setting lookup hint to and clearing it from @icq are done
	 * under queue_lock.  If it's not pointing to @icq now, it never
	 * will.  Hint assignment itself can race safely.
	 */
80
	if (rcu_access_pointer(ioc->icq_hint) == icq)
81 82
		rcu_assign_pointer(ioc->icq_hint, NULL);

83
	ioc_exit_icq(icq);
84 85 86 87 88 89 90 91 92

	/*
	 * @icq->q might have gone away by the time RCU callback runs
	 * making it impossible to determine icq_cache.  Record it in @icq.
	 */
	icq->__rcu_icq_cache = et->icq_cache;
	call_rcu(&icq->__rcu_head, icq_free_icq_rcu);
}

93 94
/*
 * Slow path for ioc release in put_io_context().  Performs double-lock
95
 * dancing to unlink all icq's and then frees ioc.
96 97
 */
static void ioc_release_fn(struct work_struct *work)
98
{
99 100
	struct io_context *ioc = container_of(work, struct io_context,
					      release_work);
101
	unsigned long flags;
102

103 104 105 106 107 108 109
	/*
	 * Exiting icq may call into put_io_context() through elevator
	 * which will trigger lockdep warning.  The ioc's are guaranteed to
	 * be different, use a different locking subclass here.  Use
	 * irqsave variant as there's no spin_lock_irq_nested().
	 */
	spin_lock_irqsave_nested(&ioc->lock, flags, 1);
110

111 112 113
	while (!hlist_empty(&ioc->icq_list)) {
		struct io_cq *icq = hlist_entry(ioc->icq_list.first,
						struct io_cq, ioc_node);
Tejun Heo's avatar
Tejun Heo committed
114 115 116
		struct request_queue *q = icq->q;

		if (spin_trylock(q->queue_lock)) {
117
			ioc_destroy_icq(icq);
Tejun Heo's avatar
Tejun Heo committed
118 119 120 121 122
			spin_unlock(q->queue_lock);
		} else {
			spin_unlock_irqrestore(&ioc->lock, flags);
			cpu_relax();
			spin_lock_irqsave_nested(&ioc->lock, flags, 1);
123 124
		}
	}
125

Tejun Heo's avatar
Tejun Heo committed
126
	spin_unlock_irqrestore(&ioc->lock, flags);
127 128

	kmem_cache_free(iocontext_cachep, ioc);
129 130
}

Tejun Heo's avatar
Tejun Heo committed
131 132 133 134 135
/**
 * put_io_context - put a reference of io_context
 * @ioc: io_context to put
 *
 * Decrement reference count of @ioc and release it if the count reaches
136
 * zero.
137
 */
138
void put_io_context(struct io_context *ioc)
139
{
140
	unsigned long flags;
141
	bool free_ioc = false;
142

143
	if (ioc == NULL)
Tejun Heo's avatar
Tejun Heo committed
144
		return;
145

Tejun Heo's avatar
Tejun Heo committed
146
	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
147

148
	/*
149 150
	 * Releasing ioc requires reverse order double locking and we may
	 * already be holding a queue_lock.  Do it asynchronously from wq.
151
	 */
152 153 154
	if (atomic_long_dec_and_test(&ioc->refcount)) {
		spin_lock_irqsave(&ioc->lock, flags);
		if (!hlist_empty(&ioc->icq_list))
155 156
			queue_work(system_power_efficient_wq,
					&ioc->release_work);
157 158
		else
			free_ioc = true;
159
		spin_unlock_irqrestore(&ioc->lock, flags);
160
	}
161 162 163

	if (free_ioc)
		kmem_cache_free(iocontext_cachep, ioc);
164
}
165
EXPORT_SYMBOL(put_io_context);
166

167 168 169 170 171 172 173 174
/**
 * put_io_context_active - put active reference on ioc
 * @ioc: ioc of interest
 *
 * Undo get_io_context_active().  If active reference reaches zero after
 * put, @ioc can never issue further IOs and ioscheds are notified.
 */
void put_io_context_active(struct io_context *ioc)
175
{
176
	struct elevator_type *et;
177
	unsigned long flags;
178
	struct io_cq *icq;
179

180
	if (!atomic_dec_and_test(&ioc->active_ref)) {
181 182 183 184 185 186 187 188 189 190 191
		put_io_context(ioc);
		return;
	}

	/*
	 * Need ioc lock to walk icq_list and q lock to exit icq.  Perform
	 * reverse double locking.  Read comment in ioc_release_fn() for
	 * explanation on the nested locking annotation.
	 */
retry:
	spin_lock_irqsave_nested(&ioc->lock, flags, 1);
192
	hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) {
193 194
		if (icq->flags & ICQ_EXITED)
			continue;
195 196 197

		et = icq->q->elevator->type;
		if (et->uses_mq) {
198 199
			ioc_exit_icq(icq);
		} else {
200 201 202 203 204 205 206 207
			if (spin_trylock(icq->q->queue_lock)) {
				ioc_exit_icq(icq);
				spin_unlock(icq->q->queue_lock);
			} else {
				spin_unlock_irqrestore(&ioc->lock, flags);
				cpu_relax();
				goto retry;
			}
208 209 210 211
		}
	}
	spin_unlock_irqrestore(&ioc->lock, flags);

212
	put_io_context(ioc);
213 214
}

215 216 217 218 219 220 221 222 223 224 225 226 227 228
/* Called by the exiting task */
void exit_io_context(struct task_struct *task)
{
	struct io_context *ioc;

	task_lock(task);
	ioc = task->io_context;
	task->io_context = NULL;
	task_unlock(task);

	atomic_dec(&ioc->nr_tasks);
	put_io_context_active(ioc);
}

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
static void __ioc_clear_queue(struct list_head *icq_list)
{
	unsigned long flags;

	while (!list_empty(icq_list)) {
		struct io_cq *icq = list_entry(icq_list->next,
					       struct io_cq, q_node);
		struct io_context *ioc = icq->ioc;

		spin_lock_irqsave(&ioc->lock, flags);
		ioc_destroy_icq(icq);
		spin_unlock_irqrestore(&ioc->lock, flags);
	}
}

244 245 246 247
/**
 * ioc_clear_queue - break any ioc association with the specified queue
 * @q: request_queue being cleared
 *
248
 * Walk @q->icq_list and exit all io_cq's.
249 250 251
 */
void ioc_clear_queue(struct request_queue *q)
{
252
	LIST_HEAD(icq_list);
253

254 255
	spin_lock_irq(q->queue_lock);
	list_splice_init(&q->icq_list, &icq_list);
256

257 258 259 260 261 262
	if (q->mq_ops) {
		spin_unlock_irq(q->queue_lock);
		__ioc_clear_queue(&icq_list);
	} else {
		__ioc_clear_queue(&icq_list);
		spin_unlock_irq(q->queue_lock);
263 264 265
	}
}

266
int create_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node)
267
{
268
	struct io_context *ioc;
269
	int ret;
270

Tejun Heo's avatar
Tejun Heo committed
271 272 273
	ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,
				    node);
	if (unlikely(!ioc))
274
		return -ENOMEM;
Tejun Heo's avatar
Tejun Heo committed
275 276 277

	/* initialize */
	atomic_long_set(&ioc->refcount, 1);
278
	atomic_set(&ioc->nr_tasks, 1);
279
	atomic_set(&ioc->active_ref, 1);
Tejun Heo's avatar
Tejun Heo committed
280
	spin_lock_init(&ioc->lock);
281 282
	INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH);
	INIT_HLIST_HEAD(&ioc->icq_list);
283
	INIT_WORK(&ioc->release_work, ioc_release_fn);
284

285 286 287 288 289 290 291
	/*
	 * Try to install.  ioc shouldn't be installed if someone else
	 * already did or @task, which isn't %current, is exiting.  Note
	 * that we need to allow ioc creation on exiting %current as exit
	 * path may issue IOs from e.g. exit_files().  The exit path is
	 * responsible for not issuing IO after exit_io_context().
	 */
292
	task_lock(task);
293 294
	if (!task->io_context &&
	    (task == current || !(task->flags & PF_EXITING)))
295
		task->io_context = ioc;
296
	else
297
		kmem_cache_free(iocontext_cachep, ioc);
298 299 300

	ret = task->io_context ? 0 : -EBUSY;

301
	task_unlock(task);
302

303
	return ret;
304 305
}

306 307 308 309 310 311 312 313 314
/**
 * get_task_io_context - get io_context of a task
 * @task: task of interest
 * @gfp_flags: allocation flags, used if allocation is necessary
 * @node: allocation node, used if allocation is necessary
 *
 * Return io_context of @task.  If it doesn't exist, it is created with
 * @gfp_flags and @node.  The returned io_context has its reference count
 * incremented.
315
 *
316
 * This function always goes through task_lock() and it's better to use
317
 * %current->io_context + get_io_context() for %current.
318
 */
319 320
struct io_context *get_task_io_context(struct task_struct *task,
				       gfp_t gfp_flags, int node)
321
{
322
	struct io_context *ioc;
323

324
	might_sleep_if(gfpflags_allow_blocking(gfp_flags));
325

326 327 328 329 330 331 332 333
	do {
		task_lock(task);
		ioc = task->io_context;
		if (likely(ioc)) {
			get_io_context(ioc);
			task_unlock(task);
			return ioc;
		}
334
		task_unlock(task);
335
	} while (!create_task_io_context(task, gfp_flags, node));
336

337
	return NULL;
338
}
339
EXPORT_SYMBOL(get_task_io_context);
340

341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
/**
 * ioc_lookup_icq - lookup io_cq from ioc
 * @ioc: the associated io_context
 * @q: the associated request_queue
 *
 * Look up io_cq associated with @ioc - @q pair from @ioc.  Must be called
 * with @q->queue_lock held.
 */
struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q)
{
	struct io_cq *icq;

	lockdep_assert_held(q->queue_lock);

	/*
	 * icq's are indexed from @ioc using radix tree and hint pointer,
	 * both of which are protected with RCU.  All removals are done
	 * holding both q and ioc locks, and we're holding q lock - if we
	 * find a icq which points to us, it's guaranteed to be valid.
	 */
	rcu_read_lock();
	icq = rcu_dereference(ioc->icq_hint);
	if (icq && icq->q == q)
		goto out;

	icq = radix_tree_lookup(&ioc->icq_tree, q->id);
	if (icq && icq->q == q)
		rcu_assign_pointer(ioc->icq_hint, icq);	/* allowed to race */
	else
		icq = NULL;
out:
	rcu_read_unlock();
	return icq;
}
EXPORT_SYMBOL(ioc_lookup_icq);

377 378
/**
 * ioc_create_icq - create and link io_cq
379
 * @ioc: io_context of interest
380 381 382
 * @q: request_queue of interest
 * @gfp_mask: allocation mask
 *
383 384
 * Make sure io_cq linking @ioc and @q exists.  If icq doesn't exist, they
 * will be created using @gfp_mask.
385 386 387 388
 *
 * The caller is responsible for ensuring @ioc won't go away and @q is
 * alive and will stay alive until this function returns.
 */
389 390
struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,
			     gfp_t gfp_mask)
391 392 393 394 395 396 397 398 399 400
{
	struct elevator_type *et = q->elevator->type;
	struct io_cq *icq;

	/* allocate stuff */
	icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO,
				    q->node);
	if (!icq)
		return NULL;

401
	if (radix_tree_maybe_preload(gfp_mask) < 0) {
402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
		kmem_cache_free(et->icq_cache, icq);
		return NULL;
	}

	icq->ioc = ioc;
	icq->q = q;
	INIT_LIST_HEAD(&icq->q_node);
	INIT_HLIST_NODE(&icq->ioc_node);

	/* lock both q and ioc and try to link @icq */
	spin_lock_irq(q->queue_lock);
	spin_lock(&ioc->lock);

	if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
		hlist_add_head(&icq->ioc_node, &ioc->icq_list);
		list_add(&icq->q_node, &q->icq_list);
418 419 420
		if (et->uses_mq && et->ops.mq.init_icq)
			et->ops.mq.init_icq(icq);
		else if (!et->uses_mq && et->ops.sq.elevator_init_icq_fn)
421
			et->ops.sq.elevator_init_icq_fn(icq);
422 423 424 425 426 427 428 429 430 431 432 433 434
	} else {
		kmem_cache_free(et->icq_cache, icq);
		icq = ioc_lookup_icq(ioc, q);
		if (!icq)
			printk(KERN_ERR "cfq: icq link failed!\n");
	}

	spin_unlock(&ioc->lock);
	spin_unlock_irq(q->queue_lock);
	radix_tree_preload_end();
	return icq;
}

Adrian Bunk's avatar
Adrian Bunk committed
435
static int __init blk_ioc_init(void)
436 437 438 439 440 441
{
	iocontext_cachep = kmem_cache_create("blkdev_ioc",
			sizeof(struct io_context), 0, SLAB_PANIC, NULL);
	return 0;
}
subsys_initcall(blk_ioc_init);