blk-ioc.c 5.78 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * Functions related to io context handling
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/bootmem.h>	/* for max_pfn/max_low_pfn */
10
#include <linux/slab.h>
11 12 13 14 15 16 17 18

#include "blk.h"

/*
 * For io context allocations
 */
static struct kmem_cache *iocontext_cachep;

19 20 21 22 23 24 25 26 27 28 29 30 31
/**
 * get_io_context - increment reference count to io_context
 * @ioc: io_context to get
 *
 * Increment reference count to @ioc.
 */
void get_io_context(struct io_context *ioc)
{
	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
	atomic_long_inc(&ioc->refcount);
}
EXPORT_SYMBOL(get_io_context);

32 33
static void cfq_dtor(struct io_context *ioc)
{
34 35 36
	if (!hlist_empty(&ioc->cic_list)) {
		struct cfq_io_context *cic;

37
		cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context,
38 39 40
								cic_list);
		cic->dtor(ioc);
	}
41 42
}

Tejun Heo's avatar
Tejun Heo committed
43 44 45 46 47 48
/**
 * put_io_context - put a reference of io_context
 * @ioc: io_context to put
 *
 * Decrement reference count of @ioc and release it if the count reaches
 * zero.
49
 */
Tejun Heo's avatar
Tejun Heo committed
50
void put_io_context(struct io_context *ioc)
51 52
{
	if (ioc == NULL)
Tejun Heo's avatar
Tejun Heo committed
53
		return;
54

Tejun Heo's avatar
Tejun Heo committed
55
	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
56

Tejun Heo's avatar
Tejun Heo committed
57 58
	if (!atomic_long_dec_and_test(&ioc->refcount))
		return;
59

Tejun Heo's avatar
Tejun Heo committed
60 61 62 63 64
	rcu_read_lock();
	cfq_dtor(ioc);
	rcu_read_unlock();

	kmem_cache_free(iocontext_cachep, ioc);
65 66 67 68 69 70 71
}
EXPORT_SYMBOL(put_io_context);

static void cfq_exit(struct io_context *ioc)
{
	rcu_read_lock();

72 73 74
	if (!hlist_empty(&ioc->cic_list)) {
		struct cfq_io_context *cic;

75
		cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context,
76 77 78 79
								cic_list);
		cic->exit(ioc);
	}
	rcu_read_unlock();
80 81
}

82
/* Called by the exiting task */
83
void exit_io_context(struct task_struct *task)
84 85 86
{
	struct io_context *ioc;

87 88 89
	/* PF_EXITING prevents new io_context from being attached to @task */
	WARN_ON_ONCE(!(current->flags & PF_EXITING));

90 91 92 93
	task_lock(task);
	ioc = task->io_context;
	task->io_context = NULL;
	task_unlock(task);
94

95
	if (atomic_dec_and_test(&ioc->nr_tasks))
96 97
		cfq_exit(ioc);

98
	put_io_context(ioc);
99 100
}

101 102 103
static struct io_context *create_task_io_context(struct task_struct *task,
						 gfp_t gfp_flags, int node,
						 bool take_ref)
104
{
105
	struct io_context *ioc;
106

Tejun Heo's avatar
Tejun Heo committed
107 108 109 110 111 112 113 114 115 116 117
	ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,
				    node);
	if (unlikely(!ioc))
		return NULL;

	/* initialize */
	atomic_long_set(&ioc->refcount, 1);
	atomic_set(&ioc->nr_tasks, 1);
	spin_lock_init(&ioc->lock);
	INIT_RADIX_TREE(&ioc->radix_root, GFP_ATOMIC | __GFP_HIGH);
	INIT_HLIST_HEAD(&ioc->cic_list);
118

119 120 121 122 123 124 125 126 127 128 129 130 131 132
	/* try to install, somebody might already have beaten us to it */
	task_lock(task);

	if (!task->io_context && !(task->flags & PF_EXITING)) {
		task->io_context = ioc;
	} else {
		kmem_cache_free(iocontext_cachep, ioc);
		ioc = task->io_context;
	}

	if (ioc && take_ref)
		get_io_context(ioc);

	task_unlock(task);
133
	return ioc;
134 135
}

Tejun Heo's avatar
Tejun Heo committed
136 137 138 139
/**
 * current_io_context - get io_context of %current
 * @gfp_flags: allocation flags, used if allocation is necessary
 * @node: allocation node, used if allocation is necessary
140
 *
Tejun Heo's avatar
Tejun Heo committed
141 142 143 144 145
 * Return io_context of %current.  If it doesn't exist, it is created with
 * @gfp_flags and @node.  The returned io_context does NOT have its
 * reference count incremented.  Because io_context is exited only on task
 * exit, %current can be sure that the returned io_context is valid and
 * alive as long as it is executing.
146 147 148
 */
struct io_context *current_io_context(gfp_t gfp_flags, int node)
{
149
	might_sleep_if(gfp_flags & __GFP_WAIT);
150

151 152 153 154
	if (current->io_context)
		return current->io_context;

	return create_task_io_context(current, gfp_flags, node, false);
155
}
156
EXPORT_SYMBOL(current_io_context);
157

158 159 160 161 162 163 164 165 166
/**
 * get_task_io_context - get io_context of a task
 * @task: task of interest
 * @gfp_flags: allocation flags, used if allocation is necessary
 * @node: allocation node, used if allocation is necessary
 *
 * Return io_context of @task.  If it doesn't exist, it is created with
 * @gfp_flags and @node.  The returned io_context has its reference count
 * incremented.
167
 *
168 169
 * This function always goes through task_lock() and it's better to use
 * current_io_context() + get_io_context() for %current.
170
 */
171 172
struct io_context *get_task_io_context(struct task_struct *task,
				       gfp_t gfp_flags, int node)
173
{
174
	struct io_context *ioc;
175

176 177 178 179 180 181 182 183 184 185 186 187
	might_sleep_if(gfp_flags & __GFP_WAIT);

	task_lock(task);
	ioc = task->io_context;
	if (likely(ioc)) {
		get_io_context(ioc);
		task_unlock(task);
		return ioc;
	}
	task_unlock(task);

	return create_task_io_context(task, gfp_flags, node, true);
188
}
189
EXPORT_SYMBOL(get_task_io_context);
190

191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
void ioc_set_changed(struct io_context *ioc, int which)
{
	struct cfq_io_context *cic;
	struct hlist_node *n;

	hlist_for_each_entry(cic, n, &ioc->cic_list, cic_list)
		set_bit(which, &cic->changed);
}

/**
 * ioc_ioprio_changed - notify ioprio change
 * @ioc: io_context of interest
 * @ioprio: new ioprio
 *
 * @ioc's ioprio has changed to @ioprio.  Set %CIC_IOPRIO_CHANGED for all
 * cic's.  iosched is responsible for checking the bit and applying it on
 * request issue path.
 */
void ioc_ioprio_changed(struct io_context *ioc, int ioprio)
{
	unsigned long flags;

	spin_lock_irqsave(&ioc->lock, flags);
	ioc->ioprio = ioprio;
	ioc_set_changed(ioc, CIC_IOPRIO_CHANGED);
	spin_unlock_irqrestore(&ioc->lock, flags);
}

/**
 * ioc_cgroup_changed - notify cgroup change
 * @ioc: io_context of interest
 *
 * @ioc's cgroup has changed.  Set %CIC_CGROUP_CHANGED for all cic's.
 * iosched is responsible for checking the bit and applying it on request
 * issue path.
 */
void ioc_cgroup_changed(struct io_context *ioc)
{
	unsigned long flags;

	spin_lock_irqsave(&ioc->lock, flags);
	ioc_set_changed(ioc, CIC_CGROUP_CHANGED);
	spin_unlock_irqrestore(&ioc->lock, flags);
}

Adrian Bunk's avatar
Adrian Bunk committed
236
static int __init blk_ioc_init(void)
237 238 239 240 241 242
{
	iocontext_cachep = kmem_cache_create("blkdev_ioc",
			sizeof(struct io_context), 0, SLAB_PANIC, NULL);
	return 0;
}
subsys_initcall(blk_ioc_init);