aboutsummaryrefslogtreecommitdiff
path: root/block/ll_rw_blk.c
diff options
context:
space:
mode:
authorJens Axboe <jens.axboe@oracle.com>2008-01-24 08:44:49 +0100
committerJens Axboe <jens.axboe@oracle.com>2008-01-28 10:50:33 +0100
commit4ac845a2e9a816ed5a7b301f56dcc0a3d0b1ba4d (patch)
tree602f15808d0f3dcdfcd7cc4491b2cc2ccd266fd2 /block/ll_rw_blk.c
parent66dac98ed0de7a1125fb0dd7907f238f6b9d2f60 (diff)
block: cfq: make the io contect sharing lockless
The io context sharing introduced a per-ioc spinlock, that would protect the cfq io context lookup. That is a regression from the original, since we never needed any locking there because the ioc/cic were process private. The cic lookup is changed from an rbtree construct to a radix tree, which we can then use RCU to make the reader side lockless. That is the performance critical path, modifying the radix tree is only done on process creation (when that process first does IO, actually) and on process exit (if that process has done IO). As it so happens, radix trees are also much faster for this type of lookup where the key is a pointer. It's a very sparse tree. Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
Diffstat (limited to 'block/ll_rw_blk.c')
-rw-r--r--block/ll_rw_blk.c49
1 files changed, 34 insertions, 15 deletions
diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c
index d4550ecae44..b901db63f6a 100644
--- a/block/ll_rw_blk.c
+++ b/block/ll_rw_blk.c
@@ -3853,6 +3853,21 @@ int __init blk_dev_init(void)
return 0;
}
+static void cfq_dtor(struct io_context *ioc)
+{
+ struct cfq_io_context *cic[1];
+ int r;
+
+ /*
+ * We don't have a specific key to lookup with, so use the gang
+ * lookup to just retrieve the first item stored. The cfq exit
+ * function will iterate the full tree, so any member will do.
+ */
+ r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
+ if (r > 0)
+ cic[0]->dtor(ioc);
+}
+
/*
* IO Context helper functions. put_io_context() returns 1 if there are no
* more users of this io context, 0 otherwise.
@@ -3865,18 +3880,11 @@ int put_io_context(struct io_context *ioc)
BUG_ON(atomic_read(&ioc->refcount) == 0);
if (atomic_dec_and_test(&ioc->refcount)) {
- struct cfq_io_context *cic;
-
rcu_read_lock();
if (ioc->aic && ioc->aic->dtor)
ioc->aic->dtor(ioc->aic);
- if (ioc->cic_root.rb_node != NULL) {
- struct rb_node *n = rb_first(&ioc->cic_root);
-
- cic = rb_entry(n, struct cfq_io_context, rb_node);
- cic->dtor(ioc);
- }
rcu_read_unlock();
+ cfq_dtor(ioc);
kmem_cache_free(iocontext_cachep, ioc);
return 1;
@@ -3885,11 +3893,26 @@ int put_io_context(struct io_context *ioc)
}
EXPORT_SYMBOL(put_io_context);
+static void cfq_exit(struct io_context *ioc)
+{
+ struct cfq_io_context *cic[1];
+ int r;
+
+ rcu_read_lock();
+ /*
+ * See comment for cfq_dtor()
+ */
+ r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
+ rcu_read_unlock();
+
+ if (r > 0)
+ cic[0]->exit(ioc);
+}
+
/* Called by the exitting task */
void exit_io_context(void)
{
struct io_context *ioc;
- struct cfq_io_context *cic;
task_lock(current);
ioc = current->io_context;
@@ -3899,11 +3922,7 @@ void exit_io_context(void)
if (atomic_dec_and_test(&ioc->nr_tasks)) {
if (ioc->aic && ioc->aic->exit)
ioc->aic->exit(ioc->aic);
- if (ioc->cic_root.rb_node != NULL) {
- cic = rb_entry(rb_first(&ioc->cic_root),
- struct cfq_io_context, rb_node);
- cic->exit(ioc);
- }
+ cfq_exit(ioc);
put_io_context(ioc);
}
@@ -3923,7 +3942,7 @@ struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
ret->last_waited = jiffies; /* doesn't matter... */
ret->nr_batch_requests = 0; /* because this is 0 */
ret->aic = NULL;
- ret->cic_root.rb_node = NULL;
+ INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH);
ret->ioc_data = NULL;
}