aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSteven Rostedt <srostedt@redhat.com>2008-05-12 21:20:45 +0200
committerThomas Gleixner <tglx@linutronix.de>2008-05-23 20:41:35 +0200
commit93a588f459da134be6ab17c4104e28441beb0d22 (patch)
tree30eb991743d7ce5698f070204d666eca98519213
parent1d4db00a5e30c7b8f8dc2a1b19e886fd942be143 (diff)
ftrace: change buffers to producer consumer
This patch changes the way the CPU trace buffers are handled. Instead of always starting from the trace page head, the logic is changed to a producer consumer logic. This allows for the buffers to be drained while they are alive. Signed-off-by: Steven Rostedt <srostedt@redhat.com> Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
-rw-r--r--kernel/trace/trace.c103
-rw-r--r--kernel/trace/trace.h6
2 files changed, 65 insertions, 44 deletions
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 6580e7ed04b..777b859e1c2 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -176,10 +176,9 @@ flip_trace(struct trace_array_cpu *tr1, struct trace_array_cpu *tr2)
INIT_LIST_HEAD(&flip_pages);
- tr1->trace_current = NULL;
- memcpy(&tr1->trace_current_idx, &tr2->trace_current_idx,
+ memcpy(&tr1->trace_head_idx, &tr2->trace_head_idx,
sizeof(struct trace_array_cpu) -
- offsetof(struct trace_array_cpu, trace_current_idx));
+ offsetof(struct trace_array_cpu, trace_head_idx));
check_pages(tr1);
check_pages(tr2);
@@ -228,7 +227,6 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
tracing_reset(max_tr.data[i]);
flip_trace(max_tr.data[cpu], data);
-
tracing_reset(data);
__update_max_tr(tr, tsk, cpu);
@@ -343,9 +341,9 @@ void unregister_tracer(struct tracer *type)
notrace void tracing_reset(struct trace_array_cpu *data)
{
data->trace_idx = 0;
- data->trace_current = head_page(data);
- data->trace_current_idx = 0;
- data->time_offset = 0;
+ data->trace_head = data->trace_tail = head_page(data);
+ data->trace_head_idx = 0;
+ data->trace_tail_idx = 0;
}
#ifdef CONFIG_FTRACE
@@ -470,38 +468,65 @@ notrace void tracing_record_cmdline(struct task_struct *tsk)
trace_save_cmdline(tsk);
}
+static inline notrace struct list_head *
+trace_next_list(struct trace_array_cpu *data, struct list_head *next)
+{
+ /*
+ * Roundrobin - but skip the head (which is not a real page):
+ */
+ next = next->next;
+ if (unlikely(next == &data->trace_pages))
+ next = next->next;
+ BUG_ON(next == &data->trace_pages);
+
+ return next;
+}
+
+static inline notrace void *
+trace_next_page(struct trace_array_cpu *data, void *addr)
+{
+ struct list_head *next;
+ struct page *page;
+
+ page = virt_to_page(addr);
+
+ next = trace_next_list(data, &page->lru);
+ page = list_entry(next, struct page, lru);
+
+ return page_address(page);
+}
+
static inline notrace struct trace_entry *
tracing_get_trace_entry(struct trace_array *tr, struct trace_array_cpu *data)
{
unsigned long idx, idx_next;
struct trace_entry *entry;
- struct list_head *next;
- struct page *page;
data->trace_idx++;
- idx = data->trace_current_idx;
+ idx = data->trace_head_idx;
idx_next = idx + 1;
BUG_ON(idx * TRACE_ENTRY_SIZE >= PAGE_SIZE);
- entry = data->trace_current + idx * TRACE_ENTRY_SIZE;
+ entry = data->trace_head + idx * TRACE_ENTRY_SIZE;
if (unlikely(idx_next >= ENTRIES_PER_PAGE)) {
- page = virt_to_page(data->trace_current);
- /*
- * Roundrobin - but skip the head (which is not a real page):
- */
- next = page->lru.next;
- if (unlikely(next == &data->trace_pages))
- next = next->next;
- BUG_ON(next == &data->trace_pages);
-
- page = list_entry(next, struct page, lru);
- data->trace_current = page_address(page);
+ data->trace_head = trace_next_page(data, data->trace_head);
idx_next = 0;
}
- data->trace_current_idx = idx_next;
+ if (data->trace_head == data->trace_tail &&
+ idx_next == data->trace_tail_idx) {
+ /* overrun */
+ data->trace_tail_idx++;
+ if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
+ data->trace_tail =
+ trace_next_page(data, data->trace_tail);
+ data->trace_tail_idx = 0;
+ }
+ }
+
+ data->trace_head_idx = idx_next;
return entry;
}
@@ -571,21 +596,11 @@ trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
return NULL;
if (!iter->next_page[cpu]) {
- /*
- * Initialize. If the count of elements in
- * this buffer is greater than the max entries
- * we had an underrun. Which means we looped around.
- * We can simply use the current pointer as our
- * starting point.
- */
- if (data->trace_idx >= tr->entries) {
- page = virt_to_page(data->trace_current);
- iter->next_page[cpu] = &page->lru;
- iter->next_page_idx[cpu] = data->trace_current_idx;
- } else {
- iter->next_page[cpu] = data->trace_pages.next;
- iter->next_page_idx[cpu] = 0;
- }
+ /* Initialize the iterator for this cpu trace buffer */
+ WARN_ON(!data->trace_tail);
+ page = virt_to_page(data->trace_tail);
+ iter->next_page[cpu] = &page->lru;
+ iter->next_page_idx[cpu] = data->trace_tail_idx;
}
page = list_entry(iter->next_page[cpu], struct page, lru);
@@ -593,6 +608,12 @@ trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
array = page_address(page);
+ /* Still possible to catch up to the tail */
+ if (iter->next_idx[cpu] && array == data->trace_tail &&
+ iter->next_page_idx[cpu] == data->trace_tail_idx)
+ return NULL;
+
+ WARN_ON(iter->next_page_idx[cpu] >= ENTRIES_PER_PAGE);
return &array[iter->next_page_idx[cpu]];
}
@@ -638,10 +659,8 @@ static void *find_next_entry_inc(struct trace_iterator *iter)
iter->next_page_idx[next_cpu] = 0;
iter->next_page[next_cpu] =
- iter->next_page[next_cpu]->next;
- if (iter->next_page[next_cpu] == &data->trace_pages)
- iter->next_page[next_cpu] =
- data->trace_pages.next;
+ trace_next_list(data, iter->next_page[next_cpu]);
+
}
}
iter->prev_ent = iter->ent;
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 5df8ff2b84a..0ce127455b4 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -53,13 +53,15 @@ struct trace_entry {
* the trace, etc.)
*/
struct trace_array_cpu {
- void *trace_current;
struct list_head trace_pages;
atomic_t disabled;
cycle_t time_offset;
/* these fields get copied into max-trace: */
- unsigned trace_current_idx;
+ unsigned trace_head_idx;
+ unsigned trace_tail_idx;
+ void *trace_head; /* producer */
+ void *trace_tail; /* consumer */
unsigned long trace_idx;
unsigned long saved_latency;
unsigned long critical_start;