name : trace_recursion.h /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_TRACE_RECURSION_H #define _LINUX_TRACE_RECURSION_H #include <linux/interrupt.h> #include <linux/sched.h> #ifdef CONFIG_TRACING /* Only current can touch trace_recursion */ /* * For function tracing recursion: * The order of these bits are important. * * When function tracing occurs, the following steps are made: * If arch does not support a ftrace feature: * call internal function (uses INTERNAL bits) which calls... * The function callback, which can use the FTRACE bits to * check for recursion. */ enum { /* Function recursion bits */ TRACE_FTRACE_BIT, TRACE_FTRACE_NMI_BIT, TRACE_FTRACE_IRQ_BIT, TRACE_FTRACE_SIRQ_BIT, TRACE_FTRACE_TRANSITION_BIT, /* Internal use recursion bits */ TRACE_INTERNAL_BIT, TRACE_INTERNAL_NMI_BIT, TRACE_INTERNAL_IRQ_BIT, TRACE_INTERNAL_SIRQ_BIT, TRACE_INTERNAL_TRANSITION_BIT, TRACE_BRANCH_BIT, /* * Abuse of the trace_recursion. * As we need a way to maintain state if we are tracing the function * graph in irq because we want to trace a particular function that * was called in irq context but we have irq tracing off. Since this * can only be modified by current, we can reuse trace_recursion. */ TRACE_IRQ_BIT, /* Set if the function is in the set_graph_function file */ TRACE_GRAPH_BIT, /* * In the very unlikely case that an interrupt came in * at a start of graph tracing, and we want to trace * the function in that interrupt, the depth can be greater * than zero, because of the preempted start of a previous * trace. In an even more unlikely case, depth could be 2 * if a softirq interrupted the start of graph tracing, * followed by an interrupt preempting a start of graph * tracing in the softirq, and depth can even be 3 * if an NMI came in at the start of an interrupt function * that preempted a softirq start of a function that * preempted normal context!!!! Luckily, it can't be * greater than 3, so the next two bits are a mask * of what the depth is when we set TRACE_GRAPH_BIT */ TRACE_GRAPH_DEPTH_START_BIT, TRACE_GRAPH_DEPTH_END_BIT, /* * To implement set_graph_notrace, if this bit is set, we ignore * function graph tracing of called functions, until the return * function is called to clear it. */ TRACE_GRAPH_NOTRACE_BIT, /* Used to prevent recursion recording from recursing. */ TRACE_RECORD_RECURSION_BIT, }; #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0) #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0) #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit))) #define trace_recursion_depth() \ (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3) #define trace_recursion_set_depth(depth) \ do { \ current->trace_recursion &= \ ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \ current->trace_recursion |= \ ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \ } while (0) #define TRACE_CONTEXT_BITS 4 #define TRACE_FTRACE_START TRACE_FTRACE_BIT #define TRACE_LIST_START TRACE_INTERNAL_BIT #define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1) /* * Used for setting context * NMI = 0 * IRQ = 1 * SOFTIRQ = 2 * NORMAL = 3 */ enum { TRACE_CTX_NMI, TRACE_CTX_IRQ, TRACE_CTX_SOFTIRQ, TRACE_CTX_NORMAL, TRACE_CTX_TRANSITION, }; static __always_inline int trace_get_context_bit(void) { unsigned char bit = interrupt_context_level(); return TRACE_CTX_NORMAL - bit; } #ifdef CONFIG_FTRACE_RECORD_RECURSION extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip); # define do_ftrace_record_recursion(ip, pip) \ do { \ if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \ trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \ ftrace_record_recursion(ip, pip); \ trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \ } \ } while (0) #else # define do_ftrace_record_recursion(ip, pip) do { } while (0) #endif #ifdef CONFIG_ARCH_WANTS_NO_INSTR # define trace_warn_on_no_rcu(ip) \ ({ \ bool __ret = !rcu_is_watching(); \ if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \ trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \ WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \ trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \ } \ __ret; \ }) #else # define trace_warn_on_no_rcu(ip) false #endif /* * Preemption is promised to be disabled when return bit >= 0. */ static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip, int start) { unsigned int val = READ_ONCE(current->trace_recursion); int bit; if (trace_warn_on_no_rcu(ip)) return -1; bit = trace_get_context_bit() + start; if (unlikely(val & (1 << bit))) { /* * If an interrupt occurs during a trace, and another trace * happens in that interrupt but before the preempt_count is * updated to reflect the new interrupt context, then this * will think a recursion occurred, and the event will be dropped. * Let a single instance happen via the TRANSITION_BIT to * not drop those events. */ bit = TRACE_CTX_TRANSITION + start; if (val & (1 << bit)) { do_ftrace_record_recursion(ip, pip); return -1; } } val |= 1 << bit; current->trace_recursion = val; barrier(); preempt_disable_notrace(); return bit; } /* * Preemption will be enabled (if it was previously enabled). */ static __always_inline void trace_clear_recursion(int bit) { preempt_enable_notrace(); barrier(); trace_recursion_clear(bit); } /** * ftrace_test_recursion_trylock - tests for recursion in same context * * Use this for ftrace callbacks. This will detect if the function * tracing recursed in the same context (normal vs interrupt), * * Returns: -1 if a recursion happened. * >= 0 if no recursion. */ static __always_inline int ftrace_test_recursion_trylock(unsigned long ip, unsigned long parent_ip) { return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START); } /** * ftrace_test_recursion_unlock - called when function callback is complete * @bit: The return of a successful ftrace_test_recursion_trylock() * * This is used at the end of a ftrace callback. */ static __always_inline void ftrace_test_recursion_unlock(int bit) { trace_clear_recursion(bit); } #endif /* CONFIG_TRACING */ #endif /* _LINUX_TRACE_RECURSION_H */