linux 内核 static-key机制分析
1、static key是什么
Linux内核的 Static Keys机制是一种高效的条件分支优化技术,主要用于在运行时动态启用或禁用特定代码路径,同时避免常规条件判断(如 if 语句)的性能开销。它通过结合编译时优化和运行时代码修补(如 Jump Label 技术)实现近乎零成本的开关切换,广泛应用于性能敏感的代码逻辑(如调试代码、跟踪点、性能监控等)。
1.1、核心原理
- 无分支执行:Static Keys将条件判断转换为对内存中某个标志位的直接跳转,绕过传统if语句的分支预测和流水线冲刷。
- 代码修补:通过修改内存中的指令(如将 nop 替换为 jmp),在运行时动态启用或禁用代码路径(依赖CPU架构支持，如x86的jmp指令替换)。
- 原子性操作:启用或禁用Static Key是原子操作，无需锁机制，保证线程安全。
1.2、核心数据结构
1.2.1、struct static_key
表示一个静态键的状态,关键字段如下：
struct static_key {atomic_t enabled;/* Jump Label 相关字段（如跳转地址） */};enabled:键的启用状态(0 禁用，1 启用)。
1.2.2、键的初始状态
DEFINE_STATIC_KEY_TRUE(key): 定义初始为true的键。
DEFINE_STATIC_KEY_FALSE(key): 定义初始为false的键。
1.2.3、使用方式
(1)、定义静态键
#include <linux/jump_label.h>
/*定义并初始化一个静态键(默认禁用)*/
DEFINE_STATIC_KEY_FALSE(my_key);
(2)、在代码中使用静态键
if (static_branch_unlikely(&my_key)) {
/*仅当 my_key 启用时执行此代码块*/
pr_info("Feature is enabled!\n");
}
- 分支预测提示：
static_branch_unlikely():暗示代码块大概率不执行(优化跳转指令位置)。
static_branch_likely():暗示代码块大概率执行。
(3)、动态启用/禁用键
/*启用键*/
static_branch_enable(&my_key);
/*禁用键*/
static_branch_disable(&my_key);
2、数据结构
typedef u32 jump_label_t;
struct jump_entry {jump_label_t code; /*被内核动态修改的nop或跳转指令地址 */jump_label_t target; /*l_yes 标号地址，arch_static_branch()函数中会描述 l_yes 标号*/
/*关联的static_key的地址，static_key的地址4字节对齐,地址值最低2bit位总是0。内核实现利用最低1位用来标识 branch 类型: 0 -> false (static_key_false()构建的 jump_entry )1 -> true (static_key_true()构建的 jump_entry )*/jump_label_t key;
};struct static_key_mod {struct static_key_mod *next;struct jump_entry *entries;struct module *mod;
};struct static_key {atomic_t enabled;
/** Note:* To make anonymous unions(匿名联合) work with old compilers, the static* initialization of them requires brackets(大括号). This creates a dependency* on the order of the struct with the initializers. If any fields* are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need* to be modified.** bit 0 => 1 if key is initially true* 0 if initially false* bit 1 => 1 if points to struct static_key_mod * 0 if points to struct jump_entry* 因为地址是4字节对齐,所以可以将低2bit作为标记使用*/union {unsigned long type;struct jump_entry *entries;struct static_key_mod *next;};
};/** Two type wrappers around static_key, such that we can use compile time type differentiation to emit the right code.* All the below code is macros in order to play type games.*/
struct static_key_true {struct static_key key;
};struct static_key_false {struct static_key key;
};enum jump_label_type {JUMP_LABEL_NOP = 0,JUMP_LABEL_JMP,
};#define JUMP_TYPE_FALSE 0UL
#define JUMP_TYPE_TRUE 1UL
#define JUMP_TYPE_LINKED 2UL
#define JUMP_TYPE_MASK 3UL下面是static key声明和定义
/** We should be using ATOMIC_INIT() for initializing .enabled, but* the inclusion of atomic.h is problematic for inclusion of jump_label.h* in 'low-level' headers. Thus, we are initializing .enabled with a* raw value, but have added a BUILD_BUG_ON() to catch any issues in* jump_label_init() see: kernel/jump_label.c.*/
/*enable为1，entries为1*/
#define STATIC_KEY_INIT_TRUE \{ .enabled = { 1 }, \{ .entries = (void *)JUMP_TYPE_TRUE } }
/*enable为0,entries为0*/
#define STATIC_KEY_INIT_FALSE \{ .enabled = { 0 }, \{ .entries = (void *)JUMP_TYPE_FALSE } }/*根据类型(true/false)来初始化.key成员变量*/
#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }#define DEFINE_STATIC_KEY_TRUE(name) \struct static_key_true name = STATIC_KEY_TRUE_INIT#define DECLARE_STATIC_KEY_TRUE(name) \extern struct static_key_true name#define DEFINE_STATIC_KEY_FALSE(name) \struct static_key_false name = STATIC_KEY_FALSE_INIT#define DECLARE_STATIC_KEY_FALSE(name) \extern struct static_key_false name#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \struct static_key_true name[count] = { \[0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \}#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \struct static_key_false name[count] = { \[0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \}enum jump_label_type {JUMP_LABEL_NOP = 0,JUMP_LABEL_JMP,
};3、接口分析
static_branch_likely()
static_branch_unlikely()
static_branch_enable()
static_branch_disable()
static_branch_inc()
static_branch_dec()
3.1、static_branch_likely/static_branch_unlikely()分析
/** Combine the right initial value (type) with the right branch order to generate the desired result.* 注意 likely(...)优化后会将成功放在前面,失败放在后面；unlikely(...)优化后会将失败放在前面,成功放在后面；* type为true表示struct static_key_true，type为false表示struct static_key_false* type\branch| likely (1) | unlikely (0)* -----------+-----------------------+------------------* | |* true (1) | ... | ...* | NOP | JMP L* | <br-stmts> | 1: ...* | L: ... |* | |* | | L: <br-stmts>* | | jmp 1b* | |* -----------+-----------------------+------------------* | |* false (0) | ... | ...* | JMP L | NOP* | <br-stmts> | 1: ...* | L: ... |* | |* | | L: <br-stmts>* | | jmp 1b* | |* -----------+-----------------------+------------------** The initial value is encoded in the LSB of static_key::entries,* type: 0 = false, 1 = true.** The branch type is encoded in the LSB of jump_entry::key,* branch: 0 = unlikely, 1 = likely.** This gives the following logic table:** enabled type branch instuction* -----------------------------+-----------* 0 0 0 | NOP* 0 0 1 | JMP* 0 1 0 | NOP* 0 1 1 | JMP** 1 0 0 | JMP* 1 0 1 | NOP* 1 1 0 | JMP* 1 1 1 | NOP** Which gives the following functions:** dynamic: instruction = enabled ^ branch (动态时取决于enable和branch的值)* static: instruction = type ^ branch (静态时取决于type和branch的值)* 上面这个规则很重要* See jump_label_type() / jump_label_init_type().*/
在Jump Label机制中,type和branch是两个关键参数,用于控制动态分支的行为和优化方向。
branch是一个布尔值(true/false 或 1/0),表示当前分支的默认状态(是否启用),它直接影响生成的指令类型(如nop或jmp),以及static_key的地址偏移计算。
决定指令初始值：
branch=true,初始指令为jmp(分支启用)。
branch=false,初始指令为nop(分支禁用)。
影响static_key的地址偏移:static_key结构体中通过偏移量区分不同的分支状态。
#define static_branch_likely(key) \
arch_static_branch(&(key)->enabled, true)
#define static_branch_unlikely(key) \
arch_static_branch(&(key)->enabled, false)
type是一个枚举值,表示static_key的类型,用于区分不同用途的键,它影响内核对键的初始化和运行时处理。
type和branch协作
初始化阶段
branch决定初始指令:若branch=true,初始指令为 jmp,对应 JUMP_LABEL_JMP。
type 决定处理方式:若type=JUMP_LABEL_TYPE_BRANCH,内核在初始化时会跳过该键,直到显式启用。
运行时修改
启用分支:调用 static_key_enable()将type设为JUMP_LABEL_JMP,并修改指令为jmp。
禁用分支:调用 static_key_disable()将type设为 JUMP_LABEL_NOP,并修改指令为nop。
地址偏移计算
branch 影响 key 的偏移:&((char *)key)[branch] 根据branch的值选择key的地址偏移,确保内核通过static_key的地址快速定位状态。
branch 决定指令的初始状态，type 决定键的生命周期和用途，两者共同实现灵活的动态分支管理。
static key的类型必须为struct static_key_true/struct static_key_false,否则会产生错误。
#define static_branch_likely(x) \
({/*__builtin_types_compatible_p GNU扩展,用来判断两个类型是否相同,sizeof(int) sizeof(char*)结果一样但是类型不同*/ \bool branch; \if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \branch = !arch_static_branch(&(x)->key, true); \else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \branch = !arch_static_branch_jump(&(x)->key, true); \else \branch = ____wrong_branch_error(); \branch; \
})
注意,static_branch_likely内部调用的arch_static_branch或者arch_static_branch_jump传递的branch都为true;#define static_branch_unlikely(x) \
({ \bool branch; \if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \branch = arch_static_branch_jump(&(x)->key, false); \else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \branch = arch_static_branch(&(x)->key, false); \else \branch = ____wrong_branch_error(); \branch; \
})
注意,static_branch_unlikely内部调用的arch_static_branch或者arch_static_branch_jump传递的branch都为false;#define static_key_enabled(x) \
({ \if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \!__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\!__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \____wrong_branch_error(); \static_key_count((struct static_key *)x) > 0; \
})int static_key_count(struct static_key *key)
{/** -1 means the first static_key_slow_inc() is in progress.* static_key_enabled() must return true, so return 1 here.*/int n = atomic_read(&key->enabled);return n >= 0 ? n : 1;
}
3.2、arch_static_branch/arch_static_branch_jump 分析
arch_static_branch/arch_static_branch_jump是和cpu体系架构相关的代码,使用asm goto内嵌汇编实现运行时的准确分支功能；
you can reference labels using the actual C label name enclosed in brackets.
For example, to reference a label named carry, you can use ‘%l[carry]’.
The label must still be listed in the GotoLabels section when using this approach.
#define JUMP_LABEL_NOP_SIZE 4
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{asm_volatile_goto("1:\n\t"/*运行时被动态修改的指令或修改为nop,或修改为跳转到 l_yes 标号的指令*//*此时为nop空操作指令*/ WASM(nop) "\n\t"/*在 __jump_table的section内,构建 jump_entry 变量：
(struct jump_entry){
.code = 上面标号1的地址,即运行是被动态修改指令的地址;
.target = 标号 l_yes 地址;
.key = key 地址,用最低1位标记branch类型: false(0), true(1),(char *)key将key地址转换为char*可以进行字节偏移&((char *)key)[branch])通过key &~1可以恢复static_key的地址,同时key地址最低bit位为branch的值(likely为true,unlikely为false);因为static key的地址是4字节对齐的,所以可以使用上面的方法,非常巧妙的方法。
}
*/".pushsection __jump_table, \"aw\"\n\t"".word 1b, %l[l_yes], %c0\n\t"".popsection\n\t": : "i" (&((char *)key)[branch]) : : l_yes);return false;
l_yes:return true;
}static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{asm_volatile_goto("1:\n\t"/*运行时被动态修改的指令或修改为nop,或修改为跳转到 l_yes 标号的指令*//*此时为跳转指令 b l_yes*/WASM(b) " %l[l_yes]\n\t"/*
在__jump_table的section内,构建 jump_entry 变量：
(struct jump_entry){
.code = 上面标号1的地址,即运行是被动态修改指令的地址;
.target = 标号 l_yes 地址;
.key = key 地址,用最低1位标记branch类型: false(0), true(1),(char *)key将key地址转换为char*可以进行字节偏移&((char *)key)[branch])通过key &~1可以恢复static_key的地址,同时key地址最低bit位为branch的值(likely为true,unlikely为false);因为static key的地址是4字节对齐的,所以可以使用上面的方法,非常巧妙的方法。
}
*/".pushsection __jump_table, \"aw\"\n\t" /*切换到__jump_table段*/".word 1b, %l[l_yes], %c0\n\t" /*往段中写入jump_entry 数据*/ ".popsection\n\t": : "i" (&((char *)key)[branch]) : : l_yes); /*切回.text段*/return false;
l_yes:return true;
}上面代码中的".word 1b, %l[l_yes], %c0\n\t"
.word 定义字空间,会依次将1b,%l[l_yes],%c0 数据放到从当前位置开始的连续地址空间上,即__jump_table段中。
跳转表与代码修补：__jump_table 段记录所有可动态修改的跳转点。
内核运行时(如调用 static_branch_enable())遍历此表，修改代码指令。
在内联汇编中,%c0是GCC的操作数修饰符,用于将输入操作数强制作为立即数(Immediate Value)嵌入到汇编指令中。
上面的Jump Label的代码中用于将static_key的地址偏移直接写入跳转表条目。
语法含义
%c0：
%0:表示第一个输入操作数(索引从 0 开始)。
c修饰符,强制将操作数视为立即数(即直接嵌入指令中的常量值)。
约束条件 "i":在输入操作数约束中,"i"表示操作数必须是整数类型的编译时常量(如 #define 定义的常量或地址偏移)。
注意:
当arch_static_branch/arch_static_branch_jump()函数被编译时,内嵌汇编中1:标签处的内容为nop指令或者b指令在内核的代码段中,内核加载时,其被加载到内存中的某个地址上。
而pushsection和popsection指令在编译时已经将定义的struct jump_entry数据(".word 1b, %l[l_yes], %c0\n\t")存入__jump_table段中,所以在后面要介绍的
jump_label_init/jump_label_update()函数将标签1:(jump_entry->code)地址中的指令修改为nop或者b指令,从而达到运行时动态启用或禁用特定代码路径的功能。
当arch_static_branch/arch_static_branch_jump()被调用时,会根据key的状态,决定是否跳转到l_yes标签。
如果跳转,返回true,否则返回false。和if判断条件相比,不用每次调用都去进行一次条件判断,提高执行效率。
这里的关键在于内联汇编如何与跳转表配合,让内核在运行时通过修改代码(如将nop替换为jmp)来改变执行路径。
static __always_inline bool static_key_false(struct static_key *key)
{return arch_static_branch(key, false);
}static __always_inline bool static_key_true(struct static_key *key)
{return !arch_static_branch(key, true);
} 对于函数对 static_key_*() 前的 __always_inline 修饰符，还得额外说明一下， __always_inline 是告诉编译器，函数代码必须就地展开。
在当前的场景下，不加__always_inline 修饰行不行？答案是不行。因为函数 arch_static_branch() 里面使用了标号 1 和 标号 l_yes 的地址，
为了保证每个 struct jump_entry 记录的标号地址唯一，必须要就地展开函数代码。
3.3、jump_label_init 分析
经过前面的分析可知,所有的struct jump_entry都放在名为__jump_table section内,在链接阶段,内核链接脚本vmlinux.lds片段
...
. = ALIGN(8); __start___jump_table = .; *(__jump_table) __stop___jump_table = .; . = ALIGN(8);
...
将所有的 __jump_table输入section,放置到__start___jump_table到 __stop___jump_table区间内,且保证每个 struct jump_entry 的地址都位于4字节边界;
ALIGN(8) 保证了第一个struct jump_entry的地址位于8字节边界，同时每个struct jump_entry 变量为 4 * 3 = 12 字节。
地址4字节对齐，意味着地址的低2位总是为0，正是利用这一点，struct static_key::type用低位的2位来存储相关struct jump_entry(即static_key::entries指向的struct jump_entry)的类型。
arch/arm/kernel/jump_label.c
start_kernel->jump_label_init
void __init jump_label_init(void)
{struct jump_entry *iter_start = __start___jump_table;struct jump_entry *iter_stop = __stop___jump_table;struct static_key *key = NULL;struct jump_entry *iter;/** Since we are initializing the static_key.enabled field with* with the 'raw' int values (to avoid pulling in atomic.h) in* jump_label.h, let's make sure that is safe. There are only two* cases to check since we initialize to 0 or 1.*/BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);if (static_key_initialized)return;cpus_read_lock();jump_label_lock();/*通过jump_entry key值来排序*/jump_label_sort_entries(iter_start, iter_stop);/*遍历所有的jump_entry*/for (iter = iter_start; iter < iter_stop; iter++) {struct static_key *iterk;/* rewrite NOPs */if (jump_label_type(iter) == JUMP_LABEL_NOP) /*nop类型,将jump_entry->code位置替换为nop指令*/arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);/*获取jump_entry对应的static_key*/iterk = jump_entry_key(iter);if (iterk == key)continue;key = iterk;/*设置static_key的entries和type值,将static_key和jump_entry建立联系*/static_key_set_entries(key, iter);}/*static_key初始化完成*/static_key_initialized = true;jump_label_unlock();cpus_read_unlock();
}/*获取jump_entry对应的static_key地址*/
static inline struct static_key *jump_entry_key(struct jump_entry *entry)
{/*在arch_static_branch/arch_static_branch_jump中设置的key值为struct static_key地址偏移一个地址(即(&((char *)key)[branch])，这里获取struct jump_entry 对应的struct static_key结构。*/return (struct static_key *)((unsigned long)entry->key & ~1UL);
}static bool jump_entry_branch(struct jump_entry *entry)
{/*通过地址判断brach为0或为1,key的值设置为(&((char *)key)[branch]。*/return (unsigned long)entry->key & 1UL;
}static enum jump_label_type jump_label_type(struct jump_entry *entry)
{/*获取jump_entry对应的static_key地址*/struct static_key *key = jump_entry_key(entry);/*static_key的enabled是否>0,即是否使能当前static key*/bool enabled = static_key_enabled(key);/*通过地址判断brach为0或为1*/bool branch = jump_entry_branch(entry);/* See the comment in linux/jump_label.h *//*计算出当前放置的是not指令还是jump指令*/return enabled ^ branch;
}/**** A 'struct static_key' uses a union such that it either points directly* to a table of 'struct jump_entry' or to a linked list of modules which in* turn point to 'struct jump_entry' tables.** The two lower bits of the pointer are used to keep track of which pointer* type is in use and to store the initial branch direction, we use an access* function which preserves these bits.*/
static void static_key_set_entries(struct static_key *key,struct jump_entry *entries)
{unsigned long type;WARN_ON_ONCE((unsigned long)entries & JUMP_TYPE_MASK);/*在对entries赋值之前将key->type值记录,记录其初始值,这里使用了地址对齐的小技巧来存放JUMP_TYPE_FALSE/JUMP_TYPE_TRUE*/type = key->type & JUMP_TYPE_MASK;/*将static_key和jump_entry建立联系*/key->entries = entries;/*将type值写回*/key->type |= type;
}void arch_jump_label_transform_static(struct jump_entry *entry,enum jump_label_type type)
{__arch_jump_label_transform(entry, type, true);
}static void __arch_jump_label_transform(struct jump_entry *entry,enum jump_label_type type,bool is_static)
{void *addr = (void *)entry->code;unsigned int insn;if (type == JUMP_LABEL_JMP) /*产生跳转指令*/insn = arm_gen_branch(entry->code, entry->target);elseinsn = arm_gen_nop(); /*产生nop指令*//*修改addr处的指令为insn*/if (is_static)__patch_text_early(addr, insn);elsepatch_text(addr, insn);
}static inline unsigned long arm_gen_nop(void)
{
#ifdef CONFIG_THUMB2_KERNELreturn 0xf3af8000; /* nop.w */
#elsereturn 0xe1a00000; /* mov r0, r0 */
#endif
}static inline unsigned long arm_gen_branch(unsigned long pc, unsigned long addr)
{return __arm_gen_branch(pc, addr, false);
}static inline unsigned long arm_gen_branch_link(unsigned long pc, unsigned long addr)
{return __arm_gen_branch(pc, addr, true);
}unsigned long __arm_gen_branch(unsigned long pc, unsigned long addr, bool link)
{if (IS_ENABLED(CONFIG_THUMB2_KERNEL))return __arm_gen_branch_thumb2(pc, addr, link);elsereturn __arm_gen_branch_arm(pc, addr, link);
}static unsigned long __arm_gen_branch_arm(unsigned long pc, unsigned long addr, bool link)
{unsigned long opcode = 0xea000000;long offset;if (link) /*link寄存器*/opcode |= 1 << 24;offset = (long)addr - (long)(pc + 8);if (unlikely(offset < -33554432 || offset > 33554428)) {WARN_ON_ONCE(1);return 0;}offset = (offset >> 2) & 0x00ffffff;return opcode | offset;
}/*静态修改内存地址对应的指令*/
static inline void __patch_text_early(void *addr, unsigned int insn)
{__patch_text_real(addr, insn, false);
}/*动态修改内存地址对应的指令*/
static inline void __patch_text(void *addr, unsigned int insn)
{__patch_text_real(addr, insn, true);
}void __kprobes __patch_text_real(void *addr, unsigned int insn, bool remap)
{bool thumb2 = IS_ENABLED(CONFIG_THUMB2_KERNEL);unsigned int uintaddr = (uintptr_t) addr;bool twopage = false;unsigned long flags;void *waddr = addr;int size;if (remap)waddr = patch_map(addr, FIX_TEXT_POKE0, &flags);else__acquire(&patch_lock);if (thumb2 && __opcode_is_thumb16(insn)) {*(u16 *)waddr = __opcode_to_mem_thumb16(insn);size = sizeof(u16);} else if (thumb2 && (uintaddr & 2)) {u16 first = __opcode_thumb32_first(insn);u16 second = __opcode_thumb32_second(insn);u16 *addrh0 = waddr;u16 *addrh1 = waddr + 2;twopage = (uintaddr & ~PAGE_MASK) == PAGE_SIZE - 2;if (twopage && remap)addrh1 = patch_map(addr + 2, FIX_TEXT_POKE1, NULL);*addrh0 = __opcode_to_mem_thumb16(first);*addrh1 = __opcode_to_mem_thumb16(second);if (twopage && addrh1 != addr + 2) {flush_kernel_vmap_range(addrh1, 2);patch_unmap(FIX_TEXT_POKE1, NULL);}size = sizeof(u32);} else {if (thumb2)insn = __opcode_to_mem_thumb32(insn);elseinsn = __opcode_to_mem_arm(insn);*(u32 *)waddr = insn;size = sizeof(u32);}if (waddr != addr) {flush_kernel_vmap_range(waddr, twopage ? size / 2 : size);patch_unmap(FIX_TEXT_POKE0, &flags);} else__release(&patch_lock);/*冲刷指令cache,避免cache不一致*/flush_icache_range((uintptr_t)(addr),(uintptr_t)(addr) + size);
}实时动态更新jump_label_update
/*更新对应的static_key*/
static void jump_label_update(struct static_key *key)
{struct jump_entry *stop = __stop___jump_table;struct jump_entry *entry;
#ifdef CONFIG_MODULESstruct module *mod;if (static_key_linked(key)) {__jump_label_mod_update(key);return;}preempt_disable();mod = __module_address((unsigned long)key);if (mod)stop = mod->jump_entries + mod->num_jump_entries;preempt_enable();
#endif/*static_key对应的jump_entry*/entry = static_key_entries(key);/* if there are no users, entry can be NULL */if (entry)__jump_label_update(key, entry, stop);
}static void __jump_label_update(struct static_key *key,struct jump_entry *entry,struct jump_entry *stop)
{for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {/** entry->code set to 0 invalidates module init text sections* kernel_text_address() verifies we are not in core kernel* init code, see jump_label_invalidate_module_init().*/if (entry->code && kernel_text_address(entry->code)) /*是否为kernel代码段地址*/arch_jump_label_transform(entry, jump_label_type(entry));}
}3.4、动态使能/失能static key
/*
* Advanced usage; refcount, branch is enabled when: count != 0
*/
#define static_branch_inc(x) static_key_slow_inc(&(x)->key)
#define static_branch_dec(x) static_key_slow_dec(&(x)->key)
#define static_branch_inc_cpuslocked(x) static_key_slow_inc_cpuslocked(&(x)->key)
#define static_branch_dec_cpuslocked(x) static_key_slow_dec_cpuslocked(&(x)->key)
/*
* Normal usage; boolean enable/disable.
*/
#define static_branch_enable(x) static_key_enable(&(x)->key)
#define static_branch_disable(x) static_key_disable(&(x)->key)
#define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key)
#define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key)
static_key_enable流程分析如下:
void static_key_enable(struct static_key *key)
{cpus_read_lock();static_key_enable_cpuslocked(key);cpus_read_unlock();
}void static_key_enable_cpuslocked(struct static_key *key)
{STATIC_KEY_CHECK_USE();/*获取key的enable值,>0则warn,理论上其值应为0*/if (atomic_read(&key->enabled) > 0) {WARN_ON_ONCE(atomic_read(&key->enabled) != 1);return;}jump_label_lock();if (atomic_read(&key->enabled) == 0) {/*将enabled设置为-1,update过程中其值为-1*/atomic_set(&key->enabled, -1);/*更新jump_label*/jump_label_update(key);/** See static_key_slow_inc().*//*设置为1*/atomic_set_release(&key->enabled, 1);}jump_label_unlock();
}/*更新对应的static_key*/
static void jump_label_update(struct static_key *key)
{struct jump_entry *stop = __stop___jump_table;struct jump_entry *entry;
#ifdef CONFIG_MODULESstruct module *mod;if (static_key_linked(key)) {__jump_label_mod_update(key);return;}preempt_disable();mod = __module_address((unsigned long)key);if (mod)stop = mod->jump_entries + mod->num_jump_entries;preempt_enable();
#endif/*static_key对应的jump_entry*/entry = static_key_entries(key);/* if there are no users, entry can be NULL */if (entry)__jump_label_update(key, entry, stop);
}static void __jump_label_update(struct static_key *key,struct jump_entry *entry,struct jump_entry *stop)
{for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {/** entry->code set to 0 invalidates module init text sections* kernel_text_address() verifies we are not in core kernel* init code, see jump_label_invalidate_module_init().*/if (entry->code && kernel_text_address(entry->code)) /*是否为kernel代码段地址*/arch_jump_label_transform(entry, jump_label_type(entry)); /*由于enabled的值发生变化jump_label_type值也会变换,根据type类型决定是nop还是b跳转指令*/}
}static_key_disable实现
void static_key_disable(struct static_key *key)
{cpus_read_lock();static_key_disable_cpuslocked(key);cpus_read_unlock();
}void static_key_disable_cpuslocked(struct static_key *key)
{STATIC_KEY_CHECK_USE();/*获取key的enable值,!=1则warn,理论上其值应为1*/if (atomic_read(&key->enabled) != 1) {WARN_ON_ONCE(atomic_read(&key->enabled) != 0);return;}jump_label_lock();if (atomic_cmpxchg(&key->enabled, 1, 0)) /*enabled值为1时,原子地修改为0,返回旧值1,否则不做任何操作*/jump_label_update(key);jump_label_unlock();
}static_key_slow_inc实现
void static_key_slow_inc(struct static_key *key)
{cpus_read_lock();static_key_slow_inc_cpuslocked(key);cpus_read_unlock();
}void static_key_slow_inc_cpuslocked(struct static_key *key)
{int v, v1;STATIC_KEY_CHECK_USE();/** Careful if we get concurrent static_key_slow_inc() calls;* later calls must wait for the first one to _finish_ the* jump_label_update() process. At the same time, however,* the jump_label_update() call below wants to see* static_key_enabled(&key) for jumps to be updated properly.** So give a special meaning to negative key->enabled: it sends* static_key_slow_inc() down the slow path, and it is non-zero* so it counts as "enabled" in jump_label_update(). Note that* atomic_inc_unless_negative() checks >= 0, so roll our own.*/for (v = atomic_read(&key->enabled); v > 0; v = v1) { /*enabled>0的情况下只做+1处理*/v1 = atomic_cmpxchg(&key->enabled, v, v + 1); /*原子加1,v1记录旧值*/if (likely(v1 == v))return;}jump_label_lock();if (atomic_read(&key->enabled) == 0) { /*enabled由0->1，需要update*/atomic_set(&key->enabled, -1);jump_label_update(key);/** Ensure that if the above cmpxchg loop observes our positive* value, it must also observe all the text changes.*/atomic_set_release(&key->enabled, 1);} else {atomic_inc(&key->enabled);}jump_label_unlock();
}void static_key_slow_dec(struct static_key *key)
{STATIC_KEY_CHECK_USE();__static_key_slow_dec(key, 0, NULL);
}static void __static_key_slow_dec(struct static_key *key,unsigned long rate_limit,struct delayed_work *work)
{cpus_read_lock();__static_key_slow_dec_cpuslocked(key, rate_limit, work);cpus_read_unlock();
}static void __static_key_slow_dec_cpuslocked(struct static_key *key,unsigned long rate_limit,struct delayed_work *work)
{/** The negative count check is valid even when a negative* key->enabled is in use by static_key_slow_inc(); a* __static_key_slow_dec() before the first static_key_slow_inc()* returns is unbalanced, because all other static_key_slow_inc()* instances block while the update is in progress.*//*enabled原子地-1后值为0,持有jump_label_mutex返回true,否则不持有jump_label_mutex，返回false*/if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {WARN(atomic_read(&key->enabled) < 0,"jump label: negative count!\n");return;}if (rate_limit) {atomic_inc(&key->enabled);schedule_delayed_work(work, rate_limit);} else {jump_label_update(key);}jump_label_unlock();
}/*** atomic_add_unless - add unless the number is already a given value* @v: pointer of type atomic_t* @a: the amount to add to v...* @u: ...unless v is equal to u.** Atomically adds @a to @v, so long as @v was not already @u.* Returns non-zero if @v was not @u, and zero otherwise.*/
static inline int atomic_add_unless(atomic_t *v, int a, int u)
{return __atomic_add_unless(v, a, u) != u;
}/*** atomic_dec_and_mutex_lock - return holding mutex if we dec to 0* @cnt: the atomic which we are to dec* @lock: the mutex to return holding if we dec to 0** return true and hold lock if we dec to 0, return false otherwise*/
int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
{/* dec if we can't possibly hit 0 *//*除非cnt为1,否则cnt值原子地-1,即cnt不能为0*/if (atomic_add_unless(cnt, -1, 1)) /*cnt不为1,则原子地-1,返回非0值*/return 0;/*下面处理cnt为1的情况*//* we might hit 0, so take the lock */mutex_lock(lock);/*cnt原子地-1,测试cnt值，如果cnt为0返回true,否则返回false*/if (!atomic_dec_and_test(cnt)) {/*cnt原子地-1后其值非0,则释放mutex*//* when we actually did the dec, we didn't hit 0 */mutex_unlock(lock);return 0;}/* we hit 0, and we hold the lock */return 1;
}