vmcore分析锁问题实例（x86-64）
问题描述：系统出现panic，dmesg有如下打印：
[122061.197311] task:irq/181-ice-enp state:D stack:0 pid:3134 ppid:2 flags:0x00004000
[122061.197315] Call Trace:
[122061.197317] <TASK>
[122061.197318] __schedule+0x34e/0xb00
[122061.197325] schedule_rtlock+0x1f/0x40
[122061.197328] rtlock_slowlock_locked+0x232/0xd40
[122061.197332] rt_read_lock+0x54/0x130
[122061.197333] ep_poll_callback+0x35/0x2a0
[122061.197337] ? ktime_get+0x39/0xa0
[122061.197340] __wake_up_common+0x7d/0x190
[122061.197343] __wake_up_common_lock+0x7c/0xc0
[122061.197345] sock_def_readable+0x42/0xc0
[122061.197349] tcp_child_process+0x199/0x1f0
[122061.197354] tcp_v4_rcv+0xaf5/0xe80
[122061.197356] ? raw_local_deliver+0xc7/0x230
[122061.197359] ip_protocol_deliver_rcu+0x32/0x160
[122061.197361] ip_local_deliver_finish+0x77/0xa0
[122061.197363] ip_sublist_rcv_finish+0x80/0x90
[122061.197364] ip_sublist_rcv+0x1a4/0x240
[122061.197365] ? __pfx_ip_rcv_finish+0x10/0x10
[122061.197367] ip_list_rcv+0x136/0x160
[122061.197368] __netif_receive_skb_list_core+0x2b1/0x2e0
[122061.197371] netif_receive_skb_list_internal+0x1d5/0x310
[122061.197373] napi_complete_done+0x73/0x1b0
[122061.197377] ice_napi_poll+0xa1f/0xd80 [ice]
[122061.197444] __napi_poll+0x29/0x1b0
[122061.197446] net_rx_action+0x29f/0x370
[122061.197447] ? plist_del+0x63/0xc0
[122061.197450] handle_softirqs.constprop.0+0xb0/0x250
[122061.197453] ? __pfx_irq_forced_thread_fn+0x10/0x10
[122061.197456] __local_bh_enable_ip+0x6f/0xa0
[122061.197457] irq_forced_thread_fn+0x77/0x90
[122061.197460] irq_thread+0xed/0x1a0
[122061.197463] ? __pfx_irq_thread_dtor+0x10/0x10
[122061.197466] ? __pfx_irq_thread+0x10/0x10
[122061.197468] kthread+0xdd/0x110
[122061.197470] ? __pfx_kthread+0x10/0x10
[122061.197471] ret_from_fork+0x31/0x50
[122061.197474] ? __pfx_kthread+0x10/0x10
[122061.197476] ret_from_fork_asm+0x1b/0x30
[122061.197479] </TASK>
进程3134在尝试获取lock，而且处于state:D，中断无法执行，导致系统出现异常。
内核代码在rtlock_slowlock_locked中尝试获取lock，代码如下：
1802 /**
1803 * rtlock_slowlock_locked - Slow path lock acquisition for RT locks
1804 * @lock: The underlying RT mutex
1805 */
1806 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)
1807 {
1808 struct rt_mutex_waiter waiter;
1809 struct task_struct *owner;
1810
1811 lockdep_assert_held(&lock->wait_lock);
1812
1813 if (try_to_take_rt_mutex(lock, current, NULL))
1814 return;
1815
1816 rt_mutex_init_rtlock_waiter(&waiter);
1817
1818 /* Save current state and set state to TASK_RTLOCK_WAIT */
1819 current_save_and_set_rtlock_wait_state();
1820
1821 trace_contention_begin(lock, LCB_F_RT);
1822
1823 task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);
1824
1825 for (;;) {
1826 /* Try to acquire the lock again */
1827 if (try_to_take_rt_mutex(lock, current, &waiter))
1828 break;
1829
1830 if (&waiter == rt_mutex_top_waiter(lock))
1831 owner = rt_mutex_owner(lock);
1832 else
1833 owner = NULL;
1834 raw_spin_unlock_irq(&lock->wait_lock);
1835
1836 if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))
1837 schedule_rtlock();
1838
1839 raw_spin_lock_irq(&lock->wait_lock);
1840 set_current_state(TASK_RTLOCK_WAIT);
1841 }
1842
1843 /* Restore the task state */
1844 current_restore_rtlock_saved_state();
1845
1846 /*
1847 * try_to_take_rt_mutex() sets the waiter bit unconditionally.
1848 * We might have to fix that up:
1849 */
1850 fixup_rt_mutex_waiters(lock, true);
1851 debug_rt_mutex_free_waiter(&waiter);
1852
1853 trace_contention_end(lock, 0);
1854 }
lock结构体如下：
struct rt_mutex_base {raw_spinlock_t wait_lock;struct rb_root_cached waiters;struct task_struct *owner;
};
解决思路：只要找到lock，就可以通过lock找到owner
使用crash打开vmcore分析：
crash> bt -f 3134
PID: 3134 TASK: ff27411db0af97c0 CPU: 0 COMMAND: "irq/181-ice-enp"#0 [ff468632c619f748] __schedule at ffffffff99f3c3ceff468632c619f750: 0000100000000000 ff27415b3e5f23c0 ff468632c619f760: ffffffff00000004 ffffffff9a2100a0 ff468632c619f770: 0000000000000000 0000000000000002 ff468632c619f780: da58a4eb0f1c7100 ff27411db0af97c0 ff468632c619f790: ff27411db0af97c0 ff27411f3bd0df00 ff468632c619f7a0: ff27411db0afa098 ff27411db0af97c0 ff468632c619f7b0: ff27411f89f8da98 ffffffff99f3d04f #1 [ff468632c619f7b8] schedule_rtlock at ffffffff99f3d04fff468632c619f7c0: ff468632c619f800 ffffffff99f43042 #2 [ff468632c619f7c8] rtlock_slowlock_locked at ffffffff99f43042ff468632c619f7d0: 0000000000000000 ff27411f3bd0e7d8 ff468632c619f7e0: ff27411f3bd0e7d8 0000000000000000 ff468632c619f7f0: ff27411db0af9ba8 0000000000000000 ff468632c619f800: ff468632cc40f3f0 0000000000000000 ff468632c619f810: 0000000000000000 ffffffff0000001d ff468632c619f820: 0000000000000000 0000000000000001 ff468632c619f830: 0000000000000000 0000000000000000 ff468632c619f840: ff27411d0000001d 0000000000000000 ff468632c619f850: ff27411db0af97c0 ff27411f89f8da98 ff468632c619f860: 27183d1600001000 0000000000000000 ff468632c619f870: da58a4eb0f1c7100 00000000000000c3 ff468632c619f880: ff27411f89f8da90 ff27411f89f8da98 ff468632c619f890: ff27411fbcd3d8d0 ff27411f89f8da90 ff468632c619f8a0: 00000000000000c3 ffffffff99f43c74
…………
需要找到lock的寄存器，进而找到在何时被压栈到何处
rt_mutex_base中wait_lock是第一个成员，也就是说函数当中许多对 lock->wait_lock 的调用可以直接用lock的值，对函数rtlock_slowlock_locked进行反汇编：
crash> dis -lr ffffffff99f43042
kernel/locking/rtmutex.c: 1807
0xffffffff99f42e10 <rtlock_slowlock_locked>: nopl 0x0(%rax,%rax,1) [FTRACE NOP]
0xffffffff99f42e15 <rtlock_slowlock_locked+5>: push %r15
kernel/locking/rtmutex.c: 1813
0xffffffff99f42e17 <rtlock_slowlock_locked+7>: xor %edx,%edx
kernel/locking/rtmutex.c: 1807
0xffffffff99f42e19 <rtlock_slowlock_locked+9>: push %r14
0xffffffff99f42e1b <rtlock_slowlock_locked+11>: push %r13
0xffffffff99f42e1d <rtlock_slowlock_locked+13>: push %r12
0xffffffff99f42e1f <rtlock_slowlock_locked+15>: push %rbp
0xffffffff99f42e20 <rtlock_slowlock_locked+16>: mov %rdi,%rbp
0xffffffff99f42e23 <rtlock_slowlock_locked+19>: push %rbx
arch/x86/include/asm/current.h: 41
0xffffffff99f42e24 <rtlock_slowlock_locked+20>: mov %gs:0x31b40,%r12
kernel/locking/rtmutex.c: 1813
0xffffffff99f42e2d <rtlock_slowlock_locked+29>: mov %r12,%rsi
kernel/locking/rtmutex.c: 1807
0xffffffff99f42e30 <rtlock_slowlock_locked+32>: sub $0xa8,%rsp
0xffffffff99f42e37 <rtlock_slowlock_locked+39>: mov %gs:0x28,%rax
0xffffffff99f42e40 <rtlock_slowlock_locked+48>: mov %rax,0xa0(%rsp)
0xffffffff99f42e48 <rtlock_slowlock_locked+56>: xor %eax,%eax
arch/x86/include/asm/current.h: 41
0xffffffff99f42e4a <rtlock_slowlock_locked+58>: call 0xffffffff99f42430 <try_to_take_rt_mutex>
kernel/locking/rtmutex.c: 1813
0xffffffff99f42e4f <rtlock_slowlock_locked+63>: mov %eax,0x4(%rsp)
0xffffffff99f42e53 <rtlock_slowlock_locked+67>: test %eax,%eax
0xffffffff99f42e55 <rtlock_slowlock_locked+69>: je 0xffffffff99f42e84 <rtlock_slowlock_locked+116>
kernel/locking/rtmutex.c: 1854
0xffffffff99f42e57 <rtlock_slowlock_locked+71>: mov 0xa0(%rsp),%rax
0xffffffff99f42e5f <rtlock_slowlock_locked+79>: sub %gs:0x28,%rax
0xffffffff99f42e68 <rtlock_slowlock_locked+88>: jne 0xffffffff99f43b44 <rtlock_slowlock_locked+3380>
0xffffffff99f42e6e <rtlock_slowlock_locked+94>: add $0xa8,%rsp
0xffffffff99f42e75 <rtlock_slowlock_locked+101>: pop %rbx
0xffffffff99f42e76 <rtlock_slowlock_locked+102>: pop %rbp
0xffffffff99f42e77 <rtlock_slowlock_locked+103>: pop %r12
0xffffffff99f42e79 <rtlock_slowlock_locked+105>: pop %r13
0xffffffff99f42e7b <rtlock_slowlock_locked+107>: pop %r14
0xffffffff99f42e7d <rtlock_slowlock_locked+109>: pop %r15
0xffffffff99f42e7f <rtlock_slowlock_locked+111>: ret
0xffffffff99f42e80 <rtlock_slowlock_locked+112>: int3
0xffffffff99f42e81 <rtlock_slowlock_locked+113>: int3
0xffffffff99f42e82 <rtlock_slowlock_locked+114>: int3
0xffffffff99f42e83 <rtlock_slowlock_locked+115>: int3
kernel/locking/rtmutex_common.h: 214
0xffffffff99f42e84 <rtlock_slowlock_locked+116>: lea 0x8d8(%r12),%r13
0xffffffff99f42e8c <rtlock_slowlock_locked+124>: lea 0x30(%rsp),%rbx
kernel/locking/rtmutex_common.h: 217
0xffffffff99f42e91 <rtlock_slowlock_locked+129>: movq $0x0,0x80(%rsp)
kernel/locking/rtmutex_common.h: 214
0xffffffff99f42e9d <rtlock_slowlock_locked+141>: lea 0x58(%rsp),%rax
kernel/locking/rtmutex.c: 1819
0xffffffff99f42ea2 <rtlock_slowlock_locked+146>: mov %r13,%rdi
kernel/locking/rtmutex_common.h: 215
0xffffffff99f42ea5 <rtlock_slowlock_locked+149>: mov %rbx,0x30(%rsp)
kernel/locking/rtmutex_common.h: 214
0xffffffff99f42eaa <rtlock_slowlock_locked+154>: mov %rax,0x58(%rsp)
kernel/locking/rtmutex_common.h: 223
0xffffffff99f42eaf <rtlock_slowlock_locked+159>: movl $0x1000,0x90(%rsp)
arch/x86/include/asm/current.h: 41
0xffffffff99f42eba <rtlock_slowlock_locked+170>: call 0xffffffff99f45c60 <_raw_spin_lock>
0xffffffff99f42ebf <rtlock_slowlock_locked+175>: mov 0x18(%r12),%eax
0xffffffff99f42ec4 <rtlock_slowlock_locked+180>: mov %r13,%rdi
0xffffffff99f42ec7 <rtlock_slowlock_locked+183>: movl $0x1000,0x18(%r12)
0xffffffff99f42ed0 <rtlock_slowlock_locked+192>: mov %eax,0x1c(%r12)
0xffffffff99f42ed5 <rtlock_slowlock_locked+197>: call 0xffffffff99f45d60 <_raw_spin_unlock>
arch/x86/include/asm/jump_label.h: 27
0xffffffff99f42eda <rtlock_slowlock_locked+202>: nopl 0x0(%rax,%rax,1)
arch/x86/include/asm/current.h: 41
0xffffffff99f42edf <rtlock_slowlock_locked+207>: mov 0x18(%rbp),%r13
0xffffffff99f42ee3 <rtlock_slowlock_locked+211>: mov %gs:0x31b40,%r15
kernel/locking/rtmutex_common.h: 161
0xffffffff99f42eec <rtlock_slowlock_locked+220>: and $0xfffffffffffffffe,%r13
0xffffffff99f42ef0 <rtlock_slowlock_locked+224>: mov %r13,%r12
kernel/locking/rtmutex.c: 1224
0xffffffff99f42ef3 <rtlock_slowlock_locked+227>: cmp %r13,%r15
0xffffffff99f42ef6 <rtlock_slowlock_locked+230>: je 0xffffffff99f4302a <rtlock_slowlock_locked+538>
kernel/locking/rtmutex.c: 1227
0xffffffff99f42efc <rtlock_slowlock_locked+236>: lea 0x8d8(%r15),%r14
0xffffffff99f42f03 <rtlock_slowlock_locked+243>: mov %r14,%rdi
0xffffffff99f42f06 <rtlock_slowlock_locked+246>: call 0xffffffff99f45c60 <_raw_spin_lock>
kernel/locking/rtmutex.c: 1228
0xffffffff99f42f0b <rtlock_slowlock_locked+251>: mov 0x6c(%r15),%ecx
kernel/locking/rtmutex.c: 350
0xffffffff99f42f0f <rtlock_slowlock_locked+255>: mov $0x78,%eax
kernel/locking/rtmutex.c: 365
0xffffffff99f42f14 <rtlock_slowlock_locked+260>: mov 0x1f8(%r15),%r9
kernel/locking/rtmutex.c: 1228
0xffffffff99f42f1b <rtlock_slowlock_locked+267>: mov %r15,0x80(%rsp)
kernel/locking/rtmutex.c: 1229
0xffffffff99f42f23 <rtlock_slowlock_locked+275>: cmp $0x64,%ecx
0xffffffff99f42f26 <rtlock_slowlock_locked+278>: mov %rbp,0x88(%rsp)
include/linux/sched/rt.h: 11
0xffffffff99f42f2e <rtlock_slowlock_locked+286>: cmovge %eax,%ecx
kernel/locking/rtmutex_common.h: 112
0xffffffff99f42f31 <rtlock_slowlock_locked+289>: mov 0x8(%rbp),%rax
kernel/locking/rtmutex.c: 365
0xffffffff99f42f35 <rtlock_slowlock_locked+293>: mov %r9,0x50(%rsp)
kernel/locking/rtmutex.c: 379
0xffffffff99f42f3a <rtlock_slowlock_locked+298>: mov %r9,0x78(%rsp)
kernel/locking/rtmutex.c: 364
0xffffffff99f42f3f <rtlock_slowlock_locked+303>: mov %ecx,0x48(%rsp)
kernel/locking/rtmutex.c: 378
0xffffffff99f42f43 <rtlock_slowlock_locked+307>: mov %ecx,0x70(%rsp)
kernel/locking/rtmutex_common.h: 112
0xffffffff99f42f47 <rtlock_slowlock_locked+311>: test %rax,%rax
0xffffffff99f42f4a <rtlock_slowlock_locked+314>: je 0xffffffff99f4321e <rtlock_slowlock_locked+1038>
kernel/locking/rtmutex_common.h: 130
0xffffffff99f42f50 <rtlock_slowlock_locked+320>: mov 0x10(%rbp),%rax
0xffffffff99f42f54 <rtlock_slowlock_locked+324>: mov %rax,0x8(%rsp)
kernel/locking/rtmutex_common.h: 135
0xffffffff99f42f59 <rtlock_slowlock_locked+329>: test %rax,%rax
0xffffffff99f42f5c <rtlock_slowlock_locked+332>: je 0xffffffff99f42f68 <rtlock_slowlock_locked+344>
kernel/locking/rtmutex_common.h: 137
0xffffffff99f42f5e <rtlock_slowlock_locked+334>: cmp 0x58(%rax),%rbp
0xffffffff99f42f62 <rtlock_slowlock_locked+338>: jne 0xffffffff99f43527 <rtlock_slowlock_locked+1815>
include/linux/rbtree.h: 168
0xffffffff99f42f68 <rtlock_slowlock_locked+344>: mov 0x8(%rbp),%rax
0xffffffff99f42f6c <rtlock_slowlock_locked+348>: lea 0x8(%rbp),%r10
include/linux/rbtree.h: 172
0xffffffff99f42f70 <rtlock_slowlock_locked+352>: test %rax,%rax
0xffffffff99f42f73 <rtlock_slowlock_locked+355>: je 0xffffffff99f4315c <rtlock_slowlock_locked+844>
include/linux/rbtree.h: 170
0xffffffff99f42f79 <rtlock_slowlock_locked+361>: mov $0x1,%edi
0xffffffff99f42f7e <rtlock_slowlock_locked+366>: jmp 0xffffffff99f42f9a <rtlock_slowlock_locked+394>
include/linux/sched/deadline.h: 15
0xffffffff99f42f80 <rtlock_slowlock_locked+368>: test %ecx,%ecx
0xffffffff99f42f82 <rtlock_slowlock_locked+370>: js 0xffffffff99f430b8 <rtlock_slowlock_locked+680>
include/linux/rbtree.h: 177
0xffffffff99f42f88 <rtlock_slowlock_locked+376>: mov 0x8(%rax),%rdx
0xffffffff99f42f8c <rtlock_slowlock_locked+380>: lea 0x8(%rax),%rsi
include/linux/rbtree.h: 178
0xffffffff99f42f90 <rtlock_slowlock_locked+384>: xor %edi,%edi
include/linux/rbtree.h: 172
0xffffffff99f42f92 <rtlock_slowlock_locked+386>: test %rdx,%rdx
0xffffffff99f42f95 <rtlock_slowlock_locked+389>: je 0xffffffff99f42fab <rtlock_slowlock_locked+411>
0xffffffff99f42f97 <rtlock_slowlock_locked+391>: mov %rdx,%rax
kernel/locking/rtmutex.c: 393
0xffffffff99f42f9a <rtlock_slowlock_locked+394>: cmp 0x18(%rax),%ecx
0xffffffff99f42f9d <rtlock_slowlock_locked+397>: jge 0xffffffff99f42f80 <rtlock_slowlock_locked+368>
include/linux/rbtree.h: 175
0xffffffff99f42f9f <rtlock_slowlock_locked+399>: lea 0x10(%rax),%rsi
include/linux/rbtree.h: 172
0xffffffff99f42fa3 <rtlock_slowlock_locked+403>: mov (%rsi),%rdx
0xffffffff99f42fa6 <rtlock_slowlock_locked+406>: test %rdx,%rdx
0xffffffff99f42fa9 <rtlock_slowlock_locked+409>: jne 0xffffffff99f42f97 <rtlock_slowlock_locked+391>
include/linux/rbtree.h: 62
0xffffffff99f42fab <rtlock_slowlock_locked+411>: mov %rax,0x30(%rsp)
include/linux/rbtree.h: 63
0xffffffff99f42fb0 <rtlock_slowlock_locked+416>: movq $0x0,0x38(%rsp)
0xffffffff99f42fb9 <rtlock_slowlock_locked+425>: movq $0x0,0x40(%rsp)
include/linux/rbtree.h: 65
0xffffffff99f42fc2 <rtlock_slowlock_locked+434>: mov %rbx,(%rsi)
include/linux/rbtree.h: 112
0xffffffff99f42fc5 <rtlock_slowlock_locked+437>: test %dil,%dil
0xffffffff99f42fc8 <rtlock_slowlock_locked+440>: jne 0xffffffff99f4317b <rtlock_slowlock_locked+875>
include/linux/rbtree.h: 114
0xffffffff99f42fce <rtlock_slowlock_locked+446>: mov %rbx,%rdi
0xffffffff99f42fd1 <rtlock_slowlock_locked+449>: mov %r10,%rsi
0xffffffff99f42fd4 <rtlock_slowlock_locked+452>: call 0xffffffff99f20d40 <rb_insert_color>
kernel/locking/rtmutex.c: 1238
0xffffffff99f42fd9 <rtlock_slowlock_locked+457>: mov %rbx,0x900(%r15)
kernel/locking/rtmutex.c: 1240
0xffffffff99f42fe0 <rtlock_slowlock_locked+464>: mov %r14,%rdi
0xffffffff99f42fe3 <rtlock_slowlock_locked+467>: call 0xffffffff99f45d60 <_raw_spin_unlock>
kernel/locking/rtmutex.c: 1257
0xffffffff99f42fe8 <rtlock_slowlock_locked+472>: test %r13,%r13
0xffffffff99f42feb <rtlock_slowlock_locked+475>: je 0xffffffff99f4302a <rtlock_slowlock_locked+538>
kernel/locking/rtmutex.c: 1260
0xffffffff99f42fed <rtlock_slowlock_locked+477>: lea 0x8d8(%r13),%r8
0xffffffff99f42ff4 <rtlock_slowlock_locked+484>: mov %r8,%rdi
0xffffffff99f42ff7 <rtlock_slowlock_locked+487>: mov %r8,0x10(%rsp)
0xffffffff99f42ffc <rtlock_slowlock_locked+492>: call 0xffffffff99f45c60 <_raw_spin_lock>
kernel/locking/rtmutex_common.h: 130
0xffffffff99f43001 <rtlock_slowlock_locked+497>: mov 0x10(%rbp),%rax
kernel/locking/rtmutex_common.h: 135
0xffffffff99f43005 <rtlock_slowlock_locked+501>: mov 0x10(%rsp),%r8
0xffffffff99f4300a <rtlock_slowlock_locked+506>: test %rax,%rax
0xffffffff99f4300d <rtlock_slowlock_locked+509>: je 0xffffffff99f43022 <rtlock_slowlock_locked+530>
kernel/locking/rtmutex_common.h: 137
0xffffffff99f4300f <rtlock_slowlock_locked+511>: cmp 0x58(%rax),%rbp
0xffffffff99f43013 <rtlock_slowlock_locked+515>: jne 0xffffffff99f435fe <rtlock_slowlock_locked+2030>
kernel/locking/rtmutex_common.h: 139
0xffffffff99f43019 <rtlock_slowlock_locked+521>: cmp %rbx,%rax
0xffffffff99f4301c <rtlock_slowlock_locked+524>: je 0xffffffff99f43228 <rtlock_slowlock_locked+1048>
kernel/locking/rtmutex.c: 1275
0xffffffff99f43022 <rtlock_slowlock_locked+530>: mov %r8,%rdi
0xffffffff99f43025 <rtlock_slowlock_locked+533>: call 0xffffffff99f45d60 <_raw_spin_unlock>
kernel/locking/rtmutex.c: 1281
0xffffffff99f4302a <rtlock_slowlock_locked+538>: mov %gs:0x31b40,%r12
arch/x86/include/asm/current.h: 41
0xffffffff99f43033 <rtlock_slowlock_locked+547>: jmp 0xffffffff99f43054 <rtlock_slowlock_locked+580>
kernel/locking/rtmutex.c: 1834
0xffffffff99f43035 <rtlock_slowlock_locked+549>: mov %rbp,%rdi
0xffffffff99f43038 <rtlock_slowlock_locked+552>: call 0xffffffff99f45da0 <_raw_spin_unlock_irq>
kernel/locking/rtmutex.c: 1837
0xffffffff99f4303d <rtlock_slowlock_locked+557>: call 0xffffffff99f3d030 <schedule_rtlock>
kernel/locking/rtmutex.c: 1839
0xffffffff99f43042 <rtlock_slowlock_locked+562>: mov %rbp,%rdi
在x86上面，当参数少于7时， 参数会依次放入寄存器: rdi, rsi, rdx, rcx, r8, r9，
mov %rdi,%rbp
这一行代码已经把参数的值放到rbp寄存器当中了，而且后续可以看到一些类似“mov %r13,%rdi”的操作，说明虽然schedule_rtlock没有参数，但是rdi寄存器已经不再存放lock的值了。
从以下反汇编的内容可以得知，lock值应该就存在rbq当中
0xffffffff99f43035 <rtlock_slowlock_locked+549>: mov %rbp,%rdi
0xffffffff99f43038 <rtlock_slowlock_locked+552>: call 0xffffffff99f45da0 <_raw_spin_unlock_irq>
因为原函数当中是这样进行调用的：
1839 raw_spin_lock_irq(&lock->wait_lock);
lock->wait_lock就是lock的值，在调用_raw_spin_unlock_irq的时候把rbp传给参数寄存器，也就是说rbq当中存的就是lock的值。
接下来对schedule_rtlock进行反汇编：
crash> dis -lr ffffffff99f3d04f
kernel/sched/core.c: 6874
0xffffffff99f3d030 <schedule_rtlock>: nopw (%rax)
kernel/sched/core.c: 6792
0xffffffff99f3d034 <schedule_rtlock+4>: push %rbx
arch/x86/include/asm/current.h: 41
0xffffffff99f3d035 <schedule_rtlock+5>: mov %gs:0x31b40,%rbx
arch/x86/include/asm/preempt.h: 80
0xffffffff99f3d03e <schedule_rtlock+14>: incl %gs:0x660f4b03(%rip) # 0x31b48
kernel/sched/core.c: 6796
0xffffffff99f3d045 <schedule_rtlock+21>: mov $0x2,%edi
0xffffffff99f3d04a <schedule_rtlock+26>: call 0xffffffff99f3c080 <__schedule>
arch/x86/include/asm/preempt.h: 85
0xffffffff99f3d04f <schedule_rtlock+31>: decl %gs:0x660f4af2(%rip) # 0x31b48
并未发现对%rbp的操作，可能是编译器进行了优化，因为schedule_rtlock并未改变寄存器%rbq的值，所以并未进行压栈。
继续对上一层函数__schedule进行反汇编
crash> dis -lr ffffffff99f3c3ce
kernel/sched/core.c: 6598
0xffffffff99f3c080 <__schedule>: push %rbp
0xffffffff99f3c081 <__schedule+1>: mov %rsp,%rbp
0xffffffff99f3c084 <__schedule+4>: push %r15
0xffffffff99f3c086 <__schedule+6>: push %r14
0xffffffff99f3c088 <__schedule+8>: mov %edi,%r14d
0xffffffff99f3c08b <__schedule+11>: push %r13
……
可以看到在最开始的时候就对寄存器%rbp进行压栈，对应的栈地址为ff468632c619f7b0，内容为ff27411f89f8da98
在crash里面查看这个地址的内容：
crash> rt_mutex_base ff27411f89f8da98
struct rt_mutex_base {wait_lock = {raw_lock = {{val = {counter = 0},{locked = 0 '\000',pending = 0 '\000'},{locked_pending = 0,tail = 0}}}},waiters = {rb_root = {rb_node = 0xff468632cc40f3f0},rb_leftmost = 0xff468632c619f800},owner = crash> rt_mutex_base ff27411f89f8da98
}
可以看到owner为0xff27411f3bd0df01，最后的1不是task_struct的地址，是在获得锁的时候或的，实际的task_struct地址是0xff27411f3bd0df00
crash> task_struct.pid 0xff27411f3bd0df00 pid = 23645,
得到锁是由23645进程执有的