Mercurial > unleashed > wips
changeset 20707:777783f32bc0 draft
WIP: kernel: lockdep
| author | Josef 'Jeff' Sipek <jeffpc@josefsipek.net> |
|---|---|
| date | Tue, 27 Jun 2017 22:39:06 +0300 |
| parents | 220e04bde1da |
| children | c611dfebe78a |
| files | arch/x86/kernel/ml/lock_prim.s include/sys/Makefile include/sys/lockdep.h include/sys/mutex.h include/sys/thread.h kernel/os/kmem.c kernel/os/lockdep.c kernel/os/main.c kernel/os/mutex.c kernel/os/vmem.c usr/src/pkg/manifests/system-header.mf usr/src/uts/common/Makefile.files |
| diffstat | 12 files changed, 812 insertions(+), 28 deletions(-) [+] |
line wrap: on
line diff
--- a/arch/x86/kernel/ml/lock_prim.s Tue Jun 27 20:20:35 2017 +0300 +++ b/arch/x86/kernel/ml/lock_prim.s Tue Jun 27 22:39:06 2017 +0300 @@ -23,8 +23,6 @@ * Use is subject to license terms. */ -#pragma ident "%Z%%M% %I% %E% SMI" - #include "assym.h" #include <sys/mutex_impl.h> @@ -42,6 +40,14 @@ * ulock_try() is for a lock in the user address space. */ +#ifdef DEBUG +#define ENTRY_NONDEBUG(x) +#define SET_SIZE_NONDEBUG(x) +#else +#define ENTRY_NONDEBUG(x) ALTENTRY(x) +#define SET_SIZE_NONDEBUG(x) SET_SIZE(x) +#endif + .globl kernelbase #if defined(__amd64) @@ -433,22 +439,22 @@ /* - * mutex_enter() and mutex_exit(). + * do_mutex_enter() and do_mutex_exit(). * * These routines handle the simple cases of mutex_enter() (adaptive * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). * If anything complicated is going on we punt to mutex_vector_enter(). * - * mutex_tryenter() is similar to mutex_enter() but returns zero if + * do_mutex_tryenter() is similar to do_mutex_enter() but returns zero if * the lock cannot be acquired, nonzero on success. * - * If mutex_exit() gets preempted in the window between checking waiters + * If do_mutex_exit() gets preempted in the window between checking waiters * and clearing the lock, we can miss wakeups. Disabling preemption * in the mutex code is prohibitively expensive, so instead we detect * mutex preemption by examining the trapped PC in the interrupt path. - * If we interrupt a thread in mutex_exit() that has not yet cleared + * If we interrupt a thread in do_mutex_exit() that has not yet cleared * the lock, cmnint() resets its PC back to the beginning of - * mutex_exit() so it will check again for waiters when it resumes. + * do_mutex_exit() so it will check again for waiters when it resumes. * * The lockstat code below is activated when the lockstat driver * calls lockstat_hot_patch() to hot-patch the kernel mutex code. @@ -458,7 +464,9 @@ #if defined(__amd64) - ENTRY_NP(mutex_enter) + ENTRY_NP(do_mutex_enter) + ENTRY_NONDEBUG(mutex_enter) + ENTRY_NONDEBUG(mutex_enter_recursive) movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ xorl %eax, %eax /* rax = 0 (unheld adaptive) */ lock @@ -497,7 +505,9 @@ movl $1, %eax /* return success if tryenter */ ret SET_SIZE(lockstat_wrapper) - SET_SIZE(mutex_enter) + SET_SIZE_NONDEBUG(mutex_enter_recursive) + SET_SIZE_NONDEBUG(mutex_enter) + SET_SIZE(do_mutex_enter) /* * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event @@ -521,7 +531,8 @@ SET_SIZE(lockstat_wrapper_arg) - ENTRY(mutex_tryenter) + ENTRY(do_mutex_tryenter) + ENTRY_NONDEBUG(mutex_tryenter) movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ xorl %eax, %eax /* rax = 0 (unheld adaptive) */ lock @@ -543,7 +554,8 @@ movq %rdi, %rsi movl $LS_MUTEX_ENTER_ACQUIRE, %edi jmp lockstat_wrapper - SET_SIZE(mutex_tryenter) + SET_SIZE_NONDEBUG(mutex_tryenter) + SET_SIZE(do_mutex_tryenter) ENTRY(mutex_adaptive_tryenter) movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ @@ -596,7 +608,8 @@ .globl mutex_exit_critical_start - ENTRY(mutex_exit) + ENTRY(do_mutex_exit) + ENTRY_NONDEBUG(mutex_exit) mutex_exit_critical_start: /* If interrupted, restart here */ movq %gs:CPU_THREAD, %rdx cmpq %rdx, (%rdi) @@ -608,7 +621,8 @@ movq %rdi, %rsi movl $LS_MUTEX_EXIT_RELEASE, %edi jmp lockstat_wrapper - SET_SIZE(mutex_exit) + SET_SIZE_NONDEBUG(mutex_exit) + SET_SIZE(do_mutex_exit) .globl mutex_exit_critical_size .type mutex_exit_critical_size, @object @@ -619,7 +633,9 @@ #else - ENTRY_NP(mutex_enter) + ENTRY_NP(do_mutex_enter) + ENTRY_NONDEBUG(mutex_enter) + ENTRY_NONDEBUG(mutex_enter_recursive) movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ movl 4(%esp), %ecx /* ecx = lock ptr */ xorl %eax, %eax /* eax = 0 (unheld adaptive) */ @@ -660,7 +676,9 @@ popl %ebp /* pop off frame */ ret SET_SIZE(lockstat_wrapper) - SET_SIZE(mutex_enter) + SET_SIZE_NONDEBUG(mutex_enter_recursive) + SET_SIZE_NONDEBUG(mutex_enter) + SET_SIZE(do_mutex_enter) ENTRY(lockstat_wrapper_arg) /* expects edx=thread, ecx=lock, */ /* eax=lockstat event, pushed arg */ @@ -689,7 +707,8 @@ SET_SIZE(lockstat_wrapper_arg) - ENTRY(mutex_tryenter) + ENTRY(do_mutex_tryenter) + ENTRY_NONDEBUG(mutex_tryenter) movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ movl 4(%esp), %ecx /* ecx = lock ptr */ xorl %eax, %eax /* eax = 0 (unheld adaptive) */ @@ -711,7 +730,8 @@ #endif /* OPTERON_WORKAROUND_6323525 */ movl $LS_MUTEX_ENTER_ACQUIRE, %eax jmp lockstat_wrapper - SET_SIZE(mutex_tryenter) + SET_SIZE_NONDEBUG(mutex_tryenter) + SET_SIZE(do_mutex_tryenter) ENTRY(mutex_adaptive_tryenter) movl %gs:CPU_THREAD, %edx /* edx = thread ptr */ @@ -767,7 +787,8 @@ .globl mutex_exit_critical_start - ENTRY(mutex_exit) + ENTRY(do_mutex_exit) + ENTRY_NONDEBUG(mutex_exit) mutex_exit_critical_start: /* If interrupted, restart here */ movl %gs:CPU_THREAD, %edx movl 4(%esp), %ecx @@ -779,7 +800,8 @@ ret movl $LS_MUTEX_EXIT_RELEASE, %eax jmp lockstat_wrapper - SET_SIZE(mutex_exit) + SET_SIZE_NONDEBUG(mutex_exit) + SET_SIZE(do_mutex_exit) .globl mutex_exit_critical_size .type mutex_exit_critical_size, @object
--- a/include/sys/Makefile Tue Jun 27 20:20:35 2017 +0300 +++ b/include/sys/Makefile Tue Jun 27 22:39:06 2017 +0300 @@ -307,6 +307,7 @@ llc1.h \ loadavg.h \ lock.h \ + lockdep.h \ lockfs.h \ lockstat.h \ lofi.h \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/include/sys/lockdep.h Tue Jun 27 22:39:06 2017 +0300 @@ -0,0 +1,28 @@ +#ifndef _LOCKDEP_H +#define _LOCKDEP_H + +#define LOCKDEP_HELD_STACK_SIZE 16 +#define LOCKDEP_HELD_STACKTRACE_SIZE 16 + +struct lock_info; + +struct held_lock { + struct lock_info *hl_info; + + /* recursive locking support */ + kmutex_rec_pol_t hl_policy; + int hl_recursion; + uintptr_t hl_rec_last; + + /* XXX: more context for of the mutex_enter? */ + /* + * really, should be a pc_t, but let's avoid including too many + * headers here + */ + uintptr_t hl_stacktrace[LOCKDEP_HELD_STACKTRACE_SIZE]; + int hl_stacktrace_size; +}; + +extern void lockdep_init(); + +#endif
--- a/include/sys/mutex.h Tue Jun 27 20:20:35 2017 +0300 +++ b/include/sys/mutex.h Tue Jun 27 22:39:06 2017 +0300 @@ -60,6 +60,11 @@ MUTEX_DEFAULT = 6 /* kernel default mutex */ } kmutex_type_t; +typedef enum { + LRP_NONE, /* not actually recursive */ + LRP_ADDR_ASC /* ascending address */ +} kmutex_rec_pol_t; + typedef struct mutex { #ifdef _LP64 void *_opaque[1]; @@ -88,6 +93,7 @@ extern void mutex_init(kmutex_t *, char *, kmutex_type_t, void *); extern void mutex_destroy(kmutex_t *); extern void mutex_enter(kmutex_t *); +extern void mutex_enter_recursive(kmutex_t *, kmutex_rec_pol_t); extern int mutex_tryenter(kmutex_t *); extern void mutex_exit(kmutex_t *); extern int mutex_owned(const kmutex_t *);
--- a/include/sys/thread.h Tue Jun 27 20:20:35 2017 +0300 +++ b/include/sys/thread.h Tue Jun 27 22:39:06 2017 +0300 @@ -34,6 +34,7 @@ #include <sys/time.h> #include <sys/signal.h> #include <sys/kcpc.h> +#include <sys/lockdep.h> #if defined(__GNUC__) && defined(_ASM_INLINES) && defined(_KERNEL) #include <asm/thread.h> #endif @@ -341,6 +342,13 @@ kmutex_t t_ctx_lock; /* protects t_ctx in removectx() */ struct waitq *t_waitq; /* wait queue */ kmutex_t t_wait_mutex; /* used in CV wait functions */ + + /* + * lockdep related fields + */ + int t_nheldlocks; /* number of held locks */ + struct held_lock t_heldlocks[LOCKDEP_HELD_STACK_SIZE]; + /* held locks array */ } kthread_t; /*
--- a/kernel/os/kmem.c Tue Jun 27 20:20:35 2017 +0300 +++ b/kernel/os/kmem.c Tue Jun 27 22:39:06 2017 +0300 @@ -1396,7 +1396,7 @@ NULL, NULL, VM_SLEEP); bzero(lhp, lhsize); - mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&lhp->lh_lock, "kmem log header", MUTEX_DEFAULT, NULL); lhp->lh_nchunks = nchunks; lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE); lhp->lh_base = vmem_alloc(kmem_log_arena, @@ -1407,7 +1407,8 @@ for (i = 0; i < max_ncpus; i++) { kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; - mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&clhp->clh_lock, "kmem log header cpu", MUTEX_DEFAULT, + NULL); clhp->clh_chunk = i; } @@ -3909,7 +3910,7 @@ /* * Initialize the rest of the slab layer. */ - mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&cp->cache_lock, "kmem cache", MUTEX_DEFAULT, NULL); avl_create(&cp->cache_partial_slabs, kmem_partial_slab_cmp, sizeof (kmem_slab_t), offsetof(kmem_slab_t, slab_link)); @@ -3930,7 +3931,7 @@ /* * Initialize the depot. */ - mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&cp->cache_depot_lock, "kmem depot", MUTEX_DEFAULT, NULL); for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++) continue; @@ -3942,7 +3943,7 @@ */ for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; - mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&ccp->cc_lock, "kmem cpu", MUTEX_DEFAULT, NULL); ccp->cc_flags = cp->cache_flags; ccp->cc_rounds = -1; ccp->cc_prounds = -1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kernel/os/lockdep.c Tue Jun 27 22:39:06 2017 +0300 @@ -0,0 +1,706 @@ +#include <sys/thread.h> +#include <sys/types.h> +#include <sys/sysmacros.h> +#include <sys/sunddi.h> +#include <sys/avl.h> +#include <sys/list.h> +#include <sys/lockdep.h> + +#define LOCKDEP_DEP_LIST_SIZE 16 +#define LOCKDEP_NUM_INFOS (16 * 1024) /* max number of live mutexes */ +#define LOCKDEP_NUM_CLASSES 256 /* max number of classes */ +#define LOCKDEP_CLASS_NAME_LEN 32 /* longest possible class name */ + +#define KEEP_GOING + +/* we never saw a mutex_init */ +#define UNKNOWN_LOCK_CLASS "<unknown>" +/* we got a NULL class name in mutex_init */ +#define NULL_LOCK_CLASS "<null>" + +// XXX: define mutex_enter_recursive in a header + +#define LAST_ACQUIRED_LOCK() (&curthread->t_heldlocks[curthread->t_nheldlocks - 1]) + +struct lock_class { + union { + avl_node_t tree; + list_node_t list; + } lc_node; + + char lc_name[LOCKDEP_CLASS_NAME_LEN]; + int lc_ndeps; +#if 0 + struct lock_class *lc_deps[LOCKDEP_DEP_LIST_SIZE]; +#endif +}; + +struct lock_info { + union { + avl_node_t tree; + list_node_t list; + } li_node; + + struct lock_class *li_class; + kmutex_t *li_lock; + void *li_ra; + boolean_t li_implicit_init; +}; + +static int lockdep_enabled; +static avl_tree_t lockinfo; +static list_t lockinfo_freelist; +static struct lock_info lockinfo_slab[LOCKDEP_NUM_INFOS]; +static avl_tree_t lockclass; +static list_t lockclass_freelist; +static struct lock_class lockclass_slab[LOCKDEP_NUM_CLASSES]; +static kmutex_t lockdep_lock; +static char lockdep_msgbuf[1024]; +static char *lockdep_msgbuf_ptr; + +#define MSG(...) lockdep_msgbuf_ptr += snprintf(lockdep_msgbuf_ptr, \ + sizeof(lockdep_msgbuf) - (lockdep_msgbuf_ptr - lockdep_msgbuf), \ + __VA_ARGS__) +#define ENTER_DEBUGGER() do { \ + debug_enter(lockdep_msgbuf); \ + lockdep_msgbuf_ptr = lockdep_msgbuf; \ + } while (0) + +static const char +__recursion_policy_char(kmutex_rec_pol_t policy) +{ + switch (policy) { + case LRP_NONE: + return '.'; + case LRP_ADDR_ASC: + return '>'; + } + + return '?'; +} + +/* + * There are various errors we may encounter. Each of the following + * functions deals with a different error. + */ +static void +__error_print_lock_info(struct lock_info *li, kmutex_rec_pol_t policy) +{ + MSG(" %p (%s) <%c%c>\n", li->li_lock, li->li_class->lc_name, + li->li_implicit_init ? 'I' : '.', + __recursion_policy_char(policy)); +} + +static void +__error_print_held_lock(struct held_lock *hl) +{ + __error_print_lock_info(hl->hl_info, hl->hl_policy); +} + +static void +error_mutex_reinitialization(kmutex_t *mutex, char *name) +{ + MSG("mutex_init(%p, \"%s\", ...) called on an already initialized " + "mutex\n", mutex, name); + ENTER_DEBUGGER(); +} + +static void +error_mutex_enter_unknown(kmutex_t *mutex) +{ + MSG("mutex_enter(%p) called on an unknown (unintialized?) lock\n", + mutex); + MSG("Assuming adaptive mutex of class \"%s\"\n", + UNKNOWN_LOCK_CLASS); + ENTER_DEBUGGER(); +} + +static void +error_mutex_msg(const char *op, const char *msg, struct lock_info *li, + kmutex_rec_pol_t policy, struct held_lock *hl) +{ + MSG("%s\n", msg); + MSG("thread %p is trying to %s lock:\n", curthread, op); + __error_print_lock_info(li, policy); + MSG("but thread is already holding lock:\n"); + __error_print_held_lock(hl); + ENTER_DEBUGGER(); +} + +static void +error_mutex_enter(const char *msg, struct lock_info *li, + kmutex_rec_pol_t policy, struct held_lock *hl) +{ + error_mutex_msg("acquire", msg, li, policy, hl); +} + +static void +error_mutex_exit_policy_violation(struct lock_info *li, struct held_lock *hl) +{ + error_mutex_msg("release", "recursive locking policy violation detected", + li, LRP_NONE, hl); +} + +static void +error_mutex_exit_unknown(kmutex_t *mutex) +{ + MSG("thread %p is trying to release an unknown mutex %p\n", + curthread, mutex); + ENTER_DEBUGGER(); +} + +static void +error_mutex_exit_notowner(struct lock_info *li) +{ + MSG("thread %p is trying to release a mutex it doesn't own:\n", + curthread); + __error_print_lock_info(li, LRP_NONE); + ENTER_DEBUGGER(); +} + +static void +error_mutex_destroy_unknown(kmutex_t *mutex) +{ + MSG("thread %p is trying to destroy an unknown mutex %p\n", + curthread, mutex); + ENTER_DEBUGGER(); +} + +static void +error_held_stack_overflow(struct lock_info *li, kmutex_rec_pol_t policy) +{ + MSG("thread %p tried to hold onto too many locks (%d max)\n", curthread, + LOCKDEP_HELD_STACK_SIZE); + __error_print_lock_info(li, policy); + ENTER_DEBUGGER(); +} + +/* + * The core part of lock dep. + */ +static int +lockinfo_cmp(const void *a, const void *b) +{ + const struct lock_info *lia = a; + const struct lock_info *lib = b; + uintptr_t va = (uintptr_t) lia->li_lock; + uintptr_t vb = (uintptr_t) lib->li_lock; + + if (va < vb) + return (-1); + if (va > vb) + return (1); + return (0); +} + +static int +lockclass_cmp(const void *a, const void *b) +{ + const struct lock_class *lca = a; + const struct lock_class *lcb = b; + int ret; + + ret = strncmp(lca->lc_name, lcb->lc_name, LOCKDEP_CLASS_NAME_LEN); + if (ret < 0) + return (-1); + if (ret > 0) + return (1); + return (0); +} + +static struct lock_info * +alloc_lock_info(kmutex_t *mutex, struct lock_class *lc) +{ + struct lock_info *li; + + li = list_remove_head(&lockinfo_freelist); + if (!li) + return (NULL); + + li->li_lock = mutex; + li->li_class = lc; + + avl_add(&lockinfo, li); + + return (li); +} + +static void +free_lock_info(struct lock_info *li) +{ + avl_remove(&lockinfo, li); + + list_insert_head(&lockinfo_freelist, li); +} + +static struct lock_class * +alloc_lock_class(char *name) +{ + struct lock_class *lc; + + lc = list_remove_head(&lockclass_freelist); + if (!lc) + return (NULL); + + if (!name) + name = NULL_LOCK_CLASS; + + (void) strlcpy(lc->lc_name, name, LOCKDEP_CLASS_NAME_LEN); + lc->lc_ndeps = 0; + + avl_add(&lockclass, lc); + + return (lc); +} + +static struct held_lock * +alloc_held_lock(struct lock_info *li, kmutex_rec_pol_t policy) +{ + struct held_lock *hl; + + curthread->t_nheldlocks++; + + if (curthread->t_nheldlocks == LOCKDEP_HELD_STACK_SIZE) { + error_held_stack_overflow(li, policy); + return (NULL); + } + + hl = &curthread->t_heldlocks[curthread->t_nheldlocks - 1]; + + hl->hl_info = li; + hl->hl_policy = policy; + hl->hl_recursion = 1; + hl->hl_stacktrace_size = getpcstack((pc_t*)hl->hl_stacktrace, + LOCKDEP_HELD_STACKTRACE_SIZE); + + return (hl); +} + +static struct lock_info * +lookup_lock_info(kmutex_t *mutex) +{ + struct lock_info key; + struct lock_info *li; + + key.li_lock = mutex; + + return (avl_find(&lockinfo, &key, NULL)); +} + +static struct lock_class * +lookup_lock_class(char *name) +{ + struct lock_class key; + + if (!name) + name = NULL_LOCK_CLASS; + + (void) strlcpy(key.lc_name, name, LOCKDEP_CLASS_NAME_LEN); + + return (avl_find(&lockclass, &key, NULL)); +} + +void +lockdep_init() +{ + int i; + + lockdep_msgbuf_ptr = lockdep_msgbuf; + + avl_create(&lockinfo, lockinfo_cmp, sizeof (struct lock_info), + offsetof(struct lock_info, li_node.tree)); + list_create(&lockinfo_freelist, sizeof (struct lock_info), + offsetof(struct lock_info, li_node.list)); + + avl_create(&lockclass, lockclass_cmp, sizeof (struct lock_class), + offsetof(struct lock_class, lc_node.tree)); + list_create(&lockclass_freelist, sizeof (struct lock_class), + offsetof(struct lock_class, lc_node.list)); + + for (i = 0; i < LOCKDEP_NUM_INFOS; i++) + list_insert_head(&lockinfo_freelist, &lockinfo_slab[i]); + + for (i = 0; i < LOCKDEP_NUM_CLASSES; i++) + list_insert_head(&lockclass_freelist, &lockclass_slab[i]); + + mutex_init(&lockdep_lock, "lockdep state lock", MUTEX_SPIN, NULL); + + /* set up the NULL and unknown lock classes */ + (void) alloc_lock_class(NULL_LOCK_CLASS); + (void) alloc_lock_class(UNKNOWN_LOCK_CLASS); + + lockdep_enabled = 1; +} + +static struct lock_info * +lockdep_mutex_init_unlocked(kmutex_t *mutex, char *name, int type, void *arg, + boolean_t implicit, void *ra) +{ + char *raname[LOCKDEP_CLASS_NAME_LEN]; + struct lock_class *lc; + struct lock_info *li; + + if (!name) { + /* if we didn't get a name, use the return address */ + snprintf(raname, sizeof(raname), "<%p>", ra); + name = raname; + } + + lc = lookup_lock_class(name); + if (!lc) { + lc = alloc_lock_class(name); + if (!lc) + goto err; + } + + /* is this a reinitialization? */ + li = lookup_lock_info(mutex); + if (li) { + error_mutex_reinitialization(mutex, name); + return (NULL); + } + + li = alloc_lock_info(mutex, lc); + if (!li) + goto err; + + li->li_ra = ra; + li->li_implicit_init = implicit; + + return (li); + +err: + MSG("failed to allocate memory\n"); + ENTER_DEBUGGER(); + + return (NULL); +} + +void +lockdep_mutex_init(kmutex_t *mutex, char *name, int type, void *arg, void *ra) +{ + struct lock_info *li; + + if (mutex == &lockdep_lock) + return; + + mutex_enter(&lockdep_lock); + + if (!lockdep_enabled) + goto out; + + li = lockdep_mutex_init_unlocked(mutex, name, type, arg, B_FALSE, ra); + if (li) + goto out; + +#ifndef KEEP_GOING + lockdep_enabled = 0; +#endif + +out: + mutex_exit(&lockdep_lock); +} + +static boolean_t +check_recursive_policy(struct lock_info *li, struct held_lock *hl) +{ + switch (hl->hl_policy) { + case LRP_NONE: + /* no one should be calling us with the none policy */ + panic("%s called with LRP_NONE policy", __func__); + break; + case LRP_ADDR_ASC: { + uintptr_t last, this; + + last = hl->hl_rec_last; + this = (uintptr_t)li->li_lock; + + if (last > this) + return (B_FALSE); + break; + } + } + + return (B_TRUE); +} + +static void +lockdep_mutex_enter_common(kmutex_t *mutex, boolean_t try, boolean_t success, + kmutex_rec_pol_t policy, void *ra) +{ + struct lock_class *lc; + struct lock_info *li; + struct held_lock *hl; + int i; + + if (mutex == &lockdep_lock) + return; + + if (try && !success) + return; + + mutex_enter(&lockdep_lock); + + if (!lockdep_enabled) + goto out; + + li = lookup_lock_info(mutex); + if (!li) { + /* probably an implicit mutex_init() */ + error_mutex_enter_unknown(mutex); + + li = lockdep_mutex_init_unlocked(mutex, UNKNOWN_LOCK_CLASS, + MUTEX_ADAPTIVE, NULL, B_TRUE, ra); + if (!li) + goto brick; + } + + /* check for recursive locking */ + if (curthread->t_nheldlocks) { + struct held_lock *cur; + + /* check for ok recursive locking */ + cur = LAST_ACQUIRED_LOCK(); + if (cur->hl_info->li_class == li->li_class) { + if (cur->hl_policy != policy) { + error_mutex_enter("recursive locking policy " + "mismatch detected", li, policy, cur); + goto brick; + } + + if ((policy != LRP_NONE) && + (cur->hl_info->li_lock != mutex)) { + /* + * someone is trying to lock a different + * lock of the same class as the last + * acquired lock, and is using + * mutex_enter_recursive correctly + */ + if (!check_recursive_policy(li, cur)) { + error_mutex_enter("recursive locking " + "policy violation detected", li, + policy, cur); + goto brick; + } + + /* + * policy was obeyed - update the state of + * the held lock to account for the newly + * held lock + */ + hl->hl_recursion++; + hl->hl_rec_last = (uintptr_t)mutex; + goto out; + } + + /* just let the deadlock detector below deal with it */ + } + + /* check for deadlocks & bad recursive locking */ + for (i = 0; i < curthread->t_nheldlocks; i++) { + cur = &curthread->t_heldlocks[i]; + + if (cur->hl_info->li_class != li->li_class) + continue; + + if (cur->hl_info->li_lock == mutex) + error_mutex_enter("possible deadlock detected", + li, policy, cur); + else + error_mutex_enter("possible recursive locking " + "detected", li, policy, cur); + goto brick; + } + } + + /* no issues, add the lock we're trying to get to the stack */ + hl = alloc_held_lock(li, policy); + if (!hl) + goto err_nomem; + +out: + mutex_exit(&lockdep_lock); + return; + +err_nomem: + MSG("failed to allocate memory\n"); + ENTER_DEBUGGER(); + +brick: +#ifndef KEEP_GOING + lockdep_enabled = 0; +#endif + mutex_exit(&lockdep_lock); +} + +static void +do_lockdep_mutex_exit(struct held_lock *hl) +{ + if (hl->hl_recursion > 1) { + /* recursively held lock */ + hl->hl_recursion--; + } else { + /* non-recursive, or last reference of a recursive */ + if (hl != LAST_ACQUIRED_LOCK()) + memmove(hl, hl + 1, LAST_ACQUIRED_LOCK() - hl); + + curthread->t_nheldlocks--; + } +} + +void +lockdep_mutex_exit(kmutex_t *mutex) +{ + struct held_lock *hl; + struct lock_info *li; + int i; + + if (mutex == &lockdep_lock) + return; + + mutex_enter(&lockdep_lock); + + if (!lockdep_enabled) + goto out; + + li = lookup_lock_info(mutex); + + for (i = 0; li && (i < curthread->t_nheldlocks); i++) { + hl = &curthread->t_heldlocks[i]; + + if (hl->hl_info->li_class != li->li_class) + continue; + + /* + * Found a class match, but we're not out of the woods yet. + * If the held lock info matches this lock exactly, we're + * set. If there is a mismatch, then then we have to check + * if this could be a recursively locked mutex being + * unlocked. + */ + + if ((hl->hl_info->li_lock != mutex) && + (hl->hl_policy == LRP_NONE)) + break; + + if ((hl->hl_policy != LRP_NONE) && + !check_recursive_policy(li, hl)) { + error_mutex_exit_policy_violation(li, hl); + goto brick; + } + + do_lockdep_mutex_exit(hl); + + goto out; + } + + /* + * We never locked this mutex. Either, this is a bogus lock (that + * we've never seen) or a different thread locked it and we're + * trying to release it. + */ + + if (!li) + error_mutex_exit_unknown(mutex); + else + error_mutex_exit_notowner(li); + +brick: +#ifndef KEEP_GOING + lockdep_enabled = 0; +#endif + +out: + mutex_exit(&lockdep_lock); +} + +void +lockdep_mutex_destroy(kmutex_t *mutex) +{ + struct lock_info *li; + + if (mutex == &lockdep_lock) + return; + + mutex_enter(&lockdep_lock); + + if (!lockdep_enabled) + goto out; + + li = lookup_lock_info(mutex); + if (!li) { + error_mutex_destroy_unknown(mutex); +#ifndef KEEP_GOING + lockdep_enabled = 0; +#endif + goto out; + } + + free_lock_info(li); + +out: + mutex_exit(&lockdep_lock); +} + +/* + * Here, depending on whether or not we're on a debug kernel, we intercept + * mutex_lock, etc. + */ +#ifdef DEBUG +extern void do_mutex_init(kmutex_t *, char *, kmutex_type_t, void *); +extern void do_mutex_destroy(kmutex_t *); +extern void do_mutex_enter(kmutex_t *); +extern void do_mutex_exit(kmutex_t *); +extern int do_mutex_tryenter(kmutex_t *); + +void +mutex_init(kmutex_t *mutex, char *name, kmutex_type_t type, void *ibc) +{ + lockdep_mutex_init(mutex, name, type, ibc, caller()); + + do_mutex_init(mutex, name, type, ibc); +} + +void +mutex_destroy(kmutex_t *mutex) +{ + lockdep_mutex_destroy(mutex); + + do_mutex_destroy(mutex); +} + +void +mutex_enter(kmutex_t *mutex) +{ + lockdep_mutex_enter_common(mutex, B_FALSE, B_FALSE, LRP_NONE, caller()); + do_mutex_enter(mutex); +} + +void +mutex_enter_recursive(kmutex_t *mutex, kmutex_rec_pol_t policy) +{ + lockdep_mutex_enter_common(mutex, B_FALSE, B_FALSE, policy, caller()); + do_mutex_enter(mutex); +} + +int +mutex_tryenter(kmutex_t *mutex) +{ + int ret; + + ret = do_mutex_tryenter(mutex); + + lockdep_mutex_enter_common(mutex, B_TRUE, (boolean_t)ret, LRP_NONE, + caller()); + + return (ret); +} + +void +mutex_exit(kmutex_t *mutex) +{ + lockdep_mutex_exit(mutex); + + do_mutex_exit(mutex); +} +#endif
--- a/kernel/os/main.c Tue Jun 27 20:20:35 2017 +0300 +++ b/kernel/os/main.c Tue Jun 27 22:39:06 2017 +0300 @@ -73,6 +73,7 @@ #include <sys/class.h> #include <sys/stack.h> #include <sys/brand.h> +#include <sys/lockdep.h> #include <vm/as.h> #include <vm/seg_kmem.h> @@ -386,12 +387,15 @@ ASSERT(curthread == CPU->cpu_thread); ASSERT_STACK_ALIGNED(); + lockdep_init(); + /* * We take the ualock until we have completed the startup * to prevent kadmin() from disrupting this work. In particular, * we don't want kadmin() to bring the system down while we are * trying to start it up. */ + mutex_init(&ualock, "ualock", MUTEX_DEFAULT, NULL); mutex_enter(&ualock); /*
--- a/kernel/os/mutex.c Tue Jun 27 20:20:35 2017 +0300 +++ b/kernel/os/mutex.c Tue Jun 27 22:39:06 2017 +0300 @@ -336,7 +336,7 @@ void (*mutex_delay)(void) = mutex_delay_default; /* - * mutex_vector_enter() is called from the assembly mutex_enter() routine + * mutex_vector_enter() is called from the assembly do_mutex_enter() routine * if the lock is held or is not of type MUTEX_ADAPTIVE. */ void @@ -562,9 +562,12 @@ * eg adaptive mutexes created as static within the BSS or allocated * by kmem_zalloc(). */ +#ifndef DEBUG +#pragma weak mutex_init = do_mutex_init +#endif /* ARGSUSED */ void -mutex_init(kmutex_t *mp, char *name, kmutex_type_t type, void *ibc) +do_mutex_init(kmutex_t *mp, char *name, kmutex_type_t type, void *ibc) { mutex_impl_t *lp = (mutex_impl_t *)mp; @@ -615,8 +618,11 @@ } } +#ifndef DEBUG +#pragma weak mutex_destroy = do_mutex_destroy +#endif void -mutex_destroy(kmutex_t *mp) +do_mutex_destroy(kmutex_t *mp) { mutex_impl_t *lp = (mutex_impl_t *)mp;
--- a/kernel/os/vmem.c Tue Jun 27 20:20:35 2017 +0300 +++ b/kernel/os/vmem.c Tue Jun 27 22:39:06 2017 +0300 @@ -1474,7 +1474,7 @@ bzero(vmp, sizeof (vmem_t)); (void) snprintf(vmp->vm_name, VMEM_NAMELEN, "%s", name); - mutex_init(&vmp->vm_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&vmp->vm_lock, "vmem arena", MUTEX_DEFAULT, NULL); cv_init(&vmp->vm_cv, NULL, CV_DEFAULT, NULL); vmp->vm_cflags = vmflag; vmflag &= VM_KMFLAGS;
--- a/usr/src/pkg/manifests/system-header.mf Tue Jun 27 20:20:35 2017 +0300 +++ b/usr/src/pkg/manifests/system-header.mf Tue Jun 27 22:39:06 2017 +0300 @@ -1139,6 +1139,7 @@ file path=usr/include/sys/llc1.h file path=usr/include/sys/loadavg.h file path=usr/include/sys/lock.h +file path=usr/include/sys/lockdep.h file path=usr/include/sys/lockfs.h file path=usr/include/sys/lofi.h file path=usr/include/sys/log.h