Index: sys/arch/amd64/conf/GENERIC.MP
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/arch/amd64/conf/GENERIC.MP,v
diff -u -p -u -p -r1.16 GENERIC.MP
--- sys/arch/amd64/conf/GENERIC.MP	9 Feb 2021 14:06:19 -0000	1.16
+++ sys/arch/amd64/conf/GENERIC.MP	15 Dec 2023 21:03:57 -0000
@@ -4,6 +4,6 @@ include "arch/amd64/conf/GENERIC"
 
 option	MULTIPROCESSOR
 #option	MP_LOCKDEBUG
-#option	WITNESS
+option	WITNESS
 
 cpu*		at mainbus?
Index: sys/arch/arm64/conf/GENERIC.MP
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/arch/arm64/conf/GENERIC.MP,v
diff -u -p -u -p -r1.5 GENERIC.MP
--- sys/arch/arm64/conf/GENERIC.MP	1 Jul 2018 21:05:07 -0000	1.5
+++ sys/arch/arm64/conf/GENERIC.MP	15 Dec 2023 21:03:57 -0000
@@ -4,6 +4,6 @@ include "arch/arm64/conf/GENERIC"
 
 option	MULTIPROCESSOR
 #option	MP_LOCKDEBUG
-#option	WITNESS
+option	WITNESS
 
 cpu*		at mainbus?
Index: sys/arch/i386/conf/GENERIC.MP
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/arch/i386/conf/GENERIC.MP,v
diff -u -p -u -p -r1.11 GENERIC.MP
--- sys/arch/i386/conf/GENERIC.MP	3 Jun 2018 05:18:33 -0000	1.11
+++ sys/arch/i386/conf/GENERIC.MP	15 Dec 2023 21:03:57 -0000
@@ -7,6 +7,6 @@ include "arch/i386/conf/GENERIC"
 
 option		MULTIPROCESSOR	# Multiple processor support
 #option		MP_LOCKDEBUG
-#option		WITNESS
+option		WITNESS
 
 cpu*		at mainbus?
Index: sys/arch/macppc/conf/GENERIC.MP
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/arch/macppc/conf/GENERIC.MP,v
diff -u -p -u -p -r1.2 GENERIC.MP
--- sys/arch/macppc/conf/GENERIC.MP	15 Apr 2020 08:09:33 -0000	1.2
+++ sys/arch/macppc/conf/GENERIC.MP	15 Dec 2023 21:03:57 -0000
@@ -4,6 +4,6 @@ include "arch/macppc/conf/GENERIC"
 
 option	MULTIPROCESSOR
 #option	MP_LOCKDEBUG
-#option	WITNESS
+option	WITNESS
 
 cpu*	at mainbus?
Index: sys/arch/powerpc64/conf/GENERIC.MP
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/arch/powerpc64/conf/GENERIC.MP,v
diff -u -p -u -p -r1.1 GENERIC.MP
--- sys/arch/powerpc64/conf/GENERIC.MP	21 Jul 2020 21:38:31 -0000	1.1
+++ sys/arch/powerpc64/conf/GENERIC.MP	15 Dec 2023 21:03:57 -0000
@@ -4,6 +4,6 @@ include "arch/powerpc64/conf/GENERIC"
 
 option	MULTIPROCESSOR
 #option	MP_LOCKDEBUG
-#option	WITNESS
+option	WITNESS
 
 cpu*		at mainbus?
Index: sys/arch/riscv64/conf/GENERIC.MP
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/arch/riscv64/conf/GENERIC.MP,v
diff -u -p -u -p -r1.1 GENERIC.MP
--- sys/arch/riscv64/conf/GENERIC.MP	29 Jun 2021 21:27:52 -0000	1.1
+++ sys/arch/riscv64/conf/GENERIC.MP	15 Dec 2023 21:03:57 -0000
@@ -4,6 +4,6 @@ include "arch/riscv64/conf/GENERIC"
 
 option	MULTIPROCESSOR
 #option	MP_LOCKDEBUG
-#option	WITNESS
+option	WITNESS
 
 cpu*		at mainbus?
Index: sys/kern/uipc_socket.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/kern/uipc_socket.c,v
diff -u -p -u -p -r1.309 uipc_socket.c
--- sys/kern/uipc_socket.c	8 Aug 2023 22:07:25 -0000	1.309
+++ sys/kern/uipc_socket.c	15 Dec 2023 21:03:51 -0000
@@ -832,8 +832,10 @@ bad:
 		*mp = NULL;
 
 	solock_shared(so);
+	pru_lock(so);
 restart:
 	if ((error = sblock(so, &so->so_rcv, SBLOCKWAIT(flags))) != 0) {
+		pru_unlock(so);
 		sounlock_shared(so);
 		return (error);
 	}
@@ -900,11 +902,13 @@ restart:
 		SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1");
 		SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1");
 		sbunlock(so, &so->so_rcv);
+		pru_unlock(so);
 		error = sbwait(so, &so->so_rcv);
 		if (error) {
 			sounlock_shared(so);
 			return (error);
 		}
+		pru_lock(so);
 		goto restart;
 	}
 dontblock:
@@ -971,11 +975,13 @@ dontblock:
 			sbsync(&so->so_rcv, nextrecord);
 			if (controlp) {
 				if (pr->pr_domain->dom_externalize) {
+					pru_unlock(so);
 					sounlock_shared(so);
 					error =
 					    (*pr->pr_domain->dom_externalize)
 					    (cm, controllen, flags);
 					solock_shared(so);
+					pru_lock(so);
 				}
 				*controlp = cm;
 			} else {
@@ -1049,9 +1055,11 @@ dontblock:
 			SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
 			SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
 			resid = uio->uio_resid;
+			pru_unlock(so);
 			sounlock_shared(so);
 			uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio);
 			solock_shared(so);
+			pru_lock(so);
 			if (uio_error)
 				uio->uio_resid = resid - len;
 		} else
@@ -1136,6 +1144,7 @@ dontblock:
 			error = sbwait(so, &so->so_rcv);
 			if (error) {
 				sbunlock(so, &so->so_rcv);
+				pru_unlock(so);
 				sounlock_shared(so);
 				return (0);
 			}
@@ -1182,6 +1191,7 @@ dontblock:
 		*flagsp |= flags;
 release:
 	sbunlock(so, &so->so_rcv);
+	pru_unlock(so);
 	sounlock_shared(so);
 	return (error);
 }
Index: sys/kern/uipc_socket2.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/kern/uipc_socket2.c,v
diff -u -p -u -p -r1.138 uipc_socket2.c
--- sys/kern/uipc_socket2.c	30 Oct 2023 13:27:53 -0000	1.138
+++ sys/kern/uipc_socket2.c	15 Dec 2023 21:03:51 -0000
@@ -368,7 +368,7 @@ solock_shared(struct socket *so)
 	case PF_INET6:
 		if (so->so_proto->pr_usrreqs->pru_lock != NULL) {
 			NET_LOCK_SHARED();
-			pru_lock(so);
+			rw_enter_write(&so->so_lock);
 		} else
 			NET_LOCK();
 		break;
@@ -427,7 +427,7 @@ sounlock_shared(struct socket *so)
 	case PF_INET:
 	case PF_INET6:
 		if (so->so_proto->pr_usrreqs->pru_unlock != NULL) {
-			pru_unlock(so);
+			rw_exit_write(&so->so_lock);
 			NET_UNLOCK_SHARED();
 		} else
 			NET_UNLOCK();
@@ -463,12 +463,12 @@ sosleep_nsec(struct socket *so, void *id
 	case PF_INET6:
 		if (so->so_proto->pr_usrreqs->pru_unlock != NULL &&
 		    rw_status(&netlock) == RW_READ) {
-			pru_unlock(so);
+			rw_exit_write(&so->so_lock);
 		}
 		ret = rwsleep_nsec(ident, &netlock, prio, wmesg, nsecs);
 		if (so->so_proto->pr_usrreqs->pru_lock != NULL &&
 		    rw_status(&netlock) == RW_READ) {
-			pru_lock(so);
+			rw_enter_write(&so->so_lock);
 		}
 		break;
 	default:
Index: sys/kern/uipc_syscalls.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/kern/uipc_syscalls.c,v
diff -u -p -u -p -r1.214 uipc_syscalls.c
--- sys/kern/uipc_syscalls.c	23 Sep 2023 09:17:21 -0000	1.214
+++ sys/kern/uipc_syscalls.c	15 Dec 2023 21:03:51 -0000
@@ -185,9 +185,9 @@ sys_bind(struct proc *p, void *v, regist
 	if (KTRPOINT(p, KTR_STRUCT))
 		ktrsockaddr(p, mtod(nam, caddr_t), SCARG(uap, namelen));
 #endif
-	solock(so);
+	solock_shared(so);
 	error = sobind(so, nam, p);
-	sounlock(so);
+	sounlock_shared(so);
 	m_freem(nam);
 out:
 	FRELE(fp, p);
Index: sys/netinet/in_pcb.h
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/in_pcb.h,v
diff -u -p -u -p -r1.144 in_pcb.h
--- sys/netinet/in_pcb.h	15 Dec 2023 00:24:56 -0000	1.144
+++ sys/netinet/in_pcb.h	15 Dec 2023 21:03:51 -0000
@@ -84,6 +84,38 @@
  *	p	inpcb_mtx		pcb mutex
  */
 
+/*
+ * The pcb table mutex guarantees that all inpcb are consistent and
+ * that bind(2) and connect(2) create unique combinations of
+ * laddr/faddr/lport/fport/rtalbleid.  This mutex is used to protect
+ * both address consistency and inpcb lookup during protocol input.
+ * All writes to inp_[lf]addr take table mutex.  A per socket lock is
+ * needed, so that socket layer input have a consistent view at these
+ * values.
+ *
+ * In soconnect() and sosend() pcb mutex cannot be used.  They eventually
+ * can call IP output which takes pf lock which is a sleeping lock.
+ * Also connect(2) does a route lookup for source selection.  There
+ * route resolve happens, which creates a route, which sends a route
+ * message, which needs route lock, which is a rw-lock.
+ *
+ * On the other hand a mutex should be used in protocol input.  It
+ * does not make sense to do a process switch per packet.  Better spin
+ * until the packet can be processed.
+ *
+ * So there are three locks.  Table mutex is for writing inp_[lf]addr/port
+ * and lookup, socket rw-lock to separate sockets in system calls, and
+ * pcb mutex to protect socket receive buffer.  Changing inp_[lf]addr/port
+ * takes both per socket rw-lock and global table mutex.  Protocol
+ * input only reads inp_[lf]addr/port during lookup and is safe.  System
+ * call only reads when holding socket rw-lock and is safe.  The socket
+ * layer needs pcb mutex only in soreceive().
+ *
+ * Function pru_lock() grabs the pcb mutex and its existence indicates
+ * that a protocol is MP safe.  Otherwise the exclusive net lock is
+ * used.
+ */
+
 struct pf_state_key;
 
 union inpaddru {
Index: sys/sys/protosw.h
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/sys/protosw.h,v
diff -u -p -u -p -r1.62 protosw.h
--- sys/sys/protosw.h	18 May 2023 09:59:44 -0000	1.62
+++ sys/sys/protosw.h	15 Dec 2023 21:03:51 -0000
@@ -284,13 +284,15 @@ pru_detach(struct socket *so)
 static inline void
 pru_lock(struct socket *so)
 {
-	(*so->so_proto->pr_usrreqs->pru_lock)(so);
+	if (so->so_proto->pr_usrreqs->pru_lock)
+		(*so->so_proto->pr_usrreqs->pru_lock)(so);
 }
 
 static inline void
 pru_unlock(struct socket *so)
 {
-	(*so->so_proto->pr_usrreqs->pru_unlock)(so);
+	if (so->so_proto->pr_usrreqs->pru_unlock)
+		(*so->so_proto->pr_usrreqs->pru_unlock)(so);
 }
 
 static inline int