From Newsgroup: muc.lists.freebsd.stable

--000000000000683f49064fe533fd
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable

Hi Mark,

I have a truncated output of procstat -kk without -a.

PID TID COMM TDNAME KSTACK

2800 105648 nfsd nfsd: master mi_switch _sleep
svc_run nfsrvd_nfsd nfssvc_nfsd sys_nfssvc amd64_syscall
fast_syscall_common

2800 105679 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105682 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105683 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105685 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sleep sbwait sosend_generic_locked sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105687 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105688 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105701 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105702 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105703 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105704 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sleep sbwait sosend_generic_locked sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105705 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105706 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105707 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105708 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105709 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105710 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105711 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105712 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105717 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105725 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105726 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105727 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105728 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105729 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105730 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sleep sbwait sosend_generic_locked sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105731 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105732 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105733 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105734 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105735 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sleep sbwait sosend_generic_locked sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105736 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105737 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sleep sbwait sosend_generic_locked sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105738 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105739 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105740 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105741 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105742 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105743 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105744 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105745 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105746 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105747 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline

2800 105748 nfsd nfsd: service mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline


Best,

-Olgun


On Mon, Apr 20, 2026 at 10:26=E2=80=AFAM Mark Johnston <markj@freebsd.org> = wrote:

On Mon, Apr 20, 2026 at 10:11:21AM -0400, Olgun Adak wrote:

Hello FreeBSD Community,

We=E2=80=99ve run into a consistent nfsd lockup after moving from 14.3-=

RELEASE-p8

to p10. We have verified this across two identical bare-metal systems. Reverting to p8 via bectl immediately restores stability on both system=

s,

so this appears to be a regression introduced in the p9/p10 cycle.

*The symptoms:*

Under NFSv3 load, the nfsd service hangs and becomes unresponsive to al=

l

clients. Looking at procstat -kk, we see a deadlock pattern where threa=

ds

are stuck waiting on soiolock:

_sx_xlock_hard -> soiolock -> sosend_generic -> sosend -> svc_vc_reply

Would you be able to share full "procstat -kka" output from an affected system?

Several threads are blocked in _sx_xlock_hard while others sit in sbwai=

t.

*The environment:*

The systems are bare-metal with 2 x dual-port Mellanox ConnectX-6 100Gb=

E

(mlx5en) cards. We see the issue regardless of MTU (1500 and 9000).

Offloads:

-

TSO: Enabled
-

LRO: The issue persists regardless of LRO state (tested with LRO
disabled and with software-only LRO). Hardware LRO is disabled in al=

l

cases.

Relevant tunables:

kern.ipc.soacceptqueue=3D1000
kern.ipc.somaxconn=3D2000
kern.ipc.maxsockbuf=3D67108864
net.inet.tcp.sendbuf_max=3D67108864
net.inet.tcp.sendspace=3D16777216
net.inet.tcp.sendbuf_inc=3D262144
net.inet.tcp.recvbuf_max=3D67108864
net.inet.tcp.recvspace=3D16777216
vfs.nfsd.srvmaxio=3D1048576

We have kept the p10 Boot Environments intact and can boot back into th=

em

to run any additional debug commands or test patches if someone can hel=

p

point us in the right direction.

Best regards,
-Olgun Adak

--=20



This message is intended only for the use of the individual or entity to=20 which it is addressed, and may contain private and confidential=20
information. If you are not the intended recipient of this message you are=
=20
hereby notified that any review, dissemination, distribution or copying of=
=20
this message is strictly prohibited. If you have received this e-mail in=20 error, please immediately notify the sender by replying to this e-mail and=
=20
delete the message and any attachment(s) from your system. This=20 communication is for information purposes only and should not be regarded=
=20
as an offer to sell or as a solicitation of an offer to buy any financial=
=20
product, an official confirmation of any transaction, or as an official=20 statement of Trexquant Investment LP. All information is subject to change=
=20
without notice.

--000000000000683f49064fe533fd
Content-Type: text/html; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable

<div dir=3D"ltr"><div>Hi Mark,</div><div><br></div><div>I have a truncated= =C2=A0output=C2=A0of procstat -kk without -a.</div><div><br></div><div><fon=
t face=3D"monospace">PID =C2=A0 =C2=A0TID COMM =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0TDNAME =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
=C2=A0KSTACK</font><br><br><font face=3D"monospace">=C2=A02800 105648 nfsd=
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: master =C2=A0=
=C2=A0 =C2=A0 =C2=A0mi_switch _sleep svc_run nfsrvd_nfsd nfssvc_nfsd sys_n= fssvc amd64_syscall fast_syscall_common <br><br>=C2=A02800 105679 nfsd =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 = =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard=
soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendr= eply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_t= rampoline <br><br>=C2=A02800 105682 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
=C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_ca= tch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend s= vc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_= internal svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 1056=
83 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: servic=
e =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_x= lock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common = svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_ex=
it fork_trampoline <br><br>=C2=A02800 105685 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch=
sleepq_catch_signals sleepq_wait_sig _sleep sbwait sosend_generic_locked s= osend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf n= fssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline <= br><br>=C2=A02800 105687 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals=
sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply=
svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal sv= c_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 105688 nfsd =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 = =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard=
soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendr= eply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_t= rampoline <br><br>=C2=A02800 105701 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
=C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_ca= tch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend s= vc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_= internal svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 1057=
02 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: servic=
e =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_x= lock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common = svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_ex=
it fork_trampoline <br><br>=C2=A02800 105703 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch=
sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generi=
c sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_progra=
m svc_run_internal svc_thread_start fork_exit fork_trampoline <br><br>=C2= =A02800 105704 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0= nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wa= it_sig _sleep sbwait sosend_generic_locked sosend_generic sosend svc_vc_rep=
ly svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal = svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 105705 nfsd = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0=
=C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_har=
d soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_send= reply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_= trampoline <br><br>=C2=A02800 105706 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq= _catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosen=
d svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_r= un_internal svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 1= 05707 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: ser= vice =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _s= x_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_comm=
on svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork= _exit fork_trampoline <br><br>=C2=A02800 105708 nfsd =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_swi= tch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_gen= eric sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_pro= gram svc_run_internal svc_thread_start fork_exit fork_trampoline <br><br>= =C2=A02800 105709 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq= _wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_se= ndreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_threa= d_start fork_exit fork_trampoline <br><br>=C2=A02800 105710 nfsd =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 = =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolo=
ck sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mb=
uf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoli=
ne <br><br>=C2=A02800 105711 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
=C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_sig= nals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_r= eply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_interna=
l svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 105712 nfsd=
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=
=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_= hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_s= endreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fo= rk_trampoline <br><br>=C2=A02800 105717 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sle= epq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic so= send svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program sv= c_run_internal svc_thread_start fork_exit fork_trampoline <br><br>=C2=A0280=
0 105725 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: = service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig=
_sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_c= ommon svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start f= ork_exit fork_trampoline <br><br>=C2=A02800 105726 nfsd =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi= _switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend= _generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc= _program svc_run_internal svc_thread_start fork_exit fork_trampoline <br><b= r>=C2=A02800 105727 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sle= epq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc= _sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_th= read_start fork_exit fork_trampoline <br><br>=C2=A02800 105728 nfsd =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0=
=C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiol= ock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_m= buf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampol= ine <br><br>=C2=A02800 105729 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_= signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_v= c_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_inte= rnal svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 105730 n= fsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service = =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sleep = sbwait sosend_generic_locked sosend_generic sosend svc_vc_reply svc_sendrep= ly_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_sta=
rt fork_exit fork_trampoline <br><br>=C2=A02800 105731 nfsd =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0=
mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sos= end_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfs= svc_program svc_run_internal svc_thread_start fork_exit fork_trampoline <br= ><br>=C2=A02800 105732 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals=
sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply=
svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal sv= c_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 105733 nfsd =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 = =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard=
soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendr= eply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_t= rampoline <br><br>=C2=A02800 105734 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
=C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_ca= tch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend s= vc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_= internal svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 1057=
35 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: servic=
e =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _slee=
p sbwait sosend_generic_locked sosend_generic sosend svc_vc_reply svc_sendr= eply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_s= tart fork_exit fork_trampoline <br><br>=C2=A02800 105736 nfsd =C2=A0 =C2=A0=
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=
=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock = sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf = nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline = <br><br>=C2=A02800 105737 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_sign= als sleepq_wait_sig _sleep sbwait sosend_generic_locked sosend_generic sose=
nd svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_= run_internal svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 = 105738 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: se= rvice =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _= sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_com= mon svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start for= k_exit fork_trampoline <br><br>=C2=A02800 105739 nfsd =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_swi= tch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_gen= eric sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_pro= gram svc_run_internal svc_thread_start fork_exit fork_trampoline <br><br>= =C2=A02800 105740 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq= _wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_se= ndreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_threa= d_start fork_exit fork_trampoline <br><br>=C2=A02800 105741 nfsd =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 = =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolo=
ck sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mb=
uf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoli=
ne <br><br>=C2=A02800 105742 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
=C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_sig= nals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_r= eply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_interna=
l svc_thread_start fork_exit fork_trampoline <br><br>=C2=A02800 105743 nfsd=
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=
=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_= hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_s= endreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fo= rk_trampoline <br><br>=C2=A02800 105744 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sle= epq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend_generic so= send svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program sv= c_run_internal svc_thread_start fork_exit fork_trampoline <br><br>=C2=A0280=
0 105745 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: = service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig=
_sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc_sendreply_c= ommon svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start f= ork_exit fork_trampoline <br><br>=C2=A02800 105746 nfsd =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi= _switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiolock sosend= _generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc= _program svc_run_internal svc_thread_start fork_exit fork_trampoline <br><b= r>=C2=A02800 105747 nfsd =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0nfsd: service =C2=A0 =C2=A0 =C2=A0 mi_switch sleepq_catch_signals sle= epq_wait_sig _sx_xlock_hard soiolock sosend_generic sosend svc_vc_reply svc= _sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_th= read_start fork_exit fork_trampoline <br><br>=C2=A02800 105748 nfsd =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0nfsd: service =C2=A0 =C2=A0=
=C2=A0 mi_switch sleepq_catch_signals sleepq_wait_sig _sx_xlock_hard soiol= ock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_m= buf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampol= ine=C2=A0</font></div><div><font face=3D"monospace"><br></font></div><div><= br></div><div><div dir=3D"ltr" class=3D"gmail_signature" data-smartmail=3D"= gmail_signature"><div dir=3D"ltr"><div><div>Best,</div><div dir=3D"ltr"><br= ></div><div dir=3D"ltr">-Olgun</div></div></div></div></div><br></div><br><= div class=3D"gmail_quote gmail_quote_container"><div dir=3D"ltr" class=3D"g= mail_attr">On Mon, Apr 20, 2026 at 10:26=E2=80=AFAM Mark Johnston &lt;<a hr= ef=3D"mailto:markj@freebsd.org">markj@freebsd.org</a>&gt; wrote:<br></div><= blockquote class=3D"gmail_quote" style=3D"margin:0px 0px 0px 0.8ex;border-l= eft:1px solid rgb(204,204,204);padding-left:1ex">On Mon, Apr 20, 2026 at 10= :11:21AM -0400, Olgun Adak wrote:<br>
&gt; Hello FreeBSD Community,<br>
&gt; <br>
&gt; We=E2=80=99ve run into a consistent nfsd lockup after moving from 14.3= -RELEASE-p8<br>
&gt; to p10. We have verified this across two identical bare-metal systems.=

&gt; Reverting to p8 via bectl immediately restores stability on both syste= ms,<br>
&gt; so this appears to be a regression introduced in the p9/p10 cycle.<br> &gt; <br>
&gt; *The symptoms:*<br>
&gt; <br>
&gt; Under NFSv3 load, the nfsd service hangs and becomes unresponsive to a= ll<br>
&gt; clients. Looking at procstat -kk, we see a deadlock pattern where thre= ads<br>
&gt; are stuck waiting on soiolock:<br>
&gt; <br>
&gt; _sx_xlock_hard -&gt; soiolock -&gt; sosend_generic -&gt; sosend -&gt; = svc_vc_reply<br>

Would you be able to share full &quot;procstat -kka&quot; output from an af= fected<br>
system?<br>

&gt; <br>
&gt; Several threads are blocked in _sx_xlock_hard while others sit in sbwa= it.<br>
&gt; <br>
&gt; *The environment:*<br>
&gt; <br>
&gt; The systems are bare-metal with 2 x dual-port Mellanox ConnectX-6 100G= bE<br>
&gt; (mlx5en) cards. We see the issue regardless of MTU (1500 and 9000).<br=

&gt; <br>
&gt; Offloads:<br>
&gt; <br>
&gt;=C2=A0 =C2=A0 -<br>
&gt; <br>
&gt;=C2=A0 =C2=A0 TSO: Enabled<br>
&gt;=C2=A0 =C2=A0 -<br>
&gt; <br>
&gt;=C2=A0 =C2=A0 LRO: The issue persists regardless of LRO state (tested w= ith LRO<br>
&gt;=C2=A0 =C2=A0 disabled and with software-only LRO). Hardware LRO is dis= abled in all cases.<br>
&gt; <br>
&gt; Relevant tunables:<br>
&gt; <br>
&gt; kern.ipc.soacceptqueue=3D1000<br>
&gt; kern.ipc.somaxconn=3D2000<br>
&gt; kern.ipc.maxsockbuf=3D67108864<br>
&gt; net.inet.tcp.sendbuf_max=3D67108864<br>
&gt; net.inet.tcp.sendspace=3D16777216<br>
&gt; net.inet.tcp.sendbuf_inc=3D262144<br>
&gt; net.inet.tcp.recvbuf_max=3D67108864<br>
&gt; net.inet.tcp.recvspace=3D16777216<br>
&gt; vfs.nfsd.srvmaxio=3D1048576<br>
&gt; <br>
&gt; We have kept the p10 Boot Environments intact and can boot back into t= hem<br>
&gt; to run any additional debug commands or test patches if someone can he= lp<br>
&gt; point us in the right direction.<br>
&gt; <br>
&gt; Best regards,<br>
&gt; -Olgun Adak<br>
</blockquote></div>


<div><hr></div><font face=3D"Arial, Helvetica, sans-serif"><span style=3D"f= ont-size:13px"><div><font face=3D"Arial, Helvetica, sans-serif"><span style= =3D"font-size:13px"><br></span></font></div>This message is intended only f=
or the use of the individual or entity to which it is addressed, and may co= ntain private and confidential information. If you are not the intended rec= ipient of this message you are hereby notified that any review, disseminati= on, distribution or copying of this message is strictly prohibited. If you = have received this e-mail in error, please immediately notify the sender by=
replying to this e-mail and delete the message and any attachment(s) from = your system. This communication is for information purposes only and should=
not be regarded as an offer to sell or as a solicitation of an offer to bu=
y any financial product, an official confirmation of any transaction, or as=
an official statement of Trexquant Investment LP. All information is subje=
ct to change without notice.</span></font>
--000000000000683f49064fe533fd--


--
Posted automagically by a mail2news gateway at muc.de e.V.
Please direct questions, flames, donations, etc. to news-admin@muc.de
--- Synchronet 3.21f-Linux NewsLink 1.2

From Newsgroup: muc.lists.freebsd.stable

--00000000000062d99d064fe8cdaa
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable

Hi Mark,

Should we capture any further diagnostics from the system? We're
considering reverting the less-critical system to p10 to reproduce the
issue. Note that the production-critical system triggered the issue
multiple times within hours, whereas the less-critical system remained operational longer, possibly due to its specific NFS workloads.

Best,

-Olgun


On Mon, Apr 20, 2026 at 2:36=E2=80=AFPM Mark Johnston <markj@freebsd.org> w= rote:

On Mon, Apr 20, 2026 at 10:34:22AM -0400, Olgun Adak wrote:

Hi Mark,

I have a truncated output of procstat -kk without -a.

PID TID COMM TDNAME KSTACK
[...]

Unfortunately this doesn't reveal much. All of the NFS threads are
trying to send data, but the socket buffer isn't being drained for some reason. Full procstat output could help shed some light.

Best,

-Olgun

On Mon, Apr 20, 2026 at 10:26=E2=80=AFAM Mark Johnston <markj@freebsd.o=

wrote:

On Mon, Apr 20, 2026 at 10:11:21AM -0400, Olgun Adak wrote:

Hello FreeBSD Community,

We=E2=80=99ve run into a consistent nfsd lockup after moving from

14.3-RELEASE-p8

to p10. We have verified this across two identical bare-metal

systems.

Reverting to p8 via bectl immediately restores stability on both

systems,

so this appears to be a regression introduced in the p9/p10 cycle.

*The symptoms:*

Under NFSv3 load, the nfsd service hangs and becomes unresponsive t=

o

all

clients. Looking at procstat -kk, we see a deadlock pattern where

threads

are stuck waiting on soiolock:

_sx_xlock_hard -> soiolock -> sosend_generic -> sosend ->

svc_vc_reply

Would you be able to share full "procstat -kka" output from an affect=

ed

system?

Several threads are blocked in _sx_xlock_hard while others sit in

sbwait.

*The environment:*

The systems are bare-metal with 2 x dual-port Mellanox ConnectX-6

100GbE

(mlx5en) cards. We see the issue regardless of MTU (1500 and 9000).

Offloads:

-

TSO: Enabled
-

LRO: The issue persists regardless of LRO state (tested with LRO
disabled and with software-only LRO). Hardware LRO is disabled i=

n

all

cases.

Relevant tunables:

kern.ipc.soacceptqueue=3D1000
kern.ipc.somaxconn=3D2000
kern.ipc.maxsockbuf=3D67108864
net.inet.tcp.sendbuf_max=3D67108864
net.inet.tcp.sendspace=3D16777216
net.inet.tcp.sendbuf_inc=3D262144
net.inet.tcp.recvbuf_max=3D67108864
net.inet.tcp.recvspace=3D16777216
vfs.nfsd.srvmaxio=3D1048576

We have kept the p10 Boot Environments intact and can boot back int=

o

them

to run any additional debug commands or test patches if someone can

help

point us in the right direction.

Best regards,
-Olgun Adak

--

This message is intended only for the use of the individual or entity t=

o

which it is addressed, and may contain private and confidential information. If you are not the intended recipient of this message you

are

hereby notified that any review, dissemination, distribution or copying

of

this message is strictly prohibited. If you have received this e-mail i=

n

error, please immediately notify the sender by replying to this e-mail

and

delete the message and any attachment(s) from your system. This communication is for information purposes only and should not be

regarded

as an offer to sell or as a solicitation of an offer to buy any

financial

product, an official confirmation of any transaction, or as an official statement of Trexquant Investment LP. All information is subject to

change

without notice.

--=20



This message is intended only for the use of the individual or entity to=20 which it is addressed, and may contain private and confidential=20
information. If you are not the intended recipient of this message you are=
=20
hereby notified that any review, dissemination, distribution or copying of=
=20
this message is strictly prohibited. If you have received this e-mail in=20 error, please immediately notify the sender by replying to this e-mail and=
=20
delete the message and any attachment(s) from your system. This=20 communication is for information purposes only and should not be regarded=
=20
as an offer to sell or as a solicitation of an offer to buy any financial=
=20
product, an official confirmation of any transaction, or as an official=20 statement of Trexquant Investment LP. All information is subject to change=
=20
without notice.

--00000000000062d99d064fe8cdaa
Content-Type: text/html; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable

<div dir=3D"ltr"><div>Hi Mark,</div><div><br></div><div>Should we capture a=
ny further diagnostics from the system? We&#39;re considering reverting the=
less-critical system to p10 to reproduce the issue. Note that the producti= on-critical system triggered the issue multiple times within hours, whereas=
the less-critical system remained operational longer, possibly due to its = specific NFS workloads.</div><div><br></div><div><div dir=3D"ltr" class=3D"= gmail_signature" data-smartmail=3D"gmail_signature"><div dir=3D"ltr"><div><= div>Best,</div><div dir=3D"ltr"><br></div><div dir=3D"ltr">-Olgun</div></di= v></div></div></div><br></div><br><div class=3D"gmail_quote"><div dir=3D"lt=
r" class=3D"gmail_attr">On Mon, Apr 20, 2026 at 2:36=E2=80=AFPM Mark Johnst=
on &lt;<a href=3D"mailto:markj@freebsd.org" target=3D"_blank">markj@freebsd= .org</a>&gt; wrote:<br></div><blockquote class=3D"gmail_quote" style=3D"mar= gin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1= ex">On Mon, Apr 20, 2026 at 10:34:22AM -0400, Olgun Adak wrote:<br>
&gt; Hi Mark,<br>
&gt; <br>
&gt; I have a truncated output of procstat -kk without -a.<br>
&gt; <br>
&gt; PID=C2=A0 =C2=A0 TID COMM=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 TDNAME=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 KSTACK<br=

&gt; [...]<br>
&gt;<br>

Unfortunately this doesn&#39;t reveal much.=C2=A0 All of the NFS threads ar= e<br>
trying to send data, but the socket buffer isn&#39;t being drained for some=

reason.=C2=A0 Full procstat output could help shed some light.<br>

&gt; Best,<br>
&gt; <br>
&gt; -Olgun<br>
&gt; <br>
&gt; <br>
&gt; On Mon, Apr 20, 2026 at 10:26=E2=80=AFAM Mark Johnston &lt;<a href=3D"= mailto:markj@freebsd.org" target=3D"_blank">markj@freebsd.org</a>&gt; wrote= :<br>
&gt; <br>
&gt; &gt; On Mon, Apr 20, 2026 at 10:11:21AM -0400, Olgun Adak wrote:<br>
&gt; &gt; &gt; Hello FreeBSD Community,<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; We=E2=80=99ve run into a consistent nfsd lockup after moving=
from 14.3-RELEASE-p8<br>
&gt; &gt; &gt; to p10. We have verified this across two identical bare-meta=
l systems.<br>
&gt; &gt; &gt; Reverting to p8 via bectl immediately restores stability on = both systems,<br>
&gt; &gt; &gt; so this appears to be a regression introduced in the p9/p10 = cycle.<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; *The symptoms:*<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; Under NFSv3 load, the nfsd service hangs and becomes unrespo= nsive to all<br>
&gt; &gt; &gt; clients. Looking at procstat -kk, we see a deadlock pattern = where threads<br>
&gt; &gt; &gt; are stuck waiting on soiolock:<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; _sx_xlock_hard -&gt; soiolock -&gt; sosend_generic -&gt; sos= end -&gt; svc_vc_reply<br>
&gt; &gt;<br>
&gt; &gt; Would you be able to share full &quot;procstat -kka&quot; output = from an affected<br>
&gt; &gt; system?<br>
&gt; &gt;<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; Several threads are blocked in _sx_xlock_hard while others s=
it in sbwait.<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; *The environment:*<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; The systems are bare-metal with 2 x dual-port Mellanox Conne= ctX-6 100GbE<br>
&gt; &gt; &gt; (mlx5en) cards. We see the issue regardless of MTU (1500 and=
9000).<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; Offloads:<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt;=C2=A0 =C2=A0 -<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt;=C2=A0 =C2=A0 TSO: Enabled<br>
&gt; &gt; &gt;=C2=A0 =C2=A0 -<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt;=C2=A0 =C2=A0 LRO: The issue persists regardless of LRO state=
(tested with LRO<br>
&gt; &gt; &gt;=C2=A0 =C2=A0 disabled and with software-only LRO). Hardware = LRO is disabled in all<br>
&gt; &gt; cases.<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; Relevant tunables:<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; kern.ipc.soacceptqueue=3D1000<br>
&gt; &gt; &gt; kern.ipc.somaxconn=3D2000<br>
&gt; &gt; &gt; kern.ipc.maxsockbuf=3D67108864<br>
&gt; &gt; &gt; net.inet.tcp.sendbuf_max=3D67108864<br>
&gt; &gt; &gt; net.inet.tcp.sendspace=3D16777216<br>
&gt; &gt; &gt; net.inet.tcp.sendbuf_inc=3D262144<br>
&gt; &gt; &gt; net.inet.tcp.recvbuf_max=3D67108864<br>
&gt; &gt; &gt; net.inet.tcp.recvspace=3D16777216<br>
&gt; &gt; &gt; vfs.nfsd.srvmaxio=3D1048576<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; We have kept the p10 Boot Environments intact and can boot b= ack into them<br>
&gt; &gt; &gt; to run any additional debug commands or test patches if some= one can help<br>
&gt; &gt; &gt; point us in the right direction.<br>
&gt; &gt; &gt;<br>
&gt; &gt; &gt; Best regards,<br>
&gt; &gt; &gt; -Olgun Adak<br>
&gt; &gt;<br>
&gt; <br>
&gt; -- <br>
&gt; <br>
&gt; <br>
&gt; <br>
&gt; This message is intended only for the use of the individual or entity =
to <br>
&gt; which it is addressed, and may contain private and confidential <br>
&gt; information. If you are not the intended recipient of this message you=
are <br>
&gt; hereby notified that any review, dissemination, distribution or copyin=
g of <br>
&gt; this message is strictly prohibited. If you have received this e-mail =
in <br>
&gt; error, please immediately notify the sender by replying to this e-mail=
and <br>
&gt; delete the message and any attachment(s) from your system. This <br>
&gt; communication is for information purposes only and should not be regar= ded <br>
&gt; as an offer to sell or as a solicitation of an offer to buy any financ= ial <br>
&gt; product, an official confirmation of any transaction, or as an officia=
l <br>
&gt; statement of Trexquant Investment LP. All information is subject to ch= ange <br>
&gt; without notice.<br>
</blockquote></div>


<div><hr></div><font face=3D"Arial, Helvetica, sans-serif"><span style=3D"f= ont-size:13px"><div><font face=3D"Arial, Helvetica, sans-serif"><span style= =3D"font-size:13px"><br></span></font></div>This message is intended only f=
or the use of the individual or entity to which it is addressed, and may co= ntain private and confidential information. If you are not the intended rec= ipient of this message you are hereby notified that any review, disseminati= on, distribution or copying of this message is strictly prohibited. If you = have received this e-mail in error, please immediately notify the sender by=
replying to this e-mail and delete the message and any attachment(s) from = your system. This communication is for information purposes only and should=
not be regarded as an offer to sell or as a solicitation of an offer to bu=
y any financial product, an official confirmation of any transaction, or as=
an official statement of Trexquant Investment LP. All information is subje=
ct to change without notice.</span></font>
--00000000000062d99d064fe8cdaa--


--
Posted automagically by a mail2news gateway at muc.de e.V.
Please direct questions, flames, donations, etc. to news-admin@muc.de
--- Synchronet 3.21f-Linux NewsLink 1.2

From Newsgroup: muc.lists.freebsd.stable

<<On Mon, 20 Apr 2026 10:26:23 -0400, Mark Johnston <markj@freebsd.org> said:

Would you be able to share full "procstat -kka" output from an affected system?

For what it's worth, I have found the following recipe to be helpful:

# procstat -hk NFSD_PID | | cut -c 54- | sort | uniq -c | sort -n

since many nfsd threads will often be sleeping with the same stack
trace, this helps to collapse them and call out the interesting
activity.

I ran this on our one active 14.3-p10 server, and the result is like
this:

1 mi_switch sleepq_timedwait _cv_timedwait_sbt zio_wait dbuf_read dbuf_hold_impl dbuf_hold dmu_buf_hold_array_by_dnode dmu_read_uio_dnode dmu_read_uio_dbuf zfs_read zfs_freebsd_read VOP_READ_APV nfsvno_read nfsrvd_read nfsrvd_dorpc nfssvc_program svc_run_internal
1 mi_switch sleepq_timedwait _cv_timedwait_sbt zio_wait dmu_buf_hold_array_by_dnode dmu_read_uio_dnode dmu_read_uio_dbuf zfs_read zfs_freebsd_read VOP_READ_APV nfsvno_read nfsrvd_read nfsrvd_dorpc nfssvc_program svc_run_internal svc_run nfsrvd_nfsd nfssvc_nfsd
1 soiolock sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline
1 uma_zalloc_arg tcp_m_copym tcp_default_output tcp_usr_send sosend_generic_locked sosend_generic sosend svc_vc_reply svc_sendreply_common svc_sendreply_mbuf nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline
2 mi_switch _cv_wait dbuf_read dbuf_hold_impl dbuf_hold dmu_buf_hold_array_by_dnode dmu_read_uio_dnode dmu_read_uio_dbuf zfs_read zfs_freebsd_read VOP_READ_APV nfsvno_read nfsrvd_read nfsrvd_dorpc nfssvc_program svc_run_internal svc_thread_start fork_exit
2 mi_switch sleepq_catch_signals sleepq_wait_sig _cv_wait_sig svc_run_internal svc_thread_start fork_exit fork_trampoline
7 mi_switch sleepq_timedwait _cv_timedwait_sbt zio_wait dmu_buf_hold_array_by_dnode dmu_read_uio_dnode dmu_read_uio_dbuf zfs_read zfs_freebsd_read VOP_READ_APV nfsvno_read nfsrvd_read nfsrvd_dorpc nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline
19 mi_switch _cv_wait dmu_buf_hold_array_by_dnode dmu_read_uio_dnode dmu_read_uio_dbuf zfs_read zfs_freebsd_read VOP_READ_APV nfsvno_read nfsrvd_read nfsrvd_dorpc nfssvc_program svc_run_internal svc_thread_start fork_exit fork_trampoline
46 mi_switch sleepq_catch_signals sleepq_timedwait_sig _cv_timedwait_sig_sbt svc_run_internal svc_thread_start fork_exit fork_trampoline

-GAWollman



--
Posted automagically by a mail2news gateway at muc.de e.V.
Please direct questions, flames, donations, etc. to news-admin@muc.de
--- Synchronet 3.21f-Linux NewsLink 1.2