add shikane-watchdog user service to auto-recover shikane deadlocks

This commit is contained in:
jozamudi committed 2026-09-28 14:04:22 -07:00
1 parent e3643c3ae2
commit e338d15922
5 files changed
+88

No files matched your search

+49
View File
@@ -0,0 +1,49 @@
#!/bin/sh
# shikane-watchdog: auto-recover shikane from its deadlock bug.
#
# Bug (shikane 1.1.1): if a variant application is discarded mid-way
# (e.g. by a failing DP atomic commit), the daemon gets stuck in
# VariantInProgress/Discarded and keeps logging
# [WARN shikane::variant] Received invalid input AtomicChangeDone at state Discarded
# `shikanectl reload` does not recover it; a service restart does
# (re-applies the active profile in ~1s).
#
# A healthy daemon's last state-machine line is "VariantApplied".
set -u
unit=shikane
stuck=0
logs3=$(journalctl --user -u "$unit" -q --since "-3min" 2>/dev/null)
[ -z "$logs3" ] && exit 0
latest3=$(printf '%s\n' "$logs3" | grep 'daemon::state_machine' | tail -1)
if [ -n "$latest3" ]; then
case "$latest3" in
*VariantApplied*)
: ;; # healthy
*)
# InProgress/Discarded within the last 3 min: only a deadlock
# if the WARN signature is present (a legitimate slow transition
# has no WARN)
printf '%s\n' "$logs3" | grep -q 'invalid input' && stuck=1
;;
esac
else
# No state change in the last 3 min — check the last known state
last10=$(journalctl --user -u "$unit" -q --since "-10min" 2>/dev/null \
| grep 'daemon::state_machine' | tail -1)
case "$last10" in
'' | *VariantApplied*)
: ;; # stable and applied, or nothing to check
*)
stuck=1 ;; # stuck in-progress/discarded for >3 min
esac
fi
if [ "$stuck" -eq 1 ]; then
echo "shikane-watchdog: shikane stuck, restarting" >&2
systemctl --user restart "$unit"
fi