Skip to content

feat: recover from repeated crashes by going into safe mode #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mateoalfaro wants to merge 2 commits into
base: main
Choose a base branch
from
+354 −43
Open

feat: recover from repeated crashes by going into safe mode #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e2c1f5f
feat: recover repeated session crashes in safe mode
mateoalfaro Sep 4, 2026
e88e872
fix: keep hardware rendering in safe mode
mateoalfaro Sep 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,7 @@ install_data(
install_dir: get_option('datadir') / 'singularity' / 'labwc',
install_mode: 'rwxr-xr-x',
)

session_test_bash = find_program('bash')
test('session-safe-mode', session_test_bash,
args: files('tests/session_safe_mode_test.sh'))
76 changes: 54 additions & 22 deletions src/singularity-desktop-session
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ BIN="$(dirname "$SELF")"
PREFIX="$(dirname "$BIN")"
LIB="$PREFIX/lib"
SHARE="$PREFIX/share"
_DESKTOP="${SINGULARITY_DESKTOP_BINARY:-$BIN/singularity-desktop}"

export LD_LIBRARY_PATH="$LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
export PATH="$BIN:$PATH"
Expand All @@ -24,6 +25,57 @@ fi

_STATE="${XDG_STATE_HOME:-$HOME/.local/state}/singularity"
mkdir -p "$_STATE"
_LOG="$_STATE/desktop.log"

run_desktop_supervisor() {
_crashes=0
_crash_window_started=0
_crash_window_seconds=300
_crash_limit=3
while true; do
_t0=$SECONDS
"$_DESKTOP" >> "$_LOG" 2>&1
_EC=$?
_dt=$((SECONDS - _t0))
if [ $_EC -eq 0 ]; then
echo "[$(date)] singularity-desktop exited 0, restarting..." >> "$_LOG"
_crashes=0
_crash_window_started=0
sleep 0.2
else
_now=$SECONDS
if [ $_crashes -eq 0 ] || [ $((_now - _crash_window_started)) -gt $_crash_window_seconds ]; then
_crashes=1
_crash_window_started=$_now
else
_crashes=$((_crashes + 1))
fi
if [ $_crashes -ge $_crash_limit ]; then
_request="$_STATE/safe-mode-request"
_request_tmp="${_request}.tmp.$$"
(umask 077; {
printf 'version=1\n'
printf 'build_id=%s\n' "${SINGULARITY_SESSION_BUILD_ID:-unknown}"
printf 'reason=repeated-shell-failure\n'
} > "$_request_tmp") && chmod 600 "$_request_tmp" &&
mv -f "$_request_tmp" "$_request"
echo "[$(date)] singularity-desktop crashed $_crashes times within ${_crash_window_seconds}s; requesting Safe Mode." >> "$_LOG"
kill "$PPID" 2>/dev/null || true
return 1
fi
echo "[$(date)] singularity-desktop exited $_EC after ${_dt}s (crash $_crashes/$_crash_limit), restarting in 2s..." >> "$_LOG"
sleep 2
fi
done
}

# Side-effect-free entry point for the supervisor black-box test. It must stay
# before process cleanup, helper startup, and user-service mutations below.
if [ "${SINGULARITY_SESSION_SUPERVISOR_ONLY:-0}" = "1" ]; then
run_desktop_supervisor
exit $?
fi

# labwc does not run /etc/xdg/autostart, so create the XDG user dirs here (idempotent,
# locale-aware): Desktop/Documents/Downloads/... on the now-decrypted /home (FB-15).
command -v xdg-user-dirs-update >/dev/null 2>&1 && xdg-user-dirs-update 2>/dev/null || true
Expand All @@ -39,7 +91,6 @@ dbus-update-activation-environment \
GSETTINGS_SCHEMA_DIR XDG_DATA_DIRS GI_TYPELIB_PATH PATH LD_LIBRARY_PATH \
2>/dev/null || true

_LOG="$_STATE/desktop.log"
[ -f "$_LOG" ] && mv -f "$_LOG" "$_LOG.1"
_SPID="${XDG_RUNTIME_DIR:-/tmp}/singularity-desktop-session.pid"
if [ -f "$_SPID" ]; then
Expand All @@ -66,24 +117,5 @@ fi
# waiting for our portal backend, which would delay the shell launch below.
systemctl --user restart --no-block xdg-desktop-portal 2>/dev/null || true

_fails=0
while true; do
_t0=$SECONDS
"$BIN/singularity-desktop" >> "$_LOG" 2>&1
_EC=$?
_dt=$((SECONDS - _t0))
if [ $_EC -eq 0 ]; then
echo "[$(date)] singularity-desktop exited 0, restarting..." >> "$_LOG"
_fails=0
sleep 0.2
else
if [ $_dt -lt 3 ]; then _fails=$((_fails + 1)); else _fails=0; fi
if [ $_fails -ge 5 ]; then
echo "[$(date)] singularity-desktop failed fast $_fails times (compositor gone?), ending the session instead of looping." >> "$_LOG"
kill "$PPID" 2>/dev/null || true
exit 1
fi
echo "[$(date)] singularity-desktop exited $_EC after ${_dt}s, restarting in 2s..." >> "$_LOG"
sleep 2
fi
done
run_desktop_supervisor
exit $?
172 changes: 151 additions & 21 deletions src/singularity-labwc-session
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,38 +26,168 @@ done
_STATE="${XDG_STATE_HOME:-$HOME/.local/state}/singularity"
mkdir -p "$_STATE"
_LLOG="$_STATE/labwc.log"
_SAFE_MARKER="$_STATE/safe-mode"
_SAFE_REQUEST="$_STATE/safe-mode-request"
[ -f "$_LLOG" ] && mv -f "$_LLOG" "$_LLOG.1"

if [ -n "${SINGULARITY_LABWC_BINARY:-}" ]; then
_LABWC="$SINGULARITY_LABWC_BINARY"
else
# Keep this literal path as a packaging seam: the Nix package replaces it
# with the compositor from PATH.
_LABWC="$BIN/labwc"
fi
_DESKTOP_SESSION="${SINGULARITY_DESKTOP_SESSION_BINARY:-$BIN/singularity-desktop-session}"
_DESKTOP="${SINGULARITY_DESKTOP_BINARY:-$BIN/singularity-desktop}"
if [ -n "${SINGULARITY_SESSION_BUILD_ID:-}" ]; then
_BUILD_ID="$SINGULARITY_SESSION_BUILD_ID"
elif command -v sha256sum >/dev/null 2>&1; then
_LABWC_IDENTITY=$(command -v "$_LABWC" 2>/dev/null || printf '%s' "$_LABWC")
_DESKTOP_IDENTITY=$(command -v "$_DESKTOP_SESSION" 2>/dev/null || printf '%s' "$_DESKTOP_SESSION")
_SHELL_IDENTITY=$(command -v "$_DESKTOP" 2>/dev/null || printf '%s' "$_DESKTOP")
_BUILD_ID="$({
for _binary in "$_LABWC_IDENTITY" "$_DESKTOP_IDENTITY" "$_SHELL_IDENTITY"; do
if [ -r "$_binary" ]; then
sha256sum "$_binary"
else
printf 'missing:%s\n' "$_binary"
fi
done
} | sha256sum | cut -d' ' -f1)"
else
_BUILD_ID="$PREFIX"
fi
export SINGULARITY_SESSION_BUILD_ID="$_BUILD_ID"

[ -r /etc/vconsole.conf ] && . /etc/vconsole.conf
[ -n "$XKBLAYOUT" ] && export XKB_DEFAULT_LAYOUT="$XKBLAYOUT"
[ -n "$XKBVARIANT" ] && export XKB_DEFAULT_VARIANT="$XKBVARIANT"
[ -n "${XKBLAYOUT:-}" ] && export XKB_DEFAULT_LAYOUT="$XKBLAYOUT"
[ -n "${XKBVARIANT:-}" ] && export XKB_DEFAULT_VARIANT="$XKBVARIANT"

run_labwc() {
if [ -n "$GDM_SESSION_DBUS_ADDRESS" ] && [ -x "/usr/libexec/gdm-wayland-session" ]; then
/usr/libexec/gdm-wayland-session "$BIN/labwc" -S "$BIN/singularity-desktop-session"
elif [ -z "$DBUS_SESSION_BUS_ADDRESS" ] && [ ! -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/bus" ] && command -v dbus-run-session >/dev/null 2>&1; then
dbus-run-session -- "$BIN/labwc" -S "$BIN/singularity-desktop-session"
if [ -n "${GDM_SESSION_DBUS_ADDRESS:-}" ] && [ -x "/usr/libexec/gdm-wayland-session" ]; then
/usr/libexec/gdm-wayland-session "$_LABWC" -S "$_DESKTOP_SESSION"
elif [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ ! -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/bus" ] && command -v dbus-run-session >/dev/null 2>&1; then
dbus-run-session -- "$_LABWC" -S "$_DESKTOP_SESSION"
else
"$BIN/labwc" -S "$BIN/singularity-desktop-session"
"$_LABWC" -S "$_DESKTOP_SESSION"
fi
}

run_attempt() {
_ATTEMPT_LABEL="$1"
_t0=$SECONDS
run_labwc >> "$_LLOG" 2>&1
_ATTEMPT_RC=$?
_ATTEMPT_DT=$((SECONDS - _t0))
_ATTEMPT_SAFE_REQUEST=0
if safe_request_matches_build; then
_ATTEMPT_SAFE_REQUEST=1
fi
[ -f "$_SAFE_REQUEST" ] && rm -f "$_SAFE_REQUEST"
echo "[$(date)] $_ATTEMPT_LABEL exited rc=$_ATTEMPT_RC after ${_ATTEMPT_DT}s" >> "$_LLOG"
}

is_recovery_failure() {
[ "${_ATTEMPT_SAFE_REQUEST:-0}" = "1" ] ||
{ [ "$1" -ne 0 ] && [ "$1" -ne 130 ] && [ "$1" -ne 143 ] && [ "$2" -lt 30 ]; }
}

marker_matches_build() {
[ -f "$_SAFE_MARKER" ] || return 1
_marker_build=$(sed -n 's/^build_id=//p' "$_SAFE_MARKER" 2>/dev/null | head -n 1)
[ -n "$_marker_build" ] && [ "$_marker_build" = "$_BUILD_ID" ]
}

safe_request_matches_build() {
[ -f "$_SAFE_REQUEST" ] || return 1
_request_build=$(sed -n 's/^build_id=//p' "$_SAFE_REQUEST" 2>/dev/null | head -n 1)
[ -n "$_request_build" ] && [ "$_request_build" = "$_BUILD_ID" ]
}

write_safe_marker() {
_failed_rc="$1"
_failed_dt="$2"
_marker_tmp="${_SAFE_MARKER}.tmp.$$"
if (umask 077; {
echo "version=1"
echo "build_id=$_BUILD_ID"
echo "reason=repeated-session-failure"
echo "exit_status=$_failed_rc"
echo "elapsed_seconds=$_failed_dt"
echo "timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} > "$_marker_tmp") && chmod 600 "$_marker_tmp" && mv -f "$_marker_tmp" "$_SAFE_MARKER"; then
return 0
fi
echo "[$(date)] could not persist Safe Mode marker at $_SAFE_MARKER" >> "$_LLOG"
rm -f "$_marker_tmp"
return 1
}

# Try the hardware renderer first (GSK default gl, labwc default drm/gl). If the
# session dies within a few seconds it is an early crash on the real GL/GPU path
# (invisible under QEMU's software path), so fall back to software rendering: the
# desktop comes up anyway, degraded but alive, and the crash stays in the log for
# diagnosis (#78). singularity-desktop-session honours an exported GSK_RENDERER.
_t0=$SECONDS
run_labwc >> "$_LLOG" 2>&1
_rc=$?
_dt=$((SECONDS - _t0))
if [ "$_dt" -lt 30 ]; then
echo "[$(date)] labwc session exited rc=$_rc after ${_dt}s; retrying with software rendering (GSK_RENDERER=cairo, WLR_RENDERER=pixman)" >> "$_LLOG"
enable_software_rendering() {
export GSK_RENDERER=cairo
export WLR_RENDERER=pixman
export LIBGL_ALWAYS_SOFTWARE=1
export GALLIUM_DRIVER=llvmpipe
run_labwc >> "$_LLOG" 2>&1
_rc=$?
}

run_safe_mode() {
# Safe mode suppresses optional shell features; it must NOT force the
# software rendering stack. On machines with a working GPU, rendering the
# whole session on the CPU maxes out cores and makes the recovery session
# unusably slow (libinput lag, crawl), and in the escalation ladder the
# software path has already failed by the time safe mode starts. Set
# SINGULARITY_SAFE_MODE_SOFTWARE=1 to force it when the GPU stack itself
# is the suspected culprit.
if [ "${SINGULARITY_SAFE_MODE_SOFTWARE:-0}" = "1" ]; then
enable_software_rendering
else
unset GSK_RENDERER WLR_RENDERER LIBGL_ALWAYS_SOFTWARE GALLIUM_DRIVER
fi
export SINGULARITY_SAFE_MODE=1
export SINGULARITY_SAFE_MODE_MARKER="$_SAFE_MARKER"
export SINGULARITY_SAFE_MODE_REASON="Repeated startup failures"
echo "[$(date)] starting Safe Mode; persisted settings remain writable" >> "$_LLOG"
run_attempt "Safe Mode session"
}

# An explicit manual Safe Mode request does not need a marker.
if [ "${SINGULARITY_SAFE_MODE:-0}" = "1" ]; then
run_safe_mode
exit "$_ATTEMPT_RC"
fi

# If the outer supervisor was terminated before it could consume an inner
# shell-crash request, honor that request on the next display-manager launch.
if safe_request_matches_build; then
rm -f "$_SAFE_REQUEST"
write_safe_marker 1 0 || true
run_safe_mode
exit "$_ATTEMPT_RC"
fi
[ -f "$_SAFE_REQUEST" ] && rm -f "$_SAFE_REQUEST"

# A marker only applies to the binaries that created it. After an update, try
# normal mode again so a fixed build cannot remain trapped in recovery mode.
if [ "${SINGULARITY_FORCE_NORMAL:-0}" != "1" ] && marker_matches_build; then
run_safe_mode
exit "$_ATTEMPT_RC"
fi
[ -f "$_SAFE_MARKER" ] && rm -f "$_SAFE_MARKER"

unset SINGULARITY_SAFE_MODE SINGULARITY_SAFE_MODE_MARKER SINGULARITY_SAFE_MODE_REASON
run_attempt "normal compositor session"
if ! is_recovery_failure "$_ATTEMPT_RC" "$_ATTEMPT_DT"; then
exit "$_ATTEMPT_RC"
fi

# Preserve the existing renderer fallback as the second normal-mode attempt.
echo "[$(date)] early compositor failure; retrying with software rendering (GSK_RENDERER=cairo, WLR_RENDERER=pixman)" >> "$_LLOG"
enable_software_rendering
run_attempt "software compositor session"
if ! is_recovery_failure "$_ATTEMPT_RC" "$_ATTEMPT_DT"; then
exit "$_ATTEMPT_RC"
fi
exit $_rc

write_safe_marker "$_ATTEMPT_RC" "$_ATTEMPT_DT" || true
run_safe_mode
exit "$_ATTEMPT_RC"
Loading