diff --git a/meson.build b/meson.build index c7f6c75..e813f74 100644 --- a/meson.build +++ b/meson.build @@ -18,3 +18,7 @@ install_data( install_dir: get_option('datadir') / 'singularity' / 'labwc', install_mode: 'rwxr-xr-x', ) + +session_test_bash = find_program('bash') +test('session-safe-mode', session_test_bash, + args: files('tests/session_safe_mode_test.sh')) diff --git a/src/singularity-desktop-session b/src/singularity-desktop-session index 3870ef5..b003059 100755 --- a/src/singularity-desktop-session +++ b/src/singularity-desktop-session @@ -4,6 +4,7 @@ BIN="$(dirname "$SELF")" PREFIX="$(dirname "$BIN")" LIB="$PREFIX/lib" SHARE="$PREFIX/share" +_DESKTOP="${SINGULARITY_DESKTOP_BINARY:-$BIN/singularity-desktop}" export LD_LIBRARY_PATH="$LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" export PATH="$BIN:$PATH" @@ -24,6 +25,57 @@ fi _STATE="${XDG_STATE_HOME:-$HOME/.local/state}/singularity" mkdir -p "$_STATE" +_LOG="$_STATE/desktop.log" + +run_desktop_supervisor() { + _crashes=0 + _crash_window_started=0 + _crash_window_seconds=300 + _crash_limit=3 + while true; do + _t0=$SECONDS + "$_DESKTOP" >> "$_LOG" 2>&1 + _EC=$? + _dt=$((SECONDS - _t0)) + if [ $_EC -eq 0 ]; then + echo "[$(date)] singularity-desktop exited 0, restarting..." >> "$_LOG" + _crashes=0 + _crash_window_started=0 + sleep 0.2 + else + _now=$SECONDS + if [ $_crashes -eq 0 ] || [ $((_now - _crash_window_started)) -gt $_crash_window_seconds ]; then + _crashes=1 + _crash_window_started=$_now + else + _crashes=$((_crashes + 1)) + fi + if [ $_crashes -ge $_crash_limit ]; then + _request="$_STATE/safe-mode-request" + _request_tmp="${_request}.tmp.$$" + (umask 077; { + printf 'version=1\n' + printf 'build_id=%s\n' "${SINGULARITY_SESSION_BUILD_ID:-unknown}" + printf 'reason=repeated-shell-failure\n' + } > "$_request_tmp") && chmod 600 "$_request_tmp" && + mv -f "$_request_tmp" "$_request" + echo "[$(date)] singularity-desktop crashed $_crashes times within ${_crash_window_seconds}s; requesting Safe Mode." >> "$_LOG" + kill "$PPID" 2>/dev/null || true + return 1 + fi + echo "[$(date)] singularity-desktop exited $_EC after ${_dt}s (crash $_crashes/$_crash_limit), restarting in 2s..." >> "$_LOG" + sleep 2 + fi + done +} + +# Side-effect-free entry point for the supervisor black-box test. It must stay +# before process cleanup, helper startup, and user-service mutations below. +if [ "${SINGULARITY_SESSION_SUPERVISOR_ONLY:-0}" = "1" ]; then + run_desktop_supervisor + exit $? +fi + # labwc does not run /etc/xdg/autostart, so create the XDG user dirs here (idempotent, # locale-aware): Desktop/Documents/Downloads/... on the now-decrypted /home (FB-15). command -v xdg-user-dirs-update >/dev/null 2>&1 && xdg-user-dirs-update 2>/dev/null || true @@ -39,7 +91,6 @@ dbus-update-activation-environment \ GSETTINGS_SCHEMA_DIR XDG_DATA_DIRS GI_TYPELIB_PATH PATH LD_LIBRARY_PATH \ 2>/dev/null || true -_LOG="$_STATE/desktop.log" [ -f "$_LOG" ] && mv -f "$_LOG" "$_LOG.1" _SPID="${XDG_RUNTIME_DIR:-/tmp}/singularity-desktop-session.pid" if [ -f "$_SPID" ]; then @@ -66,24 +117,5 @@ fi # waiting for our portal backend, which would delay the shell launch below. systemctl --user restart --no-block xdg-desktop-portal 2>/dev/null || true -_fails=0 -while true; do - _t0=$SECONDS - "$BIN/singularity-desktop" >> "$_LOG" 2>&1 - _EC=$? - _dt=$((SECONDS - _t0)) - if [ $_EC -eq 0 ]; then - echo "[$(date)] singularity-desktop exited 0, restarting..." >> "$_LOG" - _fails=0 - sleep 0.2 - else - if [ $_dt -lt 3 ]; then _fails=$((_fails + 1)); else _fails=0; fi - if [ $_fails -ge 5 ]; then - echo "[$(date)] singularity-desktop failed fast $_fails times (compositor gone?), ending the session instead of looping." >> "$_LOG" - kill "$PPID" 2>/dev/null || true - exit 1 - fi - echo "[$(date)] singularity-desktop exited $_EC after ${_dt}s, restarting in 2s..." >> "$_LOG" - sleep 2 - fi -done +run_desktop_supervisor +exit $? diff --git a/src/singularity-labwc-session b/src/singularity-labwc-session index 5392140..439b931 100755 --- a/src/singularity-labwc-session +++ b/src/singularity-labwc-session @@ -26,38 +26,168 @@ done _STATE="${XDG_STATE_HOME:-$HOME/.local/state}/singularity" mkdir -p "$_STATE" _LLOG="$_STATE/labwc.log" +_SAFE_MARKER="$_STATE/safe-mode" +_SAFE_REQUEST="$_STATE/safe-mode-request" [ -f "$_LLOG" ] && mv -f "$_LLOG" "$_LLOG.1" +if [ -n "${SINGULARITY_LABWC_BINARY:-}" ]; then + _LABWC="$SINGULARITY_LABWC_BINARY" +else + # Keep this literal path as a packaging seam: the Nix package replaces it + # with the compositor from PATH. + _LABWC="$BIN/labwc" +fi +_DESKTOP_SESSION="${SINGULARITY_DESKTOP_SESSION_BINARY:-$BIN/singularity-desktop-session}" +_DESKTOP="${SINGULARITY_DESKTOP_BINARY:-$BIN/singularity-desktop}" +if [ -n "${SINGULARITY_SESSION_BUILD_ID:-}" ]; then + _BUILD_ID="$SINGULARITY_SESSION_BUILD_ID" +elif command -v sha256sum >/dev/null 2>&1; then + _LABWC_IDENTITY=$(command -v "$_LABWC" 2>/dev/null || printf '%s' "$_LABWC") + _DESKTOP_IDENTITY=$(command -v "$_DESKTOP_SESSION" 2>/dev/null || printf '%s' "$_DESKTOP_SESSION") + _SHELL_IDENTITY=$(command -v "$_DESKTOP" 2>/dev/null || printf '%s' "$_DESKTOP") + _BUILD_ID="$({ + for _binary in "$_LABWC_IDENTITY" "$_DESKTOP_IDENTITY" "$_SHELL_IDENTITY"; do + if [ -r "$_binary" ]; then + sha256sum "$_binary" + else + printf 'missing:%s\n' "$_binary" + fi + done + } | sha256sum | cut -d' ' -f1)" +else + _BUILD_ID="$PREFIX" +fi +export SINGULARITY_SESSION_BUILD_ID="$_BUILD_ID" + [ -r /etc/vconsole.conf ] && . /etc/vconsole.conf -[ -n "$XKBLAYOUT" ] && export XKB_DEFAULT_LAYOUT="$XKBLAYOUT" -[ -n "$XKBVARIANT" ] && export XKB_DEFAULT_VARIANT="$XKBVARIANT" +[ -n "${XKBLAYOUT:-}" ] && export XKB_DEFAULT_LAYOUT="$XKBLAYOUT" +[ -n "${XKBVARIANT:-}" ] && export XKB_DEFAULT_VARIANT="$XKBVARIANT" run_labwc() { - if [ -n "$GDM_SESSION_DBUS_ADDRESS" ] && [ -x "/usr/libexec/gdm-wayland-session" ]; then - /usr/libexec/gdm-wayland-session "$BIN/labwc" -S "$BIN/singularity-desktop-session" - elif [ -z "$DBUS_SESSION_BUS_ADDRESS" ] && [ ! -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/bus" ] && command -v dbus-run-session >/dev/null 2>&1; then - dbus-run-session -- "$BIN/labwc" -S "$BIN/singularity-desktop-session" + if [ -n "${GDM_SESSION_DBUS_ADDRESS:-}" ] && [ -x "/usr/libexec/gdm-wayland-session" ]; then + /usr/libexec/gdm-wayland-session "$_LABWC" -S "$_DESKTOP_SESSION" + elif [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ ! -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/bus" ] && command -v dbus-run-session >/dev/null 2>&1; then + dbus-run-session -- "$_LABWC" -S "$_DESKTOP_SESSION" else - "$BIN/labwc" -S "$BIN/singularity-desktop-session" + "$_LABWC" -S "$_DESKTOP_SESSION" + fi +} + +run_attempt() { + _ATTEMPT_LABEL="$1" + _t0=$SECONDS + run_labwc >> "$_LLOG" 2>&1 + _ATTEMPT_RC=$? + _ATTEMPT_DT=$((SECONDS - _t0)) + _ATTEMPT_SAFE_REQUEST=0 + if safe_request_matches_build; then + _ATTEMPT_SAFE_REQUEST=1 + fi + [ -f "$_SAFE_REQUEST" ] && rm -f "$_SAFE_REQUEST" + echo "[$(date)] $_ATTEMPT_LABEL exited rc=$_ATTEMPT_RC after ${_ATTEMPT_DT}s" >> "$_LLOG" +} + +is_recovery_failure() { + [ "${_ATTEMPT_SAFE_REQUEST:-0}" = "1" ] || + { [ "$1" -ne 0 ] && [ "$1" -ne 130 ] && [ "$1" -ne 143 ] && [ "$2" -lt 30 ]; } +} + +marker_matches_build() { + [ -f "$_SAFE_MARKER" ] || return 1 + _marker_build=$(sed -n 's/^build_id=//p' "$_SAFE_MARKER" 2>/dev/null | head -n 1) + [ -n "$_marker_build" ] && [ "$_marker_build" = "$_BUILD_ID" ] +} + +safe_request_matches_build() { + [ -f "$_SAFE_REQUEST" ] || return 1 + _request_build=$(sed -n 's/^build_id=//p' "$_SAFE_REQUEST" 2>/dev/null | head -n 1) + [ -n "$_request_build" ] && [ "$_request_build" = "$_BUILD_ID" ] +} + +write_safe_marker() { + _failed_rc="$1" + _failed_dt="$2" + _marker_tmp="${_SAFE_MARKER}.tmp.$$" + if (umask 077; { + echo "version=1" + echo "build_id=$_BUILD_ID" + echo "reason=repeated-session-failure" + echo "exit_status=$_failed_rc" + echo "elapsed_seconds=$_failed_dt" + echo "timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } > "$_marker_tmp") && chmod 600 "$_marker_tmp" && mv -f "$_marker_tmp" "$_SAFE_MARKER"; then + return 0 fi + echo "[$(date)] could not persist Safe Mode marker at $_SAFE_MARKER" >> "$_LLOG" + rm -f "$_marker_tmp" + return 1 } -# Try the hardware renderer first (GSK default gl, labwc default drm/gl). If the -# session dies within a few seconds it is an early crash on the real GL/GPU path -# (invisible under QEMU's software path), so fall back to software rendering: the -# desktop comes up anyway, degraded but alive, and the crash stays in the log for -# diagnosis (#78). singularity-desktop-session honours an exported GSK_RENDERER. -_t0=$SECONDS -run_labwc >> "$_LLOG" 2>&1 -_rc=$? -_dt=$((SECONDS - _t0)) -if [ "$_dt" -lt 30 ]; then - echo "[$(date)] labwc session exited rc=$_rc after ${_dt}s; retrying with software rendering (GSK_RENDERER=cairo, WLR_RENDERER=pixman)" >> "$_LLOG" +enable_software_rendering() { export GSK_RENDERER=cairo export WLR_RENDERER=pixman export LIBGL_ALWAYS_SOFTWARE=1 export GALLIUM_DRIVER=llvmpipe - run_labwc >> "$_LLOG" 2>&1 - _rc=$? +} + +run_safe_mode() { + # Safe mode suppresses optional shell features; it must NOT force the + # software rendering stack. On machines with a working GPU, rendering the + # whole session on the CPU maxes out cores and makes the recovery session + # unusably slow (libinput lag, crawl), and in the escalation ladder the + # software path has already failed by the time safe mode starts. Set + # SINGULARITY_SAFE_MODE_SOFTWARE=1 to force it when the GPU stack itself + # is the suspected culprit. + if [ "${SINGULARITY_SAFE_MODE_SOFTWARE:-0}" = "1" ]; then + enable_software_rendering + else + unset GSK_RENDERER WLR_RENDERER LIBGL_ALWAYS_SOFTWARE GALLIUM_DRIVER + fi + export SINGULARITY_SAFE_MODE=1 + export SINGULARITY_SAFE_MODE_MARKER="$_SAFE_MARKER" + export SINGULARITY_SAFE_MODE_REASON="Repeated startup failures" + echo "[$(date)] starting Safe Mode; persisted settings remain writable" >> "$_LLOG" + run_attempt "Safe Mode session" +} + +# An explicit manual Safe Mode request does not need a marker. +if [ "${SINGULARITY_SAFE_MODE:-0}" = "1" ]; then + run_safe_mode + exit "$_ATTEMPT_RC" +fi + +# If the outer supervisor was terminated before it could consume an inner +# shell-crash request, honor that request on the next display-manager launch. +if safe_request_matches_build; then + rm -f "$_SAFE_REQUEST" + write_safe_marker 1 0 || true + run_safe_mode + exit "$_ATTEMPT_RC" +fi +[ -f "$_SAFE_REQUEST" ] && rm -f "$_SAFE_REQUEST" + +# A marker only applies to the binaries that created it. After an update, try +# normal mode again so a fixed build cannot remain trapped in recovery mode. +if [ "${SINGULARITY_FORCE_NORMAL:-0}" != "1" ] && marker_matches_build; then + run_safe_mode + exit "$_ATTEMPT_RC" +fi +[ -f "$_SAFE_MARKER" ] && rm -f "$_SAFE_MARKER" + +unset SINGULARITY_SAFE_MODE SINGULARITY_SAFE_MODE_MARKER SINGULARITY_SAFE_MODE_REASON +run_attempt "normal compositor session" +if ! is_recovery_failure "$_ATTEMPT_RC" "$_ATTEMPT_DT"; then + exit "$_ATTEMPT_RC" +fi + +# Preserve the existing renderer fallback as the second normal-mode attempt. +echo "[$(date)] early compositor failure; retrying with software rendering (GSK_RENDERER=cairo, WLR_RENDERER=pixman)" >> "$_LLOG" +enable_software_rendering +run_attempt "software compositor session" +if ! is_recovery_failure "$_ATTEMPT_RC" "$_ATTEMPT_DT"; then + exit "$_ATTEMPT_RC" fi -exit $_rc + +write_safe_marker "$_ATTEMPT_RC" "$_ATTEMPT_DT" || true +run_safe_mode +exit "$_ATTEMPT_RC" diff --git a/tests/session_safe_mode_test.sh b/tests/session_safe_mode_test.sh new file mode 100755 index 0000000..7310bff --- /dev/null +++ b/tests/session_safe_mode_test.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +set -eu + +if [ "${1:-}" = "-S" ]; then + printf '%s\n' "${SINGULARITY_SAFE_MODE:-0}" >> "$SINGULARITY_TEST_ATTEMPTS" + if [ "${SINGULARITY_SAFE_MODE:-0}" = "1" ]; then + exit 0 + fi + exit "${SINGULARITY_TEST_NORMAL_EXIT:-134}" +fi + +if [ "${SINGULARITY_TEST_FAKE_DESKTOP:-0}" = "1" ]; then + printf 'shell\n' >> "$SINGULARITY_TEST_SHELL_ATTEMPTS" + exit 1 +fi + +TEST_DIR=$(mktemp -d "${TMPDIR:-/tmp}/singularity-session-test.XXXXXX") +trap 'rm -rf "$TEST_DIR"' EXIT +mkdir -p "$TEST_DIR/state" "$TEST_DIR/runtime" +chmod 700 "$TEST_DIR/runtime" + +export XDG_STATE_HOME="$TEST_DIR/state" +export XDG_RUNTIME_DIR="$TEST_DIR/runtime" +export DBUS_SESSION_BUS_ADDRESS="test-bus" +export GDM_SESSION_DBUS_ADDRESS="" +export SINGULARITY_LABWC_BINARY="$0" +export SINGULARITY_DESKTOP_SESSION_BINARY="$0" +export SINGULARITY_SESSION_BUILD_ID="safe-mode-test-build" +export SINGULARITY_TEST_ATTEMPTS="$TEST_DIR/attempts" + +LAUNCHER="$(dirname "$0")/../src/singularity-labwc-session" +bash "$LAUNCHER" + +EXPECTED=$(printf '0\n0\n1') +ACTUAL=$(cat "$SINGULARITY_TEST_ATTEMPTS") +[ "$ACTUAL" = "$EXPECTED" ] || { + echo "unexpected first-launch attempts: $ACTUAL" >&2 + exit 1 +} + +MARKER="$XDG_STATE_HOME/singularity/safe-mode" +[ -f "$MARKER" ] +[ "$(stat -c %a "$MARKER")" = "600" ] +grep -Fx "build_id=safe-mode-test-build" "$MARKER" >/dev/null +grep -Fx "reason=repeated-session-failure" "$MARKER" >/dev/null + +: > "$SINGULARITY_TEST_ATTEMPTS" +bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "1" ] || { + echo "matching marker did not start Safe Mode directly" >&2 + exit 1 +} + +: > "$SINGULARITY_TEST_ATTEMPTS" +export SINGULARITY_SESSION_BUILD_ID="safe-mode-test-new-build" +bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "$EXPECTED" ] || { + echo "changed build did not retry normal mode" >&2 + exit 1 +} +grep -Fx "build_id=safe-mode-test-new-build" "$MARKER" >/dev/null + +: > "$SINGULARITY_TEST_ATTEMPTS" +export SINGULARITY_FORCE_NORMAL=1 +bash "$LAUNCHER" +unset SINGULARITY_FORCE_NORMAL +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "$EXPECTED" ] || { + echo "force-normal did not bypass the matching marker" >&2 + exit 1 +} + +rm -f "$MARKER" +: > "$SINGULARITY_TEST_ATTEMPTS" +SINGULARITY_SAFE_MODE=1 bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "1" ] || { + echo "explicit Safe Mode did not start directly" >&2 + exit 1 +} +[ ! -e "$MARKER" ] || { + echo "explicit Safe Mode unexpectedly created a persistent marker" >&2 + exit 1 +} + +: > "$SINGULARITY_TEST_ATTEMPTS" +export SINGULARITY_TEST_NORMAL_EXIT=0 +bash "$LAUNCHER" +unset SINGULARITY_TEST_NORMAL_EXIT +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "0" ] || { + echo "clean early exit unexpectedly retried" >&2 + exit 1 +} +[ ! -e "$MARKER" ] || { + echo "clean early exit unexpectedly entered Safe Mode" >&2 + exit 1 +} + +# The inner shell supervisor must escalate repeated failures even when each +# shell instance lives longer than the old three-second cutoff. +INNER_STATE="$TEST_DIR/inner-state" +mkdir -p "$INNER_STATE" +export XDG_STATE_HOME="$INNER_STATE" +export SINGULARITY_DESKTOP_BINARY="$0" +export SINGULARITY_TEST_FAKE_DESKTOP=1 +export SINGULARITY_TEST_SHELL_ATTEMPTS="$TEST_DIR/shell-attempts" +export SINGULARITY_SESSION_SUPERVISOR_ONLY=1 +DESKTOP_LAUNCHER="$(dirname "$0")/../src/singularity-desktop-session" +set +e +bash -c 'trap "" TERM; bash "$1" & child=$!; wait "$child"' _ "$DESKTOP_LAUNCHER" +set -e +[ "$(wc -l < "$SINGULARITY_TEST_SHELL_ATTEMPTS")" -eq 3 ] || { + echo "inner supervisor did not stop after its crash budget" >&2 + exit 1 +} +[ -f "$INNER_STATE/singularity/safe-mode-request" ] || { + echo "inner supervisor did not request Safe Mode" >&2 + exit 1 +} +[ "$(stat -c %a "$INNER_STATE/singularity/safe-mode-request")" = "600" ] +grep -Fx "build_id=safe-mode-test-new-build" \ + "$INNER_STATE/singularity/safe-mode-request" >/dev/null + +unset SINGULARITY_SESSION_SUPERVISOR_ONLY SINGULARITY_TEST_FAKE_DESKTOP +export SINGULARITY_TEST_ATTEMPTS="$TEST_DIR/request-attempts" +: > "$SINGULARITY_TEST_ATTEMPTS" +bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "1" ] || { + echo "matching shell-crash request did not start Safe Mode directly" >&2 + exit 1 +} + +# An update between the shell crash and the next launcher invocation must make +# the one-shot request stale, just like a persistent Safe Mode marker. +rm -f "$INNER_STATE/singularity/safe-mode" +printf 'version=1\nbuild_id=safe-mode-test-new-build\nreason=repeated-shell-failure\n' \ + > "$INNER_STATE/singularity/safe-mode-request" +chmod 600 "$INNER_STATE/singularity/safe-mode-request" +: > "$SINGULARITY_TEST_ATTEMPTS" +export SINGULARITY_SESSION_BUILD_ID="safe-mode-test-post-update" +bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "$EXPECTED" ] || { + echo "stale shell-crash request survived a build change" >&2 + exit 1 +} +grep -Fx "build_id=safe-mode-test-post-update" \ + "$INNER_STATE/singularity/safe-mode" >/dev/null