From e2c1f5fb64016c957af806436eb52283d4b9f518 Mon Sep 17 00:00:00 2001 From: mateoalfaro Date: Fri, 4 Sep 2026 13:33:30 -0600 Subject: [PATCH 1/2] feat: recover repeated session crashes in safe mode --- meson.build | 4 + src/singularity-desktop-session | 76 ++++++++++----- src/singularity-labwc-session | 161 +++++++++++++++++++++++++++----- tests/session_safe_mode_test.sh | 145 ++++++++++++++++++++++++++++ 4 files changed, 343 insertions(+), 43 deletions(-) create mode 100755 tests/session_safe_mode_test.sh diff --git a/meson.build b/meson.build index c7f6c75..e813f74 100644 --- a/meson.build +++ b/meson.build @@ -18,3 +18,7 @@ install_data( install_dir: get_option('datadir') / 'singularity' / 'labwc', install_mode: 'rwxr-xr-x', ) + +session_test_bash = find_program('bash') +test('session-safe-mode', session_test_bash, + args: files('tests/session_safe_mode_test.sh')) diff --git a/src/singularity-desktop-session b/src/singularity-desktop-session index 3870ef5..b003059 100755 --- a/src/singularity-desktop-session +++ b/src/singularity-desktop-session @@ -4,6 +4,7 @@ BIN="$(dirname "$SELF")" PREFIX="$(dirname "$BIN")" LIB="$PREFIX/lib" SHARE="$PREFIX/share" +_DESKTOP="${SINGULARITY_DESKTOP_BINARY:-$BIN/singularity-desktop}" export LD_LIBRARY_PATH="$LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" export PATH="$BIN:$PATH" @@ -24,6 +25,57 @@ fi _STATE="${XDG_STATE_HOME:-$HOME/.local/state}/singularity" mkdir -p "$_STATE" +_LOG="$_STATE/desktop.log" + +run_desktop_supervisor() { + _crashes=0 + _crash_window_started=0 + _crash_window_seconds=300 + _crash_limit=3 + while true; do + _t0=$SECONDS + "$_DESKTOP" >> "$_LOG" 2>&1 + _EC=$? + _dt=$((SECONDS - _t0)) + if [ $_EC -eq 0 ]; then + echo "[$(date)] singularity-desktop exited 0, restarting..." >> "$_LOG" + _crashes=0 + _crash_window_started=0 + sleep 0.2 + else + _now=$SECONDS + if [ $_crashes -eq 0 ] || [ $((_now - _crash_window_started)) -gt $_crash_window_seconds ]; then + _crashes=1 + _crash_window_started=$_now + else + _crashes=$((_crashes + 1)) + fi + if [ $_crashes -ge $_crash_limit ]; then + _request="$_STATE/safe-mode-request" + _request_tmp="${_request}.tmp.$$" + (umask 077; { + printf 'version=1\n' + printf 'build_id=%s\n' "${SINGULARITY_SESSION_BUILD_ID:-unknown}" + printf 'reason=repeated-shell-failure\n' + } > "$_request_tmp") && chmod 600 "$_request_tmp" && + mv -f "$_request_tmp" "$_request" + echo "[$(date)] singularity-desktop crashed $_crashes times within ${_crash_window_seconds}s; requesting Safe Mode." >> "$_LOG" + kill "$PPID" 2>/dev/null || true + return 1 + fi + echo "[$(date)] singularity-desktop exited $_EC after ${_dt}s (crash $_crashes/$_crash_limit), restarting in 2s..." >> "$_LOG" + sleep 2 + fi + done +} + +# Side-effect-free entry point for the supervisor black-box test. It must stay +# before process cleanup, helper startup, and user-service mutations below. +if [ "${SINGULARITY_SESSION_SUPERVISOR_ONLY:-0}" = "1" ]; then + run_desktop_supervisor + exit $? +fi + # labwc does not run /etc/xdg/autostart, so create the XDG user dirs here (idempotent, # locale-aware): Desktop/Documents/Downloads/... on the now-decrypted /home (FB-15). command -v xdg-user-dirs-update >/dev/null 2>&1 && xdg-user-dirs-update 2>/dev/null || true @@ -39,7 +91,6 @@ dbus-update-activation-environment \ GSETTINGS_SCHEMA_DIR XDG_DATA_DIRS GI_TYPELIB_PATH PATH LD_LIBRARY_PATH \ 2>/dev/null || true -_LOG="$_STATE/desktop.log" [ -f "$_LOG" ] && mv -f "$_LOG" "$_LOG.1" _SPID="${XDG_RUNTIME_DIR:-/tmp}/singularity-desktop-session.pid" if [ -f "$_SPID" ]; then @@ -66,24 +117,5 @@ fi # waiting for our portal backend, which would delay the shell launch below. systemctl --user restart --no-block xdg-desktop-portal 2>/dev/null || true -_fails=0 -while true; do - _t0=$SECONDS - "$BIN/singularity-desktop" >> "$_LOG" 2>&1 - _EC=$? - _dt=$((SECONDS - _t0)) - if [ $_EC -eq 0 ]; then - echo "[$(date)] singularity-desktop exited 0, restarting..." >> "$_LOG" - _fails=0 - sleep 0.2 - else - if [ $_dt -lt 3 ]; then _fails=$((_fails + 1)); else _fails=0; fi - if [ $_fails -ge 5 ]; then - echo "[$(date)] singularity-desktop failed fast $_fails times (compositor gone?), ending the session instead of looping." >> "$_LOG" - kill "$PPID" 2>/dev/null || true - exit 1 - fi - echo "[$(date)] singularity-desktop exited $_EC after ${_dt}s, restarting in 2s..." >> "$_LOG" - sleep 2 - fi -done +run_desktop_supervisor +exit $? diff --git a/src/singularity-labwc-session b/src/singularity-labwc-session index 5392140..b0444ad 100755 --- a/src/singularity-labwc-session +++ b/src/singularity-labwc-session @@ -26,38 +26,157 @@ done _STATE="${XDG_STATE_HOME:-$HOME/.local/state}/singularity" mkdir -p "$_STATE" _LLOG="$_STATE/labwc.log" +_SAFE_MARKER="$_STATE/safe-mode" +_SAFE_REQUEST="$_STATE/safe-mode-request" [ -f "$_LLOG" ] && mv -f "$_LLOG" "$_LLOG.1" +if [ -n "${SINGULARITY_LABWC_BINARY:-}" ]; then + _LABWC="$SINGULARITY_LABWC_BINARY" +else + # Keep this literal path as a packaging seam: the Nix package replaces it + # with the compositor from PATH. + _LABWC="$BIN/labwc" +fi +_DESKTOP_SESSION="${SINGULARITY_DESKTOP_SESSION_BINARY:-$BIN/singularity-desktop-session}" +_DESKTOP="${SINGULARITY_DESKTOP_BINARY:-$BIN/singularity-desktop}" +if [ -n "${SINGULARITY_SESSION_BUILD_ID:-}" ]; then + _BUILD_ID="$SINGULARITY_SESSION_BUILD_ID" +elif command -v sha256sum >/dev/null 2>&1; then + _LABWC_IDENTITY=$(command -v "$_LABWC" 2>/dev/null || printf '%s' "$_LABWC") + _DESKTOP_IDENTITY=$(command -v "$_DESKTOP_SESSION" 2>/dev/null || printf '%s' "$_DESKTOP_SESSION") + _SHELL_IDENTITY=$(command -v "$_DESKTOP" 2>/dev/null || printf '%s' "$_DESKTOP") + _BUILD_ID="$({ + for _binary in "$_LABWC_IDENTITY" "$_DESKTOP_IDENTITY" "$_SHELL_IDENTITY"; do + if [ -r "$_binary" ]; then + sha256sum "$_binary" + else + printf 'missing:%s\n' "$_binary" + fi + done + } | sha256sum | cut -d' ' -f1)" +else + _BUILD_ID="$PREFIX" +fi +export SINGULARITY_SESSION_BUILD_ID="$_BUILD_ID" + [ -r /etc/vconsole.conf ] && . /etc/vconsole.conf -[ -n "$XKBLAYOUT" ] && export XKB_DEFAULT_LAYOUT="$XKBLAYOUT" -[ -n "$XKBVARIANT" ] && export XKB_DEFAULT_VARIANT="$XKBVARIANT" +[ -n "${XKBLAYOUT:-}" ] && export XKB_DEFAULT_LAYOUT="$XKBLAYOUT" +[ -n "${XKBVARIANT:-}" ] && export XKB_DEFAULT_VARIANT="$XKBVARIANT" run_labwc() { - if [ -n "$GDM_SESSION_DBUS_ADDRESS" ] && [ -x "/usr/libexec/gdm-wayland-session" ]; then - /usr/libexec/gdm-wayland-session "$BIN/labwc" -S "$BIN/singularity-desktop-session" - elif [ -z "$DBUS_SESSION_BUS_ADDRESS" ] && [ ! -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/bus" ] && command -v dbus-run-session >/dev/null 2>&1; then - dbus-run-session -- "$BIN/labwc" -S "$BIN/singularity-desktop-session" + if [ -n "${GDM_SESSION_DBUS_ADDRESS:-}" ] && [ -x "/usr/libexec/gdm-wayland-session" ]; then + /usr/libexec/gdm-wayland-session "$_LABWC" -S "$_DESKTOP_SESSION" + elif [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ ! -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/bus" ] && command -v dbus-run-session >/dev/null 2>&1; then + dbus-run-session -- "$_LABWC" -S "$_DESKTOP_SESSION" else - "$BIN/labwc" -S "$BIN/singularity-desktop-session" + "$_LABWC" -S "$_DESKTOP_SESSION" + fi +} + +run_attempt() { + _ATTEMPT_LABEL="$1" + _t0=$SECONDS + run_labwc >> "$_LLOG" 2>&1 + _ATTEMPT_RC=$? + _ATTEMPT_DT=$((SECONDS - _t0)) + _ATTEMPT_SAFE_REQUEST=0 + if safe_request_matches_build; then + _ATTEMPT_SAFE_REQUEST=1 + fi + [ -f "$_SAFE_REQUEST" ] && rm -f "$_SAFE_REQUEST" + echo "[$(date)] $_ATTEMPT_LABEL exited rc=$_ATTEMPT_RC after ${_ATTEMPT_DT}s" >> "$_LLOG" +} + +is_recovery_failure() { + [ "${_ATTEMPT_SAFE_REQUEST:-0}" = "1" ] || + { [ "$1" -ne 0 ] && [ "$1" -ne 130 ] && [ "$1" -ne 143 ] && [ "$2" -lt 30 ]; } +} + +marker_matches_build() { + [ -f "$_SAFE_MARKER" ] || return 1 + _marker_build=$(sed -n 's/^build_id=//p' "$_SAFE_MARKER" 2>/dev/null | head -n 1) + [ -n "$_marker_build" ] && [ "$_marker_build" = "$_BUILD_ID" ] +} + +safe_request_matches_build() { + [ -f "$_SAFE_REQUEST" ] || return 1 + _request_build=$(sed -n 's/^build_id=//p' "$_SAFE_REQUEST" 2>/dev/null | head -n 1) + [ -n "$_request_build" ] && [ "$_request_build" = "$_BUILD_ID" ] +} + +write_safe_marker() { + _failed_rc="$1" + _failed_dt="$2" + _marker_tmp="${_SAFE_MARKER}.tmp.$$" + if (umask 077; { + echo "version=1" + echo "build_id=$_BUILD_ID" + echo "reason=repeated-session-failure" + echo "exit_status=$_failed_rc" + echo "elapsed_seconds=$_failed_dt" + echo "timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } > "$_marker_tmp") && chmod 600 "$_marker_tmp" && mv -f "$_marker_tmp" "$_SAFE_MARKER"; then + return 0 fi + echo "[$(date)] could not persist Safe Mode marker at $_SAFE_MARKER" >> "$_LLOG" + rm -f "$_marker_tmp" + return 1 } -# Try the hardware renderer first (GSK default gl, labwc default drm/gl). If the -# session dies within a few seconds it is an early crash on the real GL/GPU path -# (invisible under QEMU's software path), so fall back to software rendering: the -# desktop comes up anyway, degraded but alive, and the crash stays in the log for -# diagnosis (#78). singularity-desktop-session honours an exported GSK_RENDERER. -_t0=$SECONDS -run_labwc >> "$_LLOG" 2>&1 -_rc=$? -_dt=$((SECONDS - _t0)) -if [ "$_dt" -lt 30 ]; then - echo "[$(date)] labwc session exited rc=$_rc after ${_dt}s; retrying with software rendering (GSK_RENDERER=cairo, WLR_RENDERER=pixman)" >> "$_LLOG" +enable_software_rendering() { export GSK_RENDERER=cairo export WLR_RENDERER=pixman export LIBGL_ALWAYS_SOFTWARE=1 export GALLIUM_DRIVER=llvmpipe - run_labwc >> "$_LLOG" 2>&1 - _rc=$? +} + +run_safe_mode() { + enable_software_rendering + export SINGULARITY_SAFE_MODE=1 + export SINGULARITY_SAFE_MODE_MARKER="$_SAFE_MARKER" + export SINGULARITY_SAFE_MODE_REASON="Repeated startup failures" + echo "[$(date)] starting Safe Mode; persisted settings remain writable" >> "$_LLOG" + run_attempt "Safe Mode session" +} + +# An explicit manual Safe Mode request does not need a marker. +if [ "${SINGULARITY_SAFE_MODE:-0}" = "1" ]; then + run_safe_mode + exit "$_ATTEMPT_RC" +fi + +# If the outer supervisor was terminated before it could consume an inner +# shell-crash request, honor that request on the next display-manager launch. +if safe_request_matches_build; then + rm -f "$_SAFE_REQUEST" + write_safe_marker 1 0 || true + run_safe_mode + exit "$_ATTEMPT_RC" fi -exit $_rc +[ -f "$_SAFE_REQUEST" ] && rm -f "$_SAFE_REQUEST" + +# A marker only applies to the binaries that created it. After an update, try +# normal mode again so a fixed build cannot remain trapped in recovery mode. +if [ "${SINGULARITY_FORCE_NORMAL:-0}" != "1" ] && marker_matches_build; then + run_safe_mode + exit "$_ATTEMPT_RC" +fi +[ -f "$_SAFE_MARKER" ] && rm -f "$_SAFE_MARKER" + +unset SINGULARITY_SAFE_MODE SINGULARITY_SAFE_MODE_MARKER SINGULARITY_SAFE_MODE_REASON +run_attempt "normal compositor session" +if ! is_recovery_failure "$_ATTEMPT_RC" "$_ATTEMPT_DT"; then + exit "$_ATTEMPT_RC" +fi + +# Preserve the existing renderer fallback as the second normal-mode attempt. +echo "[$(date)] early compositor failure; retrying with software rendering (GSK_RENDERER=cairo, WLR_RENDERER=pixman)" >> "$_LLOG" +enable_software_rendering +run_attempt "software compositor session" +if ! is_recovery_failure "$_ATTEMPT_RC" "$_ATTEMPT_DT"; then + exit "$_ATTEMPT_RC" +fi + +write_safe_marker "$_ATTEMPT_RC" "$_ATTEMPT_DT" || true +run_safe_mode +exit "$_ATTEMPT_RC" diff --git a/tests/session_safe_mode_test.sh b/tests/session_safe_mode_test.sh new file mode 100755 index 0000000..7310bff --- /dev/null +++ b/tests/session_safe_mode_test.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +set -eu + +if [ "${1:-}" = "-S" ]; then + printf '%s\n' "${SINGULARITY_SAFE_MODE:-0}" >> "$SINGULARITY_TEST_ATTEMPTS" + if [ "${SINGULARITY_SAFE_MODE:-0}" = "1" ]; then + exit 0 + fi + exit "${SINGULARITY_TEST_NORMAL_EXIT:-134}" +fi + +if [ "${SINGULARITY_TEST_FAKE_DESKTOP:-0}" = "1" ]; then + printf 'shell\n' >> "$SINGULARITY_TEST_SHELL_ATTEMPTS" + exit 1 +fi + +TEST_DIR=$(mktemp -d "${TMPDIR:-/tmp}/singularity-session-test.XXXXXX") +trap 'rm -rf "$TEST_DIR"' EXIT +mkdir -p "$TEST_DIR/state" "$TEST_DIR/runtime" +chmod 700 "$TEST_DIR/runtime" + +export XDG_STATE_HOME="$TEST_DIR/state" +export XDG_RUNTIME_DIR="$TEST_DIR/runtime" +export DBUS_SESSION_BUS_ADDRESS="test-bus" +export GDM_SESSION_DBUS_ADDRESS="" +export SINGULARITY_LABWC_BINARY="$0" +export SINGULARITY_DESKTOP_SESSION_BINARY="$0" +export SINGULARITY_SESSION_BUILD_ID="safe-mode-test-build" +export SINGULARITY_TEST_ATTEMPTS="$TEST_DIR/attempts" + +LAUNCHER="$(dirname "$0")/../src/singularity-labwc-session" +bash "$LAUNCHER" + +EXPECTED=$(printf '0\n0\n1') +ACTUAL=$(cat "$SINGULARITY_TEST_ATTEMPTS") +[ "$ACTUAL" = "$EXPECTED" ] || { + echo "unexpected first-launch attempts: $ACTUAL" >&2 + exit 1 +} + +MARKER="$XDG_STATE_HOME/singularity/safe-mode" +[ -f "$MARKER" ] +[ "$(stat -c %a "$MARKER")" = "600" ] +grep -Fx "build_id=safe-mode-test-build" "$MARKER" >/dev/null +grep -Fx "reason=repeated-session-failure" "$MARKER" >/dev/null + +: > "$SINGULARITY_TEST_ATTEMPTS" +bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "1" ] || { + echo "matching marker did not start Safe Mode directly" >&2 + exit 1 +} + +: > "$SINGULARITY_TEST_ATTEMPTS" +export SINGULARITY_SESSION_BUILD_ID="safe-mode-test-new-build" +bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "$EXPECTED" ] || { + echo "changed build did not retry normal mode" >&2 + exit 1 +} +grep -Fx "build_id=safe-mode-test-new-build" "$MARKER" >/dev/null + +: > "$SINGULARITY_TEST_ATTEMPTS" +export SINGULARITY_FORCE_NORMAL=1 +bash "$LAUNCHER" +unset SINGULARITY_FORCE_NORMAL +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "$EXPECTED" ] || { + echo "force-normal did not bypass the matching marker" >&2 + exit 1 +} + +rm -f "$MARKER" +: > "$SINGULARITY_TEST_ATTEMPTS" +SINGULARITY_SAFE_MODE=1 bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "1" ] || { + echo "explicit Safe Mode did not start directly" >&2 + exit 1 +} +[ ! -e "$MARKER" ] || { + echo "explicit Safe Mode unexpectedly created a persistent marker" >&2 + exit 1 +} + +: > "$SINGULARITY_TEST_ATTEMPTS" +export SINGULARITY_TEST_NORMAL_EXIT=0 +bash "$LAUNCHER" +unset SINGULARITY_TEST_NORMAL_EXIT +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "0" ] || { + echo "clean early exit unexpectedly retried" >&2 + exit 1 +} +[ ! -e "$MARKER" ] || { + echo "clean early exit unexpectedly entered Safe Mode" >&2 + exit 1 +} + +# The inner shell supervisor must escalate repeated failures even when each +# shell instance lives longer than the old three-second cutoff. +INNER_STATE="$TEST_DIR/inner-state" +mkdir -p "$INNER_STATE" +export XDG_STATE_HOME="$INNER_STATE" +export SINGULARITY_DESKTOP_BINARY="$0" +export SINGULARITY_TEST_FAKE_DESKTOP=1 +export SINGULARITY_TEST_SHELL_ATTEMPTS="$TEST_DIR/shell-attempts" +export SINGULARITY_SESSION_SUPERVISOR_ONLY=1 +DESKTOP_LAUNCHER="$(dirname "$0")/../src/singularity-desktop-session" +set +e +bash -c 'trap "" TERM; bash "$1" & child=$!; wait "$child"' _ "$DESKTOP_LAUNCHER" +set -e +[ "$(wc -l < "$SINGULARITY_TEST_SHELL_ATTEMPTS")" -eq 3 ] || { + echo "inner supervisor did not stop after its crash budget" >&2 + exit 1 +} +[ -f "$INNER_STATE/singularity/safe-mode-request" ] || { + echo "inner supervisor did not request Safe Mode" >&2 + exit 1 +} +[ "$(stat -c %a "$INNER_STATE/singularity/safe-mode-request")" = "600" ] +grep -Fx "build_id=safe-mode-test-new-build" \ + "$INNER_STATE/singularity/safe-mode-request" >/dev/null + +unset SINGULARITY_SESSION_SUPERVISOR_ONLY SINGULARITY_TEST_FAKE_DESKTOP +export SINGULARITY_TEST_ATTEMPTS="$TEST_DIR/request-attempts" +: > "$SINGULARITY_TEST_ATTEMPTS" +bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "1" ] || { + echo "matching shell-crash request did not start Safe Mode directly" >&2 + exit 1 +} + +# An update between the shell crash and the next launcher invocation must make +# the one-shot request stale, just like a persistent Safe Mode marker. +rm -f "$INNER_STATE/singularity/safe-mode" +printf 'version=1\nbuild_id=safe-mode-test-new-build\nreason=repeated-shell-failure\n' \ + > "$INNER_STATE/singularity/safe-mode-request" +chmod 600 "$INNER_STATE/singularity/safe-mode-request" +: > "$SINGULARITY_TEST_ATTEMPTS" +export SINGULARITY_SESSION_BUILD_ID="safe-mode-test-post-update" +bash "$LAUNCHER" +[ "$(cat "$SINGULARITY_TEST_ATTEMPTS")" = "$EXPECTED" ] || { + echo "stale shell-crash request survived a build change" >&2 + exit 1 +} +grep -Fx "build_id=safe-mode-test-post-update" \ + "$INNER_STATE/singularity/safe-mode" >/dev/null From e88e8720bf92447159353f11c00693769777148b Mon Sep 17 00:00:00 2001 From: mateoalfaro Date: Fri, 4 Sep 2026 14:30:28 -0600 Subject: [PATCH 2/2] fix: keep hardware rendering in safe mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_safe_mode() forced the software rendering stack (GSK_RENDERER=cairo, WLR_RENDERER=pixman, llvmpipe). On machines with a working GPU the recovery session then renders the entire desktop on the CPU, maxing out cores and making the session unusably slow (observed: libinput event lag, glamor fallback in labwc.log) — the opposite of what a recovery mode should do. In the escalation ladder the software path has already failed by the time safe mode starts, so forcing it there cannot help either. Safe mode now runs the default rendering stack. Operators who suspect the GPU stack itself can opt back in with SINGULARITY_SAFE_MODE_SOFTWARE=1. --- src/singularity-labwc-session | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/singularity-labwc-session b/src/singularity-labwc-session index b0444ad..439b931 100755 --- a/src/singularity-labwc-session +++ b/src/singularity-labwc-session @@ -131,7 +131,18 @@ enable_software_rendering() { } run_safe_mode() { - enable_software_rendering + # Safe mode suppresses optional shell features; it must NOT force the + # software rendering stack. On machines with a working GPU, rendering the + # whole session on the CPU maxes out cores and makes the recovery session + # unusably slow (libinput lag, crawl), and in the escalation ladder the + # software path has already failed by the time safe mode starts. Set + # SINGULARITY_SAFE_MODE_SOFTWARE=1 to force it when the GPU stack itself + # is the suspected culprit. + if [ "${SINGULARITY_SAFE_MODE_SOFTWARE:-0}" = "1" ]; then + enable_software_rendering + else + unset GSK_RENDERER WLR_RENDERER LIBGL_ALWAYS_SOFTWARE GALLIUM_DRIVER + fi export SINGULARITY_SAFE_MODE=1 export SINGULARITY_SAFE_MODE_MARKER="$_SAFE_MARKER" export SINGULARITY_SAFE_MODE_REASON="Repeated startup failures"