capture: probe the display before starting capture

internal/airplay/capture.go

@@ -76,13 +76,25 @@ func StartCapture(ctx context.Context, cfg CaptureConfig) (*ScreenCapture, error
 	if err := ValidateHWAccel(cfg.HWAccel); err != nil {
 		return nil, err
 	}
-	if (cfg.X11WindowID != 0 || cfg.X11WindowName != "") && os.Getenv("DISPLAY") != "" {
+	// A display variable that is set is not necessarily a display that exists:
+	// probe it so a stale variable fails here with a clear message instead of
+	// producing a silent black stream. See probeX11Display.
+	if display := os.Getenv("DISPLAY"); (cfg.X11WindowID != 0 || cfg.X11WindowName != "") && display != "" {
+		if err := probeX11Display(display); err != nil {
+			return nil, err
+		}
 		return startX11Capture(ctx, cfg)
 	}
-	if os.Getenv("WAYLAND_DISPLAY") != "" {
+	if display := os.Getenv("WAYLAND_DISPLAY"); display != "" {
+		if err := probeWaylandDisplay(display); err != nil {
+			return nil, err
+		}
 		return startWaylandCapture(ctx, cfg)
 	}
-	if os.Getenv("DISPLAY") != "" {
+	if display := os.Getenv("DISPLAY"); display != "" {
+		if err := probeX11Display(display); err != nil {
+			return nil, err
+		}
 		return startX11Capture(ctx, cfg)
 	}
 	return nil, fmt.Errorf("no display server detected (neither WAYLAND_DISPLAY nor DISPLAY is set)")

internal/airplay/display_probe.go

@@ -0,0 +1,93 @@
+package airplay
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// displayProbeTimeout bounds the reachability check so an unresponsive
+// compositor cannot stall startup.
+const displayProbeTimeout = 2 * time.Second
+
+// probeX11Display reports whether DISPLAY points at a reachable X server.
+//
+// A display variable that is merely set is not enough. When a desktop session
+// ends, the X (or Xwayland) socket is removed while DISPLAY survives in every
+// process that outlived the session. A lingering systemd user service is the
+// common case: it keeps the DISPLAY and XAUTHORITY it inherited at login, so
+// after logout it still advertises a display that no longer exists. Capture
+// then starts against nothing, GStreamer produces no frames, and the receiver
+// shows a black screen with no error reported anywhere.
+//
+// This only checks that the display endpoint accepts a connection. It does not
+// validate authentication: a reachable server that rejects the cookie still
+// fails later in GStreamer, with GStreamer's own message.
+func probeX11Display(display string) error {
+	network, address, err := x11Endpoint(display)
+	if err != nil {
+		return err
+	}
+	conn, err := net.DialTimeout(network, address, displayProbeTimeout)
+	if err != nil {
+		return fmt.Errorf("DISPLAY=%q is set but the X server at %s is not reachable: %w "+
+			"(a process that outlives its login session keeps a stale DISPLAY)", display, address, err)
+	}
+	return conn.Close()
+}
+
+// probeWaylandDisplay reports whether WAYLAND_DISPLAY points at a reachable
+// compositor. It has the same stale-variable failure mode as probeX11Display.
+func probeWaylandDisplay(display string) error {
+	address := display
+	if !filepath.IsAbs(address) {
+		runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
+		if runtimeDir == "" {
+			return fmt.Errorf("WAYLAND_DISPLAY=%q is set but XDG_RUNTIME_DIR is empty, "+
+				"so the compositor socket cannot be located", display)
+		}
+		address = filepath.Join(runtimeDir, address)
+	}
+	conn, err := net.DialTimeout("unix", address, displayProbeTimeout)
+	if err != nil {
+		return fmt.Errorf("WAYLAND_DISPLAY=%q is set but the compositor socket %s is not reachable: %w "+
+			"(a process that outlives its login session keeps a stale WAYLAND_DISPLAY)", display, address, err)
+	}
+	return conn.Close()
+}
+
+// x11Endpoint resolves a DISPLAY value to a dialable endpoint, following the
+// same unix-socket-or-TCP rule as Xlib.
+func x11Endpoint(display string) (network, address string, err error) {
+	spec := display
+	if slash := strings.Index(spec, "/"); slash >= 0 {
+		spec = spec[slash+1:] // drop an optional protocol prefix
+	}
+	colon := strings.LastIndex(spec, ":")
+	if colon < 0 {
+		return "", "", fmt.Errorf("DISPLAY=%q is not a valid display name", display)
+	}
+	host := spec[:colon]
+	number := spec[colon+1:]
+	if dot := strings.Index(number, "."); dot >= 0 {
+		number = number[:dot] // drop the screen suffix
+	}
+	n, convErr := strconv.Atoi(number)
+	if convErr != nil {
+		return "", "", fmt.Errorf("DISPLAY=%q has no usable display number", display)
+	}
+	if host == "" || host == "unix" {
+		return "unix", fmt.Sprintf("/tmp/.X11-unix/X%d", n), nil
+	}
+	// An IPv6 literal may already be bracketed in DISPLAY. JoinHostPort adds
+	// its own brackets to any host containing a colon, so leaving these on
+	// produces an undialable "[[::1]]:6000".
+	if len(host) > 1 && host[0] == '[' && host[len(host)-1] == ']' {
+		host = host[1 : len(host)-1]
+	}
+	return "tcp", net.JoinHostPort(host, strconv.Itoa(6000+n)), nil
+}

internal/airplay/display_probe_test.go

@@ -0,0 +1,63 @@
+package airplay
+
+import "testing"
+
+// x11Endpoint follows Xlib's rule for turning a DISPLAY value into something
+// dialable: an empty or "unix" host means the local socket, anything else is
+// TCP on 6000+N. The screen suffix and an optional protocol prefix are not
+// part of the endpoint.
+func TestX11Endpoint(t *testing.T) {
+	for _, test := range []struct {
+		display     string
+		wantNetwork string
+		wantAddress string
+	}{
+		{display: ":0", wantNetwork: "unix", wantAddress: "/tmp/.X11-unix/X0"},
+		{display: ":99", wantNetwork: "unix", wantAddress: "/tmp/.X11-unix/X99"},
+		// The screen suffix selects a screen on the same server, so it must
+		// not change which socket is dialled.
+		{display: ":0.0", wantNetwork: "unix", wantAddress: "/tmp/.X11-unix/X0"},
+		{display: ":99.1", wantNetwork: "unix", wantAddress: "/tmp/.X11-unix/X99"},
+		{display: "unix:0", wantNetwork: "unix", wantAddress: "/tmp/.X11-unix/X0"},
+		// A protocol prefix is stripped before the host is read; "local"
+		// leaves an empty host, which is still the unix socket.
+		{display: "local/unix:2", wantNetwork: "unix", wantAddress: "/tmp/.X11-unix/X2"},
+		{display: "host:0", wantNetwork: "tcp", wantAddress: "host:6000"},
+		{display: "host:12.0", wantNetwork: "tcp", wantAddress: "host:6012"},
+		{display: "192.168.1.5:1", wantNetwork: "tcp", wantAddress: "192.168.1.5:6001"},
+		// An IPv6 literal keeps its brackets through JoinHostPort, and the
+		// display number is taken from the LAST colon.
+		{display: "[::1]:0", wantNetwork: "tcp", wantAddress: "[::1]:6000"},
+	} {
+		t.Run(test.display, func(t *testing.T) {
+			network, address, err := x11Endpoint(test.display)
+			if err != nil {
+				t.Fatalf("x11Endpoint(%q) returned %v, want %s %s", test.display, err, test.wantNetwork, test.wantAddress)
+			}
+			if network != test.wantNetwork || address != test.wantAddress {
+				t.Fatalf("x11Endpoint(%q) = %s %s, want %s %s", test.display, network, address, test.wantNetwork, test.wantAddress)
+			}
+		})
+	}
+}
+
+// A DISPLAY that cannot be resolved must be reported rather than guessed at,
+// since guessing is what lets a stale variable start a capture against
+// nothing.
+func TestX11EndpointRejectsUnusableDisplay(t *testing.T) {
+	for _, display := range []string{
+		"",
+		"0",         // no colon at all
+		":",         // no display number
+		":abc",      // display number is not a number
+		"host:",     // host but no number
+		"host:xy.0", // screen suffix present, number still unusable
+	} {
+		t.Run(display, func(t *testing.T) {
+			network, address, err := x11Endpoint(display)
+			if err == nil {
+				t.Fatalf("x11Endpoint(%q) = %s %s, want an error", display, network, address)
+			}
+		})
+	}
+}