From 3cb555f0cbbe7311aaf3cdead41934fec082d3ad Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:12:28 +0000 Subject: [PATCH 1/9] Add explicit display backend configuration --- server/cmd/wrapper/main.go | 11 ++++- server/lib/display/backend.go | 63 +++++++++++++++++++++++++ server/lib/display/backend_test.go | 76 ++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 server/lib/display/backend.go create mode 100644 server/lib/display/backend_test.go diff --git a/server/cmd/wrapper/main.go b/server/cmd/wrapper/main.go index 37b70e3a..fb3b7101 100644 --- a/server/cmd/wrapper/main.go +++ b/server/cmd/wrapper/main.go @@ -23,6 +23,8 @@ import ( "strings" "syscall" "time" + + "github.com/kernel/kernel-images/server/lib/display" ) const ( @@ -59,9 +61,16 @@ func profileName(p profile) string { func main() { t0 := time.Now() + displayConfig, err := display.FromEnv() + if err != nil { + fatalf("display backend configuration: %v", err) + } + if displayConfig.Backend != display.BackendX11 { + fatalf("display backend %q is not implemented in this image", displayConfig.Backend) + } prof := detectProfile() stzManaged := scaleToZeroManaged() - logf("starting wrapper (profile=%s stz=%s)", profileName(prof), stzMode(stzManaged)) + logf("starting wrapper (profile=%s display_backend=%s stz=%s)", profileName(prof), displayConfig.Backend, stzMode(stzManaged)) forkIdentityWait, err := forkIdentityWaitEnabled() if err != nil { fatalf("fork identity config: %v", err) diff --git a/server/lib/display/backend.go b/server/lib/display/backend.go new file mode 100644 index 00000000..d7465a22 --- /dev/null +++ b/server/lib/display/backend.go @@ -0,0 +1,63 @@ +// Package display contains configuration shared by the display lifecycle and +// browser launcher. Backend selection is explicit so a failed Wayland setup +// cannot silently continue on X11. +package display + +import ( + "fmt" + "os" + "strings" +) + +// Backend identifies the display protocol used by the headful image. +type Backend string + +const ( + BackendX11 Backend = "x11" + BackendWayland Backend = "wayland" +) + +// Config contains the process environment needed to connect to a display. +type Config struct { + Backend Backend + XDisplay string + WaylandDisplay string + RuntimeDir string +} + +// FromEnv reads display backend configuration. X11 remains the default for +// compatibility with existing images and callers. +func FromEnv() (Config, error) { + backend, err := ParseBackend(os.Getenv("DISPLAY_BACKEND")) + if err != nil { + return Config{}, err + } + + config := Config{ + Backend: backend, + XDisplay: strings.TrimSpace(os.Getenv("DISPLAY")), + WaylandDisplay: strings.TrimSpace(os.Getenv("WAYLAND_DISPLAY")), + RuntimeDir: strings.TrimSpace(os.Getenv("XDG_RUNTIME_DIR")), + } + if config.XDisplay == "" { + config.XDisplay = ":1" + } + if config.WaylandDisplay == "" { + config.WaylandDisplay = "wayland-0" + } + return config, nil +} + +// ParseBackend parses DISPLAY_BACKEND. An empty value selects X11. +func ParseBackend(value string) (Backend, error) { + backend := Backend(strings.ToLower(strings.TrimSpace(value))) + if backend == "" { + return BackendX11, nil + } + switch backend { + case BackendX11, BackendWayland: + return backend, nil + default: + return "", fmt.Errorf("unsupported DISPLAY_BACKEND %q (expected x11 or wayland)", value) + } +} diff --git a/server/lib/display/backend_test.go b/server/lib/display/backend_test.go new file mode 100644 index 00000000..e1fb2dbd --- /dev/null +++ b/server/lib/display/backend_test.go @@ -0,0 +1,76 @@ +package display + +import ( + "os" + "testing" +) + +func TestParseBackend(t *testing.T) { + tests := []struct { + name string + value string + want Backend + ok bool + }{ + {name: "empty defaults to x11", want: BackendX11, ok: true}, + {name: "x11", value: "x11", want: BackendX11, ok: true}, + {name: "wayland", value: "wayland", want: BackendWayland, ok: true}, + {name: "case and whitespace", value: " WayLand ", want: BackendWayland, ok: true}, + {name: "unknown", value: "mir", ok: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseBackend(tt.value) + if tt.ok { + if err != nil { + t.Fatalf("ParseBackend() error = %v", err) + } + if got != tt.want { + t.Fatalf("ParseBackend() = %q, want %q", got, tt.want) + } + return + } + if err == nil { + t.Fatal("ParseBackend() error = nil, want error") + } + }) + } +} + +func TestFromEnvDefaultsAndOverrides(t *testing.T) { + env := map[string]string{ + "DISPLAY_BACKEND": "wayland", + "DISPLAY": ":7", + "WAYLAND_DISPLAY": "wayland-2", + "XDG_RUNTIME_DIR": "/run/user/1000", + } + for key, value := range env { + t.Setenv(key, value) + } + + got, err := FromEnv() + if err != nil { + t.Fatalf("FromEnv() error = %v", err) + } + if got.Backend != BackendWayland || got.XDisplay != ":7" || got.WaylandDisplay != "wayland-2" || got.RuntimeDir != "/run/user/1000" { + t.Fatalf("FromEnv() = %#v", got) + } +} + +func TestFromEnvDefaults(t *testing.T) { + for _, key := range []string{"DISPLAY_BACKEND", "DISPLAY", "WAYLAND_DISPLAY", "XDG_RUNTIME_DIR"} { + t.Setenv(key, "") + } + + got, err := FromEnv() + if err != nil { + t.Fatalf("FromEnv() error = %v", err) + } + want := Config{Backend: BackendX11, XDisplay: ":1", WaylandDisplay: "wayland-0"} + if got != want { + t.Fatalf("FromEnv() = %#v, want %#v", got, want) + } + + _ = os.Unsetenv("DISPLAY_BACKEND") +} From b2e85a8871a18469ffbe6ca6ae01b65c71dd93cf Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:44:50 +0000 Subject: [PATCH 2/9] Add native Wayland startup and browser controls --- images/chromium-headful/Dockerfile | 1 + images/chromium-headful/run-docker.sh | 2 + images/chromium-headful/run-unikernel.sh | 2 + .../supervisor/services/weston.conf | 8 + server/cmd/api/api/api.go | 7 + server/cmd/api/api/computer.go | 18 ++ server/cmd/api/api/display.go | 61 ++++++ server/cmd/api/api/wayland_input.go | 145 ++++++++++++++ server/cmd/chromium-launcher/display.go | 80 ++++++++ server/cmd/chromium-launcher/main.go | 40 ++-- server/cmd/chromium-launcher/main_test.go | 52 +++++ server/cmd/wrapper/main.go | 57 +++++- server/lib/cdpclient/cdpclient.go | 177 ++++++++++++++++++ 13 files changed, 622 insertions(+), 28 deletions(-) create mode 100644 images/chromium-headful/supervisor/services/weston.conf create mode 100644 server/cmd/api/api/wayland_input.go create mode 100644 server/cmd/chromium-launcher/display.go diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index 57586296..c393ae16 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -189,6 +189,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=$CACHEIDPREFIX-ap imagemagick \ sudo \ mutter \ + weston \ curl \ # Network tools net-tools \ diff --git a/images/chromium-headful/run-docker.sh b/images/chromium-headful/run-docker.sh index 9e5af0ae..fdd59046 100755 --- a/images/chromium-headful/run-docker.sh +++ b/images/chromium-headful/run-docker.sh @@ -12,6 +12,7 @@ mkdir -p "$HOST_RECORDINGS_DIR" # RUN_AS_ROOT defaults to false in docker RUN_AS_ROOT="${RUN_AS_ROOT:-false}" +DISPLAY_BACKEND="${DISPLAY_BACKEND:-x11}" # Build Chromium flags file and mount CHROMIUM_FLAGS_DEFAULT="--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=*" @@ -65,6 +66,7 @@ RUN_ARGS=( -e WIDTH=1920 -e TZ=${TZ:-'America/Los_Angeles'} -e RUN_AS_ROOT="$RUN_AS_ROOT" + -e DISPLAY_BACKEND="$DISPLAY_BACKEND" --mount type=bind,src="$FLAGS_FILE",dst=/chromium/flags ) diff --git a/images/chromium-headful/run-unikernel.sh b/images/chromium-headful/run-unikernel.sh index 54feb23d..2c6e2f46 100755 --- a/images/chromium-headful/run-unikernel.sh +++ b/images/chromium-headful/run-unikernel.sh @@ -19,6 +19,7 @@ volume_name="${NAME}-flags" # volume which we then mount into the image at /chromium. # RUN_AS_ROOT defaults to true in unikernel (for now, until we figure it out) RUN_AS_ROOT="${RUN_AS_ROOT:-true}" +DISPLAY_BACKEND="${DISPLAY_BACKEND:-x11}" chromium_flags_default="--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=*" if [[ "$RUN_AS_ROOT" == "true" ]]; then @@ -76,6 +77,7 @@ deploy_args=( -e WIDTH=1920 -e TZ=${TZ:-'America/Los_Angeles'} -e RUN_AS_ROOT="$RUN_AS_ROOT" + -e DISPLAY_BACKEND="$DISPLAY_BACKEND" -e LOG_CDP_MESSAGES=true -v "$volume_name":/chromium -n "$NAME" diff --git a/images/chromium-headful/supervisor/services/weston.conf b/images/chromium-headful/supervisor/services/weston.conf new file mode 100644 index 00000000..2e6e86fb --- /dev/null +++ b/images/chromium-headful/supervisor/services/weston.conf @@ -0,0 +1,8 @@ +[program:weston] +command=/bin/bash -c 'exec weston --backend=headless-backend.so --use-pixman --debug --socket="${WAYLAND_DISPLAY:-wayland-0}" --width="${WIDTH:-1920}" --height="${HEIGHT:-1080}" --idle-time=0' +autostart=false +autorestart=true +startsecs=0 +stdout_logfile=/var/log/supervisord/weston +redirect_stderr=true +user=kernel diff --git a/server/cmd/api/api/api.go b/server/cmd/api/api/api.go index 2fd6580b..98828fa6 100644 --- a/server/cmd/api/api/api.go +++ b/server/cmd/api/api/api.go @@ -12,6 +12,7 @@ import ( "github.com/kernel/kernel-images/server/lib/cdpmonitor" "github.com/kernel/kernel-images/server/lib/devtoolsproxy" + "github.com/kernel/kernel-images/server/lib/display" "github.com/kernel/kernel-images/server/lib/events" "github.com/kernel/kernel-images/server/lib/logger" "github.com/kernel/kernel-images/server/lib/nekoclient" @@ -56,6 +57,7 @@ type ApiService struct { // Neko authenticated client nekoAuthClient *nekoclient.AuthClient + displayBackend display.Backend // DevTools upstream manager (Chromium supervisord log tailer) upstreamMgr *devtoolsproxy.UpstreamManager @@ -143,6 +145,10 @@ func New( screenshotEnabled := func() bool { return telemetrySession.CategoryEnabled(events.Screenshot) } mon := cdpmonitor.New(upstreamMgr, telemetrySession.Publish, displayNum, slog.Default(), screenshotEnabled) + displayConfig, err := display.FromEnv() + if err != nil { + return nil, fmt.Errorf("display backend configuration: %w", err) + } ctx, cancel := context.WithCancel(context.Background()) return &ApiService{ @@ -154,6 +160,7 @@ func New( upstreamMgr: upstreamMgr, stz: stz, nekoAuthClient: nekoAuthClient, + displayBackend: displayConfig.Backend, policy: &policy.Policy{}, eventStream: eventStream, telemetrySession: telemetrySession, diff --git a/server/cmd/api/api/computer.go b/server/cmd/api/api/computer.go index a7c4a3c3..3d37581a 100644 --- a/server/cmd/api/api/computer.go +++ b/server/cmd/api/api/computer.go @@ -38,6 +38,9 @@ func isValidationErr(err error) bool { } func (s *ApiService) doMoveMouse(ctx context.Context, body oapi.MoveMouseRequest) error { + if s.usesWayland() { + return s.doMoveMouseWayland(ctx, body) + } log := logger.FromContext(ctx) // Get current resolution for bounds validation @@ -213,6 +216,9 @@ func (s *ApiService) getMouseLocation(ctx context.Context) (x, y int, err error) } func (s *ApiService) doClickMouse(ctx context.Context, body oapi.ClickMouseRequest) error { + if s.usesWayland() { + return s.doClickMouseWayland(ctx, body) + } log := logger.FromContext(ctx) // Get current resolution for bounds validation @@ -341,6 +347,9 @@ func (s *ApiService) TakeScreenshot(ctx context.Context, request oapi.TakeScreen if request.Body != nil { body = *request.Body } + if s.usesWayland() { + return s.takeWaylandScreenshot(ctx, body) + } // Get current resolution for bounds validation screenWidth, screenHeight, _, err := s.getCurrentResolution(ctx) @@ -448,6 +457,9 @@ func (s *ApiService) TakeScreenshot(ctx context.Context, request oapi.TakeScreen } func (s *ApiService) doTypeText(ctx context.Context, body oapi.TypeTextRequest) error { + if s.usesWayland() { + return s.doTypeTextWayland(ctx, body) + } useSmooth := body.Smooth == nil || *body.Smooth if useSmooth { return s.doTypeTextSmooth(ctx, body) @@ -830,6 +842,9 @@ func (s *ApiService) GetMousePosition(ctx context.Context, request oapi.GetMouse } func (s *ApiService) doPressKey(ctx context.Context, body oapi.PressKeyRequest) error { + if s.usesWayland() { + return s.doPressKeyWayland(ctx, body) + } log := logger.FromContext(ctx) if len(body.Keys) == 0 { @@ -940,6 +955,9 @@ func (s *ApiService) PressKey(ctx context.Context, request oapi.PressKeyRequestO } func (s *ApiService) doScroll(ctx context.Context, body oapi.ScrollRequest) error { + if s.usesWayland() { + return s.doScrollWayland(ctx, body) + } log := logger.FromContext(ctx) // Validate deltas diff --git a/server/cmd/api/api/display.go b/server/cmd/api/api/display.go index 285ae8ff..870a9e03 100644 --- a/server/cmd/api/api/display.go +++ b/server/cmd/api/api/display.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "context" "encoding/base64" "fmt" @@ -13,6 +14,7 @@ import ( "time" "github.com/kernel/kernel-images/server/lib/cdpclient" + "github.com/kernel/kernel-images/server/lib/display" "github.com/kernel/kernel-images/server/lib/logger" oapi "github.com/kernel/kernel-images/server/lib/oapi" "github.com/kernel/kernel-images/server/lib/recorder" @@ -30,6 +32,9 @@ func (s *ApiService) PatchDisplay(ctx context.Context, req oapi.PatchDisplayRequ } func (s *ApiService) patchDisplayLocked(ctx context.Context, req oapi.PatchDisplayRequestObject) (oapi.PatchDisplayResponseObject, error) { + if s.usesWayland() { + return s.patchWaylandDisplay(ctx, req) + } log := logger.FromContext(ctx) if req.Body == nil { @@ -198,6 +203,62 @@ func (s *ApiService) patchDisplayLocked(ctx context.Context, req oapi.PatchDispl }, nil } +func (s *ApiService) usesWayland() bool { + return s.displayBackend == display.BackendWayland +} + +func (s *ApiService) patchWaylandDisplay(ctx context.Context, req oapi.PatchDisplayRequestObject) (oapi.PatchDisplayResponseObject, error) { + if req.Body == nil { + return oapi.PatchDisplay400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "missing request body"}}, nil + } + if req.Body.Width == nil && req.Body.Height == nil { + return oapi.PatchDisplay400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "no display parameters to update"}}, nil + } + if req.Body.Width == nil || req.Body.Height == nil || *req.Body.Width <= 0 || *req.Body.Height <= 0 { + return oapi.PatchDisplay400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "Wayland resize requires positive width and height"}}, nil + } + if err := s.setViewportViaCDP(ctx, *req.Body.Width, *req.Body.Height); err != nil { + return oapi.PatchDisplay500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: fmt.Sprintf("failed to resize Wayland viewport: %v", err)}}, nil + } + refreshRate := 60 + if req.Body.RefreshRate != nil { + refreshRate = int(*req.Body.RefreshRate) + } + return oapi.PatchDisplay200JSONResponse{ + Width: req.Body.Width, + Height: req.Body.Height, + RefreshRate: &refreshRate, + }, nil +} + +func (s *ApiService) takeWaylandScreenshot(ctx context.Context, body oapi.ScreenshotRequest) (oapi.TakeScreenshotResponseObject, error) { + var clip *cdpclient.ScreenshotClip + if body.Region != nil { + region := body.Region + if region.X < 0 || region.Y < 0 || region.Width <= 0 || region.Height <= 0 { + return oapi.TakeScreenshot400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "invalid region dimensions"}}, nil + } + clip = &cdpclient.ScreenshotClip{ + X: float64(region.X), + Y: float64(region.Y), + Width: float64(region.Width), + Height: float64(region.Height), + Scale: 1, + } + } + + var pngBytes []byte + err := s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { + var err error + pngBytes, err = client.CaptureScreenshot(cdpCtx, clip) + return err + }) + if err != nil { + return oapi.TakeScreenshot500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: fmt.Sprintf("failed to capture Wayland screenshot: %v", err)}}, nil + } + return oapi.TakeScreenshot200ImagepngResponse{Body: bytes.NewReader(pngBytes), ContentLength: int64(len(pngBytes))}, nil +} + // resolveDisplayParams merges the request body with the current display // state, returning the final width, height, and refresh rate plus whether // any field would actually change. Callers use the changed flag to skip diff --git a/server/cmd/api/api/wayland_input.go b/server/cmd/api/api/wayland_input.go new file mode 100644 index 00000000..6347b1f2 --- /dev/null +++ b/server/cmd/api/api/wayland_input.go @@ -0,0 +1,145 @@ +package api + +import ( + "context" + "fmt" + + "github.com/kernel/kernel-images/server/lib/cdpclient" + oapi "github.com/kernel/kernel-images/server/lib/oapi" +) + +func (s *ApiService) waylandViewport(ctx context.Context) (int, int, error) { + var size cdpclient.ViewportSize + err := s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { + var err error + size, err = client.GetViewportSize(cdpCtx) + return err + }) + if err != nil { + return 0, 0, fmt.Errorf("get Wayland viewport: %w", err) + } + if size.Width <= 0 || size.Height <= 0 { + return 0, 0, fmt.Errorf("invalid Wayland viewport %dx%d", size.Width, size.Height) + } + return size.Width, size.Height, nil +} + +func validateWaylandPoint(x, y, width, height int) error { + if x < 0 || y < 0 { + return &validationError{msg: "coordinates must be non-negative"} + } + if x >= width || y >= height { + return &validationError{msg: fmt.Sprintf("coordinates exceed screen bounds (max: %dx%d)", width-1, height-1)} + } + return nil +} + +func (s *ApiService) dispatchWaylandMouse(ctx context.Context, eventType string, x, y int, button string, clickCount int) error { + return s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { + return client.DispatchMouseEvent(cdpCtx, eventType, float64(x), float64(y), button, clickCount) + }) +} + +func (s *ApiService) doMoveMouseWayland(ctx context.Context, body oapi.MoveMouseRequest) error { + width, height, err := s.waylandViewport(ctx) + if err != nil { + return &executionError{msg: err.Error()} + } + if err := validateWaylandPoint(body.X, body.Y, width, height); err != nil { + return err + } + if err := s.dispatchWaylandMouse(ctx, "mouseMoved", body.X, body.Y, "", 0); err != nil { + return &executionError{msg: fmt.Sprintf("failed to move Wayland pointer: %v", err)} + } + return nil +} + +func (s *ApiService) doClickMouseWayland(ctx context.Context, body oapi.ClickMouseRequest) error { + width, height, err := s.waylandViewport(ctx) + if err != nil { + return &executionError{msg: err.Error()} + } + if err := validateWaylandPoint(body.X, body.Y, width, height); err != nil { + return err + } + button := "left" + if body.Button != nil { + button = map[oapi.ClickMouseRequestButton]string{ + oapi.ClickMouseRequestButtonLeft: "left", + oapi.ClickMouseRequestButtonMiddle: "middle", + oapi.ClickMouseRequestButtonRight: "right", + oapi.ClickMouseRequestButtonBack: "back", + oapi.ClickMouseRequestButtonForward: "forward", + }[*body.Button] + if button == "" { + return &validationError{msg: fmt.Sprintf("unsupported button: %s", *body.Button)} + } + } + clickCount := 1 + if body.NumClicks != nil && *body.NumClicks > 0 { + clickCount = *body.NumClicks + } + if body.ClickType != nil && *body.ClickType != oapi.Click { + return &validationError{msg: "Wayland input currently supports click only"} + } + if err := s.dispatchWaylandMouse(ctx, "mousePressed", body.X, body.Y, button, clickCount); err != nil { + return &executionError{msg: fmt.Sprintf("failed to press Wayland pointer: %v", err)} + } + if err := s.dispatchWaylandMouse(ctx, "mouseReleased", body.X, body.Y, button, clickCount); err != nil { + return &executionError{msg: fmt.Sprintf("failed to release Wayland pointer: %v", err)} + } + return nil +} + +func (s *ApiService) doTypeTextWayland(ctx context.Context, body oapi.TypeTextRequest) error { + return s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { + if err := client.InsertText(cdpCtx, body.Text); err != nil { + return &executionError{msg: fmt.Sprintf("failed to type text through Wayland: %v", err)} + } + return nil + }) +} + +func (s *ApiService) doPressKeyWayland(ctx context.Context, body oapi.PressKeyRequest) error { + if len(body.Keys) == 0 { + return &validationError{msg: "keys must contain at least one key symbol"} + } + for _, key := range body.Keys { + code := key + if len([]rune(key)) == 1 { + code = "Key" + key + } + if err := s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { + if err := client.DispatchKeyEvent(cdpCtx, "keyDown", key, code, ""); err != nil { + return err + } + return client.DispatchKeyEvent(cdpCtx, "keyUp", key, code, "") + }); err != nil { + return &executionError{msg: fmt.Sprintf("failed to press Wayland key %q: %v", key, err)} + } + } + return nil +} + +func (s *ApiService) doScrollWayland(ctx context.Context, body oapi.ScrollRequest) error { + width, height, err := s.waylandViewport(ctx) + if err != nil { + return &executionError{msg: err.Error()} + } + if err := validateWaylandPoint(body.X, body.Y, width, height); err != nil { + return err + } + deltaX, deltaY := 0, 0 + if body.DeltaX != nil { + deltaX = *body.DeltaX + } + if body.DeltaY != nil { + deltaY = *body.DeltaY + } + if deltaX == 0 && deltaY == 0 { + return &validationError{msg: "at least one of delta_x or delta_y must be non-zero"} + } + return s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { + return client.DispatchMouseWheel(cdpCtx, float64(body.X), float64(body.Y), deltaX, deltaY) + }) +} diff --git a/server/cmd/chromium-launcher/display.go b/server/cmd/chromium-launcher/display.go new file mode 100644 index 00000000..be983c75 --- /dev/null +++ b/server/cmd/chromium-launcher/display.go @@ -0,0 +1,80 @@ +package main + +import ( + "fmt" + "net" + "path/filepath" + "time" + + "github.com/kernel/kernel-images/server/lib/display" +) + +func chromiumDisplaySetup(config display.Config) (flags, env []string, err error) { + switch config.Backend { + case display.BackendX11: + return nil, []string{"DISPLAY=" + config.XDisplay}, nil + case display.BackendWayland: + if config.RuntimeDir == "" { + return nil, nil, fmt.Errorf("XDG_RUNTIME_DIR is required for Wayland") + } + return []string{ + "--enable-features=UseOzonePlatform", + "--ozone-platform=wayland", + }, []string{ + "WAYLAND_DISPLAY=" + config.WaylandDisplay, + "XDG_RUNTIME_DIR=" + config.RuntimeDir, + }, nil + default: + return nil, nil, fmt.Errorf("unsupported display backend %q", config.Backend) + } +} + +func displayConfigFromEnv() (display.Config, []string, []string, error) { + config, err := display.FromEnv() + if err != nil { + return display.Config{}, nil, nil, err + } + flags, env, err := chromiumDisplaySetup(config) + if err != nil { + return display.Config{}, nil, nil, err + } + return config, flags, env, nil +} + +func waitForDisplay(config display.Config, timeout time.Duration) time.Duration { + if config.Backend == display.BackendX11 { + return waitForXDisplay(config.XDisplay, timeout) + } + return waitForWayland(config, timeout) +} + +func waitForXDisplay(displayName string, timeout time.Duration) time.Duration { + start := time.Now() + num := displayName + if len(num) > 0 && num[0] == ':' { + num = num[1:] + } + named := "/tmp/.X11-unix/X" + num + abstract := "@/tmp/.X11-unix/X" + num + return waitForUnixSocket(start, []string{named, abstract}, timeout) +} + +func waitForWayland(config display.Config, timeout time.Duration) time.Duration { + start := time.Now() + socket := filepath.Join(config.RuntimeDir, config.WaylandDisplay) + return waitForUnixSocket(start, []string{socket}, timeout) +} + +func waitForUnixSocket(start time.Time, sockets []string, timeout time.Duration) time.Duration { + deadline := start.Add(timeout) + for time.Now().Before(deadline) { + for _, socket := range sockets { + if conn, err := net.DialTimeout("unix", socket, 200*time.Millisecond); err == nil { + _ = conn.Close() + return time.Since(start) + } + } + time.Sleep(20 * time.Millisecond) + } + return time.Since(start) +} diff --git a/server/cmd/chromium-launcher/main.go b/server/cmd/chromium-launcher/main.go index 7bce8506..b3f00f80 100644 --- a/server/cmd/chromium-launcher/main.go +++ b/server/cmd/chromium-launcher/main.go @@ -12,6 +12,7 @@ import ( "time" "github.com/kernel/kernel-images/server/lib/chromiumflags" + "github.com/kernel/kernel-images/server/lib/display" "github.com/kernel/kernel-images/server/lib/x11" ) @@ -56,19 +57,21 @@ func main() { // Wait for devtools port to be available (handles SIGKILL socket cleanup delay) waitForPort(internalPort, 5*time.Second) - // Wait for the X server. The wrapper starts chromium in parallel with - // xorg/xvfb, so the display socket may not be ready yet — without this - // gate chromium would fail on connect and supervisord would restart us. - if d := x11.WaitForDisplay(":1", 20*time.Second); d >= 20*time.Second { - fmt.Fprintf(os.Stderr, "warning: X display :1 not responsive after %s\n", d) + displayConfig, displayFlags, displayEnv, err := displayConfigFromEnv() + if err != nil { + fmt.Fprintf(os.Stderr, "display backend configuration failed: %v\n", err) + os.Exit(1) + } + + // Wait for the selected display server. The wrapper starts the display + // service in parallel with Chromium, so the socket may not be ready yet. + if d := waitForDisplay(displayConfig, 20*time.Second); d >= 20*time.Second { + fmt.Fprintf(os.Stderr, "warning: %s display is not responsive after %s\n", displayConfig.Backend, d) } - // Headful: wait for mutter to register before exec'ing chromium. If - // chromium maps its window with no WM present, the CSD hint it sends has - // no listener; mutter starts later, reparents the existing window, and - // applies default SSD — i.e., the titlebar with the close X. Headless - // has no WM, so skip. - if !*headless { + // Headful X11 needs Mutter to register before Chromium maps its window so + // Chromium's client-side decoration negotiation is handled correctly. + if !*headless && displayConfig.Backend == display.BackendX11 { if d := x11.WaitForMutter(20 * time.Second); d >= 20*time.Second { fmt.Fprintf(os.Stderr, "warning: mutter not registered after %s\n", d) } @@ -82,6 +85,7 @@ func main() { } final := chromiumflags.MergeFlagsWithRuntimeTokens(baseFlags, runtimeTokens) final = withDefaultPrivateNetworkBypass(final) + final = append(final, displayFlags...) // Diagnostics for parity with previous scripts fmt.Printf("BASE_FLAGS: %s\n", baseFlags) @@ -107,8 +111,8 @@ func main() { // recorder's sink; the root path below relies on this inherited env, while the // non-root path re-asserts them in its runuser env allowlist. env := os.Environ() + env = append(env, displayEnv...) env = append(env, - "DISPLAY=:1", "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/dbus/system_bus_socket", "PULSE_SERVER="+pulseServer, "PULSE_SINK="+pulseSink, @@ -142,17 +146,17 @@ func main() { // daemon playback lands on: Chromium's AudioManagerPulse honors it to redirect // playback into KernelOutput (see media/audio/pulse/audio_manager_pulse.cc // GetDefaultOutputDeviceID), which is the sink the recorder captures. - inner := []string{ - "env", - "DISPLAY=:1", + inner := []string{"env"} + inner = append(inner, displayEnv...) + inner = append(inner, "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/dbus/system_bus_socket", - "PULSE_SERVER=" + pulseServer, - "PULSE_SINK=" + pulseSink, + "PULSE_SERVER="+pulseServer, + "PULSE_SINK="+pulseSink, "XDG_CONFIG_HOME=/home/kernel/.config", "XDG_CACHE_HOME=/home/kernel/.cache", "HOME=/home/kernel", *chromiumPath, - } + ) inner = append(inner, chromiumArgs...) argv := append([]string{filepath.Base(runuserPath), "-u", "kernel", "--"}, inner...) if err := syscall.Exec(runuserPath, argv, env); err != nil { diff --git a/server/cmd/chromium-launcher/main_test.go b/server/cmd/chromium-launcher/main_test.go index 6e6d11b0..3b3bcf98 100644 --- a/server/cmd/chromium-launcher/main_test.go +++ b/server/cmd/chromium-launcher/main_test.go @@ -8,8 +8,60 @@ import ( "testing" "github.com/kernel/kernel-images/server/lib/chromiumflags" + "github.com/kernel/kernel-images/server/lib/display" ) +func TestChromiumDisplaySetup(t *testing.T) { + tests := []struct { + name string + config display.Config + wantFlags []string + wantEnv []string + wantErr bool + }{ + { + name: "x11", + config: display.Config{Backend: display.BackendX11, XDisplay: ":7"}, + wantEnv: []string{"DISPLAY=:7"}, + }, + { + name: "wayland", + config: display.Config{Backend: display.BackendWayland, WaylandDisplay: "wayland-2", RuntimeDir: "/run/user/1000"}, + wantFlags: []string{ + "--enable-features=UseOzonePlatform", + "--ozone-platform=wayland", + }, + wantEnv: []string{"WAYLAND_DISPLAY=wayland-2", "XDG_RUNTIME_DIR=/run/user/1000"}, + }, + { + name: "wayland requires runtime directory", + config: display.Config{Backend: display.BackendWayland, WaylandDisplay: "wayland-0"}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + flags, env, err := chromiumDisplaySetup(tt.config) + if tt.wantErr { + if err == nil { + t.Fatal("chromiumDisplaySetup() error = nil, want error") + } + return + } + if err != nil { + t.Fatalf("chromiumDisplaySetup() error = %v", err) + } + if !reflect.DeepEqual(flags, tt.wantFlags) { + t.Fatalf("flags = %#v, want %#v", flags, tt.wantFlags) + } + if !reflect.DeepEqual(env, tt.wantEnv) { + t.Fatalf("env = %#v, want %#v", env, tt.wantEnv) + } + }) + } +} + func TestWithDefaultPrivateNetworkBypass(t *testing.T) { tests := []struct { name string diff --git a/server/cmd/wrapper/main.go b/server/cmd/wrapper/main.go index fb3b7101..43ef2b67 100644 --- a/server/cmd/wrapper/main.go +++ b/server/cmd/wrapper/main.go @@ -65,10 +65,31 @@ func main() { if err != nil { fatalf("display backend configuration: %v", err) } - if displayConfig.Backend != display.BackendX11 { - fatalf("display backend %q is not implemented in this image", displayConfig.Backend) - } prof := detectProfile() + if displayConfig.Backend == display.BackendWayland { + if prof == profileHeadless { + fatalf("Wayland display backend requires the headful image") + } + if os.Getenv("ENABLE_WEBRTC") == "true" { + fatalf("Wayland display backend does not support ENABLE_WEBRTC yet") + } + if displayConfig.RuntimeDir == "" { + displayConfig.RuntimeDir = "/tmp/runtime-kernel" + _ = os.Setenv("XDG_RUNTIME_DIR", displayConfig.RuntimeDir) + } + if err := os.MkdirAll(displayConfig.RuntimeDir, 0o700); err != nil { + fatalf("create Wayland runtime directory: %v", err) + } + if err := os.Chmod(displayConfig.RuntimeDir, 0o700); err != nil { + fatalf("secure Wayland runtime directory: %v", err) + } + // Weston runs as the non-root browser user. The image creates that user + // with uid 1000, so the runtime directory must be owned by it before + // supervisor starts the compositor. + if err := os.Chown(displayConfig.RuntimeDir, 1000, 1000); err != nil { + fatalf("assign Wayland runtime directory: %v", err) + } + } stzManaged := scaleToZeroManaged() logf("starting wrapper (profile=%s display_backend=%s stz=%s)", profileName(prof), displayConfig.Backend, stzMode(stzManaged)) forkIdentityWait, err := forkIdentityWaitEnabled() @@ -128,7 +149,15 @@ func main() { startLogAggregator() // Default env that downstream services expect. - _ = os.Setenv("DISPLAY", defaultDisplay) + if displayConfig.Backend == display.BackendX11 { + _ = os.Setenv("DISPLAY", defaultDisplay) + } else { + _ = os.Unsetenv("DISPLAY") + _ = os.Setenv("WAYLAND_DISPLAY", displayConfig.WaylandDisplay) + } + if displayConfig.RuntimeDir != "" { + _ = os.Setenv("XDG_RUNTIME_DIR", displayConfig.RuntimeDir) + } if os.Getenv("INTERNAL_PORT") == "" { _ = os.Setenv("INTERNAL_PORT", defaultIntPort) } @@ -140,9 +169,13 @@ func main() { // which would otherwise spam autolaunch errors). _ = os.Setenv("DBUS_SESSION_BUS_ADDRESS", "unix:path="+dbusSocket) - // Stale X locks from prior runs. - _ = os.Remove("/tmp/.X1-lock") - _ = os.Remove("/tmp/.X11-unix/X1") + // Stale X/Wayland sockets from prior runs. + if displayConfig.Backend == display.BackendX11 { + _ = os.Remove("/tmp/.X1-lock") + _ = os.Remove("/tmp/.X11-unix/X1") + } else { + _ = os.Remove(filepath.Join(displayConfig.RuntimeDir, displayConfig.WaylandDisplay)) + } // supervisord — start in nodaemon mode so we own its lifecycle. // Without -n it forks and the parent exits with code 0, which would @@ -184,8 +217,10 @@ func main() { xServer := "xorg" if prof == profileHeadless { xServer = "xvfb" + } else if displayConfig.Backend == display.BackendWayland { + xServer = "weston" } - webrtc := prof == profileHeadful && os.Getenv("ENABLE_WEBRTC") == "true" + webrtc := prof == profileHeadful && displayConfig.Backend == display.BackendX11 && os.Getenv("ENABLE_WEBRTC") == "true" // Pre-touch chromium's supervisord log so kernel-images-api's `tail -f` // doesn't bail out and enter its 250ms retry backoff when started in @@ -194,8 +229,10 @@ func main() { browserStart := time.Now() startAll(xServer, "dbus", "chromedriver", "pulseaudio") - waitForX(defaultDisplay, 20*time.Second) - if prof == profileHeadful { + if displayConfig.Backend == display.BackendX11 { + waitForX(defaultDisplay, 20*time.Second) + } + if prof == profileHeadful && displayConfig.Backend == display.BackendX11 { startAll("mutter") } waitForSocket(pulseSocket, 10*time.Second) diff --git a/server/lib/cdpclient/cdpclient.go b/server/lib/cdpclient/cdpclient.go index ac1449d7..d3eccef1 100644 --- a/server/lib/cdpclient/cdpclient.go +++ b/server/lib/cdpclient/cdpclient.go @@ -2,6 +2,7 @@ package cdpclient import ( "context" + "encoding/base64" "encoding/json" "fmt" "net/http" @@ -263,6 +264,182 @@ func (c *Client) CountPageTargets(ctx context.Context) (int, error) { return n, nil } +// ViewportSize is the CSS pixel size of the first page target. +type ViewportSize struct { + Width int `json:"width"` + Height int `json:"height"` +} + +// GetViewportSize returns the first page target's current viewport size. +func (c *Client) GetViewportSize(ctx context.Context) (ViewportSize, error) { + sessionID, detach, err := c.attachFirstPage(ctx) + if err != nil { + return ViewportSize{}, err + } + defer detach() + + raw, err := c.send(ctx, "Runtime.evaluate", map[string]any{ + "expression": "JSON.stringify({width: window.innerWidth, height: window.innerHeight})", + "returnByValue": true, + }, sessionID) + if err != nil { + return ViewportSize{}, fmt.Errorf("Runtime.evaluate viewport size: %w", err) + } + var evaluated struct { + Result struct { + Value string `json:"value"` + } `json:"result"` + } + if err := json.Unmarshal(raw, &evaluated); err != nil { + return ViewportSize{}, fmt.Errorf("unmarshal viewport size: %w", err) + } + var size ViewportSize + if err := json.Unmarshal([]byte(evaluated.Result.Value), &size); err != nil { + return ViewportSize{}, fmt.Errorf("decode viewport size: %w", err) + } + return size, nil +} + +// DispatchMouseEvent sends a mouse event to the first page target. +func (c *Client) DispatchMouseEvent(ctx context.Context, eventType string, x, y float64, button string, clickCount int) error { + sessionID, detach, err := c.attachFirstPage(ctx) + if err != nil { + return err + } + defer detach() + + params := map[string]any{"type": eventType, "x": x, "y": y} + if button != "" { + params["button"] = button + } + if clickCount > 0 { + params["clickCount"] = clickCount + } + if _, err := c.send(ctx, "Input.dispatchMouseEvent", params, sessionID); err != nil { + return fmt.Errorf("Input.dispatchMouseEvent: %w", err) + } + return nil +} + +// DispatchMouseWheel sends a wheel event to the first page target. +func (c *Client) DispatchMouseWheel(ctx context.Context, x, y float64, deltaX, deltaY int) error { + sessionID, detach, err := c.attachFirstPage(ctx) + if err != nil { + return err + } + defer detach() + _, err = c.send(ctx, "Input.dispatchMouseEvent", map[string]any{ + "type": "mouseWheel", + "x": x, + "y": y, + "deltaX": deltaX, + "deltaY": deltaY, + }, sessionID) + if err != nil { + return fmt.Errorf("Input.dispatchMouseEvent wheel: %w", err) + } + return nil +} + +// InsertText inserts text into the focused element of the first page target. +func (c *Client) InsertText(ctx context.Context, text string) error { + sessionID, detach, err := c.attachFirstPage(ctx) + if err != nil { + return err + } + defer detach() + if _, err := c.send(ctx, "Input.insertText", map[string]string{"text": text}, sessionID); err != nil { + return fmt.Errorf("Input.insertText: %w", err) + } + return nil +} + +// DispatchKeyEvent sends a key event to the first page target. +func (c *Client) DispatchKeyEvent(ctx context.Context, eventType, key, code, text string) error { + sessionID, detach, err := c.attachFirstPage(ctx) + if err != nil { + return err + } + defer detach() + params := map[string]any{"type": eventType, "key": key, "code": code} + if text != "" { + params["text"] = text + } + if _, err := c.send(ctx, "Input.dispatchKeyEvent", params, sessionID); err != nil { + return fmt.Errorf("Input.dispatchKeyEvent: %w", err) + } + return nil +} + +// ScreenshotClip describes an optional page screenshot crop in CSS pixels. +type ScreenshotClip struct { + X float64 `json:"x"` + Y float64 `json:"y"` + Width float64 `json:"width"` + Height float64 `json:"height"` + Scale float64 `json:"scale,omitempty"` +} + +// CaptureScreenshot captures the first page target as a PNG. It uses CDP +// rather than an X11 grab so native Wayland pages can be captured too. +func (c *Client) CaptureScreenshot(ctx context.Context, clip *ScreenshotClip) ([]byte, error) { + sessionID, detach, err := c.attachFirstPage(ctx) + if err != nil { + return nil, err + } + defer detach() + + params := map[string]any{"format": "png"} + if clip != nil { + params["clip"] = clip + } + raw, err := c.send(ctx, "Page.captureScreenshot", params, sessionID) + if err != nil { + return nil, fmt.Errorf("Page.captureScreenshot: %w", err) + } + var result struct { + Data string `json:"data"` + } + if err := json.Unmarshal(raw, &result); err != nil { + return nil, fmt.Errorf("unmarshal Page.captureScreenshot: %w", err) + } + data, err := base64.StdEncoding.DecodeString(result.Data) + if err != nil { + return nil, fmt.Errorf("decode Page.captureScreenshot: %w", err) + } + return data, nil +} + +func (c *Client) attachFirstPage(ctx context.Context) (string, func(), error) { + targetID, err := c.firstPageTargetID(ctx) + if err != nil { + return "", nil, err + } + attachResult, err := c.send(ctx, "Target.attachToTarget", map[string]any{ + "targetId": targetID, + "flatten": true, + }, "") + if err != nil { + return "", nil, fmt.Errorf("Target.attachToTarget: %w", err) + } + var attach struct { + SessionID string `json:"sessionId"` + } + if err := json.Unmarshal(attachResult, &attach); err != nil { + return "", nil, fmt.Errorf("unmarshal attach: %w", err) + } + if attach.SessionID == "" { + return "", nil, fmt.Errorf("Target.attachToTarget returned no session ID") + } + return attach.SessionID, func() { + detachCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, _ = c.send(detachCtx, "Target.detachFromTarget", map[string]any{ + "sessionId": attach.SessionID, + }, "") + }, nil +} + // DispatchStartURL closes extra page targets and dispatches a navigation on the // first page target. It does not wait for lifecycle events; Chrome owns the // eventual navigation result. From 2bc6c3241bcae810f1e0a95e33faa083841d8868 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:48:05 +0000 Subject: [PATCH 3/9] Capture Wayland telemetry screenshots through CDP --- server/cmd/api/api/api.go | 15 +++++++++++++++ server/lib/cdpmonitor/monitor.go | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/server/cmd/api/api/api.go b/server/cmd/api/api/api.go index 98828fa6..d451f1ec 100644 --- a/server/cmd/api/api/api.go +++ b/server/cmd/api/api/api.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/kernel/kernel-images/server/lib/cdpclient" "github.com/kernel/kernel-images/server/lib/cdpmonitor" "github.com/kernel/kernel-images/server/lib/devtoolsproxy" "github.com/kernel/kernel-images/server/lib/display" @@ -149,6 +150,20 @@ func New( if err != nil { return nil, fmt.Errorf("display backend configuration: %w", err) } + if displayConfig.Backend == display.BackendWayland { + mon.SetScreenshotFunc(func(ctx context.Context, _ int) ([]byte, error) { + upstream := upstreamMgr.Current() + if upstream == "" { + return nil, fmt.Errorf("devtools upstream not available") + } + client, err := cdpclient.Dial(ctx, upstream) + if err != nil { + return nil, err + } + defer client.Close() + return client.CaptureScreenshot(ctx, nil) + }) + } ctx, cancel := context.WithCancel(context.Background()) return &ApiService{ diff --git a/server/lib/cdpmonitor/monitor.go b/server/lib/cdpmonitor/monitor.go index 9ae4dacb..ac969600 100644 --- a/server/lib/cdpmonitor/monitor.go +++ b/server/lib/cdpmonitor/monitor.go @@ -105,6 +105,12 @@ func New(upstreamMgr UpstreamProvider, publish PublishFunc, displayNum int, log return m } +// SetScreenshotFunc replaces the platform screenshot implementation. This is +// used by native Wayland, where an X11 grab cannot capture the compositor. +func (m *Monitor) SetScreenshotFunc(fn func(context.Context, int) ([]byte, error)) { + m.screenshotFn = fn +} + // IsRunning reports whether the monitor is actively capturing. func (m *Monitor) IsRunning() bool { return m.running.Load() From e2bbb21c277e616193422c3c11b012e7f5ddff50 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:49:48 +0000 Subject: [PATCH 4/9] Route Wayland drag actions through CDP --- server/cmd/api/api/computer.go | 3 ++ server/cmd/api/api/wayland_input.go | 43 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/server/cmd/api/api/computer.go b/server/cmd/api/api/computer.go index 3d37581a..469bc794 100644 --- a/server/cmd/api/api/computer.go +++ b/server/cmd/api/api/computer.go @@ -1041,6 +1041,9 @@ func (s *ApiService) Scroll(ctx context.Context, request oapi.ScrollRequestObjec } func (s *ApiService) doDragMouse(ctx context.Context, body oapi.DragMouseRequest) error { + if s.usesWayland() { + return s.doDragMouseWayland(ctx, body) + } log := logger.FromContext(ctx) if len(body.Path) < 2 { diff --git a/server/cmd/api/api/wayland_input.go b/server/cmd/api/api/wayland_input.go index 6347b1f2..0b7c2e42 100644 --- a/server/cmd/api/api/wayland_input.go +++ b/server/cmd/api/api/wayland_input.go @@ -143,3 +143,46 @@ func (s *ApiService) doScrollWayland(ctx context.Context, body oapi.ScrollReques return client.DispatchMouseWheel(cdpCtx, float64(body.X), float64(body.Y), deltaX, deltaY) }) } + +func (s *ApiService) doDragMouseWayland(ctx context.Context, body oapi.DragMouseRequest) error { + if len(body.Path) < 2 { + return &validationError{msg: "path must contain at least two points"} + } + width, height, err := s.waylandViewport(ctx) + if err != nil { + return &executionError{msg: err.Error()} + } + for _, point := range body.Path { + if len(point) != 2 { + return &validationError{msg: "path points must be [x,y]"} + } + if err := validateWaylandPoint(point[0], point[1], width, height); err != nil { + return err + } + } + button := "left" + if body.Button != nil { + button = map[oapi.DragMouseRequestButton]string{ + oapi.DragMouseRequestButtonLeft: "left", + oapi.DragMouseRequestButtonMiddle: "middle", + oapi.DragMouseRequestButtonRight: "right", + }[*body.Button] + if button == "" { + return &validationError{msg: fmt.Sprintf("unsupported button: %s", *body.Button)} + } + } + start := body.Path[0] + if err := s.dispatchWaylandMouse(ctx, "mousePressed", start[0], start[1], button, 1); err != nil { + return &executionError{msg: fmt.Sprintf("failed to press Wayland pointer: %v", err)} + } + for _, point := range body.Path[1:] { + if err := s.dispatchWaylandMouse(ctx, "mouseMoved", point[0], point[1], "", 0); err != nil { + _ = s.dispatchWaylandMouse(context.Background(), "mouseReleased", point[0], point[1], button, 1) + return &executionError{msg: fmt.Sprintf("failed during Wayland drag: %v", err)} + } + } + if err := s.dispatchWaylandMouse(ctx, "mouseReleased", body.Path[len(body.Path)-1][0], body.Path[len(body.Path)-1][1], button, 1); err != nil { + return &executionError{msg: fmt.Sprintf("failed to release Wayland pointer: %v", err)} + } + return nil +} From 3c10497e8478c93d3818dc9c1603073e252a6a32 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:51:20 +0000 Subject: [PATCH 5/9] Reuse CDP sessions for Wayland input sequences --- server/cmd/api/api/wayland_input.go | 38 ++++++++++++++--------- server/lib/cdpclient/cdpclient.go | 48 ++++++++++++++++++++++------- 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/server/cmd/api/api/wayland_input.go b/server/cmd/api/api/wayland_input.go index 0b7c2e42..811a95fe 100644 --- a/server/cmd/api/api/wayland_input.go +++ b/server/cmd/api/api/wayland_input.go @@ -35,8 +35,18 @@ func validateWaylandPoint(x, y, width, height int) error { } func (s *ApiService) dispatchWaylandMouse(ctx context.Context, eventType string, x, y int, button string, clickCount int) error { + return s.dispatchWaylandMouseEvents(ctx, []cdpclient.MouseEvent{{ + Type: eventType, + X: float64(x), + Y: float64(y), + Button: button, + ClickCount: clickCount, + }}) +} + +func (s *ApiService) dispatchWaylandMouseEvents(ctx context.Context, events []cdpclient.MouseEvent) error { return s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { - return client.DispatchMouseEvent(cdpCtx, eventType, float64(x), float64(y), button, clickCount) + return client.DispatchMouseEvents(cdpCtx, events) }) } @@ -82,11 +92,11 @@ func (s *ApiService) doClickMouseWayland(ctx context.Context, body oapi.ClickMou if body.ClickType != nil && *body.ClickType != oapi.Click { return &validationError{msg: "Wayland input currently supports click only"} } - if err := s.dispatchWaylandMouse(ctx, "mousePressed", body.X, body.Y, button, clickCount); err != nil { - return &executionError{msg: fmt.Sprintf("failed to press Wayland pointer: %v", err)} - } - if err := s.dispatchWaylandMouse(ctx, "mouseReleased", body.X, body.Y, button, clickCount); err != nil { - return &executionError{msg: fmt.Sprintf("failed to release Wayland pointer: %v", err)} + if err := s.dispatchWaylandMouseEvents(ctx, []cdpclient.MouseEvent{ + {Type: "mousePressed", X: float64(body.X), Y: float64(body.Y), Button: button, ClickCount: clickCount}, + {Type: "mouseReleased", X: float64(body.X), Y: float64(body.Y), Button: button, ClickCount: clickCount}, + }); err != nil { + return &executionError{msg: fmt.Sprintf("failed to click through Wayland: %v", err)} } return nil } @@ -171,18 +181,16 @@ func (s *ApiService) doDragMouseWayland(ctx context.Context, body oapi.DragMouse return &validationError{msg: fmt.Sprintf("unsupported button: %s", *body.Button)} } } + events := make([]cdpclient.MouseEvent, 0, len(body.Path)+1) start := body.Path[0] - if err := s.dispatchWaylandMouse(ctx, "mousePressed", start[0], start[1], button, 1); err != nil { - return &executionError{msg: fmt.Sprintf("failed to press Wayland pointer: %v", err)} - } + events = append(events, cdpclient.MouseEvent{Type: "mousePressed", X: float64(start[0]), Y: float64(start[1]), Button: button, ClickCount: 1}) for _, point := range body.Path[1:] { - if err := s.dispatchWaylandMouse(ctx, "mouseMoved", point[0], point[1], "", 0); err != nil { - _ = s.dispatchWaylandMouse(context.Background(), "mouseReleased", point[0], point[1], button, 1) - return &executionError{msg: fmt.Sprintf("failed during Wayland drag: %v", err)} - } + events = append(events, cdpclient.MouseEvent{Type: "mouseMoved", X: float64(point[0]), Y: float64(point[1])}) } - if err := s.dispatchWaylandMouse(ctx, "mouseReleased", body.Path[len(body.Path)-1][0], body.Path[len(body.Path)-1][1], button, 1); err != nil { - return &executionError{msg: fmt.Sprintf("failed to release Wayland pointer: %v", err)} + end := body.Path[len(body.Path)-1] + events = append(events, cdpclient.MouseEvent{Type: "mouseReleased", X: float64(end[0]), Y: float64(end[1]), Button: button, ClickCount: 1}) + if err := s.dispatchWaylandMouseEvents(ctx, events); err != nil { + return &executionError{msg: fmt.Sprintf("failed during Wayland drag: %v", err)} } return nil } diff --git a/server/lib/cdpclient/cdpclient.go b/server/lib/cdpclient/cdpclient.go index d3eccef1..72340cf5 100644 --- a/server/lib/cdpclient/cdpclient.go +++ b/server/lib/cdpclient/cdpclient.go @@ -300,27 +300,53 @@ func (c *Client) GetViewportSize(ctx context.Context) (ViewportSize, error) { return size, nil } -// DispatchMouseEvent sends a mouse event to the first page target. -func (c *Client) DispatchMouseEvent(ctx context.Context, eventType string, x, y float64, button string, clickCount int) error { +// MouseEvent describes one CDP mouse event. +type MouseEvent struct { + Type string + X float64 + Y float64 + Button string + ClickCount int +} + +// DispatchMouseEvents sends mouse events to the first page target while +// reusing one attached CDP session for the whole sequence. +func (c *Client) DispatchMouseEvents(ctx context.Context, events []MouseEvent) error { + if len(events) == 0 { + return nil + } sessionID, detach, err := c.attachFirstPage(ctx) if err != nil { return err } defer detach() - params := map[string]any{"type": eventType, "x": x, "y": y} - if button != "" { - params["button"] = button - } - if clickCount > 0 { - params["clickCount"] = clickCount - } - if _, err := c.send(ctx, "Input.dispatchMouseEvent", params, sessionID); err != nil { - return fmt.Errorf("Input.dispatchMouseEvent: %w", err) + for _, event := range events { + params := map[string]any{"type": event.Type, "x": event.X, "y": event.Y} + if event.Button != "" { + params["button"] = event.Button + } + if event.ClickCount > 0 { + params["clickCount"] = event.ClickCount + } + if _, err := c.send(ctx, "Input.dispatchMouseEvent", params, sessionID); err != nil { + return fmt.Errorf("Input.dispatchMouseEvent: %w", err) + } } return nil } +// DispatchMouseEvent sends one mouse event to the first page target. +func (c *Client) DispatchMouseEvent(ctx context.Context, eventType string, x, y float64, button string, clickCount int) error { + return c.DispatchMouseEvents(ctx, []MouseEvent{{ + Type: eventType, + X: x, + Y: y, + Button: button, + ClickCount: clickCount, + }}) +} + // DispatchMouseWheel sends a wheel event to the first page target. func (c *Client) DispatchMouseWheel(ctx context.Context, x, y float64, deltaX, deltaY int) error { sessionID, detach, err := c.attachFirstPage(ctx) From 793b3fb030d644ffdbcd85f99506b7bc714e3860 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:51:58 +0000 Subject: [PATCH 6/9] Reuse CDP sessions for Wayland key sequences --- server/cmd/api/api/wayland_input.go | 18 ++++++++------- server/lib/cdpclient/cdpclient.go | 35 ++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/server/cmd/api/api/wayland_input.go b/server/cmd/api/api/wayland_input.go index 811a95fe..6cd459bc 100644 --- a/server/cmd/api/api/wayland_input.go +++ b/server/cmd/api/api/wayland_input.go @@ -114,19 +114,21 @@ func (s *ApiService) doPressKeyWayland(ctx context.Context, body oapi.PressKeyRe if len(body.Keys) == 0 { return &validationError{msg: "keys must contain at least one key symbol"} } + events := make([]cdpclient.KeyEvent, 0, len(body.Keys)*2) for _, key := range body.Keys { code := key if len([]rune(key)) == 1 { code = "Key" + key } - if err := s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { - if err := client.DispatchKeyEvent(cdpCtx, "keyDown", key, code, ""); err != nil { - return err - } - return client.DispatchKeyEvent(cdpCtx, "keyUp", key, code, "") - }); err != nil { - return &executionError{msg: fmt.Sprintf("failed to press Wayland key %q: %v", key, err)} - } + events = append(events, + cdpclient.KeyEvent{Type: "keyDown", Key: key, Code: code}, + cdpclient.KeyEvent{Type: "keyUp", Key: key, Code: code}, + ) + } + if err := s.withCDPClient(ctx, func(cdpCtx context.Context, client *cdpclient.Client) error { + return client.DispatchKeyEvents(cdpCtx, events) + }); err != nil { + return &executionError{msg: fmt.Sprintf("failed to press Wayland key: %v", err)} } return nil } diff --git a/server/lib/cdpclient/cdpclient.go b/server/lib/cdpclient/cdpclient.go index 72340cf5..948e2bad 100644 --- a/server/lib/cdpclient/cdpclient.go +++ b/server/lib/cdpclient/cdpclient.go @@ -380,23 +380,42 @@ func (c *Client) InsertText(ctx context.Context, text string) error { return nil } -// DispatchKeyEvent sends a key event to the first page target. -func (c *Client) DispatchKeyEvent(ctx context.Context, eventType, key, code, text string) error { +// KeyEvent describes one CDP key event. +type KeyEvent struct { + Type string + Key string + Code string + Text string +} + +// DispatchKeyEvents sends key events to the first page target with one +// attached CDP session for the whole sequence. +func (c *Client) DispatchKeyEvents(ctx context.Context, events []KeyEvent) error { + if len(events) == 0 { + return nil + } sessionID, detach, err := c.attachFirstPage(ctx) if err != nil { return err } defer detach() - params := map[string]any{"type": eventType, "key": key, "code": code} - if text != "" { - params["text"] = text - } - if _, err := c.send(ctx, "Input.dispatchKeyEvent", params, sessionID); err != nil { - return fmt.Errorf("Input.dispatchKeyEvent: %w", err) + for _, event := range events { + params := map[string]any{"type": event.Type, "key": event.Key, "code": event.Code} + if event.Text != "" { + params["text"] = event.Text + } + if _, err := c.send(ctx, "Input.dispatchKeyEvent", params, sessionID); err != nil { + return fmt.Errorf("Input.dispatchKeyEvent: %w", err) + } } return nil } +// DispatchKeyEvent sends one key event to the first page target. +func (c *Client) DispatchKeyEvent(ctx context.Context, eventType, key, code, text string) error { + return c.DispatchKeyEvents(ctx, []KeyEvent{{Type: eventType, Key: key, Code: code, Text: text}}) +} + // ScreenshotClip describes an optional page screenshot crop in CSS pixels. type ScreenshotClip struct { X float64 `json:"x"` From 5a7066d36ff6de42d6d5af7f0217bc844b8bddc4 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:04:58 +0000 Subject: [PATCH 7/9] Record Wayland frames through CDP --- .../supervisor/services/weston.conf | 2 +- server/cmd/api/main.go | 32 ++++++ server/cmd/chromium-launcher/display.go | 1 + server/cmd/chromium-launcher/main_test.go | 1 + server/lib/recorder/ffmeg_test.go | 18 ++++ server/lib/recorder/ffmpeg.go | 101 ++++++++++++++---- 6 files changed, 132 insertions(+), 23 deletions(-) diff --git a/images/chromium-headful/supervisor/services/weston.conf b/images/chromium-headful/supervisor/services/weston.conf index 2e6e86fb..1403b8fb 100644 --- a/images/chromium-headful/supervisor/services/weston.conf +++ b/images/chromium-headful/supervisor/services/weston.conf @@ -1,5 +1,5 @@ [program:weston] -command=/bin/bash -c 'exec weston --backend=headless-backend.so --use-pixman --debug --socket="${WAYLAND_DISPLAY:-wayland-0}" --width="${WIDTH:-1920}" --height="${HEIGHT:-1080}" --idle-time=0' +command=/bin/bash -c 'exec weston --backend=headless-backend.so --use-pixman --debug --shell=kiosk-shell.so --socket="${WAYLAND_DISPLAY:-wayland-0}" --width="${WIDTH:-1920}" --height="${HEIGHT:-1080}" --idle-time=0' autostart=false autorestart=true startsecs=0 diff --git a/server/cmd/api/main.go b/server/cmd/api/main.go index 30155ede..3ab46574 100644 --- a/server/cmd/api/main.go +++ b/server/cmd/api/main.go @@ -27,8 +27,10 @@ import ( serverpkg "github.com/kernel/kernel-images/server" "github.com/kernel/kernel-images/server/cmd/api/api" "github.com/kernel/kernel-images/server/cmd/config" + "github.com/kernel/kernel-images/server/lib/cdpclient" "github.com/kernel/kernel-images/server/lib/chromedriverproxy" "github.com/kernel/kernel-images/server/lib/devtoolsproxy" + "github.com/kernel/kernel-images/server/lib/display" "github.com/kernel/kernel-images/server/lib/events" "github.com/kernel/kernel-images/server/lib/forkidentity" "github.com/kernel/kernel-images/server/lib/logger" @@ -83,6 +85,36 @@ func main() { AudioSource: &config.AudioSource, PulseServer: &config.PulseServer, } + displayConfig, err := display.FromEnv() + if err != nil { + slogger.Error("failed to load display backend configuration", "err", err) + os.Exit(1) + } + if displayConfig.Backend == display.BackendWayland { + versionURL := fmt.Sprintf("http://127.0.0.1:%d/json/version", config.DevToolsProxyPort) + var captureMu sync.Mutex + var captureClient *cdpclient.Client + defaultParams.CaptureFrame = func(ctx context.Context) ([]byte, error) { + captureMu.Lock() + defer captureMu.Unlock() + if captureClient == nil { + devtoolsURL, err := cdpclient.BrowserWebSocketURL(ctx, versionURL) + if err != nil { + return nil, err + } + captureClient, err = cdpclient.Dial(ctx, devtoolsURL) + if err != nil { + return nil, err + } + } + frame, err := captureClient.CaptureScreenshot(ctx, nil) + if err != nil { + _ = captureClient.Close() + captureClient = nil + } + return frame, err + } + } if err := defaultParams.Validate(); err != nil { slogger.Error("invalid default recording parameters", "err", err) os.Exit(1) diff --git a/server/cmd/chromium-launcher/display.go b/server/cmd/chromium-launcher/display.go index be983c75..4958802a 100644 --- a/server/cmd/chromium-launcher/display.go +++ b/server/cmd/chromium-launcher/display.go @@ -20,6 +20,7 @@ func chromiumDisplaySetup(config display.Config) (flags, env []string, err error return []string{ "--enable-features=UseOzonePlatform", "--ozone-platform=wayland", + "--start-fullscreen", }, []string{ "WAYLAND_DISPLAY=" + config.WaylandDisplay, "XDG_RUNTIME_DIR=" + config.RuntimeDir, diff --git a/server/cmd/chromium-launcher/main_test.go b/server/cmd/chromium-launcher/main_test.go index 3b3bcf98..bf50e0a2 100644 --- a/server/cmd/chromium-launcher/main_test.go +++ b/server/cmd/chromium-launcher/main_test.go @@ -30,6 +30,7 @@ func TestChromiumDisplaySetup(t *testing.T) { wantFlags: []string{ "--enable-features=UseOzonePlatform", "--ozone-platform=wayland", + "--start-fullscreen", }, wantEnv: []string{"WAYLAND_DISPLAY=wayland-2", "XDG_RUNTIME_DIR=/run/user/1000"}, }, diff --git a/server/lib/recorder/ffmeg_test.go b/server/lib/recorder/ffmeg_test.go index f853ff5e..94c64e90 100644 --- a/server/lib/recorder/ffmeg_test.go +++ b/server/lib/recorder/ffmeg_test.go @@ -1,6 +1,7 @@ package recorder import ( + "context" "path/filepath" "testing" "time" @@ -146,6 +147,23 @@ func TestFFmpegArgs_IncludesPulseAudioWhenEnabled(t *testing.T) { assert.NotContains(t, args, "aresample=async=1:first_pts=0") } +func TestFFmpegArgs_CaptureFrameUsesPipeInput(t *testing.T) { + tempDir := t.TempDir() + params := defaultParams(tempDir) + params.CaptureFrame = func(context.Context) ([]byte, error) { return []byte("png"), nil } + + args, err := ffmpegArgs(params, filepath.Join(tempDir, "wayland.mp4")) + require.NoError(t, err) + assert.Contains(t, args, "image2pipe") + assert.Contains(t, args, "pipe:0") + assert.Contains(t, args, "-vcodec") + assert.Contains(t, args, "png") + assert.Contains(t, args, "veryfast") + assert.Contains(t, args, "zerolatency") + assert.NotContains(t, args, "x11grab") + assert.NotContains(t, args, "-use_wallclock_as_timestamps") +} + func TestFFmpegArgs_VideoOnlyKeepsLegacyFlags(t *testing.T) { tempDir := t.TempDir() diff --git a/server/lib/recorder/ffmpeg.go b/server/lib/recorder/ffmpeg.go index f7961e9d..96d0fdff 100644 --- a/server/lib/recorder/ffmpeg.go +++ b/server/lib/recorder/ffmpeg.go @@ -71,12 +71,12 @@ type FFmpegRecorder struct { // chapter offsets to the media timeline; zero means never detected and // callers fall back to startTime (stamped before the process spawned). captureAnchor time.Time - ffmpegErr error - exitCode int - exited chan struct{} - deleted bool - markers []Marker - stz *scaletozero.Oncer + ffmpegErr error + exitCode int + exited chan struct{} + deleted bool + markers []Marker + stz *scaletozero.Oncer // flight coordinates concurrent operations using different keys: // - "stop": prevents multiple SIGINTs from being sent to ffmpeg @@ -96,6 +96,9 @@ type FFmpegRecordingParams struct { RecordAudio *bool AudioSource *string PulseServer *string + // CaptureFrame supplies PNG frames for non-X11 display backends. When nil, + // Linux recordings use x11grab as before. + CaptureFrame func(context.Context) ([]byte, error) } func (p FFmpegRecordingParams) Validate() error { @@ -178,6 +181,7 @@ func mergeFFmpegRecordingParams(config FFmpegRecordingParams, overrides FFmpegRe RecordAudio: config.RecordAudio, AudioSource: config.AudioSource, PulseServer: config.PulseServer, + CaptureFrame: config.CaptureFrame, } if overrides.FrameRate != nil { merged.FrameRate = overrides.FrameRate @@ -203,6 +207,9 @@ func mergeFFmpegRecordingParams(config FFmpegRecordingParams, overrides FFmpegRe if overrides.PulseServer != nil { merged.PulseServer = overrides.PulseServer } + if overrides.CaptureFrame != nil { + merged.CaptureFrame = overrides.CaptureFrame + } return merged } @@ -253,6 +260,7 @@ func (p FFmpegRecordingParams) clone() FFmpegRecordingParams { v := *p.PulseServer c.PulseServer = &v } + c.CaptureFrame = p.CaptureFrame return c } @@ -300,10 +308,25 @@ func (fr *FFmpegRecorder) Start(ctx context.Context) error { cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout + var frameInput io.WriteCloser + if fr.params.CaptureFrame != nil { + var err error + frameInput, err = cmd.StdinPipe() + if err != nil { + _ = fr.stz.Enable(context.WithoutCancel(ctx)) + fr.cmd = nil + close(fr.exited) + fr.mu.Unlock() + return fmt.Errorf("failed to create frame input: %w", err) + } + } fr.cmd = cmd fr.mu.Unlock() if err := cmd.Start(); err != nil { + if frameInput != nil { + _ = frameInput.Close() + } _ = fr.stz.Enable(context.WithoutCancel(ctx)) fr.mu.Lock() fr.ffmpegErr = err @@ -315,6 +338,9 @@ func (fr *FFmpegRecorder) Start(ctx context.Context) error { // Launch background waiter to capture process completion. go fr.waitForCommand(ctx) + if frameInput != nil { + go fr.streamFrames(context.WithoutCancel(ctx), frameInput) + } // Watch for ffmpeg's first output bytes to anchor chapter offsets to the // media timeline rather than the pre-spawn startTime. @@ -525,6 +551,37 @@ func (fr *FFmpegRecorder) Mark(name string) (string, int64, error) { return name, now.Sub(fr.anchorLocked()).Milliseconds(), nil } +func (fr *FFmpegRecorder) streamFrames(ctx context.Context, input io.WriteCloser) { + defer input.Close() + interval := time.Second / time.Duration(*fr.params.FrameRate) + next := time.Now() + for { + select { + case <-fr.exited: + return + default: + } + + frame, err := fr.params.CaptureFrame(ctx) + if err != nil { + return + } + if _, err := input.Write(frame); err != nil { + return + } + next = next.Add(interval) + if delay := time.Until(next); delay > 0 { + timer := time.NewTimer(delay) + select { + case <-timer.C: + case <-fr.exited: + timer.Stop() + return + } + } + } +} + // watchCaptureStart stamps captureAnchor with the moment ffmpeg writes its // first output bytes. The muxer only writes them once the capture pipeline is // fully initialized (input opened, encoder ready, first frame in flight), so @@ -705,20 +762,23 @@ func ffmpegArgs(params FFmpegRecordingParams, outputPath string) ([]string, erro "-i", fmt.Sprintf("%d:%s", *params.DisplayNum, audioDevice), } case "linux": - // When also capturing audio, give the x11grab input a larger packet queue - // so the video thread doesn't drop frames while the audio thread jitters - // during the mux (the default queue of 8 overflows with two live inputs). - // Omitted for video-only to keep that path identical to the pre-audio flags. if recordAudio { args = append(args, "-thread_queue_size", "512") } - args = append(args, - // Input options for X11 - "-f", "x11grab", - "-framerate", strconv.Itoa(*params.FrameRate), - // Input file - "-i", fmt.Sprintf(":%d", *params.DisplayNum), // X11 display - ) + if params.CaptureFrame != nil { + args = append(args, + "-f", "image2pipe", + "-framerate", strconv.Itoa(*params.FrameRate), + "-vcodec", "png", + "-i", "pipe:0", + ) + } else { + args = append(args, + "-f", "x11grab", + "-framerate", strconv.Itoa(*params.FrameRate), + "-i", fmt.Sprintf(":%d", *params.DisplayNum), + ) + } if recordAudio { args = append(args, audioInputArgs(params)...) } @@ -737,10 +797,7 @@ func ffmpegArgs(params FFmpegRecordingParams, outputPath string) ([]string, erro // Video encoding "-c:v", "libx264", ) - if recordAudio { - // Real-time-oriented encoding so ffmpeg keeps pace with the live audio+video - // mux instead of falling behind and drifting out of sync. Applied only when - // recording audio so the video-only path keeps its original encoding. + if recordAudio || params.CaptureFrame != nil { args = append(args, "-preset", "veryfast", "-tune", "zerolatency") } args = append(args, []string{ @@ -752,7 +809,7 @@ func ffmpegArgs(params FFmpegRecordingParams, outputPath string) ([]string, erro // overwrites x11grab's timestamps with wall-clock time for stable playback. // With audio we must not: it would stamp the separate video and audio inputs // independently and desync them, so we keep their input PTS instead. - if !recordAudio { + if !recordAudio && params.CaptureFrame == nil { args = append(args, "-use_wallclock_as_timestamps", "1") } args = append(args, []string{ From c415be43033480c61b846b067b82158b89603db6 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:11:04 +0000 Subject: [PATCH 8/9] Add Weston Wayland capture helper --- images/chromium-headful/Dockerfile | 24 +- images/chromium-headful/start-neko.sh | 10 + .../supervisor/services/neko.conf | 2 +- .../supervisor/services/weston.conf | 2 +- .../weston-capture/weston-capture.c | 295 ++++++++++++++++++ .../weston-capture/weston-screenshooter.xml | 9 + 6 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 images/chromium-headful/start-neko.sh create mode 100644 images/chromium-headful/weston-capture/weston-capture.c create mode 100644 images/chromium-headful/weston-capture/weston-screenshooter.xml diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index c393ae16..63e20661 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -1,3 +1,5 @@ +ARG NEKO_IMAGE=ghcr.io/kernel/neko/base:3.0.8-v1.6.0 + FROM docker.io/golang:1.25.0 AS server-builder WORKDIR /workspace/server @@ -70,7 +72,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=$CACHEIDPREFIX-ap echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; \ apt-get update; \ apt-get --no-install-recommends -y install \ - git gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev; + git gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev \ + libwayland-dev wayland-protocols; COPY images/chromium-headful/xorg-deps/ /xorg/ # build xf86-video-dummy v0.3.8 with RandR support RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=$CACHEIDPREFIX-apt-cache \ @@ -92,6 +95,20 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=$CACHEIDPREFIX-ap make -j$(nproc); \ make install; +# Build the Weston-native Wayland capture helper. It uses Weston's +# screenshooter protocol and a persistent wl_shm buffer, avoiding X11 capture. +COPY images/chromium-headful/weston-capture/ /weston-capture/ +RUN set -eux; \ + wayland-scanner client-header /weston-capture/weston-screenshooter.xml \ + /weston-capture/weston-screenshooter-client-protocol.h; \ + wayland-scanner private-code /weston-capture/weston-screenshooter.xml \ + /weston-capture/weston-screenshooter-protocol.c; \ + gcc -O2 -Wall -Wextra -Werror -o /usr/local/bin/weston-capture \ + /weston-capture/weston-capture.c \ + /weston-capture/weston-screenshooter-protocol.c \ + $(pkg-config --cflags --libs wayland-client); \ + strip /usr/local/bin/weston-capture; + FROM docker.io/ubuntu:22.04 AS ffmpeg-downloader # Allow cross-compilation when building with BuildKit platforms @@ -161,7 +178,7 @@ RUN --mount=type=cache,target=/tmp/cache/ffmpeg,sharing=locked,id=$CACHEIDPREFIX rm -rf /tmp/ffmpeg* EOT -FROM ghcr.io/kernel/neko/base:3.0.8-v1.6.0 AS neko +FROM ${NEKO_IMAGE} AS neko FROM node:22-bullseye-slim AS node-22 FROM docker.io/ubuntu:22.04 @@ -345,10 +362,13 @@ ENV WITHDOCKER=true COPY images/chromium-headful/xorg.conf /etc/neko/xorg.conf COPY images/chromium-headful/neko.yaml /etc/neko/neko.yaml +COPY images/chromium-headful/start-neko.sh /usr/local/bin/start-neko.sh +RUN chmod +x /usr/local/bin/start-neko.sh COPY --from=neko /usr/bin/neko /usr/bin/neko COPY --from=client /src/dist/ /var/www COPY --from=xorg-deps /usr/local/lib/xorg/modules/drivers/dummy_drv.so /usr/lib/xorg/modules/drivers/dummy_drv.so COPY --from=xorg-deps /usr/local/lib/xorg/modules/input/neko_drv.so /usr/lib/xorg/modules/input/neko_drv.so +COPY --from=xorg-deps /usr/local/bin/weston-capture /usr/local/bin/weston-capture COPY shared/start-pulseaudio.sh /usr/local/bin/start-pulseaudio.sh RUN chmod +x /usr/local/bin/start-pulseaudio.sh diff --git a/images/chromium-headful/start-neko.sh b/images/chromium-headful/start-neko.sh new file mode 100644 index 00000000..b2ea8c7b --- /dev/null +++ b/images/chromium-headful/start-neko.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +if [ "${DISPLAY_BACKEND:-x11}" = "wayland" ]; then + export NEKO_DESKTOP_WAYLAND=true + export NEKO_CAPTURE_VIDEO_WAYLAND=true + export NEKO_CAPTURE_VIDEO_WAYLAND_RECORDER=/usr/local/bin/weston-capture +fi + +exec /usr/bin/neko serve --server.static /var/www --server.bind 0.0.0.0:8080 diff --git a/images/chromium-headful/supervisor/services/neko.conf b/images/chromium-headful/supervisor/services/neko.conf index 9cb2677d..91ab3410 100644 --- a/images/chromium-headful/supervisor/services/neko.conf +++ b/images/chromium-headful/supervisor/services/neko.conf @@ -1,5 +1,5 @@ [program:neko] -command=/usr/bin/neko serve --server.static /var/www --server.bind 0.0.0.0:8080 +command=/usr/local/bin/start-neko.sh autostart=false autorestart=true startsecs=0 diff --git a/images/chromium-headful/supervisor/services/weston.conf b/images/chromium-headful/supervisor/services/weston.conf index 1403b8fb..d515135e 100644 --- a/images/chromium-headful/supervisor/services/weston.conf +++ b/images/chromium-headful/supervisor/services/weston.conf @@ -1,5 +1,5 @@ [program:weston] -command=/bin/bash -c 'exec weston --backend=headless-backend.so --use-pixman --debug --shell=kiosk-shell.so --socket="${WAYLAND_DISPLAY:-wayland-0}" --width="${WIDTH:-1920}" --height="${HEIGHT:-1080}" --idle-time=0' +command=/bin/bash -c 'exec weston --backend=headless-backend.so --use-pixman --debug --shell=desktop-shell.so --socket="${WAYLAND_DISPLAY:-wayland-0}" --width="${WIDTH:-1920}" --height="${HEIGHT:-1080}" --idle-time=0' autostart=false autorestart=true startsecs=0 diff --git a/images/chromium-headful/weston-capture/weston-capture.c b/images/chromium-headful/weston-capture/weston-capture.c new file mode 100644 index 00000000..8963bdc7 --- /dev/null +++ b/images/chromium-headful/weston-capture/weston-capture.c @@ -0,0 +1,295 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "weston-screenshooter-client-protocol.h" + +struct capture { + struct wl_display *display; + struct wl_shm *shm; + struct wl_output *output; + struct weston_screenshooter *screenshooter; + struct wl_buffer *buffer; + void *pixels; + size_t size; + int width; + int height; + int done; +}; + +static volatile sig_atomic_t stopping; + +static void +handle_signal(int signal_number) +{ + (void)signal_number; + stopping = 1; +} + +static int +create_anonymous_file(size_t size) +{ + int fd = syscall(SYS_memfd_create, "weston-capture", MFD_CLOEXEC); + if (fd >= 0 && ftruncate(fd, (off_t)size) == 0) + return fd; + if (fd >= 0) + close(fd); + + char name[] = "/weston-capture-XXXXXX"; + fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd < 0) + return -1; + shm_unlink(name); + if (ftruncate(fd, (off_t)size) < 0) { + close(fd); + return -1; + } + return fd; +} + +static void +output_geometry(void *data, struct wl_output *output, int32_t x, int32_t y, + int32_t physical_width, int32_t physical_height, + int32_t subpixel, const char *make, const char *model, + int32_t transform) +{ + (void)data; + (void)output; + (void)x; + (void)y; + (void)physical_width; + (void)physical_height; + (void)subpixel; + (void)make; + (void)model; + (void)transform; +} + +static void +output_mode(void *data, struct wl_output *output, uint32_t flags, + int32_t width, int32_t height, int32_t refresh) +{ + struct capture *capture = data; + (void)output; + (void)refresh; + if (flags & WL_OUTPUT_MODE_CURRENT) { + capture->width = width; + capture->height = height; + } +} + +static void +output_done(void *data, struct wl_output *output) +{ + (void)data; + (void)output; +} + +static void +output_scale(void *data, struct wl_output *output, int32_t factor) +{ + (void)data; + (void)output; + (void)factor; +} + +static const struct wl_output_listener output_listener = { + .geometry = output_geometry, + .mode = output_mode, + .done = output_done, + .scale = output_scale, +}; + +static void +screenshot_done(void *data, struct weston_screenshooter *screenshooter) +{ + struct capture *capture = data; + (void)screenshooter; + capture->done = 1; +} + +static const struct weston_screenshooter_listener screenshooter_listener = { + .done = screenshot_done, +}; + +static void +registry_global(void *data, struct wl_registry *registry, uint32_t name, + const char *interface, uint32_t version) +{ + struct capture *capture = data; + (void)version; + if (strcmp(interface, "wl_shm") == 0 && capture->shm == NULL) { + capture->shm = wl_registry_bind(registry, name, &wl_shm_interface, 1); + } else if (strcmp(interface, "wl_output") == 0 && capture->output == NULL) { + capture->output = wl_registry_bind(registry, name, &wl_output_interface, 1); + wl_output_add_listener(capture->output, &output_listener, capture); + } else if (strcmp(interface, "weston_screenshooter") == 0 && + capture->screenshooter == NULL) { + capture->screenshooter = wl_registry_bind( + registry, name, &weston_screenshooter_interface, 1); + } +} + +static void +registry_global_remove(void *data, struct wl_registry *registry, uint32_t name) +{ + (void)data; + (void)registry; + (void)name; +} + +static const struct wl_registry_listener registry_listener = { + .global = registry_global, + .global_remove = registry_global_remove, +}; + +static int +create_buffer(struct capture *capture) +{ + int fd; + int stride = capture->width * 4; + capture->size = (size_t)stride * (size_t)capture->height; + fd = create_anonymous_file(capture->size); + if (fd < 0) { + fprintf(stderr, "create capture buffer: %s\n", strerror(errno)); + return -1; + } + + capture->pixels = mmap(NULL, capture->size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (capture->pixels == MAP_FAILED) { + fprintf(stderr, "map capture buffer: %s\n", strerror(errno)); + close(fd); + return -1; + } + + struct wl_shm_pool *pool = wl_shm_create_pool(capture->shm, fd, + (int)capture->size); + close(fd); + if (pool == NULL) + return -1; + capture->buffer = wl_shm_pool_create_buffer( + pool, 0, capture->width, capture->height, stride, + WL_SHM_FORMAT_XRGB8888); + wl_shm_pool_destroy(pool); + return capture->buffer == NULL ? -1 : 0; +} + +static int +write_all(int fd, const void *data, size_t size) +{ + const char *bytes = data; + while (size > 0) { + ssize_t written = write(fd, bytes, size); + if (written < 0) { + if (errno == EINTR) + continue; + return -1; + } + bytes += written; + size -= (size_t)written; + } + return 0; +} + +static int +parse_framerate(int argc, char **argv) +{ + for (int i = 1; i + 1 < argc; i++) { + if (strcmp(argv[i], "--framerate") == 0) { + int fps = atoi(argv[i + 1]); + if (fps > 0 && fps <= 120) + return fps; + } + } + return 25; +} + +int +main(int argc, char **argv) +{ + struct capture capture = {}; + struct sigaction action = { + .sa_handler = handle_signal, + }; + int fps = parse_framerate(argc, argv); + struct timespec interval = { + .tv_sec = 0, + .tv_nsec = 1000000000L / fps, + }; + + sigemptyset(&action.sa_mask); + sigaction(SIGINT, &action, NULL); + sigaction(SIGTERM, &action, NULL); + + capture.display = wl_display_connect(NULL); + if (capture.display == NULL) { + fprintf(stderr, "connect to Wayland display: %s\n", strerror(errno)); + return 1; + } + + struct wl_registry *registry = wl_display_get_registry(capture.display); + wl_registry_add_listener(registry, ®istry_listener, &capture); + if (wl_display_roundtrip(capture.display) < 0 || + wl_display_roundtrip(capture.display) < 0) { + fprintf(stderr, "discover Wayland globals: %s\n", strerror(errno)); + return 1; + } + if (capture.shm == NULL || capture.output == NULL || + capture.screenshooter == NULL || capture.width <= 0 || + capture.height <= 0) { + fprintf(stderr, "Wayland screenshooter or output is unavailable\n"); + return 1; + } + if (create_buffer(&capture) < 0) + return 1; + weston_screenshooter_add_listener(capture.screenshooter, + &screenshooter_listener, &capture); + + while (!stopping) { + capture.done = 0; + weston_screenshooter_shoot(capture.screenshooter, capture.output, + capture.buffer); + if (wl_display_flush(capture.display) < 0 && errno != EAGAIN) + break; + while (!capture.done && !stopping) { + if (wl_display_dispatch(capture.display) < 0) { + stopping = 1; + break; + } + } + if (stopping) + break; + if (write_all(STDOUT_FILENO, capture.pixels, capture.size) < 0) + break; + nanosleep(&interval, NULL); + } + + if (capture.pixels != NULL && capture.pixels != MAP_FAILED) + munmap(capture.pixels, capture.size); + if (capture.buffer != NULL) + wl_buffer_destroy(capture.buffer); + if (capture.screenshooter != NULL) + weston_screenshooter_destroy(capture.screenshooter); + if (capture.output != NULL) + wl_output_destroy(capture.output); + if (capture.shm != NULL) + wl_shm_destroy(capture.shm); + if (registry != NULL) + wl_registry_destroy(registry); + wl_display_disconnect(capture.display); + return 0; +} diff --git a/images/chromium-headful/weston-capture/weston-screenshooter.xml b/images/chromium-headful/weston-capture/weston-screenshooter.xml new file mode 100644 index 00000000..ce952d0b --- /dev/null +++ b/images/chromium-headful/weston-capture/weston-screenshooter.xml @@ -0,0 +1,9 @@ + + + + + + + + + From a38b174889d55b7e8e74e0ab35832c87e88680af Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:02:47 +0000 Subject: [PATCH 9/9] Enable Wayland WebRTC with Neko capture --- server/cmd/wrapper/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/cmd/wrapper/main.go b/server/cmd/wrapper/main.go index 43ef2b67..4ddd8c0e 100644 --- a/server/cmd/wrapper/main.go +++ b/server/cmd/wrapper/main.go @@ -70,9 +70,6 @@ func main() { if prof == profileHeadless { fatalf("Wayland display backend requires the headful image") } - if os.Getenv("ENABLE_WEBRTC") == "true" { - fatalf("Wayland display backend does not support ENABLE_WEBRTC yet") - } if displayConfig.RuntimeDir == "" { displayConfig.RuntimeDir = "/tmp/runtime-kernel" _ = os.Setenv("XDG_RUNTIME_DIR", displayConfig.RuntimeDir) @@ -220,7 +217,7 @@ func main() { } else if displayConfig.Backend == display.BackendWayland { xServer = "weston" } - webrtc := prof == profileHeadful && displayConfig.Backend == display.BackendX11 && os.Getenv("ENABLE_WEBRTC") == "true" + webrtc := prof == profileHeadful && os.Getenv("ENABLE_WEBRTC") == "true" // Pre-touch chromium's supervisord log so kernel-images-api's `tail -f` // doesn't bail out and enter its 250ms retry backoff when started in