From c6bba86bfc5b1336e968b8ecd4b6a7fcedd72735 Mon Sep 17 00:00:00 2001 From: Chris Lee <23645059+chruffins@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:25:26 +0000 Subject: [PATCH 1/5] Add Wayland screencopy video capture --- server/internal/capture/manager.go | 23 +++- server/internal/capture/streamsink.go | 36 ++++++- server/internal/capture/wayland.go | 136 ++++++++++++++++++++++++ server/internal/capture/wayland_test.go | 46 ++++++++ server/internal/config/capture.go | 16 +++ webpage/docs/configuration/capture.md | 6 +- webpage/docs/configuration/help.json | 20 ++++ 7 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 server/internal/capture/wayland.go create mode 100644 server/internal/capture/wayland_test.go diff --git a/server/internal/capture/manager.go b/server/internal/capture/manager.go index d16919e9f..d5ac42c97 100644 --- a/server/internal/capture/manager.go +++ b/server/internal/capture/manager.go @@ -39,6 +39,9 @@ func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCt createPipeline := func() (string, error) { if pipelineConf.GstPipeline != "" { + if config.Wayland { + return "", errors.New("custom video pipelines are not supported with Wayland capture") + } // replace {display} with valid display return strings.Replace(pipelineConf.GstPipeline, "{display}", config.Display, 1), nil } @@ -49,6 +52,18 @@ func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCt return "", err } + if config.Wayland { + fps := screen.Rate + if fps <= 0 { + fps = 25 + } + return fmt.Sprintf( + "appsrc name=appsrc is-live=true format=time do-timestamp=true "+ + "caps=video/x-raw,format=BGRx,width=%d,height=%d,framerate=%d/1 "+ + "%s ! appsink name=appsink", screen.Width, screen.Height, fps, pipeline, + ), nil + } + return fmt.Sprintf( "ximagesrc display-name=%s show-pointer=%v use-damage=false "+ "%s ! appsink name=appsink", config.Display, pipelineConf.ShowPointer, pipeline, @@ -69,7 +84,13 @@ func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCt Msg("syntax check for video stream pipeline passed") // append to videos - videos[video_id] = streamSinkNew(config.VideoCodec, createPipeline, video_id) + video := streamSinkNew(config.VideoCodec, createPipeline, video_id) + if config.Wayland { + video.SetFrameSourceFactory(func() (frameSource, error) { + return newWaylandFrameSource(config.WaylandRecorder, desktop.GetScreenSize()), nil + }) + } + videos[video_id] = video } return &CaptureManagerCtx{ diff --git a/server/internal/capture/streamsink.go b/server/internal/capture/streamsink.go index a9e5146ea..14ff08dd2 100644 --- a/server/internal/capture/streamsink.go +++ b/server/internal/capture/streamsink.go @@ -32,10 +32,12 @@ type StreamSinkManagerCtx struct { mu sync.Mutex wg sync.WaitGroup - codec codec.RTPCodec - pipeline gst.Pipeline - pipelineMu sync.Mutex - pipelineFn func() (string, error) + codec codec.RTPCodec + pipeline gst.Pipeline + pipelineMu sync.Mutex + pipelineFn func() (string, error) + frameSourceFn func() (frameSource, error) + frameSource frameSource listeners map[uintptr]types.SampleListener listenersKf map[uintptr]types.SampleListener // keyframe lobby @@ -142,6 +144,10 @@ func (manager *StreamSinkManagerCtx) ID() string { return manager.id } +func (manager *StreamSinkManagerCtx) SetFrameSourceFactory(factory func() (frameSource, error)) { + manager.frameSourceFn = factory +} + func (manager *StreamSinkManagerCtx) Bitrate() uint64 { manager.listenersMu.Lock() defer manager.listenersMu.Unlock() @@ -325,9 +331,27 @@ func (manager *StreamSinkManagerCtx) CreatePipeline() error { return err } + if manager.frameSourceFn != nil { + manager.frameSource, err = manager.frameSourceFn() + if err != nil { + manager.pipeline.Destroy() + manager.pipeline = nil + return err + } + manager.pipeline.AttachAppsrc("appsrc") + } manager.pipeline.AttachAppsink("appsink") manager.pipeline.Play() + if manager.frameSource != nil { + if err := manager.frameSource.Start(manager.pipeline.Push); err != nil { + manager.pipeline.Destroy() + manager.pipeline = nil + manager.frameSource = nil + return err + } + } + manager.wg.Add(1) pipeline := manager.pipeline @@ -405,6 +429,10 @@ func (manager *StreamSinkManagerCtx) DestroyPipeline() { return } + if manager.frameSource != nil { + manager.frameSource.Stop() + manager.frameSource = nil + } manager.pipeline.Destroy() manager.logger.Info().Msgf("destroying pipeline") manager.pipeline = nil diff --git a/server/internal/capture/wayland.go b/server/internal/capture/wayland.go new file mode 100644 index 000000000..12411f248 --- /dev/null +++ b/server/internal/capture/wayland.go @@ -0,0 +1,136 @@ +package capture + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "sync" + + "github.com/rs/zerolog/log" + + "github.com/m1k1o/neko/server/pkg/types" +) + +type frameSource interface { + Start(func([]byte)) error + Stop() +} + +type waylandFrameSource struct { + recorder string + width int + height int + fps int + + mu sync.Mutex + cancel context.CancelFunc + done chan struct{} +} + +func newWaylandFrameSource(recorder string, screen types.ScreenSize) *waylandFrameSource { + fps := int(screen.Rate) + if fps <= 0 { + fps = 25 + } + + return &waylandFrameSource{ + recorder: recorder, + width: screen.Width, + height: screen.Height, + fps: fps, + } +} + +func (source *waylandFrameSource) frameSize() int { + return source.width * source.height * 4 +} + +func (source *waylandFrameSource) args() []string { + return []string{ + "--no-damage", + "--no-dmabuf", + "--framerate", strconv.Itoa(source.fps), + "--muxer", "rawvideo", + "--codec", "rawvideo", + "--pixel-format", "bgr0", + "--file", "/dev/stdout", + "--overwrite", + } +} + +func (source *waylandFrameSource) command() *exec.Cmd { + return exec.Command(source.recorder, source.args()...) +} + +func (source *waylandFrameSource) Start(push func([]byte)) error { + if push == nil { + return fmt.Errorf("frame push callback is required") + } + if source.recorder == "" { + return fmt.Errorf("Wayland recorder executable is required") + } + if source.width <= 0 || source.height <= 0 { + return fmt.Errorf("invalid Wayland output size: %dx%d", source.width, source.height) + } + + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext(ctx, source.recorder, source.args()...) + cmd.Stderr = os.Stderr + + stdout, err := cmd.StdoutPipe() + if err != nil { + cancel() + return fmt.Errorf("create Wayland recorder pipe: %w", err) + } + if err := cmd.Start(); err != nil { + cancel() + return fmt.Errorf("start Wayland recorder: %w", err) + } + + done := make(chan struct{}) + source.mu.Lock() + source.cancel = cancel + source.done = done + source.mu.Unlock() + + go func() { + defer close(done) + defer stdout.Close() + + frame := make([]byte, source.frameSize()) + for { + if _, err := io.ReadFull(stdout, frame); err != nil { + if err != io.EOF && err != io.ErrUnexpectedEOF { + log.Warn().Err(err).Msg("Wayland recorder stopped while reading a frame") + } + break + } + + push(frame) + } + + if err := cmd.Wait(); err != nil && ctx.Err() == nil { + log.Warn().Err(err).Msg("Wayland recorder exited") + } + }() + + return nil +} + +func (source *waylandFrameSource) Stop() { + source.mu.Lock() + cancel := source.cancel + done := source.done + source.cancel = nil + source.done = nil + source.mu.Unlock() + + if cancel == nil { + return + } + cancel() + <-done +} diff --git a/server/internal/capture/wayland_test.go b/server/internal/capture/wayland_test.go new file mode 100644 index 000000000..facd77211 --- /dev/null +++ b/server/internal/capture/wayland_test.go @@ -0,0 +1,46 @@ +package capture + +import ( + "reflect" + "testing" + + "github.com/m1k1o/neko/server/pkg/types" +) + +func TestWaylandFrameSourceCommand(t *testing.T) { + source := newWaylandFrameSource("wf-recorder", types.ScreenSize{ + Width: 1920, + Height: 1080, + Rate: 25, + }) + + got := source.command().Args + want := []string{ + "wf-recorder", + "--no-damage", + "--no-dmabuf", + "--framerate", "25", + "--muxer", "rawvideo", + "--codec", "rawvideo", + "--pixel-format", "bgr0", + "--file", "/dev/stdout", + "--overwrite", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("command args = %#v, want %#v", got, want) + } +} + +func TestWaylandFrameSourceDefaultsFrameRate(t *testing.T) { + source := newWaylandFrameSource("wf-recorder", types.ScreenSize{ + Width: 10, + Height: 20, + }) + + if source.fps != 25 { + t.Fatalf("fps = %d, want 25", source.fps) + } + if source.frameSize() != 800 { + t.Fatalf("frame size = %d, want 800", source.frameSize()) + } +} diff --git a/server/internal/config/capture.go b/server/internal/config/capture.go index 6bf568974..03f652b7c 100644 --- a/server/internal/config/capture.go +++ b/server/internal/config/capture.go @@ -28,6 +28,9 @@ const ( type Capture struct { Display string + Wayland bool + WaylandRecorder string + VideoCodec codec.RTPCodec VideoIDs []string VideoPipelines map[string]types.VideoConfig @@ -80,6 +83,16 @@ func (Capture) Init(cmd *cobra.Command) error { return err } + cmd.PersistentFlags().Bool("capture.video.wayland", false, "capture a Wayland compositor output") + if err := viper.BindPFlag("capture.video.wayland", cmd.PersistentFlags().Lookup("capture.video.wayland")); err != nil { + return err + } + + cmd.PersistentFlags().String("capture.video.wayland_recorder", "wf-recorder", "Wayland screencopy recorder executable") + if err := viper.BindPFlag("capture.video.wayland_recorder", cmd.PersistentFlags().Lookup("capture.video.wayland_recorder")); err != nil { + return err + } + cmd.PersistentFlags().String("capture.video.codec", "vp8", "video codec to be used") if err := viper.BindPFlag("capture.video.codec", cmd.PersistentFlags().Lookup("capture.video.codec")); err != nil { return err @@ -326,6 +339,9 @@ func (s *Capture) Set() { s.Display = os.Getenv("DISPLAY") } + s.Wayland = viper.GetBool("capture.video.wayland") + s.WaylandRecorder = viper.GetString("capture.video.wayland_recorder") + // video videoCodec := viper.GetString("capture.video.codec") s.VideoCodec, ok = codec.ParseStr(videoCodec) diff --git a/webpage/docs/configuration/capture.md b/webpage/docs/configuration/capture.md index 6349943ea..3f8fb79a9 100644 --- a/webpage/docs/configuration/capture.md +++ b/webpage/docs/configuration/capture.md @@ -31,6 +31,8 @@ The Gstreamer pipeline is started when the first client requests the video strea - is the name of the [X display](https://www.x.org/wiki/) that you want to capture. If not specified, the environment variable `DISPLAY` will be used. -- available codecs are `vp8`, `vp9`, `av1`, `h264`. [Supported video codecs](https://developer.mozilla.org/en-US/docs/Web/Media/Guides/Formats/WebRTC_codecs#supported_video_codecs) are dependent on the WebRTC implementation used by the client, `vp8` and `h264` are supported by all WebRTC implementations. +- switches the video source from `ximagesrc` to a `wf-recorder` process using the compositor's `wlr-screencopy-unstable-v1` protocol. It requires a Wayland compositor that exposes that protocol and an executable . Custom `gst_pipeline` values are not supported in this mode. +- is the executable used to produce raw `BGRx` frames on stdout. The default is `wf-recorder`. +- available codecs are `vp8`, `vp9`, `av1`, `h264`. [Supported video codecs](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/WebRTC_codecs#supported_video_codecs) are dependent on the WebRTC implementation used by the client, `vp8` and `h264` are supported by all WebRTC implementations. - is a list of pipeline ids that are defined in the section. The first pipeline in the list will be the default pipeline. - is a shorthand for defining [Gstreamer pipeline description](#video.gst_pipeline) for a single pipeline. This is option is ignored if is defined. - is a dictionary of pipeline configurations. Each pipeline configuration is defined by a unique pipeline id. They can be defined in two ways: either by building the pipeline dynamically using [Expression-Driven Configuration](#video.expression) or by defining the pipeline using a [Gstreamer Pipeline Description](#video.gst_pipeline). diff --git a/webpage/docs/configuration/help.json b/webpage/docs/configuration/help.json index 8320f7e9c..e97ccbf23 100644 --- a/webpage/docs/configuration/help.json +++ b/webpage/docs/configuration/help.json @@ -164,6 +164,26 @@ "type": "string", "description": "X display to capture" }, + { + "key": [ + "capture", + "video", + "wayland" + ], + "type": "boolean", + "defaultValue": false, + "description": "capture a Wayland compositor output" + }, + { + "key": [ + "capture", + "video", + "wayland_recorder" + ], + "type": "string", + "defaultValue": "wf-recorder", + "description": "Wayland screencopy recorder executable" + }, { "key": [ "capture", From 1d5f7674179af4e7b5ec8f90da66705be4e73f51 Mon Sep 17 00:00:00 2001 From: Chris Lee <23645059+chruffins@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:41:35 +0000 Subject: [PATCH 2/5] Add Wayland desktop input backend --- server/internal/config/desktop.go | 7 + server/internal/desktop/manager.go | 30 ++- server/internal/desktop/wayland.go | 293 ++++++++++++++++++++++++ server/internal/desktop/wayland_test.go | 41 ++++ server/internal/desktop/xorg.go | 88 +++++++ webpage/docs/configuration/desktop.md | 4 +- webpage/docs/configuration/help.json | 9 + 7 files changed, 464 insertions(+), 8 deletions(-) create mode 100644 server/internal/desktop/wayland.go create mode 100644 server/internal/desktop/wayland_test.go diff --git a/server/internal/config/desktop.go b/server/internal/config/desktop.go index 8331035e7..a1c835789 100644 --- a/server/internal/config/desktop.go +++ b/server/internal/config/desktop.go @@ -14,6 +14,7 @@ import ( type Desktop struct { Display string + Wayland bool ScreenSize types.ScreenSize @@ -31,6 +32,11 @@ func (Desktop) Init(cmd *cobra.Command) error { return err } + cmd.PersistentFlags().Bool("desktop.wayland", false, "use Wayland desktop input and screen management") + if err := viper.BindPFlag("desktop.wayland", cmd.PersistentFlags().Lookup("desktop.wayland")); err != nil { + return err + } + cmd.PersistentFlags().String("desktop.screen", "1280x720@30", "default screen size and framerate") if err := viper.BindPFlag("desktop.screen", cmd.PersistentFlags().Lookup("desktop.screen")); err != nil { return err @@ -75,6 +81,7 @@ func (Desktop) InitV2(cmd *cobra.Command) error { func (s *Desktop) Set() { s.Display = viper.GetString("desktop.display") + s.Wayland = viper.GetBool("desktop.wayland") // Display is provided by env variable unless explicitly set if s.Display == "" { diff --git a/server/internal/desktop/manager.go b/server/internal/desktop/manager.go index ccae14704..390827512 100644 --- a/server/internal/desktop/manager.go +++ b/server/internal/desktop/manager.go @@ -20,13 +20,14 @@ import ( var mu = sync.Mutex{} type DesktopManagerCtx struct { - logger zerolog.Logger - wg sync.WaitGroup - shutdown chan struct{} - emmiter events.EventEmmiter - config *config.Desktop - screenSize types.ScreenSize // cached screen size - input xinput.Driver + logger zerolog.Logger + wg sync.WaitGroup + shutdown chan struct{} + emmiter events.EventEmmiter + config *config.Desktop + screenSize types.ScreenSize // cached screen size + input xinput.Driver + waylandInput *waylandInput // Clipboard process holding the most recent clipboard data. // It must remain running to allow pasting clipboard data. @@ -54,6 +55,16 @@ func New(config *config.Desktop) *DesktopManagerCtx { } func (manager *DesktopManagerCtx) Start() { + if manager.config.Wayland { + input, err := newWaylandInput(manager.screenSize.Width, manager.screenSize.Height) + if err != nil { + manager.logger.Panic().Err(err).Msg("unable to create Wayland input device") + } + manager.waylandInput = input + manager.logger.Info().Str("screen_size", manager.screenSize.String()).Msg("using Wayland desktop backend") + return + } + if xorg.DisplayOpen(manager.config.Display) { manager.logger.Panic().Str("display", manager.config.Display).Msg("unable to open display") } @@ -139,6 +150,11 @@ func (manager *DesktopManagerCtx) Shutdown() error { manager.logger.Info().Msgf("shutdown") close(manager.shutdown) + if manager.waylandInput != nil { + manager.waylandInput.close() + manager.waylandInput = nil + return nil + } manager.replaceClipboardCommand(nil) manager.wg.Wait() diff --git a/server/internal/desktop/wayland.go b/server/internal/desktop/wayland.go new file mode 100644 index 000000000..e847e131a --- /dev/null +++ b/server/internal/desktop/wayland.go @@ -0,0 +1,293 @@ +package desktop + +import ( + "encoding/binary" + "fmt" + "os" + "sync" + "time" + + "golang.org/x/sys/unix" +) + +const ( + waylandInputPath = "/dev/uinput" + + evSyn = 0x00 + evKey = 0x01 + evRel = 0x02 + evAbs = 0x03 + + relX = 0x00 + relY = 0x01 + relWheel = 0x08 + relHWheel = 0x06 + + absX = 0x00 + absY = 0x01 + + btnLeft = 0x110 + btnMiddle = 0x112 + btnRight = 0x111 + + uiSetEvbit = 0x40045564 + uiSetKeybit = 0x40045565 + uiSetRelbit = 0x40045566 + uiSetAbsbit = 0x40045567 + uiDevCreate = 0x5501 + uiDevDestroy = 0x5502 +) + +type uinputID struct { + BusType uint16 + Vendor uint16 + Product uint16 + Version uint16 +} + +type uinputUserDev struct { + Name [80]byte + ID uinputID + FFEffectsMax uint32 + AbsMax [64]int32 + AbsMin [64]int32 + AbsFuzz [64]int32 + AbsFlat [64]int32 +} + +type inputEvent struct { + Sec int64 + Usec int64 + Type uint16 + Code uint16 + Value int32 +} + +type waylandInput struct { + fd *os.File + width int + height int + + mu sync.Mutex + cursorX int + cursorY int + pressed map[uint16]struct{} +} + +func newWaylandInput(width, height int) (*waylandInput, error) { + if width <= 0 || height <= 0 { + return nil, fmt.Errorf("invalid Wayland input size: %dx%d", width, height) + } + + fd, err := os.OpenFile(waylandInputPath, os.O_WRONLY|unix.O_NONBLOCK, 0) + if err != nil { + return nil, fmt.Errorf("open %s: %w", waylandInputPath, err) + } + + input := &waylandInput{ + fd: fd, + width: width, + height: height, + pressed: make(map[uint16]struct{}), + } + if err := input.create(); err != nil { + _ = fd.Close() + return nil, err + } + return input, nil +} + +func (input *waylandInput) ioctl(request, value uintptr) error { + _, _, errno := unix.Syscall(unix.SYS_IOCTL, input.fd.Fd(), request, value) + if errno != 0 { + return errno + } + return nil +} + +func (input *waylandInput) create() error { + for _, eventType := range []int{evKey, evRel, evAbs} { + if err := input.ioctl(uiSetEvbit, uintptr(eventType)); err != nil { + return fmt.Errorf("enable uinput event type %d: %w", eventType, err) + } + } + for key := 0; key <= 0xff; key++ { + if err := input.ioctl(uiSetKeybit, uintptr(key)); err != nil { + return fmt.Errorf("enable uinput key %d: %w", key, err) + } + } + for _, rel := range []int{relX, relY, relWheel, relHWheel} { + if err := input.ioctl(uiSetRelbit, uintptr(rel)); err != nil { + return fmt.Errorf("enable uinput relative axis %d: %w", rel, err) + } + } + for _, abs := range []int{absX, absY} { + if err := input.ioctl(uiSetAbsbit, uintptr(abs)); err != nil { + return fmt.Errorf("enable uinput absolute axis %d: %w", abs, err) + } + } + + device := uinputUserDev{ + ID: uinputID{BusType: 0x03, Vendor: 0x1, Product: 0x1, Version: 1}, + } + copy(device.Name[:], "Neko Wayland input") + device.AbsMax[absX] = int32(input.width - 1) + device.AbsMax[absY] = int32(input.height - 1) + if err := binary.Write(input.fd, binary.LittleEndian, &device); err != nil { + return fmt.Errorf("configure uinput device: %w", err) + } + if err := input.ioctl(uiDevCreate, 0); err != nil { + return fmt.Errorf("create uinput device: %w", err) + } + return nil +} + +func (input *waylandInput) close() { + input.mu.Lock() + defer input.mu.Unlock() + if input.fd == nil { + return + } + _ = input.ioctl(uiDevDestroy, 0) + _ = input.fd.Close() + input.fd = nil +} + +func (input *waylandInput) emit(eventType, code uint16, value int32) error { + now := time.Now() + event := inputEvent{ + Sec: now.Unix(), + Usec: int64(now.Nanosecond()) / 1000, + Type: eventType, + Code: code, + Value: value, + } + return binary.Write(input.fd, binary.LittleEndian, &event) +} + +func (input *waylandInput) sync() error { + return input.emit(evSyn, 0, 0) +} + +func (input *waylandInput) move(x, y int) error { + input.mu.Lock() + defer input.mu.Unlock() + if err := input.emit(evAbs, absX, int32(clamp(x, 0, input.width-1))); err != nil { + return err + } + if err := input.emit(evAbs, absY, int32(clamp(y, 0, input.height-1))); err != nil { + return err + } + if err := input.sync(); err != nil { + return err + } + input.cursorX, input.cursorY = x, y + return nil +} + +func (input *waylandInput) button(code uint32, down bool) error { + linuxCode, ok := mapButton(code) + if !ok { + return fmt.Errorf("unsupported mouse button: %d", code) + } + input.mu.Lock() + defer input.mu.Unlock() + if err := input.emit(evKey, linuxCode, boolValue(down)); err != nil { + return err + } + if err := input.sync(); err != nil { + return err + } + if down { + input.pressed[linuxCode] = struct{}{} + } else { + delete(input.pressed, linuxCode) + } + return nil +} + +func (input *waylandInput) key(code uint32, down bool) error { + linuxCode, ok := mapKey(code) + if !ok { + return fmt.Errorf("unsupported keyboard key: %d", code) + } + input.mu.Lock() + defer input.mu.Unlock() + if err := input.emit(evKey, linuxCode, boolValue(down)); err != nil { + return err + } + if err := input.sync(); err != nil { + return err + } + if down { + input.pressed[linuxCode] = struct{}{} + } else { + delete(input.pressed, linuxCode) + } + return nil +} + +func (input *waylandInput) scroll(deltaX, deltaY int) error { + input.mu.Lock() + defer input.mu.Unlock() + if deltaY != 0 { + if err := input.emit(evRel, relWheel, int32(-deltaY)); err != nil { + return err + } + } + if deltaX != 0 { + if err := input.emit(evRel, relHWheel, int32(deltaX)); err != nil { + return err + } + } + return input.sync() +} + +func (input *waylandInput) resetKeys() error { + input.mu.Lock() + defer input.mu.Unlock() + for code := range input.pressed { + if err := input.emit(evKey, code, 0); err != nil { + return err + } + } + input.pressed = make(map[uint16]struct{}) + return input.sync() +} + +func mapButton(code uint32) (uint16, bool) { + switch code { + case 1: + return btnLeft, true + case 2: + return btnMiddle, true + case 3: + return btnRight, true + default: + return 0, false + } +} + +func mapKey(code uint32) (uint16, bool) { + if code < 8 || code > 263 { + return 0, false + } + return uint16(code - 8), true +} + +func boolValue(value bool) int32 { + if value { + return 1 + } + return 0 +} + +func clamp(value, min, max int) int { + if value < min { + return min + } + if value > max { + return max + } + return value +} diff --git a/server/internal/desktop/wayland_test.go b/server/internal/desktop/wayland_test.go new file mode 100644 index 000000000..8bbd2531f --- /dev/null +++ b/server/internal/desktop/wayland_test.go @@ -0,0 +1,41 @@ +package desktop + +import "testing" + +func TestMapKeyConvertsX11Keycodes(t *testing.T) { + for _, test := range []struct { + name string + in uint32 + want uint16 + }{ + {name: "escape", in: 9, want: 1}, + {name: "a", in: 38, want: 30}, + {name: "f12", in: 96, want: 88}, + } { + t.Run(test.name, func(t *testing.T) { + got, ok := mapKey(test.in) + if !ok || got != test.want { + t.Fatalf("mapKey(%d) = (%d, %v), want (%d, true)", test.in, got, ok, test.want) + } + }) + } +} + +func TestMapButton(t *testing.T) { + for _, test := range []struct { + in uint32 + want uint16 + }{ + {in: 1, want: btnLeft}, + {in: 2, want: btnMiddle}, + {in: 3, want: btnRight}, + } { + got, ok := mapButton(test.in) + if !ok || got != test.want { + t.Fatalf("mapButton(%d) = (%d, %v), want (%d, true)", test.in, got, ok, test.want) + } + } + if _, ok := mapButton(9); ok { + t.Fatal("mapButton(9) unexpectedly succeeded") + } +} diff --git a/server/internal/desktop/xorg.go b/server/internal/desktop/xorg.go index 907f1abc1..d61f0ab03 100644 --- a/server/internal/desktop/xorg.go +++ b/server/internal/desktop/xorg.go @@ -11,14 +11,30 @@ import ( ) func (manager *DesktopManagerCtx) Move(x, y int) { + if manager.waylandInput != nil { + if err := manager.waylandInput.move(x, y); err != nil { + manager.logger.Warn().Err(err).Msg("Wayland pointer move failed") + } + return + } xorg.Move(x, y) } func (manager *DesktopManagerCtx) GetCursorPosition() (int, int) { + if manager.waylandInput != nil { + return manager.waylandInput.cursorX, manager.waylandInput.cursorY + } return xorg.GetCursorPosition() } func (manager *DesktopManagerCtx) Scroll(deltaX, deltaY int, controlKey bool) { + if manager.waylandInput != nil { + returnErr := manager.waylandInput.scroll(deltaX, deltaY) + if returnErr != nil { + manager.logger.Warn().Err(returnErr).Msg("Wayland scroll failed") + } + return + } if manager.config.UseInputDriver { // XI2.1 smooth scrolling via xf86-input-neko: set modifier before the // driver posts the motion event so the X server sees Ctrl held. @@ -37,22 +53,39 @@ func (manager *DesktopManagerCtx) Scroll(deltaX, deltaY int, controlKey bool) { } func (manager *DesktopManagerCtx) ButtonDown(code uint32) error { + if manager.waylandInput != nil { + return manager.waylandInput.button(code, true) + } return xorg.ButtonDown(code) } func (manager *DesktopManagerCtx) KeyDown(code uint32) error { + if manager.waylandInput != nil { + return manager.waylandInput.key(code, true) + } return xorg.KeyDown(code) } func (manager *DesktopManagerCtx) ButtonUp(code uint32) error { + if manager.waylandInput != nil { + return manager.waylandInput.button(code, false) + } return xorg.ButtonUp(code) } func (manager *DesktopManagerCtx) KeyUp(code uint32) error { + if manager.waylandInput != nil { + return manager.waylandInput.key(code, false) + } return xorg.KeyUp(code) } func (manager *DesktopManagerCtx) ButtonPress(code uint32) error { + if manager.waylandInput != nil { + manager.ResetKeys() + defer manager.ResetKeys() + return manager.ButtonDown(code) + } xorg.ResetKeys() defer xorg.ResetKeys() @@ -60,6 +93,16 @@ func (manager *DesktopManagerCtx) ButtonPress(code uint32) error { } func (manager *DesktopManagerCtx) KeyPress(codes ...uint32) error { + if manager.waylandInput != nil { + manager.ResetKeys() + defer manager.ResetKeys() + for _, code := range codes { + if err := manager.KeyDown(code); err != nil { + return err + } + } + return nil + } xorg.ResetKeys() defer xorg.ResetKeys() @@ -77,10 +120,19 @@ func (manager *DesktopManagerCtx) KeyPress(codes ...uint32) error { } func (manager *DesktopManagerCtx) ResetKeys() { + if manager.waylandInput != nil { + if err := manager.waylandInput.resetKeys(); err != nil { + manager.logger.Warn().Err(err).Msg("Wayland key reset failed") + } + return + } xorg.ResetKeys() } func (manager *DesktopManagerCtx) ScreenConfigurations() []types.ScreenSize { + if manager.waylandInput != nil { + return []types.ScreenSize{manager.screenSize} + } var configs []types.ScreenSize for _, size := range xorg.ScreenConfigurations { for _, fps := range size.Rates { @@ -100,6 +152,21 @@ func (manager *DesktopManagerCtx) ScreenConfigurations() []types.ScreenSize { } func (manager *DesktopManagerCtx) SetScreenSize(screenSize types.ScreenSize) (types.ScreenSize, error) { + if manager.waylandInput != nil { + input, err := newWaylandInput(screenSize.Width, screenSize.Height) + if err != nil { + return manager.screenSize, err + } + mu.Lock() + manager.emmiter.Emit("before_screen_size_change") + oldInput := manager.waylandInput + manager.waylandInput = input + manager.screenSize = screenSize + manager.emmiter.Emit("after_screen_size_change") + mu.Unlock() + oldInput.close() + return screenSize, nil + } mu.Lock() manager.emmiter.Emit("before_screen_size_change") @@ -118,10 +185,16 @@ func (manager *DesktopManagerCtx) SetScreenSize(screenSize types.ScreenSize) (ty } func (manager *DesktopManagerCtx) GetScreenSize() types.ScreenSize { + if manager.waylandInput != nil { + return manager.screenSize + } return xorg.GetScreenSize() } func (manager *DesktopManagerCtx) SetKeyboardMap(kbd types.KeyboardMap) error { + if manager.waylandInput != nil { + return nil + } // TOOD: Use native API. cmd := exec.Command("setxkbmap", "-layout", kbd.Layout, "-variant", kbd.Variant) _, err := cmd.Output() @@ -129,6 +202,9 @@ func (manager *DesktopManagerCtx) SetKeyboardMap(kbd types.KeyboardMap) error { } func (manager *DesktopManagerCtx) GetKeyboardMap() (*types.KeyboardMap, error) { + if manager.waylandInput != nil { + return &types.KeyboardMap{}, nil + } // TOOD: Use native API. cmd := exec.Command("setxkbmap", "-query") res, err := cmd.Output() @@ -154,6 +230,9 @@ func (manager *DesktopManagerCtx) GetKeyboardMap() (*types.KeyboardMap, error) { } func (manager *DesktopManagerCtx) SetKeyboardModifiers(mod types.KeyboardModifiers) { + if manager.waylandInput != nil { + return + } if mod.Shift != nil { xorg.SetKeyboardModifier(xorg.KbdModShift, *mod.Shift) } @@ -188,6 +267,9 @@ func (manager *DesktopManagerCtx) SetKeyboardModifiers(mod types.KeyboardModifie } func (manager *DesktopManagerCtx) GetKeyboardModifiers() types.KeyboardModifiers { + if manager.waylandInput != nil { + return types.KeyboardModifiers{} + } modifiers := xorg.GetKeyboardModifiers() isset := func(mod xorg.KbdMod) *bool { @@ -208,9 +290,15 @@ func (manager *DesktopManagerCtx) GetKeyboardModifiers() types.KeyboardModifiers } func (manager *DesktopManagerCtx) GetCursorImage() *types.CursorImage { + if manager.waylandInput != nil { + return nil + } return xorg.GetCursorImage() } func (manager *DesktopManagerCtx) GetScreenshotImage() *image.RGBA { + if manager.waylandInput != nil { + return nil + } return xorg.GetScreenshotImage() } diff --git a/webpage/docs/configuration/desktop.md b/webpage/docs/configuration/desktop.md index b492e9cec..41c6fa3a5 100644 --- a/webpage/docs/configuration/desktop.md +++ b/webpage/docs/configuration/desktop.md @@ -10,14 +10,16 @@ import configOptions from './help.json'; This section describes how to configure the desktop environment inside neko. -Neko uses the [X Server](https://www.x.org/archive/X11R7.6/doc/man/man1/Xserver.1.xhtml) as the display server with [Openbox](http://openbox.org/wiki/Main_Page) as the default window manager. For audio, [PulseAudio](https://www.freedesktop.org/wiki/Software/PulseAudio/) is used. +Neko uses the [X Server](https://www.x.org/archive/X11R7.6/doc/man/man1/Xserver.1.xhtml) as the display server with [Openbox](http://openbox.org/wiki/Main_Page) as the default window manager. For audio, [PulseAudio](https://www.freedesktop.org/wiki/Software/PulseAudio/) is used. Set to use a Wayland compositor and `/dev/uinput` for desktop input instead. - refers to the X server that is running on the system. If it is not specified, the environment variable `DISPLAY` is used. The same display is referred to in the [Capture](capture#video.display) configuration to capture the screen. In most cases, we want to use the same display for both. +- disables X11 desktop initialization and injects pointer and keyboard events through a `/dev/uinput` virtual device. The container must have access to `/dev/uinput`; the compositor must accept libinput devices. - refers to the screen resolution and refresh rate. The format is `x@`. If not specified, the default is `1280x720@30`. :::tip diff --git a/webpage/docs/configuration/help.json b/webpage/docs/configuration/help.json index e97ccbf23..dc5495a33 100644 --- a/webpage/docs/configuration/help.json +++ b/webpage/docs/configuration/help.json @@ -260,6 +260,15 @@ "type": "string", "description": "X display to use for desktop sharing" }, + { + "key": [ + "desktop", + "wayland" + ], + "type": "boolean", + "defaultValue": false, + "description": "use Wayland desktop input and screen management" + }, { "key": [ "desktop", From d87791554fba00b27d849a87a8f8fb2202561c05 Mon Sep 17 00:00:00 2001 From: Chris Lee <23645059+chruffins@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:45:24 +0000 Subject: [PATCH 3/5] Support Wayland output resizing --- server/internal/config/desktop.go | 18 ++++++++++++++++-- server/internal/desktop/wayland.go | 6 ++++++ server/internal/desktop/xorg.go | 11 ++++++++++- webpage/docs/configuration/desktop.md | 4 ++++ webpage/docs/configuration/help.json | 20 ++++++++++++++++++++ 5 files changed, 56 insertions(+), 3 deletions(-) diff --git a/server/internal/config/desktop.go b/server/internal/config/desktop.go index a1c835789..51a9508bf 100644 --- a/server/internal/config/desktop.go +++ b/server/internal/config/desktop.go @@ -13,8 +13,10 @@ import ( ) type Desktop struct { - Display string - Wayland bool + Display string + Wayland bool + WaylandOutput string + WaylandResizeCommand string ScreenSize types.ScreenSize @@ -37,6 +39,16 @@ func (Desktop) Init(cmd *cobra.Command) error { return err } + cmd.PersistentFlags().String("desktop.wayland.output", "HEADLESS-1", "Wayland output name used for resizing") + if err := viper.BindPFlag("desktop.wayland.output", cmd.PersistentFlags().Lookup("desktop.wayland.output")); err != nil { + return err + } + + cmd.PersistentFlags().String("desktop.wayland.resize_command", "wlr-randr", "Wayland output resize executable") + if err := viper.BindPFlag("desktop.wayland.resize_command", cmd.PersistentFlags().Lookup("desktop.wayland.resize_command")); err != nil { + return err + } + cmd.PersistentFlags().String("desktop.screen", "1280x720@30", "default screen size and framerate") if err := viper.BindPFlag("desktop.screen", cmd.PersistentFlags().Lookup("desktop.screen")); err != nil { return err @@ -82,6 +94,8 @@ func (Desktop) InitV2(cmd *cobra.Command) error { func (s *Desktop) Set() { s.Display = viper.GetString("desktop.display") s.Wayland = viper.GetBool("desktop.wayland") + s.WaylandOutput = viper.GetString("desktop.wayland.output") + s.WaylandResizeCommand = viper.GetString("desktop.wayland.resize_command") // Display is provided by env variable unless explicitly set if s.Display == "" { diff --git a/server/internal/desktop/wayland.go b/server/internal/desktop/wayland.go index e847e131a..6e200e873 100644 --- a/server/internal/desktop/wayland.go +++ b/server/internal/desktop/wayland.go @@ -169,6 +169,12 @@ func (input *waylandInput) sync() error { return input.emit(evSyn, 0, 0) } +func (input *waylandInput) position() (int, int) { + input.mu.Lock() + defer input.mu.Unlock() + return input.cursorX, input.cursorY +} + func (input *waylandInput) move(x, y int) error { input.mu.Lock() defer input.mu.Unlock() diff --git a/server/internal/desktop/xorg.go b/server/internal/desktop/xorg.go index d61f0ab03..1c18e827d 100644 --- a/server/internal/desktop/xorg.go +++ b/server/internal/desktop/xorg.go @@ -1,6 +1,7 @@ package desktop import ( + "fmt" "image" "os/exec" "regexp" @@ -22,7 +23,7 @@ func (manager *DesktopManagerCtx) Move(x, y int) { func (manager *DesktopManagerCtx) GetCursorPosition() (int, int) { if manager.waylandInput != nil { - return manager.waylandInput.cursorX, manager.waylandInput.cursorY + return manager.waylandInput.position() } return xorg.GetCursorPosition() } @@ -153,6 +154,14 @@ func (manager *DesktopManagerCtx) ScreenConfigurations() []types.ScreenSize { func (manager *DesktopManagerCtx) SetScreenSize(screenSize types.ScreenSize) (types.ScreenSize, error) { if manager.waylandInput != nil { + if manager.config.WaylandResizeCommand == "" || manager.config.WaylandOutput == "" { + return manager.screenSize, fmt.Errorf("Wayland output resize is not configured") + } + resize := fmt.Sprintf("%dx%d", screenSize.Width, screenSize.Height) + if err := exec.Command(manager.config.WaylandResizeCommand, "--output", manager.config.WaylandOutput, "--mode", resize).Run(); err != nil { + return manager.screenSize, fmt.Errorf("resize Wayland output: %w", err) + } + input, err := newWaylandInput(screenSize.Width, screenSize.Height) if err != nil { return manager.screenSize, err diff --git a/webpage/docs/configuration/desktop.md b/webpage/docs/configuration/desktop.md index 41c6fa3a5..e6635a14f 100644 --- a/webpage/docs/configuration/desktop.md +++ b/webpage/docs/configuration/desktop.md @@ -15,11 +15,15 @@ Neko uses the [X Server](https://www.x.org/archive/X11R7.6/doc/man/man1/Xserver. - refers to the X server that is running on the system. If it is not specified, the environment variable `DISPLAY` is used. The same display is referred to in the [Capture](capture#video.display) configuration to capture the screen. In most cases, we want to use the same display for both. - disables X11 desktop initialization and injects pointer and keyboard events through a `/dev/uinput` virtual device. The container must have access to `/dev/uinput`; the compositor must accept libinput devices. +- is the compositor output name passed to the resize command. The default `HEADLESS-1` matches the wlroots headless backend. +- is the executable used to resize the output. It must accept `--output --mode x`, as `wlr-randr` does. - refers to the screen resolution and refresh rate. The format is `x@`. If not specified, the default is `1280x720@30`. :::tip diff --git a/webpage/docs/configuration/help.json b/webpage/docs/configuration/help.json index dc5495a33..19162bfb8 100644 --- a/webpage/docs/configuration/help.json +++ b/webpage/docs/configuration/help.json @@ -269,6 +269,26 @@ "defaultValue": false, "description": "use Wayland desktop input and screen management" }, + { + "key": [ + "desktop", + "wayland", + "output" + ], + "type": "string", + "defaultValue": "HEADLESS-1", + "description": "Wayland output name used for resizing" + }, + { + "key": [ + "desktop", + "wayland", + "resize_command" + ], + "type": "string", + "defaultValue": "wlr-randr", + "description": "Wayland output resize executable" + }, { "key": [ "desktop", From cfd25839651aa21662b27728121ea871acdb7a5f Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:08:45 +0000 Subject: [PATCH 4/5] Harden Wayland capture and input --- server/internal/capture/manager.go | 8 +- server/internal/config/desktop.go | 2 +- server/internal/desktop/manager.go | 6 + server/internal/desktop/wayland.go | 201 +++++++++++++++++++++++- server/internal/desktop/wayland_test.go | 8 +- server/internal/desktop/xorg.go | 58 +++---- 6 files changed, 236 insertions(+), 47 deletions(-) diff --git a/server/internal/capture/manager.go b/server/internal/capture/manager.go index d5ac42c97..4cf4e6736 100644 --- a/server/internal/capture/manager.go +++ b/server/internal/capture/manager.go @@ -53,14 +53,10 @@ func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCt } if config.Wayland { - fps := screen.Rate - if fps <= 0 { - fps = 25 - } return fmt.Sprintf( "appsrc name=appsrc is-live=true format=time do-timestamp=true "+ - "caps=video/x-raw,format=BGRx,width=%d,height=%d,framerate=%d/1 "+ - "%s ! appsink name=appsink", screen.Width, screen.Height, fps, pipeline, + "caps=video/x-raw,format=BGRx,width=%d,height=%d "+ + "%s ! appsink name=appsink", screen.Width, screen.Height, pipeline, ), nil } diff --git a/server/internal/config/desktop.go b/server/internal/config/desktop.go index 51a9508bf..d22b36b01 100644 --- a/server/internal/config/desktop.go +++ b/server/internal/config/desktop.go @@ -44,7 +44,7 @@ func (Desktop) Init(cmd *cobra.Command) error { return err } - cmd.PersistentFlags().String("desktop.wayland.resize_command", "wlr-randr", "Wayland output resize executable") + cmd.PersistentFlags().String("desktop.wayland.resize_command", "", "Wayland output resize executable") if err := viper.BindPFlag("desktop.wayland.resize_command", cmd.PersistentFlags().Lookup("desktop.wayland.resize_command")); err != nil { return err } diff --git a/server/internal/desktop/manager.go b/server/internal/desktop/manager.go index 390827512..f94fdbc24 100644 --- a/server/internal/desktop/manager.go +++ b/server/internal/desktop/manager.go @@ -27,6 +27,7 @@ type DesktopManagerCtx struct { config *config.Desktop screenSize types.ScreenSize // cached screen size input xinput.Driver + waylandMu sync.RWMutex waylandInput *waylandInput // Clipboard process holding the most recent clipboard data. @@ -60,7 +61,9 @@ func (manager *DesktopManagerCtx) Start() { if err != nil { manager.logger.Panic().Err(err).Msg("unable to create Wayland input device") } + manager.waylandMu.Lock() manager.waylandInput = input + manager.waylandMu.Unlock() manager.logger.Info().Str("screen_size", manager.screenSize.String()).Msg("using Wayland desktop backend") return } @@ -150,11 +153,14 @@ func (manager *DesktopManagerCtx) Shutdown() error { manager.logger.Info().Msgf("shutdown") close(manager.shutdown) + manager.waylandMu.Lock() if manager.waylandInput != nil { manager.waylandInput.close() manager.waylandInput = nil + manager.waylandMu.Unlock() return nil } + manager.waylandMu.Unlock() manager.replaceClipboardCommand(nil) manager.wg.Wait() diff --git a/server/internal/desktop/wayland.go b/server/internal/desktop/wayland.go index 6e200e873..417f650f4 100644 --- a/server/internal/desktop/wayland.go +++ b/server/internal/desktop/wayland.go @@ -74,6 +74,12 @@ type waylandInput struct { pressed map[uint16]struct{} } +func (manager *DesktopManagerCtx) getWaylandInput() *waylandInput { + manager.waylandMu.RLock() + defer manager.waylandMu.RUnlock() + return manager.waylandInput +} + func newWaylandInput(width, height int) (*waylandInput, error) { if width <= 0 || height <= 0 { return nil, fmt.Errorf("invalid Wayland input size: %dx%d", width, height) @@ -111,7 +117,7 @@ func (input *waylandInput) create() error { return fmt.Errorf("enable uinput event type %d: %w", eventType, err) } } - for key := 0; key <= 0xff; key++ { + for key := 0; key <= 0x1ff; key++ { if err := input.ioctl(uiSetKeybit, uintptr(key)); err != nil { return fmt.Errorf("enable uinput key %d: %w", key, err) } @@ -233,9 +239,16 @@ func (input *waylandInput) key(code uint32, down bool) error { return nil } -func (input *waylandInput) scroll(deltaX, deltaY int) error { +func (input *waylandInput) scroll(deltaX, deltaY int, controlKey bool) error { input.mu.Lock() defer input.mu.Unlock() + + temporaryControl := controlKey && !input.isPressed(keyLeftCtrl) && !input.isPressed(keyRightCtrl) + if temporaryControl { + if err := input.emit(evKey, keyLeftCtrl, 1); err != nil { + return err + } + } if deltaY != 0 { if err := input.emit(evRel, relWheel, int32(-deltaY)); err != nil { return err @@ -246,9 +259,19 @@ func (input *waylandInput) scroll(deltaX, deltaY int) error { return err } } + if temporaryControl { + if err := input.emit(evKey, keyLeftCtrl, 0); err != nil { + return err + } + } return input.sync() } +func (input *waylandInput) isPressed(code uint16) bool { + _, ok := input.pressed[code] + return ok +} + func (input *waylandInput) resetKeys() error { input.mu.Lock() defer input.mu.Unlock() @@ -274,11 +297,175 @@ func mapButton(code uint32) (uint16, bool) { } } -func mapKey(code uint32) (uint16, bool) { - if code < 8 || code > 263 { - return 0, false - } - return uint16(code - 8), true +const ( + keyEsc = 1 + key1 = 2 + key2 = 3 + key3 = 4 + key4 = 5 + key5 = 6 + key6 = 7 + key7 = 8 + key8 = 9 + key9 = 10 + key0 = 11 + keyMinus = 12 + keyEqual = 13 + keyBackspace = 14 + keyTab = 15 + keyQ = 16 + keyW = 17 + keyE = 18 + keyR = 19 + keyT = 20 + keyY = 21 + keyU = 22 + keyI = 23 + keyO = 24 + keyP = 25 + keyLeftBrace = 26 + keyRightBrace = 27 + keyEnter = 28 + keyLeftCtrl = 29 + keyA = 30 + keyS = 31 + keyD = 32 + keyF = 33 + keyG = 34 + keyH = 35 + keyJ = 36 + keyK = 37 + keyL = 38 + keySemicolon = 39 + keyApostrophe = 40 + keyGrave = 41 + keyLeftShift = 42 + keyBackslash = 43 + keyZ = 44 + keyX = 45 + keyC = 46 + keyV = 47 + keyB = 48 + keyN = 49 + keyM = 50 + keyComma = 51 + keyDot = 52 + keySlash = 53 + keyRightShift = 54 + keyLeftAlt = 56 + keySpace = 57 + keyCapsLock = 58 + keyF1 = 59 + keyF10 = 68 + keyRightCtrl = 97 + keyRightAlt = 100 + keyHome = 102 + keyUp = 103 + keyPageUp = 104 + keyLeft = 105 + keyRight = 106 + keyEnd = 107 + keyDown = 108 + keyPageDown = 109 + keyInsert = 110 + keyDelete = 111 +) + +func mapKey(keysym uint32) (uint16, bool) { + if keysym >= 'a' && keysym <= 'z' { + return keyA + uint16(keysym-'a'), true + } + if keysym >= 'A' && keysym <= 'Z' { + return keyA + uint16(keysym-'A'), true + } + if keysym >= '1' && keysym <= '9' { + return key1 + uint16(keysym-'1'), true + } + if keysym == '0' { + return key0, true + } + + switch keysym { + case 0xff1b: + return keyEsc, true + case 0xff08: + return keyBackspace, true + case 0xff09: + return keyTab, true + case 0xff0d: + return keyEnter, true + case 0xffff: + return keyDelete, true + case 0xff63: + return keyInsert, true + case 0xff50: + return keyHome, true + case 0xff51: + return keyLeft, true + case 0xff52: + return keyUp, true + case 0xff53: + return keyRight, true + case 0xff54: + return keyDown, true + case 0xff55: + return keyPageUp, true + case 0xff56: + return keyPageDown, true + case 0xff57: + return keyEnd, true + case 0xffe1: + return keyLeftShift, true + case 0xffe2: + return keyRightShift, true + case 0xffe3: + return keyLeftCtrl, true + case 0xffe4: + return keyRightCtrl, true + case 0xffe9: + return keyLeftAlt, true + case 0xffea: + return keyRightAlt, true + } + if keysym >= 0xffbe && keysym <= 0xffc7 { + return keyF1 + uint16(keysym-0xffbe), true + } + if keysym == 0xffc8 { + return 87, true + } + if keysym == 0xffc9 { + return 88, true + } + + switch keysym { + case '-', '_': + return keyMinus, true + case '=', '+': + return keyEqual, true + case '[', '{': + return keyLeftBrace, true + case ']', '}': + return keyRightBrace, true + case '\\', '|': + return keyBackslash, true + case ';', ':': + return keySemicolon, true + case '\'', '"': + return keyApostrophe, true + case '`', '~': + return keyGrave, true + case ',', '<': + return keyComma, true + case '.', '>': + return keyDot, true + case '/', '?': + return keySlash, true + case ' ': + return keySpace, true + case 0xffe5: + return keyCapsLock, true + } + return 0, false } func boolValue(value bool) int32 { diff --git a/server/internal/desktop/wayland_test.go b/server/internal/desktop/wayland_test.go index 8bbd2531f..94a6afe05 100644 --- a/server/internal/desktop/wayland_test.go +++ b/server/internal/desktop/wayland_test.go @@ -2,15 +2,15 @@ package desktop import "testing" -func TestMapKeyConvertsX11Keycodes(t *testing.T) { +func TestMapKeyConvertsX11Keysyms(t *testing.T) { for _, test := range []struct { name string in uint32 want uint16 }{ - {name: "escape", in: 9, want: 1}, - {name: "a", in: 38, want: 30}, - {name: "f12", in: 96, want: 88}, + {name: "escape", in: 0xff1b, want: keyEsc}, + {name: "a", in: 'a', want: keyA}, + {name: "f12", in: 0xffc9, want: 88}, } { t.Run(test.name, func(t *testing.T) { got, ok := mapKey(test.in) diff --git a/server/internal/desktop/xorg.go b/server/internal/desktop/xorg.go index 1c18e827d..5e7667d95 100644 --- a/server/internal/desktop/xorg.go +++ b/server/internal/desktop/xorg.go @@ -12,8 +12,8 @@ import ( ) func (manager *DesktopManagerCtx) Move(x, y int) { - if manager.waylandInput != nil { - if err := manager.waylandInput.move(x, y); err != nil { + if input := manager.getWaylandInput(); input != nil { + if err := input.move(x, y); err != nil { manager.logger.Warn().Err(err).Msg("Wayland pointer move failed") } return @@ -22,15 +22,15 @@ func (manager *DesktopManagerCtx) Move(x, y int) { } func (manager *DesktopManagerCtx) GetCursorPosition() (int, int) { - if manager.waylandInput != nil { - return manager.waylandInput.position() + if input := manager.getWaylandInput(); input != nil { + return input.position() } return xorg.GetCursorPosition() } func (manager *DesktopManagerCtx) Scroll(deltaX, deltaY int, controlKey bool) { - if manager.waylandInput != nil { - returnErr := manager.waylandInput.scroll(deltaX, deltaY) + if input := manager.getWaylandInput(); input != nil { + returnErr := input.scroll(deltaX, deltaY, controlKey) if returnErr != nil { manager.logger.Warn().Err(returnErr).Msg("Wayland scroll failed") } @@ -54,35 +54,35 @@ func (manager *DesktopManagerCtx) Scroll(deltaX, deltaY int, controlKey bool) { } func (manager *DesktopManagerCtx) ButtonDown(code uint32) error { - if manager.waylandInput != nil { - return manager.waylandInput.button(code, true) + if input := manager.getWaylandInput(); input != nil { + return input.button(code, true) } return xorg.ButtonDown(code) } func (manager *DesktopManagerCtx) KeyDown(code uint32) error { - if manager.waylandInput != nil { - return manager.waylandInput.key(code, true) + if input := manager.getWaylandInput(); input != nil { + return input.key(code, true) } return xorg.KeyDown(code) } func (manager *DesktopManagerCtx) ButtonUp(code uint32) error { - if manager.waylandInput != nil { - return manager.waylandInput.button(code, false) + if input := manager.getWaylandInput(); input != nil { + return input.button(code, false) } return xorg.ButtonUp(code) } func (manager *DesktopManagerCtx) KeyUp(code uint32) error { - if manager.waylandInput != nil { - return manager.waylandInput.key(code, false) + if input := manager.getWaylandInput(); input != nil { + return input.key(code, false) } return xorg.KeyUp(code) } func (manager *DesktopManagerCtx) ButtonPress(code uint32) error { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { manager.ResetKeys() defer manager.ResetKeys() return manager.ButtonDown(code) @@ -94,7 +94,7 @@ func (manager *DesktopManagerCtx) ButtonPress(code uint32) error { } func (manager *DesktopManagerCtx) KeyPress(codes ...uint32) error { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { manager.ResetKeys() defer manager.ResetKeys() for _, code := range codes { @@ -121,8 +121,8 @@ func (manager *DesktopManagerCtx) KeyPress(codes ...uint32) error { } func (manager *DesktopManagerCtx) ResetKeys() { - if manager.waylandInput != nil { - if err := manager.waylandInput.resetKeys(); err != nil { + if input := manager.getWaylandInput(); input != nil { + if err := input.resetKeys(); err != nil { manager.logger.Warn().Err(err).Msg("Wayland key reset failed") } return @@ -131,7 +131,7 @@ func (manager *DesktopManagerCtx) ResetKeys() { } func (manager *DesktopManagerCtx) ScreenConfigurations() []types.ScreenSize { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { return []types.ScreenSize{manager.screenSize} } var configs []types.ScreenSize @@ -153,7 +153,7 @@ func (manager *DesktopManagerCtx) ScreenConfigurations() []types.ScreenSize { } func (manager *DesktopManagerCtx) SetScreenSize(screenSize types.ScreenSize) (types.ScreenSize, error) { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { if manager.config.WaylandResizeCommand == "" || manager.config.WaylandOutput == "" { return manager.screenSize, fmt.Errorf("Wayland output resize is not configured") } @@ -166,13 +166,13 @@ func (manager *DesktopManagerCtx) SetScreenSize(screenSize types.ScreenSize) (ty if err != nil { return manager.screenSize, err } - mu.Lock() + manager.waylandMu.Lock() manager.emmiter.Emit("before_screen_size_change") oldInput := manager.waylandInput manager.waylandInput = input manager.screenSize = screenSize manager.emmiter.Emit("after_screen_size_change") - mu.Unlock() + manager.waylandMu.Unlock() oldInput.close() return screenSize, nil } @@ -194,14 +194,14 @@ func (manager *DesktopManagerCtx) SetScreenSize(screenSize types.ScreenSize) (ty } func (manager *DesktopManagerCtx) GetScreenSize() types.ScreenSize { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { return manager.screenSize } return xorg.GetScreenSize() } func (manager *DesktopManagerCtx) SetKeyboardMap(kbd types.KeyboardMap) error { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { return nil } // TOOD: Use native API. @@ -211,7 +211,7 @@ func (manager *DesktopManagerCtx) SetKeyboardMap(kbd types.KeyboardMap) error { } func (manager *DesktopManagerCtx) GetKeyboardMap() (*types.KeyboardMap, error) { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { return &types.KeyboardMap{}, nil } // TOOD: Use native API. @@ -239,7 +239,7 @@ func (manager *DesktopManagerCtx) GetKeyboardMap() (*types.KeyboardMap, error) { } func (manager *DesktopManagerCtx) SetKeyboardModifiers(mod types.KeyboardModifiers) { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { return } if mod.Shift != nil { @@ -276,7 +276,7 @@ func (manager *DesktopManagerCtx) SetKeyboardModifiers(mod types.KeyboardModifie } func (manager *DesktopManagerCtx) GetKeyboardModifiers() types.KeyboardModifiers { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { return types.KeyboardModifiers{} } modifiers := xorg.GetKeyboardModifiers() @@ -299,14 +299,14 @@ func (manager *DesktopManagerCtx) GetKeyboardModifiers() types.KeyboardModifiers } func (manager *DesktopManagerCtx) GetCursorImage() *types.CursorImage { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { return nil } return xorg.GetCursorImage() } func (manager *DesktopManagerCtx) GetScreenshotImage() *image.RGBA { - if manager.waylandInput != nil { + if manager.getWaylandInput() != nil { return nil } return xorg.GetScreenshotImage() From 9766fb90b59a7e8b2b4be8e91ed1da91dcf2159b Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:15:36 +0000 Subject: [PATCH 5/5] Handle missing Wayland cursor images --- server/internal/webrtc/cursor/image.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/internal/webrtc/cursor/image.go b/server/internal/webrtc/cursor/image.go index cd0c024b6..2e004a088 100644 --- a/server/internal/webrtc/cursor/image.go +++ b/server/internal/webrtc/cursor/image.go @@ -154,6 +154,9 @@ func (manager *image) RemoveListener(listener ImageListener) { func (manager *image) fetchEntry() (*imageEntry, error) { cur := manager.desktop.GetCursorImage() + if cur == nil { + return &imageEntry{CursorImage: &types.CursorImage{}}, nil + } img, err := utils.CreatePNGImage(cur.Image) if err != nil {