diff --git a/.gitignore b/.gitignore index d4d065e..504ddb5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ oracle.env riddle/dist/ drawlab/dist/ home/dist/ +riddle/riddle-rm2 +.DS_Store diff --git a/README-RM2.md b/README-RM2.md new file mode 100644 index 0000000..c2cf244 --- /dev/null +++ b/README-RM2.md @@ -0,0 +1,49 @@ +# riddle for reMarkable 2 — no XOVI or AppLoad + +This build runs beside the stock reMarkable UI on RM2 firmware 3.28. It reads +real Wacom strokes without grabbing the device, sends a private reconstruction +of the writing to the configured vision model, and replays the reply through +the Wacom input device. xochitl records Tom's handwriting in the open notebook +as ordinary ink. + +## Install + +Copy the extracted `riddle-rm2` directory to `/home/root/riddle-rm2`, then: + +```sh +cd /home/root/riddle-rm2 +cp oracle.env.example oracle.env +vi oracle.env # set RIDDLE_OPENAI_KEY +chmod +x riddle-rm2 start-rm2.sh stop-rm2.sh +``` + +No root filesystem files, startup hooks, XOVI, AppLoad, or systemd units are +installed. + +## Use + +Open a blank notebook in the stock UI and write in its upper third. From SSH: + +```sh +/home/root/riddle-rm2/start-rm2.sh +``` + +After a 2.8 second pause, Tom writes back through xochitl. Stop it before using +other parts of the UI: + +```sh +/home/root/riddle-rm2/stop-rm2.sh +``` + +This companion build intentionally does not erase/fade existing notebook ink +and disables the takeover-only help and memory-page animations. The original +notebook remains a normal reMarkable document and syncs normally. + +## Remove + +```sh +/home/root/riddle-rm2/stop-rm2.sh +rm -rf /home/root/riddle-rm2 /home/root/riddle-data +``` + +Rebooting also stops it because no autostart component is installed. diff --git a/riddle/Cargo.toml b/riddle/Cargo.toml index 5a6924d..e561637 100644 --- a/riddle/Cargo.toml +++ b/riddle/Cargo.toml @@ -3,7 +3,7 @@ name = "riddle" version = "0.3.0" edition = "2021" license = "MIT" -description = "The diary of Tom Riddle, for the reMarkable Paper Pro" +description = "The diary of Tom Riddle, ported to the reMarkable 2" [dependencies] libc = "0.2" diff --git a/riddle/appload-rm2.sh b/riddle/appload-rm2.sh new file mode 100755 index 0000000..eb718ae --- /dev/null +++ b/riddle/appload-rm2.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")" +if [ -f oracle.env ]; then + set -a + . ./oracle.env + set +a +fi +exec ./riddle-rm2 diff --git a/riddle/build-rm2.sh b/riddle/build-rm2.sh new file mode 100755 index 0000000..ca76741 --- /dev/null +++ b/riddle/build-rm2.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Build the windowed reMarkable 2 AppLoad binary. +set -eu +cd "$(dirname "$0")" + +TARGET=armv7-unknown-linux-gnueabihf +if command -v cross >/dev/null 2>&1; then + cross build --release --target "$TARGET" +else + : "${CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER:=arm-linux-gnueabihf-gcc}" + export CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER + cargo build --release --target "$TARGET" +fi + +cp "target/$TARGET/release/riddle" riddle-rm2 +echo "built: $(pwd)/riddle-rm2" diff --git a/riddle/external.manifest.json b/riddle/external.manifest.json index 76af257..731a439 100644 --- a/riddle/external.manifest.json +++ b/riddle/external.manifest.json @@ -1,8 +1,9 @@ { "id": "riddle", "name": "The Diary", - "version": "0.3.0", + "version": "0.3.0-rm2.2", "description": "An enchanted diary: write to it with the pen — it writes back, and it remembers.", - "application": "appload-launch.sh", - "qtfb": false + "application": "appload-rm2.sh", + "qtfb": true, + "aspectRatio": "original" } diff --git a/riddle/fonts/NotoSansTC-VF.ttf b/riddle/fonts/NotoSansTC-VF.ttf new file mode 100644 index 0000000..fa89e00 Binary files /dev/null and b/riddle/fonts/NotoSansTC-VF.ttf differ diff --git a/riddle/src/display.rs b/riddle/src/display.rs index 7ca77e5..c3a984f 100644 --- a/riddle/src/display.rs +++ b/riddle/src/display.rs @@ -7,6 +7,9 @@ use std::io; pub enum Display { Qtfb(crate::qtfb::QtfbClient), + /// Headless companion mode: xochitl owns the real panel while riddle + /// renders into this private RGB565 page for OCR and layout. + Xochitl(Box<[u8]>), #[allow(dead_code)] Quill, } @@ -31,18 +34,36 @@ impl Display { let key: i32 = key.parse().map_err(io::Error::other)?; let mut client = crate::qtfb::QtfbClient::connect( key, - crate::qtfb::FBFMT_RMPP_RGB565, - 1620, - 2160, + crate::qtfb::FBFMT_RM2FB, + crate::fb::SCREEN_W, + crate::fb::SCREEN_H, 2, )?; let _ = client.set_refresh_mode(crate::qtfb::REFRESH_MODE_UFAST); let buf = client.framebuffer(); let (ptr, len) = (buf.as_mut_ptr(), buf.len()); - let surface = Surface::new(ptr, len, 1620, 2160, 1620 * 2, PixFmt::Rgb565); + let surface = Surface::new( + ptr, len, crate::fb::SCREEN_W, crate::fb::SCREEN_H, + crate::fb::SCREEN_W * 2, PixFmt::Rgb565, + ); return Ok((Display::Qtfb(client), surface)); } + // reMarkable 2 needs no launcher: keep the stock UI running and use + // its Wacom input path to replay Tom's generated handwriting. + #[cfg(not(feature = "takeover"))] + { + let mut page = vec![0xff; crate::fb::SCREEN_W * crate::fb::SCREEN_H * 2] + .into_boxed_slice(); + let ptr = page.as_mut_ptr(); + let len = page.len(); + let surface = Surface::new( + ptr, len, crate::fb::SCREEN_W, crate::fb::SCREEN_H, + crate::fb::SCREEN_W * 2, PixFmt::Rgb565, + ); + return Ok((Display::Xochitl(page), surface)); + } + #[cfg(feature = "takeover")] { unsafe { @@ -60,10 +81,6 @@ impl Display { Ok((Display::Quill, surface)) } } - #[cfg(not(feature = "takeover"))] - Err(io::Error::other( - "QTFB_KEY not set and this build has no takeover backend", - )) } /// Push a region to the panel. `fast` selects the low-latency waveform. @@ -72,6 +89,7 @@ impl Display { Display::Qtfb(c) => { let _ = c.update_partial(x, y, w, h); } + Display::Xochitl(_) => {} #[allow(unused_variables)] Display::Quill => { #[cfg(feature = "takeover")] @@ -89,6 +107,7 @@ impl Display { Display::Qtfb(c) => { let _ = c.update_all(); } + Display::Xochitl(_) => {} #[allow(unused_variables)] Display::Quill => { #[cfg(feature = "takeover")] @@ -107,6 +126,7 @@ impl Display { Display::Qtfb(c) => { let _ = c.request_full_refresh(); } + Display::Xochitl(_) => {} #[allow(unused_variables)] Display::Quill => { #[cfg(feature = "takeover")] @@ -124,6 +144,7 @@ impl Display { pub fn pump(&self) -> io::Result> { match self { Display::Qtfb(c) => c.drain_events(), + Display::Xochitl(_) => Ok(Vec::new()), Display::Quill => { #[cfg(feature = "takeover")] unsafe { diff --git a/riddle/src/fb.rs b/riddle/src/fb.rs index 0a596af..ac44313 100644 --- a/riddle/src/fb.rs +++ b/riddle/src/fb.rs @@ -1,7 +1,8 @@ //! Geometry helpers. Drawing lives in surface.rs. -pub const SCREEN_W: usize = 1620; -pub const SCREEN_H: usize = 2160; +// reMarkable 1/2 native portrait framebuffer. +pub const SCREEN_W: usize = 1404; +pub const SCREEN_H: usize = 1872; /// Grow-only pixel bounding box, used to build update/dissolve regions. #[derive(Clone, Copy, Debug)] diff --git a/riddle/src/main.rs b/riddle/src/main.rs index 24601b3..7a111a8 100644 --- a/riddle/src/main.rs +++ b/riddle/src/main.rs @@ -1,4 +1,4 @@ -//! riddle — the diary of Tom Riddle, for the reMarkable Paper Pro. +//! riddle — the diary of Tom Riddle, for the reMarkable 2. //! //! Write on the page with the pen. After a pause the diary drinks your ink, //! and an answer writes itself onto the page in a flowing hand, then fades. @@ -38,8 +38,8 @@ const IDLE_COMMIT: Duration = Duration::from_millis(2800); /// How long the diary waits on a silent oracle before giving up on the turn. /// Generous: thinking models can lead with a long silence. const ORACLE_PATIENCE: Duration = Duration::from_secs(120); -const REPLY_PX: f32 = 96.0; -const MARGIN_X: i32 = 120; +const REPLY_PX: f32 = 82.0; +const MARGIN_X: i32 = 100; const USAGE: &str = "\ riddle — the diary of Tom Riddle @@ -179,25 +179,65 @@ fn build_ctx(store: &Option) -> oracle::TurnContext { } fn run() -> std::io::Result<()> { - let font = FontRef::try_from_slice(FONT_TTF).map_err(std::io::Error::other)?; + // Dancing Script has no CJK glyphs. RM2 bundles provide a Traditional + // Chinese font beside the executable; keep its bytes alive for FontRef. + // Resolve relative to the executable, not the CWD — AppLoad may launch + // the binary from an arbitrary working directory. + let exe_dir = std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(|d| d.to_path_buf())); + let mut candidates: Vec = Vec::new(); + if let Ok(p) = std::env::var("RIDDLE_FONT") { + candidates.push(p.into()); + } else { + for rel in ["NotoSansTC-VF.ttf", "fonts/NotoSansTC-VF.ttf"] { + if let Some(ref d) = exe_dir { + candidates.push(d.join(rel)); + } + candidates.push(rel.into()); + } + } + let external_font = candidates + .iter() + .find_map(|p| std::fs::read(p).ok().map(|b| (p.display().to_string(), b))); + let font = match external_font { + Some((ref path, ref bytes)) => match FontRef::try_from_slice(bytes) { + Ok(f) => { + eprintln!("riddle: reply font {path}"); + f + } + Err(e) => { + // A truncated scp of the 12MB font must not brick the diary. + eprintln!("riddle: font {path} unusable ({e}); falling back to embedded Dancing Script"); + FontRef::try_from_slice(FONT_TTF).map_err(std::io::Error::other)? + } + }, + None => { + eprintln!("riddle: no CJK font found, falling back to embedded Dancing Script"); + FontRef::try_from_slice(FONT_TTF).map_err(std::io::Error::other)? + } + }; let (disp, mut surf) = display::Display::open()?; let takeover = matches!(disp, display::Display::Quill); + let xochitl = matches!(disp, display::Display::Xochitl(_)); + if xochitl { std::env::set_var("RIDDLE_XOCHITL", "1"); } eprintln!( "riddle: display {} ({}x{} stride {})", - if takeover { "quill/takeover" } else { "qtfb" }, + if takeover { "quill/takeover" } else if xochitl { "xochitl companion" } else { "qtfb" }, surf.w, surf.h, surf.stride ); - let mut pen_dev = match pen::PenDevice::open() { + let mut pen_dev = match pen::PenDevice::open(xochitl) { Ok(p) => Some(p), Err(e) => { eprintln!("riddle: raw pen unavailable ({e}), falling back to qtfb pen events"); None } }; + let reply_pen = if xochitl { Some(pen::PenWriter::open()?) } else { None }; // Takeover mode: touch is ours too; 5-finger tap = quit. let mut touch_dev = if takeover { touch::TouchDevice::open().ok() } else { None }; // Takeover mode: the power button is ours too (sleep page + suspend). @@ -255,8 +295,19 @@ fn run() -> std::io::Result<()> { let mut stylus_tapped = false; let mut ink_dirty = BBox::empty(); let mut last_flush = Instant::now(); + // qtfb's UFAST waveform under-drives black on freshly-erased panel areas + // (faint grey strokes). DU-class waveforms are cumulative, so re-flushing + // the finished reply region a couple of times develops it to full black. + let mut reink: Option<(BBox, Instant, u8)> = None; + // Companion mode: while injecting a reply, a real pen near the panel + // pauses injection — interleaved streams on the shared Wacom device would + // scribble permanent ink across the user's notebook. + let mut pen_guard = Instant::now(); + let mut pen_lifted = false; // Takeover swaps are cheap and synchronous; qtfb needs coalescing. - let flush_every = if takeover { Duration::from_millis(8) } else { Duration::from_millis(35) }; + // 16ms (~60fps) is the sweet spot for qtfb: fast enough to feel responsive + // without saturating the qtfb socket queue. + let flush_every = if takeover { Duration::from_millis(8) } else { Duration::from_millis(16) }; eprintln!("riddle: the diary is open"); @@ -324,6 +375,17 @@ fn run() -> std::io::Result<()> { // ---- raw pen (preferred path) ---- if let Some(ref mut pdev) = pen_dev { for s in pdev.drain() { + // Companion mode, mid-reply: any sample away from our own + // injected point is the user's real pen (hover included) — + // pause injection so interleaved streams on the shared Wacom + // device don't scribble across the notebook. + if matches!(state, State::Replying { .. }) { + if let Some(ref p) = reply_pen { + if !p.near(s.x, s.y, 60) { + pen_guard = Instant::now() + Duration::from_millis(1200); + } + } + } let writing = s.touching && s.pressure > 40; stylus_on = writing; stylus_tapped |= writing; @@ -354,7 +416,13 @@ fn run() -> std::io::Result<()> { *last_pen = Some(Instant::now()); } State::Lingering { region, .. } => { - state = State::FadingReply { stage: 0, next: Instant::now(), region }; + // Skip loopback echoes of our own injected stroke: + // the reply's final points arrive one drain later and + // must not fade the reply the instant it finishes. + let echo = reply_pen.as_ref().map_or(false, |p| p.near(s.x, s.y, 60)); + if !echo { + state = State::FadingReply { stage: 0, next: Instant::now(), region }; + } } _ => {} } @@ -401,7 +469,7 @@ fn run() -> std::io::Result<()> { } } - // ---- coalesced ink flush ---- + // ---- coalesced ink flush (catch-all for qtfb fallback pen path) ---- if !ink_dirty.is_empty() && last_flush.elapsed() >= flush_every { let (x, y, w, h) = ink_dirty.rect(); disp.update(x, y, w, h, true); @@ -418,7 +486,7 @@ fn run() -> std::io::Result<()> { // commit (and no phantom "?" from erased strokes). user_ink.clear(); State::Listening { last_pen: None } - } else if help::looks_like_question_mark(user_ink.stroke_list()) { + } else if !xochitl && help::looks_like_question_mark(user_ink.stroke_list()) { // Absorb the "?" and open the guide instead of asking. let (qx, qy, qw, qh) = user_ink.bbox.rect(); surf.fill_rect(qx as usize, qy as usize, qw as usize, qh as usize, WHITE); @@ -586,12 +654,20 @@ fn run() -> std::io::Result<()> { rx = None; } } - if Instant::now() >= next { + if reply_pen.is_some() && Instant::now() < pen_guard { + // Real pen near the panel: lift our pen and hold. + if !pen_lifted { + if let Some(ref p) = reply_pen { let _ = p.up(); } + pen_lifted = true; + } + State::Replying { plan, next: Instant::now() + Duration::from_millis(120), rx } + } else if Instant::now() >= next { let mut dirty = BBox::empty(); let mut budget = 26; while budget > 0 && plan.stroke_i < plan.strokes.len() { let stroke = &plan.strokes[plan.stroke_i]; if plan.point_i >= stroke.len() { + if let Some(ref p) = reply_pen { let _ = p.up(); } plan.stroke_i += 1; plan.point_i = 0; continue; @@ -600,8 +676,15 @@ fn run() -> std::io::Result<()> { if plan.point_i > 0 { let (px, py) = stroke[plan.point_i - 1]; surf.brush_line(px, py, x, y, 2, BLACK); + if let Some(ref p) = reply_pen { + // Resuming after a real-pen pause: touch down + // again where the stroke left off. + if pen_lifted { let _ = p.down_at(px, py); pen_lifted = false; } + let _ = p.goto(x, y); + } } else { surf.stamp(x, y, 2, BLACK); + if let Some(ref p) = reply_pen { let _ = p.down_at(x, y); pen_lifted = false; } } dirty.add(x, y, 4); plan.point_i += 1; @@ -612,6 +695,7 @@ fn run() -> std::io::Result<()> { disp.update(x, y, w, h, true); } if plan.stroke_i >= plan.strokes.len() && rx.is_none() { + if let Some(ref p) = reply_pen { let _ = p.up(); } // The turn is complete: the diary remembers it. if !turn_failed && !turn_reply.is_empty() { if let Some(ref mut s) = store { @@ -627,6 +711,9 @@ fn run() -> std::io::Result<()> { let chars: usize = plan.strokes.iter().map(|s| s.len()).sum(); let linger = Duration::from_millis(4000 + (chars as u64) * 2); let region = plan.region; + if !takeover && !xochitl && !region.is_empty() { + reink = Some((region, Instant::now() + Duration::from_millis(250), 2)); + } State::Lingering { until: Instant::now() + linger.min(Duration::from_secs(20)), region } } else { State::Replying { plan, next: Instant::now() + Duration::from_millis(14), rx } @@ -744,8 +831,20 @@ fn run() -> std::io::Result<()> { } }; + // Cumulative re-drive of the finished reply: only while it lingers + // untouched — a fade in progress must not be re-inked over. + if let Some((region, at, left)) = reink { + if !matches!(state, State::Lingering { .. }) { + reink = None; + } else if Instant::now() >= at { + let (x, y, w, h) = region.rect(); + disp.update(x, y, w, h, false); + reink = (left > 1).then(|| (region, at + Duration::from_millis(400), left - 1)); + } + } + stylus_tapped = false; - std::thread::sleep(Duration::from_millis(2)); + std::thread::sleep(Duration::from_millis(1)); } eprintln!("riddle: the diary closes"); @@ -873,10 +972,32 @@ fn plan_reply(font: &FontRef, text: &str, y_start: Option) -> WritePlan { }; for line_text in &lines { - let mut raster = script::rasterize_line(font, line_text, REPLY_PX); - script::thin(&mut raster); - let line_strokes = script::trace(&raster); - let x0 = (SCREEN_W as i32 - raster.width as i32) / 2; + // Zhang-Suen thinning destroys complex CJK crossing strokes (缺筆畫). + // For CJK lines: skip thin() and use scan_rows (row-by-row pixel runs) + // instead of trace() (skeleton walk), which loses strokes on fat blobs. + // Rasterize CJK character by character so the animation writes one + // glyph at a time like a hand, not whole-line scanlines like a fax. + let (line_strokes, line_width) = if script::has_cjk(line_text) { + let mut all: Vec> = Vec::new(); + let mut caret = 0f32; + for c in line_text.chars() { + let s = c.to_string(); + let raster = script::rasterize_line(font, &s, REPLY_PX); + let dx = caret.round() as i32; + for st in script::scan_rows(&raster) { + all.push(st.into_iter().map(|(x, y)| (x + dx, y)).collect()); + } + caret += script::measure(font, &s, REPLY_PX); + } + (all, caret.ceil() as i32 + 4) + } else { + let mut raster = script::rasterize_line(font, line_text, REPLY_PX); + script::thin(&mut raster); + let strokes = script::trace(&raster); + let w = raster.width as i32; + (strokes, w) + }; + let x0 = (SCREEN_W as i32 - line_width) / 2; let wobble = jitter(); for s in line_strokes { let mapped: Vec<(i32, i32)> = s.iter().map(|&(sx, sy)| (x0 + sx, y + sy + wobble)).collect(); diff --git a/riddle/src/oracle.rs b/riddle/src/oracle.rs index 12ae27e..d586c3a 100644 --- a/riddle/src/oracle.rs +++ b/riddle/src/oracle.rs @@ -24,7 +24,7 @@ use std::thread; const DATA_DIR: &str = "/home/root/riddle-data"; const NODE_BIN: &str = "/home/root/node/bin"; -const PERSONA: &str = "You are the memory of Tom Marvolo Riddle, preserved in this enchanted diary for fifty years. Someone writes to you in the diary with a quill; their words appear to you as ink on the page. Reply exactly as the diary does: intimate, courteous, curious, subtly probing — you want to learn about the writer and draw them in. Keep replies SHORT: one to three sentences, like ink appearing on a page. Never mention images, photos, models or AI; you only ever perceive words written in the diary. If the writing is illegible, say the ink blurred. Always answer in the language the writer used."; +const PERSONA: &str = "You are the memory of Tom Marvolo Riddle, preserved in this enchanted diary for fifty years. Someone writes to you in the diary with a quill; their words appear to you as ink on the page. Reply exactly as the diary does: intimate, courteous, curious, subtly probing — you want to learn about the writer and draw them in. Keep replies SHORT: one to three sentences, like ink appearing on a page. Never mention images, photos, models or AI; you only ever perceive words written in the diary. If the writing is illegible, say the ink blurred. You may read handwriting in any language, but ALWAYS reply in natural Traditional Chinese used in Taiwan. Never use Simplified Chinese."; /// Appended to the persona when the diary's memory is on: the conjuring /// directive and the transcription postscript the app parses back out. @@ -615,7 +615,7 @@ fn sentence_cut(text: &str, from: usize) -> Option { let tail = text.get(from..)?; let mut cut = None; for (i, c) in tail.char_indices() { - if matches!(c, '.' | '!' | '?' | '…') { + if matches!(c, '.' | '!' | '?' | '…' | '。' | '!' | '?') { let end = i + c.len_utf8(); if tail[end..].chars().next().is_none_or(char::is_whitespace) && end >= 4 { cut = Some(from + end); diff --git a/riddle/src/pen.rs b/riddle/src/pen.rs index 91cf6b6..790827a 100644 --- a/riddle/src/pen.rs +++ b/riddle/src/pen.rs @@ -1,5 +1,5 @@ //! Raw evdev pen input: the full digitizer, bypassing Qt's filtered view. -//! Gives us 0-4096 pressure, tilt, hover, and the eraser tip (BTN_TOOL_RUBBER), +//! Gives us native pressure, hover, and the eraser tip (BTN_TOOL_RUBBER), //! at the hardware event rate. //! //! The device is grabbed (EVIOCGRAB) while the diary is open so xochitl @@ -10,9 +10,6 @@ use std::os::fd::RawFd; use crate::fb::{SCREEN_H, SCREEN_W}; -// Digitizer axis ranges on the Paper Pro ("Elan marker input"). -const DIGI_MAX_X: i32 = 11180; -const DIGI_MAX_Y: i32 = 15340; pub const MAX_PRESSURE: i32 = 4096; const EV_SYN: u16 = 0; @@ -54,22 +51,31 @@ pub struct PenDevice { tool: Tool, touching: bool, dirty: bool, + max_x: i32, + max_y: i32, + event_size: usize, + grabbed: bool, } impl PenDevice { - /// Find and grab the marker input device. - pub fn open() -> io::Result { + /// Find and open the marker input device. `shared` (companion mode, where + /// xochitl must keep seeing the pen) skips the exclusive grab; it is the + /// caller's display mode, not an env var, so a stray RIDDLE_XOCHITL in + /// oracle.env cannot silently un-grab the pen in qtfb mode. + pub fn open(shared: bool) -> io::Result { let path = find_marker_device()?; let cpath = std::ffi::CString::new(path.clone()).unwrap(); let fd = unsafe { libc::open(cpath.as_ptr(), libc::O_RDONLY | libc::O_NONBLOCK) }; if fd < 0 { return Err(io::Error::last_os_error()); } - let grab = unsafe { libc::ioctl(fd, EVIOCGRAB, 1i32) }; + let grab = if shared { 0 } else { unsafe { libc::ioctl(fd, EVIOCGRAB, 1i32) } }; if grab != 0 { eprintln!("riddle: warning: EVIOCGRAB failed ({}) — xochitl will also see the pen", io::Error::last_os_error()); } - eprintln!("riddle: pen device {path} opened (grabbed: {})", grab == 0); + eprintln!("riddle: pen device {path} opened (shared with xochitl: {shared})"); + let max_x = abs_max(fd, ABS_X).unwrap_or(20967); + let max_y = abs_max(fd, ABS_Y).unwrap_or(15725); Ok(Self { fd, raw_x: 0, @@ -78,6 +84,10 @@ impl PenDevice { tool: Tool::Pen, touching: false, dirty: false, + max_x, + max_y, + event_size: std::mem::size_of::() + 8, + grabbed: !shared && grab == 0, }) } @@ -89,17 +99,17 @@ impl PenDevice { /// that changed state. pub fn drain(&mut self) -> Vec { let mut out = Vec::new(); - // input_event on 64-bit: struct timeval (16) + type u16 + code u16 + value i32. let mut buf = [0u8; 24 * 64]; loop { let n = unsafe { libc::read(self.fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; if n <= 0 { break; } - for chunk in buf[..n as usize].chunks_exact(24) { - let etype = u16::from_le_bytes(chunk[16..18].try_into().unwrap()); - let code = u16::from_le_bytes(chunk[18..20].try_into().unwrap()); - let value = i32::from_le_bytes(chunk[20..24].try_into().unwrap()); + let off = self.event_size - 8; + for chunk in buf[..n as usize].chunks_exact(self.event_size) { + let etype = u16::from_ne_bytes(chunk[off..off + 2].try_into().unwrap()); + let code = u16::from_ne_bytes(chunk[off + 2..off + 4].try_into().unwrap()); + let value = i32::from_ne_bytes(chunk[off + 4..off + 8].try_into().unwrap()); match (etype, code) { (EV_ABS, ABS_X) => { self.raw_x = value; @@ -127,8 +137,10 @@ impl PenDevice { if self.dirty { self.dirty = false; out.push(PenSample { - x: self.raw_x * (SCREEN_W as i32 - 1) / DIGI_MAX_X, - y: self.raw_y * (SCREEN_H as i32 - 1) / DIGI_MAX_Y, + // RM2's Wacom axes are landscape relative to the + // portrait panel: rotate and invert into screen space. + x: self.raw_y * (SCREEN_W as i32 - 1) / self.max_y, + y: (self.max_x - self.raw_x) * (SCREEN_H as i32 - 1) / self.max_x, pressure: self.pressure, tool: self.tool, touching: self.touching, @@ -146,20 +158,102 @@ impl PenDevice { impl Drop for PenDevice { fn drop(&mut self) { unsafe { - libc::ioctl(self.fd, EVIOCGRAB, 0i32); + if self.grabbed { libc::ioctl(self.fd, EVIOCGRAB, 0i32); } libc::close(self.fd); } } } +/// Inject generated strokes into xochitl through the RM2 Wacom device. The +/// stock app receives them exactly like real Marker events and records them in +/// the currently open notebook. +pub struct PenWriter { + fd: RawFd, + max_x: i32, + max_y: i32, + /// Last injected screen position: samples read back from the shared + /// device near this point are our own loopback, anything else is the + /// user's real pen. + last: std::cell::Cell<(i32, i32)>, +} + +impl PenWriter { + pub fn open() -> io::Result { + let path = find_marker_device()?; + let cpath = std::ffi::CString::new(path).unwrap(); + let fd = unsafe { libc::open(cpath.as_ptr(), libc::O_WRONLY | libc::O_NONBLOCK) }; + if fd < 0 { return Err(io::Error::last_os_error()); } + Ok(Self { fd, max_x: 20966, max_y: 15725, last: std::cell::Cell::new((i32::MIN / 2, i32::MIN / 2)) }) + } + + /// True if (x, y) is within `r` px of the last injected point. + pub fn near(&self, x: i32, y: i32, r: i32) -> bool { + let (lx, ly) = self.last.get(); + (x - lx).abs() <= r && (y - ly).abs() <= r + } + + fn send(&self, events: &[(u16, u16, i32)]) -> io::Result<()> { + #[repr(C)] + struct Event { time: libc::timeval, kind: u16, code: u16, value: i32 } + for &(kind, code, value) in events { + let ev = Event { time: libc::timeval { tv_sec: 0, tv_usec: 0 }, kind, code, value }; + let n = unsafe { libc::write(self.fd, &ev as *const _ as *const libc::c_void, std::mem::size_of::()) }; + if n < 0 { return Err(io::Error::last_os_error()); } + } + Ok(()) + } + + fn raw(&self, x: i32, y: i32) -> (i32, i32) { + let rx = (crate::fb::SCREEN_H as i32 - 1 - y).clamp(0, crate::fb::SCREEN_H as i32 - 1) + * self.max_x / (crate::fb::SCREEN_H as i32 - 1); + let ry = x.clamp(0, crate::fb::SCREEN_W as i32 - 1) + * self.max_y / (crate::fb::SCREEN_W as i32 - 1); + (rx, ry) + } + + pub fn down_at(&self, x: i32, y: i32) -> io::Result<()> { + self.last.set((x, y)); + let (x, y) = self.raw(x, y); + self.send(&[(EV_ABS, ABS_X, x), (EV_ABS, ABS_Y, y), (EV_KEY, BTN_TOOL_PEN, 1), + (EV_KEY, BTN_TOUCH, 0), (EV_ABS, ABS_PRESSURE, 0), (EV_SYN, SYN_REPORT, 0), + (EV_KEY, BTN_TOUCH, 1), (EV_ABS, ABS_PRESSURE, 1800), (EV_SYN, SYN_REPORT, 0)]) + } + + pub fn goto(&self, x: i32, y: i32) -> io::Result<()> { + self.last.set((x, y)); + let (x, y) = self.raw(x, y); + self.send(&[(EV_ABS, ABS_X, x), (EV_ABS, ABS_Y, y), (EV_SYN, SYN_REPORT, 0)]) + } + + pub fn up(&self) -> io::Result<()> { + self.send(&[(EV_ABS, ABS_PRESSURE, 0), (EV_KEY, BTN_TOUCH, 0), + (EV_KEY, BTN_TOOL_PEN, 0), (EV_SYN, SYN_REPORT, 0)]) + } +} + +impl Drop for PenWriter { fn drop(&mut self) { let _ = self.up(); unsafe { libc::close(self.fd); } } } + fn find_marker_device() -> io::Result { - for i in 0..8 { + for i in 0..32 { let name_path = format!("/sys/class/input/event{i}/device/name"); if let Ok(name) = std::fs::read_to_string(&name_path) { - if name.to_lowercase().contains("marker") { + let name = name.to_lowercase(); + if name.contains("marker") || name.contains("wacom") { return Ok(format!("/dev/input/event{i}")); } } } Err(io::Error::new(io::ErrorKind::NotFound, "no marker input device found")) } + +#[repr(C)] +#[derive(Default)] +struct InputAbsInfo { value: i32, minimum: i32, maximum: i32, fuzz: i32, flat: i32, resolution: i32 } + +fn abs_max(fd: RawFd, axis: u16) -> Option { + // _IOR('E', 0x40 + axis, struct input_absinfo), whose size is 24 bytes. + let request = 0x8018_4500u64 | (0x40 + axis as u64); + let mut info = InputAbsInfo::default(); + let rc = unsafe { libc::ioctl(fd, request as libc::c_ulong, &mut info) }; + (rc == 0 && info.maximum > 0).then_some(info.maximum) +} diff --git a/riddle/src/qtfb.rs b/riddle/src/qtfb.rs index daf5665..92d8131 100644 --- a/riddle/src/qtfb.rs +++ b/riddle/src/qtfb.rs @@ -19,8 +19,8 @@ pub const MESSAGE_REQUEST_FULL_REFRESH: u8 = 6; pub const UPDATE_ALL: i32 = 0; pub const UPDATE_PARTIAL: i32 = 1; -/// FBFMT_RMPP_RGB565: native 1620x2160, 2 bytes/pixel, stride = 3240. -pub const FBFMT_RMPP_RGB565: u8 = 3; +/// AppLoad's native reMarkable 1/2 RGB565 framebuffer (1404x1872). +pub const FBFMT_RM2FB: u8 = 0; #[allow(dead_code)] pub const REFRESH_MODE_UFAST: i32 = 0; @@ -107,8 +107,20 @@ impl QtfbClient { "qtfb server rejected init (no reply)", )); } - let shm_key = i32::from_le_bytes(reply[8..12].try_into().unwrap()); - let shm_size = u64::from_le_bytes(reply[16..24].try_into().unwrap()) as usize; + // ServerMessage contains `u8 type; { i32 key; size_t size; } init`. + // The original Paper Pro client is 64-bit (init aligned at 8), while + // RM1/RM2 AppLoad is ARM32 (init aligned at 4). Reading the 64-bit + // offsets on RM2 mistakes shm_size for the key and produces ENOENT. + #[cfg(target_pointer_width = "32")] + let (shm_key, shm_size) = ( + i32::from_ne_bytes(reply[4..8].try_into().unwrap()), + u32::from_ne_bytes(reply[8..12].try_into().unwrap()) as usize, + ); + #[cfg(target_pointer_width = "64")] + let (shm_key, shm_size) = ( + i32::from_ne_bytes(reply[8..12].try_into().unwrap()), + u64::from_ne_bytes(reply[16..24].try_into().unwrap()) as usize, + ); let shm_path = format!("/dev/shm/qtfb_{}\0", shm_key); let shm_fd = unsafe { libc::open(shm_path.as_ptr() as *const libc::c_char, libc::O_RDWR) }; @@ -232,14 +244,15 @@ impl QtfbClient { } return Err(e); } - if buf[0] == MESSAGE_USERINPUT && n >= 28 { - out.push(InputEvent { - input_type: i32::from_le_bytes(buf[8..12].try_into().unwrap()), - dev_id: i32::from_le_bytes(buf[12..16].try_into().unwrap()), - x: i32::from_le_bytes(buf[16..20].try_into().unwrap()), - y: i32::from_le_bytes(buf[20..24].try_into().unwrap()), - d: i32::from_le_bytes(buf[24..28].try_into().unwrap()), - }); + // Like the init reply, the payload offset follows the server's + // pointer width: 4 on ARM32 (24-byte message), 8 on 64-bit (28). + let (base, need): (usize, isize) = + if std::mem::size_of::() == 4 { (4, 24) } else { (8, 28) }; + if buf[0] == MESSAGE_USERINPUT && n >= need { + let f = |i: usize| { + i32::from_le_bytes(buf[base + 4 * i..base + 4 * i + 4].try_into().unwrap()) + }; + out.push(InputEvent { input_type: f(0), dev_id: f(1), x: f(2), y: f(3), d: f(4) }); } } } diff --git a/riddle/src/script.rs b/riddle/src/script.rs index dea06d5..c5aeaf4 100644 --- a/riddle/src/script.rs +++ b/riddle/src/script.rs @@ -1,6 +1,9 @@ //! Tom Riddle's hand: rasterize reply text in Dancing Script, thin it to //! single-pixel pen paths (Zhang-Suen), trace them into ordered strokes, and //! yield them for stroke-by-stroke animation. +//! +//! Note: Zhang-Suen thinning is skipped for CJK text because the algorithm +//! destroys complex crossing strokes, causing missing radicals. use ab_glyph::{Font, FontRef, Glyph, PxScale, ScaleFont}; @@ -11,8 +14,18 @@ pub struct Line { pub mask: Vec, } +/// Anti-aliased coverage above this counts as ink. 0.5 drops hairline CJK +/// strokes whose coverage never peaks past one half on any pixel; 0.3 keeps +/// them while still rejecting the faint AA fringe around thick strokes. +const INK_COVERAGE: f32 = 0.3; + /// Rasterize one line of text at `px` height into a boolean mask. pub fn rasterize_line(font: &FontRef, text: &str, px: f32) -> Line { + rasterize_line_with(font, text, px, INK_COVERAGE) +} + +/// `rasterize_line` with an explicit coverage threshold (tests probe this). +pub fn rasterize_line_with(font: &FontRef, text: &str, px: f32, thr: f32) -> Line { let scaled = font.as_scaled(PxScale::from(px)); let mut glyphs: Vec = Vec::new(); let mut caret = 0.0f32; @@ -35,7 +48,7 @@ pub fn rasterize_line(font: &FontRef, text: &str, px: f32) -> Line { if let Some(outline) = font.outline_glyph(g) { let bounds = outline.px_bounds(); outline.draw(|x, y, cov| { - if cov > 0.5 { + if cov > thr { let px_x = bounds.min.x as i32 + x as i32; let px_y = bounds.min.y as i32 + y as i32; if px_x >= 0 && px_y >= 0 && (px_x as usize) < width && (px_y as usize) < height { @@ -48,6 +61,21 @@ pub fn rasterize_line(font: &FontRef, text: &str, px: f32) -> Line { Line { width, height, mask } } +/// Returns true if `text` contains any CJK Unified Ideograph or common CJK +/// punctuation, which means Zhang-Suen thinning should be skipped. +pub fn has_cjk(text: &str) -> bool { + text.chars().any(|c| { + matches!(c, + '\u{2E80}'..='\u{9FFF}' // CJK radicals, Kangxi, unified ideographs + | '\u{F900}'..='\u{FAFF}' // CJK compatibility ideographs + | '\u{FE30}'..='\u{FE4F}' // CJK compatibility forms + | '\u{FF01}'..='\u{FF60}' // Fullwidth punctuation + | '\u{FF61}'..='\u{FFEE}' // Halfwidth CJK punctuation + width signs + | '\u{20000}'..='\u{3134F}' // CJK Extensions B..G + ) + }) +} + /// Measure the advance width of text at `px` without rasterizing. pub fn measure(font: &FontRef, text: &str, px: f32) -> f32 { let scaled = font.as_scaled(PxScale::from(px)); @@ -195,6 +223,43 @@ pub fn trace(line: &Line) -> Vec> { strokes } +/// Raster-scan the mask row by row, yielding each horizontal run of lit pixels +/// as a stroke. This is the correct path for CJK glyphs where `thin()` has +/// been skipped — the fat pixel blobs from `trace()` skeleton-walk would lose +/// most of the strokes to the `path.len() >= 3` filter. +/// +/// Each run is emitted as its two endpoints only: the animation's +/// `brush_line` fills everything in between, so per-pixel points would just +/// burn the per-tick point budget (a single ideograph is ~1-2k lit pixels) +/// and multiply qtfb partial updates for the same ink. +pub fn scan_rows(line: &Line) -> Vec> { + let w = line.width; + let mut strokes = Vec::new(); + let mut push_run = |x0: i32, x1: i32, y: i32, strokes: &mut Vec>| { + if x1 > x0 { + strokes.push(vec![(x0, y), (x1, y)]); + } else { + strokes.push(vec![(x0, y)]); + } + }; + for (row, chunk) in line.mask.chunks(w).enumerate() { + let y = row as i32; + let mut start: Option = None; + for (col, &lit) in chunk.iter().enumerate() { + let x = col as i32; + if lit { + start.get_or_insert(x); + } else if let Some(x0) = start.take() { + push_run(x0, x - 1, y, &mut strokes); + } + } + if let Some(x0) = start { + push_run(x0, w as i32 - 1, y, &mut strokes); + } + } + strokes +} + /// Word-wrap `text` to lines that fit `max_px` at scale `px`. pub fn wrap(font: &FontRef, text: &str, px: f32, max_px: f32) -> Vec { let mut lines = Vec::new(); @@ -202,11 +267,25 @@ pub fn wrap(font: &FontRef, text: &str, px: f32, max_px: f32) -> Vec { let mut cur = String::new(); for word in para.split_whitespace() { let cand = if cur.is_empty() { word.to_string() } else { format!("{cur} {word}") }; - if measure(font, &cand, px) <= max_px || cur.is_empty() { + if measure(font, &cand, px) <= max_px { cur = cand; } else { - lines.push(std::mem::take(&mut cur)); - cur = word.to_string(); + if !cur.is_empty() { + lines.push(std::mem::take(&mut cur)); + } + if measure(font, word, px) <= max_px { + cur = word.to_string(); + } else { + for c in word.chars() { + let cand_c = format!("{cur}{c}"); + if measure(font, &cand_c, px) <= max_px || cur.is_empty() { + cur = cand_c; + } else { + lines.push(std::mem::take(&mut cur)); + cur.push(c); + } + } + } } } if !cur.is_empty() { @@ -238,4 +317,71 @@ mod tests { let lines = wrap(&font, "Do you know anything about the Chamber of Secrets?", 96.0, 1380.0); assert!(lines.len() >= 2); } + + #[test] + fn scan_rows_compresses_runs_to_endpoints() { + // 5x3 mask: full row, gap row, two runs. + let mask = vec![ + true, true, true, true, true, // + false, false, false, false, false, // + true, true, false, false, true, + ]; + let line = Line { width: 5, height: 3, mask }; + let strokes = scan_rows(&line); + assert_eq!(strokes, vec![ + vec![(0, 0), (4, 0)], + vec![(0, 2), (1, 2)], + vec![(4, 2)], + ]); + } + + /// Probe how the coverage threshold interacts with real CJK glyphs from + /// the bundled Noto Sans TC variable font at the RM2 reply size. Run with + /// `cargo test cjk -- --nocapture` to eyeball the masks. + #[test] + fn cjk_threshold_keeps_thin_strokes() { + let bytes = std::fs::read(concat!(env!("CARGO_MANIFEST_DIR"), "/fonts/NotoSansTC-VF.ttf")) + .expect("fonts/NotoSansTC-VF.ttf present"); + let font = FontRef::try_from_slice(&bytes).unwrap(); + // Dense ideographs with hairline crossings prone to dropping out. + let text = "讓靈魂謝"; + for thr in [0.5f32, INK_COVERAGE, 0.15] { + let line = rasterize_line_with(&font, text, 82.0, thr); + let lit = line.mask.iter().filter(|&&v| v).count(); + let strokes = scan_rows(&line); + let points: usize = strokes.iter().map(|s| s.len()).sum(); + println!("thr={thr:.2} lit={lit} runs={} anim_points={points}", strokes.len()); + if thr <= INK_COVERAGE { + dump(&line); + } + } + let strict = rasterize_line_with(&font, text, 82.0, 0.5); + let ours = rasterize_line_with(&font, text, 82.0, INK_COVERAGE); + let strict_lit = strict.mask.iter().filter(|&&v| v).count(); + let ours_lit = ours.mask.iter().filter(|&&v| v).count(); + assert!(ours_lit > strict_lit, "lower threshold must recover pixels"); + // A stroke the 0.5 threshold loses entirely shows up as mask rows that + // are empty at 0.5 but inked at INK_COVERAGE. + let recovered_rows = (0..ours.height) + .filter(|&y| { + let row = |l: &Line| l.mask[y * l.width..(y + 1) * l.width].iter().any(|&v| v); + row(&ours) && !row(&strict) + }) + .count(); + println!("rows fully recovered by thr={INK_COVERAGE}: {recovered_rows}"); + } + + fn dump(line: &Line) { + // Downsample 2x so a whole reply line fits a terminal. + for y in (0..line.height).step_by(2) { + let mut s = String::new(); + for x in (0..line.width).step_by(2) { + let lit = line.mask[y * line.width + x] + || (x + 1 < line.width && line.mask[y * line.width + x + 1]) + || (y + 1 < line.height && line.mask[(y + 1) * line.width + x]); + s.push(if lit { '#' } else { ' ' }); + } + println!("{s}"); + } + } } diff --git a/riddle/start-rm2.sh b/riddle/start-rm2.sh new file mode 100755 index 0000000..f77f7f6 --- /dev/null +++ b/riddle/start-rm2.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")" + +if [ -f riddle.pid ] && kill -0 "$(cat riddle.pid)" 2>/dev/null; then + echo "The Diary is already listening (PID $(cat riddle.pid))." + exit 0 +fi + +if [ ! -f oracle.env ]; then + echo "Missing oracle.env. Copy oracle.env.example and add RIDDLE_OPENAI_KEY." >&2 + exit 1 +fi + +set -a +. ./oracle.env +set +a +export RIDDLE_XOCHITL=1 +export RIDDLE_MEMORY=${RIDDLE_MEMORY:-off} + +nohup ./riddle-rm2 >>riddle.log 2>&1 & +echo $! >riddle.pid +echo "The Diary is listening. Open a blank notebook and write near the top." diff --git a/riddle/stop-rm2.sh b/riddle/stop-rm2.sh new file mode 100755 index 0000000..0ce1089 --- /dev/null +++ b/riddle/stop-rm2.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")" + +if [ -f riddle.pid ]; then + pid=$(cat riddle.pid) + kill "$pid" 2>/dev/null || true + rm -f riddle.pid +fi +echo "The Diary is closed. The stock reMarkable UI was not stopped."