Skip to content

fix(tts): align reading position with primary content #378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
codedogQBY wants to merge 4 commits into
base: main
Choose a base branch
from
Open

fix(tts): align reading position with primary content #378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cdb53ef
fix(tts): align visible range with primary reader content
codedogQBY Jun 6, 2026
1370e63
fix(reader): keep selection popover in scroll mode
codedogQBY Jun 6, 2026
48e7123
fix(reader): simplify existing highlight popover
codedogQBY Jun 6, 2026
04d29ab
fix(tts): keep paused lyrics still
codedogQBY Jun 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 138 additions & 25 deletions packages/app-expo/assets/reader/reader.html
View file
Open in desktop

Large diffs are not rendered by default.

161 changes: 137 additions & 24 deletions packages/app-expo/assets/reader/reader.template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,11 +348,23 @@
return contents.find((content) => content.doc === doc) || null;
}

function getPrimaryRendererContent() {
const contents = getRendererContents();
if (!contents.length) return null;
const primaryIndex = view && view.renderer ? view.renderer.primaryIndex : null;
return (
contents.find((content) => content.doc && content.index === primaryIndex) ||
contents.find((content) => content.doc && content.index === currentSectionIndex) ||
contents.find((content) => content.doc) ||
null
);
}

function getRendererContentForCfi(cfi) {
const contents = getRendererContents();
if (!contents.length) return null;
if (!cfi || !view || !view.resolveCFI) {
return contents.find((content) => content.overlayer) || contents[0] || null;
return getPrimaryRendererContent() || contents.find((content) => content.overlayer) || contents[0] || null;
}

try {
Expand All @@ -373,7 +385,7 @@
console.log('[ttsHighlight] failed to resolve content for cfi:', e);
}

return contents.find((content) => content.overlayer) || contents[0] || null;
return getPrimaryRendererContent() || contents.find((content) => content.overlayer) || contents[0] || null;
}

function getAllTTSOverlayerContexts() {
Expand All @@ -390,6 +402,8 @@
}

function getCurrentTTSOverlayerContext() {
const primary = getPrimaryRendererContent();
if (primary && primary.overlayer) return primary;
return getAllTTSOverlayerContexts()[0] || null;
}

Expand Down Expand Up @@ -2713,13 +2727,102 @@
return null;
}

function isRectVisibleInReader(rect, renderer, win) {
function getPaginatedVisibleRangeCandidates(renderer) {
const start = Number(renderer && renderer.start || 0);
const end = Number(renderer && renderer.end || 0);
const size = Number(renderer && renderer.size || 0);
const candidates = [];

if (Number.isFinite(start) && Number.isFinite(size) && size > 0) {
candidates.push({ left: start - size, right: start, source: 'legacy-offset' });
candidates.push({ left: start, right: start + size, source: 'size-fallback' });
}

if (Number.isFinite(start) && Number.isFinite(end) && end > start) {
candidates.push({ left: start, right: end, source: 'renderer' });
}

return candidates.filter((candidate, index, list) => {
return candidate.right > candidate.left &&
list.findIndex((item) => item.left === candidate.left && item.right === candidate.right) === index;
});
}

function rectIntersectsPaginatedRange(rect, range) {
return rect.right > range.left && rect.left < range.right;
}

function scorePaginatedVisibleRange(doc, range) {
if (!doc || !doc.body) return 0;
const walker = doc.createTreeWalker(doc.body, NodeFilter.SHOW_TEXT, {
acceptNode: function (node) {
if (isInsideRubyAnnotation(node)) return NodeFilter.FILTER_REJECT;
if (isTTSFootnoteMarker(node.nodeValue || '')) return NodeFilter.FILTER_REJECT;
if (shouldSkipTTSNode(node.parentElement)) return NodeFilter.FILTER_REJECT;
return node.nodeValue && normalizeTTSText(node.nodeValue)
? NodeFilter.FILTER_ACCEPT
: NodeFilter.FILTER_SKIP;
},
});
let score = 0;
let visited = 0;
let textNode = walker.nextNode();
while (textNode && visited < 500) {
visited += 1;
const text = normalizeTTSText(textNode.nodeValue || '');
if (text) {
try {
const textRange = doc.createRange();
textRange.selectNodeContents(textNode);
const rects = Array.from(textRange.getClientRects ? textRange.getClientRects() : []);
if (rects.some((rect) => rect.width > 0 && rect.height > 0 && rectIntersectsPaginatedRange(rect, range))) {
score += Math.min(text.length, 120);
}
} catch {}
}
textNode = walker.nextNode();
}
return score;
}

function pickPaginatedVisibleRange(doc, renderer) {
const candidates = getPaginatedVisibleRangeCandidates(renderer);
if (candidates.length <= 1) return candidates[0] || null;

const legacyRange = candidates.find((range) => range.source === 'legacy-offset') || null;
if (legacyRange && scorePaginatedVisibleRange(doc, legacyRange) > 0) return legacyRange;

const rendererRange = candidates.find((range) => range.source === 'renderer') || null;
if (rendererRange && scorePaginatedVisibleRange(doc, rendererRange) > 0) return rendererRange;

const fallbackRange = candidates.find((range) => range.source === 'size-fallback') || null;
if (fallbackRange && scorePaginatedVisibleRange(doc, fallbackRange) > 0) return fallbackRange;

return legacyRange || rendererRange || fallbackRange || candidates[0] || null;
}

let ttsVisibleRangeByDoc = new WeakMap();

function getVisibleRangeForTTSDoc(doc, renderer) {
if (!doc) return null;
if (ttsVisibleRangeByDoc.has(doc)) return ttsVisibleRangeByDoc.get(doc) || null;
const range = pickPaginatedVisibleRange(doc, renderer);
ttsVisibleRangeByDoc.set(doc, range);
return range;
}

function getRangeDocument(range) {
const container = range && range.commonAncestorContainer;
if (!container) return null;
return container.nodeType === Node.DOCUMENT_NODE ? container : container.ownerDocument;
}

function isRectVisibleInReader(rect, renderer, win, doc) {
if (!rect || rect.width <= 0 || rect.height <= 0) return false;
const isPaginated = !renderer.scrolled;
if (isPaginated && renderer.size > 0) {
const visibleLeft = renderer.start - renderer.size;
const visibleRight = renderer.start;
return rect.right > visibleLeft && rect.left < visibleRight;
const visibleRange = getVisibleRangeForTTSDoc(doc, renderer);
return visibleRange ? rectIntersectsPaginatedRange(rect, visibleRange) : false;
}
return rect.right > 0 && rect.left < win.innerWidth && rect.bottom > 0 && rect.top < win.innerHeight;
}
Expand All @@ -2728,10 +2831,11 @@
if (!range || !renderer || !win) return false;
try {
const rects = Array.from(range.getClientRects ? range.getClientRects() : []);
const doc = getRangeDocument(range);
if (!rects.length) {
return isRectVisibleInReader(range.getBoundingClientRect(), renderer, win);
return isRectVisibleInReader(range.getBoundingClientRect(), renderer, win, doc);
}
return rects.some((rect) => isRectVisibleInReader(rect, renderer, win));
return rects.some((rect) => isRectVisibleInReader(rect, renderer, win, doc));
} catch (e) {
console.log('[visibleTTSSegments] failed to inspect range rects:', e);
return false;
Expand All @@ -2742,10 +2846,11 @@
if (!range || !renderer || !win) return false;
try {
const rects = Array.from(range.getClientRects ? range.getClientRects() : []);
const doc = getRangeDocument(range);
if (!rects.length) {
return isRectVisibleInReader(range.getBoundingClientRect(), renderer, win);
return isRectVisibleInReader(range.getBoundingClientRect(), renderer, win, doc);
}
return isRectVisibleInReader(rects[0], renderer, win);
return isRectVisibleInReader(rects[0], renderer, win, doc);
} catch (e) {
console.log('[visibleTTSSegments] failed to inspect range start rect:', e);
return false;
Expand All @@ -2765,7 +2870,7 @@
range.selectNodeContents(block);
return isRangeVisibleInReader(range, renderer, win);
} catch (e) {
return isRectVisibleInReader(block.getBoundingClientRect(), renderer, win);
return isRectVisibleInReader(block.getBoundingClientRect(), renderer, win, doc);
}
});
}
Expand Down Expand Up @@ -3191,9 +3296,13 @@
const renderer = view && view.renderer;
const contents = getRendererContents();
if (!renderer || !contents.length) return [];
ttsVisibleRangeByDoc = new WeakMap();
const primaryContent = getPrimaryRendererContent();
const scanContents = renderer.scrolled && primaryContent ? [primaryContent] : contents;
const segments = [];
const stats = {
contentsCount: contents.length,
scannedContentsCount: scanContents.length,
visibleBlockCount: 0,
rawSentenceCount: 0,
skippedTooShort: 0,
Expand All @@ -3210,8 +3319,8 @@
// Must be declared here (outer scope) so it's accessible after both loops close.
let firstVisibleRange = null;

for (var contentIndex = 0; contentIndex < contents.length; contentIndex++) {
const current = contents[contentIndex];
for (var contentIndex = 0; contentIndex < scanContents.length; contentIndex++) {
const current = scanContents[contentIndex];
if (!current || !current.doc) continue;
const doc = current.doc;
const win = doc.defaultView;
Expand Down Expand Up @@ -3289,7 +3398,7 @@
const range = doc.createRange();
range.setStart(startPos.node, startPos.offset);
range.setEnd(endPos.node, endPos.offset);
if (!isRangeVisibleInReader(range, renderer, win)) {
if (!isRangeStartVisibleInReader(range, renderer, win)) {
stats.skippedNotVisible++;
stats.skippedNotStartVisible++;
continue;
Expand Down Expand Up @@ -3331,15 +3440,17 @@
// Fall back to CFI string alignment for older tts.js builds without tts.from()
const firstVisibleCfi = segments[0]?.cfi || null;
const resolvedAlignRange = resolveRangeForCfi(alignCfi);
const alignedSegments = collectTTSSegmentsFromEngine(
alignCfi ? 500 : (segments.length || 12),
alignCfi || firstVisibleCfi,
resolvedAlignRange?.range || firstVisibleRange
);
let returnSource = alignedSegments.length ? 'aligned' : 'direct';
const alignedSegments = segments.length
? collectTTSSegmentsFromEngine(
alignCfi ? 500 : segments.length,
alignCfi || firstVisibleCfi,
resolvedAlignRange?.range || firstVisibleRange
)
: [];
let returnSource = segments.length ? 'direct-visible' : 'no-visible-segments';
let filteredAlignedCount = 0;
let filteredAlignedPreview = [];
let returnedSegments = alignedSegments.length ? alignedSegments : segments;
let returnedSegments = segments;
if (alignedSegments.length && segments.length) {
const visibleIdentities = new Set(
segments.map((segment) => getTTSSegmentIdentity(segment.cfi, segment.text))
Expand Down Expand Up @@ -3374,7 +3485,7 @@
returnedSegments = segments;
}
} else {
returnSource = 'direct-fallback';
returnSource = 'direct-visible';
returnedSegments = segments;
}
}
Expand All @@ -3398,11 +3509,13 @@
return returnedSegments;
} catch (e) {
console.log('[visibleTTSSegments] extraction error:', e);
const fallbackSegments = collectTTSSegmentsFromEngine(alignCfi ? 500 : 12, alignCfi || null, resolveRangeForCfi(alignCfi)?.range || null);
const fallbackSegments = alignCfi
? collectTTSSegmentsFromEngine(500, alignCfi, resolveRangeForCfi(alignCfi)?.range || null)
: [];
window.__lastVisibleTTSDiagnostics = {
alignCfi: alignCfi || null,
extractionError: String(e),
returnSource: 'engine-fallback',
returnSource: alignCfi ? 'engine-align-fallback' : 'no-visible-segments',
directCount: 0,
alignedCount: fallbackSegments.length,
filteredAlignedCount: 0,
Expand Down
71 changes: 41 additions & 30 deletions packages/app-expo/src/components/reader/SelectionPopover.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ export function SelectionPopover({
}
}, [selection.cfi, hasExistingHighlight]);

const buttonCount = 4 + (onNote ? 1 : 0) + (onTranslate ? 1 : 0) + (onSpeak ? 1 : 0);
const buttonCount = hasExistingHighlight
? 0
: 4 + (onNote ? 1 : 0) + (onTranslate ? 1 : 0) + (onSpeak ? 1 : 0);
const colorRowItemCount = HIGHLIGHT_COLORS.length + (canRemoveHighlight ? 2 : 0);
const colorRowWidth = showColors
? HIGHLIGHT_COLORS.length * COLOR_DOT_SIZE +
Expand All @@ -109,7 +111,12 @@ export function SelectionPopover({
: 0;
const actionRowWidth = buttonCount * (BUTTON_SIZE + GAP) + POPOVER_PADDING * 2;
const colorRowHeight = showColors ? 40 : 0;
const popoverHeight = 44 + colorRowHeight + POPOVER_PADDING * 2 + GAP;
const actionRowHeight = hasExistingHighlight ? 0 : 44;
const popoverHeight =
actionRowHeight +
colorRowHeight +
POPOVER_PADDING * 2 +
(showColors && actionRowHeight ? GAP : 0);
const popoverWidth = Math.min(
Math.max(actionRowWidth, colorRowWidth + POPOVER_PADDING * 2),
SCREEN_WIDTH - POPOVER_MARGIN * 2,
Expand Down Expand Up @@ -200,7 +207,7 @@ export function SelectionPopover({
<TouchableOpacity style={StyleSheet.absoluteFill} activeOpacity={1} onPress={onDismiss} />
<View style={[s.popover, { left: position.x, top: position.y }]}>
{showColors && (
<View style={s.colorRow}>
<View style={[s.colorRow, !hasExistingHighlight && s.colorRowWithActions]}>
{HIGHLIGHT_COLORS.map((color) => (
<TouchableOpacity
key={color}
Expand Down Expand Up @@ -228,40 +235,42 @@ export function SelectionPopover({
</View>
)}

<View style={s.actionRow}>
<TouchableOpacity
style={[s.iconBtn, showColors && s.iconBtnActive]}
onPress={handleHighlightPress}
>
<HighlighterIcon size={18} color={showColors ? colors.primary : colors.foreground} />
</TouchableOpacity>

{onNote && (
<TouchableOpacity style={s.iconBtn} onPress={handleNote}>
<NotebookPenIcon size={18} color={colors.foreground} />
{!hasExistingHighlight && (
<View style={s.actionRow}>
<TouchableOpacity
style={[s.iconBtn, showColors && s.iconBtnActive]}
onPress={handleHighlightPress}
>
<HighlighterIcon size={18} color={showColors ? colors.primary : colors.foreground} />
</TouchableOpacity>
)}

<TouchableOpacity style={s.iconBtn} onPress={handleCopy}>
<CopyIcon size={18} color={colors.foreground} />
</TouchableOpacity>
{onNote && (
<TouchableOpacity style={s.iconBtn} onPress={handleNote}>
<NotebookPenIcon size={18} color={colors.foreground} />
</TouchableOpacity>
)}

{onTranslate && (
<TouchableOpacity style={s.iconBtn} onPress={handleTranslate}>
<LanguagesIcon size={18} color={colors.foreground} />
<TouchableOpacity style={s.iconBtn} onPress={handleCopy}>
<CopyIcon size={18} color={colors.foreground} />
</TouchableOpacity>
)}

<TouchableOpacity style={s.iconBtn} onPress={onAIChat}>
<SparklesIcon size={18} color={colors.foreground} />
</TouchableOpacity>
{onTranslate && (
<TouchableOpacity style={s.iconBtn} onPress={handleTranslate}>
<LanguagesIcon size={18} color={colors.foreground} />
</TouchableOpacity>
)}

{onSpeak && (
<TouchableOpacity style={s.iconBtn} onPress={handleSpeak}>
<Volume2Icon size={18} color={colors.foreground} />
<TouchableOpacity style={s.iconBtn} onPress={onAIChat}>
<SparklesIcon size={18} color={colors.foreground} />
</TouchableOpacity>
)}
</View>

{onSpeak && (
<TouchableOpacity style={s.iconBtn} onPress={handleSpeak}>
<Volume2Icon size={18} color={colors.foreground} />
</TouchableOpacity>
)}
</View>
)}
</View>

<Modal
Expand Down Expand Up @@ -339,6 +348,8 @@ const makeStyles = (colors: ThemeColors) =>
gap: 6,
paddingVertical: 6,
paddingHorizontal: 8,
},
colorRowWithActions: {
marginBottom: GAP,
},
colorDot: {
Expand Down
Loading