Georg/ai.interactive.fiction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ink integration UI and media playback

Browse Source
This commit is contained in:
Georg Tomitsch
2026-05-15 21:23:46 +02:00
parent 44dc64f830
commit f2e786d5bc
89 changed files with 6561 additions and 556 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/js/sentence-queue-module.js
+375 -40
View File
@@ -9,7 +9,7 @@ class SentenceQueueModule extends BaseModule {
super('sentence-queue', 'Sentence Queue');
// Dependencies
this.dependencies = ['text-buffer', 'tts-factory', 'paragraph-layout', 'audio-manager'];
this.dependencies = ['text-buffer', 'tts-factory', 'paragraph-layout', 'audio-manager', 'persistence-manager'];
// Queue state
this.sentenceQueue = [];
@@ -18,6 +18,9 @@ class SentenceQueueModule extends BaseModule {
// Cache for prefetched sentences
this.preparedCache = new Map();
this.prefetchingCache = new Map();
this.activeImageWrap = null;
this.autoplay = true;
// Bind methods
this.bindMethods([
@@ -26,8 +29,18 @@ class SentenceQueueModule extends BaseModule {
'processNextSentence',
'setOnSentenceReady',
'completeSentence',
'getCacheKey',
'getPreparedSentence',
'prefetchAhead',
'isSpeechItem',
'getMediaPauseSeconds',
'readFirstFiniteNumber',
'waitForSkippableMediaPause',
'shouldAutoplay',
'waitForManualContinue',
'prepareSentence',
'prepareLayout',
'prepareImageLayout',
'extractWords',
'getDropCapText',
'extractDropCapText',
@@ -56,6 +69,16 @@ class SentenceQueueModule extends BaseModule {
});
this.reportProgress(100, "Sentence queue ready");
const persistenceManager = this.getModule('persistence-manager');
if (persistenceManager && typeof persistenceManager.getPreference === 'function') {
this.autoplay = persistenceManager.getPreference('app', 'autoplay', true) !== false;
}
this.addEventListener(document, 'preference-updated', (event) => {
const { category, key, value } = event.detail || {};
if (category === 'app' && key === 'autoplay') {
this.autoplay = value !== false;
}
});
return true;
} catch (error) {
console.error("Error initializing Sentence Queue:", error);
@@ -106,44 +129,11 @@ class SentenceQueueModule extends BaseModule {
const item = this.sentenceQueue[0];
try {
// Check if sentence is already in cache
const cacheKey = `${item.id || ''}:${item.text}`;
let sentence = this.preparedCache.get(cacheKey);
const sentence = await this.getPreparedSentence(item);
if (!sentence) {
// Prepare complete sentence object (TTS + layout in parallel)
sentence = await this.prepareSentence(item);
} else {
console.log('SentenceQueue: Using cached sentence');
this.preparedCache.delete(cacheKey);
}
// Prefetch next sentence while current displays
if (this.sentenceQueue.length > 1) {
const nextItem = this.sentenceQueue[1];
const nextCacheKey = `${nextItem.id || ''}:${nextItem.text}`;
if (!this.preparedCache.has(nextCacheKey)) {
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-generating', reason: 'prefetch-start', sentenceId: nextItem.id }
}));
console.log('Process state: playing-generating', { reason: 'prefetch-start', sentenceId: nextItem.id });
this.prepareSentence(nextItem)
.then(prepared => {
this.preparedCache.set(nextCacheKey, prepared);
console.log('SentenceQueue: Prefetched next sentence');
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: 'prefetch-complete', sentenceId: nextItem.id }
}));
console.log('Process state: playing-ready', { reason: 'prefetch-complete', sentenceId: nextItem.id });
})
.catch(err => console.warn('SentenceQueue: Prefetch failed:', err));
}
} else {
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: 'no-prefetch-needed', sentenceId: item.id }
}));
console.log('Process state: playing-ready', { reason: 'no-prefetch-needed', sentenceId: item.id });
}
// Prefetch far enough ahead that media pauses do not block TTS
// generation for the next spoken paragraph.
this.prefetchAhead();
// Notify display handler with complete sentence
if (this.onSentenceReadyCallback) {
@@ -153,6 +143,15 @@ class SentenceQueueModule extends BaseModule {
});
}
const mediaPauseSeconds = this.getMediaPauseSeconds(sentence);
if (mediaPauseSeconds > 0) {
await this.waitForSkippableMediaPause(mediaPauseSeconds, sentence.kind, sentence.id);
}
if (sentence.kind === 'paragraph' && !this.shouldAutoplay()) {
await this.waitForManualContinue(sentence.id);
}
// Remove from queue and continue
this.sentenceQueue.shift();
if (item.callback) item.callback({ success: true });
@@ -275,12 +274,17 @@ class SentenceQueueModule extends BaseModule {
}
}
const imageLayout = metadata.type === 'image'
? await this.prepareImageLayout(metadata)
: null;
return {
id,
kind: metadata.type,
text: text || '',
turnId: metadata.turnId ?? null,
status: 'ready',
metadata,
metadata: imageLayout ? { ...metadata, imageLayout } : metadata,
tts: { duration: 0, provider: null, audioData: null, play: null, stop: null, enabled: false },
animation: { wordTimings: [], cueTimings: [], totalDuration: 0 },
element: null,
@@ -309,6 +313,7 @@ class SentenceQueueModule extends BaseModule {
id,
kind: metadata.type === 'heading' ? 'heading' : 'paragraph',
text,
turnId: metadata.turnId ?? null,
paragraphIndex: metadata.paragraphIndex ?? null,
isFirstParagraphInChapter: Boolean(metadata.isFirstParagraphInChapter),
role: metadata.role || (metadata.type === 'heading' ? 'chapter-heading' : 'body'),
@@ -383,10 +388,27 @@ class SentenceQueueModule extends BaseModule {
const indentWidth = (isHeading || metadata.isFirstParagraphInChapter || metadata.addTopSpace) ? 0 : lineHeight * 1.5;
const layoutText = metadata.layoutText || text;
const layoutPlainText = metadata.dropCap ? this.extractDropCapText(layoutText) : layoutText;
const wrap = this.consumeImageWrap();
// Measures are consumed in line order by the line breaker.
const wrappedWidth = wrap ? Math.max(120, containerWidth - wrap.width) : containerWidth;
const imageLeftOffset = wrap && wrap.side !== 'right' ? wrap.width : 0;
const imageRightOffset = wrap && wrap.side === 'right' ? wrap.width : 0;
const measures = isHeading
? [containerWidth]
: wrap && metadata.dropCap
? [
Math.max(120, wrappedWidth - dropCapWidth),
Math.max(120, wrappedWidth - dropCapWidth),
...Array(Math.max(0, wrap.lines - dropCapLines)).fill(wrappedWidth),
containerWidth
]
: wrap
? [
Math.max(120, wrappedWidth - indentWidth),
...Array(Math.max(0, wrap.lines - 1)).fill(wrappedWidth),
containerWidth
]
: metadata.dropCap
? [
Math.max(120, containerWidth - dropCapWidth),
@@ -398,8 +420,34 @@ class SentenceQueueModule extends BaseModule {
containerWidth,
containerWidth
];
const lineOffsets = isHeading
? [0]
: wrap && metadata.dropCap
? [
imageLeftOffset + dropCapWidth,
imageLeftOffset + dropCapWidth,
...Array(Math.max(0, wrap.lines - dropCapLines)).fill(imageLeftOffset),
0
]
: wrap
? [
imageLeftOffset + indentWidth,
...Array(Math.max(0, wrap.lines - 1)).fill(imageLeftOffset),
0
]
: metadata.dropCap
? [
dropCapWidth,
dropCapWidth,
0
]
: [
indentWidth,
0,
0
];
console.log(`SentenceQueue: Layout calculation - indentWidth: ${indentWidth.toFixed(1)}px, measures: [${measures.map(m => m.toFixed(1)).join(', ')}]`);
console.log(`SentenceQueue: Layout calculation - indentWidth: ${indentWidth.toFixed(1)}px, imageRightOffset: ${imageRightOffset.toFixed(1)}px, measures: [${measures.map(m => m.toFixed(1)).join(', ')}], offsets: [${lineOffsets.map(m => m.toFixed(1)).join(', ')}]`);
const layout = paragraphLayout.calculateLayout(layoutPlainText, {
measures,
@@ -413,13 +461,23 @@ class SentenceQueueModule extends BaseModule {
throw new Error('Paragraph layout calculation failed');
}
if (wrap) {
const usedLines = Math.max(0, (layout.breaks?.length || 1) - 1);
const remainingLines = Math.max(0, wrap.lines - usedLines);
this.activeImageWrap = remainingLines > 0
? { ...wrap, lines: remainingLines }
: null;
}
return {
breaks: layout.breaks,
nodes: layout.nodes,
processedText: layout.processedText || text,
sourceLayoutText: layoutText,
measures,
lineOffsets,
indentWidth,
imageWrap: wrap,
dropCap: Boolean(metadata.dropCap),
dropCapText: metadata.dropCap ? this.getDropCapText(layoutText) : '',
dropCapLines,
@@ -437,6 +495,282 @@ class SentenceQueueModule extends BaseModule {
}
}
shouldAutoplay() {
const persistenceManager = this.getModule('persistence-manager');
if (persistenceManager && typeof persistenceManager.getPreference === 'function') {
return persistenceManager.getPreference('app', 'autoplay', this.autoplay) !== false;
}
return this.autoplay !== false;
}
waitForManualContinue(sentenceId) {
document.documentElement.dataset.skippablePause = 'true';
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'paused', reason: 'autoplay-disabled', sentenceId }
}));
return new Promise(resolve => {
let resolved = false;
const finish = () => {
if (resolved) return;
resolved = true;
delete document.documentElement.dataset.skippablePause;
document.removeEventListener('ui:command', onCommand);
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: 'manual-continue', sentenceId }
}));
resolve();
};
const onCommand = (event) => {
if (event.detail?.type === 'continue') {
finish();
}
};
document.addEventListener('ui:command', onCommand);
});
}
getCacheKey(item) {
return `${item?.id || ''}:${item?.text || ''}`;
}
async getPreparedSentence(item) {
const cacheKey = this.getCacheKey(item);
const cached = this.preparedCache.get(cacheKey);
if (cached) {
console.log('SentenceQueue: Using cached sentence');
this.preparedCache.delete(cacheKey);
return cached;
}
const pending = this.prefetchingCache.get(cacheKey);
if (pending) {
console.log('SentenceQueue: Awaiting active prefetch');
try {
const prepared = await pending;
return prepared || await this.prepareSentence(item);
} finally {
this.prefetchingCache.delete(cacheKey);
this.preparedCache.delete(cacheKey);
}
}
return this.prepareSentence(item);
}
prefetchAhead(maxLookahead = 4) {
if (this.sentenceQueue.length <= 1) {
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: 'no-prefetch-needed', sentenceId: this.sentenceQueue[0]?.id }
}));
console.log('Process state: playing-ready', { reason: 'no-prefetch-needed', sentenceId: this.sentenceQueue[0]?.id });
return;
}
let started = 0;
let spokenPrepared = 0;
const limit = Math.min(this.sentenceQueue.length, maxLookahead + 1);
for (let index = 1; index < limit; index += 1) {
const nextItem = this.sentenceQueue[index];
const nextCacheKey = this.getCacheKey(nextItem);
if (this.preparedCache.has(nextCacheKey) || this.prefetchingCache.has(nextCacheKey)) {
if (this.isSpeechItem(nextItem)) spokenPrepared += 1;
continue;
}
const state = this.isSpeechItem(nextItem) ? 'playing-generating' : 'playing-ready';
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state, reason: 'prefetch-start', sentenceId: nextItem.id, queueIndex: index }
}));
console.log(`Process state: ${state}`, { reason: 'prefetch-start', sentenceId: nextItem.id, queueIndex: index });
const promise = this.prepareSentence(nextItem)
.then(prepared => {
this.preparedCache.set(nextCacheKey, prepared);
console.log('SentenceQueue: Prefetched queued item', { sentenceId: nextItem.id, queueIndex: index });
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: 'prefetch-complete', sentenceId: nextItem.id, queueIndex: index }
}));
console.log('Process state: playing-ready', { reason: 'prefetch-complete', sentenceId: nextItem.id, queueIndex: index });
return prepared;
})
.catch(err => {
console.warn('SentenceQueue: Prefetch failed:', err);
return null;
})
.finally(() => {
this.prefetchingCache.delete(nextCacheKey);
});
this.prefetchingCache.set(nextCacheKey, promise);
started += 1;
if (this.isSpeechItem(nextItem)) {
spokenPrepared += 1;
}
if (spokenPrepared >= 1 && started >= 2) {
break;
}
}
if (started === 0) {
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: 'prefetch-already-ready', sentenceId: this.sentenceQueue[0]?.id }
}));
console.log('Process state: playing-ready', { reason: 'prefetch-already-ready', sentenceId: this.sentenceQueue[0]?.id });
}
}
isSpeechItem(item) {
const type = item?.type || 'paragraph';
return type === 'paragraph' || type === 'heading' || !['image', 'music'].includes(type);
}
getMediaPauseSeconds(sentence) {
if (!sentence || !['image', 'music'].includes(sentence.kind)) {
return 0;
}
const metadata = sentence.metadata || {};
const configuredPause = this.readFirstFiniteNumber(
metadata.leadInSeconds,
metadata.leadIn,
metadata.pause,
metadata.delay,
0
);
if (sentence.kind !== 'image') {
return configuredPause;
}
const revealSeconds = Number(metadata.imageRevealSeconds || metadata.revealSeconds || 0.9);
return Math.max(configuredPause, Number.isFinite(revealSeconds) ? revealSeconds : 0.9);
}
readFirstFiniteNumber(...values) {
for (const value of values) {
const number = Number(value);
if (Number.isFinite(number)) {
return Math.max(0, number);
}
}
return 0;
}
waitForSkippableMediaPause(seconds, kind = 'media', sentenceId = null) {
const duration = Math.max(0, Number(seconds) || 0) * 1000;
if (duration <= 0) return Promise.resolve(false);
const startedAt = performance.now();
console.log(`SentenceQueue: Waiting ${seconds}s for ${kind} lead`, { sentenceId });
document.documentElement.dataset.skippablePause = 'true';
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: `${kind}-pause-start`, duration, sentenceId }
}));
return new Promise(resolve => {
let finished = false;
let timeoutId = null;
const finish = (skipped, source = null) => {
if (finished) return;
finished = true;
clearTimeout(timeoutId);
document.removeEventListener('ui:command', onCommand);
delete document.documentElement.dataset.skippablePause;
const elapsedMs = Math.round(performance.now() - startedAt);
console.log(`SentenceQueue: ${kind} lead ${skipped ? 'skipped' : 'complete'}`, { sentenceId, elapsedMs, source });
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: `${kind}-pause-${skipped ? 'skipped' : 'complete'}`, duration, elapsedMs, sentenceId }
}));
resolve(skipped);
};
const onCommand = (event) => {
if (event.detail?.type === 'continue') {
finish(true, event.detail);
}
};
document.addEventListener('ui:command', onCommand);
timeoutId = setTimeout(() => finish(false), duration);
});
}
async prepareImageLayout(metadata = {}) {
const storyElement = document.getElementById('story');
if (!storyElement) {
throw new Error("Story container not found");
}
if (document.fonts && document.fonts.ready) {
await document.fonts.ready;
}
const probe = document.createElement('p');
probe.style.visibility = 'hidden';
probe.style.position = 'absolute';
probe.style.left = '-8000px';
probe.style.top = '-8000px';
storyElement.appendChild(probe);
const computedStyle = window.getComputedStyle(probe);
const lineHeight = parseFloat(computedStyle.lineHeight) || 24;
probe.remove();
const pageWidth = storyElement.clientWidth;
const size = String(metadata.size || 'landscape').toLowerCase();
const aspect = size === 'portrait' ? (9 / 16) : size === 'square' ? 1 : (16 / 9);
const imageGap = lineHeight * 0.9;
const maxWidth = size === 'portrait' ? pageWidth * 0.5 : pageWidth;
const naturalHeight = maxWidth / aspect;
const imageLineCount = Math.max(1, Math.floor(naturalHeight / lineHeight));
const height = imageLineCount * lineHeight;
const width = Math.min(maxWidth, height * aspect);
const verticalMargin = lineHeight / 2;
const lineCount = imageLineCount + 1;
if (size === 'portrait') {
this.activeImageWrap = {
lines: lineCount,
width: width + imageGap,
imageWidth: width,
gap: imageGap,
height,
lineHeight,
side: metadata.floatSide || 'left'
};
}
return {
size,
aspect,
width,
height,
gap: imageGap,
lineCount,
imageLineCount,
lineHeight,
verticalMargin,
floatSide: metadata.floatSide || 'left',
pageWidth
};
}
consumeImageWrap() {
if (!this.activeImageWrap || this.activeImageWrap.lines <= 0) {
this.activeImageWrap = null;
return null;
}
const wrap = { ...this.activeImageWrap };
this.activeImageWrap = null;
return wrap;
}
/**
* Extract words from layout nodes
* @param {Array} nodes - Layout nodes from Knuth-Plass algorithm
@@ -546,6 +880,7 @@ class SentenceQueueModule extends BaseModule {
this.sentenceQueue = [];
this.isProcessing = false;
this.preparedCache.clear();
this.activeImageWrap = null;
document.dispatchEvent(new CustomEvent('tts:queue-empty', {
detail: { reason: 'sentence-queue-cleared' }
}));