Georg/ai.interactive.fiction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Checkpoint WebGL book reveal optimization

Browse Source
This commit is contained in:
Georg Tomitsch
2026-06-08 08:19:20 +02:00
parent 7abd3387f3
commit c86a304364
13 changed files with 618 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/js/sentence-queue-module.js
+99 -27
View File
@@ -20,8 +20,10 @@ class SentenceQueueModule extends BaseModule {
this.isProcessing = false;
this.onSentenceReadyCallback = null;
// Cache in-flight TTS prefetches only. Layout belongs to the renderer.
// Cache prepared future queue items so the playback path can consume
// work that was already generated during lookahead.
this.prefetchingSpeech = new Map();
this.preparedSentenceCache = new Map();
this.autoplay = true;
this.inputMode = 'text';
this.lastContinueAt = 0;
@@ -43,6 +45,7 @@ class SentenceQueueModule extends BaseModule {
'getCacheKey',
'getPreparedSentence',
'prefetchAhead',
'prefetchWebGLBookPresentation',
'prepareSpeechMetadata',
'preloadAssetsForItem',
'normalizeTtsText',
@@ -156,9 +159,12 @@ class SentenceQueueModule extends BaseModule {
text: String(queueItem.text || '').trim()
});
// Process the queue if not already processing
// Process the queue if not already processing. If playback is already
// running, immediately start lookahead for the newly appended item.
if (!this.isProcessing) {
this.processNextSentence();
} else {
this.prefetchAhead(4, this.queueGeneration);
}
}
@@ -194,6 +200,11 @@ class SentenceQueueModule extends BaseModule {
const sentence = await this.getPreparedSentence(item);
if (!this.isCurrentQueueItem(item, queueGeneration)) return;
await this.prefetchWebGLBookPresentation(sentence, {
queueGeneration,
queueIndex: 0
});
if (!this.isCurrentQueueItem(item, queueGeneration)) return;
// Prefetch far enough ahead that media pauses do not block TTS
// generation for the next spoken paragraph.
@@ -499,14 +510,15 @@ class SentenceQueueModule extends BaseModule {
* Prepare queue metadata. This module intentionally does not create layout:
* live rendering and history rendering must go through the same renderer.
*/
async prepareSentence(item) {
async prepareSentence(item, options = {}) {
const text = typeof item === 'string' ? item : item.text;
const id = item.id || `paragraph-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
const metadata = typeof item === 'object' && item !== null ? item : {};
const blocking = options.blocking !== false;
try {
if (metadata.type && !['paragraph', 'heading'].includes(metadata.type)) {
await this.preloadAssetsForItem(metadata, { blocking: true, sentenceId: id });
await this.preloadAssetsForItem(metadata, { blocking, sentenceId: id, prefetch: Boolean(options.prefetch) });
return {
id,
@@ -529,7 +541,7 @@ class SentenceQueueModule extends BaseModule {
await this.preloadAssetsForItem({
type: 'paragraph',
cueMarkers: metadata.cueMarkers || []
}, { blocking: true, sentenceId: id });
}, { blocking, sentenceId: id, prefetch: Boolean(options.prefetch) });
}
const ttsData = await this.prepareSpeechMetadata(text, {
@@ -537,7 +549,7 @@ class SentenceQueueModule extends BaseModule {
blockId: metadata.blockId ?? null,
turnId: metadata.turnId ?? null,
ttsInstructions: Array.isArray(metadata.ttsInstructions) ? metadata.ttsInstructions : [],
blocking: true
blocking
});
console.log(`SentenceQueue: Prepared speech "${text.substring(0, 50)}..." - TTS duration: ${ttsData.duration}ms`);
@@ -834,7 +846,7 @@ class SentenceQueueModule extends BaseModule {
resolve();
};
const onCommand = (event) => {
if (event.detail?.type === 'continue') {
if (event.detail?.type === 'continue' && !this.isChoiceAwaitingPlayer()) {
finish();
}
};
@@ -846,15 +858,81 @@ class SentenceQueueModule extends BaseModule {
return `${item?.id || ''}:${item?.text || ''}`;
}
isChoiceAwaitingPlayer() {
if (this.inputMode !== 'choice') {
return false;
}
const choicePanel = document.getElementById('story_choices');
return Boolean(choicePanel && !choicePanel.hidden && choicePanel.dataset.choiceReady === 'true');
}
async getPreparedSentence(item) {
const pending = this.prefetchingSpeech.get(this.getCacheKey(item));
const cacheKey = this.getCacheKey(item);
const prepared = this.preparedSentenceCache.get(cacheKey);
if (prepared) {
this.preparedSentenceCache.delete(cacheKey);
return prepared;
}
const pending = this.prefetchingSpeech.get(cacheKey);
if (pending) {
pending.catch(() => null);
const prefetched = await pending.catch(() => null);
if (prefetched) {
this.preparedSentenceCache.delete(cacheKey);
return prefetched;
}
}
return this.prepareSentence(item);
}
async prefetchWebGLBookPresentation(sentence, options = {}) {
if (!sentence || !['paragraph', 'heading'].includes(sentence.kind || sentence.type)) return null;
const isWebGLMode = document.body?.dataset?.webglUiMode === '3d'
|| document.body?.classList?.contains('webgl-mode');
if (!isWebGLMode) return null;
const blockId = sentence.blockId ?? sentence.metadata?.blockId ?? null;
if (blockId == null) return null;
const bookPagination = this.getModule('book-pagination');
const bookTextureRenderer = this.getModule('book-texture-renderer');
if (!bookPagination || !bookTextureRenderer) return null;
if (!Array.isArray(sentence.animation?.wordTimings) || sentence.animation.wordTimings.length === 0) {
const words = String(sentence.layoutText || sentence.text || '').match(/\S+/g) || [];
sentence.animation = this.calculateAnimationTiming(words, sentence.tts?.duration || 0, sentence.cueMarkers || []);
}
await new Promise(resolve => {
const scheduler = window.requestIdleCallback || ((callback) => window.setTimeout(callback, 1));
scheduler(() => resolve(), { timeout: 120 });
});
if (options.queueGeneration !== undefined && options.queueGeneration !== this.queueGeneration) return null;
const spread = typeof bookPagination.preparePendingBlock === 'function'
? await bookPagination.preparePendingBlock(sentence, {
activate: false,
publish: false,
includeUnrenderedHistory: true
})
: null;
if (!spread) return null;
if (options.queueGeneration !== undefined && options.queueGeneration !== this.queueGeneration) return null;
if (typeof bookTextureRenderer.prepareRevealBlock === 'function') {
bookTextureRenderer.prepareRevealBlock({
id: sentence.id,
blockId,
wordTimings: sentence.animation?.wordTimings || [],
cueTimings: sentence.animation?.cueTimings || [],
totalDuration: sentence.animation?.totalDuration || 0,
spread,
preloadOnly: true
}, { preloadOnly: true });
}
return spread;
}
isCurrentQueueItem(item, queueGeneration = this.queueGeneration) {
return queueGeneration === this.queueGeneration && this.sentenceQueue[0] === item;
}
@@ -888,35 +966,28 @@ class SentenceQueueModule extends BaseModule {
const promise = (async () => {
if (queueGeneration !== this.queueGeneration) return null;
await this.preloadAssetsForItem(nextItem, {
sentenceId: nextItem.id,
const prepared = await this.prepareSentence(nextItem, {
blocking: false,
prefetch: true
prefetch: true,
queueIndex: index
});
if (queueGeneration !== this.queueGeneration) return null;
if (!this.isSpeechItem(nextItem)) {
return null;
}
return this.prepareSpeechMetadata(nextItem.text || '', {
sentenceId: nextItem.id,
blockId: nextItem.blockId ?? null,
turnId: nextItem.turnId ?? null,
ttsInstructions: Array.isArray(nextItem.ttsInstructions) ? nextItem.ttsInstructions : [],
queueIndex: index,
prefetch: true,
blocking: false
await this.prefetchWebGLBookPresentation(prepared, {
queueGeneration,
queueIndex: index
});
if (queueGeneration !== this.queueGeneration) return null;
this.preparedSentenceCache.set(nextCacheKey, prepared);
return prepared;
})()
.then(() => {
.then((prepared) => {
if (queueGeneration !== this.queueGeneration) return false;
console.log('SentenceQueue: Prefetched queued speech/media', { sentenceId: nextItem.id, queueIndex: index });
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: { state: 'playing-ready', reason: 'prefetch-complete', sentenceId: nextItem.id, queueIndex: index }
}));
console.log('Process state: playing-ready', { reason: 'prefetch-complete', sentenceId: nextItem.id, queueIndex: index });
return true;
return prepared || true;
})
.catch(err => {
console.warn('SentenceQueue: Prefetch failed:', err);
@@ -1341,6 +1412,7 @@ class SentenceQueueModule extends BaseModule {
this.cancelGenerationRequests('sentence-queue-cleared');
this.cancelAssetPreloads('sentence-queue-cleared');
this.prefetchingSpeech.clear();
this.preparedSentenceCache.clear();
this.pauseBeforeNextReason = null;
document.dispatchEvent(new CustomEvent('tts:queue-empty', {
detail: { reason: 'sentence-queue-cleared' }