From 6e908037fbc6c5ba6cfbaa865702b4784449523b Mon Sep 17 00:00:00 2001 From: Georg Tomitsch Date: Mon, 18 May 2026 11:15:39 +0200 Subject: [PATCH] Preload media assets and refine process cursors --- public/css/style.css | 14 ++--- public/js/audio-manager-module.js | 94 +++++++++++++++++++++++----- public/js/sentence-queue-module.js | 69 ++++++++++++++++---- public/js/ui-input-handler-module.js | 37 ++++++----- 4 files changed, 162 insertions(+), 52 deletions(-) diff --git a/public/css/style.css b/public/css/style.css index 8842d82..1e54c98 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -125,7 +125,7 @@ body.switched { --panel-border: rgba(62, 42, 24, 0.46); --control-radius: 0.22rem; --ui-menu-font-size: 0.82rem; - --ui-modal-font-size: calc(var(--story-line-height) * 0.68); + --ui-modal-font-size: calc(var(--story-line-height) * 0.85); font-size: calc(var(--book-height)/(34 * 1.5)); } @@ -1342,17 +1342,13 @@ html[data-process-state="ready"] .story-choices[data-choice-ready="true"] { z-index: 1; /* Ensure cursor appears above text */ } -html[data-process-state="command-waiting"], -html[data-process-state="command-waiting"] * { +html[data-process-state="command-waiting"] body { cursor: var(--process-cursor, wait) !important; } -html[data-process-state="waiting-generating"], -html[data-process-state="waiting-generating"] *, -html[data-process-state="playing-generating"], -html[data-process-state="playing-generating"] *, -html[data-process-state="playing-ready"], -html[data-process-state="playing-ready"] * { +html[data-process-state="waiting-generating"] body, +html[data-process-state="playing-generating"] body, +html[data-process-state="playing-ready"] body { cursor: var(--process-cursor, progress) !important; } diff --git a/public/js/audio-manager-module.js b/public/js/audio-manager-module.js index 44fa6b8..56f6957 100644 --- a/public/js/audio-manager-module.js +++ b/public/js/audio-manager-module.js @@ -9,6 +9,8 @@ class AudioManagerModule extends BaseModule { super('audio-manager', 'Audio Manager'); this.sounds = new Map(); this.sfxCache = new Map(); + this.musicCache = new Map(); + this.imageCache = new Map(); this.currentAudio = null; this.currentAudioRole = null; this.currentLoop = null; @@ -486,29 +488,87 @@ class AudioManagerModule extends BaseModule { async preloadSfx(filename) { const url = this.getAssetUrl('sounds', filename); - if (this.sfxCache.has(url)) { - return this.sfxCache.get(url); - } - - const promise = new Promise((resolve, reject) => { - const audio = new Audio(url); - audio.preload = 'auto'; - this.setMediaVolume(audio, this.getSfxVolume()); - audio.addEventListener('canplaythrough', () => resolve(audio), { once: true }); - audio.addEventListener('error', () => reject(new Error(`Failed to preload sound effect: ${url}`)), { once: true }); - audio.load(); - }); - + if (this.sfxCache.has(url)) return this.sfxCache.get(url); + const promise = this.preloadAudioUrl(url, 'sound effect') + .then(audio => { + this.setMediaVolume(audio, this.getSfxVolume()); + return audio; + }); this.sfxCache.set(url, promise); return promise; } + async preloadMusic(filename) { + const url = this.getAssetUrl('music', filename); + if (this.musicCache.has(url)) return this.musicCache.get(url); + const promise = this.preloadAudioUrl(url, 'music track') + .then(audio => { + this.setMediaVolume(audio, this.getMusicVolume()); + return audio; + }); + this.musicCache.set(url, promise); + return promise; + } + + preloadAudioUrl(url, label = 'audio') { + return new Promise((resolve, reject) => { + const audio = new Audio(url); + let settled = false; + const finish = (result, error = null) => { + if (settled) return; + settled = true; + audio.removeEventListener('canplaythrough', onReady); + audio.removeEventListener('loadeddata', onReady); + audio.removeEventListener('error', onError); + if (error) reject(error); + else resolve(result); + }; + const onReady = () => finish(audio); + const onError = () => finish(null, new Error(`Failed to preload ${label}: ${url}`)); + audio.preload = 'auto'; + audio.addEventListener('canplaythrough', onReady, { once: true }); + audio.addEventListener('loadeddata', onReady, { once: true }); + audio.addEventListener('error', onError, { once: true }); + audio.load(); + }); + } + + async preloadImage(filename) { + const url = this.getAssetUrl('images', filename); + if (this.imageCache.has(url)) return this.imageCache.get(url); + const promise = new Promise((resolve, reject) => { + const image = new Image(); + image.decoding = 'async'; + image.onload = () => { + if (typeof image.decode === 'function') { + image.decode().catch(() => null).then(() => resolve(image)); + } else { + resolve(image); + } + }; + image.onerror = () => reject(new Error(`Failed to preload image: ${url}`)); + image.src = url; + }); + this.imageCache.set(url, promise); + return promise; + } + + async preloadStructuredBlock(block = {}) { + const type = String(block.type || block.kind || '').toLowerCase(); + const filename = block.filename || block.metadata?.filename; + if (!filename) return null; + if (type === 'image') return this.preloadImage(filename); + if (type === 'music') return this.preloadMusic(filename); + if (type === 'sfx' || type === 'sound') return this.preloadSfx(filename); + return null; + } + async preloadMediaCues(cues = []) { const tasks = cues - .filter(cue => cue && cue.type === 'sfx' && cue.filename) - .map(cue => this.preloadSfx(cue.filename).catch(error => { - console.warn('AudioManager: SFX preload failed:', error); - return null; + .filter(cue => cue && cue.filename) + .map(cue => this.preloadStructuredBlock(cue).catch(error => { + console.warn('AudioManager: Media cue preload failed:', error); + throw error; })); await Promise.all(tasks); diff --git a/public/js/sentence-queue-module.js b/public/js/sentence-queue-module.js index ab611f3..d1b39ab 100644 --- a/public/js/sentence-queue-module.js +++ b/public/js/sentence-queue-module.js @@ -39,6 +39,7 @@ class SentenceQueueModule extends BaseModule { 'getPreparedSentence', 'prefetchAhead', 'prepareSpeechMetadata', + 'preloadAssetsForItem', 'normalizeTtsText', 'runTtsPreloadWithTimeout', 'cancelBlockingGeneration', @@ -400,12 +401,7 @@ class SentenceQueueModule extends BaseModule { try { if (metadata.type && !['paragraph', 'heading'].includes(metadata.type)) { - if (metadata.type === 'music') { - const audioManager = this.getModule('audio-manager'); - if (audioManager && typeof audioManager.playMusic === 'function') { - audioManager.getAssetUrl('music', metadata.filename); - } - } + await this.preloadAssetsForItem(metadata, { blocking: true, sentenceId: id }); return { id, @@ -425,7 +421,10 @@ class SentenceQueueModule extends BaseModule { const audioManager = this.getModule('audio-manager'); if (audioManager && typeof audioManager.preloadMediaCues === 'function') { - await audioManager.preloadMediaCues(metadata.cueMarkers || []); + await this.preloadAssetsForItem({ + type: 'paragraph', + cueMarkers: metadata.cueMarkers || [] + }, { blocking: true, sentenceId: id }); } const ttsData = await this.prepareSpeechMetadata(text, { @@ -597,6 +596,44 @@ class SentenceQueueModule extends BaseModule { } } + async preloadAssetsForItem(item = {}, context = {}) { + const audioManager = this.getModule('audio-manager'); + if (!audioManager) return; + + const tasks = []; + const type = String(item.type || item.kind || '').toLowerCase(); + if (['image', 'music', 'sfx', 'sound'].includes(type) && typeof audioManager.preloadStructuredBlock === 'function') { + tasks.push(audioManager.preloadStructuredBlock(item)); + } + if (Array.isArray(item.cueMarkers) && item.cueMarkers.length > 0 && typeof audioManager.preloadMediaCues === 'function') { + tasks.push(audioManager.preloadMediaCues(item.cueMarkers)); + } + + const pending = tasks.filter(Boolean); + if (pending.length === 0) return; + + const state = context.blocking ? 'waiting-generating' : 'playing-generating'; + document.dispatchEvent(new CustomEvent('story:process-state', { + detail: { + state, + reason: 'asset-preload-start', + sentenceId: context.sentenceId || item.id || null, + assetType: type || 'cue' + } + })); + + await Promise.all(pending); + + document.dispatchEvent(new CustomEvent('story:process-state', { + detail: { + state: 'playing-ready', + reason: 'asset-preload-complete', + sentenceId: context.sentenceId || item.id || null, + assetType: type || 'cue' + } + })); + } + shouldPauseAfterSentence(sentence) { if (sentence.kind !== 'paragraph' || this.shouldAutoplay()) { return false; @@ -684,16 +721,26 @@ class SentenceQueueModule extends BaseModule { })); console.log(`Process state: ${state}`, { reason: 'prefetch-start', sentenceId: nextItem.id, queueIndex: index }); - const promise = (this.isSpeechItem(nextItem) - ? this.prepareSpeechMetadata(nextItem.text || '', { + const promise = (async () => { + await this.preloadAssetsForItem(nextItem, { + sentenceId: nextItem.id, + blocking: false, + prefetch: true + }); + + if (!this.isSpeechItem(nextItem)) { + return null; + } + + return this.prepareSpeechMetadata(nextItem.text || '', { sentenceId: nextItem.id, blockId: nextItem.blockId ?? null, turnId: nextItem.turnId ?? null, queueIndex: index, prefetch: true, blocking: false - }) - : Promise.resolve(null)) + }); + })() .then(() => { console.log('SentenceQueue: Prefetched queued speech/media', { sentenceId: nextItem.id, queueIndex: index }); document.dispatchEvent(new CustomEvent('story:process-state', { diff --git a/public/js/ui-input-handler-module.js b/public/js/ui-input-handler-module.js index 17210a8..676d411 100644 --- a/public/js/ui-input-handler-module.js +++ b/public/js/ui-input-handler-module.js @@ -362,8 +362,9 @@ class UIInputHandlerModule extends BaseModule { return ''; } - const fallback = state === 'command-waiting' ? 'wait' : 'progress'; - return this.buildMouseCursor(state, fallback, 12, 12, this.cursorAnimationFrame); + const fallback = state === 'command-waiting' || state === 'waiting-generating' ? 'progress' : 'default'; + const usesArrowBase = state === 'command-waiting' || state === 'waiting-generating'; + return this.buildMouseCursor(state, fallback, usesArrowBase ? 4 : 5, usesArrowBase ? 3 : 24, this.cursorAnimationFrame); } buildMouseCursor(state, fallback = 'default', hotspotX = 12, hotspotY = 12, frame = 0) { @@ -398,25 +399,31 @@ class UIInputHandlerModule extends BaseModule { } getMouseCursorSvg(state, frame = 0) { - const stroke = '#222222'; - const common = `xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 24 24" fill="none" stroke="${stroke}" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"`; + const stroke = '#2a1b10'; + const common = `xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32" fill="none" stroke="${stroke}" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round"`; const spinnerSpokes = Array.from({ length: 8 }, (_, index) => { const opacity = 0.25 + (((index + frame) % 8) / 7) * 0.75; const angle = (index * 45) * Math.PI / 180; - const x1 = 12 + Math.cos(angle) * 6; - const y1 = 12 + Math.sin(angle) * 6; - const x2 = 12 + Math.cos(angle) * 9; - const y2 = 12 + Math.sin(angle) * 9; + const x1 = 24 + Math.cos(angle) * 3; + const y1 = 24 + Math.sin(angle) * 3; + const x2 = 24 + Math.cos(angle) * 5; + const y2 = 24 + Math.sin(angle) * 5; return ``; }).join(''); - const sandTop = frame % 4 < 2 ? '' : ''; + const arrow = ''; + const pointer = ''; + const feather = ''; + const speaker = ''; + const spinner = `${spinnerSpokes}`; + const hourglassSand = frame % 4 < 2 ? '' : ''; + const hourglass = `${hourglassSand}`; const icons = { - 'default': ``, - 'pointer': ``, - 'command-waiting': `${sandTop}`, - 'waiting-generating': `${spinnerSpokes}`, - 'playing-generating': `${spinnerSpokes}`, - 'playing-ready': `` + 'default': `${arrow}`, + 'pointer': `${pointer}`, + 'command-waiting': `${arrow}${hourglass}`, + 'waiting-generating': `${arrow}${spinner}`, + 'playing-generating': `${feather}${speaker}${spinner}`, + 'playing-ready': `${feather}${speaker}` }; return icons[state] || icons['waiting-generating'];