Georg/ai.interactive.fiction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preload media assets and refine process cursors

Browse Source
This commit is contained in:
Georg Tomitsch
2026-05-18 11:15:39 +02:00
parent 4f6300042c
commit 6e908037fb
4 changed files with 162 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/js/audio-manager-module.js
+77 -17
View File
@@ -9,6 +9,8 @@ class AudioManagerModule extends BaseModule {
super('audio-manager', 'Audio Manager');
this.sounds = new Map();
this.sfxCache = new Map();
this.musicCache = new Map();
this.imageCache = new Map();
this.currentAudio = null;
this.currentAudioRole = null;
this.currentLoop = null;
@@ -486,29 +488,87 @@ class AudioManagerModule extends BaseModule {
async preloadSfx(filename) {
const url = this.getAssetUrl('sounds', filename);
if (this.sfxCache.has(url)) {
return this.sfxCache.get(url);
}
const promise = new Promise((resolve, reject) => {
const audio = new Audio(url);
audio.preload = 'auto';
this.setMediaVolume(audio, this.getSfxVolume());
audio.addEventListener('canplaythrough', () => resolve(audio), { once: true });
audio.addEventListener('error', () => reject(new Error(`Failed to preload sound effect: ${url}`)), { once: true });
audio.load();
});
if (this.sfxCache.has(url)) return this.sfxCache.get(url);
const promise = this.preloadAudioUrl(url, 'sound effect')
.then(audio => {
this.setMediaVolume(audio, this.getSfxVolume());
return audio;
});
this.sfxCache.set(url, promise);
return promise;
}
async preloadMusic(filename) {
const url = this.getAssetUrl('music', filename);
if (this.musicCache.has(url)) return this.musicCache.get(url);
const promise = this.preloadAudioUrl(url, 'music track')
.then(audio => {
this.setMediaVolume(audio, this.getMusicVolume());
return audio;
});
this.musicCache.set(url, promise);
return promise;
}
preloadAudioUrl(url, label = 'audio') {
return new Promise((resolve, reject) => {
const audio = new Audio(url);
let settled = false;
const finish = (result, error = null) => {
if (settled) return;
settled = true;
audio.removeEventListener('canplaythrough', onReady);
audio.removeEventListener('loadeddata', onReady);
audio.removeEventListener('error', onError);
if (error) reject(error);
else resolve(result);
};
const onReady = () => finish(audio);
const onError = () => finish(null, new Error(`Failed to preload ${label}: ${url}`));
audio.preload = 'auto';
audio.addEventListener('canplaythrough', onReady, { once: true });
audio.addEventListener('loadeddata', onReady, { once: true });
audio.addEventListener('error', onError, { once: true });
audio.load();
});
}
async preloadImage(filename) {
const url = this.getAssetUrl('images', filename);
if (this.imageCache.has(url)) return this.imageCache.get(url);
const promise = new Promise((resolve, reject) => {
const image = new Image();
image.decoding = 'async';
image.onload = () => {
if (typeof image.decode === 'function') {
image.decode().catch(() => null).then(() => resolve(image));
} else {
resolve(image);
}
};
image.onerror = () => reject(new Error(`Failed to preload image: ${url}`));
image.src = url;
});
this.imageCache.set(url, promise);
return promise;
}
async preloadStructuredBlock(block = {}) {
const type = String(block.type || block.kind || '').toLowerCase();
const filename = block.filename || block.metadata?.filename;
if (!filename) return null;
if (type === 'image') return this.preloadImage(filename);
if (type === 'music') return this.preloadMusic(filename);
if (type === 'sfx' || type === 'sound') return this.preloadSfx(filename);
return null;
}
async preloadMediaCues(cues = []) {
const tasks = cues
.filter(cue => cue && cue.type === 'sfx' && cue.filename)
.map(cue => this.preloadSfx(cue.filename).catch(error => {
console.warn('AudioManager: SFX preload failed:', error);
return null;
.filter(cue => cue && cue.filename)
.map(cue => this.preloadStructuredBlock(cue).catch(error => {
console.warn('AudioManager: Media cue preload failed:', error);
throw error;
}));
await Promise.all(tasks);
public/js/sentence-queue-module.js
+58 -11
View File
@@ -39,6 +39,7 @@ class SentenceQueueModule extends BaseModule {
'getPreparedSentence',
'prefetchAhead',
'prepareSpeechMetadata',
'preloadAssetsForItem',
'normalizeTtsText',
'runTtsPreloadWithTimeout',
'cancelBlockingGeneration',
@@ -400,12 +401,7 @@ class SentenceQueueModule extends BaseModule {
try {
if (metadata.type && !['paragraph', 'heading'].includes(metadata.type)) {
if (metadata.type === 'music') {
const audioManager = this.getModule('audio-manager');
if (audioManager && typeof audioManager.playMusic === 'function') {
audioManager.getAssetUrl('music', metadata.filename);
}
}
await this.preloadAssetsForItem(metadata, { blocking: true, sentenceId: id });
return {
id,
@@ -425,7 +421,10 @@ class SentenceQueueModule extends BaseModule {
const audioManager = this.getModule('audio-manager');
if (audioManager && typeof audioManager.preloadMediaCues === 'function') {
await audioManager.preloadMediaCues(metadata.cueMarkers || []);
await this.preloadAssetsForItem({
type: 'paragraph',
cueMarkers: metadata.cueMarkers || []
}, { blocking: true, sentenceId: id });
}
const ttsData = await this.prepareSpeechMetadata(text, {
@@ -597,6 +596,44 @@ class SentenceQueueModule extends BaseModule {
}
}
async preloadAssetsForItem(item = {}, context = {}) {
const audioManager = this.getModule('audio-manager');
if (!audioManager) return;
const tasks = [];
const type = String(item.type || item.kind || '').toLowerCase();
if (['image', 'music', 'sfx', 'sound'].includes(type) && typeof audioManager.preloadStructuredBlock === 'function') {
tasks.push(audioManager.preloadStructuredBlock(item));
}
if (Array.isArray(item.cueMarkers) && item.cueMarkers.length > 0 && typeof audioManager.preloadMediaCues === 'function') {
tasks.push(audioManager.preloadMediaCues(item.cueMarkers));
}
const pending = tasks.filter(Boolean);
if (pending.length === 0) return;
const state = context.blocking ? 'waiting-generating' : 'playing-generating';
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: {
state,
reason: 'asset-preload-start',
sentenceId: context.sentenceId || item.id || null,
assetType: type || 'cue'
}
}));
await Promise.all(pending);
document.dispatchEvent(new CustomEvent('story:process-state', {
detail: {
state: 'playing-ready',
reason: 'asset-preload-complete',
sentenceId: context.sentenceId || item.id || null,
assetType: type || 'cue'
}
}));
}
shouldPauseAfterSentence(sentence) {
if (sentence.kind !== 'paragraph' || this.shouldAutoplay()) {
return false;
@@ -684,16 +721,26 @@ class SentenceQueueModule extends BaseModule {
}));
console.log(`Process state: ${state}`, { reason: 'prefetch-start', sentenceId: nextItem.id, queueIndex: index });
const promise = (this.isSpeechItem(nextItem)
? this.prepareSpeechMetadata(nextItem.text || '', {
const promise = (async () => {
await this.preloadAssetsForItem(nextItem, {
sentenceId: nextItem.id,
blocking: false,
prefetch: true
});
if (!this.isSpeechItem(nextItem)) {
return null;
}
return this.prepareSpeechMetadata(nextItem.text || '', {
sentenceId: nextItem.id,
blockId: nextItem.blockId ?? null,
turnId: nextItem.turnId ?? null,
queueIndex: index,
prefetch: true,
blocking: false
})
: Promise.resolve(null))
});
})()
.then(() => {
console.log('SentenceQueue: Prefetched queued speech/media', { sentenceId: nextItem.id, queueIndex: index });
document.dispatchEvent(new CustomEvent('story:process-state', {
public/js/ui-input-handler-module.js
+22 -15
View File
@@ -362,8 +362,9 @@ class UIInputHandlerModule extends BaseModule {
return '';
}
const fallback = state === 'command-waiting' ? 'wait' : 'progress';
return this.buildMouseCursor(state, fallback, 12, 12, this.cursorAnimationFrame);
const fallback = state === 'command-waiting' || state === 'waiting-generating' ? 'progress' : 'default';
const usesArrowBase = state === 'command-waiting' || state === 'waiting-generating';
return this.buildMouseCursor(state, fallback, usesArrowBase ? 4 : 5, usesArrowBase ? 3 : 24, this.cursorAnimationFrame);
}
buildMouseCursor(state, fallback = 'default', hotspotX = 12, hotspotY = 12, frame = 0) {
@@ -398,25 +399,31 @@ class UIInputHandlerModule extends BaseModule {
}
getMouseCursorSvg(state, frame = 0) {
const stroke = '#222222';
const common = `xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 24 24" fill="none" stroke="${stroke}" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"`;
const stroke = '#2a1b10';
const common = `xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32" fill="none" stroke="${stroke}" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round"`;
const spinnerSpokes = Array.from({ length: 8 }, (_, index) => {
const opacity = 0.25 + (((index + frame) % 8) / 7) * 0.75;
const angle = (index * 45) * Math.PI / 180;
const x1 = 12 + Math.cos(angle) * 6;
const y1 = 12 + Math.sin(angle) * 6;
const x2 = 12 + Math.cos(angle) * 9;
const y2 = 12 + Math.sin(angle) * 9;
const x1 = 24 + Math.cos(angle) * 3;
const y1 = 24 + Math.sin(angle) * 3;
const x2 = 24 + Math.cos(angle) * 5;
const y2 = 24 + Math.sin(angle) * 5;
return `<path opacity="${opacity.toFixed(2)}" d="M${x1.toFixed(2)} ${y1.toFixed(2)} ${x2.toFixed(2)} ${y2.toFixed(2)}"/>`;
}).join('');
const sandTop = frame % 4 < 2 ? '<path d="M9 5h6"/><path d="M10 8h4"/>' : '<path d="M10 16h4"/><path d="M9 19h6"/>';
const arrow = '<path fill="#f6efe2" d="M4 3l9 21 2.4-8.5L24 13z"/><path d="M15.4 15.5 21 21"/>';
const pointer = '<path fill="#f6efe2" d="M9 14V5a2.2 2.2 0 0 1 4.4 0v7.5"/><path d="M13.4 12V8.5a2.1 2.1 0 0 1 4.2 0v5"/><path d="M17.6 14v-1.7a2.1 2.1 0 0 1 4.2 0v4.2A8.1 8.1 0 0 1 13.7 24h-1.2a6.8 6.8 0 0 1-5.5-3L3.6 15a2.1 2.1 0 0 1 3.4-2.4l2 2.5"/>';
const feather = '<path fill="#f6efe2" d="M5 26c5.8-1.7 12.5-7.9 18.4-20.7 2.3 7.6-.2 16.1-11.8 19.9"/><path d="M5 26c4.8-4.5 8.7-9.2 13-15"/><path d="M12 25.2 5 26"/>';
const speaker = '<g transform="translate(20 2) scale(.48)"><path fill="#f6efe2" d="M11 5 6 9H2v6h4l5 4z"/><path d="M15.5 8.5a5 5 0 0 1 0 7"/><path d="M19 5a10 10 0 0 1 0 14"/></g>';
const spinner = `<g>${spinnerSpokes}</g>`;
const hourglassSand = frame % 4 < 2 ? '<path d="M5.7 4.7h4.6"/><path d="M6.7 7h2.6"/>' : '<path d="M6.7 12h2.6"/><path d="M5.7 14.3h4.6"/>';
const hourglass = `<g transform="translate(1 17) scale(.82)"><path fill="#f6efe2" d="M4 2h8M4 16h8M10.8 2v3.1a2 2 0 0 1-.6 1.4L8 8.5l-2.2-2A2 2 0 0 1 5.2 5.1V2M5.2 16v-3.1a2 2 0 0 1 .6-1.4L8 9.5l2.2 2a2 2 0 0 1 .6 1.4V16"/>${hourglassSand}</g>`;
const icons = {
'default': `<svg ${common}><path d="M4 3l7.5 18 2.1-7.4L21 11z"/><path d="M13.6 13.6 18 18"/></svg>`,
'pointer': `<svg ${common}><path d="M8 11V4a2 2 0 1 1 4 0v6"/><path d="M12 10V7a2 2 0 1 1 4 0v5"/><path d="M16 12v-1a2 2 0 1 1 4 0v3a7 7 0 0 1-7 7h-1a6 6 0 0 1-5-2.7L3 13a2 2 0 0 1 3-2.6l2 2.1"/></svg>`,
'command-waiting': `<svg ${common}><path d="M5 22h14"/><path d="M5 2h14"/><path d="M17 22v-4.172a2 2 0 0 0-.586-1.414L12 12l-4.414 4.414A2 2 0 0 0 7 17.828V22"/><path d="M7 2v4.172a2 2 0 0 0 .586 1.414L12 12l4.414-4.414A2 2 0 0 0 17 6.172V2"/>${sandTop}</svg>`,
'waiting-generating': `<svg ${common}>${spinnerSpokes}</svg>`,
'playing-generating': `<svg ${common}><path d="M11 5 6 9H2v6h4l5 4z"/><path d="M15.54 8.46a5 5 0 0 1 0 7.07"/><path d="M19.07 4.93a10 10 0 0 1 0 14.14"/>${spinnerSpokes}</svg>`,
'playing-ready': `<svg ${common}><path d="M11 5 6 9H2v6h4l5 4z"/><path d="M15.54 8.46a5 5 0 0 1 0 7.07"/><path d="M19.07 4.93a10 10 0 0 1 0 14.14"/></svg>`
'default': `<svg ${common}>${arrow}</svg>`,
'pointer': `<svg ${common}>${pointer}</svg>`,
'command-waiting': `<svg ${common}>${arrow}${hourglass}</svg>`,
'waiting-generating': `<svg ${common}>${arrow}${spinner}</svg>`,
'playing-generating': `<svg ${common}>${feather}${speaker}${spinner}</svg>`,
'playing-ready': `<svg ${common}>${feather}${speaker}</svg>`
};
return icons[state] || icons['waiting-generating'];