From 0fa8ac737926443b5e604f4181713f1388904d54 Mon Sep 17 00:00:00 2001 From: daniel-c-harvey Date: Sat, 13 Sep 2025 15:22:26 -0400 Subject: [PATCH] Front End Streaming Playback Improvements --- .claude/settings.local.json | 3 +- .../Clients/TrackMediaClient.cs | 7 +- DeepDrftWeb.Client/Pages/Home.razor | 2 +- .../Services/AudioInteropService.cs | 22 ++ DeepDrftWeb.Client/Services/IPlayerService.cs | 19 +- DeepDrftWeb/Components/App.razor | 4 +- DeepDrftWeb/Interop/wavutils.ts | 118 ++++++++ DeepDrftWeb/Interop/webaudio.ts | 253 +++++++++++++++++- 8 files changed, 417 insertions(+), 11 deletions(-) create mode 100644 DeepDrftWeb/Interop/wavutils.ts diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 7e48cac..ee5e280 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -45,7 +45,8 @@ "Read(/F:\\Development\\DeepDrftHome\\DeepDrftTests/**)", "Read(/F:\\Development\\DeepDrftHome\\DeepDrftTests/**)", "Read(/F:\\Development\\DeepDrftHome\\DeepDrftTests/**)", - "Read(//f/Development/NetBlocks/**)" + "Read(//f/Development/NetBlocks/**)", + "Read(//c/lib/NetBlocks/Models//**)" ], "deny": [], "ask": [] diff --git a/DeepDrftWeb.Client/Clients/TrackMediaClient.cs b/DeepDrftWeb.Client/Clients/TrackMediaClient.cs index 17da949..d0d8923 100644 --- a/DeepDrftWeb.Client/Clients/TrackMediaClient.cs +++ b/DeepDrftWeb.Client/Clients/TrackMediaClient.cs @@ -3,7 +3,7 @@ using NetBlocks.Models; namespace DeepDrftWeb.Client.Clients; -public class TrackMediaResponse +public class TrackMediaResponse : IDisposable { public Stream Stream { get; } public long ContentLength { get; } @@ -13,6 +13,11 @@ public class TrackMediaResponse Stream = stream; ContentLength = contentLength; } + + public void Dispose() + { + Stream?.Dispose(); + } } public class TrackMediaClient diff --git a/DeepDrftWeb.Client/Pages/Home.razor b/DeepDrftWeb.Client/Pages/Home.razor index 99279cb..97b2264 100644 --- a/DeepDrftWeb.Client/Pages/Home.razor +++ b/DeepDrftWeb.Client/Pages/Home.razor @@ -7,7 +7,7 @@ @* Hero Section *@ - + diff --git a/DeepDrftWeb.Client/Services/AudioInteropService.cs b/DeepDrftWeb.Client/Services/AudioInteropService.cs index 791d200..9c385ad 100644 --- a/DeepDrftWeb.Client/Services/AudioInteropService.cs +++ b/DeepDrftWeb.Client/Services/AudioInteropService.cs @@ -40,6 +40,21 @@ public class AudioInteropService : IAsyncDisposable return await InvokeJsAsync("DeepDrftAudio.finalizeAudioBuffer", playerId); } + // Streaming methods + public async Task InitializeStreaming(string playerId) + { + return await InvokeJsAsync("DeepDrftAudio.initializeStreaming", playerId); + } + + public async Task ProcessStreamingChunk(string playerId, byte[] audioChunk) + { + return await InvokeJsAsync("DeepDrftAudio.processStreamingChunk", playerId, audioChunk); + } + + public async Task StartStreamingPlayback(string playerId) + { + return await InvokeJsAsync("DeepDrftAudio.startStreamingPlayback", playerId); + } public async Task PlayAsync(string playerId) { @@ -216,6 +231,13 @@ public class AudioLoadResult : AudioOperationResult public double LoadProgress { get; set; } } +public class StreamingResult : AudioOperationResult +{ + public bool CanStartStreaming { get; set; } + public bool HeaderParsed { get; set; } + public int BufferCount { get; set; } +} + public class AudioPlayerState { public bool IsPlaying { get; set; } diff --git a/DeepDrftWeb.Client/Services/IPlayerService.cs b/DeepDrftWeb.Client/Services/IPlayerService.cs index 56d183c..120af1e 100644 --- a/DeepDrftWeb.Client/Services/IPlayerService.cs +++ b/DeepDrftWeb.Client/Services/IPlayerService.cs @@ -1,4 +1,5 @@ using DeepDrftModels.Entities; +using Microsoft.AspNetCore.Components; using NetBlocks.Models; namespace DeepDrftWeb.Client.Services; @@ -18,8 +19,8 @@ public interface IPlayerService string? ErrorMessage { get; } // Events for UI updates - event Action? OnStateChanged; - event Events.EventAsync OnTrackSelected; + EventCallback? OnStateChanged { get; set; } + EventCallback? OnTrackSelected { get; set; } // Control methods Task InitializeAsync(); @@ -29,5 +30,17 @@ public interface IPlayerService Task TogglePlayPause(); Task Seek(double position); Task SetVolume(double volume); - void ClearError(); + Task ClearError(); +} + +public interface IStreamingPlayerService : IPlayerService +{ + // Streaming state properties + bool IsStreamingMode { get; } + bool CanStartStreaming { get; } + bool HeaderParsed { get; } + int BufferedChunks { get; } + + // Streaming control methods + Task SelectTrackStreaming(TrackEntity track); } \ No newline at end of file diff --git a/DeepDrftWeb/Components/App.razor b/DeepDrftWeb/Components/App.razor index 43fd7d1..b5dce81 100644 --- a/DeepDrftWeb/Components/App.razor +++ b/DeepDrftWeb/Components/App.razor @@ -23,7 +23,9 @@ - + diff --git a/DeepDrftWeb/Interop/wavutils.ts b/DeepDrftWeb/Interop/wavutils.ts new file mode 100644 index 0000000..cb6f1ae --- /dev/null +++ b/DeepDrftWeb/Interop/wavutils.ts @@ -0,0 +1,118 @@ +interface WavHeader { + sampleRate: number; + channels: number; + bitsPerSample: number; + byteRate: number; + blockAlign: number; + dataSize: number; + headerSize: number; +} + +class WavUtils { + static parseHeader(chunks: Uint8Array[], totalSize: number): WavHeader | null { + if (totalSize < 44) return null; + + const concatenated = new Uint8Array(totalSize); + let offset = 0; + for (const chunk of chunks) { + concatenated.set(chunk, offset); + offset += chunk.length; + } + + const view = new DataView(concatenated.buffer, 0, 44); + + // Check RIFF header + const riff = new TextDecoder().decode(concatenated.slice(0, 4)); + if (riff !== 'RIFF') return null; + + const wave = new TextDecoder().decode(concatenated.slice(8, 12)); + if (wave !== 'WAVE') return null; + + // Find fmt chunk + let fmtOffset = 12; + while (fmtOffset < totalSize - 8) { + const chunkId = new TextDecoder().decode(concatenated.slice(fmtOffset, fmtOffset + 4)); + const chunkSize = view.getUint32(fmtOffset + 4, true); + + if (chunkId === 'fmt ') { + const channels = view.getUint16(fmtOffset + 10, true); + const sampleRate = view.getUint32(fmtOffset + 12, true); + const byteRate = view.getUint32(fmtOffset + 16, true); + const blockAlign = view.getUint16(fmtOffset + 20, true); + const bitsPerSample = view.getUint16(fmtOffset + 22, true); + + return { + sampleRate, + channels, + bitsPerSample, + byteRate, + blockAlign, + dataSize: 0, // Will be updated when we find data chunk + headerSize: 44 + }; + } + + fmtOffset += 8 + chunkSize; + } + + return null; + } + + static createHeader(wavHeader: WavHeader, dataSize: number): Uint8Array { + const header = new ArrayBuffer(44); + const view = new DataView(header); + + // RIFF header + view.setUint8(0, 0x52); view.setUint8(1, 0x49); view.setUint8(2, 0x46); view.setUint8(3, 0x46); // "RIFF" + view.setUint32(4, 36 + dataSize, true); // File size + view.setUint8(8, 0x57); view.setUint8(9, 0x41); view.setUint8(10, 0x56); view.setUint8(11, 0x45); // "WAVE" + + // fmt chunk + view.setUint8(12, 0x66); view.setUint8(13, 0x6d); view.setUint8(14, 0x74); view.setUint8(15, 0x20); // "fmt " + view.setUint32(16, 16, true); // fmt chunk size + view.setUint16(20, 1, true); // Audio format (PCM) + view.setUint16(22, wavHeader.channels, true); + view.setUint32(24, wavHeader.sampleRate, true); + view.setUint32(28, wavHeader.byteRate, true); + view.setUint16(32, wavHeader.blockAlign, true); + view.setUint16(34, wavHeader.bitsPerSample, true); + + // data chunk header + view.setUint8(36, 0x64); view.setUint8(37, 0x61); view.setUint8(38, 0x74); view.setUint8(39, 0x61); // "data" + view.setUint32(40, dataSize, true); + + return new Uint8Array(header); + } + + static extractAudioData(chunks: Uint8Array[], totalSize: number, headerSize: number, chunkSize: number): Uint8Array { + const bufferData = new Uint8Array(chunkSize + headerSize); + let dataOffset = headerSize; // Skip header space initially + let remainingSize = chunkSize; + + // Fill with audio data, skipping the header from the first chunk + let chunkIndex = 0; + let chunkOffset = headerSize; // Skip WAV header in first chunk + + while (remainingSize > 0 && chunkIndex < chunks.length) { + const chunk = chunks[chunkIndex]; + const availableInChunk = chunk.length - chunkOffset; + const toCopy = Math.min(availableInChunk, remainingSize); + + if (toCopy > 0) { + bufferData.set(chunk.slice(chunkOffset, chunkOffset + toCopy), dataOffset); + dataOffset += toCopy; + remainingSize -= toCopy; + chunkOffset += toCopy; + } + + if (chunkOffset >= chunk.length) { + chunkIndex++; + chunkOffset = 0; // No header to skip in subsequent chunks + } + } + + return bufferData.slice(0, dataOffset); + } +} + +export { WavHeader, WavUtils }; \ No newline at end of file diff --git a/DeepDrftWeb/Interop/webaudio.ts b/DeepDrftWeb/Interop/webaudio.ts index ea46266..ecf5571 100644 --- a/DeepDrftWeb/Interop/webaudio.ts +++ b/DeepDrftWeb/Interop/webaudio.ts @@ -10,6 +10,14 @@ interface LoadAudioResult extends AudioResult { loadProgress?: number; } +import { WavHeader, WavUtils } from './wavutils.js'; + +interface StreamingResult extends AudioResult { + canStartStreaming?: boolean; + headerParsed?: boolean; + bufferCount?: number; +} + interface AudioState { isPlaying: boolean; isPaused: boolean; @@ -21,10 +29,19 @@ interface AudioState { type ProgressCallback = (currentTime: number) => void; type EndCallback = () => void; +type DecodeSuccessCallback = (audioBuffer: AudioBuffer) => void; +type DecodeErrorCallback = (error: DOMException) => void; -interface Window { - webkitAudioContext?: typeof AudioContext; - DeepDrftAudio: typeof DeepDrftAudio; +declare global { + interface Window { + webkitAudioContext?: new() => AudioContext; + DeepDrftAudio: typeof DeepDrftAudio; + } + + interface AudioContext { + decodeAudioData(audioData: ArrayBuffer | SharedArrayBuffer): Promise; + decodeAudioData(audioData: ArrayBuffer | SharedArrayBuffer, successCallback?: DecodeSuccessCallback, errorCallback?: DecodeErrorCallback): Promise; + } } class AudioPlayer { @@ -43,10 +60,28 @@ class AudioPlayer { private bufferChunks: Uint8Array[] = []; private expectedSize: number = 0; private currentSize: number = 0; + + // Streaming properties + private isStreamingMode: boolean = false; + private wavHeader: WavHeader | null = null; + private bufferQueue: AudioBuffer[] = []; + private currentStreamSource: AudioBufferSourceNode | null = null; + private nextStartTime: number = 0; + private streamingStarted: boolean = false; + private minBuffersForStreaming: number = 3; + + // Buffer optimization + private cachedWavHeader: Uint8Array | null = null; + private reusableBuffer: Uint8Array | null = null; + private maxReusableBufferSize: number = 128 * 1024; // 128KB max reusable buffer async initialize(): Promise { try { - this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); + const AudioContextClass = window.AudioContext || window.webkitAudioContext; + if (!AudioContextClass) { + throw new Error('Web Audio API not supported'); + } + this.audioContext = new AudioContextClass(); this.gainNode = this.audioContext.createGain(); this.gainNode.connect(this.audioContext.destination); return { success: true }; @@ -285,6 +320,170 @@ class AudioPlayer { this.onEndCallback = callback; } + initializeStreaming(): AudioResult { + try { + this.isStreamingMode = true; + this.bufferChunks = []; + this.bufferQueue = []; + this.currentSize = 0; + this.wavHeader = null; + this.streamingStarted = false; + this.nextStartTime = 0; + return { success: true }; + } catch (error) { + return { success: false, error: (error as Error).message }; + } + } + + processStreamingChunk(audioChunk: Uint8Array): StreamingResult { + try { + this.bufferChunks.push(audioChunk); + this.currentSize += audioChunk.length; + + // Parse WAV header from first chunk if not done yet + if (!this.wavHeader && this.currentSize >= 44) { + const header = WavUtils.parseHeader(this.bufferChunks, this.currentSize); + if (header) { + this.wavHeader = header; + // Cache the WAV header for reuse + this.cachedWavHeader = WavUtils.createHeader(header, 64 * 1024); // Cache with dummy size + } + } + + // Try to create audio buffers from accumulated chunks + if (this.wavHeader) { + this.processBufferedChunks(); + } + + const canStart = this.wavHeader !== null && this.bufferQueue.length >= this.minBuffersForStreaming; + + return { + success: true, + canStartStreaming: canStart, + headerParsed: this.wavHeader !== null, + bufferCount: this.bufferQueue.length + }; + } catch (error) { + return { success: false, error: (error as Error).message }; + } + } + + startStreamingPlayback(): AudioResult { + if (!this.wavHeader || this.bufferQueue.length === 0) { + return { success: false, error: "Not ready for streaming playback" }; + } + + try { + if (this.audioContext!.state === 'suspended') { + this.audioContext!.resume(); + } + + this.streamingStarted = true; + this.isPlaying = true; + this.isPaused = false; + this.nextStartTime = this.audioContext!.currentTime; + this.startTime = this.nextStartTime; + + this.scheduleNextBuffer(); + this.startProgressTracking(); + + return { success: true }; + } catch (error) { + return { success: false, error: (error as Error).message }; + } + } + + + private processBufferedChunks(): void { + if (!this.wavHeader || this.bufferChunks.length === 0) return; + + try { + // Process chunks in groups to create audio buffers + const chunkSize = 64 * 1024; // 64KB chunks for streaming + while (this.currentSize >= chunkSize + this.wavHeader.headerSize) { + // Extract audio data using WavUtils + const audioData = WavUtils.extractAudioData(this.bufferChunks, this.currentSize, this.wavHeader.headerSize, chunkSize); + + // Reuse buffer if possible to reduce allocations + const totalSize = this.cachedWavHeader!.length + audioData.length - this.wavHeader.headerSize; + if (!this.reusableBuffer || this.reusableBuffer.length < totalSize) { + // Only allocate if we don't have a buffer or it's too small + this.reusableBuffer = new Uint8Array(Math.min(totalSize, this.maxReusableBufferSize)); + } + + // Create complete WAV buffer using cached header and reusable buffer + const completeBuffer = this.reusableBuffer.slice(0, totalSize); + completeBuffer.set(this.cachedWavHeader!.slice(0, this.wavHeader.headerSize), 0); + completeBuffer.set(audioData.subarray(this.wavHeader.headerSize), this.wavHeader.headerSize); + + // Create audio buffer from the chunk + this.createAudioBufferFromChunk(completeBuffer); + + // Remove processed data + this.removeProcessedChunks(chunkSize); + break; // Process one chunk at a time + } + } catch (error) { + console.error('Error processing buffered chunks:', error); + } + } + + private async createAudioBufferFromChunk(chunkData: Uint8Array): Promise { + try { + const arrayBuffer = chunkData.buffer.slice(chunkData.byteOffset, chunkData.byteOffset + chunkData.byteLength); + const audioBuffer = await this.audioContext!.decodeAudioData(arrayBuffer); + this.bufferQueue.push(audioBuffer); + + // Schedule buffer if streaming has started + if (this.streamingStarted) { + this.scheduleNextBuffer(); + } + } catch (error) { + console.error('Error creating audio buffer from chunk:', error); + } + } + + private scheduleNextBuffer(): void { + if (this.bufferQueue.length === 0 || !this.streamingStarted) return; + + const buffer = this.bufferQueue.shift()!; + const source = this.audioContext!.createBufferSource(); + source.buffer = buffer; + source.connect(this.gainNode!); + + source.onended = () => { + if (this.bufferQueue.length > 0) { + this.scheduleNextBuffer(); + } else if (!this.isPlaying) { + this.onEndCallback?.(); + } + }; + + source.start(this.nextStartTime); + this.nextStartTime += buffer.duration; + this.currentStreamSource = source; + } + + + private removeProcessedChunks(processedSize: number): void { + let remaining = processedSize; + + while (remaining > 0 && this.bufferChunks.length > 0) { + const chunk = this.bufferChunks[0]; + if (chunk.length <= remaining) { + remaining -= chunk.length; + this.currentSize -= chunk.length; + this.bufferChunks.shift(); + } else { + // Partial chunk removal + const newChunk = chunk.slice(remaining); + this.bufferChunks[0] = newChunk; + this.currentSize -= remaining; + remaining = 0; + } + } + } + unload(): AudioResult { try { this.stop(); @@ -294,6 +493,21 @@ class AudioPlayer { this.currentSize = 0; this.expectedSize = 0; + // Clean up streaming state + this.isStreamingMode = false; + this.wavHeader = null; + this.bufferQueue = []; + this.streamingStarted = false; + this.nextStartTime = 0; + if (this.currentStreamSource) { + this.currentStreamSource.stop(); + this.currentStreamSource = null; + } + + // Clean up cached buffers + this.cachedWavHeader = null; + this.reusableBuffer = null; + return { success: true }; } catch (error) { return { success: false, error: (error as Error).message }; @@ -311,6 +525,13 @@ class AudioPlayer { this.gainNode = null; this.bufferChunks = []; this.currentSize = 0; + + // Clean up streaming state + this.bufferQueue = []; + this.wavHeader = null; + this.currentStreamSource = null; + this.cachedWavHeader = null; + this.reusableBuffer = null; } } @@ -361,6 +582,30 @@ const DeepDrftAudio = { return await player.finalizeAudioBuffer(); }, + // Streaming methods + initializeStreaming: (playerId: string): AudioResult => { + const player = audioPlayers.get(playerId); + if (!player) { + return { success: false, error: "Player not found" }; + } + return player.initializeStreaming(); + }, + + processStreamingChunk: (playerId: string, audioChunk: Uint8Array): StreamingResult => { + const player = audioPlayers.get(playerId); + if (!player) { + return { success: false, error: "Player not found" }; + } + return player.processStreamingChunk(audioChunk); + }, + + startStreamingPlayback: (playerId: string): AudioResult => { + const player = audioPlayers.get(playerId); + if (!player) { + return { success: false, error: "Player not found" }; + } + return player.startStreamingPlayback(); + }, play: (playerId: string): AudioResult => { const player = audioPlayers.get(playerId);