Front End Streaming Playback Improvements

DeepDrftWeb/Interop/webaudio.ts

@@ -10,6 +10,14 @@ interface LoadAudioResult extends AudioResult {
     loadProgress?: number;
 }
 
+import { WavHeader, WavUtils } from './wavutils.js';
+
+interface StreamingResult extends AudioResult {
+    canStartStreaming?: boolean;
+    headerParsed?: boolean;
+    bufferCount?: number;
+}
+
 interface AudioState {
     isPlaying: boolean;
     isPaused: boolean;
@@ -21,10 +29,19 @@ interface AudioState {
 
 type ProgressCallback = (currentTime: number) => void;
 type EndCallback = () => void;
+type DecodeSuccessCallback = (audioBuffer: AudioBuffer) => void;
+type DecodeErrorCallback = (error: DOMException) => void;
 
-interface Window {
-    webkitAudioContext?: typeof AudioContext;
-    DeepDrftAudio: typeof DeepDrftAudio;
+declare global {
+    interface Window {
+        webkitAudioContext?: new() => AudioContext;
+        DeepDrftAudio: typeof DeepDrftAudio;
+    }
+    
+    interface AudioContext {
+        decodeAudioData(audioData: ArrayBuffer | SharedArrayBuffer): Promise<AudioBuffer>;
+        decodeAudioData(audioData: ArrayBuffer | SharedArrayBuffer, successCallback?: DecodeSuccessCallback, errorCallback?: DecodeErrorCallback): Promise<AudioBuffer>;
+    }
 }
 
 class AudioPlayer {
@@ -43,10 +60,28 @@ class AudioPlayer {
     private bufferChunks: Uint8Array[] = [];
     private expectedSize: number = 0;
     private currentSize: number = 0;
+    
+    // Streaming properties
+    private isStreamingMode: boolean = false;
+    private wavHeader: WavHeader | null = null;
+    private bufferQueue: AudioBuffer[] = [];
+    private currentStreamSource: AudioBufferSourceNode | null = null;
+    private nextStartTime: number = 0;
+    private streamingStarted: boolean = false;
+    private minBuffersForStreaming: number = 3;
+    
+    // Buffer optimization
+    private cachedWavHeader: Uint8Array | null = null;
+    private reusableBuffer: Uint8Array | null = null;
+    private maxReusableBufferSize: number = 128 * 1024; // 128KB max reusable buffer
 
     async initialize(): Promise<AudioResult> {
         try {
-            this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
+            const AudioContextClass = window.AudioContext || window.webkitAudioContext;
+            if (!AudioContextClass) {
+                throw new Error('Web Audio API not supported');
+            }
+            this.audioContext = new AudioContextClass();
             this.gainNode = this.audioContext.createGain();
             this.gainNode.connect(this.audioContext.destination);
             return { success: true };
@@ -285,6 +320,170 @@ class AudioPlayer {
         this.onEndCallback = callback;
     }
 
+    initializeStreaming(): AudioResult {
+        try {
+            this.isStreamingMode = true;
+            this.bufferChunks = [];
+            this.bufferQueue = [];
+            this.currentSize = 0;
+            this.wavHeader = null;
+            this.streamingStarted = false;
+            this.nextStartTime = 0;
+            return { success: true };
+        } catch (error) {
+            return { success: false, error: (error as Error).message };
+        }
+    }
+
+    processStreamingChunk(audioChunk: Uint8Array): StreamingResult {
+        try {
+            this.bufferChunks.push(audioChunk);
+            this.currentSize += audioChunk.length;
+
+            // Parse WAV header from first chunk if not done yet
+            if (!this.wavHeader && this.currentSize >= 44) {
+                const header = WavUtils.parseHeader(this.bufferChunks, this.currentSize);
+                if (header) {
+                    this.wavHeader = header;
+                    // Cache the WAV header for reuse
+                    this.cachedWavHeader = WavUtils.createHeader(header, 64 * 1024); // Cache with dummy size
+                }
+            }
+
+            // Try to create audio buffers from accumulated chunks
+            if (this.wavHeader) {
+                this.processBufferedChunks();
+            }
+
+            const canStart = this.wavHeader !== null && this.bufferQueue.length >= this.minBuffersForStreaming;
+
+            return {
+                success: true,
+                canStartStreaming: canStart,
+                headerParsed: this.wavHeader !== null,
+                bufferCount: this.bufferQueue.length
+            };
+        } catch (error) {
+            return { success: false, error: (error as Error).message };
+        }
+    }
+
+    startStreamingPlayback(): AudioResult {
+        if (!this.wavHeader || this.bufferQueue.length === 0) {
+            return { success: false, error: "Not ready for streaming playback" };
+        }
+
+        try {
+            if (this.audioContext!.state === 'suspended') {
+                this.audioContext!.resume();
+            }
+
+            this.streamingStarted = true;
+            this.isPlaying = true;
+            this.isPaused = false;
+            this.nextStartTime = this.audioContext!.currentTime;
+            this.startTime = this.nextStartTime;
+            
+            this.scheduleNextBuffer();
+            this.startProgressTracking();
+
+            return { success: true };
+        } catch (error) {
+            return { success: false, error: (error as Error).message };
+        }
+    }
+
+
+    private processBufferedChunks(): void {
+        if (!this.wavHeader || this.bufferChunks.length === 0) return;
+
+        try {
+            // Process chunks in groups to create audio buffers
+            const chunkSize = 64 * 1024; // 64KB chunks for streaming
+            while (this.currentSize >= chunkSize + this.wavHeader.headerSize) {
+                // Extract audio data using WavUtils
+                const audioData = WavUtils.extractAudioData(this.bufferChunks, this.currentSize, this.wavHeader.headerSize, chunkSize);
+                
+                // Reuse buffer if possible to reduce allocations
+                const totalSize = this.cachedWavHeader!.length + audioData.length - this.wavHeader.headerSize;
+                if (!this.reusableBuffer || this.reusableBuffer.length < totalSize) {
+                    // Only allocate if we don't have a buffer or it's too small
+                    this.reusableBuffer = new Uint8Array(Math.min(totalSize, this.maxReusableBufferSize));
+                }
+                
+                // Create complete WAV buffer using cached header and reusable buffer
+                const completeBuffer = this.reusableBuffer.slice(0, totalSize);
+                completeBuffer.set(this.cachedWavHeader!.slice(0, this.wavHeader.headerSize), 0);
+                completeBuffer.set(audioData.subarray(this.wavHeader.headerSize), this.wavHeader.headerSize);
+
+                // Create audio buffer from the chunk
+                this.createAudioBufferFromChunk(completeBuffer);
+                
+                // Remove processed data
+                this.removeProcessedChunks(chunkSize);
+                break; // Process one chunk at a time
+            }
+        } catch (error) {
+            console.error('Error processing buffered chunks:', error);
+        }
+    }
+
+    private async createAudioBufferFromChunk(chunkData: Uint8Array): Promise<void> {
+        try {
+            const arrayBuffer = chunkData.buffer.slice(chunkData.byteOffset, chunkData.byteOffset + chunkData.byteLength);
+            const audioBuffer = await this.audioContext!.decodeAudioData(arrayBuffer);
+            this.bufferQueue.push(audioBuffer);
+            
+            // Schedule buffer if streaming has started
+            if (this.streamingStarted) {
+                this.scheduleNextBuffer();
+            }
+        } catch (error) {
+            console.error('Error creating audio buffer from chunk:', error);
+        }
+    }
+
+    private scheduleNextBuffer(): void {
+        if (this.bufferQueue.length === 0 || !this.streamingStarted) return;
+
+        const buffer = this.bufferQueue.shift()!;
+        const source = this.audioContext!.createBufferSource();
+        source.buffer = buffer;
+        source.connect(this.gainNode!);
+        
+        source.onended = () => {
+            if (this.bufferQueue.length > 0) {
+                this.scheduleNextBuffer();
+            } else if (!this.isPlaying) {
+                this.onEndCallback?.();
+            }
+        };
+
+        source.start(this.nextStartTime);
+        this.nextStartTime += buffer.duration;
+        this.currentStreamSource = source;
+    }
+
+
+    private removeProcessedChunks(processedSize: number): void {
+        let remaining = processedSize;
+        
+        while (remaining > 0 && this.bufferChunks.length > 0) {
+            const chunk = this.bufferChunks[0];
+            if (chunk.length <= remaining) {
+                remaining -= chunk.length;
+                this.currentSize -= chunk.length;
+                this.bufferChunks.shift();
+            } else {
+                // Partial chunk removal
+                const newChunk = chunk.slice(remaining);
+                this.bufferChunks[0] = newChunk;
+                this.currentSize -= remaining;
+                remaining = 0;
+            }
+        }
+    }
+
     unload(): AudioResult {
         try {
             this.stop();
@@ -294,6 +493,21 @@ class AudioPlayer {
             this.currentSize = 0;
             this.expectedSize = 0;
             
+            // Clean up streaming state
+            this.isStreamingMode = false;
+            this.wavHeader = null;
+            this.bufferQueue = [];
+            this.streamingStarted = false;
+            this.nextStartTime = 0;
+            if (this.currentStreamSource) {
+                this.currentStreamSource.stop();
+                this.currentStreamSource = null;
+            }
+            
+            // Clean up cached buffers
+            this.cachedWavHeader = null;
+            this.reusableBuffer = null;
+            
             return { success: true };
         } catch (error) {
             return { success: false, error: (error as Error).message };
@@ -311,6 +525,13 @@ class AudioPlayer {
         this.gainNode = null;
         this.bufferChunks = [];
         this.currentSize = 0;
+        
+        // Clean up streaming state
+        this.bufferQueue = [];
+        this.wavHeader = null;
+        this.currentStreamSource = null;
+        this.cachedWavHeader = null;
+        this.reusableBuffer = null;
     }
 }
 
@@ -361,6 +582,30 @@ const DeepDrftAudio = {
         return await player.finalizeAudioBuffer();
     },
 
+    // Streaming methods
+    initializeStreaming: (playerId: string): AudioResult => {
+        const player = audioPlayers.get(playerId);
+        if (!player) {
+            return { success: false, error: "Player not found" };
+        }
+        return player.initializeStreaming();
+    },
+
+    processStreamingChunk: (playerId: string, audioChunk: Uint8Array): StreamingResult => {
+        const player = audioPlayers.get(playerId);
+        if (!player) {
+            return { success: false, error: "Player not found" };
+        }
+        return player.processStreamingChunk(audioChunk);
+    },
+
+    startStreamingPlayback: (playerId: string): AudioResult => {
+        const player = audioPlayers.get(playerId);
+        if (!player) {
+            return { success: false, error: "Player not found" };
+        }
+        return player.startStreamingPlayback();
+    },
 
     play: (playerId: string): AudioResult => {
         const player = audioPlayers.get(playerId);