daniel/deepdrft
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Replace broken per-segment Opus decode with WebCodecs AudioDecoder streaming pipeline

Browse Source
This commit is contained in:
daniel-c-harvey
2026-06-23 17:42:06 -04:00
parent d0118997b6
commit 7f3fb74126
13 changed files with 1270 additions and 649 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DeepDrftPublic/Interop/audio/AudioPlayer.ts
+123 -22
View File
@@ -11,11 +11,12 @@ import { AudioContextManager } from './AudioContextManager.js';
import { StreamDecoder } from './StreamDecoder.js';
import { PlaybackScheduler } from './PlaybackScheduler.js';
import { IFormatDecoder } from './IFormatDecoder.js';
import { IStreamingDecoder } from './IStreamingDecoder.js';
import { WavFormatDecoder } from './WavFormatDecoder.js';
import { Mp3FormatDecoder } from './Mp3FormatDecoder.js';
import { FlacFormatDecoder } from './FlacFormatDecoder.js';
import { OpusFormatDecoder } from './OpusFormatDecoder.js';
import { OpusSeekData, parseSidecar } from './OpusSidecar.js';
import { OpusStreamDecoder } from './OpusStreamDecoder.js';
import { OpusSeekData, parseSidecar, resolveOpusByteOffset } from './OpusSidecar.js';
export interface AudioResult {
success: boolean;
@@ -47,6 +48,15 @@ export class AudioPlayer {
private streamDecoder: StreamDecoder;
private scheduler: PlaybackScheduler;
// The Opus WebCodecs decode path (IStreamingDecoder seam), used INSTEAD of streamDecoder when the
// active stream is Ogg Opus. Null for WAV/MP3/FLAC, which keep the streamDecoder path unchanged.
// Holding both is deliberate: the change is the decode stage only; the same scheduler/Web Audio
// graph feeds from whichever decoder is active for the current stream.
private opusDecoder: IStreamingDecoder | null = null;
// The sidecar in effect for the active Opus stream (its seek index resolves byte offsets). Distinct
// from pendingOpusSidecar, which is the one set for the NEXT stream init.
private activeOpusSidecar: OpusSeekData | null = null;
// Playback state
private isPlaying: boolean = false;
private isPaused: boolean = false;
@@ -106,10 +116,24 @@ export class AudioPlayer {
this.stopProgressTracking();
this.scheduler.clear();
this.streamDecoder.reset();
this.disposeOpusDecoder();
this.resetState();
// Initialize new stream with the format decoder selected from Content-Type.
this.isStreamingMode = true;
// Opus routes to the WebCodecs streaming seam (IStreamingDecoder); WAV/MP3/FLAC keep the
// StreamDecoder wrap-and-decode path byte-for-byte. The sidecar (setup header + seek index)
// must already be set (setOpusSidecar, before init) — without it Opus cannot be decoded or
// seeked, so we fall back by leaving opusDecoder null and using the StreamDecoder path,
// which the server's C2 fallback (lossless bytes) matches. In practice the C# resolver only
// selects Opus when the sidecar parsed, so the null branch is defensive.
if (this.isOpusContentType(contentType) && this.pendingOpusSidecar) {
this.activeOpusSidecar = this.pendingOpusSidecar;
this.opusDecoder = new OpusStreamDecoder(this.contextManager, this.pendingOpusSidecar);
return { success: true };
}
// Non-Opus (or Opus-without-sidecar): the existing StreamDecoder path, unchanged.
const formatDecoder = this.createFormatDecoder(contentType);
this.streamDecoder.initialize(totalStreamLength, formatDecoder);
return { success: true };
@@ -118,6 +142,18 @@ export class AudioPlayer {
}
}
private isOpusContentType(contentType: string): boolean {
return contentType.includes('audio/ogg') || contentType.includes('audio/opus');
}
private disposeOpusDecoder(): void {
if (this.opusDecoder) {
this.opusDecoder.dispose();
this.opusDecoder = null;
}
this.activeOpusSidecar = null;
}
/**
* Inject the Opus sidecar (setup header + seek index) for the next Opus stream. Wave 18.5 calls
* this with the raw sidecar bytes (from its one-time HTTP fetch) BEFORE initializeStreaming; the
@@ -136,8 +172,9 @@ export class AudioPlayer {
}
/**
* Select a format decoder from the response Content-Type. For Opus, applies the pending sidecar
* (if 18.5 has set one) so the decoder has its setup bytes + seek index before stream init.
* Select a format decoder from the response Content-Type for the StreamDecoder (wrap-and-decode)
* path. Opus is NOT handled here — it routes to the WebCodecs IStreamingDecoder seam in
* initializeStreaming. This factory serves WAV/MP3/FLAC only.
*/
private createFormatDecoder(contentType: string): IFormatDecoder {
if (contentType.includes('audio/mpeg') || contentType.includes('audio/mp3')) {
@@ -146,13 +183,6 @@ export class AudioPlayer {
if (contentType.includes('audio/flac') || contentType.includes('audio/x-flac')) {
return new FlacFormatDecoder();
}
if (contentType.includes('audio/ogg') || contentType.includes('audio/opus')) {
const decoder = new OpusFormatDecoder();
if (this.pendingOpusSidecar) {
decoder.setSidecar(this.pendingOpusSidecar);
}
return decoder;
}
return new WavFormatDecoder(); // default (audio/wav, unknown)
}
@@ -165,10 +195,12 @@ export class AudioPlayer {
*/
async markStreamComplete(): Promise<StreamingResult> {
try {
const results = await this.streamDecoder.markStreamComplete();
const results = this.opusDecoder
? await this.opusDecoder.complete()
: (await this.streamDecoder.markStreamComplete()).map(r => r.buffer);
if (results.length > 0) {
for (const result of results) {
this.scheduler.addBuffer(result.buffer);
for (const buffer of results) {
this.scheduler.addBuffer(buffer);
}
if (this.streamingStarted && this.isPlaying) {
this.scheduler.scheduleNewBuffers();
@@ -182,6 +214,53 @@ export class AudioPlayer {
}
async processStreamingChunk(chunk: Uint8Array): Promise<StreamingResult> {
return this.opusDecoder
? this.processOpusChunk(chunk)
: this.processFormatChunk(chunk);
}
/** Opus (WebCodecs) chunk path. Mirrors processFormatChunk's add->schedule->report shape. */
private async processOpusChunk(chunk: Uint8Array): Promise<StreamingResult> {
try {
const decoder = this.opusDecoder!;
const buffers = await decoder.push(chunk);
if (buffers.length > 0) {
for (const buffer of buffers) {
this.scheduler.addBuffer(buffer);
}
// Duration is known up front from the sidecar; set once (a seek must not overwrite it).
if (this.duration === 0 && decoder.totalDuration) {
this.duration = decoder.totalDuration;
}
if (this.streamingStarted && this.isPlaying) {
this.scheduler.scheduleNewBuffers();
}
}
if (decoder.hasFatalError) {
return { success: false, error: 'Opus decode failed' };
}
// "headerParsed" maps to the decoder being configured (codec ready). canStart needs the
// min buffer count, exactly as the WAV path requires before first playback.
const headerParsed = decoder.ready;
const canStart = headerParsed && this.scheduler.hasMinimumBuffers(this.minBuffersForPlayback);
return {
success: true,
canStartStreaming: canStart,
headerParsed,
bufferCount: this.scheduler.getBufferCount(),
duration: this.duration
};
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
/** WAV/MP3/FLAC (StreamDecoder) chunk path — unchanged from before the Opus seam split. */
private async processFormatChunk(chunk: Uint8Array): Promise<StreamingResult> {
try {
const results = await this.streamDecoder.processChunk(chunk);
@@ -310,6 +389,7 @@ export class AudioPlayer {
try {
this.scheduler.clear();
this.streamDecoder.reset();
this.disposeOpusDecoder();
this.resetState();
this.stopProgressTracking();
@@ -371,8 +451,21 @@ export class AudioPlayer {
*/
private seekBeyondBuffer(position: number): AudioResult {
try {
// The header must be parsed for byte-offset math; without it we cannot
// build a valid Range request.
// Opus: resolve the offset from the precomputed seek index (the accurate VBR-safe transfer
// function). The returned offset is a real page start, so the Range continuation lands the
// demuxer/decoder Ogg-sync-aligned.
if (this.opusDecoder) {
if (!this.activeOpusSidecar) {
return { success: false, error: 'Cannot calculate byte offset' };
}
return {
success: true,
seekBeyondBuffer: true,
byteOffset: resolveOpusByteOffset(this.activeOpusSidecar, position)
};
}
// WAV/MP3/FLAC: the header must be parsed for byte-offset math.
if (!this.streamDecoder.getFormatInfo()) {
return { success: false, error: 'Cannot calculate byte offset' };
}
@@ -404,6 +497,11 @@ export class AudioPlayer {
* Calculate byte offset for a time position (for C# layer)
*/
calculateByteOffset(positionSeconds: number): number {
if (this.opusDecoder) {
return this.activeOpusSidecar
? resolveOpusByteOffset(this.activeOpusSidecar, positionSeconds)
: 0;
}
if (!this.streamDecoder.getFormatInfo()) return 0;
return this.streamDecoder.calculateByteOffset(positionSeconds);
}
@@ -416,17 +514,20 @@ export class AudioPlayer {
try {
// Stop current playback
this.stopProgressTracking();
const wasPlaying = this.isPlaying;
this.isPlaying = false;
// Clear buffers and set new offset
this.scheduler.clearForSeek();
this.scheduler.setPlaybackOffset(seekPosition);
// Reinitialize decoder for the Range-continuation stream. totalStreamLength
// here is the 206 Content-Length (range start → EOF), not the full file size —
// the decoder uses it to detect stream-complete against raw audio bytes.
this.streamDecoder.reinitializeForRangeContinuation(totalStreamLength);
// Reinitialize the active decoder for the Range-continuation stream (206 body, no header/
// setup pages). Opus resets demux + codec state (keeping the cached config); the
// StreamDecoder path uses totalStreamLength (the 206 Content-Length) to detect completion.
if (this.opusDecoder) {
this.opusDecoder.reinitializeForRangeContinuation();
} else {
this.streamDecoder.reinitializeForRangeContinuation(totalStreamLength);
}
// Update state
this.pausePosition = seekPosition;