Files
deepdrft/DeepDrftPublic/Interop/audio/AudioPlayer.ts
T
daniel-c-harvey 518479e7ae Phase 21.2: back-pressure to bound the unplayed decoded region
Shared scheduler fill signal (forward water-marks + hard byte cap) pauses
the C# read loop above high-water and, for Opus, stops the demux/decode
feed so WebCodecs queues stay near-empty. Routes through the existing
cancellation discipline; releases the latch on clear/seek.
2026-06-23 23:16:08 -04:00

721 lines
28 KiB
TypeScript

/**
* AudioPlayer - Main orchestrator for audio playback.
*
* Composes specialized managers following Single Responsibility Principle:
* - AudioContextManager: Web Audio API context and routing
* - StreamDecoder: WAV parsing and decoding
* - PlaybackScheduler: Buffer storage and playback scheduling
*/
import { AudioContextManager } from './AudioContextManager.js';
import { StreamDecoder } from './StreamDecoder.js';
import { PlaybackScheduler } from './PlaybackScheduler.js';
import { IFormatDecoder } from './IFormatDecoder.js';
import { IStreamingDecoder } from './IStreamingDecoder.js';
import { WavFormatDecoder } from './WavFormatDecoder.js';
import { Mp3FormatDecoder } from './Mp3FormatDecoder.js';
import { FlacFormatDecoder } from './FlacFormatDecoder.js';
import { OpusStreamDecoder } from './OpusStreamDecoder.js';
import { OpusSeekData, parseSidecar, resolveOpusByteOffset, OpusSeekResolution } from './OpusSidecar.js';
export interface AudioResult {
success: boolean;
error?: string;
seekBeyondBuffer?: boolean;
byteOffset?: number;
}
export interface StreamingResult extends AudioResult {
canStartStreaming?: boolean;
headerParsed?: boolean;
bufferCount?: number;
duration?: number;
// Phase 21.2a back-pressure signal piggybacked on the chunk result the C# read loop already
// awaits — true means the scheduler's forward fill is over the high-water mark and the loop
// should stop calling ReadAsync until it drains (no extra interop hop in the common case).
productionPaused?: boolean;
}
export interface AudioState {
isPlaying: boolean;
isPaused: boolean;
currentTime: number;
duration: number;
volume: number;
}
type ProgressCallback = (currentTime: number) => void;
type EndCallback = () => void;
export class AudioPlayer {
private contextManager: AudioContextManager;
private streamDecoder: StreamDecoder;
private scheduler: PlaybackScheduler;
// The Opus WebCodecs decode path (IStreamingDecoder seam), used INSTEAD of streamDecoder when the
// active stream is Ogg Opus. Null for WAV/MP3/FLAC, which keep the streamDecoder path unchanged.
// Holding both is deliberate: the change is the decode stage only; the same scheduler/Web Audio
// graph feeds from whichever decoder is active for the current stream.
private opusDecoder: IStreamingDecoder | null = null;
// The sidecar in effect for the active Opus stream (its seek index resolves byte offsets). Distinct
// from pendingOpusSidecar, which is the one set for the NEXT stream init.
private activeOpusSidecar: OpusSeekData | null = null;
// The landing time of the most recent seek-beyond-buffer page resolution (seconds). Set by
// seekBeyondBuffer, consumed by reinitializeFromOffset to trim leading decoded frames so the
// audible position matches the requested seek target (AC9 fine re-sync, §3.4a step 4).
private _seekLandingTime: number = 0;
// Playback state
private isPlaying: boolean = false;
private isPaused: boolean = false;
private pausePosition: number = 0;
private duration: number = 0;
// Streaming state
private isStreamingMode: boolean = false;
private streamingStarted: boolean = false;
private streamingCompleted: boolean = false;
private minBuffersForPlayback: number = 6;
// Callbacks
private onProgressCallback: ProgressCallback | null = null;
private onEndCallback: EndCallback | null = null;
private progressInterval: number | null = null;
// Pending Opus sidecar (setup header + seek index), parsed from the one-time sidecar fetch and
// applied to the OpusFormatDecoder when the next Opus stream initializes. Wave 18.5 sets this
// (via setOpusSidecar) before initializeStreaming; this class never fetches it.
private pendingOpusSidecar: OpusSeekData | null = null;
constructor() {
this.contextManager = new AudioContextManager();
this.streamDecoder = new StreamDecoder(this.contextManager);
this.scheduler = new PlaybackScheduler(this.contextManager);
// Wire up scheduler callbacks
this.scheduler.onPlaybackEnded = () => this.handlePlaybackEnded();
}
// ==================== Initialization ====================
async initialize(): Promise<AudioResult> {
try {
await this.contextManager.initialize();
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
async ensureAudioContextReady(): Promise<AudioResult> {
try {
await this.contextManager.ensureReady();
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
// ==================== Streaming ====================
initializeStreaming(totalStreamLength: number, contentType: string): AudioResult {
try {
// Full cleanup before starting new stream
this.stopProgressTracking();
this.scheduler.clear();
this.streamDecoder.reset();
this.disposeOpusDecoder();
this.resetState();
this.isStreamingMode = true;
// Opus routes to the WebCodecs streaming seam (IStreamingDecoder); WAV/MP3/FLAC keep the
// StreamDecoder wrap-and-decode path byte-for-byte. The sidecar (setup header + seek index)
// must already be set (setOpusSidecar, before init) — without it Opus cannot be decoded or
// seeked, so we fall back by leaving opusDecoder null and using the StreamDecoder path,
// which the server's C2 fallback (lossless bytes) matches. In practice the C# resolver only
// selects Opus when the sidecar parsed, so the null branch is defensive.
if (this.isOpusContentType(contentType) && this.pendingOpusSidecar) {
this.activeOpusSidecar = this.pendingOpusSidecar;
// Pass the shared back-pressure signal (21.2b): the Opus decoder stops demuxing/
// decoding new packets while the scheduler is full, so the WebCodecs decode queue
// and decodedQueue do not balloon behind a throttled socket (OQ7). Same signal the
// C# read loop honors — one policy, two thin hooks.
this.opusDecoder = new OpusStreamDecoder(
this.contextManager,
this.pendingOpusSidecar,
() => this.scheduler.isProductionPaused());
return { success: true };
}
// Non-Opus (or Opus-without-sidecar): the existing StreamDecoder path, unchanged.
const formatDecoder = this.createFormatDecoder(contentType);
this.streamDecoder.initialize(totalStreamLength, formatDecoder);
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
private isOpusContentType(contentType: string): boolean {
return contentType.includes('audio/ogg') || contentType.includes('audio/opus');
}
private disposeOpusDecoder(): void {
if (this.opusDecoder) {
this.opusDecoder.dispose();
this.opusDecoder = null;
}
this.activeOpusSidecar = null;
}
/**
* Inject the Opus sidecar (setup header + seek index) for the next Opus stream. Wave 18.5 calls
* this with the raw sidecar bytes (from its one-time HTTP fetch) BEFORE initializeStreaming; the
* parsed result is applied to the OpusFormatDecoder when the stream initializes. This is the
* injection seam — the player owns no transport, only the parse + hand-off.
*
* @returns success:false with an error if the bytes are not a valid sidecar blob.
*/
setOpusSidecar(sidecarBytes: Uint8Array): AudioResult {
const parsed = parseSidecar(sidecarBytes);
if (!parsed) {
return { success: false, error: 'Invalid Opus sidecar blob' };
}
this.pendingOpusSidecar = parsed;
return { success: true };
}
/**
* Select a format decoder from the response Content-Type for the StreamDecoder (wrap-and-decode)
* path. Opus is NOT handled here — it routes to the WebCodecs IStreamingDecoder seam in
* initializeStreaming. This factory serves WAV/MP3/FLAC only.
*/
private createFormatDecoder(contentType: string): IFormatDecoder {
if (contentType.includes('audio/mpeg') || contentType.includes('audio/mp3')) {
return new Mp3FormatDecoder();
}
if (contentType.includes('audio/flac') || contentType.includes('audio/x-flac')) {
return new FlacFormatDecoder();
}
return new WavFormatDecoder(); // default (audio/wav, unknown)
}
/**
* Signal to the decoder that the C# streaming loop has finished sending bytes.
* This sets streamComplete=true and flushes any remaining decoded tail segments.
* Must be called after the ReadAsync loop exits, regardless of whether
* Content-Length was known — without it the tail-decode path is dead when
* Content-Length is absent.
*/
async markStreamComplete(): Promise<StreamingResult> {
try {
const results = this.opusDecoder
? await this.opusDecoder.complete()
: (await this.streamDecoder.markStreamComplete()).map(r => r.buffer);
if (results.length > 0) {
for (const buffer of results) {
this.scheduler.addBuffer(buffer);
}
if (this.streamingStarted && this.isPlaying) {
this.scheduler.scheduleNewBuffers();
}
}
this.streamingCompleted = true;
return { success: true, bufferCount: this.scheduler.getBufferCount() };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
async processStreamingChunk(chunk: Uint8Array): Promise<StreamingResult> {
return this.opusDecoder
? this.processOpusChunk(chunk)
: this.processFormatChunk(chunk);
}
/** Opus (WebCodecs) chunk path. Mirrors processFormatChunk's add->schedule->report shape. */
private async processOpusChunk(chunk: Uint8Array): Promise<StreamingResult> {
try {
const decoder = this.opusDecoder!;
const buffers = await decoder.push(chunk);
// Duration is known up front from the sidecar — surface it as soon as the decoder reports it,
// NOT gated on the first decoded buffers. The C# layer locks Duration on the first chunk whose
// result carries a value (the `Duration == null` guard), and WebCodecs decode is async, so the
// earliest chunks can return zero buffers; gating duration on buffers means C# captures the
// initial 0 and never overwrites it — the WAV header path sets duration on chunk 1 because its
// header parses synchronously, which is the asymmetry this closes. Set once so a seek (which
// reinitialises the decoder) cannot overwrite it.
if (this.duration === 0 && decoder.totalDuration) {
this.duration = decoder.totalDuration;
}
if (buffers.length > 0) {
for (const buffer of buffers) {
this.scheduler.addBuffer(buffer);
}
if (this.streamingStarted && this.isPlaying) {
this.scheduler.scheduleNewBuffers();
}
}
if (decoder.hasFatalError) {
return { success: false, error: 'Opus decode failed' };
}
// "headerParsed" maps to the decoder being configured (codec ready). canStart needs the
// min buffer count, exactly as the WAV path requires before first playback.
const headerParsed = decoder.ready;
const canStart = headerParsed && this.scheduler.hasMinimumBuffers(this.minBuffersForPlayback);
return {
success: true,
canStartStreaming: canStart,
headerParsed,
bufferCount: this.scheduler.getBufferCount(),
duration: this.duration,
productionPaused: this.scheduler.isProductionPaused()
};
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
/** WAV/MP3/FLAC (StreamDecoder) chunk path — unchanged from before the Opus seam split. */
private async processFormatChunk(chunk: Uint8Array): Promise<StreamingResult> {
try {
const results = await this.streamDecoder.processChunk(chunk);
if (results.length > 0) {
for (const result of results) {
this.scheduler.addBuffer(result.buffer);
}
// Update duration estimate — set once only; reinitializeFromOffset does not
// reset this.duration, so after a seek-beyond-buffer the synthesised header's
// shorter DataSize cannot overwrite the original full-track duration.
const estimatedDuration = this.streamDecoder.getEstimatedDuration();
if (estimatedDuration && this.duration === 0) {
this.duration = estimatedDuration;
}
// Schedule new buffers if already playing
if (this.streamingStarted && this.isPlaying) {
this.scheduler.scheduleNewBuffers();
}
}
// Check if streaming is complete
if (this.streamDecoder.isComplete) {
this.streamingCompleted = true;
}
const canStart = this.streamDecoder.headerParsed &&
this.scheduler.hasMinimumBuffers(this.minBuffersForPlayback);
return {
success: true,
canStartStreaming: canStart,
headerParsed: this.streamDecoder.headerParsed,
bufferCount: this.scheduler.getBufferCount(),
duration: this.duration,
productionPaused: this.scheduler.isProductionPaused()
};
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
async startStreamingPlayback(): Promise<AudioResult> {
if (!this.scheduler.hasBuffers()) {
return { success: false, error: 'No buffers available' };
}
try {
// A backgrounded tab leaves AudioContext suspended. createBufferSource/start
// against a suspended context produces no audio without throwing — the same
// failure mode that was fixed for play() (resume path). Awaiting ensureReady()
// here guarantees the context is running before playFromPosition schedules
// any AudioBufferSourceNodes.
await this.contextManager.ensureReady();
this.streamingStarted = true;
this.isPlaying = true;
this.isPaused = false;
this.pausePosition = 0;
this.scheduler.playFromPosition(0);
this.startProgressTracking();
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
// ==================== Playback Control ====================
async play(): Promise<AudioResult> {
if (!this.isStreamingMode) {
return { success: false, error: 'Not in streaming mode' };
}
if (!this.streamingStarted || !this.scheduler.hasBuffers()) {
return { success: false, error: 'Streaming not ready' };
}
// Don't restart if already playing
if (this.isPlaying) {
return { success: true };
}
try {
// Must await: a backgrounded tab leaves AudioContext suspended, and
// createBufferSource/source.start against a suspended context produces
// no audio without throwing. Firing ensureReady() without await meant
// play() returned success but the user heard nothing.
await this.contextManager.ensureReady();
this.isPlaying = true;
this.isPaused = false;
// Resume from pause position. pausePosition is absolute track time;
// playFromPosition expects a buffer-relative position (excludes playbackOffset).
const bufferRelativePosition = this.pausePosition - this.scheduler.getPlaybackOffset();
this.scheduler.playFromPosition(Math.max(0, bufferRelativePosition));
this.startProgressTracking();
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
pause(): AudioResult {
if (!this.isPlaying) {
return { success: false, error: 'Not playing' };
}
try {
this.pausePosition = this.scheduler.pause();
this.isPlaying = false;
this.isPaused = true;
this.stopProgressTracking();
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
stop(): AudioResult {
try {
this.scheduler.clear();
this.streamDecoder.reset();
this.disposeOpusDecoder();
this.resetState();
this.stopProgressTracking();
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
unload(): AudioResult {
return this.stop();
}
seek(position: number): AudioResult {
if (!this.isStreamingMode || position < 0 || position > this.duration) {
return { success: false, error: 'Invalid seek position' };
}
const bufferStart = this.scheduler.getPlaybackOffset();
const bufferEnd = this.scheduler.getTotalDuration() + bufferStart;
// Position must be within [bufferStart, bufferEnd] to use buffered content.
// A lower-bound check is required: after a seek-beyond-buffer, bufferStart is
// set to the prior seek position. Seeking to a position below bufferStart would
// produce a negative bufferRelativePosition in seekWithinBuffer, silently
// clamping to position 0 of the offset buffer instead of the requested time.
if (position >= bufferStart && position <= bufferEnd) {
return this.seekWithinBuffer(position);
} else {
// Seeking outside buffered window - signal C# to fetch new stream
return this.seekBeyondBuffer(position);
}
}
/**
* Seek within currently buffered content
*/
private seekWithinBuffer(position: number): AudioResult {
try {
const wasPlaying = this.isPlaying;
this.scheduler.stopAllSources();
// Adjust position relative to buffer start (subtract playback offset)
const bufferRelativePosition = position - this.scheduler.getPlaybackOffset();
this.pausePosition = position;
if (wasPlaying) {
this.scheduler.playFromPosition(Math.max(0, bufferRelativePosition));
}
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
/**
* Seek beyond buffered content - calculate byte offset for server request
*/
private seekBeyondBuffer(position: number): AudioResult {
try {
// Opus: resolve the offset from the precomputed seek index (the accurate VBR-safe transfer
// function). The returned offset is a real page start, so the Range continuation lands the
// demuxer/decoder Ogg-sync-aligned. Also capture the landing time (t_page ≤ position) so
// reinitializeFromOffset can trim the leading decoded frames and land precisely at `position`
// (AC9 fine re-sync, §3.4a step 4).
if (this.opusDecoder) {
if (!this.activeOpusSidecar) {
return { success: false, error: 'Cannot calculate byte offset' };
}
const resolution: OpusSeekResolution = resolveOpusByteOffset(this.activeOpusSidecar, position);
this._seekLandingTime = resolution.landingTimeSeconds;
return {
success: true,
seekBeyondBuffer: true,
byteOffset: resolution.byteOffset
};
}
// WAV/MP3/FLAC: the header must be parsed for byte-offset math.
if (!this.streamDecoder.getFormatInfo()) {
return { success: false, error: 'Cannot calculate byte offset' };
}
// calculateByteOffset returns a file-absolute offset (byte position from the
// start of the file on disk, header included) — exactly what the Range request
// needs. The format decoder owns the header-offset addition and frame alignment.
const fileOffset = this.streamDecoder.calculateByteOffset(position);
// Signal that C# needs to request a new stream from this file-absolute offset
return {
success: true,
seekBeyondBuffer: true,
byteOffset: fileOffset
};
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
/**
* Get the total buffered duration (for C# to check if seek is within buffer)
*/
getBufferedDuration(): number {
return this.scheduler.getTotalDuration() + this.scheduler.getPlaybackOffset();
}
/**
* The shared back-pressure signal (Phase 21.2a), polled by the C# read loop WHILE it is
* already throttled to learn when the forward fill has drained below the low-water mark and it
* may resume reading. The steady-state (unthrottled) loop never calls this — it reads the
* piggybacked productionPaused flag off each chunk result instead, so there is no extra
* interop hop until back-pressure actually engages.
*/
isProductionPaused(): boolean {
return this.scheduler.isProductionPaused();
}
/**
* Calculate byte offset for a time position (for C# layer)
*/
calculateByteOffset(positionSeconds: number): number {
if (this.opusDecoder) {
return this.activeOpusSidecar
? resolveOpusByteOffset(this.activeOpusSidecar, positionSeconds).byteOffset
: 0;
}
if (!this.streamDecoder.getFormatInfo()) return 0;
return this.streamDecoder.calculateByteOffset(positionSeconds);
}
/**
* Reinitialize for offset streaming after seek-beyond-buffer
* Called by C# after receiving new stream from server
*/
reinitializeFromOffset(totalStreamLength: number, seekPosition: number): AudioResult {
try {
// Stop current playback
this.stopProgressTracking();
this.isPlaying = false;
// Clear buffers and set new offset
this.scheduler.clearForSeek();
this.scheduler.setPlaybackOffset(seekPosition);
// Reinitialize the active decoder for the Range-continuation stream (206 body, no header/
// setup pages). Opus resets demux + codec state (keeping the cached config) and arms the
// lead-trim so decoded audio starts at `seekPosition`, not at the page boundary (AC9). The
// StreamDecoder path uses totalStreamLength (the 206 Content-Length) to detect completion.
if (this.opusDecoder) {
this.opusDecoder.reinitializeForRangeContinuation(this._seekLandingTime, seekPosition);
} else {
this.streamDecoder.reinitializeForRangeContinuation(totalStreamLength);
}
// Update state
this.pausePosition = seekPosition;
this.streamingStarted = false; // Will restart when new buffers arrive
this.streamingCompleted = false;
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
// ==================== Volume ====================
setVolume(volume: number): AudioResult {
try {
this.contextManager.setVolume(volume);
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
// ==================== State ====================
getCurrentTime(): number {
if (this.isPlaying) {
return this.scheduler.getCurrentPosition();
}
return this.pausePosition;
}
getState(): AudioState {
return {
isPlaying: this.isPlaying,
isPaused: this.isPaused,
currentTime: this.getCurrentTime(),
duration: this.duration,
volume: this.contextManager.getVolume()
};
}
// ==================== Callbacks ====================
setOnProgressCallback(callback: ProgressCallback): void {
this.onProgressCallback = callback;
}
setOnEndCallback(callback: EndCallback): void {
this.onEndCallback = callback;
}
// ==================== Spectrum Analysis ====================
getSpectrumData(): number[] {
return this.contextManager.getSpectrumAnalyzer().getFrequencyData();
}
setSpectrumHighPass(freq: number): AudioResult {
try {
this.contextManager.getSpectrumAnalyzer().setHighPass(freq);
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
setSpectrumLowPass(freq: number): AudioResult {
try {
this.contextManager.getSpectrumAnalyzer().setLowPass(freq);
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
setSpectrumSlope(dbPerDecade: number): AudioResult {
try {
this.contextManager.getSpectrumAnalyzer().setSlopeCorrection(dbPerDecade);
return { success: true };
} catch (error) {
return { success: false, error: (error as Error).message };
}
}
startSpectrumAnimation(callbackId: string, callback: (data: number[]) => void): void {
this.contextManager.getSpectrumAnalyzer().addCallback(callbackId, callback);
}
stopSpectrumAnimation(callbackId: string): void {
this.contextManager.getSpectrumAnalyzer().removeCallback(callbackId);
}
getLevelDb(): number {
return this.contextManager.getSpectrumAnalyzer().getLevelDb();
}
startLevelAnimation(callbackId: string, callback: (db: number) => void): void {
this.contextManager.getSpectrumAnalyzer().addLevelCallback(callbackId, callback);
}
stopLevelAnimation(callbackId: string): void {
this.contextManager.getSpectrumAnalyzer().removeLevelCallback(callbackId);
}
// ==================== Private Methods ====================
private resetState(): void {
this.isPlaying = false;
this.isPaused = false;
this.pausePosition = 0;
this.duration = 0;
this.isStreamingMode = false;
this.streamingStarted = false;
this.streamingCompleted = false;
this._seekLandingTime = 0;
}
private handlePlaybackEnded(): void {
this.isPlaying = false;
this.isPaused = false;
this.pausePosition = 0;
this.stopProgressTracking();
this.onEndCallback?.();
}
private startProgressTracking(): void {
this.stopProgressTracking();
this.progressInterval = window.setInterval(() => {
if (this.onProgressCallback && this.isPlaying) {
this.onProgressCallback(this.getCurrentTime());
}
}, 100);
}
private stopProgressTracking(): void {
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
}
// ==================== Cleanup ====================
dispose(): void {
this.stop();
this.stopProgressTracking();
this.contextManager.dispose();
}
}