diff --git a/DeepDrftPublic/Interop/audio/AudioPlayer.ts b/DeepDrftPublic/Interop/audio/AudioPlayer.ts index 8e7e183..be36e90 100644 --- a/DeepDrftPublic/Interop/audio/AudioPlayer.ts +++ b/DeepDrftPublic/Interop/audio/AudioPlayer.ts @@ -14,6 +14,8 @@ import { IFormatDecoder } from './IFormatDecoder.js'; import { WavFormatDecoder } from './WavFormatDecoder.js'; import { Mp3FormatDecoder } from './Mp3FormatDecoder.js'; import { FlacFormatDecoder } from './FlacFormatDecoder.js'; +import { OpusFormatDecoder } from './OpusFormatDecoder.js'; +import { OpusSeekData, parseSidecar } from './OpusSidecar.js'; export interface AudioResult { success: boolean; @@ -62,6 +64,11 @@ export class AudioPlayer { private onEndCallback: EndCallback | null = null; private progressInterval: number | null = null; + // Pending Opus sidecar (setup header + seek index), parsed from the one-time sidecar fetch and + // applied to the OpusFormatDecoder when the next Opus stream initializes. Wave 18.5 sets this + // (via setOpusSidecar) before initializeStreaming; this class never fetches it. + private pendingOpusSidecar: OpusSeekData | null = null; + constructor() { this.contextManager = new AudioContextManager(); this.streamDecoder = new StreamDecoder(this.contextManager); @@ -103,7 +110,7 @@ export class AudioPlayer { // Initialize new stream with the format decoder selected from Content-Type. this.isStreamingMode = true; - const formatDecoder = AudioPlayer.createFormatDecoder(contentType); + const formatDecoder = this.createFormatDecoder(contentType); this.streamDecoder.initialize(totalStreamLength, formatDecoder); return { success: true }; } catch (error) { @@ -112,15 +119,40 @@ export class AudioPlayer { } /** - * Select a format decoder from the response Content-Type. + * Inject the Opus sidecar (setup header + seek index) for the next Opus stream. Wave 18.5 calls + * this with the raw sidecar bytes (from its one-time HTTP fetch) BEFORE initializeStreaming; the + * parsed result is applied to the OpusFormatDecoder when the stream initializes. This is the + * injection seam — the player owns no transport, only the parse + hand-off. + * + * @returns success:false with an error if the bytes are not a valid sidecar blob. */ - private static createFormatDecoder(contentType: string): IFormatDecoder { + setOpusSidecar(sidecarBytes: Uint8Array): AudioResult { + const parsed = parseSidecar(sidecarBytes); + if (!parsed) { + return { success: false, error: 'Invalid Opus sidecar blob' }; + } + this.pendingOpusSidecar = parsed; + return { success: true }; + } + + /** + * Select a format decoder from the response Content-Type. For Opus, applies the pending sidecar + * (if 18.5 has set one) so the decoder has its setup bytes + seek index before stream init. + */ + private createFormatDecoder(contentType: string): IFormatDecoder { if (contentType.includes('audio/mpeg') || contentType.includes('audio/mp3')) { return new Mp3FormatDecoder(); } if (contentType.includes('audio/flac') || contentType.includes('audio/x-flac')) { return new FlacFormatDecoder(); } + if (contentType.includes('audio/ogg') || contentType.includes('audio/opus')) { + const decoder = new OpusFormatDecoder(); + if (this.pendingOpusSidecar) { + decoder.setSidecar(this.pendingOpusSidecar); + } + return decoder; + } return new WavFormatDecoder(); // default (audio/wav, unknown) } diff --git a/DeepDrftPublic/Interop/audio/IFormatDecoder.ts b/DeepDrftPublic/Interop/audio/IFormatDecoder.ts index bd9fe1e..0964327 100644 --- a/DeepDrftPublic/Interop/audio/IFormatDecoder.ts +++ b/DeepDrftPublic/Interop/audio/IFormatDecoder.ts @@ -1,3 +1,5 @@ +import { OpusSeekData } from './OpusSidecar.js'; + /** * FormatInfo: parsed header data needed to stream and seek an audio file. * Populated by IFormatDecoder.tryParseHeader; used by StreamDecoder throughout playback. @@ -36,8 +38,10 @@ export interface FormatInfo { * MP3 VBR: Xing/VBRI TOC (100-entry Uint8Array, values are file-percentage * 255). * FLAC: SeekTable (array of {sampleNumber: number, streamOffset: number} — stream_offset * is bytes from the start of audio frames, i.e. after all metadata blocks). + * Opus: OpusSeekData — the precomputed granule->byte index + OpusHead/OpusTags setup bytes, + * parsed from the sidecar artifact (NOT byteRate math; see OpusFormatDecoder). */ - seekData?: Mp3VbrSeekData | FlacSeekData | null; + seekData?: Mp3VbrSeekData | FlacSeekData | OpusSeekData | null; } export interface Mp3VbrSeekData { diff --git a/DeepDrftPublic/Interop/audio/OpusCapability.ts b/DeepDrftPublic/Interop/audio/OpusCapability.ts new file mode 100644 index 0000000..d6b8719 --- /dev/null +++ b/DeepDrftPublic/Interop/audio/OpusCapability.ts @@ -0,0 +1,95 @@ +/** + * OpusCapability - runtime detection of Ogg-Opus decode support. + * + * The bespoke graph decodes segments via `AudioContext.decodeAudioData`. Ogg-Opus support there + * is long-standing in Chrome and Firefox but arrived in Safari only at 18.4 (macOS 15.4 / iOS 18.4, + * March 2025); older Safari decodes Opus only in a CAF container, not Ogg. iOS Safari is a primary + * music-listening surface, so a browser that cannot decode Ogg Opus must fall back to the lossless + * WAV path (§3.4 / OQ2). + * + * This module is the detection *seam* only — it answers "can this browser decode Ogg Opus?". The + * player (waves 18.5 / 18.6) consumes the answer to choose the delivery format; this module never + * touches the player or the stream request. + * + * Detection is a genuine probe: a tiny in-memory Ogg-Opus blob is handed to `decodeAudioData`. A + * UA/version gate was rejected because Safari's Opus story is version-specific and UA strings lie; + * a real decode attempt is authoritative. The result is cached after the first probe (capability + * does not change within a session). + */ + +/** + * A minimal, valid Ogg-Opus file: an OpusHead page, an OpusTags page, and one audio page carrying a + * single 20 ms silence packet (mono, 48 kHz). Base64-encoded; ~250 bytes decoded. This is the + * smallest blob a conformant decoder will accept and a non-supporting decoder will reject, which is + * exactly the discriminator we need. + */ +const PROBE_OGG_OPUS_BASE64 = + 'T2dnUwACAAAAAAAAAACRYwAAAAAAANieBHsBE09wdXNIZWFkAQEAAIC7AAAAAABPZ2dTAAAAAAAA' + + 'AAAAAJFjAAABAAAAUkOcUAEMT3B1c1RhZ3MAAAAAAAAAAE9nZ1MABABAAQAAAAAAkWMAAAIAAABU' + + '/9D/A2P4//////////////////////////////////////////////////////////////////8='; + +let cachedSupport: Promise | null = null; + +/** + * Resolve whether this browser can decode Ogg Opus via `decodeAudioData`. Cached after the first + * call. Never rejects — a probe failure resolves to `false` (treat as unsupported, fall back to + * lossless). Pass an existing `AudioContext`/`OfflineAudioContext` to avoid allocating one; if none + * is given, a short-lived `OfflineAudioContext` is created and torn down. + */ +export function canDecodeOggOpus(context?: BaseAudioContext): Promise { + if (cachedSupport === null) { + cachedSupport = probe(context); + } + return cachedSupport; +} + +async function probe(context?: BaseAudioContext): Promise { + let ctx = context; + let ownsContext = false; + + try { + if (!ctx) { + const OfflineCtor = + (globalThis as { OfflineAudioContext?: typeof OfflineAudioContext }).OfflineAudioContext ?? + (globalThis as { webkitOfflineAudioContext?: typeof OfflineAudioContext }).webkitOfflineAudioContext; + if (!OfflineCtor) return false; + // 1 channel, 1 frame, 48 kHz — the smallest legal context; we never render it. + ctx = new OfflineCtor(1, 1, OPUS_PROBE_SAMPLE_RATE); + ownsContext = true; + } + + const buffer = base64ToArrayBuffer(PROBE_OGG_OPUS_BASE64); + // decodeAudioData detaches the buffer; the probe blob is single-use, so that is fine. + await decode(ctx, buffer); + return true; + } catch { + // DOMException (unsupported / corrupt) or any allocation failure -> unsupported. + return false; + } finally { + // OfflineAudioContext has no close() in all engines; guard it. + if (ownsContext && ctx && 'close' in ctx && typeof (ctx as AudioContext).close === 'function') { + try { await (ctx as AudioContext).close(); } catch { /* best-effort teardown */ } + } + } +} + +const OPUS_PROBE_SAMPLE_RATE = 48000; + +/** Promisify decodeAudioData; older Safari only supports the callback form. */ +function decode(ctx: BaseAudioContext, buffer: ArrayBuffer): Promise { + return new Promise((resolve, reject) => { + const result = ctx.decodeAudioData(buffer, resolve, reject); + // Modern engines return a Promise; bridge it so a rejection isn't dropped. + if (result && typeof (result as Promise).then === 'function') { + (result as Promise).then(resolve, reject); + } + }); +} + +function base64ToArrayBuffer(b64: string): ArrayBuffer { + const binary = atob(b64); + const buffer = new ArrayBuffer(binary.length); + const out = new Uint8Array(buffer); + for (let i = 0; i < binary.length; i++) out[i] = binary.charCodeAt(i); + return buffer; +} diff --git a/DeepDrftPublic/Interop/audio/OpusFormatDecoder.test.ts b/DeepDrftPublic/Interop/audio/OpusFormatDecoder.test.ts new file mode 100644 index 0000000..67843f3 --- /dev/null +++ b/DeepDrftPublic/Interop/audio/OpusFormatDecoder.test.ts @@ -0,0 +1,261 @@ +/** + * OpusFormatDecoder / OpusSidecar tests. + * + * There is no TS test runner configured in this repo (no package.json, no jest/vitest, no other + * *.test.ts). Rather than introduce a heavy harness, this file is a self-contained, zero-dependency + * test: a ~15-line inline assert/test harness, no `node:` imports, no DOM. It exercises the pure + * parser / resolver / alignment logic (none of which touches the DOM or Web Audio). + * + * It is EXCLUDED from the production tsc build (tsconfig `exclude: Interop/**\/*.test.ts`) so it + * never ships in wwwroot/js. To run it (Node 22+ strips TS types natively — no tsc, no deps), the + * test's `.js` import specifiers must resolve to the COMPILED decoder modules, so run a copy from + * the compiled output directory: + * + * # 1. produce the compiled decoder modules (the normal build already does this): + * dotnet build DeepDrftPublic/DeepDrftPublic.csproj + * # 2. run this test next to the compiled .js siblings (Node strips the types at load): + * cp DeepDrftPublic/Interop/audio/OpusFormatDecoder.test.ts DeepDrftPublic/wwwroot/js/audio/ + * node DeepDrftPublic/wwwroot/js/audio/OpusFormatDecoder.test.ts + * + * A thrown error / non-zero exit signals failure; "ALL TESTS PASSED" signals success. (The + * copied file lives only in the gitignored wwwroot/js output; the source under Interop is the + * committed artifact.) + * + * The sidecar bytes built in `makeSidecar` reproduce the C# wire format byte-for-byte + * (DeepDrftContent.Processors.Opus.OpusSidecar.ToBytes / OggOpusSeekIndex.ToBytes): + * [uint32 setupHeaderLength][setup bytes] + * [uint64 totalByteLength][double totalDuration][uint32 count][uint16 preSkip][uint16 reserved] + * count x [uint64 granulePosition][uint64 byteOffset] — all little-endian. + * The C# serializer is the source of truth; this verifies the TS parser is its exact counterpart. + */ + +import { parseSidecar, presentationTimeSeconds, OPUS_SAMPLE_RATE } from './OpusSidecar.js'; +import type { OpusSeekData } from './OpusSidecar.js'; +import { OpusFormatDecoder } from './OpusFormatDecoder.js'; +import type { FormatInfo } from './IFormatDecoder.js'; + +// --- tiny inline harness (no dependencies) --------------------------------------------------- +let passed = 0; +const failures: string[] = []; +function test(name: string, fn: () => void): void { + try { + fn(); + passed++; + } catch (e) { + failures.push(`FAIL: ${name}\n ${(e as Error).message}`); + } +} +function assertEqual(actual: unknown, expected: unknown, msg?: string): void { + if (actual !== expected) { + throw new Error(`${msg ?? 'assertEqual'}: expected ${String(expected)}, got ${String(actual)}`); + } +} +function assertArray(actual: ArrayLike, expected: number[], msg?: string): void { + const a = Array.from(actual); + if (a.length !== expected.length || a.some((v, i) => v !== expected[i])) { + throw new Error(`${msg ?? 'assertArray'}: expected [${expected}], got [${a}]`); + } +} +function assertNull(actual: unknown, msg?: string): void { + if (actual !== null) throw new Error(`${msg ?? 'assertNull'}: expected null, got ${String(actual)}`); +} +function assertNotNull(actual: T | null, msg?: string): T { + if (actual === null) throw new Error(`${msg ?? 'assertNotNull'}: got null`); + return actual; +} + +interface SidecarSpec { + setupHeader: number[]; + totalByteLength: number; + totalDuration: number; + preSkip: number; + points: Array<{ granule: number; byteOffset: number }>; +} + +/** Serialize a sidecar blob exactly as the C# OpusSidecar/OggOpusSeekIndex writers do. */ +function makeSidecar(spec: SidecarSpec): Uint8Array { + const SEEK_INDEX_HEADER_SIZE = 24; + const SEEK_POINT_SIZE = 16; + const setupLen = spec.setupHeader.length; + const total = 4 + setupLen + SEEK_INDEX_HEADER_SIZE + spec.points.length * SEEK_POINT_SIZE; + + const bytes = new Uint8Array(total); + const view = new DataView(bytes.buffer); + + view.setUint32(0, setupLen, true); + bytes.set(spec.setupHeader, 4); + + let p = 4 + setupLen; + writeUint64(view, p, spec.totalByteLength); + view.setFloat64(p + 8, spec.totalDuration, true); + view.setUint32(p + 16, spec.points.length, true); + view.setUint16(p + 20, spec.preSkip, true); + // bytes 22-23 reserved (zero) + + p += SEEK_INDEX_HEADER_SIZE; + for (const pt of spec.points) { + writeUint64(view, p, pt.granule); + writeUint64(view, p + 8, pt.byteOffset); + p += SEEK_POINT_SIZE; + } + return bytes; +} + +function writeUint64(view: DataView, offset: number, value: number): void { + view.setUint32(offset, value >>> 0, true); + view.setUint32(offset + 4, Math.floor(value / 0x100000000), true); +} + +function formatInfoFor(sidecar: Uint8Array): FormatInfo { + const decoder = new OpusFormatDecoder(); + const parsed = assertNotNull(parseSidecar(sidecar), 'sidecar should parse'); + decoder.setSidecar(parsed); + return assertNotNull(decoder.tryParseHeader([], 0), 'tryParseHeader should build FormatInfo'); +} + +// --- parseSidecar: byte-for-byte round-trip against the C# layout ----------------------------- + +test('parseSidecar round-trips the C# binary layout exactly', () => { + const setup = [0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64]; // "OpusHead" stand-in + const spec: SidecarSpec = { + setupHeader: setup, + totalByteLength: 1_234_567, + totalDuration: 212.5, + preSkip: 312, + points: [ + { granule: 312, byteOffset: 4096 }, // first point: granule == preSkip -> t=0 + { granule: 312 + 24000, byteOffset: 9000 }, // +0.5 s + { granule: 312 + 48000, byteOffset: 14000 }, // +1.0 s + ], + }; + + const parsed: OpusSeekData = assertNotNull(parseSidecar(makeSidecar(spec))); + assertEqual(parsed.kind, 'opus-sidecar', 'kind'); + assertArray(parsed.setupHeaderBytes, setup, 'setup header bytes'); + assertEqual(parsed.totalByteLength, spec.totalByteLength, 'totalByteLength'); + assertEqual(parsed.totalDurationSeconds, spec.totalDuration, 'totalDuration'); + assertEqual(parsed.preSkip, spec.preSkip, 'preSkip'); + assertEqual(parsed.points.length, 3, 'point count'); + assertEqual(parsed.points[1].granulePosition, 312 + 24000, 'point[1].granule'); + assertEqual(parsed.points[1].byteOffset, 9000, 'point[1].byteOffset'); +}); + +test('parseSidecar honours a borrowed view byteOffset (sidecar not at buffer start)', () => { + const blob = makeSidecar({ + setupHeader: [1, 2, 3, 4], + totalByteLength: 100, + totalDuration: 1.0, + preSkip: 0, + points: [{ granule: 0, byteOffset: 8 }], + }); + const padded = new Uint8Array(blob.length + 7); + padded.set(blob, 7); + const parsed = assertNotNull(parseSidecar(padded.subarray(7))); + assertArray(parsed.setupHeaderBytes, [1, 2, 3, 4], 'borrowed setup bytes'); + assertEqual(parsed.points[0].byteOffset, 8, 'borrowed point offset'); +}); + +test('parseSidecar returns null on a truncated blob', () => { + const blob = makeSidecar({ + setupHeader: [0], + totalByteLength: 1, + totalDuration: 0, + preSkip: 0, + points: [{ granule: 0, byteOffset: 0 }], + }); + assertNull(parseSidecar(blob.subarray(0, 3)), 'short of length prefix'); + assertNull(parseSidecar(blob.subarray(0, blob.length - 4)), 'declared count overruns'); +}); + +test('presentationTimeSeconds applies preSkip and clamps at zero (RFC 7845)', () => { + assertEqual(presentationTimeSeconds(312, 312), 0, 'granule == preSkip'); + assertEqual(presentationTimeSeconds(0, 312), 0, 'below preSkip clamps'); + assertEqual(presentationTimeSeconds(312 + OPUS_SAMPLE_RATE, 312), 1.0, '+48000 -> 1 s'); +}); + +// --- calculateByteOffset: binary search over the precomputed index (exact, not interpolation) - + +test('calculateByteOffset returns the page-start of the largest entry with time <= t', () => { + const points = [0, 1, 2, 3].map(i => ({ + granule: 1000 + i * (OPUS_SAMPLE_RATE / 2), + byteOffset: 4096 + i * 5000, + })); + const info = formatInfoFor(makeSidecar({ + setupHeader: [9, 9, 9, 9], totalByteLength: 999_999, totalDuration: 1.5, preSkip: 1000, points, + })); + const d = new OpusFormatDecoder(); + assertEqual(d.calculateByteOffset(info, 0.0), 4096, 't=0 -> first point'); + assertEqual(d.calculateByteOffset(info, 0.4), 4096, 'just before bucket 1'); + assertEqual(d.calculateByteOffset(info, 0.5), 9096, 'exactly bucket 1'); + assertEqual(d.calculateByteOffset(info, 0.9), 9096, 'within bucket 1'); + assertEqual(d.calculateByteOffset(info, 1.0), 14096, 'exactly bucket 2'); + assertEqual(d.calculateByteOffset(info, 99), 19096, 'past end -> last point'); +}); + +test('calculateByteOffset never interpolates between points', () => { + const info = formatInfoFor(makeSidecar({ + setupHeader: [0], totalByteLength: 10_000, totalDuration: 1.0, preSkip: 0, + points: [{ granule: 0, byteOffset: 100 }, { granule: OPUS_SAMPLE_RATE, byteOffset: 9000 }], + })); + const d = new OpusFormatDecoder(); + assertEqual(d.calculateByteOffset(info, 0.5), 100, 'midpoint snaps to lower page start'); +}); + +test('calculateByteOffset degrades to audioDataOffset with an empty index', () => { + const info = formatInfoFor(makeSidecar({ + setupHeader: [1, 2, 3, 4, 5], totalByteLength: 0, totalDuration: 0, preSkip: 0, points: [], + })); + const d = new OpusFormatDecoder(); + assertEqual(info.audioDataOffset, 5, 'audioDataOffset == setup header length'); + assertEqual(d.calculateByteOffset(info, 10), info.audioDataOffset, 'empty index degrades'); +}); + +// --- getAlignedSegmentSize: Ogg "OggS" page-boundary alignment -------------------------------- + +function withOggS(len: number, ...pageStarts: number[]): Uint8Array { + const out = new Uint8Array(len).fill(0xaa); + for (const s of pageStarts) { out[s] = 0x4f; out[s + 1] = 0x67; out[s + 2] = 0x67; out[s + 3] = 0x53; } + return out; +} +const stubInfo = { audioDataOffset: 0 } as FormatInfo; + +test('getAlignedSegmentSize cuts at the last OggS page start within the window', () => { + const raw = withOggS(64, 4, 40); + assertEqual(new OpusFormatDecoder().getAlignedSegmentSize(stubInfo, 64, 64, false, raw), 40, 'last page start'); +}); + +test('getAlignedSegmentSize waits (returns 0) when no page boundary is found mid-stream', () => { + const raw = withOggS(64); + assertEqual(new OpusFormatDecoder().getAlignedSegmentSize(stubInfo, 64, 64, false, raw), 0, 'no boundary'); +}); + +test('getAlignedSegmentSize flushes the whole candidate on stream completion without a boundary', () => { + const raw = withOggS(64); + assertEqual(new OpusFormatDecoder().getAlignedSegmentSize(stubInfo, 64, 64, true, raw), 64, 'flush on complete'); +}); + +test('getAlignedSegmentSize ignores a page start at offset 0 (needs a real cut point)', () => { + const raw = withOggS(64, 0); + assertEqual(new OpusFormatDecoder().getAlignedSegmentSize(stubInfo, 64, 64, false, raw), 0, 'offset 0 skipped'); +}); + +// --- wrapSegment: OpusHead/OpusTags setup-header carry ---------------------------------------- + +test('wrapSegment prepends the cached setup bytes to a page run', () => { + const setup = [0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64]; // "OpusHead" + const info = formatInfoFor(makeSidecar({ + setupHeader: setup, totalByteLength: 100, totalDuration: 1, preSkip: 0, + points: [{ granule: 0, byteOffset: setup.length }], + })); + const pageRun = new Uint8Array([0x4f, 0x67, 0x67, 0x53, 0x11, 0x22]); // "OggS" + payload + const wrapped = new OpusFormatDecoder().wrapSegment(info, pageRun); + assertArray(wrapped.subarray(0, setup.length), setup, 'setup header first'); + assertArray(wrapped.subarray(setup.length), [0x4f, 0x67, 0x67, 0x53, 0x11, 0x22], 'page run follows'); +}); + +// --- report ---------------------------------------------------------------------------------- +if (failures.length > 0) { + console.error(failures.join('\n')); + throw new Error(`${failures.length} test(s) failed, ${passed} passed`); +} +console.log(`ALL ${passed} TESTS PASSED`); diff --git a/DeepDrftPublic/Interop/audio/OpusFormatDecoder.ts b/DeepDrftPublic/Interop/audio/OpusFormatDecoder.ts new file mode 100644 index 0000000..f89709a --- /dev/null +++ b/DeepDrftPublic/Interop/audio/OpusFormatDecoder.ts @@ -0,0 +1,169 @@ +/** + * OpusFormatDecoder - Ogg-Opus implementation of IFormatDecoder. + * + * Ogg Opus is a containerized, paged format — NOT raw-frame-sliceable the way WAV PCM is. Two + * things make a mid-stream byte slice decodable: (1) it must begin on an Ogg page boundary, and + * (2) the OpusHead/OpusTags setup pages must be prepended (analogous to FLAC's STREAMINFO carry). + * This decoder owns both, plus VBR-safe accurate seeking. + * + * Where the metadata comes from is the genuinely new part. WAV/MP3/FLAC parse everything out of + * the byte stream. Opus is VBR and container-paged, so a byteRate seek would be inaccurate; instead + * the seek transfer function (granule->byte) and the setup bytes are precomputed at transcode time + * (wave 18.1) and delivered as a one-time sidecar fetch (wave 18.5). The injection seam is + * `setSidecar(OpusSeekData)` — call it with the parsed sidecar BEFORE the stream is initialized so + * `tryParseHeader` can build FormatInfo from it. Without a sidecar the decoder cannot stream Opus + * (returns null from tryParseHeader); 18.5 guarantees the fetch precedes stream init. + * + * - getAlignedSegmentSize aligns to Ogg page boundaries by scanning for the "OggS" capture + * pattern (the Ogg analogue of FLAC's frame-sync scan; the interface passes rawData for this). + * - wrapSegment prepends the cached OpusHead/OpusTags setup bytes so any mid-stream page run is + * independently decodable. + * - calculateByteOffset binary-searches the precomputed index for the largest entry with + * presentation-time <= t and returns its exact page-start byte offset — NOT interpolation, + * NOT byteRate math (§3.4a A/C; C5 accurate seek). + */ + +import { FormatInfo, IFormatDecoder } from './IFormatDecoder.js'; +import { OpusSeekData, OPUS_SAMPLE_RATE, presentationTimeSeconds } from './OpusSidecar.js'; + +// "OggS" — every Ogg page begins with this 4-byte capture pattern. +const OGG_CAPTURE = [0x4f, 0x67, 0x67, 0x53]; // 'O' 'g' 'g' 'S' + +export class OpusFormatDecoder implements IFormatDecoder { + // The parsed sidecar: setup bytes + seek index + preSkip + totals. Injected by wave 18.5 via + // setSidecar before stream init. Held for the stream's lifetime (the format does not change + // across a seek/continuation), mirroring how FlacFormatDecoder retains streamInfoBytes. + private sidecar: OpusSeekData | null = null; + + /** + * Inject the parsed sidecar (setup header + seek index) for this stream. Wave 18.5 calls this + * after its one-time sidecar fetch + parseSidecar, before initializeStreaming. This is the seam + * that keeps the HTTP fetch out of the decoder: the decoder is pure and unit-testable against + * synthetic bytes, and 18.5 wires the real transport. + */ + setSidecar(sidecar: OpusSeekData): void { + this.sidecar = sidecar; + } + + tryParseHeader(_chunks: Uint8Array[], _totalSize: number): FormatInfo | null { + // Opus metadata is NOT parsed from the stream — it comes from the injected sidecar. Without + // it we cannot stream Opus; return null so StreamDecoder waits, and 18.5's contract (fetch + + // setSidecar before stream init) prevents that null from persisting. + const sidecar = this.sidecar; + if (!sidecar) return null; + + // For the initial full-file stream the server emits [setup pages][audio pages], and the + // sidecar's setup bytes are exactly those leading pages — so audio data begins right after + // them. This is the file-absolute offset of the first audio page (== the first index point's + // byteOffset by construction). + const audioDataOffset = sidecar.setupHeaderBytes.length; + + return { + // Opus always decodes at 48 kHz regardless of the source rate (RFC 7845). + sampleRate: OPUS_SAMPLE_RATE, + // Channel count is encoded in OpusHead; the decoder reads it from the prepended setup + // bytes at decode time. FormatInfo.channels is display-only here — 2 is the safe nominal. + channels: 2, + bitsPerSample: 16, + byteRate: 0, // VBR + paged; seeking uses the index, never byteRate. + blockAlign: 0, // No fixed alignment; segments align to Ogg page starts via OggS scan. + totalDuration: sidecar.totalDurationSeconds > 0 ? sidecar.totalDurationSeconds : null, + audioDataOffset, + seekData: sidecar + }; + } + + getAlignedSegmentSize( + info: FormatInfo, + availableBytes: number, + requestedSize: number, + streamComplete: boolean, + rawData?: Uint8Array + ): number { + if (availableBytes === 0) return 0; + const candidate = Math.min(requestedSize, availableBytes); + + if (!rawData || rawData.length === 0) { + // No scan data — conservative threshold to avoid tiny unusable segments (mirrors FLAC). + if (!streamComplete && availableBytes < 16384) return 0; + return candidate; + } + + // Scan backward from the candidate boundary for the start of the last Ogg page. Cutting on a + // page start keeps the next segment Ogg-sync-aligned and the current one a whole page run. + const boundary = OpusFormatDecoder.findLastOggPage(rawData, candidate); + if (boundary <= 0) { + if (streamComplete) return candidate; // flush remaining bytes (stream done) + return 0; // wait for more data — no full page boundary yet + } + return boundary; + } + + /** + * Scan backward from `maxBytes` in `rawData` for the start of the last "OggS" capture pattern. + * Returns that byte offset (the page start), or 0 if none is found (caller waits for more data). + * Skips offset 0 itself: a segment that is only "everything up to the very first page" carries + * no page and should wait, matching the FLAC frame-scan's `> 0` discipline. + */ + private static findLastOggPage(rawData: Uint8Array, maxBytes: number): number { + const limit = Math.min(maxBytes, rawData.length); + for (let i = limit - 4; i > 0; i--) { + if (rawData[i] === OGG_CAPTURE[0] && + rawData[i + 1] === OGG_CAPTURE[1] && + rawData[i + 2] === OGG_CAPTURE[2] && + rawData[i + 3] === OGG_CAPTURE[3]) { + return i; + } + } + return 0; + } + + wrapSegment(info: FormatInfo, rawBytes: Uint8Array): Uint8Array { + const sidecar = OpusFormatDecoder.opusSeekData(info); + const setupBytes = sidecar?.setupHeaderBytes; + if (!setupBytes || setupBytes.length === 0) { + // Defensive: without setup bytes a mid-stream page run is undecodable. tryParseHeader + // always populates the sidecar on success, so this path should not occur in practice. + return rawBytes; + } + + // Prepend OpusHead/OpusTags so the page run is self-contained for decodeAudioData. + const result = new Uint8Array(setupBytes.length + rawBytes.length); + result.set(setupBytes, 0); + result.set(rawBytes, setupBytes.length); + return result; + } + + calculateByteOffset(info: FormatInfo, positionSeconds: number): number { + const sidecar = OpusFormatDecoder.opusSeekData(info); + if (!sidecar || sidecar.points.length === 0) { + // No index: degrade to start of audio (seek restarts) — same graceful fallback as FLAC. + return info.audioDataOffset; + } + + const points = sidecar.points; + const preSkip = sidecar.preSkip; + + // Binary search for the largest entry whose presentation time is <= target. Presentation + // time = max(0, (granule - preSkip) / 48000), matching 18.1's RFC 7845 math exactly. + let lo = 0, hi = points.length - 1, best = 0; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const t = presentationTimeSeconds(points[mid].granulePosition, preSkip); + if (t <= positionSeconds) { + best = mid; + lo = mid + 1; + } else { + hi = mid - 1; + } + } + + // byteOffset is already a file-absolute page-start offset in the Opus file — no header math + // to add (unlike FLAC's audio-relative stream_offset). Return it directly. + return points[best].byteOffset; + } + + private static opusSeekData(info: FormatInfo): OpusSeekData | null { + return info.seekData?.kind === 'opus-sidecar' ? info.seekData : null; + } +} diff --git a/DeepDrftPublic/Interop/audio/OpusSidecar.ts b/DeepDrftPublic/Interop/audio/OpusSidecar.ts new file mode 100644 index 0000000..b952b98 --- /dev/null +++ b/DeepDrftPublic/Interop/audio/OpusSidecar.ts @@ -0,0 +1,141 @@ +/** + * OpusSidecar - parser for the per-track Opus seek/setup sidecar artifact. + * + * The sidecar is built once at transcode time (wave 18.1, C# `OpusSidecar` / + * `OggOpusSeekIndex`) and fetched once on track load (wired by wave 18.5). It carries + * everything the client needs to seek a VBR Opus stream accurately and to decode any + * mid-stream slice: + * - the verbatim OpusHead + OpusTags setup pages (prepended to every post-seek slice), + * - the precomputed granule->byte seek index (the exact time->byte transfer function), + * - the pre_skip and totals needed for presentation-time math and seek clamping. + * + * This module is the byte-for-byte counterpart to the C# serializer. It is pure: it parses + * a blob into an `OpusSeekData` accelerator with no I/O. Wave 18.5 owns the HTTP fetch and + * injects the parsed result into `OpusFormatDecoder.setSidecar`. + * + * Binary layout (all little-endian), matching DeepDrftContent.Processors.Opus: + * [uint32 setupHeaderLength] + * [setupHeaderLength bytes -> OpusHead + OpusTags pages] + * [seek-index blob]: + * header (24 bytes): + * uint64 totalByteLength + * double totalDurationSeconds (pre-skip-corrected) + * uint32 pointCount + * uint16 preSkip + * uint16 reserved + * pointCount x 16-byte points: + * uint64 granulePosition (48 kHz sample count) + * uint64 byteOffset (page-start offset in the Opus file) + */ + +/** Opus granule positions are always 48 kHz sample counts, regardless of input rate. */ +export const OPUS_SAMPLE_RATE = 48000; + +/** Size of the seek-index blob header: totalBytes(8) + duration(8) + count(4) + preSkip(2) + reserved(2). */ +const SEEK_INDEX_HEADER_SIZE = 24; +/** Size of one serialized seek point: granulepos(8) + byteOffset(8). */ +const SEEK_POINT_SIZE = 16; + +/** One (granule, byteOffset) seek-index entry. Both are page-start-accurate. */ +export interface OpusSeekPoint { + /** Page end granule position — a 48 kHz sample count. */ + granulePosition: number; + /** Byte offset of the page start in the Opus file. */ + byteOffset: number; +} + +/** + * Parsed sidecar: the `seekData` accelerator the `OpusFormatDecoder` holds for the stream's + * lifetime. Holds the setup bytes (for `wrapSegment` carry) and the index (for `calculateByteOffset`). + */ +export interface OpusSeekData { + kind: 'opus-sidecar'; + /** Verbatim OpusHead + OpusTags pages, prepended to every decodable segment. */ + setupHeaderBytes: Uint8Array; + /** Ordered (granule, byteOffset) entries, ascending by granule. */ + points: OpusSeekPoint[]; + /** Pre-skip-corrected total stream duration in seconds. */ + totalDurationSeconds: number; + /** Total Opus file byte length, for clamping a seek past the end. */ + totalByteLength: number; + /** pre_skip from OpusHead (RFC 7845 §5.1); samples to discard before presentation. */ + preSkip: number; +} + +/** + * Parse a sidecar blob produced by the C# `OpusSidecar.ToBytes`. Returns null on any structural + * inconsistency (short blob, length prefix overrun, declared point count that does not fit) — + * the format is exact, so a malformed blob is corruption, not a recoverable shape. + * + * Accepts a `Uint8Array`, an `ArrayBuffer`, or a typed-array view; copies nothing it can borrow. + */ +export function parseSidecar(input: Uint8Array | ArrayBuffer | ArrayBufferView): OpusSeekData | null { + const bytes = toUint8Array(input); + // DataView over the same backing buffer; honour the view's byteOffset so a borrowed view parses. + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + + if (bytes.byteLength < 4) return null; + + const setupLength = view.getUint32(0, true); + const indexStart = 4 + setupLength; + // Need the setup region plus at least the index header. + if (bytes.byteLength < indexStart + SEEK_INDEX_HEADER_SIZE) return null; + + // subarray is zero-copy; setup bytes are retained for wrapSegment for the stream's lifetime. + const setupHeaderBytes = bytes.subarray(4, indexStart); + + // Seek-index blob header (relative to the DataView, which is bytes-relative). + const totalByteLength = readUint64(view, indexStart); + const totalDurationSeconds = view.getFloat64(indexStart + 8, true); + const pointCount = view.getUint32(indexStart + 16, true); + const preSkip = view.getUint16(indexStart + 20, true); + // bytes 22-23: reserved — ignored on read, for forward-compatibility (matches C#). + + const pointsStart = indexStart + SEEK_INDEX_HEADER_SIZE; + const expectedEnd = pointsStart + pointCount * SEEK_POINT_SIZE; + if (bytes.byteLength < expectedEnd) return null; + + const points: OpusSeekPoint[] = new Array(pointCount); + let cursor = pointsStart; + for (let i = 0; i < pointCount; i++) { + const granulePosition = readUint64(view, cursor); + const byteOffset = readUint64(view, cursor + 8); + points[i] = { granulePosition, byteOffset }; + cursor += SEEK_POINT_SIZE; + } + + return { + kind: 'opus-sidecar', + setupHeaderBytes, + points, + totalDurationSeconds, + totalByteLength, + preSkip + }; +} + +/** + * Pre-skip-corrected presentation time for a granule position: max(0, (granule - preSkip) / 48000). + * Matches the C# `OggOpusSeekIndex.PresentationTimeSeconds` so client and server agree on the + * seek transfer function. + */ +export function presentationTimeSeconds(granulePosition: number, preSkip: number): number { + return Math.max(0, (granulePosition - preSkip) / OPUS_SAMPLE_RATE); +} + +function toUint8Array(input: Uint8Array | ArrayBuffer | ArrayBufferView): Uint8Array { + if (input instanceof Uint8Array) return input; + if (input instanceof ArrayBuffer) return new Uint8Array(input); + return new Uint8Array(input.buffer, input.byteOffset, input.byteLength); +} + +/** + * Read a little-endian uint64 as a JS number. Opus byte offsets and granule positions are exact + * to 2^53 (~8 PB / ~5,700 years of audio at 48 kHz), far beyond any real file — no BigInt needed, + * matching the FLAC seektable's same 2^53 assumption. + */ +function readUint64(view: DataView, offset: number): number { + const lo = view.getUint32(offset, true); + const hi = view.getUint32(offset + 4, true); + return hi * 0x100000000 + lo; +} diff --git a/DeepDrftPublic/Interop/audio/index.ts b/DeepDrftPublic/Interop/audio/index.ts index 3239f24..c56a134 100644 --- a/DeepDrftPublic/Interop/audio/index.ts +++ b/DeepDrftPublic/Interop/audio/index.ts @@ -3,6 +3,7 @@ */ import { AudioPlayer, AudioResult, StreamingResult, AudioState } from './AudioPlayer.js'; +import { canDecodeOggOpus } from './OpusCapability.js'; // Player instances by ID const audioPlayers = new Map(); @@ -37,6 +38,20 @@ const DeepDrftAudio = { return player.initializeStreaming(totalStreamLength, contentType); }, + // Opus injection seam (wave 18.4). Wave 18.5 fetches the per-track sidecar (setup header + + // seek index) over HTTP and hands the raw bytes here BEFORE initializeStreaming on an Opus + // stream. This module never fetches the sidecar — it only parses + stores it on the player. + setOpusSidecar: (playerId: string, sidecarBytes: Uint8Array): AudioResult => { + const player = audioPlayers.get(playerId); + if (!player) return { success: false, error: 'Player not found' }; + return player.setOpusSidecar(sidecarBytes); + }, + + // Capability seam (wave 18.4). Resolves whether this browser can decode Ogg Opus via + // decodeAudioData (Safari < 18.4 cannot). Wave 18.5 / 18.6 consume this to choose lossless + // when unsupported; this module only reports the capability. + canDecodeOggOpus: (): Promise => canDecodeOggOpus(), + processStreamingChunk: async (playerId: string, chunk: Uint8Array): Promise => { const player = audioPlayers.get(playerId); if (!player) return { success: false, error: 'Player not found' }; diff --git a/DeepDrftPublic/tsconfig.json b/DeepDrftPublic/tsconfig.json index 4d11659..4bfd47c 100644 --- a/DeepDrftPublic/tsconfig.json +++ b/DeepDrftPublic/tsconfig.json @@ -21,6 +21,7 @@ "node_modules", "bin/**/*", "obj/**/*", - "publish/**/*" + "publish/**/*", + "Interop/**/*.test.ts" ] } \ No newline at end of file