From b451dda79e1b3fd965f5370d5f6dc919c06e3e00 Mon Sep 17 00:00:00 2001 From: daniel-c-harvey Date: Mon, 15 Jun 2026 12:43:56 -0400 Subject: [PATCH] feat(visualizer): WebGL2 fragment-shader Mix renderer at parity; datum-as-texture, shader-clock rAF, drop CSS backdrop-filter (P10 W1) --- .../Controls/MixWaveformVisualizer.razor.css | 9 +- .../Interop/visualizer/MixVisualizer.ts | 699 +++++++++++------- 2 files changed, 456 insertions(+), 252 deletions(-) diff --git a/DeepDrftPublic.Client/Controls/MixWaveformVisualizer.razor.css b/DeepDrftPublic.Client/Controls/MixWaveformVisualizer.razor.css index cc4f0a8..4113ef7 100644 --- a/DeepDrftPublic.Client/Controls/MixWaveformVisualizer.razor.css +++ b/DeepDrftPublic.Client/Controls/MixWaveformVisualizer.razor.css @@ -8,17 +8,16 @@ overflow: hidden; } -/* The canvas fills the viewport. The glassy/frosted treatment is a CSS backdrop-blur on this layer - (the ribbon's luminous depth is drawn inside the canvas by the module); together they read as lit - glass moving behind the content rather than a hard chart. */ +/* The canvas fills the viewport. All ribbon shading (luminous depth, soft edges) is drawn inside the + canvas by the WebGL2 fragment shader. NO CSS backdrop-filter: it was a confirmed per-frame perf killer + on the Canvas predecessor and is exactly the cost the GPU move exists to eliminate (spec §2, §5.2); + the glass treatment returns in-shader in Wave 3. */ .mix-waveform-canvas { position: absolute; inset: 0; width: 100%; height: 100%; display: block; - backdrop-filter: blur(2px); - -webkit-backdrop-filter: blur(2px); } /* Zoom slider — a small viewing control pinned to the top-right, clear of the player bar at diff --git a/DeepDrftPublic/Interop/visualizer/MixVisualizer.ts b/DeepDrftPublic/Interop/visualizer/MixVisualizer.ts index 77ece8a..45c6b3f 100644 --- a/DeepDrftPublic/Interop/visualizer/MixVisualizer.ts +++ b/DeepDrftPublic/Interop/visualizer/MixVisualizer.ts @@ -1,5 +1,5 @@ /** - * MixVisualizer — the scrolling Mix waveform background (Phase 9, 8.K Wave 2). + * MixVisualizer — the scrolling Mix waveform background (Phase 10, Wave 1). * * What this renders: a *windowed* slice of a mix's loudness profile, scrolling * bottom-to-top, coupled to playback position. New audio enters at the bottom, @@ -7,18 +7,26 @@ * line (vertical centre by default). This is a read-only, ambient lava-lamp * background — there is no seek, no click handling, no write-back to playback. * - * Rendering tech: HTML5 Canvas 2D. This is the industry-standard, well-documented - * choice for a single flowing gradient waveform: createLinearGradient gives us the - * theme gradient directly, and layered translucent draws give the glassy look - * without any exotic tricks. Canvas 2D holds 60fps comfortably here because each - * frame draws one filled path of a few hundred points, not a per-pixel shader. - * (If this ever fails the 60fps budget at the glassy treatment, the textbook next - * step is WebGL — but we are nowhere near needing it, so we stay on Canvas 2D.) + * Rendering tech: WebGL2, fragment-shader. This is a wholesale renderer swap from + * the Canvas 2D predecessor (8.K). The reasons are forward-looking (the planned + * bulge / detach / morphing-field / glass effects are all per-pixel, per-frame + * work that Canvas is worst at and a fragment shader is best at — see + * product-notes/mix-visualizer-webgl-renderer.md). Wave 1 introduces NO new + * effects: it reproduces the predecessor's scrolling navy/moss-ish ribbon on the + * GPU at parity, holding 60 FPS, with the Blazor bridge contract unchanged. + * + * The pipeline is the textbook "shadertoy-style" full-screen pass: a single quad + * covering the canvas, a trivial pass-through vertex shader, and ALL the work in + * the fragment shader. Per fragment (pixel) the shader asks "which mix-time does + * my screen Y map to, what loudness is there, am I inside the ribbon, and what + * colour am I?" — the same scroll/zoom math the Canvas walked per screen-row, + * evaluated per-pixel in parallel on the GPU instead. * * The Blazor component owns the canvas element and the inputs (datum, playback, * zoom, theme); this module owns the requestAnimationFrame loop and all the - * drawing/scroll/zoom math. The component drives it through the small handle - * returned by `create`. + * GL/scroll/zoom math. The component drives it through the small handle returned + * by `create`. The handle shape is identical to the Canvas predecessor's, so the + * bridge (MixWaveformVisualizer.razor.cs) needs no change. */ // ── Tuning anchors (see spec §B). These are the load-bearing constants. ────────── @@ -39,27 +47,47 @@ export const DEFAULT_VISIBLE_SECONDS = 10; /** * Where the "now" line sits within the window, as a fraction from the top. * 0.5 = vertical centre (default): a short lead-in below, a short trail-out above. - * Tunable. + * Tunable. NOTE: kept in sync with the GLSL constant NOW_ANCHOR_FROM_TOP below. */ const NOW_ANCHOR_FROM_TOP = 0.5; /** Background opacity of the whole ribbon — keeps it a backdrop, not a chart. */ const RIBBON_OPACITY = 0.22; +/** + * Half-width of the ribbon at full loudness, as a fraction of half the canvas + * width (the predecessor used 0.92). Mirrors the Canvas `maxHalfWidth` factor. + */ +const RIBBON_HALF_WIDTH_FRAC = 0.92; + +/** + * Cap device-pixel-ratio at 2. Beyond that the extra fragments cost frame time for + * no visible gain on a soft glassy backdrop — this is the graceful-degrade lever + * (spec §5.1): drop internal resolution before dropping frames. + */ +const MAX_DPR = 2; + // ── Theme: the gradient stop colours, read live from the active MudBlazor palette. ─ // -// We do NOT take colours from Blazor: canvas createLinearGradient stop colours must be concrete -// CSS colour strings (it does not resolve `var(--…)`). Instead the module reads the computed -// `--mud-palette-*` custom properties straight off the canvas element, which inherits them from the -// page. The bespoke light/dark themes ("Charleston in the Day" / "Lowcountry Summer Nights") swap -// those vars when dark mode toggles, so re-reading them re-themes the gradient with no reload. The +// The shader cannot resolve `var(--mud-palette-*)` directly — uniforms are plain +// floats. So we read the computed `--mud-palette-*` custom properties straight off +// the canvas element (which inherits them from the page), parse them to RGB, and +// upload them as vec3 colour uniforms. The bespoke light/dark themes ("Charleston +// in the Day" / "Lowcountry Summer Nights") swap those vars when dark mode toggles, +// so re-reading + re-uploading them re-themes the ribbon with no reload. The // component just calls `refreshTheme()` after a dark-mode change. +// +// Parity binding (matches the Canvas predecessor): accent = --mud-palette-primary +// (brightest, at the now line), edge = --mud-palette-surface (dim, at the window +// edges so the ribbon fades into the page). Wave 3 will replace this two-stop +// gradient with the full navy↔moss morphing field; for parity we keep the +// predecessor's exact stops. interface ResolvedTheme { - /** Colour at the "now" line (brightest). Concrete CSS colour. */ - accent: string; - /** Colour at the window edges (dimmer). Concrete CSS colour. */ - edge: string; + /** RGB [0,1] at the "now" line (brightest). */ + accent: [number, number, number]; + /** RGB [0,1] at the window edges (dimmer). */ + edge: [number, number, number]; } /** Read a CSS custom property off an element, falling back if it is empty/undefined. */ @@ -68,19 +96,44 @@ function readVar(el: Element, name: string, fallback: string): string { return v.length > 0 ? v : fallback; } +/** + * Parse a CSS colour string to normalized [0,1] RGB. Handles #rgb / #rrggbb and + * rgb()/rgba() — the only forms MudBlazor emits for these palette vars. Falls back + * to mid-grey on anything unrecognised so a parse miss degrades to a visible + * ribbon rather than black. + */ +function parseColor(css: string): [number, number, number] { + const s = css.trim(); + if (s.startsWith('#')) { + let hex = s.slice(1); + if (hex.length === 3) { + hex = hex[0] + hex[0] + hex[1] + hex[1] + hex[2] + hex[2]; + } + if (hex.length >= 6) { + const r = parseInt(hex.slice(0, 2), 16); + const g = parseInt(hex.slice(2, 4), 16); + const b = parseInt(hex.slice(4, 6), 16); + if (!Number.isNaN(r) && !Number.isNaN(g) && !Number.isNaN(b)) { + return [r / 255, g / 255, b / 255]; + } + } + } + const m = s.match(/rgba?\(\s*([\d.]+)\s*,\s*([\d.]+)\s*,\s*([\d.]+)/i); + if (m) { + return [Number(m[1]) / 255, Number(m[2]) / 255, Number(m[3]) / 255]; + } + return [0.5, 0.5, 0.5]; +} + // ── Datum: the pre-downloaded loudness profile (spec §F). ──────────────────────── interface Datum { - /** Loudness samples, each already normalized to [0, 1]. */ - samples: Float32Array; - /** Total mix duration in seconds — needed to map time <-> sample index. */ + /** GPU texture holding the loudness samples (R8, 1 row tall), linear-filtered. */ + texture: WebGLTexture; + /** Number of loudness samples uploaded — used to map the texel-centre offset. */ + sampleCount: number; + /** Total mix duration in seconds — needed to map time <-> texture coordinate. */ durationSeconds: number; - /** - * samplesPerSecond = samples.length / durationSeconds. Wave 1 made the sample - * count duration-derived (~333/s), so this is NOT a fixed 2048/duration — we - * always compute it from the actual datum length. - */ - samplesPerSecond: number; } interface Playback { @@ -100,33 +153,236 @@ export interface MixVisualizerHandle { } /** - * Decode the base64 loudness datum (bytes [0,255]) into normalized [0,1] floats. - * Done once per datum, off the animation path. + * Decode the base64 loudness datum (bytes [0,255]) into a Uint8Array suitable for + * direct upload as an R8 texture. Done once per datum, off the animation path. We + * keep the bytes as [0,255] and let the GPU normalize to [0,1] on sample (R8 + * UNORM), which mirrors the predecessor's /255 and avoids a CPU float pass. */ -function decodeSamples(base64: string): Float32Array { +function decodeSamples(base64: string): Uint8Array { const binary = atob(base64); - const out = new Float32Array(binary.length); + const out = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) { - out[i] = binary.charCodeAt(i) / 255; // [0,255] -> [0,1] + out[i] = binary.charCodeAt(i); } return out; } -export function create(canvas: HTMLCanvasElement): MixVisualizerHandle { - const maybeCtx = canvas.getContext('2d'); - if (!maybeCtx) { - // No 2D context (extremely old/headless engine): hand back a no-op handle so - // the component still functions as a plain backdrop. - return { - setDatum() {}, - setPlayback() {}, - setZoom() {}, - refreshTheme() {}, - dispose() {}, - }; +// ── Shaders. ───────────────────────────────────────────────────────────────────── +// +// Vertex: trivial pass-through. We draw a single triangle that more than covers the +// clip-space box ([-1,1]²) so every pixel of the canvas is rasterized once. (One +// oversized triangle is the standard full-screen trick — cheaper than two and has +// no diagonal seam.) gl_FragCoord in the fragment shader then gives us the pixel's +// screen position directly. + +const VERTEX_SHADER = `#version 300 es +// Three clip-space vertices forming one big triangle covering [-1,1]². +const vec2 POSITIONS[3] = vec2[3]( + vec2(-1.0, -1.0), + vec2( 3.0, -1.0), + vec2(-1.0, 3.0) +); +void main() { + gl_Position = vec4(POSITIONS[gl_VertexID], 0.0, 1.0); +} +`; + +// Fragment: THE SCROLL + ZOOM MATH (spec §A, §B), ported intact from the Canvas +// predecessor's per-row loop into a per-fragment evaluation. Read this top to +// bottom to follow how a quarter-note-@-180-BPM becomes 0.333 s becomes a texture +// coordinate becomes a lit pixel. +// +// Coordinate model (matches the Canvas predecessor exactly): +// - gl_FragCoord.xy is in device pixels, origin BOTTOM-left in WebGL. The Canvas +// used a TOP-left origin with Y increasing downward. We flip Y once up front +// (screenYTop) so all the time math below reads identically to the Canvas +// version: screenYTop = 0 at the top edge, = uResolution.y at the bottom. +// - The "now" line is a fixed screen Y: nowY = height * NOW_ANCHOR_FROM_TOP. +// - Audio flows UP: newer audio is drawn lower and scrolls up past the now line. +// * audio BELOW the now line (screenYTop > nowY) is the lead-in (not yet played) +// * audio ABOVE the now line (screenYTop < nowY) is the trail-out (just played) +// +// Zoom -> time-span -> pixels: +// - uVisibleSeconds is the whole window's time span, top to bottom. At max zoom +// this is MIN_VISIBLE_SECONDS (0.333 s); at min zoom MAX_VISIBLE_SECONDS. +// - pixelsPerSecond = height / uVisibleSeconds. Smaller visibleSeconds => more px +// per second => the same audio sweeps the window faster at a fixed playback +// rate. That IS the Guitar-Hero coupling: apparent scroll speed falls straight +// out of the zoom, with no separate speed control. +// +// Time at a given screen Y: +// - At nowY the time is uPlayheadSeconds. +// - Moving DOWN by 1 px (screenYTop +1) adds (1 / pixelsPerSecond) seconds. +// - So: timeAt(y) = playhead + (screenYTop - nowY) / pixelsPerSecond +// +// Sample at a given time: +// - Texture coord = time / durationSeconds. The datum texture is LINEAR-filtered, +// so the GPU interpolates between samples in hardware — the smooth, no-stair- +// stepping read at every zoom that the design wanted, for free. +// - Outside [0, durationSeconds] we force loudness to 0. That is what gives the +// "scrolls in from empty / out to empty" behaviour at the very start and end of +// the mix (spec §A) with no special-casing. (CLAMP_TO_EDGE on the texture would +// otherwise repeat the edge sample, so we gate explicitly here.) + +const FRAGMENT_SHADER = `#version 300 es +precision highp float; + +uniform vec2 uResolution; // canvas size in device pixels +uniform float uPlayheadSeconds; // current playback position (per-frame) +uniform float uTimeSeconds; // monotonic clock (per-frame) — reserved for Wave 3 motion +uniform float uVisibleSeconds; // zoom: window time-span (per change) +uniform float uDurationSeconds; // mix length (per datum) +uniform vec3 uColorAccent; // brightest stop, at the now line (per theme) +uniform vec3 uColorEdge; // dim stop, at the window edges (per theme) +uniform float uHasDatum; // 1.0 when a datum texture is bound, else 0.0 +uniform sampler2D uDatum; // loudness profile, R8, 1 row tall, LINEAR-filtered + +out vec4 fragColor; + +const float NOW_ANCHOR_FROM_TOP = ${NOW_ANCHOR_FROM_TOP.toFixed(4)}; +const float RIBBON_OPACITY = ${RIBBON_OPACITY.toFixed(4)}; +const float RIBBON_HALF_WIDTH_FRAC = ${RIBBON_HALF_WIDTH_FRAC.toFixed(4)}; + +// Loudness at an absolute mix time, or 0 outside the mix (drives scroll-in/out). +float sampleAt(float timeSeconds) { + if (uHasDatum < 0.5) return 0.0; + if (timeSeconds < 0.0 || timeSeconds >= uDurationSeconds) return 0.0; + // u = normalized position along the mix; GPU linear filtering interpolates. + float u = timeSeconds / uDurationSeconds; + return texture(uDatum, vec2(u, 0.5)).r; +} + +void main() { + float w = uResolution.x; + float h = uResolution.y; + + // Flip to a top-left, downward-Y frame so the time math matches the Canvas port. + float screenYTop = h - gl_FragCoord.y; + float screenX = gl_FragCoord.x; + + float nowY = h * NOW_ANCHOR_FROM_TOP; + float pixelsPerSecond = h / uVisibleSeconds; + + // time at this fragment's row, then loudness there. + float t = uPlayheadSeconds + (screenYTop - nowY) / pixelsPerSecond; + float amp = sampleAt(t); + + // Ribbon silhouette: symmetric about the horizontal centre, half-width scales + // with loudness. This is the signed-distance form of the Canvas mirrored path. + float centreX = w * 0.5; + float maxHalfWidth = (w * 0.5) * RIBBON_HALF_WIDTH_FRAC; + float halfWidth = amp * maxHalfWidth; + float distFromCentre = abs(screenX - centreX); + + // Soft edge: a ~1.5px feather at the silhouette boundary so the ribbon reads as + // a glowy lit band rather than a hard-edged chart — the parity stand-in for the + // predecessor's shadowBlur halo, done with a cheap smoothstep instead of a + // per-frame CPU blur. (No CSS backdrop-filter, no shadowBlur — spec §5.2.) + float feather = 1.5; + float inside = 1.0 - smoothstep(halfWidth - feather, halfWidth + feather, distFromCentre); + + // Vertical gradient: brightest at the now line, dimming toward both edges. Same + // luminosity cue the predecessor drew (edge -> accent -> edge by screen Y). + float distFromNow = abs(screenYTop - nowY) / max(nowY, h - nowY); + vec3 ribbonColor = mix(uColorAccent, uColorEdge, clamp(distFromNow, 0.0, 1.0)); + + float alpha = inside * RIBBON_OPACITY; + // Pre-multiplied output (blend func is ONE / ONE_MINUS_SRC_ALPHA) so the + // translucent ribbon composites cleanly over the transparent canvas. + fragColor = vec4(ribbonColor * alpha, alpha); +} +`; + +/** Compile one shader stage, throwing with the info log on failure. */ +function compileShader(gl: WebGL2RenderingContext, type: number, source: string): WebGLShader { + const shader = gl.createShader(type); + if (!shader) throw new Error('MixVisualizer: gl.createShader returned null.'); + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + const log = gl.getShaderInfoLog(shader); + gl.deleteShader(shader); + throw new Error(`MixVisualizer: shader compile failed: ${log}`); } - // Non-null binding so the closures below (draw/frame) keep the narrowing. - const ctx: CanvasRenderingContext2D = maybeCtx; + return shader; +} + +/** Link the vertex + fragment shaders into a program, throwing on failure. */ +function linkProgram(gl: WebGL2RenderingContext): WebGLProgram { + const vert = compileShader(gl, gl.VERTEX_SHADER, VERTEX_SHADER); + const frag = compileShader(gl, gl.FRAGMENT_SHADER, FRAGMENT_SHADER); + const program = gl.createProgram(); + if (!program) throw new Error('MixVisualizer: gl.createProgram returned null.'); + gl.attachShader(program, vert); + gl.attachShader(program, frag); + gl.linkProgram(program); + // Shaders can be deleted after link — the program retains the compiled code. + gl.deleteShader(vert); + gl.deleteShader(frag); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + const log = gl.getProgramInfoLog(program); + gl.deleteProgram(program); + throw new Error(`MixVisualizer: program link failed: ${log}`); + } + return program; +} + +/** The no-op handle returned when WebGL2 is unavailable or setup fails. */ +function noopHandle(): MixVisualizerHandle { + return { + setDatum() {}, + setPlayback() {}, + setZoom() {}, + refreshTheme() {}, + dispose() {}, + }; +} + +export function create(canvas: HTMLCanvasElement): MixVisualizerHandle { + // premultipliedAlpha so the translucent ribbon composites correctly over the + // page; antialias off (the soft-edge smoothstep handles AA in-shader, and MSAA + // would cost fill rate we don't need for a backdrop). + const maybeGl = canvas.getContext('webgl2', { + alpha: true, + premultipliedAlpha: true, + antialias: false, + }); + if (!maybeGl) { + // No WebGL2 (old engine / disabled): hand back a no-op handle so the + // component still functions as a plain backdrop (mirrors the predecessor's + // no-2d-context fallback, now guarding against no-WebGL2). + return noopHandle(); + } + // Non-null binding so the closures below keep the narrowing (TS does not carry + // control-flow narrowing of a captured `const` into nested functions). + const gl: WebGL2RenderingContext = maybeGl; + + let program: WebGLProgram; + try { + program = linkProgram(gl); + } catch (err) { + // A compile/link failure on an exotic driver should degrade to the plain + // backdrop, not crash the page. Log for diagnosis; return the no-op handle. + console.warn(err); + return noopHandle(); + } + + // An empty VAO is still required in WebGL2 core to issue a draw; the vertex + // shader sources its positions from gl_VertexID, so no attribute buffers. + const vao = gl.createVertexArray(); + + // Cache uniform locations once. + const u = { + resolution: gl.getUniformLocation(program, 'uResolution'), + playheadSeconds: gl.getUniformLocation(program, 'uPlayheadSeconds'), + timeSeconds: gl.getUniformLocation(program, 'uTimeSeconds'), + visibleSeconds: gl.getUniformLocation(program, 'uVisibleSeconds'), + durationSeconds: gl.getUniformLocation(program, 'uDurationSeconds'), + colorAccent: gl.getUniformLocation(program, 'uColorAccent'), + colorEdge: gl.getUniformLocation(program, 'uColorEdge'), + hasDatum: gl.getUniformLocation(program, 'uHasDatum'), + datum: gl.getUniformLocation(program, 'uDatum'), + }; // ── Mutable state, fed by the component through the handle. ────────────────── let datum: Datum | null = null; @@ -137,9 +393,9 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle { function readTheme(): ResolvedTheme { return { // Brightest stop at the "now" line — the bespoke themes' primary accent. - accent: readVar(canvas, '--mud-palette-primary', '#b08d57'), - // Dim stop at the edges — the surface/background colour so the ribbon fades into the page. - edge: readVar(canvas, '--mud-palette-surface', '#1a1a1a'), + accent: parseColor(readVar(canvas, '--mud-palette-primary', '#b08d57')), + // Dim stop at the edges — the surface colour so the ribbon fades into the page. + edge: parseColor(readVar(canvas, '--mud-palette-surface', '#1a1a1a')), }; } @@ -147,6 +403,7 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle { let rafId: number | null = null; let disposed = false; + const startTimeMs = performance.now(); // Backing-store size in device pixels, tracked so we only resize the canvas // (which clears it) when the CSS box actually changed. @@ -154,199 +411,116 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle { let cssHeight = 0; let dpr = 1; + // ── One-time GL pipeline setup. ────────────────────────────────────────────── + gl.useProgram(program); + gl.disable(gl.DEPTH_TEST); + // Pre-multiplied alpha blend: src already carries colour*alpha (see frag shader). + gl.enable(gl.BLEND); + gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA); + // The datum lives in texture unit 0 for the program's lifetime. + gl.uniform1i(u.datum, 0); + // ── ResizeObserver: one-shot redraw when the container changes while idle. ──── // - // While the rAF loop is running (playing), syncCanvasSize() catches resizes each - // frame. While idle (paused/stopped), we use a ResizeObserver instead — it fires - // only when the element actually changes size, which is far cheaper than a 60fps - // tick. On each observation we do a single one-shot redraw. - const resizeObserver = new ResizeObserver(() => { - if (!playback.isPlaying && !disposed) { - // Loop is not running; draw one still frame reflecting the new size. - redrawOnce(); - } - // If the loop IS running, syncCanvasSize() inside frame() will catch it next - // tick — no action needed here. + // The ResizeObserver is the SOLE size writer (spec §2e): we never call + // getBoundingClientRect per frame. While playing, the rAF loop redraws every + // frame anyway and picks up the new backing-store size set here. While idle, the + // observer fires only on an actual size change and triggers a single redraw. + const resizeObserver = new ResizeObserver((entries) => { + if (disposed) return; + const entry = entries[0]; + // contentBoxSize is the modern, layout-thrash-free size source. Fall back to + // contentRect for engines that don't populate it. + const box = entry.contentBoxSize?.[0]; + const nextCssWidth = box ? box.inlineSize : entry.contentRect.width; + const nextCssHeight = box ? box.blockSize : entry.contentRect.height; + applySize(nextCssWidth, nextCssHeight); + // While idle, draw one still frame reflecting the new size. While playing, + // the running loop will redraw on its next tick — no action needed. + if (!playback.isPlaying) redrawOnce(); }); resizeObserver.observe(canvas); /** - * Sync the canvas backing store to its CSS size × devicePixelRatio so the draw - * is crisp on HiDPI without blurring. Returns true if a resize happened. + * Update the backing store to a CSS size × devicePixelRatio (capped at MAX_DPR) + * and the GL viewport. Only resizes when something changed — resizing clears the + * drawing buffer, so we avoid needless churn. This is the only place the canvas + * size is written (fed by the ResizeObserver, never by a per-frame measure). */ - function syncCanvasSize(): boolean { - const rect = canvas.getBoundingClientRect(); - const nextDpr = window.devicePixelRatio || 1; - // Cap DPR at 2: beyond that the extra pixels cost frame time for no visible - // gain on a soft glassy backdrop (graceful-degrade lever, spec §E). - const effectiveDpr = Math.min(nextDpr, 2); - if (rect.width === cssWidth && rect.height === cssHeight && effectiveDpr === dpr) { - return false; + function applySize(nextCssWidth: number, nextCssHeight: number): void { + const nextDpr = Math.min(window.devicePixelRatio || 1, MAX_DPR); + if (nextCssWidth === cssWidth && nextCssHeight === cssHeight && nextDpr === dpr) { + return; } - cssWidth = rect.width; - cssHeight = rect.height; - dpr = effectiveDpr; + cssWidth = nextCssWidth; + cssHeight = nextCssHeight; + dpr = nextDpr; canvas.width = Math.max(1, Math.round(cssWidth * dpr)); canvas.height = Math.max(1, Math.round(cssHeight * dpr)); - return true; + gl.viewport(0, 0, canvas.width, canvas.height); } /** - * THE SCROLL + ZOOM MATH (spec §A, §B). Read this top to bottom to follow how - * a quarter-note-@-180-BPM becomes 0.333 s becomes N samples becomes pixels. - * - * Coordinate model: - * - The canvas is `cssHeight` px tall (we draw in CSS px; the ctx is scaled by - * dpr so 1 unit == 1 CSS px). - * - The "now" line is a fixed screen Y: nowY = cssHeight * NOW_ANCHOR_FROM_TOP. - * - Audio flows UP: time increases downward in the data, but newer audio is - * drawn lower and scrolls up past the now line. So: - * * audio BELOW the now line (screen Y > nowY) is the lead-in (not yet played) - * * audio ABOVE the now line (screen Y < nowY) is the trail-out (just played) - * - * Zoom -> time-span -> pixels: - * - `visibleSeconds` is the entire window's time span, top to bottom. At max - * zoom this is MIN_VISIBLE_SECONDS (0.333 s); at min zoom MAX_VISIBLE_SECONDS. - * - pixelsPerSecond = cssHeight / visibleSeconds. Smaller visibleSeconds => - * more px per second => the same audio sweeps the window faster at a fixed - * playback rate. That IS the Guitar-Hero coupling: apparent scroll speed - * falls straight out of the zoom, with no separate speed control. - * - * Time at a given screen Y: - * - At nowY the time is playback.positionSeconds. - * - Moving DOWN by 1 px adds (1 / pixelsPerSecond) seconds (future audio). - * - So: timeAt(y) = now + (y - nowY) / pixelsPerSecond - * - * Sample at a given time (spec §F mapping, BucketCount-driven, never fixed-2048): - * - sampleIndex = round(time * samplesPerSecond), where - * samplesPerSecond = datum.samples.length / datum.durationSeconds. - * - Out-of-range indices (before 0 or past the end) draw as zero amplitude, - * which is what gives the "scrolls in from empty / out to empty" behaviour - * at the very start and end of the mix (spec §A) with no special-casing. + * Issue one GL draw with the current uniforms. The fragment shader does all the + * scroll/zoom/ribbon work; here we just push the per-frame uniforms and draw the + * full-screen triangle. */ function draw(): void { - const w = cssWidth; - const h = cssHeight; - ctx.setTransform(dpr, 0, 0, dpr, 0, 0); // draw in CSS px, crisp on HiDPI - ctx.clearRect(0, 0, w, h); + if (canvas.width <= 0 || canvas.height <= 0) return; - if (!datum || h <= 0 || w <= 0) return; + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); - const now = playback.positionSeconds; - const nowY = h * NOW_ANCHOR_FROM_TOP; - const pixelsPerSecond = h / visibleSeconds; - const samplesPerSecond = datum.samplesPerSecond; - const sampleCount = datum.samples.length; + gl.useProgram(program); + gl.bindVertexArray(vao); - // We draw one screen row per pixel of height (a few hundred points) — smooth - // at every zoom with no stair-stepping, cheap enough for 60fps. - const centreX = w / 2; - // Max half-width of the ribbon (mirrored silhouette), with a small margin. - const maxHalfWidth = (w / 2) * 0.92; + // Per-frame uniforms. + gl.uniform2f(u.resolution, canvas.width, canvas.height); + gl.uniform1f(u.playheadSeconds, playback.positionSeconds); + gl.uniform1f(u.timeSeconds, (performance.now() - startTimeMs) / 1000); + // Per-change / per-theme / per-datum uniforms (cheap to set every frame; no + // separate dirty-tracking needed for scalars/vec3s). + gl.uniform1f(u.visibleSeconds, visibleSeconds); + gl.uniform3fv(u.colorAccent, theme.accent); + gl.uniform3fv(u.colorEdge, theme.edge); - // Build the mirrored closed path: down the right edge (top->bottom), then back - // up the left edge (bottom->top). amplitude maps loudness [0,1] to half-width. - ctx.beginPath(); - - // Right edge, top to bottom. - for (let y = 0; y <= h; y++) { - const t = now + (y - nowY) / pixelsPerSecond; // time at this screen row - const amp = sampleAt(t); - const x = centreX + amp * maxHalfWidth; - if (y === 0) ctx.moveTo(x, y); - else ctx.lineTo(x, y); + if (datum) { + gl.uniform1f(u.hasDatum, 1); + gl.uniform1f(u.durationSeconds, datum.durationSeconds); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, datum.texture); + } else { + gl.uniform1f(u.hasDatum, 0); + gl.uniform1f(u.durationSeconds, 1); } - // Left edge, bottom to top (mirror). - for (let y = h; y >= 0; y--) { - const t = now + (y - nowY) / pixelsPerSecond; - const amp = sampleAt(t); - const x = centreX - amp * maxHalfWidth; - ctx.lineTo(x, y); - } - ctx.closePath(); - // ── Glassy gradient fill (spec §C). ───────────────────────────────────── - // Vertical gradient brightest at the now line, dimming toward both edges — - // this is the optional luminosity cue that reinforces the playhead without a - // hard played/unplayed boundary. Stops come from the live MudBlazor palette. - const grad = ctx.createLinearGradient(0, 0, 0, h); - const nowStop = clamp01(nowY / h); - grad.addColorStop(0, theme.edge); // top edge (trail-out), dim - grad.addColorStop(nowStop, theme.accent); // the "now" line, brightest - grad.addColorStop(1, theme.edge); // bottom edge (lead-in), dim - - // Two layered draws give the frosted/lit-glass depth: a soft wide glow under - // a crisper core, both translucent. No backdrop-filter needed on the canvas - // itself (the CSS layer adds blur); this is pure standard 2D compositing. - ctx.globalCompositeOperation = 'source-over'; - - // Soft luminous halo. - ctx.globalAlpha = RIBBON_OPACITY * 0.6; - ctx.shadowColor = theme.accent; - ctx.shadowBlur = 24 * dpr; - ctx.fillStyle = grad; - ctx.fill(); - - // Crisp core on top, no shadow. - ctx.shadowBlur = 0; - ctx.globalAlpha = RIBBON_OPACITY; - ctx.fillStyle = grad; - ctx.fill(); - - ctx.globalAlpha = 1; - } - - /** Loudness at an absolute mix time, or 0 outside the mix (drives scroll-in/out from empty). */ - function sampleAt(timeSeconds: number): number { - if (!datum) return 0; - if (timeSeconds < 0 || timeSeconds >= datum.durationSeconds) return 0; - const idx = Math.round(timeSeconds * datum.samplesPerSecond); - if (idx < 0 || idx >= datum.samples.length) return 0; - return datum.samples[idx]; - } - - function clamp01(v: number): number { - return v < 0 ? 0 : v > 1 ? 1 : v; + // One full-screen triangle (3 vertices), positions from gl_VertexID. + gl.drawArrays(gl.TRIANGLES, 0, 3); + gl.bindVertexArray(null); } // ── rAF loop lifecycle (spec §E: cool when paused/backgrounded). ───────────── // // DESIGN: The loop runs ONLY while playing. When paused or stopped, no frames - // are scheduled. The still slice stays correct via one-shot redraws triggered by - // the handle methods (setZoom, refreshTheme, setDatum) and by ResizeObserver. - // - // Start/stop contract: - // startLoop() — schedules the first frame if not already running. Safe to call - // redundantly; the rafId guard prevents double-loops. - // stopLoop() — cancels any pending frame. The current frame callback will see - // playback.isPlaying === false and will NOT reschedule itself, so - // this is belt-and-suspenders for the dispose() path. - // redrawOnce() — draw one still frame synchronously (no rAF scheduling). Used - // by setZoom/refreshTheme/setDatum/ResizeObserver while idle. + // are scheduled — the GPU is idle. The still slice stays correct via one-shot + // redraws triggered by the handle methods (setZoom/refreshTheme/setDatum) and + // by the ResizeObserver. The shader clock (uTimeSeconds) advances every frame so + // motion is smooth at 60 FPS between Blazor's ~10 Hz playback ticks, not stepping + // at that cadence (spec §2e / §5.4). - /** - * Draw one still frame immediately, without scheduling a new rAF. Syncs the - * canvas size first so zoom/theme/datum/resize changes are reflected correctly - * even when the loop is not running. - */ + /** Draw one still frame immediately, without scheduling a new rAF. */ function redrawOnce(): void { if (disposed) return; - syncCanvasSize(); draw(); } - /** - * Start the rAF loop. No-op if already running or disposed — the rafId guard - * ensures at most one loop is live at any time. - */ + /** Start the rAF loop. No-op if already running or disposed (rafId guard). */ function startLoop(): void { if (disposed || rafId !== null) return; rafId = requestAnimationFrame(frame); } - /** - * Stop the rAF loop. Safe to call when already stopped. - */ + /** Stop the rAF loop. Safe to call when already stopped. */ function stopLoop(): void { if (rafId !== null) { cancelAnimationFrame(rafId); @@ -355,54 +529,82 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle { } /** - * The animation loop. Runs only while playing. Each frame: - * 1. Syncs the canvas backing-store size (cheap no-op when nothing changed). - * 2. Redraws the scrolling waveform with the current playback position. - * 3. Reschedules itself — unless playback has stopped since this frame was - * queued, in which case it draws one final still frame and exits the loop. + * The animation loop. Runs only while playing. Each frame draws the scrolling + * waveform at the current playback position + shader clock, then reschedules + * itself — unless playback stopped since this frame was queued, in which case it + * draws one final still frame (already done above) and exits the loop. * - * A backgrounded tab gets rAF throttled by the browser automatically. On top of - * that, the loop does not run at all when paused — so a foregrounded-but-paused - * mix burns no rAF budget (spec §E acceptance criterion). + * A backgrounded tab gets rAF throttled by the browser automatically; on top of + * that the loop does not run at all when paused, so a foregrounded-but-paused + * mix burns no frames (spec §E / §5.3). */ function frame(): void { if (disposed) { rafId = null; return; } - - syncCanvasSize(); draw(); - if (playback.isPlaying) { - // Still playing — schedule the next frame. rafId = requestAnimationFrame(frame); } else { - // Playback stopped between the time this frame was queued and now. - // We already drew the final still frame above; exit the loop. + // Playback stopped between queue and now; final still frame drawn above. rafId = null; } } - // Kick off one still frame on creation so the canvas is not blank while idle - // before the first play command arrives. - redrawOnce(); + // Read the initial size synchronously (one getBoundingClientRect at setup is + // fine — it is the ResizeObserver that must not measure per-frame), then draw a + // still frame so the canvas isn't blank before the first play command. + { + const rect = canvas.getBoundingClientRect(); + applySize(rect.width, rect.height); + redrawOnce(); + } + + /** + * Upload the loudness samples as a 1-row R8 texture with LINEAR filtering and + * CLAMP_TO_EDGE. Returns the Datum, or null on an empty/invalid input. + */ + function uploadDatum(samplesBase64: string, durationSeconds: number): Datum | null { + if (durationSeconds <= 0 || !samplesBase64) return null; + const samples = decodeSamples(samplesBase64); + if (samples.length === 0) return null; + + const texture = gl.createTexture(); + if (!texture) return null; + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texture); + // 1-byte rows: relax the default 4-byte unpack alignment so an odd-length + // datum uploads without row padding corruption. + gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1); + gl.texImage2D( + gl.TEXTURE_2D, 0, gl.R8, + samples.length, 1, 0, + gl.RED, gl.UNSIGNED_BYTE, samples, + ); + // LINEAR gives the smooth, no-stair-stepping interpolation between samples + // at every zoom — in hardware, for free (spec §2b). CLAMP so a coord at the + // very edge doesn't wrap (out-of-range times are zeroed in-shader anyway). + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.bindTexture(gl.TEXTURE_2D, null); + + return { texture, sampleCount: samples.length, durationSeconds }; + } return { setDatum(samplesBase64: string, durationSeconds: number): void { - if (durationSeconds <= 0 || !samplesBase64) { + // Free the previous datum's GPU texture before replacing it (no leak + // across re-pushes / mix changes — spec §5.11). + if (datum) { + gl.deleteTexture(datum.texture); datum = null; - return; } - const samples = decodeSamples(samplesBase64); - datum = { - samples, - durationSeconds, - // samplesPerSecond from the ACTUAL datum length — never assume 2048. - samplesPerSecond: samples.length / durationSeconds, - }; - // New datum changes what is drawn — refresh the still slice immediately. - // If playing, the running loop will pick it up on the next frame automatically. + datum = uploadDatum(samplesBase64, durationSeconds); + // New datum changes what is drawn — refresh the still slice immediately + // when idle. If playing, the running loop picks it up next frame. if (!playback.isPlaying) redrawOnce(); }, @@ -411,31 +613,27 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle { playback = { positionSeconds, isPlaying }; if (isPlaying && !wasPlaying) { - // Transition: paused/stopped → playing. Start the rAF loop. + // Transition paused/stopped → playing: start the rAF loop. startLoop(); } else if (!isPlaying && wasPlaying) { - // Transition: playing → paused/stopped. The current in-flight frame - // will draw the final still position and exit the loop on its own - // (frame() checks playback.isPlaying before rescheduling). We do NOT - // call stopLoop() here — that would cancel the in-flight frame before - // it draws, leaving a stale or blank canvas. Let the frame run out. + // Transition playing → paused/stopped: the in-flight frame draws the + // final still position and exits on its own (frame() checks + // playback.isPlaying before rescheduling). We do NOT stopLoop() here — + // that would cancel the in-flight frame before it draws, leaving a + // stale canvas. Let the frame run out. } - // If isPlaying unchanged (position-only update), the running loop (if any) - // will redraw on the next frame automatically; no action needed. + // isPlaying unchanged (position-only update): the running loop (if any) + // redraws next frame; nothing to do here. }, setZoom(seconds: number): void { // Clamp into the supported span so a stray value can't break the math. visibleSeconds = Math.min(MAX_VISIBLE_SECONDS, Math.max(MIN_VISIBLE_SECONDS, seconds)); - // Zoom changes the still slice — redraw immediately when idle. - // If playing, the running loop will pick up the new value on the next frame. if (!playback.isPlaying) redrawOnce(); }, refreshTheme(): void { theme = readTheme(); - // Theme change is immediately visible — redraw the still slice when idle. - // If playing, the running loop will pick up the new theme on the next frame. if (!playback.isPlaying) redrawOnce(); }, @@ -443,6 +641,13 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle { disposed = true; stopLoop(); resizeObserver.disconnect(); + // Release all GL resources so nothing leaks on navigation (spec §5.11). + if (datum) { + gl.deleteTexture(datum.texture); + datum = null; + } + if (vao) gl.deleteVertexArray(vao); + gl.deleteProgram(program); }, }; }