daniel/deepdrft
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Merge p10-w1-renderer-fix2 into dev (P10 W1: 2-D datum texture fixes GL_MAX_TEXTURE_SIZE overflow + bridge diagnostics)

Browse Source
This commit is contained in:
daniel-c-harvey
2026-06-15 19:29:11 -04:00
parent 61d53dacff 45bf5e5d37
commit 79e7bb4799
2 changed files with 160 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DeepDrftPublic.Client/Controls/MixWaveformVisualizer.razor.cs
+44 -2
View File
@@ -54,6 +54,18 @@ public partial class MixWaveformVisualizer : ComponentBase, IAsyncDisposable
/// </summary>
[Parameter] public double PlaybackPosition { get; set; }
// Bridge-level diagnostics. Mirrors the JS-side DEBUG flag in MixVisualizer.ts: when true the
// datum-fetch / subscription / playback-coupling seams log to the browser console (prefixed
// `[MixVisualizer]`, same as the JS logs so the two interleave into one timeline). These pinpoint
// which upstream link is broken when the ribbon stays blank — set false once confirmed healthy.
private const bool Debug = true;
private const string Tag = "[MixVisualizer]";
private static void DebugLog(string message)
{
if (Debug) Console.WriteLine($"{Tag} {message}");
}
private ElementReference _canvas;
private IJSObjectReference? _module;
private IJSObjectReference? _handle;
@@ -81,12 +93,19 @@ public partial class MixWaveformVisualizer : ComponentBase, IAsyncDisposable
protected override async Task OnParametersSetAsync()
{
// Subscribe to the player's multicast side-channel once, to re-render on position/play ticks.
// Log whether the cascade is even present: a null PlayerService here means the visualizer
// never couples to playback (no StateChanged events ever reach OnPlayerStateChanged).
if (PlayerService is not null && !ReferenceEquals(PlayerService, _subscribedService))
{
if (_subscribedService is not null)
_subscribedService.StateChanged -= OnPlayerStateChanged;
PlayerService.StateChanged += OnPlayerStateChanged;
_subscribedService = PlayerService;
DebugLog($"subscribed to player StateChanged. ReleaseId={ReleaseId}, TrackId={TrackId?.ToString() ?? "null"}.");
}
else if (PlayerService is null)
{
DebugLog($"NO player cascade — playback will never couple. ReleaseId={ReleaseId}, TrackId={TrackId?.ToString() ?? "null"}.");
}
// ReleaseId is the only fetch input; fetch once per id. Position/zoom/theme changes re-render
@@ -95,11 +114,13 @@ public partial class MixWaveformVisualizer : ComponentBase, IAsyncDisposable
if (_loadedReleaseId == ReleaseId) return;
_loadedReleaseId = ReleaseId;
DebugLog($"fetching mix waveform datum for ReleaseId={ReleaseId}…");
var result = await ReleaseData.GetMixWaveform(ReleaseId);
if (result is { Success: true, Value: { } profile } && profile.BucketCount > 0 && profile.Data.Length > 0)
{
_profile = profile;
_hasDatum = true;
DebugLog($"datum fetch OK — {profile.BucketCount} buckets, base64 length {profile.Data.Length}.");
}
else
{
@@ -107,6 +128,9 @@ public partial class MixWaveformVisualizer : ComponentBase, IAsyncDisposable
// renders its content over a plain background.
_profile = null;
_hasDatum = false;
DebugLog(result.Success
? $"datum fetch returned EMPTY/absent (no stored datum for ReleaseId={ReleaseId}) — backdrop stays blank."
: $"datum fetch FAILED ({result.GetMessage() ?? "unknown error"}) — backdrop stays blank.");
}
// Push the (possibly new) datum to the module if it is already created.
@@ -117,6 +141,10 @@ public partial class MixWaveformVisualizer : ComponentBase, IAsyncDisposable
{
// Position/play-state changed: push it to the module (cheap; no re-fetch, no full re-render
// needed for the canvas itself, but StateHasChanged keeps the slider/visibility in sync).
// Log the gating inputs so a "ribbon never couples" failure shows exactly why: whether the
// player is on THIS track (IsActivePlayer), and what duration/position/play-state it reports.
var currentTrackId = PlayerService?.CurrentTrack is { } ct ? ct.Id.ToString() : "null";
DebugLog($"player StateChanged — IsActivePlayer={IsActivePlayer} (player.CurrentTrack.Id={currentTrackId}, TrackId={TrackId?.ToString() ?? "null"}), player.IsPlaying={PlayerService?.IsPlaying}, player.Duration={PlayerService?.Duration?.ToString("F2") ?? "null"}.");
await PushPlaybackAsync();
StateHasChanged();
});
@@ -173,11 +201,20 @@ public partial class MixWaveformVisualizer : ComponentBase, IAsyncDisposable
if (ReferenceEquals(_profile, _pushedProfile) && haveDuration == _pushedWithDuration)
return;
if (!haveDuration)
{
// The most common stuck state: a datum is loaded but no positive player duration has
// arrived, so we cannot map samples↔time and push an empty datum. Spell out which half
// is missing so the broken link is unambiguous in the console.
DebugLog($"datum push deferred (empty) — profile={(_profile is null ? "null" : "loaded")}, playerDuration={PlayerDurationSeconds?.ToString("F2") ?? "null"} (needs IsActivePlayer + duration>0).");
}
if (haveDuration)
{
// The mix duration must come from the player (no DTO field carries it); without a
// positive duration we cannot map samples↔time, so we hold off until it arrives.
await _handle.InvokeVoidAsync("setDatum", _profile!.Data, PlayerDurationSeconds!.Value);
DebugLog($"datum push (REAL) — base64 length {_profile!.Data.Length}, duration {PlayerDurationSeconds!.Value:F2}s.");
await _handle.InvokeVoidAsync("setDatum", _profile.Data, PlayerDurationSeconds.Value);
}
else
{
@@ -190,12 +227,17 @@ public partial class MixWaveformVisualizer : ComponentBase, IAsyncDisposable
private async Task PushPlaybackAsync()
{
if (_handle is null) return;
if (_handle is null)
{
DebugLog("PushPlayback skipped — module handle not created yet.");
return;
}
// Duration arrives via the player after the initial (duration-less) datum push; the
// idempotent PushDatumAsync re-pushes exactly once when it first becomes available.
await PushDatumAsync();
DebugLog($"setPlayback → position={CurrentPositionSeconds:F2}s, isPlaying={IsPlaying}.");
await _handle.InvokeVoidAsync("setPlayback", CurrentPositionSeconds, IsPlaying);
}
DeepDrftPublic/Interop/visualizer/MixVisualizer.ts
+116 -23
View File
@@ -152,9 +152,21 @@ function parseColor(css: string): [number, number, number] {
// ── Datum: the pre-downloaded loudness profile (spec §F). ────────────────────────
interface Datum {
/** GPU texture holding the loudness samples (R8, 1 row tall), linear-filtered. */
/**
* GPU texture holding the loudness samples (R8). Laid out as a 2-D grid that
* respects GL_MAX_TEXTURE_SIZE (see uploadDatum) rather than a 1×N row, which
* blows past the max texture width for any mix over ~49 s at the ~333 samples/s
* datum density. The shader reads it with texelFetch (integer addressing), so no
* hardware filtering is used — see sampleAt for the manual interpolation.
*/
texture: WebGLTexture;
/** Total mix duration in seconds — needed to map time <-> texture coordinate. */
/** Texture width in texels (samples per row). */
texWidth: number;
/** Texture height in texels (number of rows). */
texHeight: number;
/** Number of real samples in the datum (≤ texWidth*texHeight; the tail row is padded). */
sampleCount: number;
/** Total mix duration in seconds — needed to map time <-> sample index. */
durationSeconds: number;
}
@@ -238,9 +250,10 @@ void main() {
// - So: timeAt(y) = playhead + (screenYTop - nowY) / pixelsPerSecond
//
// Sample at a given time:
// - Texture coord = time / durationSeconds. The datum texture is LINEAR-filtered,
// so the GPU interpolates between samples in hardware — the smooth, no-stair-
// stepping read at every zoom that the design wanted, for free.
// - time / durationSeconds is the normalized position along the mix; multiplied by
// the sample count it becomes a continuous sample index. sampleAt interpolates
// between the two bracketing samples by hand (texelFetch + fract lerp) — see the
// note on its definition for WHY we can't use hardware LINEAR with the 2-D layout.
// - Outside [0, durationSeconds] we force loudness to 0. That is what gives the
// "scrolls in from empty / out to empty" behaviour at the very start and end of
// the mix (spec §A) with no special-casing. (CLAMP_TO_EDGE on the texture would
@@ -257,7 +270,9 @@ uniform float uDurationSeconds; // mix length (per datum)
uniform vec3 uColorAccent; // brightest stop, at the now line (per theme)
uniform vec3 uColorEdge; // dim stop, at the window edges (per theme)
uniform float uHasDatum; // 1.0 when a datum texture is bound, else 0.0
uniform sampler2D uDatum; // loudness profile, R8, 1 row tall, LINEAR-filtered
uniform sampler2D uDatum; // loudness profile, R8, 2-D grid, NEAREST (texelFetch)
uniform int uDatumWidth; // datum texture width in texels (samples per row)
uniform int uDatumSampleCount; // number of real samples (tail row is padded)
out vec4 fragColor;
@@ -265,13 +280,43 @@ const float NOW_ANCHOR_FROM_TOP = ${NOW_ANCHOR_FROM_TOP.toFixed(4)};
const float RIBBON_OPACITY = ${RIBBON_OPACITY.toFixed(4)};
const float RIBBON_HALF_WIDTH_FRAC = ${RIBBON_HALF_WIDTH_FRAC.toFixed(4)};
// Fetch one raw sample by its linear index, mapping the 1-D index onto the 2-D
// texture grid (col = i mod width, row = i / width). texelFetch ignores filtering
// and wrap modes — it reads the exact texel — so the row-wrap layout is invisible
// to the caller.
float fetchSample(int i) {
int col = i % uDatumWidth;
int row = i / uDatumWidth;
return texelFetch(uDatum, ivec2(col, row), 0).r;
}
// Loudness at an absolute mix time, or 0 outside the mix (drives scroll-in/out).
//
// Interpolation note: we cannot lean on hardware LINEAR filtering here. The datum
// is laid across a 2-D grid (1×N would exceed GL_MAX_TEXTURE_SIZE past ~49 s of
// mix), and a hardware 2D-LINEAR read would blend across the row-wrap seam at the
// end of every row — sample[width-1] would wrongly bleed into sample[width] of the
// next row, and bilinear would also pull in the row above/below. So we do the
// linear interpolation by hand along the TIME axis only: bracket the fractional
// sample position with the two neighbouring texels, texelFetch each (each correctly
// mapped to its own 2-D texel), and lerp. Exact, no seam artifact.
//
// Texel-centre convention: this reproduces the predecessor's 1-D LINEAR read bit for
// bit. There, u = t/duration sampled an N-texel LINEAR texture, whose texel centres
// sit at (i+0.5)/N — so u maps to texel-space position u*N - 0.5, interpolating
// between floor() and floor()+1 of that, with CLAMP_TO_EDGE at the ends. We mirror
// exactly that here: the -0.5 and the index clamps to [0, N-1] are the CLAMP_TO_EDGE
// behaviour at both extremes.
float sampleAt(float timeSeconds) {
if (uHasDatum < 0.5) return 0.0;
if (timeSeconds < 0.0 || timeSeconds >= uDurationSeconds) return 0.0;
// u = normalized position along the mix; GPU linear filtering interpolates.
float u = timeSeconds / uDurationSeconds;
return texture(uDatum, vec2(u, 0.5)).r;
float n = float(uDatumSampleCount);
// Continuous texel-space position, half-texel shifted to match LINEAR centres.
float p = (timeSeconds / uDurationSeconds) * n - 0.5;
int i0 = clamp(int(floor(p)), 0, uDatumSampleCount - 1);
int i1 = clamp(int(floor(p)) + 1, 0, uDatumSampleCount - 1);
float f = clamp(p - floor(p), 0.0, 1.0);
return mix(fetchSample(i0), fetchSample(i1), f);
}
void main() {
@@ -386,6 +431,12 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle {
// control-flow narrowing of a captured `const` into nested functions).
const gl: WebGL2RenderingContext = maybeGl;
// GL_MAX_TEXTURE_SIZE is a per-context constant — query it once. The datum is
// laid out across a 2-D grid no wider than this (see uploadDatum); a 1×N row
// would exceed it for any mix over ~49 s at the ~333 samples/s datum density,
// and texImage2D would reject the upload (the bug this fix addresses).
const maxTextureSize: number = gl.getParameter(gl.MAX_TEXTURE_SIZE) as number;
let program: WebGLProgram;
try {
program = linkProgram(gl);
@@ -416,6 +467,8 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle {
colorEdge: gl.getUniformLocation(program, 'uColorEdge'),
hasDatum: gl.getUniformLocation(program, 'uHasDatum'),
datum: gl.getUniformLocation(program, 'uDatum'),
datumWidth: gl.getUniformLocation(program, 'uDatumWidth'),
datumSampleCount: gl.getUniformLocation(program, 'uDatumSampleCount'),
};
for (const [name, loc] of Object.entries(u)) {
if (loc === null && name !== 'timeSeconds') {
@@ -534,11 +587,16 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle {
if (datum) {
gl.uniform1f(u.hasDatum, 1);
gl.uniform1f(u.durationSeconds, datum.durationSeconds);
gl.uniform1i(u.datumWidth, datum.texWidth);
gl.uniform1i(u.datumSampleCount, datum.sampleCount);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, datum.texture);
} else {
gl.uniform1f(u.hasDatum, 0);
gl.uniform1f(u.durationSeconds, 1);
// Keep the divisor safe even though sampleAt early-outs on uHasDatum<0.5.
gl.uniform1i(u.datumWidth, 1);
gl.uniform1i(u.datumSampleCount, 1);
}
// One full-screen triangle (3 vertices), positions from gl_VertexID.
@@ -624,8 +682,22 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle {
}
/**
* Upload the loudness samples as a 1-row R8 texture with LINEAR filtering and
* CLAMP_TO_EDGE. Returns the Datum, or null on an empty/invalid input.
* Upload the loudness samples as a 2-D R8 texture that respects
* GL_MAX_TEXTURE_SIZE, returning the Datum (with the grid dimensions the shader
* needs to map a sample index → texel) or null on empty/invalid input.
*
* Why 2-D and not 1×N: the mix datum runs at ~333 samples/s, so any mix over
* ~49 s produces more samples than GL_MAX_TEXTURE_SIZE (commonly 4096–16384),
* and `texImage2D(…, width=N, height=1, …)` is rejected outright
* ("Requested size at this level is unsupported"), leaving the waveform texture
* uncreated and the ribbon blank. Laying the N samples row-major across a grid
* of width = min(N, safeWidth) keeps every dimension well within the limit.
*
* Filtering: the shader reads with texelFetch and does its own time-axis
* interpolation (see sampleAt), so NEAREST is correct here — hardware LINEAR on
* a 2-D grid would bleed across the row-wrap seam. The final row is zero-padded
* (texture init is zero-filled, then we overwrite the real samples); padding is
* never read because sampleAt clamps the index to sampleCount-1.
*/
function uploadDatum(samplesBase64: string, durationSeconds: number): Datum | null {
if (durationSeconds <= 0 || !samplesBase64) {
@@ -635,34 +707,55 @@ export function create(canvas: HTMLCanvasElement): MixVisualizerHandle {
return null;
}
const samples = decodeSamples(samplesBase64);
if (samples.length === 0) {
const sampleCount = samples.length;
if (sampleCount === 0) {
console.warn(`${TAG} uploadDatum: decoded 0 samples from a non-empty base64 string — datum will not render.`);
return null;
}
debugLog(`uploadDatum — ${samples.length} samples for ${durationSeconds.toFixed(2)}s mix (${(samples.length / durationSeconds).toFixed(1)} samples/s).`);
// Width = min(N, a safe power-of-two cap). The power-of-two cap (4096) is well
// under every real GL_MAX_TEXTURE_SIZE and keeps row arithmetic clean; we
// still clamp it to the actual max in case a driver reports something smaller.
const SAFE_WIDTH = 4096;
const texWidth = Math.min(sampleCount, Math.min(SAFE_WIDTH, maxTextureSize));
const texHeight = Math.ceil(sampleCount / texWidth);
debugLog(
`uploadDatum — ${sampleCount} samples for ${durationSeconds.toFixed(2)}s mix ` +
`(${(sampleCount / durationSeconds).toFixed(1)} samples/s); ` +
`datum texture ${texWidth}x${texHeight} for N=${sampleCount} samples, maxTextureSize=${maxTextureSize}.`,
);
// Pad the final partial row with zeros so the full grid uploads in one call.
const padded = texWidth * texHeight === sampleCount
? samples
: (() => {
const buf = new Uint8Array(texWidth * texHeight);
buf.set(samples);
return buf;
})();
const texture = gl.createTexture();
if (!texture) return null;
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
// 1-byte rows: relax the default 4-byte unpack alignment so an odd-length
// datum uploads without row padding corruption.
// R8 rows are 1-byte-per-texel and texWidth is not guaranteed 4-aligned;
// relax the default 4-byte unpack alignment so rows aren't read with stride
// padding the source array doesn't have.
gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
gl.texImage2D(
gl.TEXTURE_2D, 0, gl.R8,
samples.length, 1, 0,
gl.RED, gl.UNSIGNED_BYTE, samples,
texWidth, texHeight, 0,
gl.RED, gl.UNSIGNED_BYTE, padded,
);
// LINEAR gives the smooth, no-stair-stepping interpolation between samples
// at every zoom — in hardware, for free (spec §2b). CLAMP so a coord at the
// very edge doesn't wrap (out-of-range times are zeroed in-shader anyway).
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
// NEAREST: texelFetch ignores the filter anyway, but be honest about it — the
// shader interpolates manually to avoid the row-wrap seam (see sampleAt).
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.bindTexture(gl.TEXTURE_2D, null);
return { texture, durationSeconds };
return { texture, texWidth, texHeight, sampleCount, durationSeconds };
}
return {