From 78fc5bad9431a26b7f46e921a78d389014cb087f Mon Sep 17 00:00:00 2001 From: daniel-c-harvey Date: Sun, 26 Jul 2026 17:09:58 -0400 Subject: [PATCH] fix(dedup): hash WAV fmt+data only, skipping render-varying metadata chunks hashWavContent walks RIFF chunks and feeds only fmt body + data payload through FNV-1a ('W' domain-separation prefix), skipping bext/iXML/LIST that REAPER embeds with per-render origination timestamps. Fallback to whole-file hashBytes for non-WAV. Wired into both capture commit paths. Legacy entries keep their stored whole-file hash; a re-capture will not collapse onto a pre-fix entry -- a one-time clean capture resolves it. --- src/capture.cpp | 11 ++- src/capture_paths.cpp | 91 ++++++++++++++++++++ src/capture_paths.h | 24 ++++++ src/capture_realtime.cpp | 16 ++-- tests/test_capture_paths.cpp | 160 +++++++++++++++++++++++++++++++++++ 5 files changed, 291 insertions(+), 11 deletions(-) diff --git a/src/capture.cpp b/src/capture.cpp index 2ded583..118c408 100644 --- a/src/capture.cpp +++ b/src/capture.cpp @@ -499,15 +499,18 @@ CaptureResult OfflineRenderBackend::capture(const CaptureRequest& request) { s.lengthSeconds = request.endSeconds - request.startSeconds; s.captureTempo = Master_GetTempo(); // BPM at capture time (verified ~4651) s.tier = Tier::Scratch; // captures land in scratch by default - // Content hash: FNV-1a over the rendered file bytes so hashReferencedElsewhere - // can identify copies in other banks and suppress the last-reference confirm when - // another bank still holds the same file. Best-effort: an unreadable file leaves + // Content hash: WAV-aware FNV-1a over the rendered file's fmt+data chunks so + // hashReferencedElsewhere can identify copies in other banks and suppress the + // last-reference confirm when another bank still holds the same file. Using + // hashWavContent (not the raw hashBytes) skips render-varying metadata chunks + // (bext origination timestamp, iXML, LIST/INFO, etc.) so two renders of identical + // audio collapse to the same hash. Best-effort: an unreadable file leaves // contentHash empty — the safe, confirm-eliciting direction (bank_model treats // "" as non-participating in dedup, which is the existing fallback semantics). { const std::vector fileBytes = readFileBytes(expectedPath); if (!fileBytes.empty()) { - s.contentHash = hashBytes(fileBytes.data(), fileBytes.size()); + s.contentHash = hashWavContent(fileBytes); } } s.createdTimestamp = static_cast(std::time(nullptr)); diff --git a/src/capture_paths.cpp b/src/capture_paths.cpp index 2c301dd..f328962 100644 --- a/src/capture_paths.cpp +++ b/src/capture_paths.cpp @@ -3,6 +3,8 @@ #include #include #include +#include // std::memcmp +#include namespace reasampler { @@ -23,6 +25,95 @@ std::string hashBytes(const std::uint8_t* data, std::size_t len) { return std::string(buf); } +std::string hashWavContent(const std::vector& bytes) { + // Walk the RIFF/WAVE container and feed only the `fmt ` body and `data` body + // through FNV-1a, prefixed with the domain-separation tag byte 'W' (0x57). + // Any render-varying metadata chunks (bext, iXML, LIST, SMED, etc.) are skipped. + // If the file does not parse as RIFF/WAVE with both fmt and data chunks, fall back + // to whole-file hashBytes (no prefix) so an unrecognized file still gets a hash. + // + // The chunk-walk mirrors wav_trim::parseWavLayout's structure but accumulates + // FNV state instead of recording geometry — no second parser, same logic. + + // FNV-1a 64-bit constants (same as hashBytes). + constexpr std::uint64_t kOffsetBasis = 14695981039346656037ULL; + constexpr std::uint64_t kPrime = 1099511628211ULL; + + // Minimum viable RIFF/WAVE: "RIFF"(4) size(4) "WAVE"(4) = 12 bytes. + auto tagEq = [&](std::size_t off, const char* tag) -> bool { + return off + 4 <= bytes.size() && + std::memcmp(bytes.data() + off, tag, 4) == 0; + }; + auto readU32LE = [&](std::size_t off) -> std::uint32_t { + return static_cast(bytes[off]) | + (static_cast(bytes[off + 1]) << 8) | + (static_cast(bytes[off + 2]) << 16) | + (static_cast(bytes[off + 3]) << 24); + }; + + bool isWav = bytes.size() >= 12 && + tagEq(0, "RIFF") && + tagEq(8, "WAVE"); + + if (isWav) { + // Accumulate FNV-1a starting with the domain-separation tag byte 'W'. + std::uint64_t h = kOffsetBasis; + auto feedByte = [&](std::uint8_t b) { + h ^= static_cast(b); + h *= kPrime; + }; + + bool haveFmt = false; + bool haveData = false; + + // Domain-separation prefix: 'W' (0x57) distinguishes a content hash from a + // whole-file hash of different bytes that happen to be the same length. + feedByte(static_cast('W')); + + std::size_t pos = 12; + while (pos + 8 <= bytes.size()) { + const std::size_t bodyOffset = pos + 8; + const std::uint32_t bodySize = readU32LE(pos + 4); + + if (tagEq(pos, "fmt ")) { + // Feed the entire fmt body (all fields, including format tag, channels, + // sample rate, bits-per-sample — everything that defines the audio format). + if (bodyOffset + bodySize <= bytes.size()) { + for (std::uint32_t i = 0; i < bodySize; ++i) + feedByte(bytes[bodyOffset + i]); + haveFmt = true; + } + } else if (tagEq(pos, "data")) { + // Feed the entire PCM payload. + if (bodyOffset + bodySize <= bytes.size()) { + for (std::uint32_t i = 0; i < bodySize; ++i) + feedByte(bytes[bodyOffset + i]); + haveData = true; + } + } + // All other chunks (bext, iXML, LIST, SMED, cue, etc.) are skipped. + + // Advance past this chunk's body, honoring RIFF even-byte padding. + std::size_t advance = bodySize; + if (advance & 1u) ++advance; // RIFF pad byte + if (advance > bytes.size() - bodyOffset) break; // overrun guard + pos = bodyOffset + advance; + } + + if (haveFmt && haveData) { + char buf[17]; + std::snprintf(buf, sizeof(buf), "%016llx", + static_cast(h)); + return std::string(buf); + } + // Falls through to whole-file fallback if chunks were missing/malformed. + } + + // Fallback: not a parseable RIFF/WAVE — hash the whole file (same as the old + // per-call hashBytes). No prefix tag: identical to hashBytes(data, size). + return hashBytes(bytes.data(), bytes.size()); +} + std::string normalizeSlashes(const std::string& path) { std::string out = path; for (char& c : out) { diff --git a/src/capture_paths.h b/src/capture_paths.h index 42f961a..3286287 100644 --- a/src/capture_paths.h +++ b/src/capture_paths.h @@ -15,6 +15,7 @@ #include #include #include +#include namespace reasampler { @@ -42,6 +43,29 @@ struct BankPaths { // files would share, but real WAV files are never empty). std::string hashBytes(const std::uint8_t* data, std::size_t len); +// WAV-aware content hash: hashes only the audio-defining content of a 32-bit-float +// RIFF/WAVE file — the `fmt ` chunk body + the `data` chunk payload — skipping all +// other RIFF chunks (e.g. `bext` origination timestamp, `iXML`, `LIST`/`INFO`, SMED). +// +// WHY: REAPER's offline renderer embeds render-varying metadata chunks (at minimum a +// `bext` chunk containing the origination date/time) even when the format config blob +// requests no BWF metadata. Two renders of identical audio therefore differ in those +// bytes, making whole-file hashes diverge and preventing dedup collapse. +// +// DOMAIN SEPARATION: the FNV-1a input is prefixed with the tag byte 'W' (0x57) before +// the fmt/data bytes are fed in, so a content hash can never equal a whole-file +// hashBytes result for a different file of the same size. +// +// FALLBACK: if `bytes` does not parse as a valid RIFF/WAVE with both a `fmt ` and a +// `data` chunk, the function falls back to whole-file hashBytes (no prefix tag) — +// identical to calling hashBytes(bytes.data(), bytes.size()). This ensures that an +// unrecognized or malformed file still gets a non-empty hash rather than silently +// skipping dedup. +// +// Called by both capture commit paths (offline and realtime) in place of the raw +// hashBytes call. +std::string hashWavContent(const std::vector& bytes); + // Normalizes a path to forward slashes and strips any trailing slash. Empty in // -> empty out. Pure string transform (does not consult the filesystem). std::string normalizeSlashes(const std::string& path); diff --git a/src/capture_realtime.cpp b/src/capture_realtime.cpp index 0017a11..91ce1eb 100644 --- a/src/capture_realtime.cpp +++ b/src/capture_realtime.cpp @@ -512,16 +512,18 @@ CaptureResult finalizeRecording(RealtimeCaptureState& st) { result.status = CaptureStatus::Ok; result.sample = sampleFromRecordedCapture(cap); - // Content hash: FNV-1a over the (possibly trimmed) bank file bytes so - // hashReferencedElsewhere can identify copies in other banks and suppress the - // last-reference confirm when another bank still holds the same file. - // Best-effort: an unreadable file leaves contentHash empty — the safe, - // confirm-eliciting direction (bank_model treats "" as non-participating). + // Content hash: WAV-aware FNV-1a over the (possibly trimmed) bank file's fmt+data + // chunks so hashReferencedElsewhere can identify copies in other banks and suppress + // the last-reference confirm when another bank still holds the same file. Using + // hashWavContent (not the raw hashBytes) skips render-varying metadata chunks + // (bext origination timestamp, iXML, LIST/INFO, etc.) so two records of identical + // audio collapse to the same hash. Best-effort: an unreadable file leaves + // contentHash empty — the safe, confirm-eliciting direction (bank_model treats + // "" as non-participating). { const std::vector fileBytes = readAllBytes(destPath); if (!fileBytes.empty()) { - result.sample.contentHash = - hashBytes(fileBytes.data(), fileBytes.size()); + result.sample.contentHash = hashWavContent(fileBytes); } } diff --git a/tests/test_capture_paths.cpp b/tests/test_capture_paths.cpp index 98428d1..301e8eb 100644 --- a/tests/test_capture_paths.cpp +++ b/tests/test_capture_paths.cpp @@ -7,6 +7,7 @@ #include #include +#include // std::memcpy (for putF32cp in hashWavContent tests) #include #include @@ -346,6 +347,158 @@ static void testHashBytesLargerBufferDiffersFromSmaller() { CHECK(hashBytes(short_buf, 2) != hashBytes(long_buf, 3)); } +// --- hashWavContent (WAV-aware dedup hash) ----------------------------------- +// +// Verifies that the WAV-content hash hashes only fmt+data (skipping metadata +// chunks like bext/LIST), falls back gracefully for non-WAV input, and that +// different audio data yields different hashes. + +// Minimal synthetic WAV builder (mirrors the one in test_wav_trim.cpp). +static void putU16cp(std::vector& b, std::uint16_t v) { + b.push_back(static_cast(v & 0xFF)); + b.push_back(static_cast((v >> 8) & 0xFF)); +} +static void putU32cp(std::vector& b, std::uint32_t v) { + b.push_back(static_cast(v & 0xFF)); + b.push_back(static_cast((v >> 8) & 0xFF)); + b.push_back(static_cast((v >> 16) & 0xFF)); + b.push_back(static_cast((v >> 24) & 0xFF)); +} +static void putTagcp(std::vector& b, const char* t) { + for (int i = 0; i < 4; ++i) b.push_back(static_cast(t[i])); +} +static void putF32cp(std::vector& b, float f) { + std::uint8_t tmp[4]; + std::memcpy(tmp, &f, 4); + for (int i = 0; i < 4; ++i) b.push_back(tmp[i]); +} + +// Builds a minimal 32-bit-float RIFF/WAVE with an optional metadata chunk +// inserted between "WAVE" and the fmt chunk. `metaChunkBody` and `metaTag` are +// used when `insertMeta` is true. This is the shape REAPER produces: a `bext` +// or `LIST` chunk before fmt with a render-time timestamp in the body. +static std::vector buildTestWav( + std::uint16_t channels, std::uint32_t sampleRate, + const std::vector& samples, + bool insertMeta = false, + const char* metaTag = "bext", + const std::vector& metaBody = {}) { + + std::vector chunks; + + if (insertMeta && !metaBody.empty()) { + putTagcp(chunks, metaTag); + putU32cp(chunks, static_cast(metaBody.size())); + chunks.insert(chunks.end(), metaBody.begin(), metaBody.end()); + if (metaBody.size() & 1u) chunks.push_back(0); // RIFF pad + } + + // fmt chunk (16-byte body, IEEE-float tag 3). + const std::uint32_t dataBytes = + static_cast(samples.size() * 4u); + putTagcp(chunks, "fmt "); + putU32cp(chunks, 16); + putU16cp(chunks, 3); // IEEE float + putU16cp(chunks, channels); + putU32cp(chunks, sampleRate); + putU32cp(chunks, sampleRate * channels * 4u); // byteRate + putU16cp(chunks, static_cast(channels * 4)); // blockAlign + putU16cp(chunks, 32); // bitsPerSample + + // data chunk. + putTagcp(chunks, "data"); + putU32cp(chunks, dataBytes); + for (float f : samples) putF32cp(chunks, f); + + std::vector wav; + putTagcp(wav, "RIFF"); + putU32cp(wav, static_cast(4 + chunks.size())); + putTagcp(wav, "WAVE"); + wav.insert(wav.end(), chunks.begin(), chunks.end()); + return wav; +} + +static void testHashWavContentIdenticalAudioSameHash() { + // Two WAVs with the same audio but different metadata body -> same hash. + // This is the core dedup regression: REAPER embeds a bext chunk with a + // render-time origination timestamp; without WAV-aware hashing, two renders + // of the same clip produce different file bytes -> no dedup collapse. + const std::vector audio = {0.1f, -0.2f, 0.3f, -0.4f}; + std::vector metaA(64, 0x00); // bext body, all zeros (e.g. epoch) + std::vector metaB(64, 0x00); + // Different origination timestamps: first 10 bytes of bext are ASCII date/time. + metaB[0] = '2'; metaB[1] = '0'; metaB[2] = '2'; metaB[3] = '6'; // year + + auto wavA = buildTestWav(1, 44100, audio, /*meta=*/true, "bext", metaA); + auto wavB = buildTestWav(1, 44100, audio, /*meta=*/true, "bext", metaB); + + // Files must differ (the bext body is different) to prove the test is valid. + CHECK(wavA != wavB); + + // But their content hashes must be equal: same fmt+data, different metadata. + CHECK(hashWavContent(wavA) == hashWavContent(wavB)); +} + +static void testHashWavContentDifferentAudioDifferentHash() { + // Different PCM data -> different content hashes (no false dedup). + const std::vector audioA = {0.5f, 0.5f}; + const std::vector audioB = {0.5f, 0.6f}; // last sample differs + + auto wavA = buildTestWav(1, 44100, audioA); + auto wavB = buildTestWav(1, 44100, audioB); + + CHECK(hashWavContent(wavA) != hashWavContent(wavB)); +} + +static void testHashWavContentDifferentFmtDifferentHash() { + // Different fmt fields (sample rate) -> different content hashes. + const std::vector audio = {0.1f, 0.2f}; + auto wav44 = buildTestWav(1, 44100, audio); + auto wav48 = buildTestWav(1, 48000, audio); + CHECK(hashWavContent(wav44) != hashWavContent(wav48)); +} + +static void testHashWavContentNonWavFallsBackToWholeFile() { + // Non-WAV bytes -> falls back to whole-file hashBytes; result is non-empty + // and equals hashBytes of the same bytes directly. + std::vector notWav = {0xDE, 0xAD, 0xBE, 0xEF, 0x01, 0x02}; + const std::string h = hashWavContent(notWav); + CHECK(!h.empty()); + CHECK(h.size() == 16); + CHECK(h == hashBytes(notWav.data(), notWav.size())); +} + +static void testHashWavContentEmptyFallsBackToHashBytes() { + // Empty vector -> falls back to whole-file hashBytes (the FNV offset basis). + std::vector empty; + const std::string h = hashWavContent(empty); + CHECK(!h.empty()); + CHECK(h.size() == 16); + CHECK(h == hashBytes(nullptr, 0)); +} + +static void testHashWavContentListMetaSkipped() { + // A LIST/INFO chunk (another common metadata chunk) is likewise skipped. + const std::vector audio = {1.0f, -1.0f, 0.5f}; + std::vector listBody = {'I','N','F','O', 'x','x','x','x'}; + auto wavClean = buildTestWav(1, 48000, audio); + auto wavList = buildTestWav(1, 48000, audio, true, "LIST", listBody); + + // Content hashes must match: only the LIST chunk differs. + CHECK(hashWavContent(wavClean) == hashWavContent(wavList)); +} + +static void testHashWavContentDomainSeparationFromWholeFile() { + // The content hash ('W'-prefixed) must not accidentally equal the whole-file + // hash of the SAME bytes. This guards against the domain-separation prefix + // being dropped or zeroed out. + const std::vector audio = {0.0f}; + auto wav = buildTestWav(1, 44100, audio); + const std::string contentHash = hashWavContent(wav); + const std::string wholeHash = hashBytes(wav.data(), wav.size()); + CHECK(contentHash != wholeHash); +} + int main() { testNormalizeSlashes(); testSanitizeStem(); @@ -376,6 +529,13 @@ int main() { testHashBytesDistinct(); testHashBytesEmptyBufferIsNonEmpty(); testHashBytesLargerBufferDiffersFromSmaller(); + testHashWavContentIdenticalAudioSameHash(); + testHashWavContentDifferentAudioDifferentHash(); + testHashWavContentDifferentFmtDifferentHash(); + testHashWavContentNonWavFallsBackToWholeFile(); + testHashWavContentEmptyFallsBackToHashBytes(); + testHashWavContentListMetaSkipped(); + testHashWavContentDomainSeparationFromWholeFile(); if (g_fail == 0) std::printf("capture_paths: all tests passed\n"); else std::printf("capture_paths: %d CHECK(s) FAILED\n", g_fail);