Merge fix: WAV-aware content hash — identical captures now collapse despite bext timestamps

src/capture.cpp

@@ -499,15 +499,18 @@ CaptureResult OfflineRenderBackend::capture(const CaptureRequest& request) {
     s.lengthSeconds = request.endSeconds - request.startSeconds;
     s.captureTempo  = Master_GetTempo();   // BPM at capture time (verified ~4651)
     s.tier          = Tier::Scratch;       // captures land in scratch by default
-    // Content hash: FNV-1a over the rendered file bytes so hashReferencedElsewhere
+    // Content hash: WAV-aware FNV-1a over the rendered file's fmt+data chunks so
-    // can identify copies in other banks and suppress the last-reference confirm when
+    // hashReferencedElsewhere can identify copies in other banks and suppress the
-    // another bank still holds the same file. Best-effort: an unreadable file leaves
+    // last-reference confirm when another bank still holds the same file. Using
+    // hashWavContent (not the raw hashBytes) skips render-varying metadata chunks
+    // (bext origination timestamp, iXML, LIST/INFO, etc.) so two renders of identical
+    // audio collapse to the same hash. Best-effort: an unreadable file leaves
     // contentHash empty — the safe, confirm-eliciting direction (bank_model treats
     // "" as non-participating in dedup, which is the existing fallback semantics).
     {
         const std::vector<std::uint8_t> fileBytes = readFileBytes(expectedPath);
         if (!fileBytes.empty()) {
-            s.contentHash = hashBytes(fileBytes.data(), fileBytes.size());
+            s.contentHash = hashWavContent(fileBytes);
         }
     }
     s.createdTimestamp = static_cast<std::int64_t>(std::time(nullptr));

src/capture_paths.cpp

@@ -3,6 +3,8 @@
 #include <cassert>
 #include <cstdint>
 #include <cstdio>
+#include <cstring>  // std::memcmp
+#include <vector>
 
 namespace reasampler {
 
@@ -23,6 +25,95 @@ std::string hashBytes(const std::uint8_t* data, std::size_t len) {
     return std::string(buf);
 }
 
+std::string hashWavContent(const std::vector<std::uint8_t>& bytes) {
+    // Walk the RIFF/WAVE container and feed only the `fmt ` body and `data` body
+    // through FNV-1a, prefixed with the domain-separation tag byte 'W' (0x57).
+    // Any render-varying metadata chunks (bext, iXML, LIST, SMED, etc.) are skipped.
+    // If the file does not parse as RIFF/WAVE with both fmt and data chunks, fall back
+    // to whole-file hashBytes (no prefix) so an unrecognized file still gets a hash.
+    //
+    // The chunk-walk mirrors wav_trim::parseWavLayout's structure but accumulates
+    // FNV state instead of recording geometry — no second parser, same logic.
+
+    // FNV-1a 64-bit constants (same as hashBytes).
+    constexpr std::uint64_t kOffsetBasis = 14695981039346656037ULL;
+    constexpr std::uint64_t kPrime       = 1099511628211ULL;
+
+    // Minimum viable RIFF/WAVE: "RIFF"(4) size(4) "WAVE"(4) = 12 bytes.
+    auto tagEq = [&](std::size_t off, const char* tag) -> bool {
+        return off + 4 <= bytes.size() &&
+               std::memcmp(bytes.data() + off, tag, 4) == 0;
+    };
+    auto readU32LE = [&](std::size_t off) -> std::uint32_t {
+        return static_cast<std::uint32_t>(bytes[off]) |
+               (static_cast<std::uint32_t>(bytes[off + 1]) << 8) |
+               (static_cast<std::uint32_t>(bytes[off + 2]) << 16) |
+               (static_cast<std::uint32_t>(bytes[off + 3]) << 24);
+    };
+
+    bool isWav = bytes.size() >= 12 &&
+                 tagEq(0, "RIFF") &&
+                 tagEq(8, "WAVE");
+
+    if (isWav) {
+        // Accumulate FNV-1a starting with the domain-separation tag byte 'W'.
+        std::uint64_t h = kOffsetBasis;
+        auto feedByte = [&](std::uint8_t b) {
+            h ^= static_cast<std::uint64_t>(b);
+            h *= kPrime;
+        };
+
+        bool haveFmt  = false;
+        bool haveData = false;
+
+        // Domain-separation prefix: 'W' (0x57) distinguishes a content hash from a
+        // whole-file hash of different bytes that happen to be the same length.
+        feedByte(static_cast<std::uint8_t>('W'));
+
+        std::size_t pos = 12;
+        while (pos + 8 <= bytes.size()) {
+            const std::size_t bodyOffset = pos + 8;
+            const std::uint32_t bodySize = readU32LE(pos + 4);
+
+            if (tagEq(pos, "fmt ")) {
+                // Feed the entire fmt body (all fields, including format tag, channels,
+                // sample rate, bits-per-sample — everything that defines the audio format).
+                if (bodyOffset + bodySize <= bytes.size()) {
+                    for (std::uint32_t i = 0; i < bodySize; ++i)
+                        feedByte(bytes[bodyOffset + i]);
+                    haveFmt = true;
+                }
+            } else if (tagEq(pos, "data")) {
+                // Feed the entire PCM payload.
+                if (bodyOffset + bodySize <= bytes.size()) {
+                    for (std::uint32_t i = 0; i < bodySize; ++i)
+                        feedByte(bytes[bodyOffset + i]);
+                    haveData = true;
+                }
+            }
+            // All other chunks (bext, iXML, LIST, SMED, cue, etc.) are skipped.
+
+            // Advance past this chunk's body, honoring RIFF even-byte padding.
+            std::size_t advance = bodySize;
+            if (advance & 1u) ++advance;  // RIFF pad byte
+            if (advance > bytes.size() - bodyOffset) break;  // overrun guard
+            pos = bodyOffset + advance;
+        }
+
+        if (haveFmt && haveData) {
+            char buf[17];
+            std::snprintf(buf, sizeof(buf), "%016llx",
+                          static_cast<unsigned long long>(h));
+            return std::string(buf);
+        }
+        // Falls through to whole-file fallback if chunks were missing/malformed.
+    }
+
+    // Fallback: not a parseable RIFF/WAVE — hash the whole file (same as the old
+    // per-call hashBytes). No prefix tag: identical to hashBytes(data, size).
+    return hashBytes(bytes.data(), bytes.size());
+}
+
 std::string normalizeSlashes(const std::string& path) {
     std::string out = path;
     for (char& c : out) {

src/capture_paths.h

@@ -15,6 +15,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <string>
+#include <vector>
 
 namespace reasampler {
 
@@ -42,6 +43,29 @@ struct BankPaths {
 // files would share, but real WAV files are never empty).
 std::string hashBytes(const std::uint8_t* data, std::size_t len);
 
+// WAV-aware content hash: hashes only the audio-defining content of a 32-bit-float
+// RIFF/WAVE file — the `fmt ` chunk body + the `data` chunk payload — skipping all
+// other RIFF chunks (e.g. `bext` origination timestamp, `iXML`, `LIST`/`INFO`, SMED).
+//
+// WHY: REAPER's offline renderer embeds render-varying metadata chunks (at minimum a
+// `bext` chunk containing the origination date/time) even when the format config blob
+// requests no BWF metadata. Two renders of identical audio therefore differ in those
+// bytes, making whole-file hashes diverge and preventing dedup collapse.
+//
+// DOMAIN SEPARATION: the FNV-1a input is prefixed with the tag byte 'W' (0x57) before
+// the fmt/data bytes are fed in, so a content hash can never equal a whole-file
+// hashBytes result for a different file of the same size.
+//
+// FALLBACK: if `bytes` does not parse as a valid RIFF/WAVE with both a `fmt ` and a
+// `data` chunk, the function falls back to whole-file hashBytes (no prefix tag) —
+// identical to calling hashBytes(bytes.data(), bytes.size()). This ensures that an
+// unrecognized or malformed file still gets a non-empty hash rather than silently
+// skipping dedup.
+//
+// Called by both capture commit paths (offline and realtime) in place of the raw
+// hashBytes call.
+std::string hashWavContent(const std::vector<std::uint8_t>& bytes);
+
 // Normalizes a path to forward slashes and strips any trailing slash. Empty in
 // -> empty out. Pure string transform (does not consult the filesystem).
 std::string normalizeSlashes(const std::string& path);

src/capture_realtime.cpp

@@ -512,16 +512,18 @@ CaptureResult finalizeRecording(RealtimeCaptureState& st) {
     result.status  = CaptureStatus::Ok;
     result.sample  = sampleFromRecordedCapture(cap);
 
-    // Content hash: FNV-1a over the (possibly trimmed) bank file bytes so
+    // Content hash: WAV-aware FNV-1a over the (possibly trimmed) bank file's fmt+data
-    // hashReferencedElsewhere can identify copies in other banks and suppress the
+    // chunks so hashReferencedElsewhere can identify copies in other banks and suppress
-    // last-reference confirm when another bank still holds the same file.
+    // the last-reference confirm when another bank still holds the same file. Using
-    // Best-effort: an unreadable file leaves contentHash empty — the safe,
+    // hashWavContent (not the raw hashBytes) skips render-varying metadata chunks
-    // confirm-eliciting direction (bank_model treats "" as non-participating).
+    // (bext origination timestamp, iXML, LIST/INFO, etc.) so two records of identical
+    // audio collapse to the same hash. Best-effort: an unreadable file leaves
+    // contentHash empty — the safe, confirm-eliciting direction (bank_model treats
+    // "" as non-participating).
     {
         const std::vector<std::uint8_t> fileBytes = readAllBytes(destPath);
         if (!fileBytes.empty()) {
-            result.sample.contentHash =
+            result.sample.contentHash = hashWavContent(fileBytes);
-                hashBytes(fileBytes.data(), fileBytes.size());
         }
     }
 

tests/test_capture_paths.cpp

@@ -7,6 +7,7 @@
 
 #include <cstdint>
 #include <cstdio>
+#include <cstring>  // std::memcpy (for putF32cp in hashWavContent tests)
 #include <string>
 #include <vector>
 
@@ -346,6 +347,158 @@ static void testHashBytesLargerBufferDiffersFromSmaller() {
     CHECK(hashBytes(short_buf, 2) != hashBytes(long_buf, 3));
 }
 
+// --- hashWavContent (WAV-aware dedup hash) -----------------------------------
+//
+// Verifies that the WAV-content hash hashes only fmt+data (skipping metadata
+// chunks like bext/LIST), falls back gracefully for non-WAV input, and that
+// different audio data yields different hashes.
+
+// Minimal synthetic WAV builder (mirrors the one in test_wav_trim.cpp).
+static void putU16cp(std::vector<std::uint8_t>& b, std::uint16_t v) {
+    b.push_back(static_cast<std::uint8_t>(v & 0xFF));
+    b.push_back(static_cast<std::uint8_t>((v >> 8) & 0xFF));
+}
+static void putU32cp(std::vector<std::uint8_t>& b, std::uint32_t v) {
+    b.push_back(static_cast<std::uint8_t>(v & 0xFF));
+    b.push_back(static_cast<std::uint8_t>((v >> 8) & 0xFF));
+    b.push_back(static_cast<std::uint8_t>((v >> 16) & 0xFF));
+    b.push_back(static_cast<std::uint8_t>((v >> 24) & 0xFF));
+}
+static void putTagcp(std::vector<std::uint8_t>& b, const char* t) {
+    for (int i = 0; i < 4; ++i) b.push_back(static_cast<std::uint8_t>(t[i]));
+}
+static void putF32cp(std::vector<std::uint8_t>& b, float f) {
+    std::uint8_t tmp[4];
+    std::memcpy(tmp, &f, 4);
+    for (int i = 0; i < 4; ++i) b.push_back(tmp[i]);
+}
+
+// Builds a minimal 32-bit-float RIFF/WAVE with an optional metadata chunk
+// inserted between "WAVE" and the fmt chunk. `metaChunkBody` and `metaTag` are
+// used when `insertMeta` is true. This is the shape REAPER produces: a `bext`
+// or `LIST` chunk before fmt with a render-time timestamp in the body.
+static std::vector<std::uint8_t> buildTestWav(
+        std::uint16_t channels, std::uint32_t sampleRate,
+        const std::vector<float>& samples,
+        bool insertMeta = false,
+        const char* metaTag = "bext",
+        const std::vector<std::uint8_t>& metaBody = {}) {
+
+    std::vector<std::uint8_t> chunks;
+
+    if (insertMeta && !metaBody.empty()) {
+        putTagcp(chunks, metaTag);
+        putU32cp(chunks, static_cast<std::uint32_t>(metaBody.size()));
+        chunks.insert(chunks.end(), metaBody.begin(), metaBody.end());
+        if (metaBody.size() & 1u) chunks.push_back(0);  // RIFF pad
+    }
+
+    // fmt chunk (16-byte body, IEEE-float tag 3).
+    const std::uint32_t dataBytes =
+        static_cast<std::uint32_t>(samples.size() * 4u);
+    putTagcp(chunks, "fmt ");
+    putU32cp(chunks, 16);
+    putU16cp(chunks, 3);                                            // IEEE float
+    putU16cp(chunks, channels);
+    putU32cp(chunks, sampleRate);
+    putU32cp(chunks, sampleRate * channels * 4u);                   // byteRate
+    putU16cp(chunks, static_cast<std::uint16_t>(channels * 4));    // blockAlign
+    putU16cp(chunks, 32);                                           // bitsPerSample
+
+    // data chunk.
+    putTagcp(chunks, "data");
+    putU32cp(chunks, dataBytes);
+    for (float f : samples) putF32cp(chunks, f);
+
+    std::vector<std::uint8_t> wav;
+    putTagcp(wav, "RIFF");
+    putU32cp(wav, static_cast<std::uint32_t>(4 + chunks.size()));
+    putTagcp(wav, "WAVE");
+    wav.insert(wav.end(), chunks.begin(), chunks.end());
+    return wav;
+}
+
+static void testHashWavContentIdenticalAudioSameHash() {
+    // Two WAVs with the same audio but different metadata body -> same hash.
+    // This is the core dedup regression: REAPER embeds a bext chunk with a
+    // render-time origination timestamp; without WAV-aware hashing, two renders
+    // of the same clip produce different file bytes -> no dedup collapse.
+    const std::vector<float> audio = {0.1f, -0.2f, 0.3f, -0.4f};
+    std::vector<std::uint8_t> metaA(64, 0x00);  // bext body, all zeros (e.g. epoch)
+    std::vector<std::uint8_t> metaB(64, 0x00);
+    // Different origination timestamps: first 10 bytes of bext are ASCII date/time.
+    metaB[0] = '2'; metaB[1] = '0'; metaB[2] = '2'; metaB[3] = '6';  // year
+
+    auto wavA = buildTestWav(1, 44100, audio, /*meta=*/true, "bext", metaA);
+    auto wavB = buildTestWav(1, 44100, audio, /*meta=*/true, "bext", metaB);
+
+    // Files must differ (the bext body is different) to prove the test is valid.
+    CHECK(wavA != wavB);
+
+    // But their content hashes must be equal: same fmt+data, different metadata.
+    CHECK(hashWavContent(wavA) == hashWavContent(wavB));
+}
+
+static void testHashWavContentDifferentAudioDifferentHash() {
+    // Different PCM data -> different content hashes (no false dedup).
+    const std::vector<float> audioA = {0.5f, 0.5f};
+    const std::vector<float> audioB = {0.5f, 0.6f};  // last sample differs
+
+    auto wavA = buildTestWav(1, 44100, audioA);
+    auto wavB = buildTestWav(1, 44100, audioB);
+
+    CHECK(hashWavContent(wavA) != hashWavContent(wavB));
+}
+
+static void testHashWavContentDifferentFmtDifferentHash() {
+    // Different fmt fields (sample rate) -> different content hashes.
+    const std::vector<float> audio = {0.1f, 0.2f};
+    auto wav44 = buildTestWav(1, 44100, audio);
+    auto wav48 = buildTestWav(1, 48000, audio);
+    CHECK(hashWavContent(wav44) != hashWavContent(wav48));
+}
+
+static void testHashWavContentNonWavFallsBackToWholeFile() {
+    // Non-WAV bytes -> falls back to whole-file hashBytes; result is non-empty
+    // and equals hashBytes of the same bytes directly.
+    std::vector<std::uint8_t> notWav = {0xDE, 0xAD, 0xBE, 0xEF, 0x01, 0x02};
+    const std::string h = hashWavContent(notWav);
+    CHECK(!h.empty());
+    CHECK(h.size() == 16);
+    CHECK(h == hashBytes(notWav.data(), notWav.size()));
+}
+
+static void testHashWavContentEmptyFallsBackToHashBytes() {
+    // Empty vector -> falls back to whole-file hashBytes (the FNV offset basis).
+    std::vector<std::uint8_t> empty;
+    const std::string h = hashWavContent(empty);
+    CHECK(!h.empty());
+    CHECK(h.size() == 16);
+    CHECK(h == hashBytes(nullptr, 0));
+}
+
+static void testHashWavContentListMetaSkipped() {
+    // A LIST/INFO chunk (another common metadata chunk) is likewise skipped.
+    const std::vector<float> audio = {1.0f, -1.0f, 0.5f};
+    std::vector<std::uint8_t> listBody = {'I','N','F','O', 'x','x','x','x'};
+    auto wavClean = buildTestWav(1, 48000, audio);
+    auto wavList  = buildTestWav(1, 48000, audio, true, "LIST", listBody);
+
+    // Content hashes must match: only the LIST chunk differs.
+    CHECK(hashWavContent(wavClean) == hashWavContent(wavList));
+}
+
+static void testHashWavContentDomainSeparationFromWholeFile() {
+    // The content hash ('W'-prefixed) must not accidentally equal the whole-file
+    // hash of the SAME bytes. This guards against the domain-separation prefix
+    // being dropped or zeroed out.
+    const std::vector<float> audio = {0.0f};
+    auto wav = buildTestWav(1, 44100, audio);
+    const std::string contentHash  = hashWavContent(wav);
+    const std::string wholeHash    = hashBytes(wav.data(), wav.size());
+    CHECK(contentHash != wholeHash);
+}
+
 int main() {
     testNormalizeSlashes();
     testSanitizeStem();
@@ -376,6 +529,13 @@ int main() {
     testHashBytesDistinct();
     testHashBytesEmptyBufferIsNonEmpty();
     testHashBytesLargerBufferDiffersFromSmaller();
+    testHashWavContentIdenticalAudioSameHash();
+    testHashWavContentDifferentAudioDifferentHash();
+    testHashWavContentDifferentFmtDifferentHash();
+    testHashWavContentNonWavFallsBackToWholeFile();
+    testHashWavContentEmptyFallsBackToHashBytes();
+    testHashWavContentListMetaSkipped();
+    testHashWavContentDomainSeparationFromWholeFile();
 
     if (g_fail == 0) std::printf("capture_paths: all tests passed\n");
     else std::printf("capture_paths: %d CHECK(s) FAILED\n", g_fail);