fix(dedup): hash WAV fmt+data only, skipping render-varying metadata chunks

hashWavContent walks RIFF chunks and feeds only fmt body + data payload
through FNV-1a ('W' domain-separation prefix), skipping bext/iXML/LIST that
REAPER embeds with per-render origination timestamps. Fallback to whole-file
hashBytes for non-WAV. Wired into both capture commit paths.

Legacy entries keep their stored whole-file hash; a re-capture will not
collapse onto a pre-fix entry -- a one-time clean capture resolves it.
This commit is contained in:
2026-07-26 17:09:58 -04:00
parent 6a6d305cf2
commit 78fc5bad94
5 changed files with 291 additions and 11 deletions
+7 -4
View File
@@ -499,15 +499,18 @@ CaptureResult OfflineRenderBackend::capture(const CaptureRequest& request) {
s.lengthSeconds = request.endSeconds - request.startSeconds;
s.captureTempo = Master_GetTempo(); // BPM at capture time (verified ~4651)
s.tier = Tier::Scratch; // captures land in scratch by default
// Content hash: FNV-1a over the rendered file bytes so hashReferencedElsewhere
// can identify copies in other banks and suppress the last-reference confirm when
// another bank still holds the same file. Best-effort: an unreadable file leaves
// Content hash: WAV-aware FNV-1a over the rendered file's fmt+data chunks so
// hashReferencedElsewhere can identify copies in other banks and suppress the
// last-reference confirm when another bank still holds the same file. Using
// hashWavContent (not the raw hashBytes) skips render-varying metadata chunks
// (bext origination timestamp, iXML, LIST/INFO, etc.) so two renders of identical
// audio collapse to the same hash. Best-effort: an unreadable file leaves
// contentHash empty — the safe, confirm-eliciting direction (bank_model treats
// "" as non-participating in dedup, which is the existing fallback semantics).
{
const std::vector<std::uint8_t> fileBytes = readFileBytes(expectedPath);
if (!fileBytes.empty()) {
s.contentHash = hashBytes(fileBytes.data(), fileBytes.size());
s.contentHash = hashWavContent(fileBytes);
}
}
s.createdTimestamp = static_cast<std::int64_t>(std::time(nullptr));
+91
View File
@@ -3,6 +3,8 @@
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <cstring> // std::memcmp
#include <vector>
namespace reasampler {
@@ -23,6 +25,95 @@ std::string hashBytes(const std::uint8_t* data, std::size_t len) {
return std::string(buf);
}
std::string hashWavContent(const std::vector<std::uint8_t>& bytes) {
// Walk the RIFF/WAVE container and feed only the `fmt ` body and `data` body
// through FNV-1a, prefixed with the domain-separation tag byte 'W' (0x57).
// Any render-varying metadata chunks (bext, iXML, LIST, SMED, etc.) are skipped.
// If the file does not parse as RIFF/WAVE with both fmt and data chunks, fall back
// to whole-file hashBytes (no prefix) so an unrecognized file still gets a hash.
//
// The chunk-walk mirrors wav_trim::parseWavLayout's structure but accumulates
// FNV state instead of recording geometry — no second parser, same logic.
// FNV-1a 64-bit constants (same as hashBytes).
constexpr std::uint64_t kOffsetBasis = 14695981039346656037ULL;
constexpr std::uint64_t kPrime = 1099511628211ULL;
// Minimum viable RIFF/WAVE: "RIFF"(4) size(4) "WAVE"(4) = 12 bytes.
auto tagEq = [&](std::size_t off, const char* tag) -> bool {
return off + 4 <= bytes.size() &&
std::memcmp(bytes.data() + off, tag, 4) == 0;
};
auto readU32LE = [&](std::size_t off) -> std::uint32_t {
return static_cast<std::uint32_t>(bytes[off]) |
(static_cast<std::uint32_t>(bytes[off + 1]) << 8) |
(static_cast<std::uint32_t>(bytes[off + 2]) << 16) |
(static_cast<std::uint32_t>(bytes[off + 3]) << 24);
};
bool isWav = bytes.size() >= 12 &&
tagEq(0, "RIFF") &&
tagEq(8, "WAVE");
if (isWav) {
// Accumulate FNV-1a starting with the domain-separation tag byte 'W'.
std::uint64_t h = kOffsetBasis;
auto feedByte = [&](std::uint8_t b) {
h ^= static_cast<std::uint64_t>(b);
h *= kPrime;
};
bool haveFmt = false;
bool haveData = false;
// Domain-separation prefix: 'W' (0x57) distinguishes a content hash from a
// whole-file hash of different bytes that happen to be the same length.
feedByte(static_cast<std::uint8_t>('W'));
std::size_t pos = 12;
while (pos + 8 <= bytes.size()) {
const std::size_t bodyOffset = pos + 8;
const std::uint32_t bodySize = readU32LE(pos + 4);
if (tagEq(pos, "fmt ")) {
// Feed the entire fmt body (all fields, including format tag, channels,
// sample rate, bits-per-sample — everything that defines the audio format).
if (bodyOffset + bodySize <= bytes.size()) {
for (std::uint32_t i = 0; i < bodySize; ++i)
feedByte(bytes[bodyOffset + i]);
haveFmt = true;
}
} else if (tagEq(pos, "data")) {
// Feed the entire PCM payload.
if (bodyOffset + bodySize <= bytes.size()) {
for (std::uint32_t i = 0; i < bodySize; ++i)
feedByte(bytes[bodyOffset + i]);
haveData = true;
}
}
// All other chunks (bext, iXML, LIST, SMED, cue, etc.) are skipped.
// Advance past this chunk's body, honoring RIFF even-byte padding.
std::size_t advance = bodySize;
if (advance & 1u) ++advance; // RIFF pad byte
if (advance > bytes.size() - bodyOffset) break; // overrun guard
pos = bodyOffset + advance;
}
if (haveFmt && haveData) {
char buf[17];
std::snprintf(buf, sizeof(buf), "%016llx",
static_cast<unsigned long long>(h));
return std::string(buf);
}
// Falls through to whole-file fallback if chunks were missing/malformed.
}
// Fallback: not a parseable RIFF/WAVE — hash the whole file (same as the old
// per-call hashBytes). No prefix tag: identical to hashBytes(data, size).
return hashBytes(bytes.data(), bytes.size());
}
std::string normalizeSlashes(const std::string& path) {
std::string out = path;
for (char& c : out) {
+24
View File
@@ -15,6 +15,7 @@
#include <cstddef>
#include <cstdint>
#include <string>
#include <vector>
namespace reasampler {
@@ -42,6 +43,29 @@ struct BankPaths {
// files would share, but real WAV files are never empty).
std::string hashBytes(const std::uint8_t* data, std::size_t len);
// WAV-aware content hash: hashes only the audio-defining content of a 32-bit-float
// RIFF/WAVE file — the `fmt ` chunk body + the `data` chunk payload — skipping all
// other RIFF chunks (e.g. `bext` origination timestamp, `iXML`, `LIST`/`INFO`, SMED).
//
// WHY: REAPER's offline renderer embeds render-varying metadata chunks (at minimum a
// `bext` chunk containing the origination date/time) even when the format config blob
// requests no BWF metadata. Two renders of identical audio therefore differ in those
// bytes, making whole-file hashes diverge and preventing dedup collapse.
//
// DOMAIN SEPARATION: the FNV-1a input is prefixed with the tag byte 'W' (0x57) before
// the fmt/data bytes are fed in, so a content hash can never equal a whole-file
// hashBytes result for a different file of the same size.
//
// FALLBACK: if `bytes` does not parse as a valid RIFF/WAVE with both a `fmt ` and a
// `data` chunk, the function falls back to whole-file hashBytes (no prefix tag) —
// identical to calling hashBytes(bytes.data(), bytes.size()). This ensures that an
// unrecognized or malformed file still gets a non-empty hash rather than silently
// skipping dedup.
//
// Called by both capture commit paths (offline and realtime) in place of the raw
// hashBytes call.
std::string hashWavContent(const std::vector<std::uint8_t>& bytes);
// Normalizes a path to forward slashes and strips any trailing slash. Empty in
// -> empty out. Pure string transform (does not consult the filesystem).
std::string normalizeSlashes(const std::string& path);
+9 -7
View File
@@ -512,16 +512,18 @@ CaptureResult finalizeRecording(RealtimeCaptureState& st) {
result.status = CaptureStatus::Ok;
result.sample = sampleFromRecordedCapture(cap);
// Content hash: FNV-1a over the (possibly trimmed) bank file bytes so
// hashReferencedElsewhere can identify copies in other banks and suppress the
// last-reference confirm when another bank still holds the same file.
// Best-effort: an unreadable file leaves contentHash empty — the safe,
// confirm-eliciting direction (bank_model treats "" as non-participating).
// Content hash: WAV-aware FNV-1a over the (possibly trimmed) bank file's fmt+data
// chunks so hashReferencedElsewhere can identify copies in other banks and suppress
// the last-reference confirm when another bank still holds the same file. Using
// hashWavContent (not the raw hashBytes) skips render-varying metadata chunks
// (bext origination timestamp, iXML, LIST/INFO, etc.) so two records of identical
// audio collapse to the same hash. Best-effort: an unreadable file leaves
// contentHash empty — the safe, confirm-eliciting direction (bank_model treats
// "" as non-participating).
{
const std::vector<std::uint8_t> fileBytes = readAllBytes(destPath);
if (!fileBytes.empty()) {
result.sample.contentHash =
hashBytes(fileBytes.data(), fileBytes.size());
result.sample.contentHash = hashWavContent(fileBytes);
}
}