fix(dedup): hash WAV fmt+data only, skipping render-varying metadata chunks
hashWavContent walks RIFF chunks and feeds only fmt body + data payload
through FNV-1a ('W' domain-separation prefix), skipping bext/iXML/LIST that
REAPER embeds with per-render origination timestamps. Fallback to whole-file
hashBytes for non-WAV. Wired into both capture commit paths.
Legacy entries keep their stored whole-file hash; a re-capture will not
collapse onto a pre-fix entry -- a one-time clean capture resolves it.
This commit is contained in:
+7
-4
@@ -499,15 +499,18 @@ CaptureResult OfflineRenderBackend::capture(const CaptureRequest& request) {
|
||||
s.lengthSeconds = request.endSeconds - request.startSeconds;
|
||||
s.captureTempo = Master_GetTempo(); // BPM at capture time (verified ~4651)
|
||||
s.tier = Tier::Scratch; // captures land in scratch by default
|
||||
// Content hash: FNV-1a over the rendered file bytes so hashReferencedElsewhere
|
||||
// can identify copies in other banks and suppress the last-reference confirm when
|
||||
// another bank still holds the same file. Best-effort: an unreadable file leaves
|
||||
// Content hash: WAV-aware FNV-1a over the rendered file's fmt+data chunks so
|
||||
// hashReferencedElsewhere can identify copies in other banks and suppress the
|
||||
// last-reference confirm when another bank still holds the same file. Using
|
||||
// hashWavContent (not the raw hashBytes) skips render-varying metadata chunks
|
||||
// (bext origination timestamp, iXML, LIST/INFO, etc.) so two renders of identical
|
||||
// audio collapse to the same hash. Best-effort: an unreadable file leaves
|
||||
// contentHash empty — the safe, confirm-eliciting direction (bank_model treats
|
||||
// "" as non-participating in dedup, which is the existing fallback semantics).
|
||||
{
|
||||
const std::vector<std::uint8_t> fileBytes = readFileBytes(expectedPath);
|
||||
if (!fileBytes.empty()) {
|
||||
s.contentHash = hashBytes(fileBytes.data(), fileBytes.size());
|
||||
s.contentHash = hashWavContent(fileBytes);
|
||||
}
|
||||
}
|
||||
s.createdTimestamp = static_cast<std::int64_t>(std::time(nullptr));
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring> // std::memcmp
|
||||
#include <vector>
|
||||
|
||||
namespace reasampler {
|
||||
|
||||
@@ -23,6 +25,95 @@ std::string hashBytes(const std::uint8_t* data, std::size_t len) {
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
std::string hashWavContent(const std::vector<std::uint8_t>& bytes) {
|
||||
// Walk the RIFF/WAVE container and feed only the `fmt ` body and `data` body
|
||||
// through FNV-1a, prefixed with the domain-separation tag byte 'W' (0x57).
|
||||
// Any render-varying metadata chunks (bext, iXML, LIST, SMED, etc.) are skipped.
|
||||
// If the file does not parse as RIFF/WAVE with both fmt and data chunks, fall back
|
||||
// to whole-file hashBytes (no prefix) so an unrecognized file still gets a hash.
|
||||
//
|
||||
// The chunk-walk mirrors wav_trim::parseWavLayout's structure but accumulates
|
||||
// FNV state instead of recording geometry — no second parser, same logic.
|
||||
|
||||
// FNV-1a 64-bit constants (same as hashBytes).
|
||||
constexpr std::uint64_t kOffsetBasis = 14695981039346656037ULL;
|
||||
constexpr std::uint64_t kPrime = 1099511628211ULL;
|
||||
|
||||
// Minimum viable RIFF/WAVE: "RIFF"(4) size(4) "WAVE"(4) = 12 bytes.
|
||||
auto tagEq = [&](std::size_t off, const char* tag) -> bool {
|
||||
return off + 4 <= bytes.size() &&
|
||||
std::memcmp(bytes.data() + off, tag, 4) == 0;
|
||||
};
|
||||
auto readU32LE = [&](std::size_t off) -> std::uint32_t {
|
||||
return static_cast<std::uint32_t>(bytes[off]) |
|
||||
(static_cast<std::uint32_t>(bytes[off + 1]) << 8) |
|
||||
(static_cast<std::uint32_t>(bytes[off + 2]) << 16) |
|
||||
(static_cast<std::uint32_t>(bytes[off + 3]) << 24);
|
||||
};
|
||||
|
||||
bool isWav = bytes.size() >= 12 &&
|
||||
tagEq(0, "RIFF") &&
|
||||
tagEq(8, "WAVE");
|
||||
|
||||
if (isWav) {
|
||||
// Accumulate FNV-1a starting with the domain-separation tag byte 'W'.
|
||||
std::uint64_t h = kOffsetBasis;
|
||||
auto feedByte = [&](std::uint8_t b) {
|
||||
h ^= static_cast<std::uint64_t>(b);
|
||||
h *= kPrime;
|
||||
};
|
||||
|
||||
bool haveFmt = false;
|
||||
bool haveData = false;
|
||||
|
||||
// Domain-separation prefix: 'W' (0x57) distinguishes a content hash from a
|
||||
// whole-file hash of different bytes that happen to be the same length.
|
||||
feedByte(static_cast<std::uint8_t>('W'));
|
||||
|
||||
std::size_t pos = 12;
|
||||
while (pos + 8 <= bytes.size()) {
|
||||
const std::size_t bodyOffset = pos + 8;
|
||||
const std::uint32_t bodySize = readU32LE(pos + 4);
|
||||
|
||||
if (tagEq(pos, "fmt ")) {
|
||||
// Feed the entire fmt body (all fields, including format tag, channels,
|
||||
// sample rate, bits-per-sample — everything that defines the audio format).
|
||||
if (bodyOffset + bodySize <= bytes.size()) {
|
||||
for (std::uint32_t i = 0; i < bodySize; ++i)
|
||||
feedByte(bytes[bodyOffset + i]);
|
||||
haveFmt = true;
|
||||
}
|
||||
} else if (tagEq(pos, "data")) {
|
||||
// Feed the entire PCM payload.
|
||||
if (bodyOffset + bodySize <= bytes.size()) {
|
||||
for (std::uint32_t i = 0; i < bodySize; ++i)
|
||||
feedByte(bytes[bodyOffset + i]);
|
||||
haveData = true;
|
||||
}
|
||||
}
|
||||
// All other chunks (bext, iXML, LIST, SMED, cue, etc.) are skipped.
|
||||
|
||||
// Advance past this chunk's body, honoring RIFF even-byte padding.
|
||||
std::size_t advance = bodySize;
|
||||
if (advance & 1u) ++advance; // RIFF pad byte
|
||||
if (advance > bytes.size() - bodyOffset) break; // overrun guard
|
||||
pos = bodyOffset + advance;
|
||||
}
|
||||
|
||||
if (haveFmt && haveData) {
|
||||
char buf[17];
|
||||
std::snprintf(buf, sizeof(buf), "%016llx",
|
||||
static_cast<unsigned long long>(h));
|
||||
return std::string(buf);
|
||||
}
|
||||
// Falls through to whole-file fallback if chunks were missing/malformed.
|
||||
}
|
||||
|
||||
// Fallback: not a parseable RIFF/WAVE — hash the whole file (same as the old
|
||||
// per-call hashBytes). No prefix tag: identical to hashBytes(data, size).
|
||||
return hashBytes(bytes.data(), bytes.size());
|
||||
}
|
||||
|
||||
std::string normalizeSlashes(const std::string& path) {
|
||||
std::string out = path;
|
||||
for (char& c : out) {
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace reasampler {
|
||||
|
||||
@@ -42,6 +43,29 @@ struct BankPaths {
|
||||
// files would share, but real WAV files are never empty).
|
||||
std::string hashBytes(const std::uint8_t* data, std::size_t len);
|
||||
|
||||
// WAV-aware content hash: hashes only the audio-defining content of a 32-bit-float
|
||||
// RIFF/WAVE file — the `fmt ` chunk body + the `data` chunk payload — skipping all
|
||||
// other RIFF chunks (e.g. `bext` origination timestamp, `iXML`, `LIST`/`INFO`, SMED).
|
||||
//
|
||||
// WHY: REAPER's offline renderer embeds render-varying metadata chunks (at minimum a
|
||||
// `bext` chunk containing the origination date/time) even when the format config blob
|
||||
// requests no BWF metadata. Two renders of identical audio therefore differ in those
|
||||
// bytes, making whole-file hashes diverge and preventing dedup collapse.
|
||||
//
|
||||
// DOMAIN SEPARATION: the FNV-1a input is prefixed with the tag byte 'W' (0x57) before
|
||||
// the fmt/data bytes are fed in, so a content hash can never equal a whole-file
|
||||
// hashBytes result for a different file of the same size.
|
||||
//
|
||||
// FALLBACK: if `bytes` does not parse as a valid RIFF/WAVE with both a `fmt ` and a
|
||||
// `data` chunk, the function falls back to whole-file hashBytes (no prefix tag) —
|
||||
// identical to calling hashBytes(bytes.data(), bytes.size()). This ensures that an
|
||||
// unrecognized or malformed file still gets a non-empty hash rather than silently
|
||||
// skipping dedup.
|
||||
//
|
||||
// Called by both capture commit paths (offline and realtime) in place of the raw
|
||||
// hashBytes call.
|
||||
std::string hashWavContent(const std::vector<std::uint8_t>& bytes);
|
||||
|
||||
// Normalizes a path to forward slashes and strips any trailing slash. Empty in
|
||||
// -> empty out. Pure string transform (does not consult the filesystem).
|
||||
std::string normalizeSlashes(const std::string& path);
|
||||
|
||||
@@ -512,16 +512,18 @@ CaptureResult finalizeRecording(RealtimeCaptureState& st) {
|
||||
result.status = CaptureStatus::Ok;
|
||||
result.sample = sampleFromRecordedCapture(cap);
|
||||
|
||||
// Content hash: FNV-1a over the (possibly trimmed) bank file bytes so
|
||||
// hashReferencedElsewhere can identify copies in other banks and suppress the
|
||||
// last-reference confirm when another bank still holds the same file.
|
||||
// Best-effort: an unreadable file leaves contentHash empty — the safe,
|
||||
// confirm-eliciting direction (bank_model treats "" as non-participating).
|
||||
// Content hash: WAV-aware FNV-1a over the (possibly trimmed) bank file's fmt+data
|
||||
// chunks so hashReferencedElsewhere can identify copies in other banks and suppress
|
||||
// the last-reference confirm when another bank still holds the same file. Using
|
||||
// hashWavContent (not the raw hashBytes) skips render-varying metadata chunks
|
||||
// (bext origination timestamp, iXML, LIST/INFO, etc.) so two records of identical
|
||||
// audio collapse to the same hash. Best-effort: an unreadable file leaves
|
||||
// contentHash empty — the safe, confirm-eliciting direction (bank_model treats
|
||||
// "" as non-participating).
|
||||
{
|
||||
const std::vector<std::uint8_t> fileBytes = readAllBytes(destPath);
|
||||
if (!fileBytes.empty()) {
|
||||
result.sample.contentHash =
|
||||
hashBytes(fileBytes.data(), fileBytes.size());
|
||||
result.sample.contentHash = hashWavContent(fileBytes);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user