daniel/reasampler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dedup): hash WAV fmt+data only, skipping render-varying metadata chunks

Browse Source 
hashWavContent walks RIFF chunks and feeds only fmt body + data payload
through FNV-1a ('W' domain-separation prefix), skipping bext/iXML/LIST that
REAPER embeds with per-render origination timestamps. Fallback to whole-file
hashBytes for non-WAV. Wired into both capture commit paths.

Legacy entries keep their stored whole-file hash; a re-capture will not
collapse onto a pre-fix entry -- a one-time clean capture resolves it.
This commit is contained in:
Daniel Harvey
2026-07-26 17:09:58 -04:00
parent 6a6d305cf2
commit 78fc5bad94
5 changed files with 291 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_capture_paths.cpp
+160
View File
@@ -7,6 +7,7 @@
#include <cstdint>
#include <cstdio>
#include <cstring> // std::memcpy (for putF32cp in hashWavContent tests)
#include <string>
#include <vector>
@@ -346,6 +347,158 @@ static void testHashBytesLargerBufferDiffersFromSmaller() {
CHECK(hashBytes(short_buf, 2) != hashBytes(long_buf, 3));
}
// --- hashWavContent (WAV-aware dedup hash) -----------------------------------
//
// Verifies that the WAV-content hash hashes only fmt+data (skipping metadata
// chunks like bext/LIST), falls back gracefully for non-WAV input, and that
// different audio data yields different hashes.
// Minimal synthetic WAV builder (mirrors the one in test_wav_trim.cpp).
static void putU16cp(std::vector<std::uint8_t>& b, std::uint16_t v) {
b.push_back(static_cast<std::uint8_t>(v & 0xFF));
b.push_back(static_cast<std::uint8_t>((v >> 8) & 0xFF));
}
static void putU32cp(std::vector<std::uint8_t>& b, std::uint32_t v) {
b.push_back(static_cast<std::uint8_t>(v & 0xFF));
b.push_back(static_cast<std::uint8_t>((v >> 8) & 0xFF));
b.push_back(static_cast<std::uint8_t>((v >> 16) & 0xFF));
b.push_back(static_cast<std::uint8_t>((v >> 24) & 0xFF));
}
static void putTagcp(std::vector<std::uint8_t>& b, const char* t) {
for (int i = 0; i < 4; ++i) b.push_back(static_cast<std::uint8_t>(t[i]));
}
static void putF32cp(std::vector<std::uint8_t>& b, float f) {
std::uint8_t tmp[4];
std::memcpy(tmp, &f, 4);
for (int i = 0; i < 4; ++i) b.push_back(tmp[i]);
}
// Builds a minimal 32-bit-float RIFF/WAVE with an optional metadata chunk
// inserted between "WAVE" and the fmt chunk. `metaChunkBody` and `metaTag` are
// used when `insertMeta` is true. This is the shape REAPER produces: a `bext`
// or `LIST` chunk before fmt with a render-time timestamp in the body.
static std::vector<std::uint8_t> buildTestWav(
std::uint16_t channels, std::uint32_t sampleRate,
const std::vector<float>& samples,
bool insertMeta = false,
const char* metaTag = "bext",
const std::vector<std::uint8_t>& metaBody = {}) {
std::vector<std::uint8_t> chunks;
if (insertMeta && !metaBody.empty()) {
putTagcp(chunks, metaTag);
putU32cp(chunks, static_cast<std::uint32_t>(metaBody.size()));
chunks.insert(chunks.end(), metaBody.begin(), metaBody.end());
if (metaBody.size() & 1u) chunks.push_back(0); // RIFF pad
}
// fmt chunk (16-byte body, IEEE-float tag 3).
const std::uint32_t dataBytes =
static_cast<std::uint32_t>(samples.size() * 4u);
putTagcp(chunks, "fmt ");
putU32cp(chunks, 16);
putU16cp(chunks, 3); // IEEE float
putU16cp(chunks, channels);
putU32cp(chunks, sampleRate);
putU32cp(chunks, sampleRate * channels * 4u); // byteRate
putU16cp(chunks, static_cast<std::uint16_t>(channels * 4)); // blockAlign
putU16cp(chunks, 32); // bitsPerSample
// data chunk.
putTagcp(chunks, "data");
putU32cp(chunks, dataBytes);
for (float f : samples) putF32cp(chunks, f);
std::vector<std::uint8_t> wav;
putTagcp(wav, "RIFF");
putU32cp(wav, static_cast<std::uint32_t>(4 + chunks.size()));
putTagcp(wav, "WAVE");
wav.insert(wav.end(), chunks.begin(), chunks.end());
return wav;
}
static void testHashWavContentIdenticalAudioSameHash() {
// Two WAVs with the same audio but different metadata body -> same hash.
// This is the core dedup regression: REAPER embeds a bext chunk with a
// render-time origination timestamp; without WAV-aware hashing, two renders
// of the same clip produce different file bytes -> no dedup collapse.
const std::vector<float> audio = {0.1f, -0.2f, 0.3f, -0.4f};
std::vector<std::uint8_t> metaA(64, 0x00); // bext body, all zeros (e.g. epoch)
std::vector<std::uint8_t> metaB(64, 0x00);
// Different origination timestamps: first 10 bytes of bext are ASCII date/time.
metaB[0] = '2'; metaB[1] = '0'; metaB[2] = '2'; metaB[3] = '6'; // year
auto wavA = buildTestWav(1, 44100, audio, /*meta=*/true, "bext", metaA);
auto wavB = buildTestWav(1, 44100, audio, /*meta=*/true, "bext", metaB);
// Files must differ (the bext body is different) to prove the test is valid.
CHECK(wavA != wavB);
// But their content hashes must be equal: same fmt+data, different metadata.
CHECK(hashWavContent(wavA) == hashWavContent(wavB));
}
static void testHashWavContentDifferentAudioDifferentHash() {
// Different PCM data -> different content hashes (no false dedup).
const std::vector<float> audioA = {0.5f, 0.5f};
const std::vector<float> audioB = {0.5f, 0.6f}; // last sample differs
auto wavA = buildTestWav(1, 44100, audioA);
auto wavB = buildTestWav(1, 44100, audioB);
CHECK(hashWavContent(wavA) != hashWavContent(wavB));
}
static void testHashWavContentDifferentFmtDifferentHash() {
// Different fmt fields (sample rate) -> different content hashes.
const std::vector<float> audio = {0.1f, 0.2f};
auto wav44 = buildTestWav(1, 44100, audio);
auto wav48 = buildTestWav(1, 48000, audio);
CHECK(hashWavContent(wav44) != hashWavContent(wav48));
}
static void testHashWavContentNonWavFallsBackToWholeFile() {
// Non-WAV bytes -> falls back to whole-file hashBytes; result is non-empty
// and equals hashBytes of the same bytes directly.
std::vector<std::uint8_t> notWav = {0xDE, 0xAD, 0xBE, 0xEF, 0x01, 0x02};
const std::string h = hashWavContent(notWav);
CHECK(!h.empty());
CHECK(h.size() == 16);
CHECK(h == hashBytes(notWav.data(), notWav.size()));
}
static void testHashWavContentEmptyFallsBackToHashBytes() {
// Empty vector -> falls back to whole-file hashBytes (the FNV offset basis).
std::vector<std::uint8_t> empty;
const std::string h = hashWavContent(empty);
CHECK(!h.empty());
CHECK(h.size() == 16);
CHECK(h == hashBytes(nullptr, 0));
}
static void testHashWavContentListMetaSkipped() {
// A LIST/INFO chunk (another common metadata chunk) is likewise skipped.
const std::vector<float> audio = {1.0f, -1.0f, 0.5f};
std::vector<std::uint8_t> listBody = {'I','N','F','O', 'x','x','x','x'};
auto wavClean = buildTestWav(1, 48000, audio);
auto wavList = buildTestWav(1, 48000, audio, true, "LIST", listBody);
// Content hashes must match: only the LIST chunk differs.
CHECK(hashWavContent(wavClean) == hashWavContent(wavList));
}
static void testHashWavContentDomainSeparationFromWholeFile() {
// The content hash ('W'-prefixed) must not accidentally equal the whole-file
// hash of the SAME bytes. This guards against the domain-separation prefix
// being dropped or zeroed out.
const std::vector<float> audio = {0.0f};
auto wav = buildTestWav(1, 44100, audio);
const std::string contentHash = hashWavContent(wav);
const std::string wholeHash = hashBytes(wav.data(), wav.size());
CHECK(contentHash != wholeHash);
}
int main() {
testNormalizeSlashes();
testSanitizeStem();
@@ -376,6 +529,13 @@ int main() {
testHashBytesDistinct();
testHashBytesEmptyBufferIsNonEmpty();
testHashBytesLargerBufferDiffersFromSmaller();
testHashWavContentIdenticalAudioSameHash();
testHashWavContentDifferentAudioDifferentHash();
testHashWavContentDifferentFmtDifferentHash();
testHashWavContentNonWavFallsBackToWholeFile();
testHashWavContentEmptyFallsBackToHashBytes();
testHashWavContentListMetaSkipped();
testHashWavContentDomainSeparationFromWholeFile();
if (g_fail == 0) std::printf("capture_paths: all tests passed\n");
else std::printf("capture_paths: %d CHECK(s) FAILED\n", g_fail);