fix(api): stream audio store path to eliminate whole-file buffering (OOM)

Processors now emit a ProcessedAudio plan with a streamed writer instead of a whole-file AudioBinary; vault writes stream via RegisterResourceStreamingAsync. Header parsing is bounded. Wave 2 (waveform/Opus) still re-reads the full file by design.
This commit is contained in:
daniel-c-harvey
2026-06-25 15:27:28 -04:00
parent 1e063d95f4
commit 79bbbd4956
13 changed files with 920 additions and 168 deletions
@@ -178,6 +178,42 @@ public class FileDatabase : DirectoryIndexDirectory, IDisposable
return false;
}
/// <summary>
/// Registers a resource by streaming its bytes into the vault, without materializing the whole
/// file in a managed <c>byte[]</c> (the store-path OOM fix). The caller supplies the index
/// <paramref name="metaData"/> and a <paramref name="writeContent"/> callback that emits bytes to
/// the backing stream. Swallows exceptions and returns false, matching
/// <see cref="RegisterResourceAsync"/>'s contract — callers check the bool.
/// </summary>
public async Task<bool> RegisterResourceStreamingAsync(
string vaultId,
string entryId,
MetaData metaData,
Func<Stream, CancellationToken, Task> writeContent,
CancellationToken cancellationToken = default)
{
try
{
var directoryVault = _vaults.Get(vaultId);
if (directoryVault != null)
{
var written = await directoryVault.AddEntryStreamingAsync(entryId, metaData, writeContent, cancellationToken);
_logger.LogInformation(
"Streamed {Bytes} bytes into vault {VaultId} entry {EntryId} (no whole-file buffer).",
written, vaultId, entryId);
return true;
}
}
catch (Exception ex)
{
// Swallow and return false, matching RegisterResourceAsync. Logged (unlike the buffered
// path) because a streamed write failure can leave a partial backing file worth noticing.
_logger.LogError(ex, "RegisterResourceStreamingAsync failed for vault {VaultId} entry {EntryId}", vaultId, entryId);
}
return false;
}
/// <summary>
/// Removes a resource from a specific vault. Returns null if the vault does not exist,
/// false if the entry was not found, true if the entry was removed. Distinguishing
@@ -56,6 +56,37 @@ public abstract class MediaVault : VaultIndexDirectory
await FileUtils.PutFileAsync(mediaPath, buffer);
}
/// <summary>
/// Streams an entry's bytes into the vault without ever materializing the whole file in memory:
/// records the supplied <paramref name="metaData"/> in the index, then invokes
/// <paramref name="writeContent"/> to emit bytes directly to the backing <see cref="FileStream"/>.
/// The metadata is supplied by the caller (there is no in-memory <see cref="FileBinary"/> to infer
/// it from) — the store path (upload / replace-audio) sources its bytes from a staging file, not a
/// buffer. Returns the number of bytes written, for the caller to log.
///
/// Index-then-file ordering matches <see cref="AddEntryAsync"/>; a mid-write failure therefore
/// leaves an index entry over a partial/missing file, the same exposure the buffered path has on
/// an I/O fault. The caller treats a thrown exception as a failed register.
/// </summary>
public async Task<long> AddEntryStreamingAsync(
string entryId,
MetaData metaData,
Func<Stream, CancellationToken, Task> writeContent,
CancellationToken cancellationToken = default)
{
var mediaPath = GetMediaPathFromEntryKey(entryId, metaData.Extension);
await AddToIndexAsync(entryId, metaData);
await using var fileStream = new FileStream(
mediaPath, FileMode.Create, FileAccess.Write, FileShare.None,
bufferSize: 81920, useAsync: true);
await writeContent(fileStream, cancellationToken);
await fileStream.FlushAsync(cancellationToken);
return fileStream.Length;
}
/// <summary>
/// Retrieves an entry from the vault (MediaVaultType inferred from T)
/// </summary>