fix(api): stream audio store path to eliminate whole-file buffering (OOM)
Processors now emit a ProcessedAudio plan with a streamed writer instead of a whole-file AudioBinary; vault writes stream via RegisterResourceStreamingAsync. Header parsing is bounded. Wave 2 (waveform/Opus) still re-reads the full file by design.
This commit is contained in:
@@ -178,6 +178,42 @@ public class FileDatabase : DirectoryIndexDirectory, IDisposable
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a resource by streaming its bytes into the vault, without materializing the whole
|
||||
/// file in a managed <c>byte[]</c> (the store-path OOM fix). The caller supplies the index
|
||||
/// <paramref name="metaData"/> and a <paramref name="writeContent"/> callback that emits bytes to
|
||||
/// the backing stream. Swallows exceptions and returns false, matching
|
||||
/// <see cref="RegisterResourceAsync"/>'s contract — callers check the bool.
|
||||
/// </summary>
|
||||
public async Task<bool> RegisterResourceStreamingAsync(
|
||||
string vaultId,
|
||||
string entryId,
|
||||
MetaData metaData,
|
||||
Func<Stream, CancellationToken, Task> writeContent,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var directoryVault = _vaults.Get(vaultId);
|
||||
if (directoryVault != null)
|
||||
{
|
||||
var written = await directoryVault.AddEntryStreamingAsync(entryId, metaData, writeContent, cancellationToken);
|
||||
_logger.LogInformation(
|
||||
"Streamed {Bytes} bytes into vault {VaultId} entry {EntryId} (no whole-file buffer).",
|
||||
written, vaultId, entryId);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Swallow and return false, matching RegisterResourceAsync. Logged (unlike the buffered
|
||||
// path) because a streamed write failure can leave a partial backing file worth noticing.
|
||||
_logger.LogError(ex, "RegisterResourceStreamingAsync failed for vault {VaultId} entry {EntryId}", vaultId, entryId);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a resource from a specific vault. Returns null if the vault does not exist,
|
||||
/// false if the entry was not found, true if the entry was removed. Distinguishing
|
||||
|
||||
@@ -56,6 +56,37 @@ public abstract class MediaVault : VaultIndexDirectory
|
||||
await FileUtils.PutFileAsync(mediaPath, buffer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Streams an entry's bytes into the vault without ever materializing the whole file in memory:
|
||||
/// records the supplied <paramref name="metaData"/> in the index, then invokes
|
||||
/// <paramref name="writeContent"/> to emit bytes directly to the backing <see cref="FileStream"/>.
|
||||
/// The metadata is supplied by the caller (there is no in-memory <see cref="FileBinary"/> to infer
|
||||
/// it from) — the store path (upload / replace-audio) sources its bytes from a staging file, not a
|
||||
/// buffer. Returns the number of bytes written, for the caller to log.
|
||||
///
|
||||
/// Index-then-file ordering matches <see cref="AddEntryAsync"/>; a mid-write failure therefore
|
||||
/// leaves an index entry over a partial/missing file, the same exposure the buffered path has on
|
||||
/// an I/O fault. The caller treats a thrown exception as a failed register.
|
||||
/// </summary>
|
||||
public async Task<long> AddEntryStreamingAsync(
|
||||
string entryId,
|
||||
MetaData metaData,
|
||||
Func<Stream, CancellationToken, Task> writeContent,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var mediaPath = GetMediaPathFromEntryKey(entryId, metaData.Extension);
|
||||
|
||||
await AddToIndexAsync(entryId, metaData);
|
||||
|
||||
await using var fileStream = new FileStream(
|
||||
mediaPath, FileMode.Create, FileAccess.Write, FileShare.None,
|
||||
bufferSize: 81920, useAsync: true);
|
||||
await writeContent(fileStream, cancellationToken);
|
||||
await fileStream.FlushAsync(cancellationToken);
|
||||
|
||||
return fileStream.Length;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves an entry from the vault (MediaVaultType inferred from T)
|
||||
/// </summary>
|
||||
|
||||
Reference in New Issue
Block a user