Fix large CMS upload timeout with idle heartbeat and add per-file progress meter

Replace the 100s default HttpClient timeout (set Timeout=Infinite) with an idle/heartbeat
deadline driven by a ProgressStreamContent wrapper that reports bytes-on-the-wire. Each tick
resets the idle window and advances a MudProgressLinear per upload row. Idle window is
configurable via Upload:IdleTimeoutSeconds (default 90s).
This commit is contained in:
daniel-c-harvey
2026-06-17 11:07:19 -04:00
parent ec3989c354
commit c9c6286571
10 changed files with 328 additions and 8 deletions
+33 -2
View File
@@ -20,19 +20,29 @@ public class CmsTrackService : ICmsTrackService
private const string ContentCmsClientName = "DeepDrft.Content.Cms";
private const string UploadPath = "api/track/upload";
// Idle/heartbeat window: abort an upload only after this long with zero bytes written to the wire.
// The window resets on every progress tick, so a slow-but-moving half-gig upload never trips it;
// a genuinely stalled socket does. Operator-tunable via Upload:IdleTimeoutSeconds.
private const int DefaultIdleTimeoutSeconds = 90;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<CmsTrackService> _logger;
private readonly TimeSpan _uploadIdleTimeout;
public CmsTrackService(
IHttpClientFactory httpClientFactory,
IConfiguration configuration,
ILogger<CmsTrackService> logger)
{
_httpClientFactory = httpClientFactory;
_logger = logger;
var seconds = configuration.GetValue<int?>("Upload:IdleTimeoutSeconds") ?? DefaultIdleTimeoutSeconds;
_uploadIdleTimeout = TimeSpan.FromSeconds(seconds > 0 ? seconds : DefaultIdleTimeoutSeconds);
}
public async Task<ResultContainer<TrackDto>> UploadTrackAsync(
Stream wavStream,
long contentLength,
string fileName,
string contentType,
string trackName,
@@ -46,12 +56,25 @@ public class CmsTrackService : ICmsTrackService
ReleaseType releaseType,
int trackNumber,
ReleaseMedium medium = ReleaseMedium.Cut,
IProgress<long>? progress = null,
CancellationToken ct = default)
{
// Idle/heartbeat cancellation: HttpClient.Timeout is a whole-request cap and cannot express
// "no bytes for N seconds", so the named client runs with InfiniteTimeSpan and the deadline
// lives here. Each ProgressStreamContent tick resets CancelAfter(idle); a stalled socket lets
// the window elapse and cancels the send. Linked to the caller's ct so a page cancel still wins.
using var idleCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
idleCts.CancelAfter(_uploadIdleTimeout);
// Rebuild the multipart container so the boundary is owned by HttpClient and the
// caller-supplied stream (already buffered by the SignalR upload) is the source.
using var multipart = new MultipartFormDataContent();
var wavContent = new StreamContent(wavStream);
var wavContent = new ProgressStreamContent(wavStream, contentLength, written =>
{
// One mechanism, two consumers: advance the UI meter and reset the idle heartbeat.
progress?.Report(written);
idleCts.CancelAfter(_uploadIdleTimeout);
});
wavContent.Headers.ContentType = new MediaTypeHeaderValue(
string.IsNullOrWhiteSpace(contentType) ? "audio/wav" : contentType);
multipart.Add(wavContent, "audioFile", fileName);
@@ -76,7 +99,15 @@ public class CmsTrackService : ICmsTrackService
HttpResponseMessage response;
try
{
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct);
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, idleCts.Token);
}
catch (OperationCanceledException) when (idleCts.IsCancellationRequested && !ct.IsCancellationRequested)
{
// Idle window elapsed with no bytes moving — a stalled connection, not a caller cancel.
_logger.LogWarning("Upload of {TrackName} stalled — no progress for {IdleSeconds}s; aborting.",
trackName, _uploadIdleTimeout.TotalSeconds);
return ResultContainer<TrackDto>.CreateFailResult(
$"Upload stalled — no data transferred for {_uploadIdleTimeout.TotalSeconds:0}s. Please retry.");
}
catch (Exception ex)
{
@@ -21,9 +21,14 @@ public interface ICmsTrackService
/// <paramref name="medium"/> sets the parent release's <see cref="ReleaseMedium"/> when this upload
/// creates the release. The medium is authoritative only on creation — adding a track to an existing
/// release never changes its medium (that is the edit path, <see cref="UpdateAsync"/>).
/// <paramref name="contentLength"/> is the total payload size (the browser file's <c>Size</c>); it
/// sets Content-Length and is the denominator for <paramref name="progress"/>, which reports cumulative
/// bytes pushed to the wire. Each progress tick also resets the idle/heartbeat upload timeout, so a
/// stalled connection aborts without a fixed total-duration cap.
/// </summary>
Task<ResultContainer<TrackDto>> UploadTrackAsync(
Stream wavStream,
long contentLength,
string fileName,
string contentType,
string trackName,
@@ -37,6 +42,7 @@ public interface ICmsTrackService
ReleaseType releaseType,
int trackNumber,
ReleaseMedium medium = ReleaseMedium.Cut,
IProgress<long>? progress = null,
CancellationToken ct = default);
/// <summary>
@@ -0,0 +1,64 @@
using System.Net;
namespace DeepDrftManager.Services;
/// <summary>
/// An <see cref="HttpContent"/> that streams a source stream to the wire while reporting cumulative
/// bytes written after each chunk. This is the single source of truth for both the upload progress
/// meter and the idle/heartbeat timeout: every reported tick advances the UI <em>and</em> resets the
/// idle deadline, so one mechanism feeds both concerns.
/// </summary>
/// <remarks>
/// Wrap the audio payload (not the whole multipart container) so <see cref="TryComputeLength"/>
/// returns the file length and the reported byte counts map directly onto "bytes of this file".
/// </remarks>
public sealed class ProgressStreamContent : HttpContent
{
// 80 KB: large enough to keep the socket fed on a healthy link, small enough that a stalled
// connection trips the idle window without a multi-MB write swallowing the whole heartbeat budget.
private const int CopyBufferSize = 81_920;
private readonly Stream _source;
private readonly long _length;
private readonly Action<long> _onBytesWritten;
/// <param name="source">The payload stream. Read once, sequentially — not seekable-rewound.</param>
/// <param name="length">Total bytes the source will yield; sets Content-Length and the meter denominator.</param>
/// <param name="onBytesWritten">Invoked after each chunk with the cumulative bytes written so far.</param>
public ProgressStreamContent(Stream source, long length, Action<long> onBytesWritten)
{
_source = source;
_length = length;
_onBytesWritten = onBytesWritten;
}
// Token-aware overload (.NET 5+): HttpClient calls this on the send path and passes the request's
// CancellationToken, so the idle-heartbeat CTS aborts an in-flight read/write promptly — not just
// between chunks. The parameterless base override delegates here with CancellationToken.None.
protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context, CancellationToken cancellationToken)
=> CopyAsync(stream, cancellationToken);
protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context)
=> CopyAsync(stream, CancellationToken.None);
private async Task CopyAsync(Stream stream, CancellationToken cancellationToken)
{
var buffer = new byte[CopyBufferSize];
long written = 0;
int read;
while ((read = await _source.ReadAsync(buffer, cancellationToken)) > 0)
{
await stream.WriteAsync(buffer.AsMemory(0, read), cancellationToken);
written += read;
// Report after the bytes are on the wire — a tick means real forward progress, which is
// exactly the signal the idle heartbeat must reset on.
_onBytesWritten(written);
}
}
protected override bool TryComputeLength(out long length)
{
length = _length;
return true;
}
}