Fix large CMS upload timeout with idle heartbeat and add per-file progress meter
Replace the 100s default HttpClient timeout (set Timeout=Infinite) with an idle/heartbeat deadline driven by a ProgressStreamContent wrapper that reports bytes-on-the-wire. Each tick resets the idle window and advances a MudProgressLinear per upload row. Idle window is configurable via Upload:IdleTimeoutSeconds (default 90s).
This commit is contained in:
@@ -20,19 +20,29 @@ public class CmsTrackService : ICmsTrackService
|
||||
private const string ContentCmsClientName = "DeepDrft.Content.Cms";
|
||||
private const string UploadPath = "api/track/upload";
|
||||
|
||||
// Idle/heartbeat window: abort an upload only after this long with zero bytes written to the wire.
|
||||
// The window resets on every progress tick, so a slow-but-moving half-gig upload never trips it;
|
||||
// a genuinely stalled socket does. Operator-tunable via Upload:IdleTimeoutSeconds.
|
||||
private const int DefaultIdleTimeoutSeconds = 90;
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly ILogger<CmsTrackService> _logger;
|
||||
private readonly TimeSpan _uploadIdleTimeout;
|
||||
|
||||
public CmsTrackService(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IConfiguration configuration,
|
||||
ILogger<CmsTrackService> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_logger = logger;
|
||||
var seconds = configuration.GetValue<int?>("Upload:IdleTimeoutSeconds") ?? DefaultIdleTimeoutSeconds;
|
||||
_uploadIdleTimeout = TimeSpan.FromSeconds(seconds > 0 ? seconds : DefaultIdleTimeoutSeconds);
|
||||
}
|
||||
|
||||
public async Task<ResultContainer<TrackDto>> UploadTrackAsync(
|
||||
Stream wavStream,
|
||||
long contentLength,
|
||||
string fileName,
|
||||
string contentType,
|
||||
string trackName,
|
||||
@@ -46,12 +56,25 @@ public class CmsTrackService : ICmsTrackService
|
||||
ReleaseType releaseType,
|
||||
int trackNumber,
|
||||
ReleaseMedium medium = ReleaseMedium.Cut,
|
||||
IProgress<long>? progress = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
// Idle/heartbeat cancellation: HttpClient.Timeout is a whole-request cap and cannot express
|
||||
// "no bytes for N seconds", so the named client runs with InfiniteTimeSpan and the deadline
|
||||
// lives here. Each ProgressStreamContent tick resets CancelAfter(idle); a stalled socket lets
|
||||
// the window elapse and cancels the send. Linked to the caller's ct so a page cancel still wins.
|
||||
using var idleCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
idleCts.CancelAfter(_uploadIdleTimeout);
|
||||
|
||||
// Rebuild the multipart container so the boundary is owned by HttpClient and the
|
||||
// caller-supplied stream (already buffered by the SignalR upload) is the source.
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var wavContent = new StreamContent(wavStream);
|
||||
var wavContent = new ProgressStreamContent(wavStream, contentLength, written =>
|
||||
{
|
||||
// One mechanism, two consumers: advance the UI meter and reset the idle heartbeat.
|
||||
progress?.Report(written);
|
||||
idleCts.CancelAfter(_uploadIdleTimeout);
|
||||
});
|
||||
wavContent.Headers.ContentType = new MediaTypeHeaderValue(
|
||||
string.IsNullOrWhiteSpace(contentType) ? "audio/wav" : contentType);
|
||||
multipart.Add(wavContent, "audioFile", fileName);
|
||||
@@ -76,7 +99,15 @@ public class CmsTrackService : ICmsTrackService
|
||||
HttpResponseMessage response;
|
||||
try
|
||||
{
|
||||
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct);
|
||||
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, idleCts.Token);
|
||||
}
|
||||
catch (OperationCanceledException) when (idleCts.IsCancellationRequested && !ct.IsCancellationRequested)
|
||||
{
|
||||
// Idle window elapsed with no bytes moving — a stalled connection, not a caller cancel.
|
||||
_logger.LogWarning("Upload of {TrackName} stalled — no progress for {IdleSeconds}s; aborting.",
|
||||
trackName, _uploadIdleTimeout.TotalSeconds);
|
||||
return ResultContainer<TrackDto>.CreateFailResult(
|
||||
$"Upload stalled — no data transferred for {_uploadIdleTimeout.TotalSeconds:0}s. Please retry.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -21,9 +21,14 @@ public interface ICmsTrackService
|
||||
/// <paramref name="medium"/> sets the parent release's <see cref="ReleaseMedium"/> when this upload
|
||||
/// creates the release. The medium is authoritative only on creation — adding a track to an existing
|
||||
/// release never changes its medium (that is the edit path, <see cref="UpdateAsync"/>).
|
||||
/// <paramref name="contentLength"/> is the total payload size (the browser file's <c>Size</c>); it
|
||||
/// sets Content-Length and is the denominator for <paramref name="progress"/>, which reports cumulative
|
||||
/// bytes pushed to the wire. Each progress tick also resets the idle/heartbeat upload timeout, so a
|
||||
/// stalled connection aborts without a fixed total-duration cap.
|
||||
/// </summary>
|
||||
Task<ResultContainer<TrackDto>> UploadTrackAsync(
|
||||
Stream wavStream,
|
||||
long contentLength,
|
||||
string fileName,
|
||||
string contentType,
|
||||
string trackName,
|
||||
@@ -37,6 +42,7 @@ public interface ICmsTrackService
|
||||
ReleaseType releaseType,
|
||||
int trackNumber,
|
||||
ReleaseMedium medium = ReleaseMedium.Cut,
|
||||
IProgress<long>? progress = null,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
using System.Net;
|
||||
|
||||
namespace DeepDrftManager.Services;
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="HttpContent"/> that streams a source stream to the wire while reporting cumulative
|
||||
/// bytes written after each chunk. This is the single source of truth for both the upload progress
|
||||
/// meter and the idle/heartbeat timeout: every reported tick advances the UI <em>and</em> resets the
|
||||
/// idle deadline, so one mechanism feeds both concerns.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Wrap the audio payload (not the whole multipart container) so <see cref="TryComputeLength"/>
|
||||
/// returns the file length and the reported byte counts map directly onto "bytes of this file".
|
||||
/// </remarks>
|
||||
public sealed class ProgressStreamContent : HttpContent
|
||||
{
|
||||
// 80 KB: large enough to keep the socket fed on a healthy link, small enough that a stalled
|
||||
// connection trips the idle window without a multi-MB write swallowing the whole heartbeat budget.
|
||||
private const int CopyBufferSize = 81_920;
|
||||
|
||||
private readonly Stream _source;
|
||||
private readonly long _length;
|
||||
private readonly Action<long> _onBytesWritten;
|
||||
|
||||
/// <param name="source">The payload stream. Read once, sequentially — not seekable-rewound.</param>
|
||||
/// <param name="length">Total bytes the source will yield; sets Content-Length and the meter denominator.</param>
|
||||
/// <param name="onBytesWritten">Invoked after each chunk with the cumulative bytes written so far.</param>
|
||||
public ProgressStreamContent(Stream source, long length, Action<long> onBytesWritten)
|
||||
{
|
||||
_source = source;
|
||||
_length = length;
|
||||
_onBytesWritten = onBytesWritten;
|
||||
}
|
||||
|
||||
// Token-aware overload (.NET 5+): HttpClient calls this on the send path and passes the request's
|
||||
// CancellationToken, so the idle-heartbeat CTS aborts an in-flight read/write promptly — not just
|
||||
// between chunks. The parameterless base override delegates here with CancellationToken.None.
|
||||
protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context, CancellationToken cancellationToken)
|
||||
=> CopyAsync(stream, cancellationToken);
|
||||
|
||||
protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context)
|
||||
=> CopyAsync(stream, CancellationToken.None);
|
||||
|
||||
private async Task CopyAsync(Stream stream, CancellationToken cancellationToken)
|
||||
{
|
||||
var buffer = new byte[CopyBufferSize];
|
||||
long written = 0;
|
||||
int read;
|
||||
while ((read = await _source.ReadAsync(buffer, cancellationToken)) > 0)
|
||||
{
|
||||
await stream.WriteAsync(buffer.AsMemory(0, read), cancellationToken);
|
||||
written += read;
|
||||
// Report after the bytes are on the wire — a tick means real forward progress, which is
|
||||
// exactly the signal the idle heartbeat must reset on.
|
||||
_onBytesWritten(written);
|
||||
}
|
||||
}
|
||||
|
||||
protected override bool TryComputeLength(out long length)
|
||||
{
|
||||
length = _length;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user