daniel/deepdrft
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

fix(cms-upload): scope InfiniteTimeSpan to upload client; add response-wait budget after body completes

Browse Source
This commit is contained in:
daniel-c-harvey
2026-06-17 11:14:15 -04:00
parent c9c6286571
commit 803bc7840a
3 changed files with 177 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DeepDrftManager/Services/CmsTrackService.cs
+72 -20
View File
@@ -18,16 +18,26 @@ namespace DeepDrftManager.Services;
public class CmsTrackService : ICmsTrackService
{
private const string ContentCmsClientName = "DeepDrft.Content.Cms";
private const string UploadClientName = "DeepDrft.Content.Cms.Upload";
private const string UploadPath = "api/track/upload";
// Idle/heartbeat window: abort an upload only after this long with zero bytes written to the wire.
// The window resets on every progress tick, so a slow-but-moving half-gig upload never trips it;
// a genuinely stalled socket does. Operator-tunable via Upload:IdleTimeoutSeconds.
// a genuinely stalled socket does. Governs the BODY-STREAMING phase only.
// Operator-tunable via Upload:IdleTimeoutSeconds.
private const int DefaultIdleTimeoutSeconds = 90;
// Response-wait budget: once the request body is fully on the wire the server runs AudioProcessor
// decode → vault write → SQL persist. For a several-hundred-MB WAV this can take many minutes.
// The idle heartbeat goes silent after the last byte, so a separate, larger deadline governs the
// response-wait phase so a fully-uploaded file is never killed mid-persist.
// Operator-tunable via Upload:ResponseTimeoutSeconds.
private const int DefaultResponseTimeoutSeconds = 600; // 10 minutes
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<CmsTrackService> _logger;
private readonly TimeSpan _uploadIdleTimeout;
private readonly TimeSpan _uploadResponseTimeout;
public CmsTrackService(
IHttpClientFactory httpClientFactory,
@@ -36,8 +46,10 @@ public class CmsTrackService : ICmsTrackService
{
_httpClientFactory = httpClientFactory;
_logger = logger;
var seconds = configuration.GetValue<int?>("Upload:IdleTimeoutSeconds") ?? DefaultIdleTimeoutSeconds;
_uploadIdleTimeout = TimeSpan.FromSeconds(seconds > 0 ? seconds : DefaultIdleTimeoutSeconds);
var idleSeconds = configuration.GetValue<int?>("Upload:IdleTimeoutSeconds") ?? DefaultIdleTimeoutSeconds;
_uploadIdleTimeout = TimeSpan.FromSeconds(idleSeconds > 0 ? idleSeconds : DefaultIdleTimeoutSeconds);
var responseSeconds = configuration.GetValue<int?>("Upload:ResponseTimeoutSeconds") ?? DefaultResponseTimeoutSeconds;
_uploadResponseTimeout = TimeSpan.FromSeconds(responseSeconds > 0 ? responseSeconds : DefaultResponseTimeoutSeconds);
}
public async Task<ResultContainer<TrackDto>> UploadTrackAsync(
@@ -59,22 +71,52 @@ public class CmsTrackService : ICmsTrackService
IProgress<long>? progress = null,
CancellationToken ct = default)
{
// Idle/heartbeat cancellation: HttpClient.Timeout is a whole-request cap and cannot express
// "no bytes for N seconds", so the named client runs with InfiniteTimeSpan and the deadline
// lives here. Each ProgressStreamContent tick resets CancelAfter(idle); a stalled socket lets
// the window elapse and cancels the send. Linked to the caller's ct so a page cancel still wins.
// Two-phase cancellation for the upload send:
//
// BODY-STREAMING phase (while bytes are on the wire):
// idleCts fires if no progress tick arrives within the idle window. Each
// ProgressStreamContent chunk resets CancelAfter(idle), so a slow-but-moving
// upload never trips it; a genuinely stalled socket does.
//
// RESPONSE-WAIT phase (after the last byte, while the server persists):
// The idle heartbeat goes silent once the body is fully sent. responseCts is
// armed at that moment with a larger budget so a fully-uploaded file is never
// killed mid-persist. idleCts is simultaneously disarmed (CancelAfter(Infinite))
// so it cannot misfire during the response-wait.
//
// sendCts links both so either deadline — plus the caller's ct — cancels the send.
using var idleCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
idleCts.CancelAfter(_uploadIdleTimeout);
// responseCts starts disarmed; the body-complete callback below arms it.
using var responseCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
// Umbrella token passed to SendAsync — either phase token (or the caller) can cancel.
using var sendCts = CancellationTokenSource.CreateLinkedTokenSource(idleCts.Token, responseCts.Token);
// Rebuild the multipart container so the boundary is owned by HttpClient and the
// caller-supplied stream (already buffered by the SignalR upload) is the source.
using var multipart = new MultipartFormDataContent();
var wavContent = new ProgressStreamContent(wavStream, contentLength, written =>
{
// One mechanism, two consumers: advance the UI meter and reset the idle heartbeat.
progress?.Report(written);
idleCts.CancelAfter(_uploadIdleTimeout);
});
var wavContent = new ProgressStreamContent(
wavStream,
contentLength,
written =>
{
// One mechanism, three consumers: advance the UI meter, reset the idle heartbeat,
// and on body-complete transition to the response-wait budget.
progress?.Report(written);
if (written < contentLength)
{
// Body still in flight — keep the idle heartbeat alive.
idleCts.CancelAfter(_uploadIdleTimeout);
}
else
{
// Last byte on the wire. Disarm the idle timer and start the response budget.
idleCts.CancelAfter(Timeout.InfiniteTimeSpan);
responseCts.CancelAfter(_uploadResponseTimeout);
}
});
wavContent.Headers.ContentType = new MediaTypeHeaderValue(
string.IsNullOrWhiteSpace(contentType) ? "audio/wav" : contentType);
multipart.Add(wavContent, "audioFile", fileName);
@@ -93,21 +135,31 @@ public class CmsTrackService : ICmsTrackService
// for an unrecognised value). Authoritative only when this upload creates the release.
multipart.Add(new StringContent(medium.ToString()), "medium");
var client = _httpClientFactory.CreateClient(ContentCmsClientName);
// Use the dedicated upload client (InfiniteTimeSpan) so the two-phase CTS logic above is the
// sole timeout authority. Non-upload operations use the bounded "DeepDrft.Content.Cms" client.
var client = _httpClientFactory.CreateClient(UploadClientName);
using var request = new HttpRequestMessage(HttpMethod.Post, UploadPath) { Content = multipart };
HttpResponseMessage response;
try
{
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, idleCts.Token);
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, sendCts.Token);
}
catch (OperationCanceledException) when (idleCts.IsCancellationRequested && !ct.IsCancellationRequested)
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
// Idle window elapsed with no bytes moving — a stalled connection, not a caller cancel.
_logger.LogWarning("Upload of {TrackName} stalled — no progress for {IdleSeconds}s; aborting.",
trackName, _uploadIdleTimeout.TotalSeconds);
// Either idle window (body-streaming stall) or response-wait budget (server persist too slow).
if (idleCts.IsCancellationRequested)
{
_logger.LogWarning("Upload of {TrackName} stalled — no progress for {IdleSeconds}s; aborting.",
trackName, _uploadIdleTimeout.TotalSeconds);
return ResultContainer<TrackDto>.CreateFailResult(
$"Upload stalled — no data transferred for {_uploadIdleTimeout.TotalSeconds:0}s. Please retry.");
}
// responseCts fired: body reached the server but persist timed out.
_logger.LogWarning("Upload of {TrackName} timed out waiting for server response after {ResponseSeconds}s.",
trackName, _uploadResponseTimeout.TotalSeconds);
return ResultContainer<TrackDto>.CreateFailResult(
$"Upload stalled — no data transferred for {_uploadIdleTimeout.TotalSeconds:0}s. Please retry.");
$"Upload timed out waiting for the server to respond after {_uploadResponseTimeout.TotalSeconds:0}s. Please retry.");
}
catch (Exception ex)
{