From 605fc94fbbbe01dc1b8b504c1c66ed3ff48f2620 Mon Sep 17 00:00:00 2001 From: daniel-c-harvey Date: Mon, 15 Sep 2025 17:03:36 -0400 Subject: [PATCH] True Streaming Support Draft --- CLI.sln | 40 ++ .../Processors/AudioProcessor.cs | 256 ++++++--- .../Clients/TrackMediaClient.cs | 3 +- .../AudioPlayerBar/AudioPlayerBar.razor | 4 +- .../AudioPlayerBar/AudioPlayerBar.razor.cs | 7 +- .../Controls/AudioPlayerProvider.razor.cs | 10 +- .../Services/AudioInteropService.cs | 24 +- .../Services/AudioPlayerService.cs | 106 ++-- .../Services/StreamingAudioPlayerService.cs | 301 ++++++++++ .../Services/StreamingErrorHandler.cs | 30 + DeepDrftWeb/Interop/wavutils.ts | 64 ++- DeepDrftWeb/Interop/webaudio.ts | 542 +++++++++++++----- DeepDrftWeb/Program.cs | 12 + DeepDrftWeb/appsettings.Development.json | 9 +- DeepDrftWeb/tsconfig.json | 4 +- global.json | 7 + 16 files changed, 1124 insertions(+), 295 deletions(-) create mode 100644 CLI.sln create mode 100644 DeepDrftWeb.Client/Services/StreamingAudioPlayerService.cs create mode 100644 DeepDrftWeb.Client/Services/StreamingErrorHandler.cs create mode 100644 global.json diff --git a/CLI.sln b/CLI.sln new file mode 100644 index 0000000..182fd88 --- /dev/null +++ b/CLI.sln @@ -0,0 +1,40 @@ +ο»Ώ +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeepDrftCli", "DeepDrftCli\DeepDrftCli.csproj", "{84844B37-FD15-4AFC-850B-DD432AA33B4C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeepDrftContent.Services", "DeepDrftContent.Services\DeepDrftContent.Services.csproj", "{169D5D3E-DAEC-46BE-98EE-CC5EBF5E3E8A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeepDrftWeb.Services", "DeepDrftWeb.Services\DeepDrftWeb.Services.csproj", "{A3DA341B-589E-4705-AB66-6B22652A9B36}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetBlocks", "C:\lib\NetBlocks\NetBlocks.csproj", "{41FC69D0-F60D-41B4-AA41-C2382C83DFE8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeepDrftModels", "DeepDrftModels\DeepDrftModels.csproj", "{AEA0B3A0-722E-4D34-B2F6-F8179A4DD45A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {84844B37-FD15-4AFC-850B-DD432AA33B4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84844B37-FD15-4AFC-850B-DD432AA33B4C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84844B37-FD15-4AFC-850B-DD432AA33B4C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84844B37-FD15-4AFC-850B-DD432AA33B4C}.Release|Any CPU.Build.0 = Release|Any CPU + {169D5D3E-DAEC-46BE-98EE-CC5EBF5E3E8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {169D5D3E-DAEC-46BE-98EE-CC5EBF5E3E8A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {169D5D3E-DAEC-46BE-98EE-CC5EBF5E3E8A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {169D5D3E-DAEC-46BE-98EE-CC5EBF5E3E8A}.Release|Any CPU.Build.0 = Release|Any CPU + {A3DA341B-589E-4705-AB66-6B22652A9B36}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A3DA341B-589E-4705-AB66-6B22652A9B36}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A3DA341B-589E-4705-AB66-6B22652A9B36}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A3DA341B-589E-4705-AB66-6B22652A9B36}.Release|Any CPU.Build.0 = Release|Any CPU + {41FC69D0-F60D-41B4-AA41-C2382C83DFE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {41FC69D0-F60D-41B4-AA41-C2382C83DFE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {41FC69D0-F60D-41B4-AA41-C2382C83DFE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {41FC69D0-F60D-41B4-AA41-C2382C83DFE8}.Release|Any CPU.Build.0 = Release|Any CPU + {AEA0B3A0-722E-4D34-B2F6-F8179A4DD45A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AEA0B3A0-722E-4D34-B2F6-F8179A4DD45A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AEA0B3A0-722E-4D34-B2F6-F8179A4DD45A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AEA0B3A0-722E-4D34-B2F6-F8179A4DD45A}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/DeepDrftContent.Services/Processors/AudioProcessor.cs b/DeepDrftContent.Services/Processors/AudioProcessor.cs index ba22e46..ffac64a 100644 --- a/DeepDrftContent.Services/Processors/AudioProcessor.cs +++ b/DeepDrftContent.Services/Processors/AudioProcessor.cs @@ -46,101 +46,198 @@ public class AudioProcessor } /// - /// Extracts metadata from WAV file buffer + /// Extracts metadata from WAV file buffer with comprehensive validation /// private WavMetadata ExtractWavMetadata(byte[] buffer) { try { - // WAV file format parsing - // RIFF header starts at byte 0 - if (buffer.Length < 44) + var validationResult = ValidateWavStructure(buffer); + if (!validationResult.IsValid) { - throw new InvalidDataException("WAV file too short to contain valid header"); + throw new InvalidDataException($"WAV validation failed: {validationResult.ErrorMessage}"); } - // Check RIFF signature - var riffSignature = System.Text.Encoding.ASCII.GetString(buffer, 0, 4); - if (riffSignature != "RIFF") - { - throw new InvalidDataException("Invalid WAV file: Missing RIFF signature"); - } - - // Check WAVE format - var waveSignature = System.Text.Encoding.ASCII.GetString(buffer, 8, 4); - if (waveSignature != "WAVE") - { - throw new InvalidDataException("Invalid WAV file: Missing WAVE signature"); - } - - // Find fmt chunk - var fmtChunkPos = FindChunk(buffer, "fmt "); - if (fmtChunkPos == -1) - { - throw new InvalidDataException("Invalid WAV file: Missing fmt chunk"); - } - - // Parse fmt chunk - var fmtChunkSize = BitConverter.ToUInt32(buffer, fmtChunkPos + 4); - var sampleRate = BitConverter.ToUInt32(buffer, fmtChunkPos + 12); - var byteRate = BitConverter.ToUInt32(buffer, fmtChunkPos + 16); - var channels = BitConverter.ToUInt16(buffer, fmtChunkPos + 10); - var bitsPerSample = BitConverter.ToUInt16(buffer, fmtChunkPos + 22); - - // Find data chunk - var dataChunkPos = FindChunk(buffer, "data"); - if (dataChunkPos == -1) - { - throw new InvalidDataException("Invalid WAV file: Missing data chunk"); - } - - var dataSize = BitConverter.ToUInt32(buffer, dataChunkPos + 4); - - // Calculate duration - var duration = (double)dataSize / byteRate; + var metadata = ParseWavMetadata(buffer, validationResult); + ValidateAudioParameters(metadata); - // Calculate bitrate (bits per second / 1000 for kbps) - var bitrate = (int)((sampleRate * channels * bitsPerSample) / 1000); - - return new WavMetadata - { - Duration = duration, - Bitrate = bitrate, - SampleRate = (int)sampleRate, - Channels = channels, - BitsPerSample = bitsPerSample - }; + return metadata; } catch (Exception ex) { - // Fallback to basic metadata if parsing fails - Console.WriteLine($"Warning: Could not parse WAV metadata: {ex.Message}"); - return new WavMetadata - { - Duration = 180.0, // Default 3 minutes - Bitrate = 1411, // Default CD quality bitrate for WAV - SampleRate = 44100, - Channels = 2, - BitsPerSample = 16 - }; + Console.WriteLine($"Warning: WAV parsing failed, using defaults: {ex.Message}"); + return GetDefaultWavMetadata(); } } + + /// + /// Validates WAV file structure and returns parsing information + /// + private WavValidationResult ValidateWavStructure(byte[] buffer) + { + if (buffer.Length < 44) + { + return new WavValidationResult { IsValid = false, ErrorMessage = "File too short" }; + } + + // Validate RIFF signature + var riffSignature = System.Text.Encoding.ASCII.GetString(buffer, 0, 4); + if (riffSignature != "RIFF") + { + return new WavValidationResult { IsValid = false, ErrorMessage = "Invalid RIFF signature" }; + } + + // Validate WAVE signature + var waveSignature = System.Text.Encoding.ASCII.GetString(buffer, 8, 4); + if (waveSignature != "WAVE") + { + return new WavValidationResult { IsValid = false, ErrorMessage = "Invalid WAVE signature" }; + } + + // Find and validate fmt chunk + var fmtChunkPos = FindChunk(buffer, "fmt "); + if (fmtChunkPos == -1) + { + return new WavValidationResult { IsValid = false, ErrorMessage = "Missing fmt chunk" }; + } + + var fmtChunkSize = BitConverter.ToUInt32(buffer, fmtChunkPos + 4); + if (fmtChunkSize < 16) + { + return new WavValidationResult { IsValid = false, ErrorMessage = "fmt chunk too small" }; + } + + // Validate audio format (PCM only) + var audioFormat = BitConverter.ToUInt16(buffer, fmtChunkPos + 8); + if (audioFormat != 1) + { + return new WavValidationResult { IsValid = false, ErrorMessage = "Only PCM format supported" }; + } + + // Find data chunk + var dataChunkPos = FindChunk(buffer, "data"); + if (dataChunkPos == -1) + { + return new WavValidationResult { IsValid = false, ErrorMessage = "Missing data chunk" }; + } + + return new WavValidationResult + { + IsValid = true, + FmtChunkPos = fmtChunkPos, + DataChunkPos = dataChunkPos + }; + } + + /// + /// Parses WAV metadata from validated buffer + /// + private WavMetadata ParseWavMetadata(byte[] buffer, WavValidationResult validation) + { + var channels = BitConverter.ToUInt16(buffer, validation.FmtChunkPos + 10); + var sampleRate = BitConverter.ToUInt32(buffer, validation.FmtChunkPos + 12); + var byteRate = BitConverter.ToUInt32(buffer, validation.FmtChunkPos + 16); + var blockAlign = BitConverter.ToUInt16(buffer, validation.FmtChunkPos + 20); + var bitsPerSample = BitConverter.ToUInt16(buffer, validation.FmtChunkPos + 22); + var dataSize = BitConverter.ToUInt32(buffer, validation.DataChunkPos + 4); + + var duration = byteRate > 0 ? (double)dataSize / byteRate : 0.0; + var bitrate = (int)((sampleRate * channels * bitsPerSample) / 1000); + + return new WavMetadata + { + Duration = duration, + Bitrate = bitrate, + SampleRate = (int)sampleRate, + Channels = channels, + BitsPerSample = bitsPerSample, + BlockAlign = blockAlign, + DataSize = (int)dataSize + }; + } + + /// + /// Validates audio parameters for reasonableness + /// + private void ValidateAudioParameters(WavMetadata metadata) + { + var validSampleRates = new[] { 8000, 11025, 16000, 22050, 44100, 48000, 88200, 96000, 176400, 192000 }; + var validBitDepths = new[] { 8, 16, 24, 32 }; + + if (metadata.Channels < 1 || metadata.Channels > 8) + { + throw new InvalidDataException($"Invalid channel count: {metadata.Channels}"); + } + + if (!validSampleRates.Contains(metadata.SampleRate)) + { + throw new InvalidDataException($"Unsupported sample rate: {metadata.SampleRate}"); + } + + if (!validBitDepths.Contains(metadata.BitsPerSample)) + { + throw new InvalidDataException($"Unsupported bit depth: {metadata.BitsPerSample}"); + } + + var expectedBlockAlign = metadata.Channels * (metadata.BitsPerSample / 8); + if (metadata.BlockAlign != expectedBlockAlign) + { + throw new InvalidDataException($"Invalid block align: expected {expectedBlockAlign}, got {metadata.BlockAlign}"); + } + } + + /// + /// Returns default WAV metadata for fallback scenarios + /// + private WavMetadata GetDefaultWavMetadata() + { + return new WavMetadata + { + Duration = 180.0, + Bitrate = 1411, + SampleRate = 44100, + Channels = 2, + BitsPerSample = 16, + BlockAlign = 4, + DataSize = 0 + }; + } /// - /// Finds a chunk in the WAV file buffer + /// Finds a chunk in the WAV file buffer with proper alignment handling /// private int FindChunk(byte[] buffer, string chunkId) { var chunkBytes = System.Text.Encoding.ASCII.GetBytes(chunkId); + int offset = 12; // Start after RIFF header - for (int i = 12; i < buffer.Length - 8; i += 4) + while (offset <= buffer.Length - 8) { - if (buffer[i] == chunkBytes[0] && - buffer[i + 1] == chunkBytes[1] && - buffer[i + 2] == chunkBytes[2] && - buffer[i + 3] == chunkBytes[3]) + // Check for chunk signature match + bool match = true; + for (int i = 0; i < 4; i++) { - return i; + if (buffer[offset + i] != chunkBytes[i]) + { + match = false; + break; + } + } + + if (match) + { + return offset; + } + + // Move to next chunk with proper alignment + if (offset + 4 < buffer.Length) + { + var chunkSize = BitConverter.ToUInt32(buffer, offset + 4); + offset += 8 + (int)((chunkSize + 1) & ~1U); // Ensure even alignment + } + else + { + break; } } @@ -148,7 +245,7 @@ public class AudioProcessor } /// - /// WAV file metadata + /// WAV file metadata with complete audio information /// private class WavMetadata { @@ -157,5 +254,18 @@ public class AudioProcessor public int SampleRate { get; set; } public int Channels { get; set; } public int BitsPerSample { get; set; } + public int BlockAlign { get; set; } + public int DataSize { get; set; } + } + + /// + /// Result of WAV structure validation + /// + private class WavValidationResult + { + public bool IsValid { get; set; } + public string ErrorMessage { get; set; } = string.Empty; + public int FmtChunkPos { get; set; } + public int DataChunkPos { get; set; } } } \ No newline at end of file diff --git a/DeepDrftWeb.Client/Clients/TrackMediaClient.cs b/DeepDrftWeb.Client/Clients/TrackMediaClient.cs index d0d8923..ad74b9e 100644 --- a/DeepDrftWeb.Client/Clients/TrackMediaClient.cs +++ b/DeepDrftWeb.Client/Clients/TrackMediaClient.cs @@ -33,7 +33,8 @@ public class TrackMediaClient { try { - var response = await _http.GetAsync($"api/track/{trackId}"); + // Use HttpCompletionOption.ResponseHeadersRead to get stream immediately + var response = await _http.GetAsync($"api/track/{trackId}", HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); var contentLength = response.Content.Headers.ContentLength ?? 0; diff --git a/DeepDrftWeb.Client/Controls/AudioPlayerBar/AudioPlayerBar.razor b/DeepDrftWeb.Client/Controls/AudioPlayerBar/AudioPlayerBar.razor index 2765ee0..5e65363 100644 --- a/DeepDrftWeb.Client/Controls/AudioPlayerBar/AudioPlayerBar.razor +++ b/DeepDrftWeb.Client/Controls/AudioPlayerBar/AudioPlayerBar.razor @@ -23,7 +23,7 @@ else IsLoaded="IsLoaded" TogglePlayPause="@TogglePlayPause" Stop="@Stop"/> - @if (IsLoading) + @if (IsLoading && !IsStreaming) { - @if (IsLoading) + @if (IsLoading && !IsStreaming) { PlayerService.IsLoaded; private bool IsLoading => PlayerService.IsLoading; + private bool IsStreaming => PlayerService.CanStartStreaming; private bool IsPlaying => PlayerService.IsPlaying; private bool IsPaused => PlayerService.IsPaused; private double CurrentTime => PlayerService.CurrentTime; @@ -23,8 +24,8 @@ public partial class AudioPlayerBar : ComponentBase protected override async Task OnInitializedAsync() { await base.OnInitializedAsync(); - PlayerService.OnStateChanged += StateHasChanged; - PlayerService.OnTrackSelected += Expand; + // Set up EventCallback for track selection + PlayerService.OnTrackSelected = new EventCallback(this, Expand); } private async Task Expand() diff --git a/DeepDrftWeb.Client/Controls/AudioPlayerProvider.razor.cs b/DeepDrftWeb.Client/Controls/AudioPlayerProvider.razor.cs index 1356683..38e5d5e 100644 --- a/DeepDrftWeb.Client/Controls/AudioPlayerProvider.razor.cs +++ b/DeepDrftWeb.Client/Controls/AudioPlayerProvider.razor.cs @@ -1,6 +1,7 @@ ο»Ώusing DeepDrftWeb.Client.Services; using DeepDrftWeb.Client.Clients; using Microsoft.AspNetCore.Components; +using Microsoft.Extensions.Logging; namespace DeepDrftWeb.Client.Controls; @@ -8,15 +9,20 @@ public partial class AudioPlayerProvider : ComponentBase { [Inject] public required AudioInteropService AudioInterop { get; set; } [Inject] public required TrackMediaClient TrackMediaClient { get; set; } + [Inject] public required ILogger Logger { get; set; } - private AudioPlayerService? _audioPlayerService; + private StreamingAudioPlayerService? _audioPlayerService; [Parameter] public RenderFragment? ChildContent { get; set; } protected override void OnInitialized() { // Create the service immediately (but don't initialize yet) - _audioPlayerService = new AudioPlayerService(AudioInterop, TrackMediaClient); + _audioPlayerService = new StreamingAudioPlayerService(AudioInterop, TrackMediaClient, Logger); + + // Set up EventCallback to properly marshal UI updates back to UI thread + _audioPlayerService.OnStateChanged = new EventCallback(this, StateHasChanged); + // OnTrackSelected will be set by individual child components that need it } protected override async Task OnAfterRenderAsync(bool firstRender) diff --git a/DeepDrftWeb.Client/Services/AudioInteropService.cs b/DeepDrftWeb.Client/Services/AudioInteropService.cs index 9c385ad..e1218b3 100644 --- a/DeepDrftWeb.Client/Services/AudioInteropService.cs +++ b/DeepDrftWeb.Client/Services/AudioInteropService.cs @@ -41,9 +41,9 @@ public class AudioInteropService : IAsyncDisposable } // Streaming methods - public async Task InitializeStreaming(string playerId) + public async Task InitializeStreaming(string playerId, long totalStreamLength) { - return await InvokeJsAsync("DeepDrftAudio.initializeStreaming", playerId); + return await InvokeJsAsync("DeepDrftAudio.initializeStreaming", playerId, totalStreamLength); } public async Task ProcessStreamingChunk(string playerId, byte[] audioChunk) @@ -56,6 +56,11 @@ public class AudioInteropService : IAsyncDisposable return await InvokeJsAsync("DeepDrftAudio.startStreamingPlayback", playerId); } + public async Task EnsureAudioContextReady(string playerId) + { + return await InvokeJsAsync("DeepDrftAudio.ensureAudioContextReady", playerId); + } + public async Task PlayAsync(string playerId) { return await InvokeJsAsync("DeepDrftAudio.play", playerId); @@ -122,11 +127,6 @@ public class AudioInteropService : IAsyncDisposable wrapper => wrapper.OnEnd = callback); } - public async Task SetOnLoadProgressCallbackAsync(string playerId, Func callback) - { - return await SetCallbackAsync(playerId, "_loadprogress", "setOnLoadProgressCallback", "OnLoadProgressCallback", - wrapper => wrapper.OnLoadProgress = callback); - } public async Task DisposePlayerAsync(string playerId) { @@ -146,6 +146,8 @@ public class AudioInteropService : IAsyncDisposable return (T)(object)new AudioOperationResult { Success = false, Error = ex.Message }; if (typeof(T) == typeof(AudioLoadResult)) return (T)(object)new AudioLoadResult { Success = false, Error = ex.Message }; + if (typeof(T) == typeof(StreamingResult)) + return (T)(object)new StreamingResult { Success = false, Error = ex.Message }; throw; } } @@ -193,7 +195,6 @@ public class AudioPlayerCallback { public Func? OnProgress { get; set; } public Func? OnEnd { get; set; } - public Func? OnLoadProgress { get; set; } [JSInvokable] public async Task OnProgressCallback(double currentTime) @@ -208,13 +209,6 @@ public class AudioPlayerCallback if (OnEnd != null) await OnEnd(); } - - [JSInvokable] - public async Task OnLoadProgressCallback(double progress) - { - if (OnLoadProgress != null) - await OnLoadProgress(progress); - } } public class AudioOperationResult diff --git a/DeepDrftWeb.Client/Services/AudioPlayerService.cs b/DeepDrftWeb.Client/Services/AudioPlayerService.cs index b023972..7e8b859 100644 --- a/DeepDrftWeb.Client/Services/AudioPlayerService.cs +++ b/DeepDrftWeb.Client/Services/AudioPlayerService.cs @@ -1,33 +1,34 @@ using DeepDrftModels.Entities; using DeepDrftWeb.Client.Clients; +using Microsoft.AspNetCore.Components; using NetBlocks.Models; namespace DeepDrftWeb.Client.Services; -public class AudioPlayerService : IPlayerService, IAsyncDisposable +public abstract class AudioPlayerService : IPlayerService, IAsyncDisposable { - private readonly AudioInteropService _audioInterop; - private readonly TrackMediaClient _trackMediaClient; + protected readonly AudioInteropService _audioInterop; + protected readonly TrackMediaClient _trackMediaClient; public string PlayerId { get; private set; } = Guid.NewGuid().ToString(); // State properties - public bool IsInitialized { get; private set; } = false; - public bool IsLoaded { get; private set; } = false; - public bool IsLoading { get; private set; } = false; - public bool IsPlaying { get; private set; } = false; - public bool IsPaused { get; private set; } = false; - public double CurrentTime { get; private set; } = 0; - public double? Duration { get; private set; } = null; - public double Volume { get; private set; } = 0.8; - public double LoadProgress { get; private set; } = 0; - public string? ErrorMessage { get; private set; } + public bool IsInitialized { get; protected set; } = false; + public bool IsLoaded { get; protected set; } = false; + public bool IsLoading { get; protected set; } = false; + public bool IsPlaying { get; protected set; } = false; + public bool IsPaused { get; protected set; } = false; + public double CurrentTime { get; protected set; } = 0; + public double? Duration { get; protected set; } = null; + public double Volume { get; protected set; } = 0.8; + public double LoadProgress { get; protected set; } = 0; + public string? ErrorMessage { get; protected set; } // Events - public event Action? OnStateChanged; - public event Events.EventAsync? OnTrackSelected; + public EventCallback? OnStateChanged { get; set; } + public EventCallback? OnTrackSelected { get; set; } - public AudioPlayerService(AudioInteropService audioInterop, TrackMediaClient trackMediaClient) + protected AudioPlayerService(AudioInteropService audioInterop, TrackMediaClient trackMediaClient) { _audioInterop = audioInterop; _trackMediaClient = trackMediaClient; @@ -43,38 +44,37 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable if (!result.Success) { ErrorMessage = $"Failed to initialize audio player: {result.Error}"; - NotifyStateChanged(); + await NotifyStateChanged(); return; } await _audioInterop.SetOnProgressCallbackAsync(PlayerId, OnProgressCallback); await _audioInterop.SetOnEndCallbackAsync(PlayerId, OnPlaybackEndCallback); - await _audioInterop.SetOnLoadProgressCallbackAsync(PlayerId, OnLoadProgressCallback); await _audioInterop.SetVolumeAsync(PlayerId, Volume); IsInitialized = true; ErrorMessage = null; - NotifyStateChanged(); + await NotifyStateChanged(); } catch (Exception ex) { ErrorMessage = $"Failed to initialize audio player: {ex.Message}"; - NotifyStateChanged(); + await NotifyStateChanged(); } } - public async Task SelectTrack(TrackEntity track) + public virtual async Task SelectTrack(TrackEntity track) { await EnsureInitializedAsync(); - NotifyStateChanged(); + await NotifyStateChanged(); - if (OnTrackSelected != null) - await OnTrackSelected.Invoke(); + if (OnTrackSelected.HasValue) + await OnTrackSelected.Value.InvokeAsync(); await LoadTrack(track); - NotifyStateChanged(); + await NotifyStateChanged(); } private async Task LoadTrack(TrackEntity track) @@ -95,7 +95,7 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable IsLoading = true; Duration = null; CurrentTime = 0; - NotifyStateChanged(); + await NotifyStateChanged(); var loadResult = await _audioInterop.InitializeBufferedPlayerAsync(PlayerId); if (loadResult?.Success != true) @@ -129,7 +129,7 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable finally { IsLoading = false; - NotifyStateChanged(); + await NotifyStateChanged(); } } @@ -166,7 +166,7 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable if (audio.ContentLength > 0) { LoadProgress = Math.Min(1.0, (double)totalBytesRead / audio.ContentLength); - NotifyStateChanged(); + await NotifyStateChanged(); } } } while (currentBytes > 0); @@ -181,14 +181,14 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable LoadProgress = 1.0; IsLoaded = true; ErrorMessage = null; - NotifyStateChanged(); + await NotifyStateChanged(); } catch (Exception ex) { ErrorMessage = $"Error streaming audio: {ex.Message}"; LoadProgress = 0; IsLoaded = false; - NotifyStateChanged(); + await NotifyStateChanged(); throw; } } @@ -229,12 +229,12 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable ErrorMessage = null; } - NotifyStateChanged(); + await NotifyStateChanged(); } catch (Exception ex) { ErrorMessage = $"Error controlling playback: {ex.Message}"; - NotifyStateChanged(); + await NotifyStateChanged(); } } @@ -257,16 +257,16 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable ErrorMessage = $"Stop error: {result.Error}"; } - NotifyStateChanged(); + await NotifyStateChanged(); } catch (Exception ex) { ErrorMessage = $"Error stopping playback: {ex.Message}"; - NotifyStateChanged(); + await NotifyStateChanged(); } } - public async Task Unload() + public virtual async Task Unload() { if (!IsLoaded) return; @@ -289,12 +289,12 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable ErrorMessage = $"Unload error: {result.Error}"; } - NotifyStateChanged(); + await NotifyStateChanged(); } catch (Exception ex) { ErrorMessage = $"Error unloading track: {ex.Message}"; - NotifyStateChanged(); + await NotifyStateChanged(); } } @@ -315,12 +315,12 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable ErrorMessage = $"Seek error: {result.Error}"; } - NotifyStateChanged(); + await NotifyStateChanged(); } catch (Exception ex) { ErrorMessage = $"Error seeking: {ex.Message}"; - NotifyStateChanged(); + await NotifyStateChanged(); } } @@ -348,19 +348,19 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable } } - NotifyStateChanged(); + await NotifyStateChanged(); } - public void ClearError() + public async Task ClearError() { ErrorMessage = null; - NotifyStateChanged(); + await NotifyStateChanged(); } private async Task OnProgressCallback(double currentTime) { CurrentTime = currentTime; - NotifyStateChanged(); + await NotifyStateChanged(); } private async Task OnPlaybackEndCallback() @@ -368,16 +368,11 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable IsPlaying = false; IsPaused = false; CurrentTime = 0; - NotifyStateChanged(); + await NotifyStateChanged(); } - private async Task OnLoadProgressCallback(double progress) - { - LoadProgress = progress; - NotifyStateChanged(); - } - private async Task EnsureInitializedAsync() + protected async Task EnsureInitializedAsync() { if (!IsInitialized) { @@ -385,9 +380,16 @@ public class AudioPlayerService : IPlayerService, IAsyncDisposable } } - private void NotifyStateChanged() + protected async Task NotifyStateChanged() { - OnStateChanged?.Invoke(); + if (OnStateChanged.HasValue) + await OnStateChanged.Value.InvokeAsync(); + } + + protected async Task NotifyTrackSelected() + { + if (OnTrackSelected.HasValue) + await OnTrackSelected.Value.InvokeAsync(); } public async ValueTask DisposeAsync() diff --git a/DeepDrftWeb.Client/Services/StreamingAudioPlayerService.cs b/DeepDrftWeb.Client/Services/StreamingAudioPlayerService.cs new file mode 100644 index 0000000..0b54902 --- /dev/null +++ b/DeepDrftWeb.Client/Services/StreamingAudioPlayerService.cs @@ -0,0 +1,301 @@ +using DeepDrftModels.Entities; +using DeepDrftWeb.Client.Clients; +using System.Buffers; +using Microsoft.Extensions.Logging; + +namespace DeepDrftWeb.Client.Services; + +public class StreamingAudioPlayerService : AudioPlayerService, IStreamingPlayerService +{ + // Configuration constants + private const int DefaultBufferSize = 32 * 1024; // 32KB chunks + private const int NotificationThrottleMs = 100; // Throttle UI updates to max 10 per second + + // Adaptive chunk sizing + private const int MinBufferSize = 16 * 1024; // 16KB minimum + private const int MaxBufferSize = 64 * 1024; // 64KB maximum + private int _currentBufferSize = DefaultBufferSize; + private int _consecutiveSlowReads = 0; + + + // Streaming state properties + public bool IsStreamingMode { get; private set; } = false; + public bool CanStartStreaming { get; private set; } = false; + public bool HeaderParsed { get; private set; } = false; + public int BufferedChunks { get; private set; } = 0; + + private bool _streamingPlaybackStarted = false; + private CancellationTokenSource? _streamingCancellation; + private DateTime _lastNotification = DateTime.MinValue; + private readonly ILogger _logger; + + public StreamingAudioPlayerService( + AudioInteropService audioInterop, + TrackMediaClient trackMediaClient, + ILogger logger) + : base(audioInterop, trackMediaClient) + { + _logger = logger; + } + + public override async Task SelectTrack(TrackEntity track) + { + await SelectTrackStreaming(track); + } + + public async Task SelectTrackStreaming(TrackEntity track) + { + await EnsureInitializedAsync(); + + // Resume AudioContext immediately on track selection (user interaction) to avoid clicks later + await _audioInterop.EnsureAudioContextReady(PlayerId); + + // NotifyStateChanged(); + + await NotifyTrackSelected(); + + await LoadTrackStreaming(track); + await NotifyStateChanged(); + } + + private async Task LoadTrackStreaming(TrackEntity track) + { + // Cancel and replace any previous streaming operation atomically + var oldCancellation = _streamingCancellation; + _streamingCancellation = new CancellationTokenSource(); + + // Cancel the old operation after we've replaced it + oldCancellation?.Cancel(); + oldCancellation?.Dispose(); + + try + { + // No need to check IsLoading - we cancel previous operations + + if (IsPlaying || IsPaused) + { + await Unload(); + } + + // Reset state to indicate streaming has started + ErrorMessage = null; + LoadProgress = 0; + IsLoaded = false; + IsLoading = true; + IsStreamingMode = true; + CanStartStreaming = false; + HeaderParsed = false; + BufferedChunks = 0; + _streamingPlaybackStarted = false; + Duration = null; + CurrentTime = 0; + + // Reset adaptive buffer sizing + _currentBufferSize = DefaultBufferSize; + _consecutiveSlowReads = 0; + + await NotifyStateChanged(); + + var mediaResult = await _trackMediaClient.GetTrackMedia(track.EntryKey); + if (!mediaResult.Success) + { + var technicalError = mediaResult.GetMessage(); + _logger.LogError("Failed to get track media for {TrackId}: {Error}", + track.EntryKey, technicalError); + ErrorMessage = StreamingErrorHandler.GetUserFriendlyMessage(technicalError); + return; + } + + if (mediaResult.Value == null) + { + const string technicalError = "No audio returned from server"; + _logger.LogError("No audio data returned for track {TrackId}", track.EntryKey); + ErrorMessage = StreamingErrorHandler.GetUserFriendlyMessage(technicalError); + return; + } + + using var audio = mediaResult.Value; + + // Initialize streaming mode with content length + var streamingResult = await _audioInterop.InitializeStreaming(PlayerId, audio.ContentLength); + if (!streamingResult.Success) + { + var technicalError = $"Failed to initialize streaming: {streamingResult.Error}"; + _logger.LogError("Streaming initialization failed for track {TrackId}: {Error}", + track.EntryKey, technicalError); + ErrorMessage = StreamingErrorHandler.GetUserFriendlyMessage(technicalError); + return; + } + + await StreamAudioWithEarlyPlayback(audio, _streamingCancellation.Token); + } + catch (OperationCanceledException) + { + // Cancellation is expected, reset state + _logger.LogDebug("Audio streaming cancelled for track {TrackId}", track.EntryKey); + IsLoaded = false; + IsStreamingMode = false; + } + catch (Exception ex) + { + StreamingErrorHandler.LogError(_logger, ex, "LoadTrackStreaming", track.EntryKey); + ErrorMessage = StreamingErrorHandler.GetUserFriendlyMessage(ex.Message); + LoadProgress = 0; + IsLoaded = false; + IsStreamingMode = false; + } + finally + { + IsLoading = false; + await NotifyStateChanged(); + } + } + + private async Task StreamAudioWithEarlyPlayback(TrackMediaResponse audio, CancellationToken cancellationToken) + { + byte[]? buffer = null; + try + { + long totalBytesRead = 0; + buffer = ArrayPool.Shared.Rent(MaxBufferSize); // Rent larger buffer to accommodate adaptive sizing + int currentBytes; + var readTimer = System.Diagnostics.Stopwatch.StartNew(); + + do + { + readTimer.Restart(); + currentBytes = await audio.Stream.ReadAsync(buffer, 0, _currentBufferSize, cancellationToken); + readTimer.Stop(); + + // Adapt buffer size based on read performance + AdaptBufferSize(currentBytes, readTimer.ElapsedMilliseconds); + + if (currentBytes > 0) + { + totalBytesRead += currentBytes; + + // Use only the actual bytes read, no copying needed + var actualBuffer = currentBytes == _currentBufferSize ? buffer : buffer[..currentBytes]; + + // Process chunk for streaming + var chunkResult = await _audioInterop.ProcessStreamingChunk(PlayerId, actualBuffer); + if (!chunkResult.Success) + { + var error = $"Failed to process streaming chunk: {chunkResult.Error}"; + _logger.LogWarning("Chunk processing failed: {Error}", error); + throw new Exception(error); + } + + // Update streaming state + CanStartStreaming = chunkResult.CanStartStreaming; + HeaderParsed = chunkResult.HeaderParsed; + BufferedChunks = chunkResult.BufferCount; + + // Start playback as soon as we can + if (!_streamingPlaybackStarted && CanStartStreaming) + { + var playbackResult = await _audioInterop.StartStreamingPlayback(PlayerId); + if (playbackResult.Success) + { + _streamingPlaybackStarted = true; + IsPlaying = true; + IsPaused = false; + IsLoaded = true; // Track is loaded and ready to play (even if still downloading) + ErrorMessage = null; + await NotifyStateChanged(); // Immediate notification for critical state change + } + else + { + var technicalError = $"Failed to start streaming playback: {playbackResult.Error}"; + _logger.LogError("Failed to start playback: {Error}", technicalError); + ErrorMessage = StreamingErrorHandler.GetUserFriendlyMessage(technicalError); + } + } + + // Update progress + if (audio.ContentLength > 0) + { + LoadProgress = Math.Min(1.0, (double)totalBytesRead / audio.ContentLength); + } + + await ThrottledNotifyStateChanged(); + } + } while (currentBytes > 0); + + // Mark as fully loaded + LoadProgress = 1.0; + await NotifyStateChanged(); + } + catch (Exception ex) + { + StreamingErrorHandler.LogError(_logger, ex, "StreamAudioWithEarlyPlayback"); + ErrorMessage = StreamingErrorHandler.GetUserFriendlyMessage(ex.Message); + LoadProgress = 0; + IsLoaded = false; + IsStreamingMode = false; + await NotifyStateChanged(); + throw; + } + finally + { + if (buffer != null) + { + ArrayPool.Shared.Return(buffer); + } + } + } + + public override async Task Unload() + { + // Cancel any ongoing streaming operation + _streamingCancellation?.Cancel(); + _streamingCancellation?.Dispose(); + _streamingCancellation = null; + + IsStreamingMode = false; + CanStartStreaming = false; + HeaderParsed = false; + BufferedChunks = 0; + _streamingPlaybackStarted = false; + + await base.Unload(); + } + + private async Task ThrottledNotifyStateChanged() + { + var now = DateTime.UtcNow; + if ((now - _lastNotification).TotalMilliseconds >= NotificationThrottleMs) + { + _lastNotification = now; + await NotifyStateChanged(); + } + } + + private void AdaptBufferSize(int bytesRead, long readTimeMs) + { + // Adaptive buffer sizing based on network performance + if (readTimeMs > 100) // Slow read (>100ms) + { + _consecutiveSlowReads++; + if (_consecutiveSlowReads >= 3 && _currentBufferSize > MinBufferSize) + { + // Reduce buffer size for slow connections + _currentBufferSize = Math.Max(MinBufferSize, _currentBufferSize / 2); + _consecutiveSlowReads = 0; + } + } + else if (readTimeMs < 20 && bytesRead == _currentBufferSize) // Fast read, buffer fully utilized + { + _consecutiveSlowReads = 0; + if (_currentBufferSize < MaxBufferSize) + { + // Increase buffer size for fast connections + _currentBufferSize = Math.Min(MaxBufferSize, _currentBufferSize * 2); + } + } + else + { + _consecutiveSlowReads = 0; + } + } +} \ No newline at end of file diff --git a/DeepDrftWeb.Client/Services/StreamingErrorHandler.cs b/DeepDrftWeb.Client/Services/StreamingErrorHandler.cs new file mode 100644 index 0000000..84ba02a --- /dev/null +++ b/DeepDrftWeb.Client/Services/StreamingErrorHandler.cs @@ -0,0 +1,30 @@ +using Microsoft.Extensions.Logging; + +namespace DeepDrftWeb.Client.Services; + +public static class StreamingErrorHandler +{ + public static string GetUserFriendlyMessage(string technicalError) + { + var lowerError = technicalError.ToLowerInvariant(); + + return lowerError switch + { + _ when lowerError.Contains("network") || lowerError.Contains("connection") || lowerError.Contains("timeout") => + "Unable to load audio. Please check your connection and try again.", + + _ when lowerError.Contains("audio") || lowerError.Contains("decode") || lowerError.Contains("format") => + "This audio file may be corrupted or in an unsupported format.", + + _ when lowerError.Contains("cancel") || lowerError.Contains("abort") => + "Audio loading was cancelled.", + + _ => "Unable to play audio. Please try again." + }; + } + + public static void LogError(ILogger logger, Exception ex, string operation, string trackId = "") + { + logger.LogError(ex, "Streaming error in {Operation} for track {TrackId}", operation, trackId); + } +} \ No newline at end of file diff --git a/DeepDrftWeb/Interop/wavutils.ts b/DeepDrftWeb/Interop/wavutils.ts index cb6f1ae..a83913e 100644 --- a/DeepDrftWeb/Interop/wavutils.ts +++ b/DeepDrftWeb/Interop/wavutils.ts @@ -28,19 +28,29 @@ class WavUtils { const wave = new TextDecoder().decode(concatenated.slice(8, 12)); if (wave !== 'WAVE') return null; - // Find fmt chunk + // Find fmt chunk with better alignment handling let fmtOffset = 12; while (fmtOffset < totalSize - 8) { const chunkId = new TextDecoder().decode(concatenated.slice(fmtOffset, fmtOffset + 4)); const chunkSize = view.getUint32(fmtOffset + 4, true); if (chunkId === 'fmt ') { + // Validate minimum fmt chunk size + if (chunkSize < 16) return null; + + const audioFormat = view.getUint16(fmtOffset + 8, true); + if (audioFormat !== 1) return null; // Only PCM supported + const channels = view.getUint16(fmtOffset + 10, true); const sampleRate = view.getUint32(fmtOffset + 12, true); const byteRate = view.getUint32(fmtOffset + 16, true); const blockAlign = view.getUint16(fmtOffset + 20, true); const bitsPerSample = view.getUint16(fmtOffset + 22, true); + // Basic validation + if (channels < 1 || channels > 8) return null; + if (blockAlign !== channels * (bitsPerSample / 8)) return null; + return { sampleRate, channels, @@ -52,7 +62,8 @@ class WavUtils { }; } - fmtOffset += 8 + chunkSize; + // Move to next chunk with proper alignment + fmtOffset += 8 + ((chunkSize + 1) & ~1); // Ensure even alignment } return null; @@ -84,12 +95,15 @@ class WavUtils { return new Uint8Array(header); } - static extractAudioData(chunks: Uint8Array[], totalSize: number, headerSize: number, chunkSize: number): Uint8Array { - const bufferData = new Uint8Array(chunkSize + headerSize); - let dataOffset = headerSize; // Skip header space initially - let remainingSize = chunkSize; + static copyAudioDataDirect(chunks: Uint8Array[], targetBuffer: Uint8Array, targetOffset: number, headerSize: number, audioDataSize: number): number { + // Clear audio data area completely to prevent contamination - KEY FIX + for (let i = targetOffset; i < targetOffset + audioDataSize; i++) { + targetBuffer[i] = 0; + } - // Fill with audio data, skipping the header from the first chunk + // Direct copy of audio data to target buffer, skipping WAV header in first chunk only + let targetPos = targetOffset; + let remainingSize = audioDataSize; let chunkIndex = 0; let chunkOffset = headerSize; // Skip WAV header in first chunk @@ -99,8 +113,8 @@ class WavUtils { const toCopy = Math.min(availableInChunk, remainingSize); if (toCopy > 0) { - bufferData.set(chunk.slice(chunkOffset, chunkOffset + toCopy), dataOffset); - dataOffset += toCopy; + targetBuffer.set(chunk.subarray(chunkOffset, chunkOffset + toCopy), targetPos); + targetPos += toCopy; remainingSize -= toCopy; chunkOffset += toCopy; } @@ -110,8 +124,38 @@ class WavUtils { chunkOffset = 0; // No header to skip in subsequent chunks } } + + return targetPos - targetOffset; // Return actual bytes copied + } - return bufferData.slice(0, dataOffset); + static patchHeaderSizes(buffer: Uint8Array, audioDataSize: number): void { + // Patch file size (offset 4) and data chunk size (offset 40) - little endian, 4 bytes each + const fileSize = 36 + audioDataSize; + buffer[4] = fileSize & 0xFF; + buffer[5] = (fileSize >> 8) & 0xFF; + buffer[6] = (fileSize >> 16) & 0xFF; + buffer[7] = (fileSize >> 24) & 0xFF; + buffer[40] = audioDataSize & 0xFF; + buffer[41] = (audioDataSize >> 8) & 0xFF; + buffer[42] = (audioDataSize >> 16) & 0xFF; + buffer[43] = (audioDataSize >> 24) & 0xFF; + } + + static getSampleAlignedChunkSize(header: WavHeader, maxChunkSize: number, availableDataSize: number): number { + const frameSize = header.blockAlign; + + // Much smaller minimum for streaming - just enough for Web Audio API + const minAudioBytes = Math.max(512, frameSize * 10); // At least 512 bytes or 10 frames + + // If we don't have enough data, return 0 to wait for more + if (availableDataSize < minAudioBytes) { + return 0; + } + + // Calculate frames for the available data + const requestedSize = Math.min(maxChunkSize, availableDataSize); + const frames = Math.floor(requestedSize / frameSize); + return frames * frameSize; } } diff --git a/DeepDrftWeb/Interop/webaudio.ts b/DeepDrftWeb/Interop/webaudio.ts index ecf5571..af0908d 100644 --- a/DeepDrftWeb/Interop/webaudio.ts +++ b/DeepDrftWeb/Interop/webaudio.ts @@ -58,9 +58,9 @@ class AudioPlayer { private onEndCallback: EndCallback | null = null; private progressInterval: number | null = null; private bufferChunks: Uint8Array[] = []; - private expectedSize: number = 0; private currentSize: number = 0; - + private processedBytes: number = 0; // Track how many bytes we've already processed + // Streaming properties private isStreamingMode: boolean = false; private wavHeader: WavHeader | null = null; @@ -68,12 +68,10 @@ class AudioPlayer { private currentStreamSource: AudioBufferSourceNode | null = null; private nextStartTime: number = 0; private streamingStarted: boolean = false; - private minBuffersForStreaming: number = 3; + private streamingCompleted: boolean = false; // Track if streaming is finished + private totalStreamLength: number = 0; // Total bytes expected in stream + private minBuffersForStreaming: number = 6; // Increased for better buffering - // Buffer optimization - private cachedWavHeader: Uint8Array | null = null; - private reusableBuffer: Uint8Array | null = null; - private maxReusableBufferSize: number = 128 * 1024; // 128KB max reusable buffer async initialize(): Promise { try { @@ -81,9 +79,26 @@ class AudioPlayer { if (!AudioContextClass) { throw new Error('Web Audio API not supported'); } - this.audioContext = new AudioContextClass(); + + // Initialize with 44.1kHz for music (most common rate) to avoid recreation + this.audioContext = new AudioContextClass({ sampleRate: 44100 }); this.gainNode = this.audioContext.createGain(); this.gainNode.connect(this.audioContext.destination); + + console.log(`AudioContext initialized: sampleRate=${this.audioContext.sampleRate}Hz, state=${this.audioContext.state}`); + return { success: true }; + } catch (error) { + return { success: false, error: (error as Error).message }; + } + } + + async ensureAudioContextReady(): Promise { + try { + if (this.audioContext!.state === 'suspended') { + console.log('πŸ”Š Resuming AudioContext on track selection (user interaction)'); + await this.audioContext!.resume(); + console.log(`βœ… AudioContext resumed: state=${this.audioContext!.state}`); + } return { success: true }; } catch (error) { return { success: false, error: (error as Error).message }; @@ -94,7 +109,6 @@ class AudioPlayer { try { this.bufferChunks = []; this.currentSize = 0; - this.expectedSize = 0; return { success: true }; } catch (error) { return { success: false, error: (error as Error).message }; @@ -320,39 +334,44 @@ class AudioPlayer { this.onEndCallback = callback; } - initializeStreaming(): AudioResult { + initializeStreaming(totalStreamLength: number): AudioResult { try { this.isStreamingMode = true; this.bufferChunks = []; this.bufferQueue = []; this.currentSize = 0; + this.processedBytes = 0; // Reset stream position + this.totalStreamLength = totalStreamLength; // Set total expected stream length this.wavHeader = null; this.streamingStarted = false; + this.streamingCompleted = false; // Reset completion flag this.nextStartTime = 0; + + console.log(`Streaming initialized: expecting ${this.totalStreamLength} total bytes`); + return { success: true }; } catch (error) { return { success: false, error: (error as Error).message }; } } - processStreamingChunk(audioChunk: Uint8Array): StreamingResult { + private chunkCounter = 0; + + async processStreamingChunk(audioChunk: Uint8Array): Promise { try { - this.bufferChunks.push(audioChunk); - this.currentSize += audioChunk.length; + this.chunkCounter++; + console.log(`\n=== CHUNK ${this.chunkCounter} ===`); + console.log(`Incoming chunk size: ${audioChunk.length}`); + console.log(`Chunk preview:`, Array.from(audioChunk.slice(0, 32)).map(b => b.toString(16).padStart(2, '0')).join(' ')); + console.log(`Buffer queue length before processing: ${this.bufferQueue.length}`); - // Parse WAV header from first chunk if not done yet - if (!this.wavHeader && this.currentSize >= 44) { - const header = WavUtils.parseHeader(this.bufferChunks, this.currentSize); - if (header) { - this.wavHeader = header; - // Cache the WAV header for reuse - this.cachedWavHeader = WavUtils.createHeader(header, 64 * 1024); // Cache with dummy size - } - } + await this.processChunk(audioChunk); - // Try to create audio buffers from accumulated chunks - if (this.wavHeader) { - this.processBufferedChunks(); + // Check if we've received all expected data + console.log(`Stream check: ${this.currentSize}/${this.totalStreamLength} bytes, completed=${this.streamingCompleted}`); + if (this.totalStreamLength > 0 && this.currentSize >= this.totalStreamLength) { + console.log(`Stream complete: received ${this.currentSize}/${this.totalStreamLength} bytes`); + this.streamingCompleted = true; } const canStart = this.wavHeader !== null && this.bufferQueue.length >= this.minBuffersForStreaming; @@ -368,115 +387,166 @@ class AudioPlayer { } } - startStreamingPlayback(): AudioResult { - if (!this.wavHeader || this.bufferQueue.length === 0) { - return { success: false, error: "Not ready for streaming playback" }; + + private isFirstChunk = true; + + private async processChunk(audioChunk: Uint8Array): Promise { + if (this.isFirstChunk) { + const audioData = await this.extractAudioFromFirstChunk(audioChunk); + this.addToAudioStream(audioData); + this.isFirstChunk = false; + } else { + // Continuation chunks are pure audio data + this.addToAudioStream(audioChunk); } - - try { - if (this.audioContext!.state === 'suspended') { - this.audioContext!.resume(); - } - - this.streamingStarted = true; - this.isPlaying = true; - this.isPaused = false; - this.nextStartTime = this.audioContext!.currentTime; - this.startTime = this.nextStartTime; - - this.scheduleNextBuffer(); - this.startProgressTracking(); - - return { success: true }; - } catch (error) { - return { success: false, error: (error as Error).message }; - } - } - - - private processBufferedChunks(): void { - if (!this.wavHeader || this.bufferChunks.length === 0) return; - - try { - // Process chunks in groups to create audio buffers - const chunkSize = 64 * 1024; // 64KB chunks for streaming - while (this.currentSize >= chunkSize + this.wavHeader.headerSize) { - // Extract audio data using WavUtils - const audioData = WavUtils.extractAudioData(this.bufferChunks, this.currentSize, this.wavHeader.headerSize, chunkSize); - - // Reuse buffer if possible to reduce allocations - const totalSize = this.cachedWavHeader!.length + audioData.length - this.wavHeader.headerSize; - if (!this.reusableBuffer || this.reusableBuffer.length < totalSize) { - // Only allocate if we don't have a buffer or it's too small - this.reusableBuffer = new Uint8Array(Math.min(totalSize, this.maxReusableBufferSize)); - } - - // Create complete WAV buffer using cached header and reusable buffer - const completeBuffer = this.reusableBuffer.slice(0, totalSize); - completeBuffer.set(this.cachedWavHeader!.slice(0, this.wavHeader.headerSize), 0); - completeBuffer.set(audioData.subarray(this.wavHeader.headerSize), this.wavHeader.headerSize); - - // Create audio buffer from the chunk - this.createAudioBufferFromChunk(completeBuffer); - - // Remove processed data - this.removeProcessedChunks(chunkSize); - break; // Process one chunk at a time - } - } catch (error) { - console.error('Error processing buffered chunks:', error); - } - } - - private async createAudioBufferFromChunk(chunkData: Uint8Array): Promise { - try { - const arrayBuffer = chunkData.buffer.slice(chunkData.byteOffset, chunkData.byteOffset + chunkData.byteLength); - const audioBuffer = await this.audioContext!.decodeAudioData(arrayBuffer); - this.bufferQueue.push(audioBuffer); - - // Schedule buffer if streaming has started - if (this.streamingStarted) { - this.scheduleNextBuffer(); - } - } catch (error) { - console.error('Error creating audio buffer from chunk:', error); - } - } - - private scheduleNextBuffer(): void { - if (this.bufferQueue.length === 0 || !this.streamingStarted) return; - - const buffer = this.bufferQueue.shift()!; - const source = this.audioContext!.createBufferSource(); - source.buffer = buffer; - source.connect(this.gainNode!); - source.onended = () => { - if (this.bufferQueue.length > 0) { - this.scheduleNextBuffer(); - } else if (!this.isPlaying) { - this.onEndCallback?.(); - } - }; - - source.start(this.nextStartTime); - this.nextStartTime += buffer.duration; - this.currentStreamSource = source; + await this.processAudioStream(); } + private async extractAudioFromFirstChunk(chunkData: Uint8Array): Promise { + console.log('\n--- EXTRACTING AUDIO FROM FIRST CHUNK ---'); + + // Parse header and setup AudioContext + const header = WavUtils.parseHeader([chunkData], chunkData.length); + if (!header) { + throw new Error('Invalid WAV header in first chunk'); + } + + this.wavHeader = header; + console.log(`WAV format: ${header.bitsPerSample}-bit, ${header.channels}ch, ${header.sampleRate}Hz`); + console.log(`Header details: blockAlign=${header.blockAlign}, byteRate=${header.byteRate}, headerSize=${header.headerSize}`); + + // Recreate AudioContext with correct sample rate if needed (only during initial setup) + if (this.audioContext!.sampleRate !== header.sampleRate) { + console.log(`πŸ”„ AudioContext sample rate mismatch: ${this.audioContext!.sampleRate}Hz -> ${header.sampleRate}Hz`); - private removeProcessedChunks(processedSize: number): void { + // Only recreate if we haven't started playing yet AND AudioContext is already running + if (!this.streamingStarted && !this.isPlaying && this.audioContext!.state === 'running') { + console.log(`⚠️ Recreating AudioContext for proper sample rate matching`); + await this.audioContext!.close(); + + const AudioContextClass = window.AudioContext || window.webkitAudioContext; + this.audioContext = new AudioContextClass({ sampleRate: header.sampleRate }); + + this.gainNode = this.audioContext.createGain(); + this.gainNode.connect(this.audioContext.destination); + + console.log(`βœ… AudioContext recreated: ${this.audioContext.sampleRate}Hz (should eliminate resampling artifacts)`); + } else { + console.log(`ℹ️ Keeping existing AudioContext - using Web Audio API sample rate conversion`); + } + } + + // Extract pure audio data (skip WAV header) + const audioData = chunkData.subarray(header.headerSize); + console.log(`Extracted ${audioData.length} bytes of audio data (skipped ${header.headerSize} byte header)`); + + return audioData; + } + + private async ensureCorrectSampleRate(sampleRate: number): Promise { + if (this.audioContext!.sampleRate !== sampleRate) { + console.log(`πŸ”Š AUDIO CONTEXT CHANGE START: ${this.audioContext!.sampleRate}Hz -> ${sampleRate}Hz`); + console.log(`⚠️ This may cause an audible pop/click!`); + + await this.audioContext!.close(); + console.log(`βœ… Old AudioContext closed`); + + const AudioContextClass = window.AudioContext || window.webkitAudioContext; + this.audioContext = new AudioContextClass({ sampleRate }); + console.log(`βœ… New AudioContext created: actual=${this.audioContext.sampleRate}Hz (requested=${sampleRate}Hz)`); + + this.gainNode = this.audioContext.createGain(); + this.gainNode.connect(this.audioContext.destination); + console.log(`πŸ”Š AUDIO CONTEXT CHANGE COMPLETE`); + } + } + + private addToAudioStream(audioData: Uint8Array): void { + this.bufferChunks.push(audioData); + this.currentSize += audioData.length; + console.log(`Added ${audioData.length} bytes to audio stream (total: ${this.currentSize} bytes)`); + } + + private async processAudioStream(): Promise { + if (!this.wavHeader) return; + + // Process available data (but don't over-process during active playback) + if (this.streamingStarted && this.bufferQueue.length >= 2) { + console.log(`Buffer queue has cushion (${this.bufferQueue.length}), minimal processing`); + // Still process but be less aggressive + } + + // Create sample-aligned segments from continuous audio stream + const maxSegmentSize = 64 * 1024; // 64KB segments to match C# chunks better + const availableBytes = this.currentSize - this.processedBytes; // Only count unprocessed bytes + const alignedSize = WavUtils.getSampleAlignedChunkSize(this.wavHeader, maxSegmentSize, availableBytes); + + if (alignedSize > 0) { + console.log(`\n--- CREATING ALIGNED AUDIO SEGMENT ---`); + console.log(`Available: ${availableBytes} bytes, requesting: ${alignedSize} bytes (frame-aligned, frame size: ${this.wavHeader.blockAlign})`); + console.log(`Buffer queue: ${this.bufferQueue.length}, processing chunk`); + + // Extract sample-aligned segment from continuous stream + const alignedSegment = this.extractAlignedData(alignedSize); + const wavFile = this.createWavFromRawData(alignedSegment); + + await this.createAudioBufferFromChunk(wavFile); + // Note: No longer removing processed data - we track position instead + } + } + + private extractAlignedData(alignedSize: number): Uint8Array { + const extracted = new Uint8Array(alignedSize); + let extractedOffset = 0; + let remaining = alignedSize; + let streamPosition = this.processedBytes; // Start from where we left off + let currentPos = 0; + + for (const chunk of this.bufferChunks) { + if (remaining <= 0) break; + + // Skip chunks that are entirely before our current stream position + if (currentPos + chunk.length <= streamPosition) { + currentPos += chunk.length; + continue; + } + + // Calculate the offset within this chunk to start extracting + const chunkStartOffset = Math.max(0, streamPosition - currentPos); + const availableInChunk = chunk.length - chunkStartOffset; + const toCopy = Math.min(availableInChunk, remaining); + + if (toCopy > 0) { + extracted.set(chunk.subarray(chunkStartOffset, chunkStartOffset + toCopy), extractedOffset); + extractedOffset += toCopy; + remaining -= toCopy; + } + + currentPos += chunk.length; + } + + // Update processed bytes position + this.processedBytes += alignedSize; + console.log(`Extracted ${alignedSize} bytes from stream position ${streamPosition} -> ${this.processedBytes}`); + + return extracted; + } + + private removeProcessedData(processedSize: number): void { let remaining = processedSize; while (remaining > 0 && this.bufferChunks.length > 0) { - const chunk = this.bufferChunks[0]; - if (chunk.length <= remaining) { - remaining -= chunk.length; - this.currentSize -= chunk.length; + const firstChunk = this.bufferChunks[0]; + + if (firstChunk.length <= remaining) { + // Remove entire chunk + remaining -= firstChunk.length; + this.currentSize -= firstChunk.length; this.bufferChunks.shift(); } else { - // Partial chunk removal - const newChunk = chunk.slice(remaining); + // Partially remove chunk + const newChunk = firstChunk.subarray(remaining); this.bufferChunks[0] = newChunk; this.currentSize -= remaining; remaining = 0; @@ -484,6 +554,203 @@ class AudioPlayer { } } + private concatenateChunks(): Uint8Array { + const totalSize = this.currentSize; + const concatenated = new Uint8Array(totalSize); + let offset = 0; + + for (const chunk of this.bufferChunks) { + concatenated.set(chunk, offset); + offset += chunk.length; + } + + return concatenated; + } + + private createWavFromRawData(rawData: Uint8Array): Uint8Array { + const header = WavUtils.createHeader(this.wavHeader!, rawData.length); + const wavFile = new Uint8Array(header.length + rawData.length); + wavFile.set(header, 0); + wavFile.set(rawData, header.length); + + console.log(`Created WAV: header=${header.length} bytes, data=${rawData.length} bytes, total=${wavFile.length} bytes`); + console.log(`Expected duration: ${rawData.length / this.wavHeader!.byteRate} seconds`); + + return wavFile; + } + + + + + startStreamingPlayback(): AudioResult { + if (!this.wavHeader || this.bufferQueue.length === 0) { + return { success: false, error: "Not ready for streaming playback" }; + } + + try { + console.log(`\n=== STARTING STREAMING PLAYBACK ===`); + console.log(`AudioContext state: ${this.audioContext!.state}`); + console.log(`AudioContext sample rate: ${this.audioContext!.sampleRate}Hz`); + console.log(`Current time precision: ${this.audioContext!.currentTime.toFixed(6)}s`); + console.log(`Queue ready: ${this.bufferQueue.length} buffers, ${this.bufferQueue.reduce((sum, b) => sum + b.duration, 0).toFixed(3)}s total`); + + // AudioContext should already be resumed during track selection + + const startTimestamp = performance.now(); + const audioContextTime = this.audioContext!.currentTime; + + this.streamingStarted = true; + this.isPlaying = true; + this.isPaused = false; + this.nextStartTime = audioContextTime; + this.startTime = this.nextStartTime; + + console.log(`▢️ Playback timing: audioContext=${audioContextTime.toFixed(6)}s, performance=${startTimestamp.toFixed(3)}ms`); + console.log(`🎡 Initial nextStartTime set to: ${this.nextStartTime.toFixed(6)}s`); + + this.scheduleNextBuffer(); + this.startProgressTracking(); + + console.log(`βœ… Streaming playback started successfully`); + console.log(`=====================================\n`); + + return { success: true }; + } catch (error) { + console.error(`❌ Failed to start streaming playback:`, error); + return { success: false, error: (error as Error).message }; + } + } + + + + private async createAudioBufferFromChunk(chunkData: Uint8Array): Promise { + try { + console.log(`createAudioBufferFromChunk: chunkData.length=${chunkData.length}`); + + // Create a clean ArrayBuffer with exact size (avoid reusable buffer issues) + const cleanBuffer = new ArrayBuffer(chunkData.length); + new Uint8Array(cleanBuffer).set(chunkData); + + console.log(`Decoding ${cleanBuffer.byteLength} bytes with Web Audio API`); + console.log('Starting decode...'); + + // Try with timeout to catch hanging decodes + const decodePromise = this.audioContext!.decodeAudioData(cleanBuffer); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Decode timeout after 5 seconds')), 5000); + }); + + const audioBuffer = await Promise.race([decodePromise, timeoutPromise]); + console.log("AFTER Promise.race - this should always appear after 5 seconds max"); + console.log(`\n--- DECODE SUCCESS ---`); + console.log(`Buffer duration: ${audioBuffer.duration}s`); + console.log(`Buffer channels: ${audioBuffer.numberOfChannels}`); + console.log(`Buffer sample rate: ${audioBuffer.sampleRate}`); + console.log(`Buffer length: ${audioBuffer.length} samples`); + + // Check if buffer contains actual audio data or silence/noise + const channel0 = audioBuffer.getChannelData(0); + const firstSamples = Array.from(channel0.slice(0, 10)).map(v => v.toFixed(4)); + const maxValue = Math.max(...Array.from(channel0).map(Math.abs)); + const avgValue = Array.from(channel0).reduce((sum, val) => sum + Math.abs(val), 0) / channel0.length; + console.log(`First 10 samples:`, firstSamples); + console.log(`Max amplitude: ${maxValue.toFixed(4)}`); + console.log(`Average amplitude: ${avgValue.toFixed(4)}`); + + this.bufferQueue.push(audioBuffer); + + console.log(`\n=== BUFFER QUEUE UPDATE ===`); + console.log(`βœ“ Added buffer: duration=${audioBuffer.duration.toFixed(6)}s, samples=${audioBuffer.length}`); + console.log(`Queue state: ${this.bufferQueue.length} buffers (${this.bufferQueue.map(b => b.duration.toFixed(3)).join('s, ')}s)`); + console.log(`Total queued audio: ${this.bufferQueue.reduce((sum, b) => sum + b.duration, 0).toFixed(3)}s`); + console.log(`Streaming: started=${this.streamingStarted}, completed=${this.streamingCompleted}`); + console.log(`Current playback time: ${this.audioContext!.currentTime.toFixed(6)}s`); + + // Schedule immediately when streaming has started (for gapless playback) + if (this.streamingStarted) { + console.log(`⏩ Triggering proactive schedule (streaming active)`); + this.scheduleNextBuffer(); + } else { + console.log(`⏸️ Not scheduling yet (streaming not started)`); + } + console.log(`===========================\n`); + } catch (error) { + console.error('Error creating audio buffer from chunk:', error); + console.error('Failed chunk size:', chunkData.length); + // Log first few bytes of the chunk for debugging + const preview = Array.from(chunkData.slice(0, 16)).map(b => b.toString(16).padStart(2, '0')).join(' '); + console.error('Chunk preview (first 16 bytes):', preview); + } + } + + private scheduleNextBuffer(): void { + // Schedule all available buffers proactively instead of waiting for onended + while (this.bufferQueue.length > 0 && this.streamingStarted) { + const scheduleStartTime = performance.now(); + const buffer = this.bufferQueue.shift()!; + const source = this.audioContext!.createBufferSource(); + source.buffer = buffer; + source.connect(this.gainNode!); + + // Critical: Use precise timing for gapless playback + const currentTime = this.audioContext!.currentTime; + // For the very first buffer, add small lookahead to avoid startup glitches + const startTime = this.nextStartTime > 0 ? this.nextStartTime : currentTime + 0.01; + const schedulingDelay = currentTime - startTime; + + console.log(`🎡 Scheduling buffer: start=${startTime.toFixed(3)}s, duration=${buffer.duration.toFixed(3)}s, delay=${(schedulingDelay * 1000).toFixed(1)}ms ${schedulingDelay > 0.005 ? '⚠️' : 'βœ“'}, queue=${this.bufferQueue.length}`); + + // Only log timing issues for debugging + const gap = Math.abs(startTime - this.nextStartTime); + if (gap > 0.001) { + console.warn(`⚠️ TIMING GAP: ${(gap * 1000).toFixed(3)}ms between expected and actual start time`); + } + + source.onended = () => { + const endTime = this.audioContext!.currentTime; + const expectedEndTime = startTime + buffer.duration; + const timingError = Math.abs(endTime - expectedEndTime); + + console.log(`🏁 Buffer ended: timing error=${(timingError * 1000).toFixed(1)}ms`); + + this.currentStreamSource = null; + + // Check for end-of-stream + if (this.bufferQueue.length === 0) { + if (this.streamingCompleted) { + console.log(`βœ“ End-of-stream: All buffers played at ${endTime.toFixed(3)}s (expected)`); + } else { + console.warn(`❌ Buffer underrun! Queue empty at ${endTime.toFixed(3)}s (unexpected during streaming)`); + } + + if (!this.isPlaying) { + this.onEndCallback?.(); + } + } + }; + + + source.start(startTime); + + // Calculate next start time with sample-perfect precision + this.nextStartTime = startTime + buffer.duration; + this.currentStreamSource = source; + + const scheduleEndTime = performance.now(); + const scheduleProcessingTime = scheduleEndTime - scheduleStartTime; + + + // Stop scheduling when we have enough buffered ahead + const lookaheadTime = this.nextStartTime - currentTime; + if (lookaheadTime > 0.5) { // Stop when we have 500ms of audio scheduled ahead + console.log(`πŸ“‹ Sufficient lookahead: ${(lookaheadTime * 1000).toFixed(0)}ms scheduled ahead`); + break; + } + } + } + + + unload(): AudioResult { try { this.stop(); @@ -491,22 +758,21 @@ class AudioPlayer { this.duration = 0; this.bufferChunks = []; this.currentSize = 0; - this.expectedSize = 0; - + this.processedBytes = 0; // Reset stream position + // Clean up streaming state this.isStreamingMode = false; this.wavHeader = null; this.bufferQueue = []; this.streamingStarted = false; + this.streamingCompleted = false; + this.totalStreamLength = 0; this.nextStartTime = 0; if (this.currentStreamSource) { this.currentStreamSource.stop(); this.currentStreamSource = null; } - // Clean up cached buffers - this.cachedWavHeader = null; - this.reusableBuffer = null; return { success: true }; } catch (error) { @@ -530,8 +796,6 @@ class AudioPlayer { this.bufferQueue = []; this.wavHeader = null; this.currentStreamSource = null; - this.cachedWavHeader = null; - this.reusableBuffer = null; } } @@ -583,20 +847,20 @@ const DeepDrftAudio = { }, // Streaming methods - initializeStreaming: (playerId: string): AudioResult => { + initializeStreaming: (playerId: string, totalStreamLength: number): AudioResult => { const player = audioPlayers.get(playerId); if (!player) { return { success: false, error: "Player not found" }; } - return player.initializeStreaming(); + return player.initializeStreaming(totalStreamLength); }, - processStreamingChunk: (playerId: string, audioChunk: Uint8Array): StreamingResult => { + processStreamingChunk: async (playerId: string, audioChunk: Uint8Array): Promise => { const player = audioPlayers.get(playerId); if (!player) { return { success: false, error: "Player not found" }; } - return player.processStreamingChunk(audioChunk); + return await player.processStreamingChunk(audioChunk); }, startStreamingPlayback: (playerId: string): AudioResult => { @@ -607,6 +871,14 @@ const DeepDrftAudio = { return player.startStreamingPlayback(); }, + ensureAudioContextReady: async (playerId: string): Promise => { + const player = audioPlayers.get(playerId); + if (!player) { + return { success: false, error: "Player not found" }; + } + return await player.ensureAudioContextReady(); + }, + play: (playerId: string): AudioResult => { const player = audioPlayers.get(playerId); if (!player) { diff --git a/DeepDrftWeb/Program.cs b/DeepDrftWeb/Program.cs index 39fe4b2..43a1bfa 100644 --- a/DeepDrftWeb/Program.cs +++ b/DeepDrftWeb/Program.cs @@ -91,6 +91,18 @@ if (app.Environment.IsDevelopment()) } app.MapStaticAssets(); + +// Serve TypeScript source files for debugging in development +if (app.Environment.IsDevelopment()) +{ + app.UseStaticFiles(new StaticFileOptions + { + FileProvider = new Microsoft.Extensions.FileProviders.PhysicalFileProvider( + Path.Combine(app.Environment.ContentRootPath, "Interop")), + RequestPath = "/Interop" + }); +} + app.MapControllers(); app.MapRazorComponents() .AddInteractiveServerRenderMode() diff --git a/DeepDrftWeb/appsettings.Development.json b/DeepDrftWeb/appsettings.Development.json index cef5f32..88066b5 100644 --- a/DeepDrftWeb/appsettings.Development.json +++ b/DeepDrftWeb/appsettings.Development.json @@ -2,7 +2,14 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Warning", + "DeepDrftWeb.Client.Services.StreamingAudioPlayerService": "Debug" + }, + "Console": { + "FormatterName": "simple", + "LogLevel": { + "Default": "Information" + } } }, "DetailedErrors": true, diff --git a/DeepDrftWeb/tsconfig.json b/DeepDrftWeb/tsconfig.json index bed2f8c..4d11659 100644 --- a/DeepDrftWeb/tsconfig.json +++ b/DeepDrftWeb/tsconfig.json @@ -10,7 +10,9 @@ "noEmitOnError": true, "removeComments": false, "sourceMap": true, - "outDir": "wwwroot/js" + "outDir": "wwwroot/js", + "sourceRoot": "/Interop", + "mapRoot": "/js" }, "include": [ "Interop/**/*.ts" diff --git a/global.json b/global.json new file mode 100644 index 0000000..f4fd385 --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "9.0.0", + "rollForward": "latestMajor", + "allowPrerelease": true + } +} \ No newline at end of file