diff --git a/CLAUDE.md b/CLAUDE.md index 8ff5090..78cff39 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,10 +20,12 @@ C# .NET 10 console app for scraping and converting audiobooks (M4B/MP3) from mul IScraperStrategy (Scraper/Abstractions/) └── BaseScraperStrategy (Scraper/Base/) ├── TokybookStrategy (Scraper/Strategies/) — tokybook.com - ├── ZAudiobooksStrategy (Scraper/Strategies/) — zaudiobooks / freeaudiobooks.top - ├── GoldenAudiobookStrategy (Scraper/Strategies/) — all sites with `` structure: + ├── DropboxTracksStrategy (Scraper/Strategies/) — sites with JS `tracks = [{ chapter_link_dropbox }]` structure: + │ zaudiobooks.com, freeaudiobooks.top + ├── AudioSourceTagStrategy (Scraper/Strategies/) — sites with `` or `` structure: │ goldenaudiobook.net, fulllengthaudiobooks.net, bigaudiobooks.net, - │ findaudiobook.com, bookaudiobook.net, hotaudiobooks.com, audiozaic.com + │ findaudiobook.com, bookaudiobook.net, hotaudiobooks.com, audiozaic.com, + │ appaudiobooks.com ├── PlaylistAudiobookStrategy (Scraper/Strategies/) — all sites with `data-playlist` JSON attribute: │ hdaudiobooks.net └── AudioAzStrategy (Scraper/Strategies/) — Next.js site with tracks JSON in streaming data: @@ -44,16 +46,47 @@ Downloads and conversions run decoupled via `Channel` + `SemaphoreSlim`: ### Track Types - `SegmentedTrackData` — for HLS streams (`.m3u8` → `.ts` segments → merge via FFmpeg concat) -- `DirectFileTrackData` — for direct MP3/audio downloads (zaudiobooks, goldenaudiobook); conversion is skipped when source file is already in the target format +- `DirectFileTrackData` — for direct MP3/audio downloads (zaudiobooks, goldenaudiobook); when the source is already in the target format, a copy-conversion runs to embed metadata without re-encoding + +### Metadata Pipeline + +`BaseScraperStrategy` provides shared metadata infrastructure used by all strategies. + +For all non-Tokybook strategies, metadata is collected in two stages before downloading: + +**Stage 1 — MP3 tag enrichment** (`EnrichFromFirstTrackTagsAsync`): +- Runs `ffprobe` on the first chapter URL to read existing ID3 tags without downloading the file +- Maps: `artist` → `Author`, `date` → `Year`, `comment` → `Description` (skips chapter references like "Chapter 1") +- Only fills empty fields — never overwrites + +**Stage 2 — HTML extraction** (`ExtractCommonMetadata`): +- Only fills fields still empty after Stage 1 +- `og:image` → `CoverArtUrl` +- `og:description` → `Description` +- `", RegexOptions.Singleline | RegexOptions.IgnoreCase)] + private static partial Regex LdJsonRegex(); + + [GeneratedRegex(@"]*\bas=""image""[^>]*/?>", RegexOptions.IgnoreCase)] + private static partial Regex PreloadImageTagRegex(); + + [GeneratedRegex(@"\bhref=""([^""]+)""")] + private static partial Regex HrefAttributeRegex(); + + [GeneratedRegex(@"<[^>]+>")] + private static partial Regex HtmlTagsRegex(); + + private static readonly string[] _headlineSuffixes = + ["Audiobook", "Audio Book", "Free", "Online", "Streaming", "Download", "(AUDIOBOOK)"]; + protected static string SanitizeName(string fileName) { return SanitizeRegex().Replace(fileName, "_"); @@ -38,121 +65,457 @@ protected static string ExtractTitleFromH1(string html) { var match = H1Regex().Match(html); return match.Success - ? System.Net.WebUtility.HtmlDecode(match.Groups[1].Value.Trim()) + ? WebUtility.HtmlDecode(match.Groups[1].Value.Trim()) : "Unknown Title"; } + protected static string CleanupBookTitle(string headline) + { + if (string.IsNullOrEmpty(headline)) return headline; + + // Pattern: "Title [Audiobook] by Author" → title is everything before " by " + var byIndex = headline.IndexOf(" by ", StringComparison.OrdinalIgnoreCase); + if (byIndex > 0) + return StripHeadlineSuffixes(headline[..byIndex].Trim()); + + // Pattern: "Author – Title [Audiobook]" or "Title [Audiobook] – Author" + string[] separators = [" \u2013 ", " \u2014 ", " - "]; + foreach (var sep in separators) + { + var idx = headline.IndexOf(sep, StringComparison.Ordinal); + if (idx <= 0) continue; + + var before = headline[..idx].Trim(); + var after = headline[(idx + sep.Length)..].Trim(); + + // Whichever segment contains a known suffix is the title segment + if (_headlineSuffixes.Any(s => after.Contains(s, StringComparison.OrdinalIgnoreCase))) + return StripHeadlineSuffixes(after); + + if (_headlineSuffixes.Any(s => before.Contains(s, StringComparison.OrdinalIgnoreCase))) + return StripHeadlineSuffixes(before); + + // No suffix found — common pattern is "Author – Title", so take the after segment + return after; + } + + // No separator → just strip suffixes from the whole title + return StripHeadlineSuffixes(headline); + } + + protected static void ExtractCommonMetadata(string html, AudiobookMetadata metadata) + { + // og:image → CoverArtUrl + if (string.IsNullOrEmpty(metadata.CoverArtUrl)) + { + var ogImageTag = OgImageTagRegex().Match(html); + if (ogImageTag.Success) + { + var contentMatch = ContentAttributeRegex().Match(ogImageTag.Value); + if (contentMatch.Success) + metadata.CoverArtUrl = WebUtility.HtmlDecode(contentMatch.Groups[1].Value); + } + } + + // og:description → Description + if (string.IsNullOrEmpty(metadata.Description)) + { + var ogDescTag = OgDescriptionTagRegex().Match(html); + if (ogDescTag.Success) + { + var contentMatch = ContentAttributeRegex().Match(ogDescTag.Value); + if (contentMatch.Success) + metadata.Description = WebUtility.HtmlDecode(contentMatch.Groups[1].Value); + } + } + + // ld+json blocks: look for @type "Audiobook" first, then "Article"/"WebPage" for author via headline + var ldJsonMatches = LdJsonRegex().Matches(html); + string headlineForAuthor = string.Empty; + + foreach (Match ldMatch in ldJsonMatches) + { + try + { + var json = JObject.Parse(ldMatch.Groups[1].Value); + var type = json["@type"]?.ToString(); + + if (type == "Audiobook") + { + if (string.IsNullOrEmpty(metadata.Author)) + metadata.Author = json["author"]?["name"]?.ToString() ?? string.Empty; + if (string.IsNullOrEmpty(metadata.Narrator)) + metadata.Narrator = json["readBy"]?["name"]?.ToString() ?? string.Empty; + if (string.IsNullOrEmpty(metadata.Description)) + metadata.Description = json["description"]?.ToString() ?? string.Empty; + if (string.IsNullOrEmpty(metadata.CoverArtUrl)) + metadata.CoverArtUrl = json["image"]?.ToString() ?? string.Empty; + break; + } + + if (string.IsNullOrEmpty(headlineForAuthor) && (type == "Article" || type == "WebPage")) + headlineForAuthor = json["headline"]?.ToString() ?? json["name"]?.ToString() ?? string.Empty; + } + catch { } + } + + // Fallback: cover art from + if (string.IsNullOrEmpty(metadata.CoverArtUrl)) + { + var preloadTag = PreloadImageTagRegex().Match(html); + if (preloadTag.Success) + { + var hrefMatch = HrefAttributeRegex().Match(preloadTag.Value); + if (hrefMatch.Success) + { + var href = hrefMatch.Groups[1].Value; + if (href.Contains(".jpg", StringComparison.OrdinalIgnoreCase) + || href.Contains(".jpeg", StringComparison.OrdinalIgnoreCase) + || href.Contains(".png", StringComparison.OrdinalIgnoreCase) + || href.Contains(".webp", StringComparison.OrdinalIgnoreCase)) + metadata.CoverArtUrl = href; + } + } + } + + // Fallback: extract author from ld+json headline, then from H1 title + if (string.IsNullOrEmpty(metadata.Author)) + { + var source = !string.IsNullOrEmpty(headlineForAuthor) ? headlineForAuthor : metadata.Title; + metadata.Author = ExtractAuthorFromHeadline(source); + } + } + + protected static string ExtractAuthorFromHeadline(string headline) + { + if (string.IsNullOrEmpty(headline)) return string.Empty; + + // Pattern: "Title [Audiobook] by Author [suffix]" + var byIndex = headline.IndexOf(" by ", StringComparison.OrdinalIgnoreCase); + if (byIndex > 0) + { + var after = StripHeadlineSuffixes(headline[(byIndex + 4)..].Trim()); + if (!string.IsNullOrEmpty(after)) + return after; + } + + // Pattern: "Author – Title" or "Author - Title" (en dash, em dash, hyphen) + string[] separators = [" \u2013 ", " \u2014 ", " - "]; + foreach (var sep in separators) + { + var idx = headline.IndexOf(sep, StringComparison.Ordinal); + if (idx <= 0) continue; + + var before = headline[..idx].Trim(); + var after = headline[(idx + sep.Length)..].Trim(); + + // If the first segment contains "Audiobook" the author is in the second segment + if (before.Contains("Audiobook", StringComparison.OrdinalIgnoreCase) || before.Contains('[')) + return StripHeadlineSuffixes(after); + + return StripHeadlineSuffixes(before); + } + + return string.Empty; + } + + protected async Task EnrichFromFirstTrackTagsAsync(SimpleAudiobookMetadata metadata, StatusContext? ctx = null) + { + if (metadata.ChapterUrls.Count == 0 || string.IsNullOrEmpty(_settings.FFmpegDirectory)) + return; + + try + { + var ffprobeName = OperatingSystem.IsWindows() ? "ffprobe.exe" : "ffprobe"; + var ffprobePath = Path.Combine(_settings.FFmpegDirectory, ffprobeName); + if (!File.Exists(ffprobePath)) return; + + ctx?.Status("Reading original MP3 tags..."); + + // -probesize and -analyzeduration 0 limit ffprobe to reading only the file header + // (ID3v2 tags are at the start of MP3 files, so 64 KB is more than enough). + // -show_entries format_tags avoids fetching duration/bitrate which can require + // reading much further into the file or even to the end. + using var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = ffprobePath, + Arguments = $"-v quiet -probesize 65536 -analyzeduration 0 -print_format json -show_entries format_tags \"{metadata.ChapterUrls[0]}\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.Start(); + var output = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync(); + + if (string.IsNullOrWhiteSpace(output)) return; + + var json = JObject.Parse(output); + var tags = json["format"]?["tags"] as JObject; + if (tags == null) return; + + if (string.IsNullOrEmpty(metadata.Author)) + { + var artist = tags["artist"]?.ToString(); + if (!string.IsNullOrEmpty(artist)) + metadata.Author = artist; + } + + if (string.IsNullOrEmpty(metadata.Year)) + { + var date = tags["date"]?.ToString(); + if (!string.IsNullOrEmpty(date) && date.Length >= 4 && date[..4].All(char.IsDigit)) + metadata.Year = date[..4]; + } + + if (string.IsNullOrEmpty(metadata.Description)) + { + var comment = tags["comment"]?.ToString(); + if (!string.IsNullOrEmpty(comment) && !IsChapterReference(comment)) + metadata.Description = comment; + } + } + catch { } + } + + private static bool IsChapterReference(string comment) + { + const string prefix = "Chapter "; + return comment.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + && comment.Length < 20 + && comment[prefix.Length..].All(char.IsDigit); + } + + private static string StripHeadlineSuffixes(string text) + { + var result = text.Trim(); + bool stripped; + do + { + stripped = false; + foreach (var suffix in _headlineSuffixes) + { + if (result.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) + { + result = result[..^suffix.Length].Trim().TrimEnd('-', '\u2013', ':', ',').Trim(); + stripped = true; + break; + } + } + } while (stripped && !string.IsNullOrEmpty(result)); + return result; + } + + protected static string BuildMetadataParams(AudiobookMetadata? bookMetadata, TrackData? trackData, bool hasCoverArt) + { + if (bookMetadata == null) return string.Empty; + + var parts = new List(); + + var trackTitle = trackData?.TrackTitle; + if (!string.IsNullOrEmpty(trackTitle)) + { + var ext = Path.GetExtension(trackTitle); + if (!string.IsNullOrEmpty(ext)) + { + var nameWithoutExt = Path.GetFileNameWithoutExtension(trackTitle); + trackTitle = int.TryParse(nameWithoutExt, out _) + ? $"Chapter {trackData!.TrackNumber}" + : nameWithoutExt; + } + parts.Add($"-metadata title=\"{EscapeMetadata(trackTitle)}\""); + } + + if (!string.IsNullOrEmpty(bookMetadata.Title)) + parts.Add($"-metadata album=\"{EscapeMetadata(bookMetadata.Title)}\""); + + if (!string.IsNullOrEmpty(bookMetadata.Author)) + parts.Add($"-metadata artist=\"{EscapeMetadata(bookMetadata.Author)}\""); + + if (!string.IsNullOrEmpty(bookMetadata.Narrator)) + parts.Add($"-metadata album_artist=\"{EscapeMetadata(bookMetadata.Narrator)}\""); + + if (trackData != null && trackData.TotalTracks > 0) + parts.Add($"-metadata track=\"{trackData.TrackNumber}/{trackData.TotalTracks}\""); + + parts.Add("-metadata genre=\"Audiobook\""); + + if (!string.IsNullOrEmpty(bookMetadata.Description)) + { + var desc = StripHtml(bookMetadata.Description); + if (desc.Length > 500) desc = desc[..500]; + parts.Add($"-metadata comment=\"{EscapeMetadata(desc)}\""); + } + + if (!string.IsNullOrEmpty(bookMetadata.Publisher)) + parts.Add($"-metadata publisher=\"{EscapeMetadata(bookMetadata.Publisher)}\""); + + if (!string.IsNullOrEmpty(bookMetadata.Year)) + parts.Add($"-metadata date=\"{EscapeMetadata(bookMetadata.Year)}\""); + + if (hasCoverArt) + parts.Add("-c:v copy -metadata:s:v comment=\"Cover (front)\" -disposition:v attached_pic"); + + return string.Join(" ", parts); + } + + private static string EscapeMetadata(string value) => + value.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\n", " ").Replace("\r", ""); + + protected static string StripHtml(string html) => + WebUtility.HtmlDecode(HtmlTagsRegex().Replace(html, " ")).Replace(" ", " ").Trim(); + + protected async Task DownloadCoverArtAsync(string coverArtUrl, string folder) + { + try + { + var uriPath = new Uri(coverArtUrl).AbsolutePath.Split('?')[0]; + var ext = Path.GetExtension(uriPath); + if (string.IsNullOrEmpty(ext)) ext = ".jpg"; + var coverPath = Path.Combine(folder, $"_cover{ext}"); + + var response = await _httpService.GetAsync(coverArtUrl); + if (!response.IsSuccessStatusCode) return null; + + await using var stream = await response.Content.ReadAsStreamAsync(); + await using var file = File.Create(coverPath); + await stream.CopyToAsync(file); + return coverPath; + } + catch + { + return null; + } + } + protected void SetFFmpegPath() => FFmpeg.SetExecutablesPath(_settings.FFmpegDirectory); protected async Task ProcessDirectFilesInParallelAsync(SimpleAudiobookMetadata metadata, string folderPath) { - var conversionChannel = Channel.CreateBounded(new BoundedChannelOptions(10) + string? coverArtPath = null; + if (!string.IsNullOrEmpty(metadata.CoverArtUrl)) + coverArtPath = await DownloadCoverArtAsync(metadata.CoverArtUrl, folderPath); + + try { - FullMode = BoundedChannelFullMode.Wait - }); - - var downloadSemaphore = new SemaphoreSlim(_config.MaxParallelDownloads); - var completedDownloads = 0; - var completedConversions = 0; - var totalTracks = metadata.ChapterUrls.Count; - var lockObj = new object(); - - await _console.Progress() - .AutoRefresh(true) - .HideCompleted(false) - .Columns( - new TaskDescriptionColumn(), - new ProgressBarColumn(), - new PercentageColumn(), - new DownloadedColumn(), - new SpinnerColumn()) - .StartAsync(async ctx => + var conversionChannel = Channel.CreateBounded(new BoundedChannelOptions(10) { - var downloadTasks = metadata.ChapterUrls.Select((chapterUrl, index) => Task.Run(async () => - { - var trackTitle = Path.GetFileName(new Uri(chapterUrl.Split('?')[0]).LocalPath); - var progressTask = ctx.AddTask($"[green]DL {index + 1}/{totalTracks}[/] {Markup.Escape(trackTitle)}", autoStart: false); + FullMode = BoundedChannelFullMode.Wait + }); - await downloadSemaphore.WaitAsync(); - try + var downloadSemaphore = new SemaphoreSlim(_config.MaxParallelDownloads); + var completedDownloads = 0; + var completedConversions = 0; + var totalTracks = metadata.ChapterUrls.Count; + var lockObj = new object(); + + await _console.Progress() + .AutoRefresh(true) + .HideCompleted(false) + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new DownloadedColumn(), + new SpinnerColumn()) + .StartAsync(async ctx => + { + var downloadTasks = metadata.ChapterUrls.Select((chapterUrl, index) => Task.Run(async () => { - await Task.Delay(100 * (index + 1)); - progressTask.StartTask(); - - var filePath = await DownloadDirectFileWithProgressAsync(chapterUrl, trackTitle, folderPath, progressTask); + var trackTitle = Path.GetFileName(new Uri(chapterUrl.Split('?')[0]).LocalPath); + var progressTask = ctx.AddTask($"[green]DL {index + 1}/{totalTracks}[/] {Markup.Escape(trackTitle)}", autoStart: false); - if (!string.IsNullOrEmpty(filePath)) + await downloadSemaphore.WaitAsync(); + try { - var trackData = new DirectFileTrackData + await Task.Delay(100 * (index + 1)); + progressTask.StartTask(); + + var filePath = await DownloadDirectFileWithProgressAsync(chapterUrl, trackTitle, folderPath, progressTask); + + if (!string.IsNullOrEmpty(filePath)) { - FilePath = filePath, - FolderPath = folderPath, - TrackTitle = trackTitle, - SanitizedTitle = SanitizeName(trackTitle), - TrackNumber = index + 1, - TotalTracks = totalTracks - }; + var trackData = new DirectFileTrackData + { + FilePath = filePath, + FolderPath = folderPath, + TrackTitle = trackTitle, + SanitizedTitle = SanitizeName(trackTitle), + TrackNumber = index + 1, + TotalTracks = totalTracks + }; - await conversionChannel.Writer.WriteAsync(trackData); + await conversionChannel.Writer.WriteAsync(trackData); + lock (lockObj) + { + completedDownloads++; + progressTask.Description = $"[green]Done {completedDownloads}/{totalTracks}[/] {Markup.Escape(trackTitle)}"; + } + } + } + catch (Exception ex) + { lock (lockObj) { - completedDownloads++; - progressTask.Description = $"[green]Done {completedDownloads}/{totalTracks}[/] {Markup.Escape(trackTitle)}"; + progressTask.Description = $"[red]Failed[/] {Markup.Escape(trackTitle)}"; + _console.MarkupLine($"[red]Download error for {Markup.Escape(trackTitle)}: {Markup.Escape(ex.Message)}[/]"); } } - } - catch (Exception ex) - { - lock (lockObj) + finally { - progressTask.Description = $"[red]Failed[/] {Markup.Escape(trackTitle)}"; - _console.MarkupLine($"[red]Download error for {Markup.Escape(trackTitle)}: {Markup.Escape(ex.Message)}[/]"); + downloadSemaphore.Release(); } - } - finally - { - downloadSemaphore.Release(); - } - })).ToList(); + })).ToList(); - var conversionTasks = Enumerable.Range(0, _config.MaxParallelConversions) - .Select(_ => Task.Run(async () => - { - await foreach (var track in conversionChannel.Reader.ReadAllAsync()) + var conversionTasks = Enumerable.Range(0, _config.MaxParallelConversions) + .Select(_ => Task.Run(async () => { - var convTask = ctx.AddTask($"[cyan]Converting[/] {Markup.Escape(track.TrackTitle)}"); - convTask.IsIndeterminate = true; - - try + await foreach (var track in conversionChannel.Reader.ReadAllAsync()) { - await ConvertDirectFileTrackAsync(track); + var convTask = ctx.AddTask($"[cyan]Converting[/] {Markup.Escape(track.TrackTitle)}"); + convTask.IsIndeterminate = true; - lock (lockObj) + try { - completedConversions++; - convTask.IsIndeterminate = false; - convTask.Value = 100; - convTask.Description = $"[cyan]Converted {completedConversions}/{totalTracks}[/] {Markup.Escape(track.TrackTitle)}"; + await ConvertDirectFileTrackAsync(track, metadata, coverArtPath); + + lock (lockObj) + { + completedConversions++; + convTask.IsIndeterminate = false; + convTask.Value = 100; + convTask.Description = $"[cyan]Converted {completedConversions}/{totalTracks}[/] {Markup.Escape(track.TrackTitle)}"; + } } - } - catch (Exception ex) - { - lock (lockObj) + catch (Exception ex) { - convTask.IsIndeterminate = false; - convTask.Value = 100; - convTask.Description = $"[red]Conv. failed[/] {Markup.Escape(track.TrackTitle)}"; - _console.MarkupLine($"[red]Conversion error for {Markup.Escape(track.TrackTitle)}: {Markup.Escape(ex.Message)}[/]"); + lock (lockObj) + { + convTask.IsIndeterminate = false; + convTask.Value = 100; + convTask.Description = $"[red]Conv. failed[/] {Markup.Escape(track.TrackTitle)}"; + _console.MarkupLine($"[red]Conversion error for {Markup.Escape(track.TrackTitle)}: {Markup.Escape(ex.Message)}[/]"); + } } } - } - })).ToList(); + })).ToList(); - await Task.WhenAll(downloadTasks); - conversionChannel.Writer.Complete(); - await Task.WhenAll(conversionTasks); - }); + await Task.WhenAll(downloadTasks); + conversionChannel.Writer.Complete(); + await Task.WhenAll(conversionTasks); + }); + } + finally + { + if (coverArtPath != null && File.Exists(coverArtPath)) + File.Delete(coverArtPath); + } } private async Task DownloadDirectFileWithProgressAsync(string trackSrc, string trackTitle, string folderPath, ProgressTask progressTask) @@ -221,22 +584,43 @@ protected async Task DownloadDirectFileAsync(string trackSrc, string tra } } - protected async Task ConvertDirectFileTrackAsync(DirectFileTrackData track) + protected async Task ConvertDirectFileTrackAsync(DirectFileTrackData track, + AudiobookMetadata? bookMetadata = null, string? coverArtPath = null) { try { var ext = Path.GetExtension(track.FilePath).ToLowerInvariant(); - if (_settings.ConvertToMp3 && ext != ".mp3") + if (_settings.ConvertToMp3) { - var mp3Output = Path.ChangeExtension(track.FilePath, ".mp3"); - await ConvertToFormatAsync(track.FilePath, mp3Output, "-c:a libmp3lame -b:a 128k"); + if (ext != ".mp3") + { + var mp3Output = Path.ChangeExtension(track.FilePath, ".mp3"); + await ConvertToFormatAsync(track.FilePath, mp3Output, "-c:a libmp3lame -b:a 128k", bookMetadata, track, coverArtPath); + } + else if (bookMetadata != null) + { + var taggedPath = track.FilePath + ".tmp.mp3"; + await ConvertToFormatAsync(track.FilePath, taggedPath, "-c:a copy", bookMetadata, track, coverArtPath); + File.Delete(track.FilePath); + File.Move(taggedPath, track.FilePath); + } } - if (_settings.ConvertToM4b && ext != ".m4b") + if (_settings.ConvertToM4b) { - var m4bOutput = Path.ChangeExtension(track.FilePath, ".m4b"); - await ConvertToFormatAsync(track.FilePath, m4bOutput, "-c:a aac -b:a 64k"); + if (ext != ".m4b") + { + var m4bOutput = Path.ChangeExtension(track.FilePath, ".m4b"); + await ConvertToFormatAsync(track.FilePath, m4bOutput, "-c:a aac -b:a 64k", bookMetadata, track, coverArtPath); + } + else if (bookMetadata != null) + { + var taggedPath = track.FilePath + ".tmp.m4b"; + await ConvertToFormatAsync(track.FilePath, taggedPath, "-c:a copy", bookMetadata, track, coverArtPath); + File.Delete(track.FilePath); + File.Move(taggedPath, track.FilePath); + } } } catch (Exception ex) @@ -245,20 +629,21 @@ protected async Task ConvertDirectFileTrackAsync(DirectFileTrackData track) } } - protected async Task ConvertTrackToFormatsAsync(string inputFile, string outputFolder, string baseFileName) + protected async Task ConvertTrackToFormatsAsync(string inputFile, string outputFolder, string baseFileName, + AudiobookMetadata? bookMetadata = null, TrackData? trackData = null, string? coverArtPath = null) { SetFFmpegPath(); if (_settings.ConvertToMp3) { var mp3Output = Path.Combine(outputFolder, $"{baseFileName}.mp3"); - await ConvertToFormatAsync(inputFile, mp3Output, "-c:a libmp3lame -b:a 128k"); + await ConvertToFormatAsync(inputFile, mp3Output, "-c:a libmp3lame -b:a 128k", bookMetadata, trackData, coverArtPath); } if (_settings.ConvertToM4b) { var m4bOutput = Path.Combine(outputFolder, $"{baseFileName}.m4b"); - await ConvertToFormatAsync(inputFile, m4bOutput, "-c:a aac -b:a 64k"); + await ConvertToFormatAsync(inputFile, m4bOutput, "-c:a aac -b:a 64k", bookMetadata, trackData, coverArtPath); } } @@ -266,7 +651,10 @@ protected async Task MergeTsSegmentsAsync( string tempFolder, List segments, string outputFile, - string codecParams) + string codecParams, + AudiobookMetadata? bookMetadata = null, + TrackData? trackData = null, + string? coverArtPath = null) { var concatFile = Path.Combine(tempFolder, "concat.txt"); var concatLines = new List(); @@ -288,18 +676,39 @@ protected async Task MergeTsSegmentsAsync( await File.WriteAllLinesAsync(concatFile, concatLines); + var hasCover = !string.IsNullOrEmpty(coverArtPath) && File.Exists(coverArtPath); + IConversion conversion = FFmpeg.Conversions.New(); conversion.AddParameter($"-f concat -safe 0 -i \"{concatFile}\""); + if (hasCover) + { + conversion.AddParameter($"-i \"{coverArtPath}\""); + conversion.AddParameter("-map 0:a -map 1:v"); + } conversion.AddParameter(codecParams); + var metaParams = BuildMetadataParams(bookMetadata, trackData, hasCover); + if (!string.IsNullOrEmpty(metaParams)) + conversion.AddParameter(metaParams); conversion.SetOutput(outputFile); await conversion.Start(); } - protected async Task ConvertToFormatAsync(string inputFile, string outputFile, string codecParams) + protected async Task ConvertToFormatAsync(string inputFile, string outputFile, string codecParams, + AudiobookMetadata? bookMetadata = null, TrackData? trackData = null, string? coverArtPath = null) { + var hasCover = !string.IsNullOrEmpty(coverArtPath) && File.Exists(coverArtPath); + IConversion conversion = FFmpeg.Conversions.New(); conversion.AddParameter($"-i \"{inputFile}\""); + if (hasCover) + { + conversion.AddParameter($"-i \"{coverArtPath}\""); + conversion.AddParameter("-map 0:a -map 1:v"); + } conversion.AddParameter(codecParams); + var metaParams = BuildMetadataParams(bookMetadata, trackData, hasCover); + if (!string.IsNullOrEmpty(metaParams)) + conversion.AddParameter(metaParams); conversion.SetOutput(outputFile); await conversion.Start(); } diff --git a/TokyBay/Scraper/Strategies/AudioAzStrategy.cs b/TokyBay/Scraper/Strategies/AudioAzStrategy.cs index ed67808..567b17f 100644 --- a/TokyBay/Scraper/Strategies/AudioAzStrategy.cs +++ b/TokyBay/Scraper/Strategies/AudioAzStrategy.cs @@ -58,13 +58,13 @@ await _console.Status() .StartAsync("Preparing download...", async ctx => { ctx.Status("Fetching page..."); - metadata = await GetChapterUrlsAsync(bookUrl); + metadata = await GetChapterUrlsAsync(bookUrl, ctx); }); return metadata; } - private async Task GetChapterUrlsAsync(string bookUrl) + private async Task GetChapterUrlsAsync(string bookUrl, StatusContext ctx) { try { @@ -91,11 +91,14 @@ await _console.Status() if (chapterUrls.Count == 0) return null; - return new SimpleAudiobookMetadata + var result = new SimpleAudiobookMetadata { - Title = ExtractTitleFromH1(html), + Title = CleanupBookTitle(ExtractTitleFromH1(html)), ChapterUrls = chapterUrls }; + await EnrichFromFirstTrackTagsAsync(result, ctx); + ExtractCommonMetadata(html, result); + return result; } catch (Exception ex) { diff --git a/TokyBay/Scraper/Strategies/GoldenAudiobookStrategy.cs b/TokyBay/Scraper/Strategies/AudioSourceTagStrategy.cs similarity index 69% rename from TokyBay/Scraper/Strategies/GoldenAudiobookStrategy.cs rename to TokyBay/Scraper/Strategies/AudioSourceTagStrategy.cs index 4550616..f82d071 100644 --- a/TokyBay/Scraper/Strategies/GoldenAudiobookStrategy.cs +++ b/TokyBay/Scraper/Strategies/AudioSourceTagStrategy.cs @@ -7,7 +7,7 @@ namespace TokyBay.Scraper.Strategies { - public partial class GoldenAudiobookStrategy( + public partial class AudioSourceTagStrategy( IAnsiConsole console, IHttpService httpService, ISettingsService settingsService, @@ -20,15 +20,20 @@ public partial class GoldenAudiobookStrategy( [GeneratedRegex(@"\bsrc=""([^""]+)""")] private static partial Regex SrcAttributeRegex(); + // Matches links + [GeneratedRegex(@"]*\bhref=""([^""]+\.mp3[^""]*?)""[^>]*>", RegexOptions.IgnoreCase)] + private static partial Regex AnchorMp3Regex(); + public override bool CanHandle(string bookUrl) { - return bookUrl.Contains("goldenaudiobook.net", StringComparison.OrdinalIgnoreCase) - || bookUrl.Contains("fulllengthaudiobooks.net", StringComparison.OrdinalIgnoreCase) + return bookUrl.Contains("appaudiobooks.com", StringComparison.OrdinalIgnoreCase) + || bookUrl.Contains("audiozaic.com", StringComparison.OrdinalIgnoreCase) || bookUrl.Contains("bigaudiobooks.net", StringComparison.OrdinalIgnoreCase) - || bookUrl.Contains("findaudiobook.com", StringComparison.OrdinalIgnoreCase) || bookUrl.Contains("bookaudiobook.net", StringComparison.OrdinalIgnoreCase) - || bookUrl.Contains("hotaudiobooks.com", StringComparison.OrdinalIgnoreCase) - || bookUrl.Contains("audiozaic.com", StringComparison.OrdinalIgnoreCase); + || bookUrl.Contains("findaudiobook.com", StringComparison.OrdinalIgnoreCase) + || bookUrl.Contains("fulllengthaudiobooks.net", StringComparison.OrdinalIgnoreCase) + || bookUrl.Contains("goldenaudiobook.net", StringComparison.OrdinalIgnoreCase) + || bookUrl.Contains("hotaudiobooks.com", StringComparison.OrdinalIgnoreCase); } public override async Task DownloadBookAsync(string bookUrl) @@ -62,13 +67,13 @@ await _console.Status() .StartAsync("Preparing download...", async ctx => { ctx.Status("Fetching page..."); - metadata = await GetChapterUrlsAsync(bookUrl); + metadata = await GetChapterUrlsAsync(bookUrl, ctx); }); return metadata; } - private async Task GetChapterUrlsAsync(string bookUrl) + private async Task GetChapterUrlsAsync(string bookUrl, StatusContext ctx) { try { @@ -77,23 +82,22 @@ await _console.Status() var html = await response.Content.ReadAsStringAsync(); - var chapterUrls = AudioSourceTagRegex() - .Matches(html) - .Select(m => SrcAttributeRegex().Match(m.Value)) - .Where(m => m.Success) - .Select(m => m.Groups[1].Value) - .ToList(); + var chapterUrls = ExtractFromSourceTags(html); + + if (chapterUrls.Count == 0) + chapterUrls = ExtractFromAnchorLinks(html); if (chapterUrls.Count == 0) - { return null; - } - return new SimpleAudiobookMetadata + var result = new SimpleAudiobookMetadata { - Title = ExtractTitleFromH1(html), + Title = CleanupBookTitle(ExtractTitleFromH1(html)), ChapterUrls = chapterUrls }; + await EnrichFromFirstTrackTagsAsync(result, ctx); + ExtractCommonMetadata(html, result); + return result; } catch (Exception ex) { @@ -101,5 +105,24 @@ await _console.Status() return null; } } + + private static List ExtractFromSourceTags(string html) + { + return AudioSourceTagRegex() + .Matches(html) + .Select(m => SrcAttributeRegex().Match(m.Value)) + .Where(m => m.Success) + .Select(m => m.Groups[1].Value) + .ToList(); + } + + private static List ExtractFromAnchorLinks(string html) + { + return AnchorMp3Regex() + .Matches(html) + .Select(m => m.Groups[1].Value) + .Distinct() + .ToList(); + } } } diff --git a/TokyBay/Scraper/Strategies/ZAudiobooksStrategy.cs b/TokyBay/Scraper/Strategies/DropboxTracksStrategy.cs similarity index 87% rename from TokyBay/Scraper/Strategies/ZAudiobooksStrategy.cs rename to TokyBay/Scraper/Strategies/DropboxTracksStrategy.cs index 0d27189..c3be77c 100644 --- a/TokyBay/Scraper/Strategies/ZAudiobooksStrategy.cs +++ b/TokyBay/Scraper/Strategies/DropboxTracksStrategy.cs @@ -6,7 +6,7 @@ namespace TokyBay.Scraper.Strategies { - public class ZAudiobooksStrategy( + public class DropboxTracksStrategy( IAnsiConsole console, IHttpService httpService, ISettingsService settingsService, @@ -16,8 +16,8 @@ public class ZAudiobooksStrategy( public override bool CanHandle(string bookUrl) { - return bookUrl.Contains("freeaudiobooks", StringComparison.OrdinalIgnoreCase) || - bookUrl.Contains("zaudiobooks", StringComparison.OrdinalIgnoreCase); + return bookUrl.Contains("freeaudiobooks.top", StringComparison.OrdinalIgnoreCase) || + bookUrl.Contains("zaudiobooks.com", StringComparison.OrdinalIgnoreCase); } public override async Task DownloadBookAsync(string bookUrl) @@ -53,14 +53,14 @@ await _console.Status() .SpinnerStyle(Style.Parse("blue bold")) .StartAsync("Preparing download...", async ctx => { - ctx.Status("Getting title and chapters..."); - metadata = await GetChapterUrlsAsync(bookUrl); + ctx.Status("Fetching page..."); + metadata = await GetChapterUrlsAsync(bookUrl, ctx); }); return metadata; } - private async Task GetChapterUrlsAsync(string bookUrl) + private async Task GetChapterUrlsAsync(string bookUrl, StatusContext ctx) { try { @@ -110,11 +110,14 @@ await _console.Status() } } - return new SimpleAudiobookMetadata + var result = new SimpleAudiobookMetadata { - Title = ExtractTitleFromH1(html), + Title = CleanupBookTitle(ExtractTitleFromH1(html)), ChapterUrls = chapterUrls }; + await EnrichFromFirstTrackTagsAsync(result, ctx); + ExtractCommonMetadata(html, result); + return result; } catch (Exception ex) { diff --git a/TokyBay/Scraper/Strategies/PlaylistAudiobookStrategy.cs b/TokyBay/Scraper/Strategies/PlaylistAudiobookStrategy.cs index 17db6a5..589b880 100644 --- a/TokyBay/Scraper/Strategies/PlaylistAudiobookStrategy.cs +++ b/TokyBay/Scraper/Strategies/PlaylistAudiobookStrategy.cs @@ -53,13 +53,13 @@ await _console.Status() .StartAsync("Preparing download...", async ctx => { ctx.Status("Fetching page..."); - metadata = await GetChapterUrlsAsync(bookUrl); + metadata = await GetChapterUrlsAsync(bookUrl, ctx); }); return metadata; } - private async Task GetChapterUrlsAsync(string bookUrl) + private async Task GetChapterUrlsAsync(string bookUrl, StatusContext ctx) { try { @@ -84,11 +84,14 @@ await _console.Status() if (chapterUrls.Count == 0) return null; - return new SimpleAudiobookMetadata + var result = new SimpleAudiobookMetadata { - Title = ExtractTitleFromH1(html), + Title = CleanupBookTitle(ExtractTitleFromH1(html)), ChapterUrls = chapterUrls }; + await EnrichFromFirstTrackTagsAsync(result, ctx); + ExtractCommonMetadata(html, result); + return result; } catch (Exception ex) { diff --git a/TokyBay/Scraper/Strategies/TokybookStrategy.cs b/TokyBay/Scraper/Strategies/TokybookStrategy.cs index ea588f9..9241167 100644 --- a/TokyBay/Scraper/Strategies/TokybookStrategy.cs +++ b/TokyBay/Scraper/Strategies/TokybookStrategy.cs @@ -104,9 +104,23 @@ await _console.Status() return; } + var authors = (postDetails["authors"] as JArray) + ?.Select(a => a["name"]?.ToString() ?? string.Empty) + .Where(n => !string.IsNullOrEmpty(n)) + .ToList() ?? []; + var narrators = (postDetails["narrators"] as JArray) + ?.Select(n => n["name"]?.ToString() ?? string.Empty) + .Where(n => !string.IsNullOrEmpty(n)) + .ToList() ?? []; + metadata = new StreamingAudiobookMetadata { Title = bookTitle, + Author = string.Join(", ", authors), + Narrator = string.Join(", ", narrators), + CoverArtUrl = postDetails["coverImage"]?.ToString() ?? string.Empty, + Description = StripHtml(postDetails["description"]?.ToString() ?? string.Empty), + Publisher = postDetails["publisher"]?.ToString() ?? string.Empty, AudioBookId = audioBookId, StreamToken = streamToken, Tracks = tracks.Select(t => new TrackInfo @@ -122,6 +136,12 @@ await _console.Status() private async Task ProcessTracksInParallelAsync(StreamingAudiobookMetadata metadata, string folderPath) { + var coverArtPath = !string.IsNullOrEmpty(metadata.CoverArtUrl) + ? await DownloadCoverArtAsync(metadata.CoverArtUrl, folderPath) + : null; + + try + { var conversionChannel = Channel.CreateBounded(new BoundedChannelOptions(10) { FullMode = BoundedChannelFullMode.Wait @@ -202,7 +222,7 @@ await _console.Progress() try { - await ConvertSegmentedTrackAsync(track); + await ConvertSegmentedTrackAsync(track, metadata, coverArtPath); lock (lockObj) { @@ -229,6 +249,12 @@ await _console.Progress() conversionChannel.Writer.Complete(); await Task.WhenAll(conversionTasks); }); + } + finally + { + if (coverArtPath != null && File.Exists(coverArtPath)) + File.Delete(coverArtPath); + } } private async Task DownloadTrackWithProgressAsync( @@ -372,7 +398,8 @@ await RetryAsync(async () => } } - private async Task ConvertSegmentedTrackAsync(SegmentedTrackData track) + private async Task ConvertSegmentedTrackAsync(SegmentedTrackData track, + StreamingAudiobookMetadata? bookMetadata = null, string? coverArtPath = null) { try { @@ -385,13 +412,13 @@ private async Task ConvertSegmentedTrackAsync(SegmentedTrackData track) if (_settings.ConvertToMp3) { var mp3Output = Path.Combine(track.FolderPath, $"{track.SanitizedTitle}.mp3"); - await MergeTsSegmentsAsync(track.TempFolder, track.TsSegments, mp3Output, "-c:a libmp3lame -b:a 128k"); + await MergeTsSegmentsAsync(track.TempFolder, track.TsSegments, mp3Output, "-c:a libmp3lame -b:a 128k", bookMetadata, track, coverArtPath); } if (_settings.ConvertToM4b) { var m4bOutput = Path.Combine(track.FolderPath, $"{track.SanitizedTitle}.m4b"); - await MergeTsSegmentsAsync(track.TempFolder, track.TsSegments, m4bOutput, "-c:a aac -b:a 64k"); + await MergeTsSegmentsAsync(track.TempFolder, track.TsSegments, m4bOutput, "-c:a aac -b:a 64k", bookMetadata, track, coverArtPath); } SafeDeleteDirectory(track.TempFolder); diff --git a/TokyBay/ScraperServiceExtensions.cs b/TokyBay/ScraperServiceExtensions.cs index 6b593dc..ddc54c8 100644 --- a/TokyBay/ScraperServiceExtensions.cs +++ b/TokyBay/ScraperServiceExtensions.cs @@ -18,8 +18,8 @@ public static IServiceCollection AddScraperServices( services.AddSingleton(config); services.AddTransient(); - services.AddTransient(); - services.AddTransient(); + services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddTransient(); diff --git a/TokyBay/Services/DownloadService.cs b/TokyBay/Services/DownloadService.cs index ffd79f9..d5fb8c6 100644 --- a/TokyBay/Services/DownloadService.cs +++ b/TokyBay/Services/DownloadService.cs @@ -30,6 +30,7 @@ public IEnumerable GetSupportedDomains() { return new[] { + "appaudiobooks.com", "audioaz.com", "audiozaic.com", "bigaudiobooks.net", diff --git a/TokyBay/TokyBay.csproj b/TokyBay/TokyBay.csproj index a0f8d9f..c2bab05 100644 --- a/TokyBay/TokyBay.csproj +++ b/TokyBay/TokyBay.csproj @@ -5,9 +5,9 @@ net10.0 enable enable - 0.6.2 - 0.6.2 - 0.6.2 + 0.6.3 + 0.6.3 + 0.6.3