diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d00e7c6..e49f3b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,14 @@ jobs: git clone "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" . git checkout ${{ github.event.pull_request.merge_commit_sha }} + - name: Install .NET 10 SDK + run: | + wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh + chmod +x dotnet-install.sh + ./dotnet-install.sh --channel 10.0 + echo "$HOME/.dotnet" >> $GITHUB_PATH + echo "DOTNET_ROOT=$HOME/.dotnet" >> $GITHUB_ENV + - name: Extract version from csproj id: version run: | diff --git a/CLAUDE.md b/CLAUDE.md index 510d910..8ff5090 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,16 +1,18 @@ # TokyBay -C# .NET 9 Console App zum Scrapen und Konvertieren von Audiobüchern (M4B/MP3) von mehreren Websites. +C# .NET 10 console app for scraping and converting audiobooks (M4B/MP3) from multiple websites. ## Stack -- **.NET 9**, C# mit Nullable-Referenztypen und Primary Constructors -- **Spectre.Console** — alle Konsolenausgaben (Markup mit `[green]...[/]` etc., Status-Spinner) -- **Xabe.FFmpeg** — Audio-Konvertierung und Segment-Merging; Binaries via `Xabe.FFmpeg.Downloader` -- **Newtonsoft.Json** — JSON-Parsing der API-Antworten -- **Microsoft.Extensions.DependencyInjection** — DI-Container +- **.NET 10**, C# with nullable reference types and primary constructors +- **Spectre.Console** — all console output (markup with `[green]...[/]` etc., status spinners, tables, figlet) +- **Xabe.FFmpeg** — audio conversion and segment merging; binaries via `Xabe.FFmpeg.Downloader` +- **Newtonsoft.Json** — JSON parsing of API responses +- **Microsoft.Extensions.DependencyInjection** — DI container +- **Microsoft.Extensions.Configuration** — configuration via `appsettings.json` with binder +- **Microsoft.Extensions.Http** — `IHttpClientFactory` for typed HttpClients -## Architektur +## Architecture ### Strategy Pattern (Scraper) @@ -19,43 +21,109 @@ IScraperStrategy (Scraper/Abstractions/) └── BaseScraperStrategy (Scraper/Base/) ├── TokybookStrategy (Scraper/Strategies/) — tokybook.com ├── ZAudiobooksStrategy (Scraper/Strategies/) — zaudiobooks / freeaudiobooks.top - ├── GoldenAudiobookStrategy (Scraper/Strategies/) — alle Sites mit `` Struktur: + ├── GoldenAudiobookStrategy (Scraper/Strategies/) — all sites with `` structure: │ goldenaudiobook.net, fulllengthaudiobooks.net, bigaudiobooks.net, │ findaudiobook.com, bookaudiobook.net, hotaudiobooks.com, audiozaic.com - └── PlaylistAudiobookStrategy (Scraper/Strategies/) — alle Sites mit `data-playlist` JSON-Attribut: - hdaudiobooks.net + ├── PlaylistAudiobookStrategy (Scraper/Strategies/) — all sites with `data-playlist` JSON attribute: + │ hdaudiobooks.net + └── AudioAzStrategy (Scraper/Strategies/) — Next.js site with tracks JSON in streaming data: + audioaz.com ``` -- `ScraperFactory` wählt per `CanHandle(url)` die passende Strategie aus -- `ScraperConfig` steuert Parallelismus-Parameter (Defaults: 3 parallele Downloads, 2 Konvertierungen, 5 Segmente/Track) +- `ScraperFactory` selects the matching strategy via `CanHandle(url)` +- `ScraperConfig` controls parallelism parameters (defaults: 3 parallel downloads, 2 conversions, 5 segments/track) +- Runtime overrides in `Program.cs`: MaxParallelDownloads=5, MaxParallelConversions=3, MaxSegmentsPerTrack=8 -### Download-Pipeline +### Download Pipeline -Downloads und Konvertierungen laufen entkoppelt über `Channel` + `SemaphoreSlim`: -1. Download-Tasks schreiben fertige Tracks in einen bounded Channel -2. Konvertierungs-Tasks lesen aus dem Channel und rufen FFmpeg auf -3. Channel wird nach `Task.WhenAll(downloadTasks)` geschlossen +Downloads and conversions run decoupled via `Channel` + `SemaphoreSlim`: +1. Download tasks write completed tracks into a bounded channel +2. Conversion tasks read from the channel and invoke FFmpeg +3. Channel is closed after `Task.WhenAll(downloadTasks)` -### Track-Typen +### Track Types -- `SegmentedTrackData` — für HLS-Streams (`.m3u8` → `.ts`-Segmente → merge via FFmpeg concat) -- `DirectFileTrackData` — für direkte MP3/Audio-Downloads (zaudiobooks, goldenaudiobook); Konvertierung wird übersprungen wenn Quelldatei bereits im Zielformat vorliegt +- `SegmentedTrackData` — for HLS streams (`.m3u8` → `.ts` segments → merge via FFmpeg concat) +- `DirectFileTrackData` — for direct MP3/audio downloads (zaudiobooks, goldenaudiobook); conversion is skipped when source file is already in the target format -## Neue Website hinzufügen +### Data Model -1. Neue Klasse in `TokyBay/Scraper/Strategies/` anlegen, die von `BaseScraperStrategy` erbt -2. `CanHandle(string url)` implementieren — URL-basierte Erkennung -3. `DownloadBookAsync(string url)` implementieren — Metadata fetch, dann `ProcessTracksInParallelAsync` -4. In `ScraperServiceExtensions.cs` registrieren: +``` +AudiobookMetadata (abstract) +├── SimpleAudiobookMetadata — ChapterUrls: List +└── StreamingAudiobookMetadata — Tracks: List, StreamToken, AudioBookId + +TrackData (abstract) +├── SegmentedTrackData — TempFolder, FolderPath, TsSegments: List +└── DirectFileTrackData — FilePath, FolderPath + +TrackInfo — Src, TrackTitle +UserSettings — DownloadPath, FFmpegDirectory, ConvertToMp3, ConvertToM4b +``` + +### Services + +- `HttpService` (`IHttpService`) — GET/POST via `HttpClient` (10 min timeout, DI via `IHttpClientFactory`) +- `DownloadService` — orchestrates strategy selection and execution, provides supported domains +- `SettingsService` (`ISettingsService`) — loads/saves `UserSettings` in `appsettings.json`, auto-downloads FFmpeg +- `IpifyService` (`IIpifyService`) — fetches IP address via `api.ipify.org` (required for Tokybook API) +- `PageService` (`IPageService`) — Spectre.Console UI wrapper with figlet banner (`Bulbhead.flf`), ESC cancellation + +### Pages (UI Flow) + +``` +Application.RunAsync(args) + → InitializeAsync() — load settings, ensure FFmpeg + → Parse CLI args (-d / --directory) + → MainPage.ShowAsync() — main menu loop + ├── "Search book on Tokybook.com" → SearchTokybookPage (API: /api/v1/search, pagination) + ├── "Download from URL" → DownloadPage (shows supported domains, prompts for URL) + ├── "Settings" → SettingsPage (download path, FFmpeg path, MP3/M4B toggles) + └── "Exit" +``` + +### Dependency Injection + +Registration in `Program.cs` → `ConfigureServices()` and `ScraperServiceExtensions.AddScraperServices()`: + +- Singleton: `IConfiguration`, `IAnsiConsole`, `EscapeCancellableConsole`, `IIpifyService`, `IPageService`, `ISettingsService`, `ScraperFactory`, `DownloadService` +- Typed HttpClient: `IHttpService` → `HttpService` +- Transient: all `IScraperStrategy` implementations + +## Configuration + +### appsettings.json + +```json +{ + "UserSettings": { + "DownloadPath": "", + "FFmpegDirectory": "", + "ConvertToMp3": true, + "ConvertToM4b": false + } +} +``` + +### CLI Arguments + +- `-d` / `--directory` `` — sets download directory + +## Adding a New Website + +1. Create a new class in `TokyBay/Scraper/Strategies/` extending `BaseScraperStrategy` +2. Implement `CanHandle(string url)` — URL-based detection +3. Implement `DownloadBookAsync(string url)` — fetch metadata, then call `ProcessTracksInParallelAsync` or `ProcessDirectFilesInParallelAsync` +4. Register in `ScraperServiceExtensions.cs`: ```csharp - services.AddTransient(); + services.AddTransient(); ``` ## Build & Run ```sh dotnet build -dotnet run --project TokyBay -- -d "C:\Pfad\zum\Download" +dotnet run --project TokyBay -- -d "C:\Path\To\Downloads" ``` ### Publish (Cross-Platform) @@ -68,10 +136,23 @@ dotnet publish -c Release -r osx-x64 --self-contained dotnet publish -c Release -r osx-arm64 --self-contained ``` -## Konventionen +Publish profiles are in `TokyBay/Properties/PublishProfiles/` (WinX64, LinuxX64, LinuxArm64). +Flags: `PublishSingleFile=true`, `PublishTrimmed=false`, `SelfContained=true`. + +## CI/CD (GitHub Actions) + +Workflow: `.github/workflows/release.yml` + +- **Trigger:** Pull request merged into `master` +- **Constraint:** Org `z00mable` only allows actions owned by the org — no third-party actions like `actions/checkout`, `actions/setup-dotnet`, `softprops/action-gh-release`. Use `git clone`/`run`-based alternatives instead (e.g. `dotnet-install.sh` for SDK setup, `gh release create` instead of `softprops/action-gh-release`). +- **Steps:** git clone → install .NET 10 SDK (via `dotnet-install.sh`) → extract version from `.csproj` → publish for 5 platforms (win-x64, linux-x64, linux-arm64, osx-x64, osx-arm64) → ZIP → git tag `v{version}` → GitHub Release via `gh release create` +- **Release notes:** From PR body + +## Conventions -- Konsolenausgaben immer via `_console.MarkupLine(...)` (Spectre), nie `Console.WriteLine` -- Fehlermeldungen in `[red]`, Erfolg in `[green]`, Info in `[blue]`, Konvertierungen in `[cyan]` -- Dateinamen werden via `SanitizeName()` bereinigt (`[^A-Za-z0-9]+` → `_`) -- Retry-Logik: `RetryAsync()` aus `BaseScraperStrategy` verwenden (exponentielles Delay) -- Temp-Verzeichnisse immer via `SafeDeleteDirectory()` aufräumen +- Console output always via `_console.MarkupLine(...)` (Spectre), never `Console.WriteLine` +- Error messages in `[red]`, success in `[green]`, info in `[blue]`, conversions in `[cyan]`, warnings in `[yellow]`, secondary in `[grey]` +- Filenames are sanitized via `SanitizeName()` (`[^A-Za-z0-9]+` → `_`) +- Retry logic: use `RetryAsync()` from `BaseScraperStrategy` (exponential delay) +- Temp directories always cleaned up via `SafeDeleteDirectory()` +- ESC cancellation in UI prompts via `EscapeCancellableConsole` diff --git a/README.md b/README.md index 93eb0fe..7d5f8ef 100644 --- a/README.md +++ b/README.md @@ -68,21 +68,22 @@ That's it. TokyBay will guide you through the rest. ## Supported Sites -*Last checked: 2026-04-07* +*Last checked: 2026-04-08* | Site | Status | |------|--------| -| [tokybook.com](https://tokybook.com) | Website temporarily offline, API operational | -| [hdaudiobooks.net](https://hdaudiobooks.net) | Working | -| [goldenaudiobook.net](https://goldenaudiobook.net) | Working | -| [fulllengthaudiobooks.net](https://fulllengthaudiobooks.net) | Working | -| [bigaudiobooks.net](https://bigaudiobooks.net) | Working | -| [findaudiobook.com](https://findaudiobook.com) | Working | -| [bookaudiobook.net](https://bookaudiobook.net) | Working | -| [hotaudiobooks.com](https://hotaudiobooks.com) | Working | -| [audiozaic.com](https://audiozaic.com) | Working | -| [zaudiobooks.com](https://zaudiobooks.com) | Currently unreachable | -| [freeaudiobooks.top](https://freeaudiobooks.top) | Currently unreachable | +| [audioaz.com](https://audioaz.com) | 🟢 Working | +| [audiozaic.com](https://audiozaic.com) | 🟢 Working | +| [bigaudiobooks.net](https://bigaudiobooks.net) | 🟢 Working | +| [bookaudiobook.net](https://bookaudiobook.net) | 🟢 Working | +| [findaudiobook.com](https://findaudiobook.com) | 🟢 Working | +| [freeaudiobooks.top](https://freeaudiobooks.top) | 🔴 Currently unreachable | +| [fulllengthaudiobooks.net](https://fulllengthaudiobooks.net) | 🟢 Working | +| [goldenaudiobook.net](https://goldenaudiobook.net) | 🟢 Working | +| [hdaudiobooks.net](https://hdaudiobooks.net) | 🟢 Working | +| [hotaudiobooks.com](https://hotaudiobooks.com) | 🟢 Working | +| [tokybook.com](https://tokybook.com) | 🔴 Website temporarily offline, 🟢 API operational | +| [zaudiobooks.com](https://zaudiobooks.com) | 🔴 Currently unreachable | ## Features @@ -116,7 +117,7 @@ That's it. TokyBay will guide you through the rest. - Change the FFmpeg binary path > [!Tip] -> By default, TokyBay downloads to the folder it is run from. You can change this inside the app's settings, or pass a custom path via `-d "C:\Users\User\Music"` when launching from the terminal. +> By default, TokyBay downloads to your **Music** folder on Windows (`C:\Users\\Music`) or your **home directory** on Linux/macOS. You can change this inside the app's settings, or pass a custom path via `-d "C:\Path\To\Downloads"` when launching from the terminal. --- @@ -127,7 +128,7 @@ That's it. TokyBay will guide you through the rest. ### Prerequisites -- [.NET 9 SDK](https://dotnet.microsoft.com/download/dotnet/9.0) +- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) ### Clone and run diff --git a/TokyBay/Properties/PublishProfiles/LinuxArm64.pubxml b/TokyBay/Properties/PublishProfiles/LinuxArm64.pubxml index a819816..49d217d 100644 --- a/TokyBay/Properties/PublishProfiles/LinuxArm64.pubxml +++ b/TokyBay/Properties/PublishProfiles/LinuxArm64.pubxml @@ -4,10 +4,10 @@ Release Any CPU - bin\Release\net9.0\publish\linux-arm64\ + bin\Release\net10.0\publish\linux-arm64\ FileSystem <_TargetId>Folder - net9.0 + net10.0 linux-arm64 true true diff --git a/TokyBay/Properties/PublishProfiles/LinuxX64.pubxml b/TokyBay/Properties/PublishProfiles/LinuxX64.pubxml index 64c8bd7..ece28fa 100644 --- a/TokyBay/Properties/PublishProfiles/LinuxX64.pubxml +++ b/TokyBay/Properties/PublishProfiles/LinuxX64.pubxml @@ -4,10 +4,10 @@ Release Any CPU - bin\Release\net9.0\publish\linux-x64\ + bin\Release\net10.0\publish\linux-x64\ FileSystem <_TargetId>Folder - net9.0 + net10.0 linux-x64 true true diff --git a/TokyBay/Properties/PublishProfiles/OsxArm64.pubxml b/TokyBay/Properties/PublishProfiles/OsxArm64.pubxml new file mode 100644 index 0000000..ebc4a52 --- /dev/null +++ b/TokyBay/Properties/PublishProfiles/OsxArm64.pubxml @@ -0,0 +1,16 @@ + + + + + Release + Any CPU + bin\Release\net10.0\publish\osx-arm64\ + FileSystem + <_TargetId>Folder + net10.0 + osx-arm64 + true + true + false + + \ No newline at end of file diff --git a/TokyBay/Properties/PublishProfiles/OsxX64.pubxml b/TokyBay/Properties/PublishProfiles/OsxX64.pubxml new file mode 100644 index 0000000..dce9489 --- /dev/null +++ b/TokyBay/Properties/PublishProfiles/OsxX64.pubxml @@ -0,0 +1,16 @@ + + + + + Release + Any CPU + bin\Release\net10.0\publish\osx-x64\ + FileSystem + <_TargetId>Folder + net10.0 + osx-x64 + true + true + false + + \ No newline at end of file diff --git a/TokyBay/Properties/PublishProfiles/WinX64.pubxml b/TokyBay/Properties/PublishProfiles/WinX64.pubxml index 266c45b..88f5dc6 100644 --- a/TokyBay/Properties/PublishProfiles/WinX64.pubxml +++ b/TokyBay/Properties/PublishProfiles/WinX64.pubxml @@ -4,10 +4,10 @@ Release Any CPU - bin\Release\net9.0\publish\win-x64\ + bin\Release\net10.0\publish\win-x64\ FileSystem <_TargetId>Folder - net9.0 + net10.0 win-x64 true true diff --git a/TokyBay/Scraper/Base/BaseScraperStrategy.cs b/TokyBay/Scraper/Base/BaseScraperStrategy.cs index 64a5c14..4f8a455 100644 --- a/TokyBay/Scraper/Base/BaseScraperStrategy.cs +++ b/TokyBay/Scraper/Base/BaseScraperStrategy.cs @@ -57,79 +57,143 @@ protected async Task ProcessDirectFilesInParallelAsync(SimpleAudiobookMetadata m var totalTracks = metadata.ChapterUrls.Count; var lockObj = new object(); - var downloadTasks = metadata.ChapterUrls.Select((chapterUrl, index) => Task.Run(async () => - { - var trackTitle = Path.GetFileName(new Uri(chapterUrl.Split('?')[0]).LocalPath); - - await downloadSemaphore.WaitAsync(); - try + await _console.Progress() + .AutoRefresh(true) + .HideCompleted(false) + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new DownloadedColumn(), + new SpinnerColumn()) + .StartAsync(async ctx => { - await Task.Delay(100 * (index + 1)); - - var filePath = await DownloadDirectFileAsync(chapterUrl, trackTitle, folderPath); - - if (!string.IsNullOrEmpty(filePath)) + var downloadTasks = metadata.ChapterUrls.Select((chapterUrl, index) => Task.Run(async () => { - var trackData = new DirectFileTrackData - { - FilePath = filePath, - FolderPath = folderPath, - TrackTitle = trackTitle, - SanitizedTitle = SanitizeName(trackTitle), - TrackNumber = index + 1, - TotalTracks = totalTracks - }; + var trackTitle = Path.GetFileName(new Uri(chapterUrl.Split('?')[0]).LocalPath); + var progressTask = ctx.AddTask($"[green]DL {index + 1}/{totalTracks}[/] {Markup.Escape(trackTitle)}", autoStart: false); - await conversionChannel.Writer.WriteAsync(trackData); - - lock (lockObj) - { - completedDownloads++; - _console.MarkupLine($"[green]Downloaded:[/] {completedDownloads}/{totalTracks} - {trackTitle}"); - } - } - } - catch (Exception ex) - { - lock (lockObj) - { - _console.MarkupLine($"[red]Download error for {trackTitle}: {ex.Message}[/]"); - } - } - finally - { - downloadSemaphore.Release(); - } - })).ToList(); - - var conversionTasks = Enumerable.Range(0, _config.MaxParallelConversions) - .Select(_ => Task.Run(async () => - { - await foreach (var track in conversionChannel.Reader.ReadAllAsync()) - { + await downloadSemaphore.WaitAsync(); try { - await ConvertDirectFileTrackAsync(track); + await Task.Delay(100 * (index + 1)); + progressTask.StartTask(); - lock (lockObj) + var filePath = await DownloadDirectFileWithProgressAsync(chapterUrl, trackTitle, folderPath, progressTask); + + if (!string.IsNullOrEmpty(filePath)) { - completedConversions++; - _console.MarkupLine($"[cyan]Converted:[/] {completedConversions}/{totalTracks} - {track.TrackTitle}"); + var trackData = new DirectFileTrackData + { + FilePath = filePath, + FolderPath = folderPath, + TrackTitle = trackTitle, + SanitizedTitle = SanitizeName(trackTitle), + TrackNumber = index + 1, + TotalTracks = totalTracks + }; + + await conversionChannel.Writer.WriteAsync(trackData); + + lock (lockObj) + { + completedDownloads++; + progressTask.Description = $"[green]Done {completedDownloads}/{totalTracks}[/] {Markup.Escape(trackTitle)}"; + } } } catch (Exception ex) { lock (lockObj) { - _console.MarkupLine($"[red]Conversion error for {track.TrackTitle}: {ex.Message}[/]"); + progressTask.Description = $"[red]Failed[/] {Markup.Escape(trackTitle)}"; + _console.MarkupLine($"[red]Download error for {Markup.Escape(trackTitle)}: {Markup.Escape(ex.Message)}[/]"); } } - } - })).ToList(); + finally + { + downloadSemaphore.Release(); + } + })).ToList(); - await Task.WhenAll(downloadTasks); - conversionChannel.Writer.Complete(); - await Task.WhenAll(conversionTasks); + var conversionTasks = Enumerable.Range(0, _config.MaxParallelConversions) + .Select(_ => Task.Run(async () => + { + await foreach (var track in conversionChannel.Reader.ReadAllAsync()) + { + var convTask = ctx.AddTask($"[cyan]Converting[/] {Markup.Escape(track.TrackTitle)}"); + convTask.IsIndeterminate = true; + + try + { + await ConvertDirectFileTrackAsync(track); + + lock (lockObj) + { + completedConversions++; + convTask.IsIndeterminate = false; + convTask.Value = 100; + convTask.Description = $"[cyan]Converted {completedConversions}/{totalTracks}[/] {Markup.Escape(track.TrackTitle)}"; + } + } + catch (Exception ex) + { + lock (lockObj) + { + convTask.IsIndeterminate = false; + convTask.Value = 100; + convTask.Description = $"[red]Conv. failed[/] {Markup.Escape(track.TrackTitle)}"; + _console.MarkupLine($"[red]Conversion error for {Markup.Escape(track.TrackTitle)}: {Markup.Escape(ex.Message)}[/]"); + } + } + } + })).ToList(); + + await Task.WhenAll(downloadTasks); + conversionChannel.Writer.Complete(); + await Task.WhenAll(conversionTasks); + }); + } + + private async Task DownloadDirectFileWithProgressAsync(string trackSrc, string trackTitle, string folderPath, ProgressTask progressTask) + { + var filePath = Path.Combine(folderPath, trackTitle); + try + { + var response = await _httpService.GetAsync(trackSrc); + if (!response.IsSuccessStatusCode) + { + _console.MarkupLine($"[red]Could not download chapter: {Markup.Escape(trackTitle)}[/]"); + return string.Empty; + } + + var totalBytes = response.Content.Headers.ContentLength ?? 0; + progressTask.MaxValue = totalBytes > 0 ? totalBytes : 1; + + await using var contentStream = await response.Content.ReadAsStreamAsync(); + await using var fileStream = File.Create(filePath); + + var buffer = new byte[81920]; + long totalRead = 0; + int bytesRead; + + while ((bytesRead = await contentStream.ReadAsync(buffer)) > 0) + { + await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)); + totalRead += bytesRead; + progressTask.Value = totalRead; + } + + if (totalBytes == 0) + progressTask.Value = progressTask.MaxValue; + + return filePath; + } + catch (Exception ex) + { + _console.MarkupLine($"[red]Error downloading chapter {Markup.Escape(trackSrc)}: {Markup.Escape(ex.Message)}[/]"); + return string.Empty; + } } protected async Task DownloadDirectFileAsync(string trackSrc, string trackTitle, string folderPath) diff --git a/TokyBay/Scraper/Strategies/AudioAzStrategy.cs b/TokyBay/Scraper/Strategies/AudioAzStrategy.cs new file mode 100644 index 0000000..ed67808 --- /dev/null +++ b/TokyBay/Scraper/Strategies/AudioAzStrategy.cs @@ -0,0 +1,107 @@ +using Newtonsoft.Json.Linq; +using Spectre.Console; +using System.Text.RegularExpressions; +using TokyBay.Models; +using TokyBay.Scraper.Base; +using TokyBay.Scraper.Configuration; +using TokyBay.Services; + +namespace TokyBay.Scraper.Strategies +{ + public partial class AudioAzStrategy( + IAnsiConsole console, + IHttpService httpService, + ISettingsService settingsService, + ScraperConfig? config = null) : BaseScraperStrategy(console, httpService, settingsService, config) + { + // Matches the "tracks":[...] JSON array after unescaping + [GeneratedRegex(@"""tracks"":\s*(\[\{.*?\}\])", RegexOptions.Singleline)] + private static partial Regex TracksJsonRegex(); + + // Strips escape backslashes preceding JSON structural characters + [GeneratedRegex(@"\\+(?=[""[\]{}/])")] + private static partial Regex EscapeBackslashRegex(); + + public override bool CanHandle(string bookUrl) + { + return bookUrl.Contains("audioaz.com", StringComparison.OrdinalIgnoreCase); + } + + public override async Task DownloadBookAsync(string bookUrl) + { + var metadata = await FetchMetadataAsync(bookUrl); + if (metadata == null || metadata.ChapterUrls.Count == 0) + { + ShowErrorMessage("No valid tracks found"); + return; + } + + SetFFmpegPath(); + + var folderPath = PrepareOutputFolder(metadata.Title); + + _console.MarkupLine($"[green]Found {metadata.ChapterUrls.Count} tracks[/]"); + _console.MarkupLine($"[blue]Parallel downloads:[/] {_config.MaxParallelDownloads}"); + _console.MarkupLine($"[blue]Parallel conversions:[/] {_config.MaxParallelConversions}"); + + await ProcessDirectFilesInParallelAsync(metadata, folderPath); + + ShowCompletionMessage(folderPath); + } + + private async Task FetchMetadataAsync(string bookUrl) + { + SimpleAudiobookMetadata? metadata = null; + + await _console.Status() + .SpinnerStyle(Style.Parse("blue bold")) + .StartAsync("Preparing download...", async ctx => + { + ctx.Status("Fetching page..."); + metadata = await GetChapterUrlsAsync(bookUrl); + }); + + return metadata; + } + + private async Task GetChapterUrlsAsync(string bookUrl) + { + try + { + var response = await _httpService.GetAsync(bookUrl); + response.EnsureSuccessStatusCode(); + + var html = await response.Content.ReadAsStringAsync(); + + var normalized = EscapeBackslashRegex().Replace(html, ""); + + var match = TracksJsonRegex().Match(normalized); + if (!match.Success) + return null; + + var tracksArray = JArray.Parse(match.Groups[1].Value); + + var chapterUrls = tracksArray + .OrderBy(t => t["order"]?.Value() ?? 0) + .Select(t => t["audio_url"]?.ToString()) + .Where(url => !string.IsNullOrEmpty(url)) + .Select(url => url!) + .ToList(); + + if (chapterUrls.Count == 0) + return null; + + return new SimpleAudiobookMetadata + { + Title = ExtractTitleFromH1(html), + ChapterUrls = chapterUrls + }; + } + catch (Exception ex) + { + _console.MarkupLine($"[red]Error fetching chapter URLs: {ex.Message}[/]"); + return null; + } + } + } +} diff --git a/TokyBay/Scraper/Strategies/TokybookStrategy.cs b/TokyBay/Scraper/Strategies/TokybookStrategy.cs index 71c1206..ea588f9 100644 --- a/TokyBay/Scraper/Strategies/TokybookStrategy.cs +++ b/TokyBay/Scraper/Strategies/TokybookStrategy.cs @@ -133,84 +133,113 @@ private async Task ProcessTracksInParallelAsync(StreamingAudiobookMetadata metad var totalTracks = metadata.Tracks.Count; var lockObj = new object(); - var downloadTasks = metadata.Tracks.Select((track, index) => Task.Run(async () => - { - await downloadSemaphore.WaitAsync(); - try + await _console.Progress() + .AutoRefresh(true) + .HideCompleted(false) + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new SpinnerColumn()) + .StartAsync(async ctx => { - await Task.Delay(10 * (index + 1)); - - var downloadedTrack = await DownloadTrackAsync( - metadata.AudioBookId, - metadata.StreamToken, - track.Src, - track.TrackTitle, - folderPath, - index + 1, - totalTracks); - - if (downloadedTrack != null) + var downloadTasks = metadata.Tracks.Select((track, index) => Task.Run(async () => { - await conversionChannel.Writer.WriteAsync(downloadedTrack); + var progressTask = ctx.AddTask($"[green]DL {index + 1}/{totalTracks}[/] {Markup.Escape(track.TrackTitle)}", autoStart: false); - lock (lockObj) - { - completedDownloads++; - _console.MarkupLine($"[green]Downloaded:[/] {completedDownloads}/{totalTracks} - {track.TrackTitle}"); - } - } - } - catch (Exception ex) - { - lock (lockObj) - { - _console.MarkupLine($"[red]Download error for {track.TrackTitle}: {ex.Message}[/]"); - } - } - finally - { - downloadSemaphore.Release(); - } - })).ToList(); - - var conversionTasks = Enumerable.Range(0, _config.MaxParallelConversions) - .Select(_ => Task.Run(async () => - { - await foreach (var track in conversionChannel.Reader.ReadAllAsync()) - { + await downloadSemaphore.WaitAsync(); try { - await ConvertSegmentedTrackAsync(track); + await Task.Delay(10 * (index + 1)); + progressTask.StartTask(); + + var downloadedTrack = await DownloadTrackWithProgressAsync( + metadata.AudioBookId, + metadata.StreamToken, + track.Src, + track.TrackTitle, + folderPath, + index + 1, + totalTracks, + progressTask); + + if (downloadedTrack != null) + { + await conversionChannel.Writer.WriteAsync(downloadedTrack); - lock (lockObj) + lock (lockObj) + { + completedDownloads++; + progressTask.Description = $"[green]Done {completedDownloads}/{totalTracks}[/] {Markup.Escape(track.TrackTitle)}"; + } + } + else { - completedConversions++; - _console.MarkupLine($"[cyan]Converted:[/] {completedConversions}/{totalTracks} - {track.TrackTitle}"); + progressTask.Description = $"[red]Failed[/] {Markup.Escape(track.TrackTitle)}"; } } catch (Exception ex) { lock (lockObj) { - _console.MarkupLine($"[red]Conversion error for {track.TrackTitle}: {ex.Message}[/]"); + progressTask.Description = $"[red]Failed[/] {Markup.Escape(track.TrackTitle)}"; + _console.MarkupLine($"[red]Download error for {Markup.Escape(track.TrackTitle)}: {Markup.Escape(ex.Message)}[/]"); } } - } - })).ToList(); + finally + { + downloadSemaphore.Release(); + } + })).ToList(); - await Task.WhenAll(downloadTasks); - conversionChannel.Writer.Complete(); - await Task.WhenAll(conversionTasks); + var conversionTasks = Enumerable.Range(0, _config.MaxParallelConversions) + .Select(_ => Task.Run(async () => + { + await foreach (var track in conversionChannel.Reader.ReadAllAsync()) + { + var convTask = ctx.AddTask($"[cyan]Converting[/] {Markup.Escape(track.TrackTitle)}"); + convTask.IsIndeterminate = true; + + try + { + await ConvertSegmentedTrackAsync(track); + + lock (lockObj) + { + completedConversions++; + convTask.IsIndeterminate = false; + convTask.Value = 100; + convTask.Description = $"[cyan]Converted {completedConversions}/{totalTracks}[/] {Markup.Escape(track.TrackTitle)}"; + } + } + catch (Exception ex) + { + lock (lockObj) + { + convTask.IsIndeterminate = false; + convTask.Value = 100; + convTask.Description = $"[red]Conv. failed[/] {Markup.Escape(track.TrackTitle)}"; + _console.MarkupLine($"[red]Conversion error for {Markup.Escape(track.TrackTitle)}: {Markup.Escape(ex.Message)}[/]"); + } + } + } + })).ToList(); + + await Task.WhenAll(downloadTasks); + conversionChannel.Writer.Complete(); + await Task.WhenAll(conversionTasks); + }); } - private async Task DownloadTrackAsync( + private async Task DownloadTrackWithProgressAsync( string audioBookId, string streamToken, string trackSrc, string trackTitle, string folderPath, int trackNumber, - int totalTracks) + int totalTracks, + ProgressTask progressTask) { try { @@ -223,6 +252,8 @@ private async Task ProcessTracksInParallelAsync(StreamingAudiobookMetadata metad var escapedTrackSrc = basePath + Uri.EscapeDataString(trackSrc[(lastSlash + 1)..]); var m3u8Url = TokybookBaseUrl + AudioBaseApiPath + escapedTrackSrc; + progressTask.IsIndeterminate = true; + var m3u8Content = await RetryAsync(async () => { var content = await DownloadM3u8PlaylistAsync(audioBookId, streamToken, m3u8Url, escapedTrackSrc); @@ -232,6 +263,7 @@ private async Task ProcessTracksInParallelAsync(StreamingAudiobookMetadata metad if (string.IsNullOrEmpty(m3u8Content)) { SafeDeleteDirectory(tempFolder); + progressTask.Value = progressTask.MaxValue; return null; } @@ -239,15 +271,20 @@ private async Task ProcessTracksInParallelAsync(StreamingAudiobookMetadata metad if (tsSegments.Count == 0) { SafeDeleteDirectory(tempFolder); + progressTask.Value = progressTask.MaxValue; return null; } - await DownloadTsSegmentsAsync(audioBookId, streamToken, basePath, tsSegments, tempFolder); + progressTask.IsIndeterminate = false; + progressTask.MaxValue = tsSegments.Count; + progressTask.Value = 0; + + await DownloadTsSegmentsWithProgressAsync(audioBookId, streamToken, basePath, tsSegments, tempFolder, progressTask); var downloadedFiles = Directory.GetFiles(tempFolder, "*.ts").Length; if (downloadedFiles < tsSegments.Count) { - _console.MarkupLine($"[yellow]Warning: Only {downloadedFiles}/{tsSegments.Count} segments found for {trackTitle}[/]"); + _console.MarkupLine($"[yellow]Warning: Only {downloadedFiles}/{tsSegments.Count} segments found for {Markup.Escape(trackTitle)}[/]"); SafeDeleteDirectory(tempFolder); return null; } @@ -265,7 +302,7 @@ private async Task ProcessTracksInParallelAsync(StreamingAudiobookMetadata metad } catch (Exception ex) { - _console.MarkupLine($"[red]Error downloading track {trackTitle}: {ex.Message}[/]"); + _console.MarkupLine($"[red]Error downloading track {Markup.Escape(trackTitle)}: {Markup.Escape(ex.Message)}[/]"); return null; } } @@ -284,12 +321,13 @@ private static List ParseTsSegments(string m3u8Content) .ToList(); } - private async Task DownloadTsSegmentsAsync( + private async Task DownloadTsSegmentsWithProgressAsync( string audioBookId, string streamToken, string basePath, List segments, - string outputFolder) + string outputFolder, + ProgressTask progressTask) { var semaphore = new SemaphoreSlim(_config.MaxSegmentsPerTrack); var successCount = 0; @@ -314,6 +352,7 @@ await RetryAsync(async () => var bytes = await response.Content.ReadAsByteArrayAsync(); await File.WriteAllBytesAsync(segmentPath, bytes); Interlocked.Increment(ref successCount); + progressTask.Increment(1); return true; } throw new InvalidOperationException("Segment download failed"); diff --git a/TokyBay/ScraperServiceExtensions.cs b/TokyBay/ScraperServiceExtensions.cs index 96d6186..6b593dc 100644 --- a/TokyBay/ScraperServiceExtensions.cs +++ b/TokyBay/ScraperServiceExtensions.cs @@ -21,6 +21,7 @@ public static IServiceCollection AddScraperServices( services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddSingleton(); services.AddSingleton(); diff --git a/TokyBay/Services/DownloadService.cs b/TokyBay/Services/DownloadService.cs index 192e53a..ffd79f9 100644 --- a/TokyBay/Services/DownloadService.cs +++ b/TokyBay/Services/DownloadService.cs @@ -30,16 +30,17 @@ public IEnumerable GetSupportedDomains() { return new[] { - "tokybook.com", - "goldenaudiobook.net", - "fulllengthaudiobooks.net", + "audioaz.com", + "audiozaic.com", "bigaudiobooks.net", - "findaudiobook.com", "bookaudiobook.net", - "hotaudiobooks.com", - "hdaudiobooks.net", - "audiozaic.com", + "findaudiobook.com", "freeaudiobooks.top", + "fulllengthaudiobooks.net", + "goldenaudiobook.net", + "hdaudiobooks.net", + "hotaudiobooks.com", + "tokybook.com", "zaudiobooks.com" }; } diff --git a/TokyBay/TokyBay.csproj b/TokyBay/TokyBay.csproj index ab9d5ac..a0f8d9f 100644 --- a/TokyBay/TokyBay.csproj +++ b/TokyBay/TokyBay.csproj @@ -2,12 +2,12 @@ Exe - net9.0 + net10.0 enable enable - 0.6.1 - 0.6.1 - 0.6.1 + 0.6.2 + 0.6.2 + 0.6.2