diff --git a/docs/pages/configuration/index.md b/docs/pages/configuration/index.md index 4454f682..6a697b47 100644 --- a/docs/pages/configuration/index.md +++ b/docs/pages/configuration/index.md @@ -38,14 +38,11 @@ In der Weboberfläche können Bilddateien hochgeladen werden. Diese werden mit e | `Turnierplan__ImageQuality` | Die Qualitätseinstellung für Bild-Uploads. Ein Wert von `100` entspricht einer verlustfreien Komprimierung. | `80` | | `ImageStorage__StoragePath` | Das Verzeichnis innerhalb vom Container, hochgeladene Bilder gespeichert werden. | `/var/turnierplan/images` | -Alternativ können externe Services zum Speichern der Bilder konfiguriert werden. Dies hat den Vorteil, dass das Bereitstellen von Bilddateien keine CPU- und Netzwerkresourcen vom turnierplan.NET-Server beansprucht. Aktuell werden die folgenden externen Services unterstützt: +Alternativ können externe Services zum Speichern der Bilder konfiguriert werden. Dies hat den Vorteil, dass das Bereitstellen von Bilddateien keine CPU- und Netzwerkressourcen vom turnierplan.NET-Server beansprucht. Aktuell werden die folgenden externen Services unterstützt: - **AWS S3** (oder kompatibler Dienst) - **Azure Blob Storage** -!!! warning - Die nachfolgend vorgestellten Alternativen verwenden nicht zwangsläufig eine identische Verzeichnisstruktur zur Organisation der Dateien. Dadurch wird eine nachträgliche Umstellung ggf. erschwert! - ### AWS S3 Um Bilder in einem AWS S3 oder S3-kompatiblen Bucket zu speichern, müssen die folgenden Umgebungsvariablen gesetzt werden: diff --git a/src/Turnierplan.App/Helpers/DatabaseImageProvider.cs b/src/Turnierplan.App/Helpers/DatabaseImageProvider.cs new file mode 100644 index 00000000..be0ccb95 --- /dev/null +++ b/src/Turnierplan.App/Helpers/DatabaseImageProvider.cs @@ -0,0 +1,13 @@ +using Turnierplan.Core.Image; +using Turnierplan.Dal.Repositories; +using Turnierplan.ImageStorage; + +namespace Turnierplan.App.Helpers; + +internal sealed class DatabaseImageProvider(IImageRepository imageRepository) : IImageProvider +{ + public async Task> GetImagesAsync() + { + return (await imageRepository.GetAllImagesAsync()).AsReadOnly(); + } +} diff --git a/src/Turnierplan.App/Program.cs b/src/Turnierplan.App/Program.cs index 070f34b4..c92aeec8 100644 --- a/src/Turnierplan.App/Program.cs +++ b/src/Turnierplan.App/Program.cs @@ -37,7 +37,7 @@ builder.Services.AddTurnierplanMonitoring(builder.Configuration); builder.Services.AddTurnierplanDataAccessLayer(builder.Configuration); builder.Services.AddTurnierplanDocumentRendering(); -builder.Services.AddTurnierplanImageStorage(builder.Configuration.GetSection("ImageStorage")); +builder.Services.AddTurnierplanImageStorage(builder.Configuration.GetSection("ImageStorage")); builder.Services.AddTurnierplanLocalization(); builder.Services.AddTurnierplanSecurity(builder.Configuration.GetSection("Identity")); @@ -116,4 +116,7 @@ // Migrate database and create admin user if DB is empty await app.InitializeDatabaseAsync(); +// Run image storage migrations if required by the configured image storage +await app.MigrateImageStorageAsync(); + await app.RunAsync(); diff --git a/src/Turnierplan.Dal/Repositories/ImageRepository.cs b/src/Turnierplan.Dal/Repositories/ImageRepository.cs index 1357e440..52430df4 100644 --- a/src/Turnierplan.Dal/Repositories/ImageRepository.cs +++ b/src/Turnierplan.Dal/Repositories/ImageRepository.cs @@ -6,6 +6,8 @@ namespace Turnierplan.Dal.Repositories; public interface IImageRepository : IRepositoryWithPublicId { + Task> GetAllImagesAsync(); + Task CountNumberOfReferencingTournamentsAsync(long imageId); } @@ -22,6 +24,11 @@ internal sealed class ImageRepository(TurnierplanContext context) : RepositoryBa .FirstOrDefaultAsync(); } + public Task> GetAllImagesAsync() + { + return DbSet.ToListAsync(); + } + public async Task CountNumberOfReferencingTournamentsAsync(long imageId) { var count = await _context.Tournaments diff --git a/src/Turnierplan.ImageStorage.Test.Migration/Local/LocalImageStorageMigrationTest.cs b/src/Turnierplan.ImageStorage.Test.Migration/Local/LocalImageStorageMigrationTest.cs new file mode 100644 index 00000000..5a5585fc --- /dev/null +++ b/src/Turnierplan.ImageStorage.Test.Migration/Local/LocalImageStorageMigrationTest.cs @@ -0,0 +1,330 @@ +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using System.Text.RegularExpressions; +using FluentAssertions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Turnierplan.Core.Image; +using Turnierplan.Core.PublicId; +using Turnierplan.ImageStorage.Extensions; +using Xunit; + +namespace Turnierplan.ImageStorage.Test.Migration.Local; + +public sealed class LocalImageStorageMigrationTest : IDisposable +{ + private const string ImageExtension = "jpg"; + + private const int LogEventIdUsingDirectoryForStorage = 1; + private const int LogEventIdDirectoryStructureIsUpToDate = 8; + private const int LogEventIdMigrationWillBeAttempted = 9; + private const int LogEventIdImageProviderReturnedNoImages = 10; + private const int LogEventIdEncounteredFileWithoutCorrespondingImageFromProvider = 13; + private const int LogEventIdEncounteredTargetFileThatAlreadyExists = 14; + private const int LogEventIdSuccessfullyMovedFileTo = 15; + + private readonly string _basePath; + private readonly TestLoggerProvider _loggerProvider; + private readonly StateForTestImageProvider _imageProviderState; + private readonly ServiceProvider _serviceProvider; + + public LocalImageStorageMigrationTest() + { + _basePath = Path.Join(Path.GetTempPath(), $"{nameof(LocalImageStorageMigrationTest)}-{Guid.NewGuid()}"); + + var config = new ConfigurationBuilder().AddInMemoryCollection([ + new KeyValuePair("ImageStorage:Type", "Local"), + new KeyValuePair("ImageStorage:StoragePath", _basePath) + ]).Build().GetSection("ImageStorage"); + + var services = new ServiceCollection(); + + _loggerProvider = new TestLoggerProvider(); + services.AddLogging(builder => builder.AddProvider(_loggerProvider)); + + _imageProviderState = new StateForTestImageProvider(); + services.AddSingleton(_imageProviderState); + services.AddTurnierplanImageStorage(config); + + _serviceProvider = services.BuildServiceProvider(); + } + + public void Dispose() + { + _serviceProvider.Dispose(); + + if (Directory.Exists(_basePath)) + { + Directory.Delete(_basePath, recursive: true); + } + } + + [Fact] + public async Task Migration___With_Empty_Directory_And_No_Images___Works_As_Expected() + { + await RunMigrationAsync(); + + ExpectLogMessages(LogEventIdUsingDirectoryForStorage, LogEventIdDirectoryStructureIsUpToDate); + } + + [Fact] + public async Task Migration___With_Images_In_Provider_But_Storage_Directory_Is_Empty___Works_As_Expected() + { + var image1 = AddNewImage(new DateTime(2025, 07, 03), createImageFile: false); + var image2 = AddNewImage(new DateTime(2025, 08, 15), createImageFile: false); + + // No image files exist in either old/new structure + CheckImageFileInOldStructure(image1, expectExists: false); + CheckImageFileInOldStructure(image2, expectExists: false); + CheckImageFileInNewStructure(image1, expectExists: false); + CheckImageFileInNewStructure(image2, expectExists: false); + + await RunMigrationAsync(); + + // If no physical files exist, the image storage migration will never query the image provider. + _imageProviderState.WasQueried.Should().BeFalse(); + + ExpectLogMessages(LogEventIdUsingDirectoryForStorage, LogEventIdDirectoryStructureIsUpToDate); + + // No image files exist in either old/new structure + CheckImageFileInOldStructure(image1, expectExists: false); + CheckImageFileInOldStructure(image2, expectExists: false); + CheckImageFileInNewStructure(image1, expectExists: false); + CheckImageFileInNewStructure(image2, expectExists: false); + } + + [Fact] + public async Task Migration___With_Images_In_Old_Structure___Works_As_Expected() + { + var image1 = AddNewImage(new DateTime(2023, 07, 03)); + var image2 = AddNewImage(new DateTime(2024, 08, 15)); + var image3 = AddNewImage(new DateTime(2024, 08, 17)); + var image4 = AddNewImage(new DateTime(2025, 03, 12)); + var image5 = AddNewImage(new DateTime(2025, 03, 05)); + var image6 = AddNewImage(new DateTime(2025, 06, 29)); + var image7 = AddNewImage(new DateTime(2026, 04, 21)); + var image8 = AddNewImage(new DateTime(2026, 06, 04)); + + CheckImageFiles([image1, image2, image3, image4, image5, image6, image7, image8], []); + await RunMigrationAsync(); + + _imageProviderState.WasQueried.Should().BeTrue(); + + ExpectLogMessages(LogEventIdUsingDirectoryForStorage, LogEventIdMigrationWillBeAttempted, LogEventIdSuccessfullyMovedFileTo); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2023[^\d]+{image1.ResourceIdentifier}\.{ImageExtension}' to '.+2023.+07.+{image1.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2024[^\d]+{image2.ResourceIdentifier}\.{ImageExtension}' to '.+2024.+08.+{image2.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2024[^\d]+{image3.ResourceIdentifier}\.{ImageExtension}' to '.+2024.+08.+{image3.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2025[^\d]+{image4.ResourceIdentifier}\.{ImageExtension}' to '.+2025.+03.+{image4.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2025[^\d]+{image5.ResourceIdentifier}\.{ImageExtension}' to '.+2025.+03.+{image5.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2025[^\d]+{image6.ResourceIdentifier}\.{ImageExtension}' to '.+2025.+06.+{image6.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2026[^\d]+{image7.ResourceIdentifier}\.{ImageExtension}' to '.+2026.+04.+{image7.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2026[^\d]+{image8.ResourceIdentifier}\.{ImageExtension}' to '.+2026.+06.+{image8.ResourceIdentifier}\.{ImageExtension}'\.$"); + + // Images should all be moved to the new structure - the old files should have been deleted + CheckImageFiles([], [image1, image2, image3, image4, image5, image6, image7, image8]); + } + + [Fact] + public async Task Migration___With_Images_In_Old_Structure_But_Image_Is_Missing_One_Specific_Image___Works_As_Expected() + { + var image1 = AddNewImage(new DateTime(2025, 05, 03)); + var image2 = AddNewImage(new DateTime(2025, 05, 15)); + var image3 = AddNewImage(new DateTime(2026, 02, 17)); + + _imageProviderState.Images.Remove(image2); + + CheckImageFiles([image1, image2, image3], []); + + await RunMigrationAsync(); + + _imageProviderState.WasQueried.Should().BeTrue(); + + ExpectLogMessages(LogEventIdUsingDirectoryForStorage, LogEventIdMigrationWillBeAttempted, LogEventIdSuccessfullyMovedFileTo, LogEventIdEncounteredFileWithoutCorrespondingImageFromProvider); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2025[^\d]+{image1.ResourceIdentifier}\.{ImageExtension}' to '.+2025.+05.+{image1.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2026[^\d]+{image3.ResourceIdentifier}\.{ImageExtension}' to '.+2026.+02.+{image3.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdEncounteredFileWithoutCorrespondingImageFromProvider, $@"^Encountered a file in the image storage directory for which there exists no corresponding entry from the image provider: '.+2025[^\d]+{image2.ResourceIdentifier}\.{ImageExtension}'$"); + + // Images 1 & 3 are moved to the new structure, while image 2 remains in the old structure + CheckImageFiles([image2], [image1, image3]); + } + + [Fact] + public async Task Migration___With_Images_In_Old_Structure_But_Image_Provider_Returns_No_Images___Works_As_Expected() + { + var image1 = AddNewImage(new DateTime(2025, 05, 03)); + var image2 = AddNewImage(new DateTime(2025, 05, 15)); + var image3 = AddNewImage(new DateTime(2026, 02, 17)); + + CheckImageFiles([image1, image2, image3], []); + + _imageProviderState.Images.Clear(); + + await RunMigrationAsync(); + + _imageProviderState.WasQueried.Should().BeTrue(); + + ExpectLogMessages(LogEventIdUsingDirectoryForStorage, LogEventIdMigrationWillBeAttempted, LogEventIdImageProviderReturnedNoImages); + + // Images should still exist in the old structure + CheckImageFiles([image1, image2, image3], []); + } + + [Fact] + public async Task Migration___With_Images_In_Old_Structure_But_One_Image_File_Already_Exists_In_New_Structure___Works_As_Expected() + { + var image1 = AddNewImage(new DateTime(2025, 05, 03)); + var image2 = AddNewImage(new DateTime(2025, 05, 15)); + var image3 = AddNewImage(new DateTime(2026, 02, 17)); + + CreateImageFileInNewStructure(image2.CreatedAt.Year, image2.CreatedAt.Month, GetImageFileName(image2), "data"); + + // Images 1 & 3 only exist in the old structure. Image 2 exists in both structures + CheckImageFiles([image1, image3], []); + CheckImageFileInOldStructure(image2, expectExists: true); + CheckImageFileInNewStructure(image2, expectExists: true); + + await RunMigrationAsync(); + + _imageProviderState.WasQueried.Should().BeTrue(); + + ExpectLogMessages(LogEventIdUsingDirectoryForStorage, LogEventIdMigrationWillBeAttempted, LogEventIdSuccessfullyMovedFileTo, LogEventIdEncounteredTargetFileThatAlreadyExists); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2025[^\d]+{image1.ResourceIdentifier}\.{ImageExtension}' to '.+2025.+05.+{image1.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2025[^\d]+{image2.ResourceIdentifier}\.{ImageExtension}' to '.+2025.+05.+{image2.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdSuccessfullyMovedFileTo, $@"^Successfully moved image file '.+2026[^\d]+{image3.ResourceIdentifier}\.{ImageExtension}' to '.+2026.+02.+{image3.ResourceIdentifier}\.{ImageExtension}'\.$"); + ExpectLogMessageExists(LogEventIdEncounteredTargetFileThatAlreadyExists, $@"^Encountered an already existing target file '.+2025.+05.+{image2.ResourceIdentifier}\.{ImageExtension}' while migrating '.+2025[^\d]+{image2.ResourceIdentifier}\.{ImageExtension}' - the move operation will be attempted again\.$"); + + // All images only exist in the new structure + CheckImageFiles([], [image1, image2, image3]); + } + + private async Task RunMigrationAsync() + { + await using var scope = _serviceProvider.CreateAsyncScope(); + var migrator = scope.ServiceProvider.GetRequiredService(); + await migrator.MigrateAsync(TestContext.Current.CancellationToken); + } + + private Image AddNewImage(DateTime createdAt, bool createImageFile = true) + { + var ctor = typeof(Image).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, [typeof(long), typeof(Guid), typeof(PublicId), typeof(DateTime), typeof(string), typeof(string), typeof(long), typeof(ushort), typeof(ushort)]); + var id = (long)_imageProviderState.Images.Count + 1; + var image = (Image)ctor!.Invoke([id, Guid.NewGuid(), PublicId.Empty, createdAt, $"Image{_imageProviderState.Images.Count + 1}", ImageExtension, (long)1, (ushort)1, (ushort)1]); + + _imageProviderState.Images.Add(image); + + if (createImageFile) + { + CreateImageFile(image.CreatedAt.Year, GetImageFileName(image), image.Name); + } + + return image; + } + + private void CreateImageFile(int year, string fileName, string contents) + { + var directory = Path.Join(_basePath, $"{year}"); + Directory.CreateDirectory(directory); + File.WriteAllText(Path.Join(directory, fileName), contents); + } + + private void CreateImageFileInNewStructure(int year, int month, string fileName, string contents) + { + var directory = Path.Join(_basePath, $"{year}", $"{month:D2}"); + Directory.CreateDirectory(directory); + File.WriteAllText(Path.Join(directory, fileName), contents); + } + + private void CheckImageFileInOldStructure(int year, string fileName, bool expectExists = true) + { + var path = Path.Join(_basePath, $"{year}", fileName); + File.Exists(path).Should().Be(expectExists); + } + + private void CheckImageFileInOldStructure(Image image, bool expectExists = true) + { + CheckImageFileInOldStructure(image.CreatedAt.Year, GetImageFileName(image), expectExists); + } + + private void CheckImageFileInNewStructure(int year, int month, string fileName, bool expectExists = true) + { + var path = Path.Join(_basePath, $"{year}", $"{month:D2}", fileName); + File.Exists(path).Should().Be(expectExists); + } + + private void CheckImageFileInNewStructure(Image image, bool expectExists = true) + { + CheckImageFileInNewStructure(image.CreatedAt.Year, image.CreatedAt.Month, GetImageFileName(image), expectExists); + } + + private void CheckImageFiles(Image[] imagesInOldStructure, Image[] imagesInNewStructure) + { + foreach (var image in imagesInOldStructure) + { + CheckImageFileInOldStructure(image, expectExists: true); + CheckImageFileInNewStructure(image, expectExists: false); + } + + foreach (var image in imagesInNewStructure) + { + CheckImageFileInOldStructure(image, expectExists: false); + CheckImageFileInNewStructure(image, expectExists: true); + } + } + + private void ExpectLogMessages(params int[] expectedEventIds) + { + var actualEventIds = _loggerProvider.Logger.Messages.Keys; + actualEventIds.Except(expectedEventIds).Should().BeEmpty(because: "all expected log event IDs should be present at least once in the actual log messages"); + expectedEventIds.Except(actualEventIds).Should().BeEmpty(because: "the actual log messages should only contain expected log event IDs"); + } + + private void ExpectLogMessageExists(int eventId, [StringSyntax(StringSyntaxAttribute.Regex)] string pattern) + { + _loggerProvider.Logger.Messages.TryGetValue(eventId, out var list).Should().BeTrue(); + var regex = new Regex(pattern); + list.Should().ContainSingle(x => regex.IsMatch(x)); + } + + private static string GetImageFileName(Image image) => $"{image.ResourceIdentifier}.{ImageExtension}"; + + private sealed class TestLogger : ILogger + { + public readonly Dictionary> Messages = []; + + public IDisposable BeginScope(TState state) where TState : notnull => throw new NotSupportedException(); + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + Messages.TryAdd(eventId.Id, []); + Messages[eventId.Id].Add(formatter(state, exception)); + } + } + + private sealed class TestLoggerProvider : ILoggerProvider + { + public readonly TestLogger Logger = new(); + + public ILogger CreateLogger(string name) => Logger; + + public void Dispose() { } + } + + private sealed class StateForTestImageProvider + { + public readonly List Images = []; + public bool WasQueried; + } + + private sealed class TestImageProvider(StateForTestImageProvider state) : IImageProvider + { + public Task> GetImagesAsync() + { + state.WasQueried = true; + + return Task.FromResult>(state.Images.AsReadOnly()); + } + } +} diff --git a/src/Turnierplan.ImageStorage.Test.Migration/Turnierplan.ImageStorage.Test.Migration.csproj b/src/Turnierplan.ImageStorage.Test.Migration/Turnierplan.ImageStorage.Test.Migration.csproj new file mode 100644 index 00000000..2d56fe69 --- /dev/null +++ b/src/Turnierplan.ImageStorage.Test.Migration/Turnierplan.ImageStorage.Test.Migration.csproj @@ -0,0 +1,15 @@ + + + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/Turnierplan.ImageStorage/AssemblyInfo.cs b/src/Turnierplan.ImageStorage/AssemblyInfo.cs new file mode 100644 index 00000000..3533c6d3 --- /dev/null +++ b/src/Turnierplan.ImageStorage/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Turnierplan.ImageStorage.Test.Migration")] diff --git a/src/Turnierplan.ImageStorage/Extensions/EndpointRouteBuilderExtensions.cs b/src/Turnierplan.ImageStorage/Extensions/EndpointRouteBuilderExtensions.cs deleted file mode 100644 index 171c10c8..00000000 --- a/src/Turnierplan.ImageStorage/Extensions/EndpointRouteBuilderExtensions.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; -using Turnierplan.ImageStorage.Local; - -namespace Turnierplan.ImageStorage.Extensions; - -public static class EndpointRouteBuilderExtensions -{ - public static void MapImageStorageEndpoint(this IApplicationBuilder builder) - { - var imageStorage = builder.ApplicationServices.GetRequiredService(); - - if (imageStorage is not ILocalImageStorage localImageStorage) - { - return; - } - - localImageStorage.MapEndpoint(builder); - } -} diff --git a/src/Turnierplan.ImageStorage/Extensions/ServiceCollectionExtensions.cs b/src/Turnierplan.ImageStorage/Extensions/ServiceCollectionExtensions.cs index b7cf35b2..b78e8acf 100644 --- a/src/Turnierplan.ImageStorage/Extensions/ServiceCollectionExtensions.cs +++ b/src/Turnierplan.ImageStorage/Extensions/ServiceCollectionExtensions.cs @@ -8,7 +8,8 @@ namespace Turnierplan.ImageStorage.Extensions; public static class ServiceCollectionExtensions { - public static void AddTurnierplanImageStorage(this IServiceCollection services, IConfigurationSection configuration) + public static void AddTurnierplanImageStorage(this IServiceCollection services, IConfigurationSection configuration) + where T : IImageProvider { var type = configuration.GetValue("Type"); @@ -29,5 +30,7 @@ public static void AddTurnierplanImageStorage(this IServiceCollection services, default: throw new InvalidOperationException($"Invalid image storage type specified: '{type}'"); } + + services.AddTransient(sp => new ImageStorageMigrator(sp.GetRequiredService(), ActivatorUtilities.CreateInstance(sp))); } } diff --git a/src/Turnierplan.ImageStorage/Extensions/WebApplicationExtensions.cs b/src/Turnierplan.ImageStorage/Extensions/WebApplicationExtensions.cs new file mode 100644 index 00000000..c7833596 --- /dev/null +++ b/src/Turnierplan.ImageStorage/Extensions/WebApplicationExtensions.cs @@ -0,0 +1,28 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +namespace Turnierplan.ImageStorage.Extensions; + +public static class WebApplicationExtensions +{ + public static void MapImageStorageEndpoint(this WebApplication application) + { + var imageStorage = application.Services.GetRequiredService(); + + if (imageStorage is not IHostedImageStorage localImageStorage) + { + return; + } + + localImageStorage.MapEndpoint(application); + } + + public static async Task MigrateImageStorageAsync(this WebApplication application) + { + await using var scope = application.Services.CreateAsyncScope(); + + var migrator = scope.ServiceProvider.GetRequiredService(); + + await migrator.MigrateAsync(application.Lifetime.ApplicationStopping); + } +} diff --git a/src/Turnierplan.ImageStorage/IHostedImageStorage.cs b/src/Turnierplan.ImageStorage/IHostedImageStorage.cs new file mode 100644 index 00000000..387393a4 --- /dev/null +++ b/src/Turnierplan.ImageStorage/IHostedImageStorage.cs @@ -0,0 +1,8 @@ +using Microsoft.AspNetCore.Builder; + +namespace Turnierplan.ImageStorage; + +internal interface IHostedImageStorage +{ + void MapEndpoint(IApplicationBuilder applicationBuilder); +} diff --git a/src/Turnierplan.ImageStorage/IImageProvider.cs b/src/Turnierplan.ImageStorage/IImageProvider.cs new file mode 100644 index 00000000..c11dc0d9 --- /dev/null +++ b/src/Turnierplan.ImageStorage/IImageProvider.cs @@ -0,0 +1,8 @@ +using Turnierplan.Core.Image; + +namespace Turnierplan.ImageStorage; + +public interface IImageProvider +{ + Task> GetImagesAsync(); +} diff --git a/src/Turnierplan.ImageStorage/IMigratableImageStorage.cs b/src/Turnierplan.ImageStorage/IMigratableImageStorage.cs new file mode 100644 index 00000000..e75fe58e --- /dev/null +++ b/src/Turnierplan.ImageStorage/IMigratableImageStorage.cs @@ -0,0 +1,6 @@ +namespace Turnierplan.ImageStorage; + +internal interface IMigratableImageStorage +{ + Task MigrateAsync(IImageProvider imageProvider, CancellationToken cancellationToken); +} diff --git a/src/Turnierplan.ImageStorage/ImageStorageMigrator.cs b/src/Turnierplan.ImageStorage/ImageStorageMigrator.cs new file mode 100644 index 00000000..b739ad5a --- /dev/null +++ b/src/Turnierplan.ImageStorage/ImageStorageMigrator.cs @@ -0,0 +1,14 @@ +namespace Turnierplan.ImageStorage; + +internal sealed class ImageStorageMigrator(IImageStorage imageStorage, IImageProvider imageProvider) +{ + public async Task MigrateAsync(CancellationToken cancellationToken = default) + { + if (imageStorage is not IMigratableImageStorage migratableImageStorage) + { + return; + } + + await migratableImageStorage.MigrateAsync(imageProvider, cancellationToken); + } +} diff --git a/src/Turnierplan.ImageStorage/Local/ILocalImageStorage.cs b/src/Turnierplan.ImageStorage/Local/ILocalImageStorage.cs deleted file mode 100644 index fd04324b..00000000 --- a/src/Turnierplan.ImageStorage/Local/ILocalImageStorage.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Microsoft.AspNetCore.Builder; - -namespace Turnierplan.ImageStorage.Local; - -public interface ILocalImageStorage : IImageStorage -{ - void MapEndpoint(IApplicationBuilder builder); -} diff --git a/src/Turnierplan.ImageStorage/Local/LocalImageStorage.cs b/src/Turnierplan.ImageStorage/Local/LocalImageStorage.cs index e4b12d11..c973c9c8 100644 --- a/src/Turnierplan.ImageStorage/Local/LocalImageStorage.cs +++ b/src/Turnierplan.ImageStorage/Local/LocalImageStorage.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Text.RegularExpressions; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Logging; @@ -7,31 +8,34 @@ namespace Turnierplan.ImageStorage.Local; -internal sealed class LocalImageStorage : ILocalImageStorage +internal sealed partial class LocalImageStorage : IImageStorage, IHostedImageStorage, IMigratableImageStorage { - private readonly ILogger _logger; + private readonly LocalImageStorageLogger _logger; private readonly string _storagePath; + private readonly SemaphoreSlim _migrationSemaphore = new(1, 1); + private readonly bool _skipMigration; public LocalImageStorage(ILogger logger, IOptions options) { - _logger = logger; + _logger = new LocalImageStorageLogger(logger); ArgumentException.ThrowIfNullOrWhiteSpace(options.Value.StoragePath); _storagePath = Path.GetFullPath(options.Value.StoragePath); - _logger.LogInformation("Using the following directory for local image storage: '{LocalImageStoragePath}'", _storagePath); + _logger.UsingDirectoryForStorage(_storagePath); Directory.CreateDirectory(_storagePath); if (!Directory.Exists(_storagePath)) { - _logger.LogCritical("The directory for local image storage does not exist and could not be created."); + _logger.RootDirectoryDoesNotExistAndCouldNotBeCreated(); + _skipMigration = true; } } public string GetFullImageUrl(Image image) { - return $"/images/{image.CreatedAt.Year}/{GetImageFileName(image)}"; + return $"/images/{image.CreatedAt.Year}/{image.CreatedAt.Month:D2}/{GetImageFileName(image)}"; } public Task SaveImageAsync(Image image, MemoryStream imageData) @@ -50,13 +54,13 @@ public Task SaveImageAsync(Image image, MemoryStream imageData) if (!Directory.Exists(imageDirectoryPath)) { - _logger.LogCritical("The directory for the image could not be created: '{ImageDirectory}'.", imageDirectoryPath); + _logger.ImageDirectoryCouldNotBeCreated(imageDirectoryPath); return Task.FromResult(false); } } - _logger.LogDebug("Writing image to: {FilePath}", filePath); + _logger.WritingImageToPath(filePath); using var destination = new FileStream(filePath, FileMode.Create); imageData.CopyTo(destination); @@ -65,7 +69,7 @@ public Task SaveImageAsync(Image image, MemoryStream imageData) } catch (Exception ex) { - _logger.LogError(ex, "Failed to save image to file to '{FilePath}'.", filePath); + _logger.FailedToWriteImageToPath(filePath, ex); return Task.FromResult(false); } @@ -86,7 +90,7 @@ public Task DeleteImageAsync(Image image) } catch (Exception ex) { - _logger.LogError(ex, "Failed to delete image file '{FilePath}'.", filePath); + _logger.FailedToDeleteImageFile(filePath, ex); return Task.FromResult(false); } @@ -103,17 +107,124 @@ public void MapEndpoint(IApplicationBuilder builder) }); } + public async Task MigrateAsync(IImageProvider imageProvider, CancellationToken cancellationToken) + { + if (_skipMigration) + { + _logger.SkippingMigration(); + return; + } + + /* + * This migration copies files from the previous folder structure to the new one: + * + * Previous folder structure: + * /2025/5b22d459-58b8-4aac-8ad0-1b0ba5e453b3.webp + * /2026/7e669676-443b-4d01-a12e-676c9c097739.webp + * + * New folder structure: + * /2025/07/5b22d459-58b8-4aac-8ad0-1b0ba5e453b3.webp + * /2026/04/7e669676-443b-4d01-a12e-676c9c097739.webp + * + * The new folder structure introduces a month-based subdirectory. This is done to be aligned with the + * other image storage implementations, which should make switching between implementations easier. + */ + + try + { + await _migrationSemaphore.WaitAsync(cancellationToken); + + // First, we figure out if a migration needs to be run by checking if any files with the old naming scheme exist. + var hasOldFiles = Directory.GetDirectories(_storagePath).SelectMany(Directory.GetFiles).Any(); + if (!hasOldFiles) + { + _logger.DirectoryStructureIsUpToDate(); + return; + } + + _logger.MigrationWillBeAttempted(); + + // Load images from DB because we need the months of creation + var images = await imageProvider.GetImagesAsync(); + + // Consistency check - there might be an error in the image provider implementation causing invalid data to be returned. + if (images.Count == 0) + { + _logger.ImageProviderReturnedNoImages(); + return; + } + + // Iterate over all old files and handle them accordingly + foreach (var directory in Directory.GetDirectories(_storagePath)) + { + foreach (var file in Directory.GetFiles(directory)) + { + var fileName = Path.GetFileName(file); + var match = FileNameRegex().Match(fileName); + + if (!match.Success) + { + _logger.EncounteredFileWhichDoesNotMatchPattern(Path.GetFullPath(file)); + continue; + } + + if (!Guid.TryParse(match.Groups["identifier"].Value, out var resourceIdentifier)) + { + _logger.EncounteredFileThatHasNoValidGuidInName(Path.GetFullPath(file)); + continue; + } + + var image = images.FirstOrDefault(x => x.ResourceIdentifier == resourceIdentifier); + + if (image is null) + { + _logger.EncounteredFileWithoutCorrespondingImageFromProvider(Path.GetFullPath(file)); + continue; + } + + var targetDirectory = Path.Join(_storagePath, image.CreatedAt.Year.ToString(CultureInfo.InvariantCulture), image.CreatedAt.Month.ToString("D2", CultureInfo.InvariantCulture)); + var targetPath = Path.Join(targetDirectory, fileName); + + Directory.CreateDirectory(targetDirectory); + + if (File.Exists(targetPath)) + { + _logger.EncounteredTargetFileThatAlreadyExists(targetPath, file); + } + + File.Move(file, targetPath, overwrite: true); + _logger.SuccessfullyMovedFileTo(file, targetPath); + } + } + } + catch (Exception ex) + { + _logger.ExceptionOccurredDuringMigration(ex); + } + finally + { + _migrationSemaphore.Release(); + } + } + public void Dispose() { } private string GetImageFullPath(Image image) { - return Path.Join(_storagePath, image.CreatedAt.Year.ToString(CultureInfo.InvariantCulture), GetImageFileName(image)); + return Path.Join( + _storagePath, + image.CreatedAt.Year.ToString(CultureInfo.InvariantCulture), + image.CreatedAt.Month.ToString("D2", CultureInfo.InvariantCulture), + GetImageFileName(image)); } private static string GetImageFileName(Image image) { return $"{image.ResourceIdentifier}.{image.FileExtension}"; } + + [GeneratedRegex(@"(?[0-9a-f-]{36})\.(?[a-z]+)$")] + private static partial Regex FileNameRegex(); } diff --git a/src/Turnierplan.ImageStorage/Local/LocalImageStorageLogger.cs b/src/Turnierplan.ImageStorage/Local/LocalImageStorageLogger.cs new file mode 100644 index 00000000..9cd0e8b9 --- /dev/null +++ b/src/Turnierplan.ImageStorage/Local/LocalImageStorageLogger.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Logging; + +namespace Turnierplan.ImageStorage.Local; + +internal sealed partial class LocalImageStorageLogger(ILogger logger) +{ + [LoggerMessage(LogLevel.Information, "Using the following directory for local image storage: '{LocalImageStoragePath}'", EventId = 1)] + public partial void UsingDirectoryForStorage(string localImageStoragePath); + + [LoggerMessage(LogLevel.Critical, "The directory for local image storage does not exist and could not be created.", EventId = 2)] + public partial void RootDirectoryDoesNotExistAndCouldNotBeCreated(); + + [LoggerMessage(LogLevel.Critical, "The directory for the image could not be created: '{ImageDirectory}'.", EventId = 3)] + public partial void ImageDirectoryCouldNotBeCreated(string imageDirectory); + + [LoggerMessage(LogLevel.Debug, "Writing image to: {FilePath}", EventId = 4)] + public partial void WritingImageToPath(string filePath); + + [LoggerMessage(LogLevel.Error, "Failed to save image to file to '{FilePath}'.", EventId = 5)] + public partial void FailedToWriteImageToPath(string filePath, Exception exception); + + [LoggerMessage(LogLevel.Error, "Failed to delete image file '{FilePath}'.", EventId = 6)] + public partial void FailedToDeleteImageFile(string filePath, Exception exception); + + [LoggerMessage(LogLevel.Critical, "A previous initialization error of local image storage will cause the migrations to be skipped.", EventId = 7)] + public partial void SkippingMigration(); + + [LoggerMessage(LogLevel.Information, "The directory structure is up to date, no migration is needed.", EventId = 8)] + public partial void DirectoryStructureIsUpToDate(); + + [LoggerMessage(LogLevel.Information, "A migration will be attempted because at least one file in the storage directory matches the old file pattern.", EventId = 9)] + public partial void MigrationWillBeAttempted(); + + [LoggerMessage(LogLevel.Warning, "The image provider returned no images even though the image storage folder contains old image files - migration is skipped.", EventId = 10)] + public partial void ImageProviderReturnedNoImages(); + + [LoggerMessage(LogLevel.Warning, "Encountered a file in the image storage directory which does not match the file name pattern: '{ImagePath}'", EventId = 11)] + public partial void EncounteredFileWhichDoesNotMatchPattern(string imagePath); + + [LoggerMessage(LogLevel.Warning, "Encountered a file in the image storage directory which does not have a valid GUID in the file name: '{ImagePath}'", EventId = 12)] + public partial void EncounteredFileThatHasNoValidGuidInName(string imagePath); + + [LoggerMessage(LogLevel.Warning, "Encountered a file in the image storage directory for which there exists no corresponding entry from the image provider: '{ImagePath}'", EventId = 13)] + public partial void EncounteredFileWithoutCorrespondingImageFromProvider(string imagePath); + + [LoggerMessage(LogLevel.Warning, "Encountered an already existing target file '{TargetPath}' while migrating '{SourcePath}' - the move operation will be attempted again.", EventId = 14)] + public partial void EncounteredTargetFileThatAlreadyExists(string targetPath, string sourcePath); + + [LoggerMessage(LogLevel.Information, "Successfully moved image file '{SourcePath}' to '{TargetPath}'.", EventId = 15)] + public partial void SuccessfullyMovedFileTo(string sourcePath, string targetPath); + + [LoggerMessage(LogLevel.Critical, "An unexpected exception occurred while trying to run local image storage migrations.", EventId = 16)] + public partial void ExceptionOccurredDuringMigration(Exception exception); +} diff --git a/src/turnierplan.NET.slnx b/src/turnierplan.NET.slnx index b163cf28..31da8eed 100644 --- a/src/turnierplan.NET.slnx +++ b/src/turnierplan.NET.slnx @@ -10,6 +10,7 @@ +