diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8eec027 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +TELEGRAM_BOT_TOKEN=your-telegram-bot-token + +POSTGRES_DB=linktracker +POSTGRES_USER=linktracker +POSTGRES_PASSWORD=linktracker \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4fd0f7e..7e2fd40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: bot: build: @@ -23,4 +21,30 @@ services: environment: - ASPNETCORE_ENVIRONMENT=Development - ASPNETCORE_HTTP_PORTS=8080 - - BotUrl=http://bot:8080 \ No newline at end of file + - BotUrl=http://bot:8080 + - Database__AccessType=SQL + - Database__ConnectionString=Host=postgres;Port=5432;Database=${POSTGRES_DB};Username=${POSTGRES_USER};Password=${POSTGRES_PASSWORD} + - Database__RunMigrations=true + depends_on: + postgres: + condition: service_healthy + + postgres: + image: postgres:16 + container_name: linktracker-postgres + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + ports: + - "5433:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 + +volumes: + postgres-data: \ No newline at end of file diff --git a/migrations/001_initial_schema.sql b/migrations/001_initial_schema.sql new file mode 100644 index 0000000..210d635 --- /dev/null +++ b/migrations/001_initial_schema.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS chats ( + id BIGINT PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS links ( + id BIGSERIAL PRIMARY KEY, + url TEXT NOT NULL UNIQUE, + last_checked_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chat_links ( + chat_id BIGINT NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + link_id BIGINT NOT NULL REFERENCES links(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (chat_id, link_id) +); + +CREATE TABLE IF NOT EXISTS tags ( + id BIGSERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chat_link_tags ( + chat_id BIGINT NOT NULL, + link_id BIGINT NOT NULL, + tag_id BIGINT NOT NULL REFERENCES tags(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, link_id, tag_id), + FOREIGN KEY (chat_id, link_id) + REFERENCES chat_links(chat_id, link_id) + ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS ix_links_url ON links(url); +CREATE INDEX IF NOT EXISTS ix_chat_links_link_id ON chat_links(link_id); +CREATE INDEX IF NOT EXISTS ix_chat_link_tags_tag_id ON chat_link_tags(tag_id); +CREATE INDEX IF NOT EXISTS ix_tags_name ON tags(name); \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Configuration/DatabaseOptions.cs b/src/LinkTracker.Scrapper/Configuration/DatabaseOptions.cs new file mode 100644 index 0000000..fcc08e8 --- /dev/null +++ b/src/LinkTracker.Scrapper/Configuration/DatabaseOptions.cs @@ -0,0 +1,12 @@ +namespace LinkTracker.Scrapper.Configuration; + +public sealed class DatabaseOptions +{ + public const string SectionName = "Database"; + + public string AccessType { get; init; } = "SQL"; + + public string ConnectionString { get; init; } = string.Empty; + + public bool RunMigrations { get; init; } = true; +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Database/Entities/ChatEntity.cs b/src/LinkTracker.Scrapper/Database/Entities/ChatEntity.cs new file mode 100644 index 0000000..a5a7675 --- /dev/null +++ b/src/LinkTracker.Scrapper/Database/Entities/ChatEntity.cs @@ -0,0 +1,10 @@ +namespace LinkTracker.Scrapper.Database.Entities; + +public class ChatEntity +{ + public long Id { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public ICollection ChatLinks { get; set; } = new List(); +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Database/Entities/ChatLinkEntity.cs b/src/LinkTracker.Scrapper/Database/Entities/ChatLinkEntity.cs new file mode 100644 index 0000000..b5a766e --- /dev/null +++ b/src/LinkTracker.Scrapper/Database/Entities/ChatLinkEntity.cs @@ -0,0 +1,16 @@ +namespace LinkTracker.Scrapper.Database.Entities; + +public class ChatLinkEntity +{ + public long ChatId { get; set; } + + public long LinkId { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public ChatEntity Chat { get; set; } = null!; + + public LinkEntity Link { get; set; } = null!; + + public ICollection ChatLinkTags { get; set; } = new List(); +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Database/Entities/ChatLinkTagEntity.cs b/src/LinkTracker.Scrapper/Database/Entities/ChatLinkTagEntity.cs new file mode 100644 index 0000000..c655b7d --- /dev/null +++ b/src/LinkTracker.Scrapper/Database/Entities/ChatLinkTagEntity.cs @@ -0,0 +1,14 @@ +namespace LinkTracker.Scrapper.Database.Entities; + +public class ChatLinkTagEntity +{ + public long ChatId { get; set; } + + public long LinkId { get; set; } + + public long TagId { get; set; } + + public ChatLinkEntity ChatLink { get; set; } = null!; + + public TagEntity Tag { get; set; } = null!; +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Database/Entities/LinkEntity.cs b/src/LinkTracker.Scrapper/Database/Entities/LinkEntity.cs new file mode 100644 index 0000000..f706182 --- /dev/null +++ b/src/LinkTracker.Scrapper/Database/Entities/LinkEntity.cs @@ -0,0 +1,14 @@ +namespace LinkTracker.Scrapper.Database.Entities; + +public class LinkEntity +{ + public long Id { get; set; } + + public string Url { get; set; } = string.Empty; + + public DateTimeOffset LastCheckedAt { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public ICollection ChatLinks { get; set; } = new List(); +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Database/Entities/TagEntity.cs b/src/LinkTracker.Scrapper/Database/Entities/TagEntity.cs new file mode 100644 index 0000000..13f4eea --- /dev/null +++ b/src/LinkTracker.Scrapper/Database/Entities/TagEntity.cs @@ -0,0 +1,12 @@ +namespace LinkTracker.Scrapper.Database.Entities; + +public class TagEntity +{ + public long Id { get; set; } + + public string Name { get; set; } = string.Empty; + + public DateTimeOffset CreatedAt { get; set; } + + public ICollection ChatLinkTags { get; set; } = new List(); +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Database/LinkTrackerDbContext.cs b/src/LinkTracker.Scrapper/Database/LinkTrackerDbContext.cs new file mode 100644 index 0000000..b18ee4c --- /dev/null +++ b/src/LinkTracker.Scrapper/Database/LinkTrackerDbContext.cs @@ -0,0 +1,143 @@ +using Microsoft.EntityFrameworkCore; +using LinkTracker.Scrapper.Database.Entities; + +namespace LinkTracker.Scrapper.Database; + +public class LinkTrackerDbContext(DbContextOptions options) : DbContext(options) +{ + public DbSet Chats => Set(); + + public DbSet Links => Set(); + + public DbSet ChatLinks => Set(); + + public DbSet Tags => Set(); + + public DbSet ChatLinkTags => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("chats"); + + entity.HasKey(chat => chat.Id); + + entity.Property(chat => chat.Id) + .HasColumnName("id") + .ValueGeneratedNever(); + + entity.Property(chat => chat.CreatedAt) + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("links"); + + entity.HasKey(link => link.Id); + + entity.Property(link => link.Id) + .HasColumnName("id"); + + entity.Property(link => link.Url) + .HasColumnName("url") + .IsRequired(); + + entity.Property(link => link.LastCheckedAt) + .HasColumnName("last_checked_at") + .HasDefaultValueSql("NOW()"); + + entity.Property(link => link.CreatedAt) + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + entity.HasIndex(link => link.Url) + .IsUnique(); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("chat_links"); + + entity.HasKey(chatLink => new { chatLink.ChatId, chatLink.LinkId }); + + entity.Property(chatLink => chatLink.ChatId) + .HasColumnName("chat_id"); + + entity.Property(chatLink => chatLink.LinkId) + .HasColumnName("link_id"); + + entity.Property(chatLink => chatLink.CreatedAt) + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + entity.HasOne(chatLink => chatLink.Chat) + .WithMany(chat => chat.ChatLinks) + .HasForeignKey(chatLink => chatLink.ChatId) + .OnDelete(DeleteBehavior.Cascade); + + entity.HasOne(chatLink => chatLink.Link) + .WithMany(link => link.ChatLinks) + .HasForeignKey(chatLink => chatLink.LinkId) + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("tags"); + + entity.HasKey(tag => tag.Id); + + entity.Property(tag => tag.Id) + .HasColumnName("id"); + + entity.Property(tag => tag.Name) + .HasColumnName("name") + .IsRequired(); + + entity.Property(tag => tag.CreatedAt) + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + entity.HasIndex(tag => tag.Name) + .IsUnique(); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("chat_link_tags"); + + entity.HasKey(chatLinkTag => new + { + chatLinkTag.ChatId, + chatLinkTag.LinkId, + chatLinkTag.TagId + }); + + entity.Property(chatLinkTag => chatLinkTag.ChatId) + .HasColumnName("chat_id"); + + entity.Property(chatLinkTag => chatLinkTag.LinkId) + .HasColumnName("link_id"); + + entity.Property(chatLinkTag => chatLinkTag.TagId) + .HasColumnName("tag_id"); + + entity.HasOne(chatLinkTag => chatLinkTag.ChatLink) + .WithMany(chatLink => chatLink.ChatLinkTags) + .HasForeignKey(chatLinkTag => new + { + chatLinkTag.ChatId, + chatLinkTag.LinkId + }) + .OnDelete(DeleteBehavior.Cascade); + + entity.HasOne(chatLinkTag => chatLinkTag.Tag) + .WithMany(tag => tag.ChatLinkTags) + .HasForeignKey(chatLinkTag => chatLinkTag.TagId) + .OnDelete(DeleteBehavior.Cascade); + }); + } +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Database/MigrationRunner.cs b/src/LinkTracker.Scrapper/Database/MigrationRunner.cs new file mode 100644 index 0000000..9be8c6f --- /dev/null +++ b/src/LinkTracker.Scrapper/Database/MigrationRunner.cs @@ -0,0 +1,47 @@ +using DbUp; +using LinkTracker.Scrapper.Configuration; +using Microsoft.Extensions.Options; + +namespace LinkTracker.Scrapper.Database; + +public static class MigrationRunner +{ + public static void Run(IServiceProvider services) + { + using var scope = services.CreateScope(); + + var options = scope.ServiceProvider + .GetRequiredService>() + .Value; + + if (!options.RunMigrations) + { + return; + } + + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new InvalidOperationException("Database connection string is not configured."); + } + + var migrationsPath = Path.Combine(AppContext.BaseDirectory, "migrations"); + + if (!Directory.Exists(migrationsPath)) + { + throw new DirectoryNotFoundException($"Migrations directory not found: {migrationsPath}"); + } + + var upgrader = DeployChanges.To + .PostgresqlDatabase(options.ConnectionString) + .WithScriptsFromFileSystem(migrationsPath) + .LogToConsole() + .Build(); + + var result = upgrader.PerformUpgrade(); + + if (!result.Successful) + { + throw new Exception($"Failed to upgrade migrations: {result.Error}"); + } + } +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/LinkTracker.Scrapper.csproj b/src/LinkTracker.Scrapper/LinkTracker.Scrapper.csproj index 5888887..c206ac9 100644 --- a/src/LinkTracker.Scrapper/LinkTracker.Scrapper.csproj +++ b/src/LinkTracker.Scrapper/LinkTracker.Scrapper.csproj @@ -11,8 +11,12 @@ - + + + + + @@ -25,4 +29,10 @@ + + + migrations\%(RecursiveDir)%(Filename)%(Extension) + PreserveNewest + + diff --git a/src/LinkTracker.Scrapper/Program.cs b/src/LinkTracker.Scrapper/Program.cs index 064ebc6..4a014bb 100644 --- a/src/LinkTracker.Scrapper/Program.cs +++ b/src/LinkTracker.Scrapper/Program.cs @@ -2,13 +2,17 @@ using LinkTracker.Scrapper.Repositories; using LinkTracker.Scrapper.Clients; using LinkTracker.Scrapper.Jobs; +using LinkTracker.Scrapper.Configuration; +using LinkTracker.Scrapper.Database; +using Microsoft.EntityFrameworkCore.Migrations; var builder = WebApplication.CreateBuilder(args); builder.Services.AddControllers(); builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); - +builder.Services.Configure( + builder.Configuration.GetSection(DatabaseOptions.SectionName)); builder.Services.AddSingleton(); @@ -37,6 +41,8 @@ var app = builder.Build(); +MigrationRunner.Run(app.Services); + if (app.Environment.IsDevelopment()) { app.UseSwagger(); app.UseSwaggerUI(); diff --git a/src/LinkTracker.Scrapper/Repositories/ILinkRepository.cs b/src/LinkTracker.Scrapper/Repositories/ILinkRepository.cs index 7274456..85a3598 100644 --- a/src/LinkTracker.Scrapper/Repositories/ILinkRepository.cs +++ b/src/LinkTracker.Scrapper/Repositories/ILinkRepository.cs @@ -9,7 +9,7 @@ public interface ILinkRepository bool ChatExists(long chatId); LinkResponse? AddLink(long chatId, string url, string[]? tags); bool RemoveLink(long chatId, string url); - IEnumerable GetLinks(long chatId); - IEnumerable<(string Url, long[] ChatIds, DateTimeOffset LastUpdate)> GetLinksForUpdate(); + IEnumerable GetLinks(long chatId, string? tag = null, int offset = 0, int limit = 100); + IEnumerable<(string Url, long[] ChatIds, DateTimeOffset LastUpdate)> GetLinksForUpdate(int offset = 0, int limit = 100); void UpdateLastCheckTime(string url, DateTimeOffset lastUpdate); } \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Repositories/ITagRepository.cs b/src/LinkTracker.Scrapper/Repositories/ITagRepository.cs new file mode 100644 index 0000000..4f2224f --- /dev/null +++ b/src/LinkTracker.Scrapper/Repositories/ITagRepository.cs @@ -0,0 +1,12 @@ +using LinkTracker.Shared.Models; + +namespace LinkTracker.Scrapper.Repositories; + +public interface ITagRepository +{ + TagResponse Create(string name); + TagResponse? Get(long Id); + IEnumerable GetAll(int offset = 0, int limit = 100); + TagResponse? Update(long id, string name); + bool Delete(long id); +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Repositories/InMemoryLinkRepository.cs b/src/LinkTracker.Scrapper/Repositories/InMemoryLinkRepository.cs index 971ca55..e63d8d5 100644 --- a/src/LinkTracker.Scrapper/Repositories/InMemoryLinkRepository.cs +++ b/src/LinkTracker.Scrapper/Repositories/InMemoryLinkRepository.cs @@ -6,52 +6,103 @@ namespace LinkTracker.Scrapper.Repositories; public class InMemoryLinkRepository : ILinkRepository { private readonly ConcurrentDictionary> _userLinks = new(); - private readonly ConcurrentDictionary _linkMetadata= new(); + private readonly ConcurrentDictionary _linkMetadata = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _linkIdsByUrl = new(StringComparer.OrdinalIgnoreCase); + private readonly object _lock = new(); + + private long _nextLinkId; + + public void AddChat(long chatId) => _userLinks.TryAdd(chatId, new List()); - public void AddChat(long chatId) => _userLinks.TryAdd(chatId, new()); public void RemoveChat(long chatId) => _userLinks.TryRemove(chatId, out _); + public bool ChatExists(long chatId) => _userLinks.ContainsKey(chatId); public LinkResponse? AddLink(long chatId, string url, string[]? tags) { - if (!_userLinks.ContainsKey(chatId)) + lock (_lock) { - return null; + if (!_userLinks.TryGetValue(chatId, out var links)) + { + return null; + } + + if (links.Any(link => string.Equals(link.Url, url, StringComparison.OrdinalIgnoreCase))) + { + return null; + } + + var linkId = _linkIdsByUrl.GetOrAdd(url, _ => Interlocked.Increment(ref _nextLinkId)); + var link = new LinkResponse(linkId, url, tags ?? Array.Empty()); + + links.Add(link); + _linkMetadata.TryAdd(url, DateTimeOffset.UtcNow); + + return link; } + } - if (_userLinks[chatId].Any(l => l.Url == url)) + public bool RemoveLink(long chatId, string url) + { + lock (_lock) { - return null; + return _userLinks.TryGetValue(chatId, out var links) + && links.RemoveAll(link => string.Equals(link.Url, url, StringComparison.OrdinalIgnoreCase)) > 0; } - - var link = new LinkResponse(Random.Shared.Next(1, 10000), url, tags ?? Array.Empty()); - _userLinks[chatId].Add(link); - _linkMetadata.TryAdd(url, DateTimeOffset.UtcNow); - return link; } - public bool RemoveLink(long chatId, string url) => - _userLinks.TryGetValue(chatId, out var links) && links.RemoveAll(l => l.Url == url) > 0; + public IEnumerable GetLinks(long chatId, string? tag = null, int offset = 0, int limit = 100) + { + lock (_lock) + { + if (!_userLinks.TryGetValue(chatId, out var links)) + { + return Enumerable.Empty(); + } + + var query = links.AsEnumerable(); - public IEnumerable GetLinks(long chatId) => - _userLinks.TryGetValue(chatId, out var links) ? links.ToList() : Enumerable.Empty(); + if (!string.IsNullOrWhiteSpace(tag)) + { + query = query.Where(link => + link.Tags.Any(linkTag => string.Equals(linkTag, tag, StringComparison.OrdinalIgnoreCase))); + } - public IEnumerable<(string Url, long[] ChatIds, DateTimeOffset LastUpdate)> GetLinksForUpdate() + return query + .Skip(offset) + .Take(limit) + .ToList(); + } + } + + public IEnumerable<(string Url, long[] ChatIds, DateTimeOffset LastUpdate)> GetLinksForUpdate( + int offset = 0, + int limit = 100) { - return _linkMetadata.Select(m => + lock (_lock) { - var url = m.Key; - var lastUpdate = m.Value; + return _linkMetadata + .Select(metadata => + { + var url = metadata.Key; + var chatIds = _userLinks + .Where(userLinks => userLinks.Value.Any(link => + string.Equals(link.Url, url, StringComparison.OrdinalIgnoreCase))) + .Select(userLinks => userLinks.Key) + .Distinct() + .ToArray(); - var chatIds = _userLinks - .Where(u => u.Value.Any(l => l.Url == url)) - .Select(u => u.Key) - .Distinct() - .ToArray(); + return (Url: url, ChatIds: chatIds, LastUpdate: metadata.Value); + }) + .Where(link => link.ChatIds.Length > 0) + .Skip(offset) + .Take(limit) + .ToList(); + } + } - return (url, chatIds, lastUpdate); - }).ToList(); + public void UpdateLastCheckTime(string url, DateTimeOffset lastUpdate) + { + _linkMetadata[url] = lastUpdate; } - - public void UpdateLastCheckTime(string url, DateTimeOffset lastUpdate) => _linkMetadata[url] = lastUpdate; } \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/appsettings.json b/src/LinkTracker.Scrapper/appsettings.json index 10f68b8..63e7dc9 100644 --- a/src/LinkTracker.Scrapper/appsettings.json +++ b/src/LinkTracker.Scrapper/appsettings.json @@ -5,5 +5,10 @@ "Microsoft.AspNetCore": "Warning" } }, + "Database": { + "AccessType": "SQL", + "ConnectionString": "Host=localhost;Port=5433;Database=linktracker;Username=linktracker;Password=linktracker", + "RunMigrations": true + }, "AllowedHosts": "*" -} +} \ No newline at end of file diff --git a/src/LinkTracker.Shared/Models.cs b/src/LinkTracker.Shared/Models.cs index 0b57563..97104b3 100644 --- a/src/LinkTracker.Shared/Models.cs +++ b/src/LinkTracker.Shared/Models.cs @@ -4,4 +4,7 @@ public record AddLinkRequest(string Link, string[]? Tags = null); public record RemoveLinkRequest(string Link); public record LinkResponse(long Id, string Url, string[] Tags); public record LinkUpdate(long Id, string Url, string Description, long[] TgChatIds); -public record ListLinksResponse(LinkResponse[] Links, int Size); \ No newline at end of file +public record ListLinksResponse(LinkResponse[] Links, int Size); +public record CreateTagRequest(string Name); +public record UpdateTagRequest(string Name); +public record TagResponse(long Id, string Name); \ No newline at end of file