diff --git a/scripts/backfill_reviews.rs b/scripts/backfill_reviews.rs index 722c1c3..c811dfc 100644 --- a/scripts/backfill_reviews.rs +++ b/scripts/backfill_reviews.rs @@ -33,7 +33,6 @@ async fn main() { let config = load_config(); let discord = DiscordAPI::default().await; - let channel_ids = [config.teatro_channel_id, config.artes_channel_id]; let total = records.len(); async { @@ -55,11 +54,18 @@ async fn main() { let vote_emoji = &config.voting_emojis[(record.rating - 1) as usize]; match discord - .send_backfill_review(user_id, &record.url, vote_emoji, comment, &channel_ids) + .send_backfill_review( + user_id, + &record.url, + vote_emoji, + comment, + &config.venue_ticket_shop_url, + &config.ticket_shop_icon_url, + ) .await { Ok(true) => info!("Sent review for {}", record.url), - Ok(false) => warn!("Skipped (already sent or event not found): {}", record.url), + Ok(false) => warn!("Skipped (already sent): {}", record.url), Err(_) => error!("Failed to send review for {}", record.url), } } diff --git a/src/agenda_cultural/api.rs b/src/agenda_cultural/api.rs index 4a5e620..80b6c89 100644 --- a/src/agenda_cultural/api.rs +++ b/src/agenda_cultural/api.rs @@ -1,5 +1,5 @@ use super::{dto::EventResponse, model::Event}; -use crate::agenda_cultural::model::Category; +use crate::agenda_cultural::model::{Category, EventDetails, Schedule}; use chrono::{Datelike, NaiveDate, TimeDelta, Utc}; use futures::TryFutureExt; use lazy_static::lazy_static; @@ -32,6 +32,10 @@ lazy_static! { .build(); static ref EVENT_DESCRIPTION_SELECTOR: Selector = Selector::parse(".entry-container > :not(.event__extra-info):not(.section-title):not(.section-title--venue):not(.venue):not(.post__share)").unwrap(); + static ref OG_TITLE_SELECTOR: Selector = Selector::parse(r#"meta[property="og:title"]"#).unwrap(); + static ref OG_IMAGE_SELECTOR: Selector = Selector::parse(r#"meta[property="og:image"]"#).unwrap(); + static ref VENUE_NAME_SELECTOR: Selector = Selector::parse(".venue__name").unwrap(); + static ref EVENT_DATES_SELECTOR: Selector = Selector::parse(".signpost__date").unwrap(); } pub struct AgendaCulturalAPI; @@ -215,9 +219,79 @@ impl AgendaCulturalAPI { fn clean_description(description: &str) -> String { strip_tags(description) .replace(" ", " ") - .trim_end_matches("\n") + .trim() .to_owned() } + + /// Scrapes title, venue, dates and image directly off the event page, for events + /// no longer present in the upcoming-events API (e.g. when backfilling old reviews). + pub async fn scrape_event(link: &str) -> Option { + let full_page: Result = REST_CLIENT.get(link).send().await; + + let body = match full_page { + Ok(full_page) => full_page + .text() + .await + .inspect_err(|err| warn!("Failed to get event page text: {}", err)) + .ok()?, + Err(err) => { + warn!("Failed to get event page: {:?}", err); + return None; + } + }; + + let description = Self::extract_full_description(&body).unwrap_or_else(|| { + warn!("Unable to extract description for '{}'", link); + String::new() + }); + + let document = Html::parse_document(&body); + + let title = Self::extract_meta_content(&document, &OG_TITLE_SELECTOR).unwrap_or_else(|| { + warn!("Unable to extract title for '{}'", link); + String::new() + }); + let image_url = Self::extract_meta_content(&document, &OG_IMAGE_SELECTOR) + .unwrap_or_else(|| { + warn!("Unable to extract image for '{}'", link); + String::new() + }); + let venue = Self::extract_text(&document, &VENUE_NAME_SELECTOR).unwrap_or_else(|| { + warn!("Unable to extract venue for '{}'", link); + String::new() + }); + let dates = Self::extract_text(&document, &EVENT_DATES_SELECTOR).unwrap_or_else(|| { + warn!("Unable to extract dates for '{}'", link); + String::new() + }); + + Some(Event::new( + title, + EventDetails::new(String::new(), description, image_url), + link.to_string(), + Schedule::new(dates, String::new()), + venue, + Vec::new(), + )) + } + + fn extract_meta_content(document: &Html, selector: &Selector) -> Option { + document + .select(selector) + .next() + .and_then(|element| element.value().attr("content")) + .map(str::to_string) + } + + fn extract_text(document: &Html, selector: &Selector) -> Option { + document.select(selector).next().map(|element| { + element + .text() + .collect::() + .trim() + .to_string() + }) + } } #[cfg(test)] diff --git a/src/discord/api.rs b/src/discord/api.rs index 4929076..fe1b0d4 100644 --- a/src/discord/api.rs +++ b/src/discord/api.rs @@ -1,3 +1,4 @@ +use crate::agenda_cultural::api::AgendaCulturalAPI; use crate::agenda_cultural::model::Event; use crate::config::model::EmojiConfig; use crate::metrics::{record_dm_review_rewrite, record_dm_review_sent, MetricResult}; @@ -17,6 +18,7 @@ use serenity::cache::Settings; use serenity::model::id::ChannelId; use serenity::prelude::SerenityError; use serenity::Client; +use std::collections::HashMap; use std::env; use std::fmt::Debug; use tracing::field::debug; @@ -104,6 +106,25 @@ impl DiscordAPI { ) -> Result { info!(channel_id = %channel_id, event = %event.title, "Sending event"); + let title = event.title.clone(); + let embed = Self::build_event_embed(event, ticket_shop_url, ticket_shop_icon_url); + + let message_builder = CreateMessage::new().add_embed(embed.clone()); + + channel_id + .send_message(&self.client.http, message_builder) + .await + .map_err(|err| { + error!("Failed sending event '{}' due to '{}'", title, err); + DiscordError::Api + }) + } + + fn build_event_embed( + event: Event, + ticket_shop_url: Option, + ticket_shop_icon_url: &str, + ) -> CreateEmbed { let mut description = event.details.description; if event.is_for_children { @@ -117,26 +138,15 @@ impl DiscordAPI { } let embed_description = Self::truncate_embed_description(description); - let embed_title = event.title.clone(); - let embed = CreateEmbed::new() - .title(embed_title) + CreateEmbed::new() + .title(event.title) .url(event.link) .description(embed_description) .author(author) .color(Colour::new(0x005eeb)) .field("Datas", event.occurring_at.dates, true) - .image(event.details.image_url); - - let message_builder = CreateMessage::new().add_embed(embed.clone()); - - channel_id - .send_message(&self.client.http, message_builder) - .await - .map_err(|err| { - error!("Failed sending event '{}' due to '{}'", event.title, err); - DiscordError::Api - }) + .image(event.details.image_url) } pub async fn add_custom_reaction(&self, message: &Message, emoji: &EmojiConfig) { @@ -575,32 +585,22 @@ impl DiscordAPI { } } - /// Sends a review DM identical in format to `send_user_review_in_dm`, but with an - /// explicit comment string instead of reading the user's last DM message. - /// Used to backfill historical reviews that predate (or bypassed) the normal - /// reaction -> DM flow. pub async fn send_backfill_review( &self, user_id: UserId, event_url: &str, vote_emoji: &EmojiConfig, comment: Option<&str>, - channel_ids: &[ChannelId], + venue_ticket_shop_url: &HashMap, + ticket_shop_icon_url: &str, ) -> Result { - let mut event_embed = None; - - for channel_id in channel_ids { - if let Some(found) = self.find_event_embed(*channel_id, event_url).await { - event_embed = Some(found); - break; - } - } - - let Some(event_embed) = event_embed else { - warn!("Could not find event message for url '{}'", event_url); + let Some(event) = AgendaCulturalAPI::scrape_event(event_url).await else { + warn!("Could not scrape event details for '{}'", event_url); return Ok(false); }; + let ticket_shop_url = venue_ticket_shop_url.get(&event.venue).cloned(); + let dm = match user_id.create_dm_channel(&self.client.http).await { Ok(dm) => dm, Err(e) => { @@ -618,7 +618,13 @@ impl DiscordAPI { Ok(false) => {} } - let embed = Self::create_user_review_embed(vote_emoji, event_embed, comment); + let mut embed = + Self::build_event_embed(event, ticket_shop_url, ticket_shop_icon_url) + .field("Voto", vote_emoji.to_string(), true); + + if let Some(comment) = comment { + embed = embed.field("Comentários", comment, true); + } match dm .send_message(&self.client.http, CreateMessage::new().embed(embed)) @@ -651,9 +657,10 @@ impl DiscordAPI { Err(_) => return 0, }; - // Snowflake IDs are chronologically ordered; sort ascending so the latest - // reply to any given review embed is always processed last. - messages.sort_by_key(|message| message.id); + // fetch_all_dm_messages returns newest-to-oldest with no gaps or overlaps; + // reverse to oldest-first so the latest reply to any given review embed is + // always processed last. + messages.reverse(); let mut rewritten_count = 0; @@ -711,9 +718,9 @@ impl DiscordAPI { e })?; - match page.first() { + match page.last() { None => break, - Some(message) => last_message_id = Some(message.id), + Some(oldest_in_page) => last_message_id = Some(oldest_in_page.id), } all_messages.extend(page); @@ -782,23 +789,6 @@ impl DiscordAPI { } } - async fn find_event_embed(&self, channel_id: ChannelId, event_url: &str) -> Option { - let guild = self.get_guild(channel_id).await; - let threads = self.get_channel_threads(&guild, channel_id).await; - - for thread in threads { - let messages = self.get_all_messages(thread.id).await; - - if let Some(message) = messages.into_iter().find(|m| { - m.embeds.first().and_then(|e| e.url.clone()).as_deref() == Some(event_url) - }) { - return message.embeds.into_iter().next(); - } - } - - None - } - async fn get_user_last_comment(&self, dm: &PrivateChannel) -> Option { match dm.last_message_id { Some(last_message_id) => { @@ -859,11 +849,11 @@ impl DiscordAPI { return Ok(true); } - match messages_iter.first() { + match messages_iter.last() { None => { searched_all_dms = true; } - Some(last_message) => last_message_id = Some(last_message.id), + Some(oldest_in_page) => last_message_id = Some(oldest_in_page.id), } } diff --git a/tests/agenda_cultural_api_tests.rs b/tests/agenda_cultural_api_tests.rs index 3559f51..11db9f8 100644 --- a/tests/agenda_cultural_api_tests.rs +++ b/tests/agenda_cultural_api_tests.rs @@ -25,4 +25,18 @@ mod agenda_cultural { assert_eq!(res.len(), 2); } + + #[test_log::test(tokio::test)] + async fn should_scrape_single_event_page() { + let event = AgendaCulturalAPI::scrape_event("https://www.agendalx.pt/events/event/maes/") + .await + .expect("Failed to scrape event"); + + assert_eq!(event.title, "Mães"); + assert_eq!(event.link, "https://www.agendalx.pt/events/event/maes/"); + assert_eq!(event.venue, "Teatro Villaret"); + assert_eq!(event.occurring_at.dates, "14 março a 30 junho 2024"); + assert_eq!(event.details.description, "Três mães e uma grávida juntas num musical hilariante e ternurento onde ficamos a conhecer a poderosa amizade de quatro mulheres…"); + assert_eq!(event.details.image_url, "https://www.agendalx.pt/content/uploads/2024/02/Maes.jpg"); + } }