//! F-Droid mod index; use atrium_api::app::bsky::richtext::facet::{ByteSliceData, LinkData, MainData, MainFeaturesItem}; use atrium_api::types::{Union, string::Language}; use reqwest::{ StatusCode, header::{ETAG, IF_NONE_MATCH}, }; use serde::{Deserialize, Serialize}; use unicode_segmentation::UnicodeSegmentation; use self::index::{AntiFeature, EN_LOCALE, Index, PHONE_DEVICE_TYPE, Package, Screenshot}; use crate::{ MAX_IMAGE_SIZE, MAX_POST_GRAPHEMES, MAX_POST_IMAGES, http_client, report::{Post, PostImage, Report}, }; const REPOSITORY_URL: &str = "https://f-droid.org/repo"; const PACKAGE_URL_PREFIX: &str = "https://f-droid.org/packages/"; const ANTI_FEATURES_EXCLUDING_APPS: &[AntiFeature] = &[AntiFeature::KnownVuln, AntiFeature::NoSourceSince]; #[derive(Debug, Default, Deserialize, Serialize)] pub struct BreakingState { pub last_repo_timestamp: Option, pub index_etag: Option, } pub struct Breaking; impl Report for Breaking { type State = BreakingState; const STATE_FILE: &'static str = "fdroid-breaking.json"; async fn generate(state: Self::State) -> Result<(Vec, Self::State), String> { let mut request = http_client().get(format!("{REPOSITORY_URL}/index-v2.json")); if let Some(etag) = state.index_etag.as_deref() { request = request.header(IF_NONE_MATCH, etag); } let response = request .send() .await .map_err(|error| format!("failed to fetch F-Droid index: {error}"))?; if response.status() == StatusCode::NOT_MODIFIED { return Ok((Vec::new(), state)); } let response = response .error_for_status() .map_err(|error| format!("failed to fetch F-Droid index: {error}"))?; let index_etag = response .headers() .get(ETAG) .and_then(|value| value.to_str().ok()) .map(str::to_owned); let index = response .json::() .await .map_err(|error| format!("failed to parse F-Droid index: {error}"))?; let Some(timestamp) = state.last_repo_timestamp else { // We don't know the timestamp of the last index version. tracing::info!( timestamp = index.repo.timestamp, "recording initial repository timestamp; will announce new apps with the next index version" ); let new_state = BreakingState { last_repo_timestamp: Some(index.repo.timestamp), index_etag, }; return Ok((Vec::new(), new_state)); }; if index.repo.timestamp < timestamp { let error = format!( "F-Droid repository timestamp moved backwards from {timestamp} to {}", index.repo.timestamp ); tracing::error!("{error}"); return Err(error); } let mut apps: Vec<_> = index .packages .iter() .filter_map(|(id, package)| { (package.metadata.added > timestamp) .then(|| App::from_package(id.as_str(), package)) }) .collect(); tracing::info!( timestamp = index.repo.timestamp, new_apps = apps.len(), total_apps = index.packages.len(), "building posts" ); apps.sort_unstable_by(|left, right| { left.name .to_lowercase() .cmp(&right.name.to_lowercase()) .then_with(|| left.id.cmp(right.id)) }); Ok(( download_images(build_announcement_posts(&apps)).await?, BreakingState { last_repo_timestamp: Some(index.repo.timestamp), index_etag, }, )) } } struct App<'a> { id: &'a str, name: &'a str, summary: &'a str, screenshot: Option<&'a Screenshot>, anti_features: Vec, } impl<'a> App<'a> { fn from_package(id: &'a str, package: &'a Package) -> Self { Self { id, name: package .metadata .name .get(EN_LOCALE) .map_or(id, String::as_str), summary: package .metadata .summary .get(EN_LOCALE) .map_or("", String::as_str), screenshot: package .metadata .screenshots .get(PHONE_DEVICE_TYPE) .and_then(|screenshots| screenshots.get(EN_LOCALE)) .and_then(|screenshots| { screenshots .iter() // We could resize images but realistically they should fit. .find(|screenshot| screenshot.size < MAX_IMAGE_SIZE) }), anti_features: package .versions .values() .flat_map(|version| version.anti_features.keys()) // "Tethered network service" means an app depends on a network service, // which isn't easy to self-host. There's a lot of such network services out there // be it OpenStreetMap or a public transit API ... not much apps can do about that. // So we just disregard this anti-feature in this bot. .filter(|&anti_feature| anti_feature != &AntiFeature::TetheredNet) .cloned() .collect(), } } } fn build_announcement_posts(apps: &[App<'_>]) -> Vec { let apps: Vec<_> = apps .iter() .filter(|app| { let Some(anti_feature) = ANTI_FEATURES_EXCLUDING_APPS .iter() .find(|anti_feature| app.anti_features.contains(anti_feature)) else { return true; }; tracing::info!( app_id = app.id, app_name = app.name, "omitting F-Droid app due to anti-feature {anti_feature:?}" ); false }) .collect(); if apps.is_empty() { return Vec::new(); } let mut builders = vec![PostBuilder::new(format!( "{} new F-Droid apps are available:", apps.len() ))]; let (without_anti_features, with_anti_features): (Vec<_>, Vec<_>) = apps .into_iter() .partition(|app| app.anti_features.is_empty()); let (with_screenshots, without_screenshots): (Vec<_>, Vec<_>) = without_anti_features .into_iter() .partition(|app| app.screenshot.is_some()); for app in with_screenshots { let name = truncate(app.name, MAX_POST_GRAPHEMES); let screenshot = app.screenshot.unwrap(); let image = PendingImage { url: format!("{REPOSITORY_URL}{}", screenshot.name), // Using the summary for the alt text isn't great // but the Bluesky lexicon provides no alternative. alt: format!("{}: {}", app.name, app.summary), }; let post = builders.last().unwrap(); let separator = if post.text.is_empty() { "" } else if post.images.is_empty() { "\n" } else { ", " }; if !post.can_fit(separator, &name) || post.images.len() == MAX_POST_IMAGES { builders.push(PostBuilder::new(String::new())); } let post = builders.last_mut().unwrap(); let separator = if post.text.is_empty() { "" } else if post.images.is_empty() { "\n" } else { ", " }; post.push_app(separator, &name, &name, app.id, Some(image)); } for app in without_screenshots { push_bullet_app(&mut builders, app); } if let Some((first, rest)) = with_anti_features.split_first() { const ANTI_FEATURES_HEADING: &str = "New apps with an anti-feature (other than tethered network):"; let line_limit = MAX_POST_GRAPHEMES - grapheme_count(ANTI_FEATURES_HEADING) - grapheme_count("\n"); let (name, line) = app_line(first, line_limit); // Keep anti-feature apps in their own section, starting at a post boundary. builders.push(PostBuilder::new(ANTI_FEATURES_HEADING.to_owned())); builders .last_mut() .unwrap() .push_app("\n", &line, &name, first.id, None); for app in rest { push_bullet_app(&mut builders, app); } } builders.into_iter().map(PostBuilder::finish).collect() } fn push_bullet_app(builders: &mut Vec, app: &App<'_>) { let (name, line) = app_line(app, MAX_POST_GRAPHEMES); let post = builders.last().unwrap(); let separator = if post.text.is_empty() { "" } else { "\n" }; if !post.can_fit(separator, &line) { builders.push(PostBuilder::new(String::new())); } let post = builders.last_mut().unwrap(); let separator = if post.text.is_empty() { "" } else { "\n" }; post.push_app(separator, &line, &name, app.id, None); } /// Build a bullet that fits within `limit`, preserving as much of the summary as possible. fn app_line(app: &App<'_>, limit: usize) -> (String, String) { let prefix = "* "; let anti_features = anti_feature_annotation(&app.anti_features); let suffix = ": "; let fixed = grapheme_count(prefix) + grapheme_count(app.name) + grapheme_count(&anti_features) + grapheme_count(suffix); if fixed <= limit { let summary = truncate(app.summary, limit - fixed); ( app.name.to_owned(), format!("{prefix}{}{anti_features}{suffix}{summary}", app.name), ) } else { let reserved = grapheme_count(prefix) + grapheme_count(&anti_features); let name = truncate(app.name, limit.saturating_sub(reserved)); let line = format!("{prefix}{name}{anti_features}"); (name, line) } } fn anti_feature_annotation(anti_features: &[AntiFeature]) -> String { if anti_features.is_empty() { return String::new(); } let mut icons: Vec<_> = anti_features .iter() .map(|anti_feature| match anti_feature { AntiFeature::Tracking => (0, "๐Ÿ‘๏ธ"), AntiFeature::Ads => (1, "๐Ÿชง"), AntiFeature::NonFreeNet => (2, "โ˜๏ธ"), AntiFeature::NonFreeDep => (3, "๐Ÿ“ฆ"), AntiFeature::NonFreeAssets => (4, "๐Ÿ–ผ๏ธ"), AntiFeature::NonFreeAdd => (5, "๐Ÿงฉ"), AntiFeature::DisabledAlgorithm => (6, "๐Ÿ”“"), // shouldn't happen for new apps AntiFeature::Other(_) => (7, "โ“"), AntiFeature::KnownVuln | AntiFeature::NoSourceSince | AntiFeature::TetheredNet => { unreachable!() } }) .collect(); icons.sort_unstable_by_key(|(order, _)| *order); icons.dedup_by_key(|(order, _)| *order); format!( " ({})", icons .into_iter() .map(|(_, icon)| icon) .collect::>() .join("") ) } fn truncate(text: &str, limit: usize) -> String { let length = grapheme_count(text); if length <= limit { return text.to_owned(); } if limit == 0 { return String::new(); } let mut shortened: String = text.graphemes(true).take(limit - 1).collect(); shortened.push('โ€ฆ'); shortened } fn grapheme_count(text: &str) -> usize { text.graphemes(true).count() } struct PendingPost { text: String, facets: Vec, images: Vec, } struct PendingImage { url: String, alt: String, } struct PostBuilder { text: String, graphemes: usize, facets: Vec, images: Vec, } impl PostBuilder { fn new(text: String) -> Self { let graphemes = grapheme_count(&text); Self { text, graphemes, facets: Vec::new(), images: Vec::new(), } } fn can_fit(&self, separator: &str, text: &str) -> bool { self.graphemes + grapheme_count(separator) + grapheme_count(text) <= MAX_POST_GRAPHEMES } fn push_app( &mut self, separator: &str, line: &str, name: &str, id: &str, image: Option, ) { self.text.push_str(separator); let byte_start = self.text.len() + line.find(name).unwrap_or(0); self.text.push_str(line); let byte_end = byte_start + name.len(); self.graphemes += grapheme_count(separator) + grapheme_count(line); self.facets.push( MainData { features: vec![Union::Refs(MainFeaturesItem::Link(Box::new( LinkData { uri: format!("{PACKAGE_URL_PREFIX}{id}/"), } .into(), )))], index: ByteSliceData { byte_start, byte_end, } .into(), } .into(), ); if let Some(image) = image { self.images.push(image); } } fn finish(self) -> PendingPost { PendingPost { text: self.text, facets: self.facets, images: self.images, } } } async fn download_images(pending_posts: Vec) -> Result, String> { let client = http_client(); let english: Language = "en".parse().unwrap(); let mut posts = Vec::with_capacity(pending_posts.len()); for (post_index, post) in pending_posts.into_iter().enumerate() { let mut images = Vec::with_capacity(post.images.len()); for image in post.images { let data = client .get(&image.url) .send() .await .and_then(reqwest::Response::error_for_status) .map_err(|error| { format!( "failed to download image {} for post {}: {error}", image.url, post_index + 1 ) })? .bytes() .await .map_err(|error| { format!( "failed to download image {} for post {}: {error}", image.url, post_index + 1 ) })?; images.push(PostImage { data: data.to_vec(), alt: image.alt, }); } posts.push(Post { text: post.text, facets: post.facets, images, langs: vec![english.clone()], tags: vec!["F-Droid".to_owned()], }); } Ok(posts) } #[cfg(test)] mod tests { use super::*; #[test] fn builds_announcement_posts() { assert!(build_announcement_posts(&[]).is_empty()); let screenshot = Screenshot { name: "/screenshots/camera.png".to_owned(), size: 123, }; let apps = [ App { id: "org.example.camera", name: "Camera ๐Ÿ“ท", summary: "Take photos", screenshot: Some(&screenshot), anti_features: Vec::new(), }, App { id: "org.example.notes", name: "Notes", summary: "Take notes", screenshot: None, anti_features: Vec::new(), }, App { id: "org.example.tracker", name: "Tracker", summary: "Tracks users", screenshot: None, anti_features: vec![ AntiFeature::Other("UnknownFeature".to_owned()), AntiFeature::Tracking, AntiFeature::NonFreeNet, AntiFeature::NonFreeDep, AntiFeature::NonFreeAssets, AntiFeature::NonFreeAdd, AntiFeature::DisabledAlgorithm, AntiFeature::Ads, ], }, App { id: "org.example.vulnerable", name: "Vulnerable", summary: "Has a known vulnerability", screenshot: None, anti_features: vec![AntiFeature::KnownVuln], }, App { id: "org.example.abandoned", name: "Abandoned", summary: "No longer has source available", screenshot: None, anti_features: vec![AntiFeature::NoSourceSince], }, ]; let posts = build_announcement_posts(&apps); assert_eq!(posts.len(), 2); assert_eq!( posts[0].text, "3 new F-Droid apps are available:\n\ Camera ๐Ÿ“ท\n\ * Notes: Take notes" ); assert_eq!( posts[1].text, "New apps with an anti-feature (other than tethered network):\n\ * Tracker (๐Ÿ‘๏ธ๐Ÿชงโ˜๏ธ๐Ÿ“ฆ๐Ÿ–ผ๏ธ๐Ÿงฉ๐Ÿ”“โ“): Tracks users" ); assert_eq!(posts[0].images.len(), 1); assert_eq!( posts[0].images[0].url, "https://f-droid.org/repo/screenshots/camera.png" ); assert_eq!(posts[0].images[0].alt, "Camera ๐Ÿ“ท: Take photos"); assert!(posts[1].images.is_empty()); let expected_links = [ (&posts[0], "Camera ๐Ÿ“ท", "org.example.camera"), (&posts[0], "Notes", "org.example.notes"), (&posts[1], "Tracker", "org.example.tracker"), ]; for (post, name, id) in expected_links { let facet = post .facets .iter() .find(|facet| &post.text[facet.index.byte_start..facet.index.byte_end] == name) .unwrap(); match facet.features.as_slice() { [Union::Refs(MainFeaturesItem::Link(link))] => { assert_eq!(link.uri, format!("{PACKAGE_URL_PREFIX}{id}/")); } features => panic!("expected one link feature, got {features:?}"), } } } }