From 41212a7a15198e53d205252a02069a7dd6ddadeb Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Sun, 25 Oct 2026 09:08:01 +0100 Subject: initial commit --- src/bots/fdroid/index.rs | 97 ++++++++ src/bots/fdroid/mod.rs | 572 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 669 insertions(+) create mode 100644 src/bots/fdroid/index.rs create mode 100644 src/bots/fdroid/mod.rs (limited to 'src/bots/fdroid') diff --git a/src/bots/fdroid/index.rs b/src/bots/fdroid/index.rs new file mode 100644 index 0000000..61cc44d --- /dev/null +++ b/src/bots/fdroid/index.rs @@ -0,0 +1,97 @@ +//! Types for the F-Droid repository index. + +use std::collections::HashMap; + +use serde::{Deserialize, Deserializer}; + +pub(super) const PHONE_DEVICE_TYPE: &str = "phone"; +pub(super) const EN_LOCALE: &str = "en-US"; + +#[derive(Debug, Deserialize)] +pub(super) struct Index { + pub(super) repo: Repo, + pub(super) packages: HashMap, +} + +#[derive(Debug, Deserialize)] +pub(super) struct Repo { + pub(super) timestamp: u64, +} + +#[derive(Debug, Deserialize)] +pub(super) struct Package { + pub(super) metadata: Metadata, + /// Releases keyed by the APK's SHA-256 digest. + pub(super) versions: HashMap, +} + +#[derive(Debug, Deserialize)] +pub(super) struct Version { + /// Anti-feature explanations keyed first by anti-feature name and then by locale. + #[serde(default, rename = "antiFeatures")] + pub(super) anti_features: HashMap>, +} + +/// An [anti-feature](https://f-droid.org/docs/Anti-Features/). +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(super) enum AntiFeature { + Ads, + DisabledAlgorithm, + KnownVuln, + NoSourceSince, + NonFreeAdd, + NonFreeAssets, + NonFreeDep, + NonFreeNet, + TetheredNet, + Tracking, + Other(String), +} + +impl<'de> Deserialize<'de> for AntiFeature { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let name = String::deserialize(deserializer)?; + Ok(match name.as_str() { + "Ads" => Self::Ads, + "DisabledAlgorithm" => Self::DisabledAlgorithm, + "KnownVuln" => Self::KnownVuln, + "NoSourceSince" => Self::NoSourceSince, + "NonFreeAdd" => Self::NonFreeAdd, + "NonFreeAssets" => Self::NonFreeAssets, + "NonFreeDep" => Self::NonFreeDep, + "NonFreeNet" => Self::NonFreeNet, + "TetheredNet" => Self::TetheredNet, + "Tracking" => Self::Tracking, + _ => { + tracing::warn!("observed unknown F-Droid anti-feature {name}"); + Self::Other(name) + } + }) + } +} + +#[derive(Debug, Deserialize)] +pub(super) struct Metadata { + pub(super) added: u64, + /// App names keyed by locale. Every package has an `en-US` name. + #[serde(default)] + pub(super) name: HashMap, + /// Short app descriptions keyed by locale. + #[serde(default)] + pub(super) summary: HashMap, + /// Screenshots keyed first by device type and then by locale. + #[serde(default)] + pub(super) screenshots: HashMap>>, +} + +#[derive(Debug, Deserialize)] +pub(super) struct Screenshot { + /// Path to the screenshot relative to the repository address. + pub(super) name: String, + /// Screenshot file size in bytes, retained for enforcing upload limits. + #[allow(dead_code)] + pub(super) size: u64, +} diff --git a/src/bots/fdroid/mod.rs b/src/bots/fdroid/mod.rs new file mode 100644 index 0000000..c0e7198 --- /dev/null +++ b/src/bots/fdroid/mod.rs @@ -0,0 +1,572 @@ +//! F-Droid + +mod index; + +use atrium_api::app::bsky::richtext::facet::{ByteSliceData, LinkData, MainData, MainFeaturesItem}; +use atrium_api::types::Union; +use reqwest::{ + StatusCode, + header::{ETAG, IF_NONE_MATCH}, +}; +use serde::{Deserialize, Serialize}; +use unicode_segmentation::UnicodeSegmentation; + +use self::index::{AntiFeature, EN_LOCALE, Index, PHONE_DEVICE_TYPE, Package, Screenshot}; +use crate::{ + MAX_IMAGE_SIZE, MAX_POST_GRAPHEMES, MAX_POST_IMAGES, http_client, + report::{Post, PostImage, Report}, +}; + +const REPOSITORY_URL: &str = "https://f-droid.org/repo"; +const PACKAGE_URL_PREFIX: &str = "https://f-droid.org/packages/"; +const ANTI_FEATURES_EXCLUDING_APPS: &[AntiFeature] = + &[AntiFeature::KnownVuln, AntiFeature::NoSourceSince]; + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct BreakingState { + pub last_repo_timestamp: Option, + pub index_etag: Option, +} + +pub struct Breaking; + +impl Report for Breaking { + type State = BreakingState; + + const STATE_FILE: &'static str = "fdroid-breaking.json"; + + async fn generate(state: Self::State) -> Result<(Vec, Self::State), String> { + let mut request = http_client().get(format!("{REPOSITORY_URL}/index-v2.json")); + if let Some(etag) = state.index_etag.as_deref() { + request = request.header(IF_NONE_MATCH, etag); + } + + let response = request + .send() + .await + .map_err(|error| format!("failed to fetch F-Droid index: {error}"))?; + if response.status() == StatusCode::NOT_MODIFIED { + return Ok((Vec::new(), state)); + } + + let response = response + .error_for_status() + .map_err(|error| format!("failed to fetch F-Droid index: {error}"))?; + let index_etag = response + .headers() + .get(ETAG) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let index = response + .json::() + .await + .map_err(|error| format!("failed to parse F-Droid index: {error}"))?; + + let Some(timestamp) = state.last_repo_timestamp else { + // We don't know the timestamp of the last index version. + tracing::info!( + timestamp = index.repo.timestamp, + "recording initial repository timestamp; will announce new apps with the next index version" + ); + let new_state = BreakingState { + last_repo_timestamp: Some(index.repo.timestamp), + index_etag, + }; + return Ok((Vec::new(), new_state)); + }; + + if index.repo.timestamp < timestamp { + let error = format!( + "F-Droid repository timestamp moved backwards from {timestamp} to {}", + index.repo.timestamp + ); + tracing::error!("{error}"); + return Err(error); + } + + let mut apps: Vec<_> = index + .packages + .iter() + .filter_map(|(id, package)| { + (package.metadata.added > timestamp) + .then(|| App::from_package(id.as_str(), package)) + }) + .collect(); + + tracing::info!( + timestamp = index.repo.timestamp, + new_apps = apps.len(), + total_apps = index.packages.len(), + "building posts" + ); + + apps.sort_unstable_by(|left, right| { + left.name + .to_lowercase() + .cmp(&right.name.to_lowercase()) + .then_with(|| left.id.cmp(right.id)) + }); + + Ok(( + download_images(build_announcement_posts(&apps)).await?, + BreakingState { + last_repo_timestamp: Some(index.repo.timestamp), + index_etag, + }, + )) + } +} + +struct App<'a> { + id: &'a str, + name: &'a str, + summary: &'a str, + screenshot: Option<&'a Screenshot>, + anti_features: Vec, +} + +impl<'a> App<'a> { + fn from_package(id: &'a str, package: &'a Package) -> Self { + Self { + id, + name: package + .metadata + .name + .get(EN_LOCALE) + .map_or(id, String::as_str), + summary: package + .metadata + .summary + .get(EN_LOCALE) + .map_or("", String::as_str), + screenshot: package + .metadata + .screenshots + .get(PHONE_DEVICE_TYPE) + .and_then(|screenshots| screenshots.get(EN_LOCALE)) + .and_then(|screenshots| { + screenshots + .iter() + // We could resize images but realistically they should fit. + .find(|screenshot| screenshot.size < MAX_IMAGE_SIZE) + }), + anti_features: package + .versions + .values() + .flat_map(|version| version.anti_features.keys()) + // "Tethered network service" means an app depends on a network service, + // which isn't easy to self-host. There's a lot of such network services out there + // be it OpenStreetMap or a public transit API ... not much apps can do about that. + // So we just disregard this anti-feature in this bot. + .filter(|&anti_feature| anti_feature != &AntiFeature::TetheredNet) + .cloned() + .collect(), + } + } +} + +fn build_announcement_posts(apps: &[App<'_>]) -> Vec { + let apps: Vec<_> = apps + .iter() + .filter(|app| { + let Some(anti_feature) = ANTI_FEATURES_EXCLUDING_APPS + .iter() + .find(|anti_feature| app.anti_features.contains(anti_feature)) + else { + return true; + }; + + tracing::info!( + app_id = app.id, + app_name = app.name, + "omitting F-Droid app due to anti-feature {anti_feature:?}" + ); + false + }) + .collect(); + + if apps.is_empty() { + return Vec::new(); + } + + let mut builders = vec![PostBuilder::new(format!( + "{} new F-Droid packages are available:", + apps.len() + ))]; + + let (without_anti_features, with_anti_features): (Vec<_>, Vec<_>) = apps + .into_iter() + .partition(|app| app.anti_features.is_empty()); + let (with_screenshots, without_screenshots): (Vec<_>, Vec<_>) = without_anti_features + .into_iter() + .partition(|app| app.screenshot.is_some()); + + for app in with_screenshots { + let name = truncate(app.name, MAX_POST_GRAPHEMES); + let screenshot = app.screenshot.unwrap(); + let image = PendingImage { + url: format!("{REPOSITORY_URL}{}", screenshot.name), + // Using the summary for the alt text isn't great + // but the Bluesky lexicon provides no alternative. + alt: format!("{}: {}", app.name, app.summary), + }; + + let post = builders.last().unwrap(); + let separator = if post.text.is_empty() { + "" + } else if post.images.is_empty() { + "\n" + } else { + ", " + }; + if !post.can_fit(separator, &name) || post.images.len() == MAX_POST_IMAGES { + builders.push(PostBuilder::new(String::new())); + } + + let post = builders.last_mut().unwrap(); + let separator = if post.text.is_empty() { + "" + } else if post.images.is_empty() { + "\n" + } else { + ", " + }; + post.push_app(separator, &name, &name, app.id, Some(image)); + } + + for app in without_screenshots { + push_bullet_app(&mut builders, app); + } + + if let Some((first, rest)) = with_anti_features.split_first() { + const ANTI_FEATURES_HEADING: &str = + "New apps with an anti-feature (other than tethered network):"; + let line_limit = + MAX_POST_GRAPHEMES - grapheme_count(ANTI_FEATURES_HEADING) - grapheme_count("\n"); + let (name, line) = app_line(first, line_limit); + + // Keep anti-feature apps in their own section, starting at a post boundary. + builders.push(PostBuilder::new(ANTI_FEATURES_HEADING.to_owned())); + builders + .last_mut() + .unwrap() + .push_app("\n", &line, &name, first.id, None); + + for app in rest { + push_bullet_app(&mut builders, app); + } + } + + builders.into_iter().map(PostBuilder::finish).collect() +} + +fn push_bullet_app(builders: &mut Vec, app: &App<'_>) { + let (name, line) = app_line(app, MAX_POST_GRAPHEMES); + let post = builders.last().unwrap(); + let separator = if post.text.is_empty() { "" } else { "\n" }; + if !post.can_fit(separator, &line) { + builders.push(PostBuilder::new(String::new())); + } + + let post = builders.last_mut().unwrap(); + let separator = if post.text.is_empty() { "" } else { "\n" }; + post.push_app(separator, &line, &name, app.id, None); +} + +/// Build a bullet that fits within `limit`, preserving as much of the summary as possible. +fn app_line(app: &App<'_>, limit: usize) -> (String, String) { + let prefix = "* "; + let anti_features = anti_feature_annotation(&app.anti_features); + let suffix = ": "; + let fixed = grapheme_count(prefix) + + grapheme_count(app.name) + + grapheme_count(&anti_features) + + grapheme_count(suffix); + + if fixed <= limit { + let summary = truncate(app.summary, limit - fixed); + ( + app.name.to_owned(), + format!("{prefix}{}{anti_features}{suffix}{summary}", app.name), + ) + } else { + let reserved = grapheme_count(prefix) + grapheme_count(&anti_features); + let name = truncate(app.name, limit.saturating_sub(reserved)); + let line = format!("{prefix}{name}{anti_features}"); + (name, line) + } +} + +fn anti_feature_annotation(anti_features: &[AntiFeature]) -> String { + if anti_features.is_empty() { + return String::new(); + } + + let mut icons: Vec<_> = anti_features + .iter() + .map(|anti_feature| match anti_feature { + AntiFeature::Tracking => (0, "๐Ÿ‘๏ธ"), + AntiFeature::Ads => (1, "๐Ÿชง"), + AntiFeature::NonFreeNet => (2, "โ˜๏ธ"), + AntiFeature::NonFreeAssets => (3, "๐Ÿ–ผ๏ธ"), + AntiFeature::NonFreeAdd => (4, "๐Ÿงฉ"), + AntiFeature::NonFreeDep => (5, "๐Ÿ“ฆ"), + AntiFeature::DisabledAlgorithm => (6, "๐Ÿ”“"), // shouldn't happen for new apps + AntiFeature::Other(_) => (7, "โ“"), + AntiFeature::KnownVuln | AntiFeature::NoSourceSince | AntiFeature::TetheredNet => { + unreachable!() + } + }) + .collect(); + icons.sort_unstable_by_key(|(order, _)| *order); + icons.dedup_by_key(|(order, _)| *order); + + format!( + " ({})", + icons + .into_iter() + .map(|(_, icon)| icon) + .collect::>() + .join("") + ) +} + +fn truncate(text: &str, limit: usize) -> String { + let length = grapheme_count(text); + if length <= limit { + return text.to_owned(); + } + if limit == 0 { + return String::new(); + } + + let mut shortened: String = text.graphemes(true).take(limit - 1).collect(); + shortened.push('โ€ฆ'); + shortened +} + +fn grapheme_count(text: &str) -> usize { + text.graphemes(true).count() +} + +struct PendingPost { + text: String, + facets: Vec, + images: Vec, +} + +struct PendingImage { + url: String, + alt: String, +} + +struct PostBuilder { + text: String, + graphemes: usize, + facets: Vec, + images: Vec, +} + +impl PostBuilder { + fn new(text: String) -> Self { + let graphemes = grapheme_count(&text); + Self { + text, + graphemes, + facets: Vec::new(), + images: Vec::new(), + } + } + + fn can_fit(&self, separator: &str, text: &str) -> bool { + self.graphemes + grapheme_count(separator) + grapheme_count(text) <= MAX_POST_GRAPHEMES + } + + fn push_app( + &mut self, + separator: &str, + line: &str, + name: &str, + id: &str, + image: Option, + ) { + self.text.push_str(separator); + let byte_start = self.text.len() + line.find(name).unwrap_or(0); + self.text.push_str(line); + let byte_end = byte_start + name.len(); + self.graphemes += grapheme_count(separator) + grapheme_count(line); + + self.facets.push( + MainData { + features: vec![Union::Refs(MainFeaturesItem::Link(Box::new( + LinkData { + uri: format!("{PACKAGE_URL_PREFIX}{id}/"), + } + .into(), + )))], + index: ByteSliceData { + byte_start, + byte_end, + } + .into(), + } + .into(), + ); + if let Some(image) = image { + self.images.push(image); + } + } + + fn finish(self) -> PendingPost { + PendingPost { + text: self.text, + facets: self.facets, + images: self.images, + } + } +} + +async fn download_images(pending_posts: Vec) -> Result, String> { + let client = http_client(); + let mut posts = Vec::with_capacity(pending_posts.len()); + + for (post_index, post) in pending_posts.into_iter().enumerate() { + let mut images = Vec::with_capacity(post.images.len()); + for image in post.images { + let data = client + .get(&image.url) + .send() + .await + .and_then(reqwest::Response::error_for_status) + .map_err(|error| { + format!( + "failed to download image {} for post {}: {error}", + image.url, + post_index + 1 + ) + })? + .bytes() + .await + .map_err(|error| { + format!( + "failed to download image {} for post {}: {error}", + image.url, + post_index + 1 + ) + })?; + images.push(PostImage { + data: data.to_vec(), + alt: image.alt, + }); + } + posts.push(Post { + text: post.text, + facets: post.facets, + images, + }); + } + + Ok(posts) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builds_announcement_posts() { + assert!(build_announcement_posts(&[]).is_empty()); + + let screenshot = Screenshot { + name: "/screenshots/camera.png".to_owned(), + size: 123, + }; + let apps = [ + App { + id: "org.example.camera", + name: "Camera ๐Ÿ“ท", + summary: "Take photos", + screenshot: Some(&screenshot), + anti_features: Vec::new(), + }, + App { + id: "org.example.notes", + name: "Notes", + summary: "Take notes", + screenshot: None, + anti_features: Vec::new(), + }, + App { + id: "org.example.tracker", + name: "Tracker", + summary: "Tracks users", + screenshot: None, + anti_features: vec![ + AntiFeature::Other("UnknownFeature".to_owned()), + AntiFeature::Tracking, + AntiFeature::NonFreeNet, + AntiFeature::NonFreeDep, + AntiFeature::NonFreeAssets, + AntiFeature::NonFreeAdd, + AntiFeature::DisabledAlgorithm, + AntiFeature::Ads, + ], + }, + App { + id: "org.example.vulnerable", + name: "Vulnerable", + summary: "Has a known vulnerability", + screenshot: None, + anti_features: vec![AntiFeature::KnownVuln], + }, + App { + id: "org.example.abandoned", + name: "Abandoned", + summary: "No longer has source available", + screenshot: None, + anti_features: vec![AntiFeature::NoSourceSince], + }, + ]; + + let posts = build_announcement_posts(&apps); + + assert_eq!(posts.len(), 2); + assert_eq!( + posts[0].text, + "3 new F-Droid packages are available:\n\ + Camera ๐Ÿ“ท\n\ + * Notes: Take notes" + ); + assert_eq!( + posts[1].text, + "New apps with an anti-feature (other than tethered network):\n\ + * Tracker (๐Ÿ‘๏ธ๐Ÿชงโ˜๏ธ๐Ÿ–ผ๏ธ๐Ÿงฉ๐Ÿ“ฆ๐Ÿ”“โ“): Tracks users" + ); + assert_eq!(posts[0].images.len(), 1); + assert_eq!( + posts[0].images[0].url, + "https://f-droid.org/repo/screenshots/camera.png" + ); + assert_eq!(posts[0].images[0].alt, "Camera ๐Ÿ“ท: Take photos"); + assert!(posts[1].images.is_empty()); + + let expected_links = [ + (&posts[0], "Camera ๐Ÿ“ท", "org.example.camera"), + (&posts[0], "Notes", "org.example.notes"), + (&posts[1], "Tracker", "org.example.tracker"), + ]; + for (post, name, id) in expected_links { + let facet = post + .facets + .iter() + .find(|facet| &post.text[facet.index.byte_start..facet.index.byte_end] == name) + .unwrap(); + match facet.features.as_slice() { + [Union::Refs(MainFeaturesItem::Link(link))] => { + assert_eq!(link.uri, format!("{PACKAGE_URL_PREFIX}{id}/")); + } + features => panic!("expected one link feature, got {features:?}"), + } + } + } +} -- cgit v1.3.1