initial commit

3 files changed, 670 insertions, 0 deletions

diff --git a/src/bots/fdroid/index.rs b/src/bots/fdroid/index.rs
new file mode 100644
index 0000000..61cc44d
--- /dev/null
+++ b/src/bots/fdroid/index.rs
@@ -0,0 +1,97 @@
+//! Types for the F-Droid repository index.
+
+use std::collections::HashMap;
+
+use serde::{Deserialize, Deserializer};
+
+pub(super) const PHONE_DEVICE_TYPE: &str = "phone";
+pub(super) const EN_LOCALE: &str = "en-US";
+
+#[derive(Debug, Deserialize)]
+pub(super) struct Index {
+    pub(super) repo: Repo,
+    pub(super) packages: HashMap<String, Package>,
+}
+
+#[derive(Debug, Deserialize)]
+pub(super) struct Repo {
+    pub(super) timestamp: u64,
+}
+
+#[derive(Debug, Deserialize)]
+pub(super) struct Package {
+    pub(super) metadata: Metadata,
+    /// Releases keyed by the APK's SHA-256 digest.
+    pub(super) versions: HashMap<String, Version>,
+}
+
+#[derive(Debug, Deserialize)]
+pub(super) struct Version {
+    /// Anti-feature explanations keyed first by anti-feature name and then by locale.
+    #[serde(default, rename = "antiFeatures")]
+    pub(super) anti_features: HashMap<AntiFeature, HashMap<String, String>>,
+}
+
+/// An [anti-feature](https://f-droid.org/docs/Anti-Features/).
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub(super) enum AntiFeature {
+    Ads,
+    DisabledAlgorithm,
+    KnownVuln,
+    NoSourceSince,
+    NonFreeAdd,
+    NonFreeAssets,
+    NonFreeDep,
+    NonFreeNet,
+    TetheredNet,
+    Tracking,
+    Other(String),
+}
+
+impl<'de> Deserialize<'de> for AntiFeature {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let name = String::deserialize(deserializer)?;
+        Ok(match name.as_str() {
+            "Ads" => Self::Ads,
+            "DisabledAlgorithm" => Self::DisabledAlgorithm,
+            "KnownVuln" => Self::KnownVuln,
+            "NoSourceSince" => Self::NoSourceSince,
+            "NonFreeAdd" => Self::NonFreeAdd,
+            "NonFreeAssets" => Self::NonFreeAssets,
+            "NonFreeDep" => Self::NonFreeDep,
+            "NonFreeNet" => Self::NonFreeNet,
+            "TetheredNet" => Self::TetheredNet,
+            "Tracking" => Self::Tracking,
+            _ => {
+                tracing::warn!("observed unknown F-Droid anti-feature {name}");
+                Self::Other(name)
+            }
+        })
+    }
+}
+
+#[derive(Debug, Deserialize)]
+pub(super) struct Metadata {
+    pub(super) added: u64,
+    /// App names keyed by locale. Every package has an `en-US` name.
+    #[serde(default)]
+    pub(super) name: HashMap<String, String>,
+    /// Short app descriptions keyed by locale.
+    #[serde(default)]
+    pub(super) summary: HashMap<String, String>,
+    /// Screenshots keyed first by device type and then by locale.
+    #[serde(default)]
+    pub(super) screenshots: HashMap<String, HashMap<String, Vec<Screenshot>>>,
+}
+
+#[derive(Debug, Deserialize)]
+pub(super) struct Screenshot {
+    /// Path to the screenshot relative to the repository address.
+    pub(super) name: String,
+    /// Screenshot file size in bytes, retained for enforcing upload limits.
+    #[allow(dead_code)]
+    pub(super) size: u64,
+}
diff --git a/src/bots/fdroid/mod.rs b/src/bots/fdroid/mod.rs
new file mode 100644
index 0000000..c0e7198
--- /dev/null
+++ b/src/bots/fdroid/mod.rs
@@ -0,0 +1,572 @@
+//! F-Droid
+
+mod index;
+
+use atrium_api::app::bsky::richtext::facet::{ByteSliceData, LinkData, MainData, MainFeaturesItem};
+use atrium_api::types::Union;
+use reqwest::{
+    StatusCode,
+    header::{ETAG, IF_NONE_MATCH},
+};
+use serde::{Deserialize, Serialize};
+use unicode_segmentation::UnicodeSegmentation;
+
+use self::index::{AntiFeature, EN_LOCALE, Index, PHONE_DEVICE_TYPE, Package, Screenshot};
+use crate::{
+    MAX_IMAGE_SIZE, MAX_POST_GRAPHEMES, MAX_POST_IMAGES, http_client,
+    report::{Post, PostImage, Report},
+};
+
+const REPOSITORY_URL: &str = "https://f-droid.org/repo";
+const PACKAGE_URL_PREFIX: &str = "https://f-droid.org/packages/";
+const ANTI_FEATURES_EXCLUDING_APPS: &[AntiFeature] =
+    &[AntiFeature::KnownVuln, AntiFeature::NoSourceSince];
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct BreakingState {
+    pub last_repo_timestamp: Option<u64>,
+    pub index_etag: Option<String>,
+}
+
+pub struct Breaking;
+
+impl Report for Breaking {
+    type State = BreakingState;
+
+    const STATE_FILE: &'static str = "fdroid-breaking.json";
+
+    async fn generate(state: Self::State) -> Result<(Vec<Post>, Self::State), String> {
+        let mut request = http_client().get(format!("{REPOSITORY_URL}/index-v2.json"));
+        if let Some(etag) = state.index_etag.as_deref() {
+            request = request.header(IF_NONE_MATCH, etag);
+        }
+
+        let response = request
+            .send()
+            .await
+            .map_err(|error| format!("failed to fetch F-Droid index: {error}"))?;
+        if response.status() == StatusCode::NOT_MODIFIED {
+            return Ok((Vec::new(), state));
+        }
+
+        let response = response
+            .error_for_status()
+            .map_err(|error| format!("failed to fetch F-Droid index: {error}"))?;
+        let index_etag = response
+            .headers()
+            .get(ETAG)
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned);
+        let index = response
+            .json::<Index>()
+            .await
+            .map_err(|error| format!("failed to parse F-Droid index: {error}"))?;
+
+        let Some(timestamp) = state.last_repo_timestamp else {
+            // We don't know the timestamp of the last index version.
+            tracing::info!(
+                timestamp = index.repo.timestamp,
+                "recording initial repository timestamp; will announce new apps with the next index version"
+            );
+            let new_state = BreakingState {
+                last_repo_timestamp: Some(index.repo.timestamp),
+                index_etag,
+            };
+            return Ok((Vec::new(), new_state));
+        };
+
+        if index.repo.timestamp < timestamp {
+            let error = format!(
+                "F-Droid repository timestamp moved backwards from {timestamp} to {}",
+                index.repo.timestamp
+            );
+            tracing::error!("{error}");
+            return Err(error);
+        }
+
+        let mut apps: Vec<_> = index
+            .packages
+            .iter()
+            .filter_map(|(id, package)| {
+                (package.metadata.added > timestamp)
+                    .then(|| App::from_package(id.as_str(), package))
+            })
+            .collect();
+
+        tracing::info!(
+            timestamp = index.repo.timestamp,
+            new_apps = apps.len(),
+            total_apps = index.packages.len(),
+            "building posts"
+        );
+
+        apps.sort_unstable_by(|left, right| {
+            left.name
+                .to_lowercase()
+                .cmp(&right.name.to_lowercase())
+                .then_with(|| left.id.cmp(right.id))
+        });
+
+        Ok((
+            download_images(build_announcement_posts(&apps)).await?,
+            BreakingState {
+                last_repo_timestamp: Some(index.repo.timestamp),
+                index_etag,
+            },
+        ))
+    }
+}
+
+struct App<'a> {
+    id: &'a str,
+    name: &'a str,
+    summary: &'a str,
+    screenshot: Option<&'a Screenshot>,
+    anti_features: Vec<AntiFeature>,
+}
+
+impl<'a> App<'a> {
+    fn from_package(id: &'a str, package: &'a Package) -> Self {
+        Self {
+            id,
+            name: package
+                .metadata
+                .name
+                .get(EN_LOCALE)
+                .map_or(id, String::as_str),
+            summary: package
+                .metadata
+                .summary
+                .get(EN_LOCALE)
+                .map_or("", String::as_str),
+            screenshot: package
+                .metadata
+                .screenshots
+                .get(PHONE_DEVICE_TYPE)
+                .and_then(|screenshots| screenshots.get(EN_LOCALE))
+                .and_then(|screenshots| {
+                    screenshots
+                        .iter()
+                        // We could resize images but realistically they should fit.
+                        .find(|screenshot| screenshot.size < MAX_IMAGE_SIZE)
+                }),
+            anti_features: package
+                .versions
+                .values()
+                .flat_map(|version| version.anti_features.keys())
+                // "Tethered network service" means an app depends on a network service,
+                // which isn't easy to self-host. There's a lot of such network services out there
+                // be it OpenStreetMap or a public transit API ... not much apps can do about that.
+                // So we just disregard this anti-feature in this bot.
+                .filter(|&anti_feature| anti_feature != &AntiFeature::TetheredNet)
+                .cloned()
+                .collect(),
+        }
+    }
+}
+
+fn build_announcement_posts(apps: &[App<'_>]) -> Vec<PendingPost> {
+    let apps: Vec<_> = apps
+        .iter()
+        .filter(|app| {
+            let Some(anti_feature) = ANTI_FEATURES_EXCLUDING_APPS
+                .iter()
+                .find(|anti_feature| app.anti_features.contains(anti_feature))
+            else {
+                return true;
+            };
+
+            tracing::info!(
+                app_id = app.id,
+                app_name = app.name,
+                "omitting F-Droid app due to anti-feature {anti_feature:?}"
+            );
+            false
+        })
+        .collect();
+
+    if apps.is_empty() {
+        return Vec::new();
+    }
+
+    let mut builders = vec![PostBuilder::new(format!(
+        "{} new F-Droid packages are available:",
+        apps.len()
+    ))];
+
+    let (without_anti_features, with_anti_features): (Vec<_>, Vec<_>) = apps
+        .into_iter()
+        .partition(|app| app.anti_features.is_empty());
+    let (with_screenshots, without_screenshots): (Vec<_>, Vec<_>) = without_anti_features
+        .into_iter()
+        .partition(|app| app.screenshot.is_some());
+
+    for app in with_screenshots {
+        let name = truncate(app.name, MAX_POST_GRAPHEMES);
+        let screenshot = app.screenshot.unwrap();
+        let image = PendingImage {
+            url: format!("{REPOSITORY_URL}{}", screenshot.name),
+            // Using the summary for the alt text isn't great
+            // but the Bluesky lexicon provides no alternative.
+            alt: format!("{}: {}", app.name, app.summary),
+        };
+
+        let post = builders.last().unwrap();
+        let separator = if post.text.is_empty() {
+            ""
+        } else if post.images.is_empty() {
+            "\n"
+        } else {
+            ", "
+        };
+        if !post.can_fit(separator, &name) || post.images.len() == MAX_POST_IMAGES {
+            builders.push(PostBuilder::new(String::new()));
+        }
+
+        let post = builders.last_mut().unwrap();
+        let separator = if post.text.is_empty() {
+            ""
+        } else if post.images.is_empty() {
+            "\n"
+        } else {
+            ", "
+        };
+        post.push_app(separator, &name, &name, app.id, Some(image));
+    }
+
+    for app in without_screenshots {
+        push_bullet_app(&mut builders, app);
+    }
+
+    if let Some((first, rest)) = with_anti_features.split_first() {
+        const ANTI_FEATURES_HEADING: &str =
+            "New apps with an anti-feature (other than tethered network):";
+        let line_limit =
+            MAX_POST_GRAPHEMES - grapheme_count(ANTI_FEATURES_HEADING) - grapheme_count("\n");
+        let (name, line) = app_line(first, line_limit);
+
+        // Keep anti-feature apps in their own section, starting at a post boundary.
+        builders.push(PostBuilder::new(ANTI_FEATURES_HEADING.to_owned()));
+        builders
+            .last_mut()
+            .unwrap()
+            .push_app("\n", &line, &name, first.id, None);
+
+        for app in rest {
+            push_bullet_app(&mut builders, app);
+        }
+    }
+
+    builders.into_iter().map(PostBuilder::finish).collect()
+}
+
+fn push_bullet_app(builders: &mut Vec<PostBuilder>, app: &App<'_>) {
+    let (name, line) = app_line(app, MAX_POST_GRAPHEMES);
+    let post = builders.last().unwrap();
+    let separator = if post.text.is_empty() { "" } else { "\n" };
+    if !post.can_fit(separator, &line) {
+        builders.push(PostBuilder::new(String::new()));
+    }
+
+    let post = builders.last_mut().unwrap();
+    let separator = if post.text.is_empty() { "" } else { "\n" };
+    post.push_app(separator, &line, &name, app.id, None);
+}
+
+/// Build a bullet that fits within `limit`, preserving as much of the summary as possible.
+fn app_line(app: &App<'_>, limit: usize) -> (String, String) {
+    let prefix = "* ";
+    let anti_features = anti_feature_annotation(&app.anti_features);
+    let suffix = ": ";
+    let fixed = grapheme_count(prefix)
+        + grapheme_count(app.name)
+        + grapheme_count(&anti_features)
+        + grapheme_count(suffix);
+
+    if fixed <= limit {
+        let summary = truncate(app.summary, limit - fixed);
+        (
+            app.name.to_owned(),
+            format!("{prefix}{}{anti_features}{suffix}{summary}", app.name),
+        )
+    } else {
+        let reserved = grapheme_count(prefix) + grapheme_count(&anti_features);
+        let name = truncate(app.name, limit.saturating_sub(reserved));
+        let line = format!("{prefix}{name}{anti_features}");
+        (name, line)
+    }
+}
+
+fn anti_feature_annotation(anti_features: &[AntiFeature]) -> String {
+    if anti_features.is_empty() {
+        return String::new();
+    }
+
+    let mut icons: Vec<_> = anti_features
+        .iter()
+        .map(|anti_feature| match anti_feature {
+            AntiFeature::Tracking => (0, "👁️"),
+            AntiFeature::Ads => (1, "🪧"),
+            AntiFeature::NonFreeNet => (2, "☁️"),
+            AntiFeature::NonFreeAssets => (3, "🖼️"),
+            AntiFeature::NonFreeAdd => (4, "🧩"),
+            AntiFeature::NonFreeDep => (5, "📦"),
+            AntiFeature::DisabledAlgorithm => (6, "🔓"), // shouldn't happen for new apps
+            AntiFeature::Other(_) => (7, "❓"),
+            AntiFeature::KnownVuln | AntiFeature::NoSourceSince | AntiFeature::TetheredNet => {
+                unreachable!()
+            }
+        })
+        .collect();
+    icons.sort_unstable_by_key(|(order, _)| *order);
+    icons.dedup_by_key(|(order, _)| *order);
+
+    format!(
+        " ({})",
+        icons
+            .into_iter()
+            .map(|(_, icon)| icon)
+            .collect::<Vec<_>>()
+            .join("")
+    )
+}
+
+fn truncate(text: &str, limit: usize) -> String {
+    let length = grapheme_count(text);
+    if length <= limit {
+        return text.to_owned();
+    }
+    if limit == 0 {
+        return String::new();
+    }
+
+    let mut shortened: String = text.graphemes(true).take(limit - 1).collect();
+    shortened.push('…');
+    shortened
+}
+
+fn grapheme_count(text: &str) -> usize {
+    text.graphemes(true).count()
+}
+
+struct PendingPost {
+    text: String,
+    facets: Vec<atrium_api::app::bsky::richtext::facet::Main>,
+    images: Vec<PendingImage>,
+}
+
+struct PendingImage {
+    url: String,
+    alt: String,
+}
+
+struct PostBuilder {
+    text: String,
+    graphemes: usize,
+    facets: Vec<atrium_api::app::bsky::richtext::facet::Main>,
+    images: Vec<PendingImage>,
+}
+
+impl PostBuilder {
+    fn new(text: String) -> Self {
+        let graphemes = grapheme_count(&text);
+        Self {
+            text,
+            graphemes,
+            facets: Vec::new(),
+            images: Vec::new(),
+        }
+    }
+
+    fn can_fit(&self, separator: &str, text: &str) -> bool {
+        self.graphemes + grapheme_count(separator) + grapheme_count(text) <= MAX_POST_GRAPHEMES
+    }
+
+    fn push_app(
+        &mut self,
+        separator: &str,
+        line: &str,
+        name: &str,
+        id: &str,
+        image: Option<PendingImage>,
+    ) {
+        self.text.push_str(separator);
+        let byte_start = self.text.len() + line.find(name).unwrap_or(0);
+        self.text.push_str(line);
+        let byte_end = byte_start + name.len();
+        self.graphemes += grapheme_count(separator) + grapheme_count(line);
+
+        self.facets.push(
+            MainData {
+                features: vec![Union::Refs(MainFeaturesItem::Link(Box::new(
+                    LinkData {
+                        uri: format!("{PACKAGE_URL_PREFIX}{id}/"),
+                    }
+                    .into(),
+                )))],
+                index: ByteSliceData {
+                    byte_start,
+                    byte_end,
+                }
+                .into(),
+            }
+            .into(),
+        );
+        if let Some(image) = image {
+            self.images.push(image);
+        }
+    }
+
+    fn finish(self) -> PendingPost {
+        PendingPost {
+            text: self.text,
+            facets: self.facets,
+            images: self.images,
+        }
+    }
+}
+
+async fn download_images(pending_posts: Vec<PendingPost>) -> Result<Vec<Post>, String> {
+    let client = http_client();
+    let mut posts = Vec::with_capacity(pending_posts.len());
+
+    for (post_index, post) in pending_posts.into_iter().enumerate() {
+        let mut images = Vec::with_capacity(post.images.len());
+        for image in post.images {
+            let data = client
+                .get(&image.url)
+                .send()
+                .await
+                .and_then(reqwest::Response::error_for_status)
+                .map_err(|error| {
+                    format!(
+                        "failed to download image {} for post {}: {error}",
+                        image.url,
+                        post_index + 1
+                    )
+                })?
+                .bytes()
+                .await
+                .map_err(|error| {
+                    format!(
+                        "failed to download image {} for post {}: {error}",
+                        image.url,
+                        post_index + 1
+                    )
+                })?;
+            images.push(PostImage {
+                data: data.to_vec(),
+                alt: image.alt,
+            });
+        }
+        posts.push(Post {
+            text: post.text,
+            facets: post.facets,
+            images,
+        });
+    }
+
+    Ok(posts)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn builds_announcement_posts() {
+        assert!(build_announcement_posts(&[]).is_empty());
+
+        let screenshot = Screenshot {
+            name: "/screenshots/camera.png".to_owned(),
+            size: 123,
+        };
+        let apps = [
+            App {
+                id: "org.example.camera",
+                name: "Camera 📷",
+                summary: "Take photos",
+                screenshot: Some(&screenshot),
+                anti_features: Vec::new(),
+            },
+            App {
+                id: "org.example.notes",
+                name: "Notes",
+                summary: "Take notes",
+                screenshot: None,
+                anti_features: Vec::new(),
+            },
+            App {
+                id: "org.example.tracker",
+                name: "Tracker",
+                summary: "Tracks users",
+                screenshot: None,
+                anti_features: vec![
+                    AntiFeature::Other("UnknownFeature".to_owned()),
+                    AntiFeature::Tracking,
+                    AntiFeature::NonFreeNet,
+                    AntiFeature::NonFreeDep,
+                    AntiFeature::NonFreeAssets,
+                    AntiFeature::NonFreeAdd,
+                    AntiFeature::DisabledAlgorithm,
+                    AntiFeature::Ads,
+                ],
+            },
+            App {
+                id: "org.example.vulnerable",
+                name: "Vulnerable",
+                summary: "Has a known vulnerability",
+                screenshot: None,
+                anti_features: vec![AntiFeature::KnownVuln],
+            },
+            App {
+                id: "org.example.abandoned",
+                name: "Abandoned",
+                summary: "No longer has source available",
+                screenshot: None,
+                anti_features: vec![AntiFeature::NoSourceSince],
+            },
+        ];
+
+        let posts = build_announcement_posts(&apps);
+
+        assert_eq!(posts.len(), 2);
+        assert_eq!(
+            posts[0].text,
+            "3 new F-Droid packages are available:\n\
+             Camera 📷\n\
+             * Notes: Take notes"
+        );
+        assert_eq!(
+            posts[1].text,
+            "New apps with an anti-feature (other than tethered network):\n\
+             * Tracker (👁️🪧☁️🖼️🧩📦🔓❓): Tracks users"
+        );
+        assert_eq!(posts[0].images.len(), 1);
+        assert_eq!(
+            posts[0].images[0].url,
+            "https://f-droid.org/repo/screenshots/camera.png"
+        );
+        assert_eq!(posts[0].images[0].alt, "Camera 📷: Take photos");
+        assert!(posts[1].images.is_empty());
+
+        let expected_links = [
+            (&posts[0], "Camera 📷", "org.example.camera"),
+            (&posts[0], "Notes", "org.example.notes"),
+            (&posts[1], "Tracker", "org.example.tracker"),
+        ];
+        for (post, name, id) in expected_links {
+            let facet = post
+                .facets
+                .iter()
+                .find(|facet| &post.text[facet.index.byte_start..facet.index.byte_end] == name)
+                .unwrap();
+            match facet.features.as_slice() {
+                [Union::Refs(MainFeaturesItem::Link(link))] => {
+                    assert_eq!(link.uri, format!("{PACKAGE_URL_PREFIX}{id}/"));
+                }
+                features => panic!("expected one link feature, got {features:?}"),
+            }
+        }
+    }
+}
diff --git a/src/bots/mod.rs b/src/bots/mod.rs
new file mode 100644
index 0000000..63414d3
--- /dev/null
+++ b/src/bots/mod.rs
@@ -0,0 +1 @@
+pub mod fdroid;