feat: automatic backups

9 files changed, 221 insertions, 2 deletions

diff --git a/nixos/hosts/ev/default.nix b/nixos/hosts/ev/default.nix
index 353d761..252e5c8 100644
--- a/nixos/hosts/ev/default.nix
+++ b/nixos/hosts/ev/default.nix
@@ -7,6 +7,7 @@
 let
   ports = import ./ports.nix;
   helpers = import <top/helpers.nix> { inherit config lib pkgs; };
+  backupHost = "u518695-sub2@u518695-sub2.your-storagebox.de";
 in
 {
   imports = [
@@ -17,6 +18,7 @@ in
     <top/shared/tailscale.nix>
     <top/shared/basics-physical.nix>
     <top/shared/vpn.nix>
+    <top/shared/restic-database-backups.nix>
     ./exporters.nix
     ./home-automation.nix
     ./kodi.nix
@@ -73,6 +75,38 @@ in
     };
   };
 
+  # Backups
+  age.secrets.restic-db.file = ./secrets/restic-db.age;
+  age.secrets.restic-media.file = ./secrets/restic-media.age;
+
+  restic-database-backups = {
+    repository = "sftp://${backupHost}:23" + "/db";
+    passwordFile = config.age.secrets.restic-db.path;
+    postgresDatabases = [
+      "miniflux"
+    ];
+    sqliteDatabases = {
+      navidrome = "/var/lib/navidrome/navidrome.db";
+    };
+    timerConfig = { OnCalendar = "00:20"; };
+    pruneOpts = ["--keep-daily 30" "--keep-weekly 8" "--keep-monthly 12"];
+  };
+
+  services.restic.backups = {
+    media = {
+      repository = "sftp://${backupHost}:23" + "/media";
+      passwordFile = config.age.secrets.restic-media.path;
+      paths = [
+        "/mnt/personal/photos"
+        "/mnt/personal/music"
+      ];
+      timerConfig = { OnCalendar = "00:30"; };
+      pruneOpts = ["--keep-daily 30" "--keep-weekly 8"];
+      extraBackupArgs = ["--skip-if-unchanged"];
+      progressFps = 0.0166; # once per minute
+    };
+  };
+
   boot.loader.systemd-boot.enable = true;
   boot.loader.efi.canTouchEfiVariables = true;
 
diff --git a/nixos/hosts/ev/secrets/restic-db.age b/nixos/hosts/ev/secrets/restic-db.age
new file mode 100644
index 0000000..eadf2ef
--- /dev/null
+++ b/nixos/hosts/ev/secrets/restic-db.age
@@ -0,0 +1,10 @@
+age-encryption.org/v1
+-> ssh-ed25519 PHC5tQ y9tYkynZ58Jy0MPKAVVIDEIwF9yUiGMqryAEbo2q+28
+PSRxZdfZmcIol+FMoPh0bda/bUfkMSChF5Y72GcGfIg
+-> ssh-ed25519 PMTW+A jMWBSj3z0EdY72S2XeCUGCnjEH26asIxM2mR7UJszRc
+DaV91wChTq41w+Ij6aiAPDNcCi/20eWV3AaTxO1nrec
+--- w0m1n/71+Ux37APY3bTxb8NhL9SpZJbKhvzj8tTA30k
+v�QK�7���
+���>�p
+���{
+�
�嵦_����+�.��l�܀�Nʚ)^
\ No newline at end of file
diff --git a/nixos/hosts/ev/secrets/restic-media.age b/nixos/hosts/ev/secrets/restic-media.age
new file mode 100644
index 0000000..745db28
--- /dev/null
+++ b/nixos/hosts/ev/secrets/restic-media.age
diff --git a/nixos/hosts/tente/default.nix b/nixos/hosts/tente/default.nix
index a54de6c..f0aa773 100644
--- a/nixos/hosts/tente/default.nix
+++ b/nixos/hosts/tente/default.nix
@@ -7,6 +7,7 @@
 let
   baseDomain = "push-f.com";
   acmeEmail = "martin@push-f.com";
+  backupHost = "u518695-sub1@u518695-sub1.your-storagebox.de";
   ports = import ./ports.nix;
   sources = import <top/npins>;
   helpers = import <top/helpers.nix> { inherit config lib pkgs; };
@@ -20,6 +21,7 @@ in
     <top/shared/postgresql.nix>
     <top/shared/tailscale.nix>
     <top/shared/monitoring.nix>
+    <top/shared/restic-database-backups.nix>
     ./web-personal.nix
     ./git.nix
     ./headscale.nix
@@ -129,6 +131,23 @@ in
   headscale.port = ports.headscale;
   matrix.port = ports.matrix;
 
+  # Backups
+  git.backupRsyncDest = "${backupHost}:repos";
+  git.backupOnCalendar = "00:00";
+
+  age.secrets.restic-db.file = ./secrets/restic-db.age;
+
+  restic-database-backups = {
+    repository = "sftp://${backupHost}:23" + "/db";
+    passwordFile = config.age.secrets.restic-db.path;
+    postgresDatabases = [];
+    sqliteDatabases = {
+      grafana = "/var/lib/grafana/data/grafana.db";
+    };
+    timerConfig = { OnCalendar = "00:10"; };
+    pruneOpts = ["--keep-daily 30" "--keep-weekly 8" "--keep-monthly 12"];
+  };
+
   # Use the GRUB 2 boot loader.
   boot.loader.grub.enable = true;
   # boot.loader.grub.efiSupport = true;
diff --git a/nixos/hosts/tente/exporters.nix b/nixos/hosts/tente/exporters.nix
index 1c03c35..a2e799e 100644
--- a/nixos/hosts/tente/exporters.nix
+++ b/nixos/hosts/tente/exporters.nix
@@ -37,6 +37,7 @@ in
           targets = [ "localhost:${toString cfg.sqlExporterPort}" ];
         }];
       }
+      # TODO: add exporter for storagebox
     ];
   };
 }
diff --git a/nixos/hosts/tente/git.nix b/nixos/hosts/tente/git.nix
index 467ba30..01fa3f4 100644
--- a/nixos/hosts/tente/git.nix
+++ b/nixos/hosts/tente/git.nix
@@ -9,9 +9,19 @@ in
     webUiDomain = lib.mkOption {
       type = lib.types.str;
     };
+
+    backupRsyncDest = lib.mkOption {
+      type = lib.types.str;
+    };
+
+    backupOnCalendar = lib.mkOption {
+      type = lib.types.str;
+    };
   };
 
-  config = {
+  config = let
+    reposPath = "${config.services.gitolite.dataDir}/repositories";
+  in {
     services = {
       gitolite = {
         enable = true;
@@ -49,7 +59,7 @@ in
         user = config.services.gitolite.user;
         group = config.services.gitolite.group;
         nginx.virtualHost = cfg.webUiDomain;
-        scanPath = "${config.services.gitolite.dataDir}/repositories";
+        scanPath = reposPath;
         settings = {
           remove-suffix = 1;
           enable-git-config = 1;
@@ -74,5 +84,13 @@ in
         };
       };
     };
+
+    systemd.services.backup-repos = {
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${pkgs.rsync}/bin/rsync -a --delete ${reposPath}/ ${cfg.backupRsyncDest} -e '${pkgs.openssh}/bin/ssh -p23'";
+      };
+      startAt = cfg.backupOnCalendar;
+    };
   };
 }
diff --git a/nixos/hosts/tente/secrets/restic-db.age b/nixos/hosts/tente/secrets/restic-db.age
new file mode 100644
index 0000000..f5b5312
--- /dev/null
+++ b/nixos/hosts/tente/secrets/restic-db.age
@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 aqtcmw oIqAvifgxLP5LINVJ5s9qHjM5bSCpcCcJRtLfXDvTG8
+vX5EWOPifb8hgpJiox4Vlk3aw41bBtDoxx9mB9YI8JU
+-> ssh-ed25519 PMTW+A oeJuzrKccPTwrHtOBrwowsV669McJs3voo7Gw/RQ1UQ
+ajo7AmdJlaQ4hnOCoCmhqqkiER4tXJUv+OogO68V/fc
+--- H9+ZM0qj8sqcv46czovIyta763wGodK/zjAsRD4Aors
+�A�܉Az׃`Ǎ�ߋ�g�� _8�'��&L�v��0�ܽX�V��#��\_��
\ No newline at end of file
diff --git a/nixos/profiles/workstation/default.nix b/nixos/profiles/workstation/default.nix
index d0323f8..38b4723 100644
--- a/nixos/profiles/workstation/default.nix
+++ b/nixos/profiles/workstation/default.nix
@@ -42,6 +42,7 @@ in
       (callPackage "${sources.agenix}/pkgs/agenix.nix" {})
       (callPackage sources.my-vdf {})
       nixfmt-rfc-style
+      restic
 
       vim-full
       skim
diff --git a/nixos/shared/restic-database-backups.nix b/nixos/shared/restic-database-backups.nix
new file mode 100644
index 0000000..f1f6f36
--- /dev/null
+++ b/nixos/shared/restic-database-backups.nix
@@ -0,0 +1,129 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.restic-database-backups;
+in
+{
+  options.restic-database-backups = {
+    repository = mkOption {
+      type = types.str;
+      description = "Restic repository URL.";
+    };
+
+    passwordFile = mkOption {
+      type = types.path;
+      description = "Path to restic password file.";
+    };
+
+    timerConfig = mkOption {
+      type = types.attrs;
+      description = "systemd timer configuration.";
+    };
+
+    pruneOpts = mkOption {
+      type = types.listOf types.str;
+      default = [];
+      description = "Prune options passed to restic.";
+    };
+
+    postgresDatabases = mkOption {
+      type = types.listOf types.str;
+      default = [];
+      description = "List of PostgreSQL database names to back up.";
+    };
+
+    sqliteDatabases = mkOption {
+      type = types.attrsOf types.path;
+      default = {};
+      description = "SQLite databases to back up (name -> path).";
+    };
+  };
+
+  config =
+    {
+      services.restic.backups =
+        let
+          backup = {
+            repository = cfg.repository;
+            passwordFile = cfg.passwordFile;
+            # I want the backups to run one after the other so I'm disabling
+            # the timer creation here and then later set up "after".
+            timerConfig = null;
+            extraBackupArgs = ["--skip-if-unchanged"];
+            progressFps = 0.0166; # once per minute
+          };
+        in
+        # TODO: pg_dumpall --globals-only
+        (listToAttrs (
+          map (db: {
+            name = "postgres-${db}";
+            value = backup //
+            {
+              backupPrepareCommand = ''
+                mkdir -p /var/backup
+                chmod 700 /var/backup
+                ${pkgs.sudo}/bin/sudo ${pkgs.su}/bin/su postgres -c '${pkgs.postgresql}/bin/pg_dump ${db}' > /var/backup/${db}.sql
+              '';
+              paths = ["/var/backup/${db}.sql"];
+              backupCleanupCommand = ''
+                rm -f /var/backup/${db}.sql
+              '';
+            };
+          }) cfg.postgresDatabases
+        ))
+        // (mapAttrs' (name: path: {
+          name = "sqlite-${name}";
+          value = backup // {
+            backupPrepareCommand = ''
+              mkdir -p /var/backup
+              chmod 700 /var/backup
+              ${pkgs.sqlite}/bin/sqlite3 ${path} ".backup /var/backup/${name}.sqlite"
+            '';
+            paths = ["/var/backup/${name}.sqlite"];
+            backupCleanupCommand = ''
+              rm -f /var/backup/${name}.sqlite
+            '';
+          };
+        }) cfg.sqliteDatabases)
+        // {
+          database-prune = backup // {
+            pruneOpts = cfg.pruneOpts;
+          };
+        };
+
+      systemd.services =
+        let
+          serviceNames =
+            (map (db: "restic-backups-postgres-${db}") cfg.postgresDatabases)
+            ++ (map (name: "restic-backups-sqlite-${name}") (attrNames cfg.sqliteDatabases))
+            ++ ["restic-backups-database-prune"];
+          fullServiceNames = map (name: "${name}.service") serviceNames;
+          services = listToAttrs (
+            imap0 (idx: name: {
+              name = name;
+              value = mkIf (idx > 0) {
+                after = [(elemAt fullServiceNames (idx - 1))];
+              };
+            }) serviceNames
+          );
+        in
+        services
+        // {
+          database-backups = {
+            serviceConfig = {
+              Type = "oneshot";
+              ExecStart = "${pkgs.coreutils}/bin/true";
+            };
+            wants = fullServiceNames;
+            after = fullServiceNames;
+          };
+        };
+
+      systemd.timers.database-backups = {
+        wantedBy = ["timers.target"];
+        timerConfig = cfg.timerConfig;
+      };
+    };
+}