diff --git a/checks/borgbackup-legacy/default.nix b/checks/borgbackup-legacy/default.nix new file mode 100644 index 000000000..85a6dba46 --- /dev/null +++ b/checks/borgbackup-legacy/default.nix @@ -0,0 +1,51 @@ +( + { ... }: + { + name = "borgbackup"; + + nodes.machine = + { self, pkgs, ... }: + { + imports = [ + self.clanModules.borgbackup + self.nixosModules.clanCore + { + services.openssh.enable = true; + services.borgbackup.repos.testrepo = { + authorizedKeys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + }; + } + { + clan.core.settings.directory = ./.; + clan.core.state.testState.folders = [ "/etc/state" ]; + environment.etc.state.text = "hello world"; + systemd.tmpfiles.settings."vmsecrets" = { + "/etc/secrets/borgbackup/borgbackup.ssh" = { + C.argument = "${../assets/ssh/privkey}"; + z = { + mode = "0400"; + user = "root"; + }; + }; + "/etc/secrets/borgbackup/borgbackup.repokey" = { + C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345"); + z = { + mode = "0400"; + user = "root"; + }; + }; + }; + # clan.core.facts.secretStore = "vm"; + clan.core.vars.settings.secretStore = "vm"; + + clan.borgbackup.destinations.test.repo = "borg@localhost:."; + } + ]; + }; + testScript = '' + start_all() + machine.systemctl("start --wait borgbackup-job-test.service") + assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list") + ''; + } +) diff --git a/checks/borgbackup/default.nix b/checks/borgbackup/default.nix index 85a6dba46..e0cb8a890 100644 --- a/checks/borgbackup/default.nix +++ b/checks/borgbackup/default.nix @@ -1,51 +1,118 @@ -( - { ... }: - { - name = "borgbackup"; +{ + pkgs, + self, + clanLib, + ... +}: - nodes.machine = - { self, pkgs, ... }: - { - imports = [ - self.clanModules.borgbackup - self.nixosModules.clanCore - { - services.openssh.enable = true; - services.borgbackup.repos.testrepo = { - authorizedKeys = [ (builtins.readFile ../assets/ssh/pubkey) ]; - }; - } - { - clan.core.settings.directory = ./.; - clan.core.state.testState.folders = [ "/etc/state" ]; - environment.etc.state.text = "hello world"; - systemd.tmpfiles.settings."vmsecrets" = { - "/etc/secrets/borgbackup/borgbackup.ssh" = { - C.argument = "${../assets/ssh/privkey}"; - z = { - mode = "0400"; - user = "root"; - }; - }; - "/etc/secrets/borgbackup/borgbackup.repokey" = { - C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345"); - z = { - mode = "0400"; - user = "root"; - }; - }; - }; - # clan.core.facts.secretStore = "vm"; - clan.core.vars.settings.secretStore = "vm"; +clanLib.test.makeTestClan { + inherit pkgs self; + useContainers = true; - clan.borgbackup.destinations.test.repo = "borg@localhost:."; - } - ]; + nixosTest = ( + { ... }: + + { + name = "borgbackup"; + + clan = { + directory = ./.; + modules."@clan/borgbackup" = ../../clanServices/borgbackup/default.nix; + inventory = { + + machines.clientone = { }; + machines.serverone = { }; + + instances = { + borgone = { + + module.name = "@clan/borgbackup"; + + roles.client.machines."clientone" = { }; + roles.server.machines."serverone".settings.directory = "/tmp/borg-test"; + }; + }; + }; }; - testScript = '' - start_all() - machine.systemctl("start --wait borgbackup-job-test.service") - assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list") - ''; - } -) + + nodes = { + + serverone = { + services.openssh.enable = true; + # Needed so PAM doesn't see the user as locked + users.users.borg.password = "borg"; + }; + + clientone = + { config, pkgs, ... }: + let + dependencies = [ + self + pkgs.stdenv.drvPath + ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); + closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; + + in + { + + services.openssh.enable = true; + + users.users.root.openssh.authorizedKeys.keyFiles = [ ../assets/ssh/pubkey ]; + + clan.core.networking.targetHost = config.networking.hostName; + + environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ]; + + environment.etc.install-closure.source = "${closureInfo}/store-paths"; + nix.settings = { + substituters = pkgs.lib.mkForce [ ]; + hashed-mirrors = null; + connect-timeout = pkgs.lib.mkForce 3; + flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}''; + }; + system.extraDependencies = dependencies; + + clan.core.state.test-backups.folders = [ "/var/test-backups" ]; + }; + + }; + + testScript = '' + import json + start_all() + + machines = [clientone, serverone] + + for m in machines: + m.systemctl("start network-online.target") + + for m in machines: + m.wait_for_unit("network-online.target") + + # dummy data + clientone.succeed("mkdir -p /var/test-backups /var/test-service") + clientone.succeed("echo testing > /var/test-backups/somefile") + + clientone.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519") + clientone.succeed("${pkgs.coreutils}/bin/touch /root/.ssh/known_hosts") + clientone.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new localhost hostname") + clientone.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new $(hostname) hostname") + + # create + clientone.succeed("borgbackup-create >&2") + clientone.wait_until_succeeds("! systemctl is-active borgbackup-job-serverone >&2") + + # list + backup_id = json.loads(clientone.succeed("borg-job-serverone list --json"))["archives"][0]["archive"] + out = clientone.succeed("borgbackup-list").strip() + print(out) + assert backup_id in out, f"backup {backup_id} not found in {out}" + + # borgbackup restore + clientone.succeed("rm -f /var/test-backups/somefile") + clientone.succeed(f"NAME='serverone::borg@serverone:.::{backup_id}' borgbackup-restore >&2") + assert clientone.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed" + ''; + } + ); +} diff --git a/checks/borgbackup/sops/machines/clientone/key.json b/checks/borgbackup/sops/machines/clientone/key.json new file mode 100755 index 000000000..28a250742 --- /dev/null +++ b/checks/borgbackup/sops/machines/clientone/key.json @@ -0,0 +1,6 @@ +[ + { + "publickey": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c", + "type": "age" + } +] diff --git a/checks/borgbackup/sops/secrets/clientone-age.key/secret b/checks/borgbackup/sops/secrets/clientone-age.key/secret new file mode 100644 index 000000000..384ad184a --- /dev/null +++ b/checks/borgbackup/sops/secrets/clientone-age.key/secret @@ -0,0 +1,15 @@ +{ + "data": "ENC[AES256_GCM,data:wCKoKuJo4uXycfqEUYAXDlRRMGJaWgOFiaQa4Wigs0jx1eCI80lP3cEZ1QKyrU/9m9POoZz0JlaKHcuhziTKUqaevHvGfVq2y00=,iv:pH5a90bJbK9Ro6zndNJ18qd4/rU+Tdm+y+jJZtY7UGg=,tag:9lHZJ9C/zIfy8nFrYt9JBQ==,type:str]", + "sops": { + "age": [ + { + "recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwUDhpd1ZqbWFqR0I3dVFI\nOHlyZnFUYXJnWElrRWhoUHVNMzdKd0VrcGdRCkphQVhuYzlJV0p1MG9MSW5ncWJ3\nREp1OEJxMzQzS2MxTk9aMkJ1a3B0Q0kKLS0tIENweVJ2Tk1yeXlFc2F5cTNIV3F3\nTkRFOVZ1amRIYmg1K3hGWUFSTTl4Wk0KHJRJ7756Msod7Bsmn9SgtwRo53B8Ilp3\nhsAPv+TtdmOD8He9MvGV+BElKEXCsLUwhp/Py6n6CJCczu0VIr8owg==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-05-20T13:33:56Z", + "mac": "ENC[AES256_GCM,data:FyfxXhnI6o4SVGJY2e1eMDnfkbMWiCkP4JL/G4PQvzz+c7OIuz8xaa03P3VW7b7o85NP2Tln4FMNTZ0FYtQwd0kKypLUnIxAHsixAHFCv4X8ul1gtZynzgbFbmc0GkfVWW8Lf+U+vvDwT+UrEVfcmksCjdvAOwP26PvlEhYEkSw=,iv:H+VrWYL+kLOLezCZrI8ZgeCsaUdpb7LxDMiLotezVPs=,tag:B/cbPdiEFumGKQHby5inCA==,type:str]", + "unencrypted_suffix": "_unencrypted", + "version": "3.10.2" + } +} diff --git a/checks/borgbackup/sops/secrets/clientone-age.key/users/admin b/checks/borgbackup/sops/secrets/clientone-age.key/users/admin new file mode 120000 index 000000000..9e21a9938 --- /dev/null +++ b/checks/borgbackup/sops/secrets/clientone-age.key/users/admin @@ -0,0 +1 @@ +../../../users/admin \ No newline at end of file diff --git a/checks/borgbackup/sops/users/admin/key.json b/checks/borgbackup/sops/users/admin/key.json new file mode 100644 index 000000000..e408aa96b --- /dev/null +++ b/checks/borgbackup/sops/users/admin/key.json @@ -0,0 +1,4 @@ +{ + "publickey": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg", + "type": "age" +} diff --git a/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/machines/clientone b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/machines/clientone new file mode 120000 index 000000000..230a86f98 --- /dev/null +++ b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/machines/clientone @@ -0,0 +1 @@ +../../../../../../sops/machines/clientone \ No newline at end of file diff --git a/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/secret b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/secret new file mode 100644 index 000000000..ea1243ade --- /dev/null +++ b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/secret @@ -0,0 +1,19 @@ +{ + "data": "ENC[AES256_GCM,data:52vY68gqbwiZRMUBKc9SeXR06fuKAhuAPciLpxXgEOxI,iv:Y34AVoHaZzRiFFTDbekXP1X3W8zSXJmzVCYODYkdxnY=,tag:8WQaGEHQKT/n+auHUZCE0w==,type:str]", + "sops": { + "age": [ + { + "recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOdUFUZUZ2M00zTGlhNjF4\nL0VlMVY4Z2xMbWRWR29zZlFwdm1XRk12NGtBCnkrb3A4M3BkalMyeWdDaUdQdStt\nUWY3SXJROXdpRzN0NlBJNEpjTEZ0aFkKLS0tIGZkMGhsTXB2RnRqVHVrUFQwL2lw\nZnBreWhWa3Jrcm4yOXBiaUlPWFM1aDAKRE+Zzrja7KeANEJUbmFYuVoO3qGyi4iH\n0cfH0W8irRe9vsKMXz7YJxtByYLwRulrT8tXtElHvIEVJG0mwwaf0Q==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsNEljUFdnQ0tTQ1IxZ2Zo\nYkc4V2dCaUk0YXh5SzlSazhsRTVKVzFvVXhFCkRyMlMxR3EyWEZIRzFQV3d2dVpz\na3NPbk9XdWR1NmtMQlZsNlBuU0NkQWMKLS0tIDlDYzMzOExVL1g5SVRHYlpUQlBV\na2lpdTUwaEd4OXhWUWxuV04xRVVKNHcK9coohAD1IoarLOXSGg3MIRXQ3BsTIA4y\nKrcS/PxITKJs7ihg93RZin70R79Qsij1RHZLKGfgGJ67i8ZCxc4N0g==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-05-20T13:33:59Z", + "mac": "ENC[AES256_GCM,data:eABMaIe07dwAMMlgrIUUpfpj73q1H5Keafql91MBQ5NN9Znr5lI/ennQsQsuLO8ZTCC34US/MJndliW34SqVM9y53p0jjPzqBxSKYq74iNcBz7+TxbjlY1aapgTRPr6Ta8I/5loohnxlHqjvLL70ZzfbChDN0/4jZsDVXYNfbIk=,iv:41Mz2u40JN0iE5zPUK6siaxo0rTtlk7fGWq7TF5NyUI=,tag:1A+h6XPH7DeQ6kxGDV3PgQ==,type:str]", + "unencrypted_suffix": "_unencrypted", + "version": "3.10.2" + } +} diff --git a/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/users/admin b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/users/admin new file mode 120000 index 000000000..ca714e122 --- /dev/null +++ b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/users/admin @@ -0,0 +1 @@ +../../../../../../sops/users/admin \ No newline at end of file diff --git a/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh.pub/value b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh.pub/value new file mode 100644 index 000000000..af244e2c1 --- /dev/null +++ b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh.pub/value @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE3clYF6BDZ0PxfDdprx7YYM4U4PKEZkWUuhpre0wb7w nixbld@kiwi diff --git a/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/machines/clientone b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/machines/clientone new file mode 120000 index 000000000..230a86f98 --- /dev/null +++ b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/machines/clientone @@ -0,0 +1 @@ +../../../../../../sops/machines/clientone \ No newline at end of file diff --git a/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/secret b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/secret new file mode 100644 index 000000000..c9a6c8bd0 --- /dev/null +++ b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/secret @@ -0,0 +1,19 @@ +{ + "data": "ENC[AES256_GCM,data:tAjfBW75XDS8lfJCf/+9rPYH3aMjRX1nmdN5dPMxnrlhuEPM3Smv9AM93Tz36k7BKk31bUWcV/99ax+KaIK1Rzgym/CwKGGxIUziuVOEOwrCOBeOw7amZ9YGsgiLUTLIhoeO6SjfdZ4q2JxGPw7KqNfUM9kiZT01vx5JTLa24JdvBKpizbtHRlL1lappTRVt0dG2WhT9/YhQUGu9ZFqPs8+bPOBclc78qjCm2DAPgsprK+JCBuq+r+qHgAx4Ee1QHI7FC39e5NeGBTBeZfZ5d95+0klKuTx9FCPs6QRBkQ0tN29OpwzkdSuRAXGGHpzPkZ+FupbETtSQWCmnjma6jPzEl8oDUTWooKK0mUEz8icvTQvRfyM3Qt3mQpkX3e0rTEbZzoLdWCwTufP/tRQNDCWvI/NV7OjIHpNPjymqE5uPmiBpA6y6hhCH7zL1eDo11ICSIX3hkyFJH2svvFQn6oLrPAoByvNutfetKhd8z7NFpVeIOWwtuPzO7wU5M7zESHww0JF78vjFwimQYYhQ,iv:fVjeVez4dTGSrANi5ZeP9PJhsSySqeqqJzBDbd0gFW4=,tag:Aa89+bWLljxV1tlSHtpddw==,type:str]", + "sops": { + "age": [ + { + "recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVaW94M3VwcFJ2elcrRGlv\nUGdzVk9vU2ZweFpIVVlIRUEyRVlSMlEyeHpVCnJuV0xIS3hMLy9IbG92S0pvL2RP\nL0J0WkVuWVhQdldHekdYNTVXdFkrUlEKLS0tIFQzdGErZVBwQUFNMXErbDBQVURZ\naHlsY2hDa1Zud1E2dFh0ZHl4VEJ2S0kKVABqwRcCUTcsBInfo9CpFtoM3kl4KMyU\nGXDjHOSjlX5df7OKZAvYukgX7Q2penvq+Fq4fa4A1Cmkqga7cHdJ+A==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnbHRSVEg3Vi9qTnAwWGF6\nbEdIR2gvZ2laZnJMbVF3NjcvN25OdXF3WXowCnVUODdEa1NWU3JISXlrNldOMjVi\ndUlMTVdBaWxvZHlwSTdJY3NCcll4SjAKLS0tIEp6ZVlDTklqVXdNYzJ2dElCR21o\nUWphMDdyVVppVnFHOVlHZTNtajZzOXMKRB61lUrAkUXSYl3ffOOK8k4QgLA4bFln\naQ7GOol8f8W5H68zXBMZrhjP6k4kZDfknc9jgyoWM7jaZNSWC5J19Q==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-05-20T13:33:59Z", + "mac": "ENC[AES256_GCM,data:NjVpDweqxTSQGt9VKR/CMfvbvHQJHCi8P7XbOuKLZKQ4GVoeZ5r4PsC6nxKHHikN6YL1oJCmaSxr0mJRk/sFZg/+wdW8L7F5aQeFRiWo9jCjH0MDMnfiu5a0xjRt21uPl/7LUJ9jNon5nyxPTlZMeYSvTP2Q9spnNuN8vqipP68=,iv:DPvbN9IvWiUfxiJk6mey/us8N1GGVJcSJrT8Bty4kB4=,tag:+emK8uSkfIGUXoYpaWeu3A==,type:str]", + "unencrypted_suffix": "_unencrypted", + "version": "3.10.2" + } +} diff --git a/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/users/admin b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/users/admin new file mode 120000 index 000000000..ca714e122 --- /dev/null +++ b/checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/users/admin @@ -0,0 +1 @@ +../../../../../../sops/users/admin \ No newline at end of file diff --git a/checks/clan-core-for-checks.nix b/checks/clan-core-for-checks.nix index 348136ee4..b0899704f 100644 --- a/checks/clan-core-for-checks.nix +++ b/checks/clan-core-for-checks.nix @@ -1,6 +1,6 @@ { fetchgit }: fetchgit { url = "https://git.clan.lol/clan/clan-core.git"; - rev = "1523ac18c9c575d32033dcf1e769fccc324f248e"; - sha256 = "0nxhw5s9lva4g1rgx6pgczh3vxrskmmlxay48wvn2pnkrlvhr9j8"; + rev = "cb4f2ab014aa01f249e852da9e1b92cdc44c2a66"; + sha256 = "sha256-8zlAW+iWKrxxozSNEZAVlDPOMLzetd36RL1GLnAvvjg="; } diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 18805f3f6..a47edd491 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -41,7 +41,7 @@ in # Base Tests secrets = self.clanLib.test.baseTest ./secrets nixosTestArgs; - borgbackup = self.clanLib.test.baseTest ./borgbackup nixosTestArgs; + borgbackup-legacy = self.clanLib.test.baseTest ./borgbackup-legacy nixosTestArgs; wayland-proxy-virtwl = self.clanLib.test.baseTest ./wayland-proxy-virtwl nixosTestArgs; # Container Tests @@ -53,6 +53,7 @@ in # Clan Tests dummy-inventory-test = import ./dummy-inventory-test nixosTestArgs; admin = import ./admin nixosTestArgs; + borgbackup = import ./borgbackup nixosTestArgs; data-mesher = import ./data-mesher nixosTestArgs; syncthing = import ./syncthing nixosTestArgs; } @@ -108,11 +109,13 @@ in cat $schemaFile > $out/allSchemas.json ''; - clan-core-for-checks = pkgs.runCommand "clan-core-for-checks" { } '' - cp -r ${pkgs.callPackage ./clan-core-for-checks.nix { }} $out - chmod +w $out/flake.lock - cp ${../flake.lock} $out/flake.lock - ''; + clan-core-for-checks = self; + + # pkgs.runCommand "clan-core-for-checks" { } '' + # cp -r ${pkgs.callPackage ./clan-core-for-checks.nix { }} $out + # chmod +w $out/flake.lock + # cp ${../flake.lock} $out/flake.lock + # ''; }; legacyPackages = { nixosTests = diff --git a/clanModules/borgbackup/README.md b/clanModules/borgbackup/README.md index ef4649210..a5d03d83e 100644 --- a/clanModules/borgbackup/README.md +++ b/clanModules/borgbackup/README.md @@ -1,7 +1,7 @@ --- description = "Efficient, deduplicating backup program with optional compression and secure encryption." categories = ["System"] -features = [ "inventory" ] +features = [ "inventory", "deprecated" ] --- BorgBackup (short: Borg) gives you: diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index 9515f7517..236b1ee29 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -106,7 +106,8 @@ in systemd.services = lib.mapAttrs' ( _: dest: lib.nameValuePair "borgbackup-job-${dest.name}" { - # since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files. + # since borgbackup mounts the system read-only, we need to run in a + # ExecStartPre script, so we can generate additional files. serviceConfig.ExecStartPre = [ ''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}'' ]; diff --git a/clanServices/borgbackup/README.md b/clanServices/borgbackup/README.md new file mode 100644 index 000000000..846360c8c --- /dev/null +++ b/clanServices/borgbackup/README.md @@ -0,0 +1,9 @@ +BorgBackup (short: Borg) gives you: + +- Space efficient storage of backups. +- Secure, authenticated encryption. +- Compression: lz4, zstd, zlib, lzma or none. +- Mountable backups with FUSE. +- Easy installation on multiple platforms: Linux, macOS, BSD, … +- Free software (BSD license). +- Backed by a large and active open-source community. diff --git a/clanServices/borgbackup/default.nix b/clanServices/borgbackup/default.nix new file mode 100644 index 000000000..b292a0bdb --- /dev/null +++ b/clanServices/borgbackup/default.nix @@ -0,0 +1,313 @@ +{ ... }: +{ + _class = "clan.service"; + manifest.name = "borgbackup"; + manifest.description = "Efficient, deduplicating backup program with optional compression and secure encryption."; + manifest.categories = [ "System" ]; + manifest.readme = builtins.readFile ./README.md; + + # TODO: a client can only be in one instance, add constraint + + roles.server = { + + interface = + { lib, ... }: + { + + options.directory = lib.mkOption { + type = lib.types.str; + default = "/var/lib/borgbackup"; + description = '' + The directory where the borgbackup repositories are stored. + ''; + }; + + }; + + perInstance = + { + roles, + settings, + ... + }: + { + nixosModule = + { + config, + ... + }: + { + + config.services.openssh.enable = true; + + config.services.borgbackup.repos = + let + borgbackupIpMachinePath = + machine: + config.clan.core.settings.directory + + "/vars/per-machine/${machine}/borgbackup/borgbackup.ssh.pub/value"; + + hosts = builtins.mapAttrs (machineName: _machineSettings: { + # name = "${instanceName}-${machineName}"; + # value = { + path = "${settings.directory}/${machineName}"; + authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machineName)) ]; + # }; + # }) machinesWithKey; + }) roles.client.machines; + in + hosts; + }; + }; + }; + + roles.client = { + interface = + { + lib, + ... + }: + { + + options.destinations = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + type = lib.types.strMatching "^[a-zA-Z0-9._-]+$"; + default = name; + description = "the name of the backup job"; + }; + repo = lib.mkOption { + type = lib.types.str; + description = "the borgbackup repository to backup to"; + }; + rsh = lib.mkOption { + type = lib.types.str; + defaultText = "ssh -i \${config.clan.core.vars.generators.borgbackup.files.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"; + description = "the rsh to use for the backup"; + }; + }; + } + ) + ); + default = { }; + description = '' + external destinations where the machine should be backuped to + ''; + }; + + options.exclude = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = [ "*.pyc" ]; + default = [ ]; + description = '' + Directories/Files to exclude from the backup. + Use * as a wildcard. + ''; + }; + }; + + perInstance = + { + extendSettings, + roles, + ... + }: + { + nixosModule = + { + config, + lib, + pkgs, + ... + }: + let + settings = extendSettings { + + # Adding default value with option merging, because it depends on + # generators, which we can reference here. + options.destinations = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + rsh = lib.mkOption { + default = "ssh -i ${ + config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh".path + } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes -o PasswordAuthentication=no"; + }; + }; + } + ); + }; + }; + + in + + { + config = + let + preBackupScript = '' + declare -A preCommandErrors + + ${lib.concatMapStringsSep "\n" ( + state: + lib.optionalString (state.preBackupCommand != null) '' + echo "Running pre-backup command for ${state.name}" + if ! /run/current-system/sw/bin/${state.preBackupCommand}; then + preCommandErrors["${state.name}"]=1 + fi + '' + ) (lib.attrValues config.clan.core.state)} + + if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then + echo "pre-backup commands failed for the following services:" + for state in "''${!preCommandErrors[@]}"; do + echo " $state" + done + exit 1 + fi + ''; + + # The destinations from server.roles.machines.* + # name is the server, machine can only be in one instance + internalDestinations = + let + destinations = builtins.map (serverName: { + name = "${serverName}"; + value = { + # inherit name; + name = "${serverName}"; + repo = "borg@${serverName}:."; + # rsh = ""; + + rsh = "ssh -i ${ + config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh".path + } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes"; + }; + }) (builtins.attrNames roles.server.machines); + in + (builtins.listToAttrs destinations); + + # The destinations specified via roles.client.machines.*.settings.destinations. + # name is the + externalDestinations = lib.mapAttrs' ( + name: dest: lib.nameValuePair name dest + ) settings.destinations; + + allDestinations = + lib.warnIf ((builtins.intersectAttrs externalDestinations internalDestinations) != { }) + "You are overwriting an internalDestinations through an externalDestination configuration." + (internalDestinations // externalDestinations); + + in + { + services.openssh.enable = true; + + # Derived from the destinations + systemd.services = lib.mapAttrs' ( + destName: _dest: + lib.nameValuePair "borgbackup-job-${destName}" { + # since borgbackup mounts the system read-only, we need to + # run in a ExecStartPre script, so we can generate + # additional files. + serviceConfig.ExecStartPre = [ + ''+${pkgs.writeShellScript "borgbackup-job-${destName}-pre-backup-commands" preBackupScript}'' + ]; + } + ) allDestinations; + + services.borgbackup.jobs = lib.mapAttrs (_: dest: { + paths = lib.unique ( + lib.flatten (map (state: state.folders) (lib.attrValues config.clan.core.state)) + ); + exclude = settings.exclude; + repo = dest.repo; + environment.BORG_RSH = dest.rsh; + compression = "auto,zstd"; + startAt = "*-*-* 01:00:00"; + persistentTimer = true; + + encryption = { + mode = "repokey"; + passCommand = "cat ${config.clan.core.vars.generators.borgbackup.files."borgbackup.repokey".path}"; + }; + + prune.keep = { + within = "1d"; # Keep all archives from the last day + daily = 7; + weekly = 4; + monthly = 0; + }; + }) allDestinations; + + clan.core.vars.generators.borgbackup = { + + files."borgbackup.ssh.pub".secret = false; + files."borgbackup.ssh" = { }; + files."borgbackup.repokey" = { }; + + migrateFact = "borgbackup"; + runtimeInputs = [ + pkgs.coreutils + pkgs.openssh + pkgs.xkcdpass + ]; + script = '' + ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh + xkcdpass -n 4 -d - > "$out"/borgbackup.repokey + ''; + }; + + clan.core.backups.providers.borgbackup = { + list = "borgbackup-list"; + create = "borgbackup-create"; + restore = "borgbackup-restore"; + }; + + environment.systemPackages = [ + (pkgs.writeShellApplication { + name = "borgbackup-create"; + runtimeInputs = [ config.systemd.package ]; + text = '' + ${lib.concatMapStringsSep "\n" (dest: '' + systemctl start borgbackup-job-${dest} + '') (lib.attrNames allDestinations)} + ''; + }) + (pkgs.writeShellApplication { + name = "borgbackup-list"; + runtimeInputs = [ pkgs.jq ]; + text = '' + (${ + lib.concatMapStringsSep "\n" ( + dest: + # we need yes here to skip the changed url verification + ''echo y | /run/current-system/sw/bin/borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' '' + ) (lib.attrValues allDestinations) + }) | jq -s 'add // []' + ''; + }) + (pkgs.writeShellApplication { + name = "borgbackup-restore"; + runtimeInputs = [ pkgs.gawk ]; + text = '' + cd / + IFS=':' read -ra FOLDER <<< "''${FOLDERS-}" + job_name=$(echo "$NAME" | awk -F'::' '{print $1}') + backup_name=''${NAME#"$job_name"::} + if [[ ! -x /run/current-system/sw/bin/borg-job-"$job_name" ]]; then + echo "borg-job-$job_name not found: Backup name is invalid" >&2 + exit 1 + fi + echo y | /run/current-system/sw/bin/borg-job-"$job_name" extract "$backup_name" "''${FOLDER[@]}" + ''; + }) + ]; + }; + }; + }; + }; +} diff --git a/clanServices/borgbackup/flake-module.nix b/clanServices/borgbackup/flake-module.nix new file mode 100644 index 000000000..e8eb5dd02 --- /dev/null +++ b/clanServices/borgbackup/flake-module.nix @@ -0,0 +1,6 @@ +{ lib, ... }: +{ + clan.modules = { + borgbackup = lib.modules.importApply ./default.nix { }; + }; +} diff --git a/clanServices/flake-module.nix b/clanServices/flake-module.nix index d8a27aa7b..f430446d8 100644 --- a/clanServices/flake-module.nix +++ b/clanServices/flake-module.nix @@ -4,5 +4,6 @@ ./admin/flake-module.nix ./hello-world/flake-module.nix ./wifi/flake-module.nix + ./borgbackup/flake-module.nix ]; } diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 63f368245..72f5db72f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -83,6 +83,7 @@ nav: - Clan Services: - Overview: reference/clanServices/index.md - reference/clanServices/admin.md + - reference/clanServices/borgbackup.md - reference/clanServices/hello-world.md - reference/clanServices/wifi.md - Clan Modules: diff --git a/docs/nix/render_options/__init__.py b/docs/nix/render_options/__init__.py index 662f19ef3..2bb9c9532 100644 --- a/docs/nix/render_options/__init__.py +++ b/docs/nix/render_options/__init__.py @@ -494,6 +494,9 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](.. output += render_categories( module_info["manifest"]["categories"], fm.categories_info ) + + output += f"{module_info['manifest']['readme']}\n" + output += "\n---\n\n## Roles\n" output += f"The {module_name} module has the following roles:\n\n"