yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'main' of git.clan.lol:RuboGubo/clan-core

Browse Source
This commit is contained in:
RuboGubo
2025-06-03 14:33:12 +01:00
parent 2f0714770a 354ba3df70
commit 5e43876f8e
40 changed files with 805 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
checks/borgbackup-legacy/default.nix Normal file
View File

@@ -0,0 +1,51 @@
(
{ ... }:
{
name = "borgbackup";
nodes.machine =
{ self, pkgs, ... }:
{
imports = [
self.clanModules.borgbackup
self.nixosModules.clanCore
{
services.openssh.enable = true;
services.borgbackup.repos.testrepo = {
authorizedKeys = [ (builtins.readFile ../assets/ssh/pubkey) ];
};
}
{
clan.core.settings.directory = ./.;
clan.core.state.testState.folders = [ "/etc/state" ];
environment.etc.state.text = "hello world";
systemd.tmpfiles.settings."vmsecrets" = {
"/etc/secrets/borgbackup/borgbackup.ssh" = {
C.argument = "${../assets/ssh/privkey}";
z = {
mode = "0400";
user = "root";
};
};
"/etc/secrets/borgbackup/borgbackup.repokey" = {
C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345");
z = {
mode = "0400";
user = "root";
};
};
};
# clan.core.facts.secretStore = "vm";
clan.core.vars.settings.secretStore = "vm";
clan.borgbackup.destinations.test.repo = "borg@localhost:.";
}
];
};
testScript = ''
start_all()
machine.systemctl("start --wait borgbackup-job-test.service")
assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list")
'';
}
)

149
checks/borgbackup/default.nix
View File

@@ -1,51 +1,118 @@
(
{
pkgs,
self,
clanLib,
...
}:
clanLib.test.makeTestClan {
inherit pkgs self;
useContainers = true;
nixosTest = (
{ ... }:
{
name = "borgbackup";
nodes.machine =
{ self, pkgs, ... }:
{
imports = [
self.clanModules.borgbackup
self.nixosModules.clanCore
{
services.openssh.enable = true;
services.borgbackup.repos.testrepo = {
authorizedKeys = [ (builtins.readFile ../assets/ssh/pubkey) ];
};
}
{
clan.core.settings.directory = ./.;
clan.core.state.testState.folders = [ "/etc/state" ];
environment.etc.state.text = "hello world";
systemd.tmpfiles.settings."vmsecrets" = {
"/etc/secrets/borgbackup/borgbackup.ssh" = {
C.argument = "${../assets/ssh/privkey}";
z = {
mode = "0400";
user = "root";
};
};
"/etc/secrets/borgbackup/borgbackup.repokey" = {
C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345");
z = {
mode = "0400";
user = "root";
};
};
};
# clan.core.facts.secretStore = "vm";
clan.core.vars.settings.secretStore = "vm";
clan = {
directory = ./.;
modules."@clan/borgbackup" = ../../clanServices/borgbackup/default.nix;
inventory = {
clan.borgbackup.destinations.test.repo = "borg@localhost:.";
}
];
machines.clientone = { };
machines.serverone = { };
instances = {
borgone = {
module.name = "@clan/borgbackup";
roles.client.machines."clientone" = { };
roles.server.machines."serverone".settings.directory = "/tmp/borg-test";
};
};
};
};
nodes = {
serverone = {
services.openssh.enable = true;
# Needed so PAM doesn't see the user as locked
users.users.borg.password = "borg";
};
clientone =
{ config, pkgs, ... }:
let
dependencies = [
self
pkgs.stdenv.drvPath
] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
in
{
services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keyFiles = [ ../assets/ssh/pubkey ];
clan.core.networking.targetHost = config.networking.hostName;
environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
environment.etc.install-closure.source = "${closureInfo}/store-paths";
nix.settings = {
substituters = pkgs.lib.mkForce [ ];
hashed-mirrors = null;
connect-timeout = pkgs.lib.mkForce 3;
flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
};
system.extraDependencies = dependencies;
clan.core.state.test-backups.folders = [ "/var/test-backups" ];
};
};
testScript = ''
import json
start_all()
machine.systemctl("start --wait borgbackup-job-test.service")
assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list")
machines = [clientone, serverone]
for m in machines:
m.systemctl("start network-online.target")
for m in machines:
m.wait_for_unit("network-online.target")
# dummy data
clientone.succeed("mkdir -p /var/test-backups /var/test-service")
clientone.succeed("echo testing > /var/test-backups/somefile")
clientone.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519")
clientone.succeed("${pkgs.coreutils}/bin/touch /root/.ssh/known_hosts")
clientone.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new localhost hostname")
clientone.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new $(hostname) hostname")
# create
clientone.succeed("borgbackup-create >&2")
clientone.wait_until_succeeds("! systemctl is-active borgbackup-job-serverone >&2")
# list
backup_id = json.loads(clientone.succeed("borg-job-serverone list --json"))["archives"][0]["archive"]
out = clientone.succeed("borgbackup-list").strip()
print(out)
assert backup_id in out, f"backup {backup_id} not found in {out}"
# borgbackup restore
clientone.succeed("rm -f /var/test-backups/somefile")
clientone.succeed(f"NAME='serverone::borg@serverone:.::{backup_id}' borgbackup-restore >&2")
assert clientone.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
'';
}
)
);
}

6
checks/borgbackup/sops/machines/clientone/key.json Executable file
View File

@@ -0,0 +1,6 @@
[
{
"publickey": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c",
"type": "age"
}
]

15
checks/borgbackup/sops/secrets/clientone-age.key/secret Normal file
View File

@@ -0,0 +1,15 @@
{
"data": "ENC[AES256_GCM,data:wCKoKuJo4uXycfqEUYAXDlRRMGJaWgOFiaQa4Wigs0jx1eCI80lP3cEZ1QKyrU/9m9POoZz0JlaKHcuhziTKUqaevHvGfVq2y00=,iv:pH5a90bJbK9Ro6zndNJ18qd4/rU+Tdm+y+jJZtY7UGg=,tag:9lHZJ9C/zIfy8nFrYt9JBQ==,type:str]",
"sops": {
"age": [
{
"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwUDhpd1ZqbWFqR0I3dVFI\nOHlyZnFUYXJnWElrRWhoUHVNMzdKd0VrcGdRCkphQVhuYzlJV0p1MG9MSW5ncWJ3\nREp1OEJxMzQzS2MxTk9aMkJ1a3B0Q0kKLS0tIENweVJ2Tk1yeXlFc2F5cTNIV3F3\nTkRFOVZ1amRIYmg1K3hGWUFSTTl4Wk0KHJRJ7756Msod7Bsmn9SgtwRo53B8Ilp3\nhsAPv+TtdmOD8He9MvGV+BElKEXCsLUwhp/Py6n6CJCczu0VIr8owg==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2025-05-20T13:33:56Z",
"mac": "ENC[AES256_GCM,data:FyfxXhnI6o4SVGJY2e1eMDnfkbMWiCkP4JL/G4PQvzz+c7OIuz8xaa03P3VW7b7o85NP2Tln4FMNTZ0FYtQwd0kKypLUnIxAHsixAHFCv4X8ul1gtZynzgbFbmc0GkfVWW8Lf+U+vvDwT+UrEVfcmksCjdvAOwP26PvlEhYEkSw=,iv:H+VrWYL+kLOLezCZrI8ZgeCsaUdpb7LxDMiLotezVPs=,tag:B/cbPdiEFumGKQHby5inCA==,type:str]",
"unencrypted_suffix": "_unencrypted",
"version": "3.10.2"
}
}

1
checks/borgbackup/sops/secrets/clientone-age.key/users/admin Symbolic link
View File

@@ -0,0 +1 @@
../../../users/admin

4
checks/borgbackup/sops/users/admin/key.json Normal file
View File

@@ -0,0 +1,4 @@
{
"publickey": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
"type": "age"
}

1
checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/machines/clientone Symbolic link
View File

@@ -0,0 +1 @@
../../../../../../sops/machines/clientone

19
checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/secret Normal file
View File

@@ -0,0 +1,19 @@
{
"data": "ENC[AES256_GCM,data:52vY68gqbwiZRMUBKc9SeXR06fuKAhuAPciLpxXgEOxI,iv:Y34AVoHaZzRiFFTDbekXP1X3W8zSXJmzVCYODYkdxnY=,tag:8WQaGEHQKT/n+auHUZCE0w==,type:str]",
"sops": {
"age": [
{
"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOdUFUZUZ2M00zTGlhNjF4\nL0VlMVY4Z2xMbWRWR29zZlFwdm1XRk12NGtBCnkrb3A4M3BkalMyeWdDaUdQdStt\nUWY3SXJROXdpRzN0NlBJNEpjTEZ0aFkKLS0tIGZkMGhsTXB2RnRqVHVrUFQwL2lw\nZnBreWhWa3Jrcm4yOXBiaUlPWFM1aDAKRE+Zzrja7KeANEJUbmFYuVoO3qGyi4iH\n0cfH0W8irRe9vsKMXz7YJxtByYLwRulrT8tXtElHvIEVJG0mwwaf0Q==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsNEljUFdnQ0tTQ1IxZ2Zo\nYkc4V2dCaUk0YXh5SzlSazhsRTVKVzFvVXhFCkRyMlMxR3EyWEZIRzFQV3d2dVpz\na3NPbk9XdWR1NmtMQlZsNlBuU0NkQWMKLS0tIDlDYzMzOExVL1g5SVRHYlpUQlBV\na2lpdTUwaEd4OXhWUWxuV04xRVVKNHcK9coohAD1IoarLOXSGg3MIRXQ3BsTIA4y\nKrcS/PxITKJs7ihg93RZin70R79Qsij1RHZLKGfgGJ67i8ZCxc4N0g==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2025-05-20T13:33:59Z",
"mac": "ENC[AES256_GCM,data:eABMaIe07dwAMMlgrIUUpfpj73q1H5Keafql91MBQ5NN9Znr5lI/ennQsQsuLO8ZTCC34US/MJndliW34SqVM9y53p0jjPzqBxSKYq74iNcBz7+TxbjlY1aapgTRPr6Ta8I/5loohnxlHqjvLL70ZzfbChDN0/4jZsDVXYNfbIk=,iv:41Mz2u40JN0iE5zPUK6siaxo0rTtlk7fGWq7TF5NyUI=,tag:1A+h6XPH7DeQ6kxGDV3PgQ==,type:str]",
"unencrypted_suffix": "_unencrypted",
"version": "3.10.2"
}
}

1
checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/users/admin Symbolic link
View File

@@ -0,0 +1 @@
../../../../../../sops/users/admin

1
checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh.pub/value Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE3clYF6BDZ0PxfDdprx7YYM4U4PKEZkWUuhpre0wb7w nixbld@kiwi

1
checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/machines/clientone Symbolic link
View File

@@ -0,0 +1 @@
../../../../../../sops/machines/clientone

19
checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/secret Normal file
View File

@@ -0,0 +1,19 @@
{
"data": "ENC[AES256_GCM,data:tAjfBW75XDS8lfJCf/+9rPYH3aMjRX1nmdN5dPMxnrlhuEPM3Smv9AM93Tz36k7BKk31bUWcV/99ax+KaIK1Rzgym/CwKGGxIUziuVOEOwrCOBeOw7amZ9YGsgiLUTLIhoeO6SjfdZ4q2JxGPw7KqNfUM9kiZT01vx5JTLa24JdvBKpizbtHRlL1lappTRVt0dG2WhT9/YhQUGu9ZFqPs8+bPOBclc78qjCm2DAPgsprK+JCBuq+r+qHgAx4Ee1QHI7FC39e5NeGBTBeZfZ5d95+0klKuTx9FCPs6QRBkQ0tN29OpwzkdSuRAXGGHpzPkZ+FupbETtSQWCmnjma6jPzEl8oDUTWooKK0mUEz8icvTQvRfyM3Qt3mQpkX3e0rTEbZzoLdWCwTufP/tRQNDCWvI/NV7OjIHpNPjymqE5uPmiBpA6y6hhCH7zL1eDo11ICSIX3hkyFJH2svvFQn6oLrPAoByvNutfetKhd8z7NFpVeIOWwtuPzO7wU5M7zESHww0JF78vjFwimQYYhQ,iv:fVjeVez4dTGSrANi5ZeP9PJhsSySqeqqJzBDbd0gFW4=,tag:Aa89+bWLljxV1tlSHtpddw==,type:str]",
"sops": {
"age": [
{
"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVaW94M3VwcFJ2elcrRGlv\nUGdzVk9vU2ZweFpIVVlIRUEyRVlSMlEyeHpVCnJuV0xIS3hMLy9IbG92S0pvL2RP\nL0J0WkVuWVhQdldHekdYNTVXdFkrUlEKLS0tIFQzdGErZVBwQUFNMXErbDBQVURZ\naHlsY2hDa1Zud1E2dFh0ZHl4VEJ2S0kKVABqwRcCUTcsBInfo9CpFtoM3kl4KMyU\nGXDjHOSjlX5df7OKZAvYukgX7Q2penvq+Fq4fa4A1Cmkqga7cHdJ+A==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnbHRSVEg3Vi9qTnAwWGF6\nbEdIR2gvZ2laZnJMbVF3NjcvN25OdXF3WXowCnVUODdEa1NWU3JISXlrNldOMjVi\ndUlMTVdBaWxvZHlwSTdJY3NCcll4SjAKLS0tIEp6ZVlDTklqVXdNYzJ2dElCR21o\nUWphMDdyVVppVnFHOVlHZTNtajZzOXMKRB61lUrAkUXSYl3ffOOK8k4QgLA4bFln\naQ7GOol8f8W5H68zXBMZrhjP6k4kZDfknc9jgyoWM7jaZNSWC5J19Q==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2025-05-20T13:33:59Z",
"mac": "ENC[AES256_GCM,data:NjVpDweqxTSQGt9VKR/CMfvbvHQJHCi8P7XbOuKLZKQ4GVoeZ5r4PsC6nxKHHikN6YL1oJCmaSxr0mJRk/sFZg/+wdW8L7F5aQeFRiWo9jCjH0MDMnfiu5a0xjRt21uPl/7LUJ9jNon5nyxPTlZMeYSvTP2Q9spnNuN8vqipP68=,iv:DPvbN9IvWiUfxiJk6mey/us8N1GGVJcSJrT8Bty4kB4=,tag:+emK8uSkfIGUXoYpaWeu3A==,type:str]",
"unencrypted_suffix": "_unencrypted",
"version": "3.10.2"
}
}

1
checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/users/admin Symbolic link
View File

@@ -0,0 +1 @@
../../../../../../sops/users/admin

4
checks/clan-core-for-checks.nix
View File

@@ -1,6 +1,6 @@
{ fetchgit }:
fetchgit {
url = "https://git.clan.lol/clan/clan-core.git";
rev = "1523ac18c9c575d32033dcf1e769fccc324f248e";
sha256 = "0nxhw5s9lva4g1rgx6pgczh3vxrskmmlxay48wvn2pnkrlvhr9j8";
rev = "13a9b1719835ef4510e4adb6941ddfe9a91d41cb";
sha256 = "sha256-M+pLnpuX+vIsxTFtbBZaNA1OwGQPeSbsMbTiDl1t4vY=";
}

3
checks/flake-module.nix
View File

@@ -41,7 +41,7 @@ in
# Base Tests
secrets = self.clanLib.test.baseTest ./secrets nixosTestArgs;
borgbackup = self.clanLib.test.baseTest ./borgbackup nixosTestArgs;
borgbackup-legacy = self.clanLib.test.baseTest ./borgbackup-legacy nixosTestArgs;
wayland-proxy-virtwl = self.clanLib.test.baseTest ./wayland-proxy-virtwl nixosTestArgs;
# Container Tests
@@ -53,6 +53,7 @@ in
# Clan Tests
dummy-inventory-test = import ./dummy-inventory-test nixosTestArgs;
admin = import ./admin nixosTestArgs;
borgbackup = import ./borgbackup nixosTestArgs;
data-mesher = import ./data-mesher nixosTestArgs;
syncthing = import ./syncthing nixosTestArgs;
}

2
clanModules/borgbackup/README.md
View File

@@ -1,7 +1,7 @@
---
description = "Efficient, deduplicating backup program with optional compression and secure encryption."
categories = ["System"]
features = [ "inventory" ]
features = [ "inventory", "deprecated" ]
---
BorgBackup (short: Borg) gives you:

3
clanModules/borgbackup/roles/client.nix
View File

@@ -106,7 +106,8 @@ in
systemd.services = lib.mapAttrs' (
_: dest:
lib.nameValuePair "borgbackup-job-${dest.name}" {
# since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
# since borgbackup mounts the system read-only, we need to run in a
# ExecStartPre script, so we can generate additional files.
serviceConfig.ExecStartPre = [
''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}''
];

9
clanServices/borgbackup/README.md Normal file
View File

@@ -0,0 +1,9 @@
BorgBackup (short: Borg) gives you:
- Space efficient storage of backups.
- Secure, authenticated encryption.
- Compression: lz4, zstd, zlib, lzma or none.
- Mountable backups with FUSE.
- Easy installation on multiple platforms: Linux, macOS, BSD, …
- Free software (BSD license).
- Backed by a large and active open-source community.

313
clanServices/borgbackup/default.nix Normal file
View File

@@ -0,0 +1,313 @@
{ ... }:
{
_class = "clan.service";
manifest.name = "borgbackup";
manifest.description = "Efficient, deduplicating backup program with optional compression and secure encryption.";
manifest.categories = [ "System" ];
manifest.readme = builtins.readFile ./README.md;
# TODO: a client can only be in one instance, add constraint
roles.server = {
interface =
{ lib, ... }:
{
options.directory = lib.mkOption {
type = lib.types.str;
default = "/var/lib/borgbackup";
description = ''
The directory where the borgbackup repositories are stored.
'';
};
};
perInstance =
{
roles,
settings,
...
}:
{
nixosModule =
{
config,
...
}:
{
config.services.openssh.enable = true;
config.services.borgbackup.repos =
let
borgbackupIpMachinePath =
machine:
config.clan.core.settings.directory
+ "/vars/per-machine/${machine}/borgbackup/borgbackup.ssh.pub/value";
hosts = builtins.mapAttrs (machineName: _machineSettings: {
# name = "${instanceName}-${machineName}";
# value = {
path = "${settings.directory}/${machineName}";
authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machineName)) ];
# };
# }) machinesWithKey;
}) roles.client.machines;
in
hosts;
};
};
};
roles.client = {
interface =
{
lib,
...
}:
{
options.destinations = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options = {
name = lib.mkOption {
type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
default = name;
description = "the name of the backup job";
};
repo = lib.mkOption {
type = lib.types.str;
description = "the borgbackup repository to backup to";
};
rsh = lib.mkOption {
type = lib.types.str;
defaultText = "ssh -i \${config.clan.core.vars.generators.borgbackup.files.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
description = "the rsh to use for the backup";
};
};
}
)
);
default = { };
description = ''
external destinations where the machine should be backuped to
'';
};
options.exclude = lib.mkOption {
type = lib.types.listOf lib.types.str;
example = [ "*.pyc" ];
default = [ ];
description = ''
Directories/Files to exclude from the backup.
Use * as a wildcard.
'';
};
};
perInstance =
{
extendSettings,
roles,
...
}:
{
nixosModule =
{
config,
lib,
pkgs,
...
}:
let
settings = extendSettings {
# Adding default value with option merging, because it depends on
# generators, which we can reference here.
options.destinations = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule {
options = {
rsh = lib.mkOption {
default = "ssh -i ${
config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh".path
} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes -o PasswordAuthentication=no";
};
};
}
);
};
};
in
{
config =
let
preBackupScript = ''
declare -A preCommandErrors
${lib.concatMapStringsSep "\n" (
state:
lib.optionalString (state.preBackupCommand != null) ''
echo "Running pre-backup command for ${state.name}"
if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
preCommandErrors["${state.name}"]=1
fi
''
) (lib.attrValues config.clan.core.state)}
if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
echo "pre-backup commands failed for the following services:"
for state in "''${!preCommandErrors[@]}"; do
echo " $state"
done
exit 1
fi
'';
# The destinations from server.roles.machines.*
# name is the server, machine can only be in one instance
internalDestinations =
let
destinations = builtins.map (serverName: {
name = "${serverName}";
value = {
# inherit name;
name = "${serverName}";
repo = "borg@${serverName}:.";
# rsh = "";
rsh = "ssh -i ${
config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh".path
} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes";
};
}) (builtins.attrNames roles.server.machines);
in
(builtins.listToAttrs destinations);
# The destinations specified via roles.client.machines.*.settings.destinations.<name>
# name is the <name>
externalDestinations = lib.mapAttrs' (
name: dest: lib.nameValuePair name dest
) settings.destinations;
allDestinations =
lib.warnIf ((builtins.intersectAttrs externalDestinations internalDestinations) != { })
"You are overwriting an internalDestinations through an externalDestination configuration."
(internalDestinations // externalDestinations);
in
{
services.openssh.enable = true;
# Derived from the destinations
systemd.services = lib.mapAttrs' (
destName: _dest:
lib.nameValuePair "borgbackup-job-${destName}" {
# since borgbackup mounts the system read-only, we need to
# run in a ExecStartPre script, so we can generate
# additional files.
serviceConfig.ExecStartPre = [
''+${pkgs.writeShellScript "borgbackup-job-${destName}-pre-backup-commands" preBackupScript}''
];
}
) allDestinations;
services.borgbackup.jobs = lib.mapAttrs (_: dest: {
paths = lib.unique (
lib.flatten (map (state: state.folders) (lib.attrValues config.clan.core.state))
);
exclude = settings.exclude;
repo = dest.repo;
environment.BORG_RSH = dest.rsh;
compression = "auto,zstd";
startAt = "*-*-* 01:00:00";
persistentTimer = true;
encryption = {
mode = "repokey";
passCommand = "cat ${config.clan.core.vars.generators.borgbackup.files."borgbackup.repokey".path}";
};
prune.keep = {
within = "1d"; # Keep all archives from the last day
daily = 7;
weekly = 4;
monthly = 0;
};
}) allDestinations;
clan.core.vars.generators.borgbackup = {
files."borgbackup.ssh.pub".secret = false;
files."borgbackup.ssh" = { };
files."borgbackup.repokey" = { };
migrateFact = "borgbackup";
runtimeInputs = [
pkgs.coreutils
pkgs.openssh
pkgs.xkcdpass
];
script = ''
ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh
xkcdpass -n 4 -d - > "$out"/borgbackup.repokey
'';
};
clan.core.backups.providers.borgbackup = {
list = "borgbackup-list";
create = "borgbackup-create";
restore = "borgbackup-restore";
};
environment.systemPackages = [
(pkgs.writeShellApplication {
name = "borgbackup-create";
runtimeInputs = [ config.systemd.package ];
text = ''
${lib.concatMapStringsSep "\n" (dest: ''
systemctl start borgbackup-job-${dest}
'') (lib.attrNames allDestinations)}
'';
})
(pkgs.writeShellApplication {
name = "borgbackup-list";
runtimeInputs = [ pkgs.jq ];
text = ''
(${
lib.concatMapStringsSep "\n" (
dest:
# we need yes here to skip the changed url verification
''echo y | /run/current-system/sw/bin/borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' ''
) (lib.attrValues allDestinations)
}) | jq -s 'add // []'
'';
})
(pkgs.writeShellApplication {
name = "borgbackup-restore";
runtimeInputs = [ pkgs.gawk ];
text = ''
cd /
IFS=':' read -ra FOLDER <<< "''${FOLDERS-}"
job_name=$(echo "$NAME" | awk -F'::' '{print $1}')
backup_name=''${NAME#"$job_name"::}
if [[ ! -x /run/current-system/sw/bin/borg-job-"$job_name" ]]; then
echo "borg-job-$job_name not found: Backup name is invalid" >&2
exit 1
fi
echo y | /run/current-system/sw/bin/borg-job-"$job_name" extract "$backup_name" "''${FOLDER[@]}"
'';
})
];
};
};
};
};
}

6
clanServices/borgbackup/flake-module.nix Normal file
View File

@@ -0,0 +1,6 @@
{ lib, ... }:
{
clan.modules = {
borgbackup = lib.modules.importApply ./default.nix { };
};
}

1
clanServices/flake-module.nix
View File

@@ -4,5 +4,6 @@
./admin/flake-module.nix
./hello-world/flake-module.nix
./wifi/flake-module.nix
./borgbackup/flake-module.nix
];
}

1
docs/mkdocs.yml
View File

@@ -83,6 +83,7 @@ nav:
- Clan Services:
- Overview: reference/clanServices/index.md
- reference/clanServices/admin.md
- reference/clanServices/borgbackup.md
- reference/clanServices/hello-world.md
- reference/clanServices/wifi.md
- Clan Modules:

3
docs/nix/render_options/__init__.py
View File

@@ -494,6 +494,9 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](..
output += render_categories(
module_info["manifest"]["categories"], fm.categories_info
)
output += f"{module_info['manifest']['readme']}\n"
output += "\n---\n\n## Roles\n"
output += f"The {module_name} module has the following roles:\n\n"

26
flake.lock generated
View File

@@ -16,11 +16,11 @@
]
},
"locked": {
"lastModified": 1748244631,
"narHash": "sha256-fLJu837n0aP6ky+3qYUCaQN2ZESOh2Tvho8RA73DLZw=",
"rev": "f52e3eef263617f5b15a4486720e98a7aada8de3",
"lastModified": 1748824882,
"narHash": "sha256-DnBR3hpUtaEtidCTIyiPzTfXsrY5huYo6ny6XIxaZFs=",
"rev": "bca54baa18fcbfb73dada430cfdac8e55c0532a4",
"type": "tarball",
"url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/f52e3eef263617f5b15a4486720e98a7aada8de3.tar.gz"
"url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/bca54baa18fcbfb73dada430cfdac8e55c0532a4.tar.gz"
},
"original": {
"type": "tarball",
@@ -34,11 +34,11 @@
]
},
"locked": {
"lastModified": 1748225455,
"narHash": "sha256-AzlJCKaM4wbEyEpV3I/PUq5mHnib2ryEy32c+qfj6xk=",
"lastModified": 1748832438,
"narHash": "sha256-/CtyLVfNaFP7PrOPrTEuGOJBIhcBKVQ91KiEbtXJi0A=",
"owner": "nix-community",
"repo": "disko",
"rev": "a894f2811e1ee8d10c50560551e50d6ab3c392ba",
"rev": "58d6e5a83fff9982d57e0a0a994d4e5c0af441e4",
"type": "github"
},
"original": {
@@ -54,11 +54,11 @@
]
},
"locked": {
"lastModified": 1743550720,
"narHash": "sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY=",
"lastModified": 1748821116,
"narHash": "sha256-F82+gS044J1APL0n4hH50GYdPRv/5JWm34oCJYmVKdE=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "c621e8422220273271f52058f618c94e405bb0f5",
"rev": "49f0870db23e8c1ca0b5259734a02cd9e1e371a1",
"type": "github"
},
"original": {
@@ -118,10 +118,10 @@
"nixpkgs": {
"locked": {
"lastModified": 315532800,
"narHash": "sha256-A6ddIFRZ6y1IQQY0Gu868W9le0wuJfI1qYcVpq++f+I=",
"rev": "bdac72d387dca7f836f6ef1fe547755fb0e9df61",
"narHash": "sha256-zZGnqFOaEHRxiSAqTnBHQ+HNZCxMLumNzqtxRMhS4bk=",
"rev": "5929de975bcf4c7c8d8b5ca65c8cd9ef9e44523e",
"type": "tarball",
"url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre805949.bdac72d387dc/nixexprs.tar.xz"
"url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre809520.5929de975bcf/nixexprs.tar.xz"
},
"original": {
"type": "tarball",

5
lib/inventory/build-inventory/builder/default.nix
View File

@@ -50,7 +50,10 @@ let
{
machineImports = [
(lib.optionalAttrs (machineConfig.deploy.targetHost or null != null) {
config.clan.core.networking.targetHost = machineConfig.deploy.targetHost;
config.clan.core.networking.targetHost = lib.mkForce machineConfig.deploy.targetHost;
})
(lib.optionalAttrs (machineConfig.deploy.buildHost or null != null) {
config.clan.core.networking.buildHost = lib.mkForce machineConfig.deploy.buildHost;
})
];
assertions = { };

7
lib/inventory/build-inventory/interface.nix
View File

@@ -347,7 +347,12 @@ in
type = types.listOf types.str;
};
deploy.targetHost = lib.mkOption {
description = "Configuration for the deployment of the machine";
description = "SSH address of the host to deploy the machine to";
default = null;
type = types.nullOr types.str;
};
deploy.buildHost = lib.mkOption {
description = "SSH address of the host to build the machine on";
default = null;
type = types.nullOr types.str;
};

2
pkgs/clan-app/ui/.storybook/preview.ts
View File

@@ -1,5 +1,7 @@
import type { Preview } from "@kachurun/storybook-solid";
import "../src/index.css";
export const preview: Preview = {
tags: ["autodocs"],
parameters: {

18
pkgs/clan-app/ui/src/components/Button/Button.stories.tsx
View File

@@ -1,6 +1,6 @@
import type { Meta, StoryObj } from "@kachurun/storybook-solid";
import { Button, ButtonProps } from "./Button";
import FlashIcon from "@/icons/flash.svg";
import Icon from "../icon";
const meta: Meta<ButtonProps> = {
title: "Components/Button",
@@ -12,12 +12,10 @@ export default meta;
type Story = StoryObj<ButtonProps>;
const children = "click me";
const startIcon = <FlashIcon width={16} height={16} viewBox="0 0 48 48" />;
export const Default: Story = {
args: {
children,
startIcon,
},
};
@@ -41,3 +39,17 @@ export const Ghost: Story = {
variant: "ghost",
},
};
export const StartIcon: Story = {
args: {
...Default.args,
startIcon: <Icon size={12} icon="Flash" />,
},
};
export const EndIcon: Story = {
args: {
...Default.args,
endIcon: <Icon size={12} icon="Flash" />,
},
};

7
pkgs/clan-app/ui/src/components/TagList/TagList.css Normal file
View File

@@ -0,0 +1,7 @@
div.tag-list {
@apply flex flex-wrap gap-2;
span.tag {
@apply w-fit rounded-full px-3 py-2 bg-inv-4 fg-inv-1;
}
}

36
pkgs/clan-app/ui/src/components/TagList/TagList.stories.tsx Normal file
View File

@@ -0,0 +1,36 @@
import type { Meta, StoryObj } from "@kachurun/storybook-solid";
import { TagList, TagListProps } from "./TagList";
const meta: Meta<TagListProps> = {
title: "Components/TagList",
component: TagList,
decorators: [
// wrap in a fixed width div so we can check that it wraps
(Story) => {
return (
<div style={{ width: "20em" }}>
<Story />
</div>
);
},
],
};
export default meta;
type Story = StoryObj<TagListProps>;
export const Default: Story = {
args: {
values: [
"Titan",
"Enceladus",
"Mimas",
"Dione",
"Iapetus",
"Tethys",
"Hyperion",
"Epimetheus",
],
},
};

21
pkgs/clan-app/ui/src/components/TagList/TagList.tsx Normal file
View File

@@ -0,0 +1,21 @@
import { Component, For } from "solid-js";
import { Typography } from "@/src/components/Typography";
import "./TagList.css";
export interface TagListProps {
values: string[];
}
export const TagList: Component<TagListProps> = (props) => {
return (
<div class="tag-list">
<For each={props.values}>
{(tag) => (
<Typography hierarchy="label" size="s" inverted={true} class="tag">
{tag}
</Typography>
)}
</For>
</div>
);
};

2
pkgs/clan-app/ui/src/components/inputBase/index.tsx
View File

@@ -137,7 +137,7 @@ export const InputLabel = (props: InputLabelProps) => {
weight="bold"
size="xs"
>
{"∗"}
<>&#42;</>
</Typography>
)}
{props.help && (

28
pkgs/clan-app/ui/src/routes/machines/details.tsx
View File

@@ -32,6 +32,7 @@ import {
FileSelectorField,
} from "@/src/components/fileSelect";
import { useClanContext } from "@/src/contexts/clan";
import { TagList } from "@/src/components/TagList/TagList";
type MachineFormInterface = MachineData & {
sshKey?: File;
@@ -77,10 +78,12 @@ const LoadingBar = () => (
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
interface InstallMachineProps {
name?: string;
machine: MachineData;
}
const InstallMachine = (props: InstallMachineProps) => {
const { activeClanURI } = useClanContext();
@@ -381,6 +384,7 @@ const InstallMachine = (props: InstallMachineProps) => {
interface MachineDetailsProps {
initialData: MachineData;
}
const MachineForm = (props: MachineDetailsProps) => {
const [formStore, { Form, Field }] =
// TODO: retrieve the correct initial values from API
@@ -595,28 +599,25 @@ const MachineForm = (props: MachineDetailsProps) => {
</Field>
<Field name="machine.tags" type="string[]">
{(field, props) => (
<div class="flex items-center gap-4">
<Typography hierarchy="label" size="default" weight="bold">
Tags{" "}
</Typography>
<For each={field.value}>
{(tag) => (
<span class="mx-2 w-fit rounded-full px-3 py-0.5 bg-inv-4 fg-inv-1">
<div class="grid grid-cols-10 items-center">
<Typography
hierarchy="label"
size="s"
inverted={true}
size="default"
weight="bold"
class="col-span-5"
>
{tag}
Tags{" "}
</Typography>
</span>
)}
</For>
<div class="col-span-5 justify-self-end">
{/* alphabetically sort the tags */}
<TagList values={[...(field.value || [])].sort()} />
</div>
</div>
)}
</Field>
</Fieldset>
<Typography hierarchy={"body"} size={"s"}>
<Fieldset legend="Hardware">
<Field name="hw_config">
{(field, props) => (
@@ -638,6 +639,7 @@ const MachineForm = (props: MachineDetailsProps) => {
)}
</Field>
</Fieldset>
</Typography>
<Accordion title="Connection Settings">
<Fieldset>

2
pkgs/clan-cli/clan_lib/api/modules.py
View File

@@ -149,7 +149,7 @@ class ModuleManifest(TypedDict):
@dataclass
class ModuleInfo:
class ModuleInfo(TypedDict):
manifest: ModuleManifest
roles: dict[str, None]

12
pkgs/clan-cli/clan_lib/nix/__init__.py
View File

@@ -119,6 +119,13 @@ class Packages:
else:
allowed_packages = cls.allowed_packages
if "#" in package:
log.warning(
"Allowing package %s for debugging as it looks like a flakeref",
package,
)
return
if package not in allowed_packages:
msg = f"Package not allowed: '{package}', allowed packages are:\n{'\n'.join(allowed_packages)}"
raise ClanError(msg)
@@ -133,6 +140,9 @@ class Packages:
os.environ.get("CLAN_PROVIDED_PACKAGES", "").split(":")
)
if "#" in program:
return True
if program in cls.static_packages:
if shutil.which(program) is None:
log.warning(
@@ -157,7 +167,7 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
f"nixpkgs#{package}"
for package in packages
if not Packages.is_provided(package)
]
] + [package for package in packages if "#" in package]
if not missing_packages:
return cmd
return [

2
pkgs/clan-cli/clan_lib/nix_models/clan.py
View File

@@ -58,9 +58,11 @@ class InventoryInstance(TypedDict):
InventoryMachineDeployBuildhostType = str | None
InventoryMachineDeployTargethostType = str | None
class InventoryMachineDeploy(TypedDict):
buildHost: NotRequired[InventoryMachineDeployBuildhostType]
targetHost: NotRequired[InventoryMachineDeployTargethostType]

2
pkgs/clan-cli/clan_lib/nix_models/inventory.py
View File

@@ -58,9 +58,11 @@ class Instance(TypedDict):
MachineDeployBuildhostType = str
MachineDeployTargethostType = str
class MachineDeploy(TypedDict):
buildHost: NotRequired[MachineDeployBuildhostType]
targetHost: NotRequired[MachineDeployTargethostType]

27
pkgs/clan-cli/clan_lib/ssh/remote.py
View File

@@ -271,13 +271,6 @@ class Remote:
self,
control_master: bool = True,
) -> list[str]:
effective_control_path_dir = self._control_path_dir
if self._control_path_dir is None and not control_master:
effective_control_path_dir = None
elif self._control_path_dir is None and control_master:
msg = "Bug! Control path directory is not set. Please use Remote.ssh_control_master() or set control_master to false."
raise ClanError(msg)
ssh_opts = ["-A"] if self.forward_agent else []
if self.port:
ssh_opts.extend(["-p", str(self.port)])
@@ -287,11 +280,21 @@ class Remote:
if self.private_key:
ssh_opts.extend(["-i", str(self.private_key)])
if effective_control_path_dir:
socket_path = effective_control_path_dir / "socket"
ssh_opts.extend(["-o", "ControlPersist=30m"])
ssh_opts.extend(["-o", f"ControlPath={socket_path}"])
ssh_opts.extend(["-o", "ControlMaster=auto"])
if control_master:
if self._control_path_dir is None:
msg = "Bug! Control path directory is not set. Please use Remote.ssh_control_master() or set control_master to False."
raise ClanError(msg)
socket_path = self._control_path_dir / "socket"
ssh_opts.extend(
[
"-o",
"ControlMaster=auto",
"-o",
"ControlPersist=30m",
"-o",
f"ControlPath={socket_path}",
]
)
return ssh_opts
def ssh_cmd(

87
pkgs/clan-cli/clan_lib/tests/test_create.py
View File

@@ -4,7 +4,7 @@ import shutil
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from typing import Any, cast
import clan_cli.clan.create
import pytest
@@ -17,15 +17,23 @@ from clan_cli.ssh.host_key import HostKeyCheck
from clan_cli.vars.generate import generate_vars_for_machine, get_generators_closure
from clan_lib.api.disk import hw_main_disk_options, set_machine_disk_schema
from clan_lib.api.modules import list_modules
from clan_lib.api.network import check_machine_online
from clan_lib.cmd import RunOpts, run
from clan_lib.dirs import specific_machine_dir
from clan_lib.errors import ClanError
from clan_lib.flake import Flake
from clan_lib.inventory import InventoryStore
from clan_lib.machines.machines import Machine
from clan_lib.nix import nix_command
from clan_lib.nix_models.clan import InventoryMachine
from clan_lib.nix_models.clan import (
InventoryInstancesType,
InventoryMachine,
InventoryServicesType,
Unknown,
)
from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy
from clan_lib.persist.util import set_value_by_path
from clan_lib.ssh.remote import Remote
log = logging.getLogger(__name__)
@@ -33,8 +41,8 @@ log = logging.getLogger(__name__)
@dataclass
class InventoryWrapper:
services: dict[str, Any]
instances: dict[str, Any]
services: InventoryServicesType
instances: InventoryInstancesType
@dataclass
@@ -58,16 +66,6 @@ def create_base_inventory(ssh_keys_pairs: list[SSHKeyPair]) -> InventoryWrapper:
"""Create the base inventory structure."""
legacy_services: dict[str, Any] = {
"sshd": {
"someid": {
"roles": {
"server": {
"tags": ["all"],
"config": {},
}
}
}
},
"state-version": {
"someid": {
"roles": {
@@ -78,21 +76,27 @@ def create_base_inventory(ssh_keys_pairs: list[SSHKeyPair]) -> InventoryWrapper:
}
},
}
instances = {
"admin-1": {
"module": {"name": "admin"},
instances = InventoryInstancesType(
{
"admin-inst": {
"module": {"name": "admin", "input": "clan-core"},
"roles": {
"default": {
"tags": {"all": {}},
"settings": {
"settings": cast(
Unknown,
{
"allowedKeys": {
key.username: key.ssh_pubkey_txt for key in ssh_keys
}
},
},
),
},
},
}
}
)
return InventoryWrapper(services=legacy_services, instances=instances)
@@ -187,7 +191,6 @@ def test_clan_create_api(
clan_dir_flake, inv_machine, target_host=f"{host.target}:{ssh_port_var}"
)
)
machine = Machine(
name=vm_name,
flake=clan_dir_flake,
@@ -203,22 +206,35 @@ def test_clan_create_api(
result = check_machine_online(machine)
assert result == "Online", f"Machine {machine.name} is not online"
# ssh_keys = [
# SSHKeyPair(
# private=private_key,
# public=public_key,
# )
# ]
ssh_keys = [
SSHKeyPair(
private=private_key,
public=public_key,
)
]
# ===== CREATE BASE INVENTORY ======
# TODO(@Qubasa): This seems unused?
# inventory = create_base_inventory(ssh_keys)
# patch_inventory_with(Flake(str(dest_clan_dir)), "services", inventory.services)
inventory_conf = create_base_inventory(ssh_keys)
store = InventoryStore(clan_dir_flake)
inventory = store.read()
modules = list_modules(str(clan_dir_flake.path))
assert (
modules["modulesPerSource"]["clan-core"]["admin"]["manifest"]["name"]
== "clan-core/admin"
)
set_value_by_path(inventory, "services", inventory_conf.services)
set_value_by_path(inventory, "instances", inventory_conf.instances)
store.write(
inventory,
"base config",
)
# Invalidate cache because of new inventory
clan_dir_flake.invalidate_cache()
generators = get_generators_closure(machine.name, dest_clan_dir)
generators = get_generators_closure(machine.name, machine.flake.path)
all_prompt_values = {}
for generator in generators:
prompt_values = {}
@@ -232,9 +248,9 @@ def test_clan_create_api(
all_prompt_values[generator.name] = prompt_values
generate_vars_for_machine(
machine.name,
machine_name=machine.name,
base_dir=machine.flake.path,
generators=[gen.name for gen in generators],
base_dir=dest_clan_dir,
all_prompt_values=all_prompt_values,
)
@@ -269,7 +285,6 @@ def test_clan_create_api(
set_machine_disk_schema(machine, "single-disk", placeholders)
clan_dir_flake.invalidate_cache()
# @Qubasa what does this assert check, why does it raise?
# with pytest.raises(ClanError) as exc_info:
# machine.build_nix("config.system.build.toplevel")
# assert "nixos-system-test-clan" in str(exc_info.value)
with pytest.raises(ClanError) as exc_info:
machine.build_nix("config.system.build.toplevel")
assert "nixos-system-test-clan" in str(exc_info.value)

1
pkgs/clan-cli/flake-module.nix
View File

@@ -22,6 +22,7 @@
"nixosModules"
"flake.lock"
"templates"
"clanServices"
];
};
};