clan-cli: add --json and --png flags to machine install

Allow using `clan machines [machine] install` in conjunction with the
`--json` and `--png` flags.

Fixes #1180

.envrc

@@ -2,4 +2,10 @@ if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then
   source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4="
 fi
 
+watch_file .direnv/selected-shell 
+
+if [ -e .direnv/selected-shell ]; then
+  use flake .#$(cat .direnv/selected-shell)
+else
   use flake
+fi

.gitea/workflows/checks-impure.yaml

@@ -1,12 +0,0 @@
-name: checks-impure
-on:
-  pull_request:
-  push:
-    branches: main
-jobs:
-  test:
-    if: ${{ github.actor != 'ui-asset-bot' }}
-    runs-on: nix
-    steps:
-      - uses: actions/checkout@v3
-      - run: nix run .#impure-checks

.gitea/workflows/checks.yaml

@@ -2,11 +2,16 @@ name: checks
 on:
   pull_request:
   push:
-    branches: main
+    branches:
+      - main
 jobs:
-  test:
-    if: ${{ github.actor != 'ui-asset-bot' }}
+  checks:
     runs-on: nix
     steps:
       - uses: actions/checkout@v3
-      - run: nix run --refresh github:Mic92/nix-fast-build -- --no-nom --eval-workers 20
+      - run: nix run --refresh github:Mic92/nix-fast-build -- --no-nom --eval-workers 10
+  checks-impure:
+    runs-on: nix
+    steps:
+      - uses: actions/checkout@v3
+      - run: nix run .#impure-checks

.gitignore

@@ -9,9 +9,9 @@ example_clan
 result*
 /pkgs/clan-cli/clan_cli/nixpkgs
 /pkgs/clan-cli/clan_cli/webui/assets
-/machines
 nixos.qcow2
 **/*.glade~
+/docs/out
 
 # python
 __pycache__
@@ -21,3 +21,10 @@ __pycache__
 .reports
 .ruff_cache
 htmlcov
+
+# flatpak
+.flatpak-builder
+build
+build-dir
+repo
+.env

CONTRIBUTING.md

@@ -0,0 +1,21 @@
+# Contributing to cLAN
+
+## Live-reloading documentation
+
+Enter the `docs` directory:
+
+```shell-session
+cd docs
+```
+
+Enter the development shell or enable `direnv`:
+
+```shell-session
+direnv allow
+```
+
+Run a local server:
+
+```shell-session
+mkdocs serve
+```

README.md

@@ -1,23 +1,45 @@
 # cLAN Core Repository
 
-Welcome to the cLAN Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo houses all the essential packages, NixOS modules, CLI tools, and tests you need to contribute and work with the cLAN project.
+Welcome to the cLAN Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the cLAN project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
 
-## Getting Started
+## Why Clan?
 
-If you're new to cLAN and eager to dive in, start with our quickstart guide:
+Our mission is simple: to democratize computing by providing tools that empower users, foster innovation, and challenge outdated paradigms. Clan represents our contribution to a future where technology serves humanity, not the other way around. By participating in Clan, you're joining a movement dedicated to creating a secure, user-empowered digital future.
 
-- **Quickstart Guide**: Check out [quickstart.md](docs/quickstart.md) to get up and running with cLAN in no time.
+## Features of Clan
 
-## Managing Secrets
+- **Full-Stack System Deployment:** Utilize Clan’s toolkit alongside Nix's reliability to build and manage systems effortlessly.
+- **Overlay Networks:** Secure, private communication channels between devices.
+- **Virtual Machine Integration:** Seamless operation of VM applications within the main operating system.
+- **Robust Backup Management:** Long-term, self-hosted data preservation.
+- **Intuitive Secret Management:** Simplified encryption and password management processes.
 
-Security is paramount, and cLAN provides guidelines for handling secrets effectively:
+## Getting Started with cLAN
 
-- **Secrets Management**: Learn how to manage secrets securely by reading [secrets-management.md](docs/secrets-management.md).
+If you're new to cLAN and eager to dive in, start with our quickstart guide and explore the core functionalities that Clan offers:
 
-## Contributing to cLAN
+- **Quickstart Guide**: Check out [getting started](https://docs.clan.lol/#starting-with-a-new-clan-project)<!-- [docs/site/index.md](docs/site/index.md) --> to get up and running with cLAN in no time.
 
-We welcome contributions from the community, and we've prepared a comprehensive guide to help you get started:
+### Managing Secrets
 
-- **Contribution Guidelines**: Find out how to contribute and make a meaningful impact on the cLAN project by reading [contributing.md](docs/contributing.md).
+In the Clan ecosystem, security is paramount. Learn how to handle secrets effectively:
+
+- **Secrets Management**: Securely manage secrets by consulting [secrets](https://docs.clan.lol/getting-started/secrets/)<!-- [secrets.md](docs/site/getting-started/secrets.md) -->.
+
+### Contributing to cLAN
+
+The Clan project thrives on community contributions. We welcome everyone to contribute and collaborate:
+
+- **Contribution Guidelines**: Make a meaningful impact by following the steps in [contributing](https://docs.clan.lol/contributing/contributing/)<!-- [contributing.md](docs/CONTRIBUTING.md) -->.
+
+## Join the Revolution
+
+Clan is more than a tool; it's a movement towards a better digital future. By contributing to the cLAN project, you're part of changing technology for the better, together.
+
+### Community and Support
+
+Connect with us and the Clan community for support and discussion:
+
+- [Matrix channel](https://matrix.to/#/!djzOHBBBHnwQkgNgdV:matrix.org?via=blog.clan.lol) for live discussions.
+- IRC bridges (coming soon) for real-time chat support.
 
-Whether you're a newcomer or a seasoned developer, we look forward to your contributions and collaboration on the cLAN project. Let's build amazing things together!

checks/backups/flake-module.nix

@@ -1,53 +1,85 @@
 { self, ... }:
-let
-  clan = self.lib.buildClan {
-    clanName = "testclan";
-    directory = ../..;
-    machines = {
-      test_backup_client = {
-        clan.networking.targetHost = "client";
-        imports = [ self.nixosModules.test_backup_client ];
+{
+  clan.machines.test-backup = {
+    imports = [ self.nixosModules.test-backup ];
     fileSystems."/".device = "/dev/null";
     boot.loader.grub.device = "/dev/null";
   };
-    };
-  };
-in
-{
-  flake.nixosConfigurations = { inherit (clan.nixosConfigurations) test_backup_client; };
-  flake.clanInternals = clan.clanInternals;
   flake.nixosModules = {
-    test_backup_server = { ... }: {
-      imports = [
-        self.clanModules.borgbackup
-      ];
-      services.sshd.enable = true;
-      services.borgbackup.repos.testrepo = {
-        authorizedKeys = [
-          (builtins.readFile ../lib/ssh/pubkey)
-        ];
-      };
-    };
-    test_backup_client = { pkgs, lib, config, ... }:
+    test-backup =
+      {
+        pkgs,
+        lib,
+        config,
+        ...
+      }:
       let
         dependencies = [
           self
           pkgs.stdenv.drvPath
-          clan.clanInternals.machines.x86_64-linux.test_backup_client.config.system.clan.deployment.file
+          self.clanInternals.machines.${pkgs.hostPlatform.system}.test-backup.config.system.clan.deployment.file
         ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
         closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
       in
       {
         imports = [
           self.clanModules.borgbackup
+          self.clanModules.localbackup
+          self.clanModules.sshd
         ];
-        networking.hostName = "client";
-        services.sshd.enable = true;
-        users.users.root.openssh.authorizedKeys.keyFiles = [
-          ../lib/ssh/pubkey
+        clan.networking.targetHost = "machine";
+        networking.hostName = "machine";
+        services.openssh.settings.UseDns = false;
+
+        programs.ssh.knownHosts = {
+          machine.hostNames = [ "machine" ];
+          machine.publicKey = builtins.readFile ../lib/ssh/pubkey;
+        };
+
+        users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ];
+
+        systemd.tmpfiles.settings."vmsecrets" = {
+          "/root/.ssh/id_ed25519" = {
+            C.argument = "${../lib/ssh/privkey}";
+            z = {
+              mode = "0400";
+              user = "root";
+            };
+          };
+          "/etc/secrets/ssh.id_ed25519" = {
+            C.argument = "${../lib/ssh/privkey}";
+            z = {
+              mode = "0400";
+              user = "root";
+            };
+          };
+          "/etc/secrets/borgbackup.ssh" = {
+            C.argument = "${../lib/ssh/privkey}";
+            z = {
+              mode = "0400";
+              user = "root";
+            };
+          };
+          "/etc/secrets/borgbackup.repokey" = {
+            C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345");
+            z = {
+              mode = "0400";
+              user = "root";
+            };
+          };
+        };
+        clanCore.facts.secretStore = "vm";
+
+        environment.systemPackages = [
+          self.packages.${pkgs.system}.clan-cli
+          (pkgs.writeShellScriptBin "pre-restore-command" ''
+            touch /var/test-service/pre-restore-command
+          '')
+          (pkgs.writeShellScriptBin "post-restore-command" ''
+            touch /var/test-service/post-restore-command
+          '')
         ];
-        environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
-        environment.etc."install-closure".source = "${closureInfo}/store-paths";
+        environment.etc.install-closure.source = "${closureInfo}/store-paths";
         nix.settings = {
           substituters = lib.mkForce [ ];
           hashed-mirrors = null;
@@ -56,74 +88,91 @@ in
         };
         system.extraDependencies = dependencies;
         clanCore.state.test-backups.folders = [ "/var/test-backups" ];
-        clan.borgbackup = {
-          enable = true;
-          destinations.test_backup_server = {
-            repo = "borg@server:.";
-            rsh = "ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no";
+
+        clanCore.state.test-service = {
+          preRestoreCommand = "pre-restore-command";
+          postRestoreCommand = "post-restore-command";
+          folders = [ "/var/test-service" ];
+        };
+        clan.borgbackup.destinations.test-backup.repo = "borg@machine:.";
+
+        fileSystems."/mnt/external-disk" = {
+          device = "/dev/vdb"; # created in tests with virtualisation.emptyDisks
+          autoFormat = true;
+          fsType = "ext4";
+          options = [
+            "defaults"
+            "noauto"
+          ];
+        };
+
+        clan.localbackup.targets.hdd = {
+          directory = "/mnt/external-disk";
+          preMountHook = ''
+            touch /run/mount-external-disk
+          '';
+          postUnmountHook = ''
+            touch /run/unmount-external-disk
+          '';
+        };
+
+        services.borgbackup.repos.test-backups = {
+          path = "/var/lib/borgbackup/test-backups";
+          authorizedKeys = [ (builtins.readFile ../lib/ssh/pubkey) ];
         };
       };
   };
-  };
-  perSystem = { nodes, pkgs, ... }: {
+  perSystem =
+    { nodes, pkgs, ... }:
+    {
       checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
-      test-backups =
-        (import ../lib/test-base.nix)
-          {
+        test-backups = (import ../lib/test-base.nix) {
           name = "test-backups";
-            nodes.server = {
+          nodes.machine = {
             imports = [
-                self.nixosModules.test_backup_server
               self.nixosModules.clanCore
-                {
-                  clanCore.machineName = "server";
-                  clanCore.clanDir = ../..;
-                }
-              ];
-            };
-            nodes.client = {
-              imports = [
-                self.nixosModules.test_backup_client
-                self.nixosModules.clanCore
-                {
-                  clanCore.machineName = "client";
-                  clanCore.clanDir = ../..;
-                }
+              self.nixosModules.test-backup
             ];
+            virtualisation.emptyDiskImages = [ 256 ];
           };
 
           testScript = ''
             import json
             start_all()
 
-              # setup
-              client.succeed("mkdir -m 700 /root/.ssh")
-              client.succeed(
-                  "cat ${../lib/ssh/privkey} > /root/.ssh/id_ed25519"
-              )
-              client.succeed("chmod 600 /root/.ssh/id_ed25519")
-              client.wait_for_unit("sshd", timeout=30)
-              client.succeed("ssh -o StrictHostKeyChecking=accept-new root@client hostname")
-
             # dummy data
-              client.succeed("mkdir /var/test-backups")
-              client.succeed("echo testing > /var/test-backups/somefile")
+            machine.succeed("mkdir -p /var/test-backups /var/test-service")
+            machine.succeed("echo testing > /var/test-backups/somefile")
 
             # create
-              client.succeed("clan --debug --flake ${../..} backups create test_backup_client")
-              client.wait_until_succeeds("! systemctl is-active borgbackup-job-test_backup_server")
+            machine.succeed("clan --debug --flake ${self} backups create test-backup")
+            machine.wait_until_succeeds("! systemctl is-active borgbackup-job-test-backup >&2")
+            machine.succeed("test -f /run/mount-external-disk")
+            machine.succeed("test -f /run/unmount-external-disk")
 
             # list
-              backup_id = json.loads(client.succeed("borg-job-test_backup_server list --json"))["archives"][0]["archive"]
-              assert(backup_id in client.succeed("clan --debug --flake ${../..} backups list test_backup_client"))
+            backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"]
+            out = machine.succeed("clan --debug --flake ${self} backups list test-backup").strip()
+            print(out)
+            assert backup_id in out, f"backup {backup_id} not found in {out}"
+            localbackup_id = "hdd::/mnt/external-disk/snapshot.0"
+            assert localbackup_id in out, "localbackup not found in {out}"
 
-              # restore
-              client.succeed("rm -f /var/test-backups/somefile")
-              client.succeed(f"clan --debug --flake ${../..} backups restore test_backup_client borgbackup {backup_id}")
-              assert(client.succeed("cat /var/test-backups/somefile").strip() == "testing")
+            ## borgbackup restore
+            machine.succeed("rm -f /var/test-backups/somefile")
+            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
+            assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
+            machine.succeed("test -f /var/test-service/pre-restore-command")
+            machine.succeed("test -f /var/test-service/post-restore-command")
+
+            ## localbackup restore
+            machine.succeed("rm -f /var/test-backups/somefile /var/test-service/{pre,post}-restore-command")
+            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup localbackup '{localbackup_id}' >&2")
+            assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
+            machine.succeed("test -f /var/test-service/pre-restore-command")
+            machine.succeed("test -f /var/test-service/post-restore-command")
           '';
-          }
-          { inherit pkgs self; };
+        } { inherit pkgs self; };
       };
     };
 }

checks/borgbackup/default.nix

@@ -1,16 +1,18 @@
-(import ../lib/test-base.nix) ({ ... }: {
+(import ../lib/test-base.nix) (
+  { ... }:
+  {
     name = "borgbackup";
 
-  nodes.machine = { self, ... }: {
+    nodes.machine =
+      { self, pkgs, ... }:
+      {
         imports = [
           self.clanModules.borgbackup
           self.nixosModules.clanCore
           {
             services.openssh.enable = true;
             services.borgbackup.repos.testrepo = {
-          authorizedKeys = [
-            (builtins.readFile ../lib/ssh/pubkey)
-          ];
+              authorizedKeys = [ (builtins.readFile ../lib/ssh/pubkey) ];
             };
           }
           {
@@ -18,22 +20,25 @@
             clanCore.clanDir = ./.;
             clanCore.state.testState.folders = [ "/etc/state" ];
             environment.etc.state.text = "hello world";
-        systemd.tmpfiles.settings = {
-          "ssh-key"."/root/.ssh/id_ed25519" = {
+            systemd.tmpfiles.settings."vmsecrets" = {
+              "/etc/secrets/borgbackup.ssh" = {
                 C.argument = "${../lib/ssh/privkey}";
                 z = {
                   mode = "0400";
                   user = "root";
                 };
               };
-        };
-        clan.borgbackup = {
-          enable = true;
-          destinations.test = {
-            repo = "borg@localhost:.";
-            rsh = "ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no";
+              "/etc/secrets/borgbackup.repokey" = {
+                C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345");
+                z = {
+                  mode = "0400";
+                  user = "root";
                 };
               };
+            };
+            clanCore.facts.secretStore = "vm";
+
+            clan.borgbackup.destinations.test.repo = "borg@localhost:.";
           }
         ];
       };
@@ -42,4 +47,5 @@
       machine.systemctl("start --wait borgbackup-job-test.service")
       assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list")
     '';
-})
+  }
+)

checks/container/default.nix

@@ -1,7 +1,11 @@
-(import ../lib/container-test.nix) ({ ... }: {
+(import ../lib/container-test.nix) (
+  { ... }:
+  {
     name = "secrets";
 
-  nodes.machine = { ... }: {
+    nodes.machine =
+      { ... }:
+      {
         networking.hostName = "machine";
         services.openssh.enable = true;
         services.openssh.startWhenNeeded = false;
@@ -11,4 +15,5 @@
       machine.succeed("systemctl status sshd")
       machine.wait_for_unit("sshd")
     '';
-})
+  }
+)

checks/deltachat/default.nix

@@ -1,7 +1,11 @@
-(import ../lib/container-test.nix) ({ pkgs, ... }: {
+(import ../lib/container-test.nix) (
+  { pkgs, ... }:
+  {
     name = "secrets";
 
-  nodes.machine = { self, ... }: {
+    nodes.machine =
+      { self, ... }:
+      {
         imports = [
           self.clanModules.deltachat
           self.nixosModules.clanCore
@@ -21,4 +25,5 @@
       # smtp
       machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 25")
     '';
-})
+  }
+)

checks/flake-module.nix

@@ -1,12 +1,34 @@
-{ self, ... }: {
+{ self, ... }:
+{
   imports = [
     ./impure/flake-module.nix
     ./backups/flake-module.nix
     ./installation/flake-module.nix
+    ./flash/flake-module.nix
   ];
-  perSystem = { pkgs, lib, self', ... }: {
+  perSystem =
+    {
+      pkgs,
+      lib,
+      self',
+      ...
+    }:
+    {
       checks =
         let
+          # ensure all options can be rendered after importing clan into nixos
+          renderClanOptions =
+            let
+              docs = pkgs.nixosOptionsDoc {
+                options =
+                  (pkgs.nixos {
+                    imports = [ self.nixosModules.clanCore ];
+                    clanCore.clanDir = ./.;
+                  }).options;
+                warningsAreErrors = false;
+              };
+            in
+            docs.optionsJSON;
           nixosTestArgs = {
             # reference to nixpkgs for the current system
             inherit pkgs;
@@ -18,22 +40,25 @@
             secrets = import ./secrets nixosTestArgs;
             container = import ./container nixosTestArgs;
             deltachat = import ./deltachat nixosTestArgs;
-          meshnamed = import ./meshnamed nixosTestArgs;
+            matrix-synapse = import ./matrix-synapse nixosTestArgs;
             zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
             borgbackup = import ./borgbackup nixosTestArgs;
             syncthing = import ./syncthing nixosTestArgs;
             wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
           };
-        schemaTests = pkgs.callPackages ./schemas.nix {
-          inherit self;
-        };
+          schemaTests = pkgs.callPackages ./schemas.nix { inherit self; };
 
-        flakeOutputs = lib.mapAttrs' (name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel) self.nixosConfigurations
+          flakeOutputs =
+            lib.mapAttrs' (
+              name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel
+            ) self.nixosConfigurations
             // lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages
             // lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells
-          // lib.mapAttrs' (name: config: lib.nameValuePair "home-manager-${name}" config.activation-script) (self'.legacyPackages.homeConfigurations or { });
+            // lib.mapAttrs' (name: config: lib.nameValuePair "home-manager-${name}" config.activation-script) (
+              self'.legacyPackages.homeConfigurations or { }
+            );
         in
-      nixosTests // schemaTests // flakeOutputs;
+        { inherit renderClanOptions; } // nixosTests // schemaTests // flakeOutputs;
       legacyPackages = {
         nixosTests =
           let

checks/flash/flake-module.nix

@@ -0,0 +1,49 @@
+{ self, ... }:
+{
+  perSystem =
+    {
+      nodes,
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      dependencies = [
+        self
+        pkgs.stdenv.drvPath
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.toplevel
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.diskoScript
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.clan.deployment.file
+        self.inputs.nixpkgs.legacyPackages.${pkgs.hostPlatform.system}.disko
+      ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+      closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
+    in
+    {
+      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
+        flash = (import ../lib/test-base.nix) {
+          name = "flash";
+          nodes.target = {
+            virtualisation.emptyDiskImages = [ 4096 ];
+            virtualisation.memorySize = 3000;
+            environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
+            environment.etc."install-closure".source = "${closureInfo}/store-paths";
+
+            nix.settings = {
+              substituters = lib.mkForce [ ];
+              hashed-mirrors = null;
+              connect-timeout = lib.mkForce 3;
+              flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
+              experimental-features = [
+                "nix-command"
+                "flakes"
+              ];
+            };
+          };
+          testScript = ''
+            start_all()
+            machine.succeed("clan --flake ${../..} flash --debug --yes --disk main /dev/vdb test_install_machine")
+          '';
+        } { inherit pkgs self; };
+      };
+    };
+}

checks/impure/flake-module.nix

@@ -1,15 +1,19 @@
 {
-  perSystem = { pkgs, lib, ... }: {
+  perSystem =
+    { pkgs, lib, ... }:
+    {
       # a script that executes all other checks
       packages.impure-checks = pkgs.writeShellScriptBin "impure-checks" ''
         #!${pkgs.bash}/bin/bash
         set -euo pipefail
 
-      export PATH="${lib.makeBinPath [
+        export PATH="${
+          lib.makeBinPath [
             pkgs.gitMinimal
             pkgs.nix
             pkgs.rsync # needed to have rsync installed on the dummy ssh server
-      ]}"
+          ]
+        }"
         ROOT=$(git rev-parse --show-toplevel)
         cd "$ROOT/pkgs/clan-cli"
         nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -s -m impure ./tests $@"

checks/installation/flake-module.nix

@@ -1,65 +1,54 @@
-{ self, ... }:
-let
-  clan = self.lib.buildClan {
-    clanName = "testclan";
-    directory = ../..;
-    machines = {
-      test_install_machine = {
+{ self, lib, ... }:
+{
+  clan.machines.test_install_machine = {
     clan.networking.targetHost = "test_install_machine";
+    fileSystems."/".device = lib.mkDefault "/dev/null";
+    boot.loader.grub.device = lib.mkDefault "/dev/null";
+
     imports = [ self.nixosModules.test_install_machine ];
   };
-    };
-  };
-in
-{
-  flake.nixosConfigurations = { inherit (clan.nixosConfigurations) test_install_machine; };
-  flake.clanInternals = clan.clanInternals;
   flake.nixosModules = {
-    test_install_machine = { lib, modulesPath, ... }: {
+    test_install_machine =
+      { lib, modulesPath, ... }:
+      {
         imports = [
           self.clanModules.diskLayouts
           (modulesPath + "/testing/test-instrumentation.nix") # we need these 2 modules always to be able to run the tests
           (modulesPath + "/profiles/qemu-guest.nix")
         ];
-      fileSystems."/nix/store" = lib.mkForce {
-        device = "nix-store";
-        fsType = "9p";
-        neededForBoot = true;
-        options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
-      };
         clan.diskLayouts.singleDiskExt4.device = "/dev/vdb";
 
         environment.etc."install-successful".text = "ok";
 
         boot.consoleLogLevel = lib.mkForce 100;
-      boot.kernelParams = [
-        "boot.shell_on_fail"
-      ];
+        boot.kernelParams = [ "boot.shell_on_fail" ];
       };
   };
-  perSystem = { nodes, pkgs, lib, ... }:
+  perSystem =
+    {
+      nodes,
+      pkgs,
+      lib,
+      ...
+    }:
     let
       dependencies = [
         self
+        self.nixosConfigurations.test_install_machine.config.system.build.toplevel
+        self.nixosConfigurations.test_install_machine.config.system.build.diskoScript
+        self.nixosConfigurations.test_install_machine.config.system.clan.deployment.file
         pkgs.stdenv.drvPath
-        clan.clanInternals.machines.x86_64-linux.test_install_machine.config.system.build.toplevel
-        clan.clanInternals.machines.x86_64-linux.test_install_machine.config.system.build.diskoScript
-        clan.clanInternals.machines.x86_64-linux.test_install_machine.config.system.clan.deployment.file
         pkgs.nixos-anywhere
       ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
       closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
     in
     {
       checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
-        test-installation =
-          (import ../lib/test-base.nix)
-            {
+        test-installation = (import ../lib/test-base.nix) {
           name = "test-installation";
           nodes.target = {
             services.openssh.enable = true;
-                users.users.root.openssh.authorizedKeys.keyFiles = [
-                  ../lib/ssh/pubkey
-                ];
+            users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ];
             system.nixos.variant_id = "installer";
             virtualisation.emptyDiskImages = [ 4096 ];
             nix.settings = {
@@ -92,22 +81,22 @@ in
 
           testScript = ''
             def create_test_machine(oldmachine=None, args={}): # taken from <nixpkgs/nixos/tests/installer.nix>
+                startCommand = "${pkgs.qemu_test}/bin/qemu-kvm"
+                startCommand += " -cpu max -m 1024 -virtfs local,path=/nix/store,security_model=none,mount_tag=nix-store"
+                startCommand += f' -drive file={oldmachine.state_dir}/empty0.qcow2,id=drive1,if=none,index=1,werror=report'
+                startCommand += ' -device virtio-blk-pci,drive=drive1'
                 machine = create_machine({
-                      "qemuFlags":
-                        '-cpu max -m 1024 -virtfs local,path=/nix/store,security_model=none,mount_tag=nix-store,'
-                        f' -drive file={oldmachine.state_dir}/empty0.qcow2,id=drive1,if=none,index=1,werror=report'
-                        f' -device virtio-blk-pci,drive=drive1',
+                  "startCommand": startCommand,
                 } | args)
                 driver.machines.append(machine)
                 return machine
 
-
             start_all()
 
             client.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../lib/ssh/privkey} /root/.ssh/id_ed25519")
             client.wait_until_succeeds("ssh -o StrictHostKeyChecking=accept-new -v root@target hostname")
 
-                client.succeed("clan --debug --flake ${../..} machines install test_install_machine root@target >&2")
+            client.succeed("clan --debug --flake ${../..} machines install --yes test_install_machine root@target >&2")
             try:
               target.shutdown()
             except BrokenPipeError:
@@ -117,8 +106,7 @@ in
             new_machine = create_test_machine(oldmachine=target, args={ "name": "new_machine" })
             assert(new_machine.succeed("cat /etc/install-successful").strip() == "ok")
           '';
-            }
-            { inherit pkgs self; };
+        } { inherit pkgs self; };
       };
     };
 }

checks/lib/container-driver/module.nix

@@ -1,17 +1,23 @@
-{ hostPkgs, lib, config, ... }:
+{
+  hostPkgs,
+  lib,
+  config,
+  ...
+}:
 let
   testDriver = hostPkgs.python3.pkgs.callPackage ./package.nix {
     inherit (config) extraPythonPackages;
     inherit (hostPkgs.pkgs) util-linux systemd;
   };
   containers = map (m: m.system.build.toplevel) (lib.attrValues config.nodes);
-  pythonizeName = name:
+  pythonizeName =
+    name:
     let
       head = lib.substring 0 1 name;
       tail = lib.substring 1 (-1) name;
     in
-    (if builtins.match "[A-z_]" head == null then "_" else head) +
-    lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail;
+    (if builtins.match "[A-z_]" head == null then "_" else head)
+    + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail;
   nodeHostNames =
     let
       nodesList = map (c: c.system.name) (lib.attrValues config.nodes);
@@ -21,7 +27,8 @@ let
   pythonizedNames = map pythonizeName nodeHostNames;
 in
 {
-  driver = lib.mkForce (hostPkgs.runCommand "nixos-test-driver-${config.name}"
+  driver = lib.mkForce (
+    hostPkgs.runCommand "nixos-test-driver-${config.name}"
       {
         nativeBuildInputs = [
           hostPkgs.makeWrapper
@@ -61,9 +68,11 @@ in
         wrapProgram $out/bin/nixos-test-driver \
           ${lib.concatStringsSep " " (map (name: "--add-flags '--container ${name}'") containers)} \
           --add-flags "--test-script '$out/test-script'"
-    '');
+      ''
+  );
 
-  test = lib.mkForce (lib.lazyDerivation {
+  test = lib.mkForce (
+    lib.lazyDerivation {
       # lazyDerivation improves performance when only passthru items and/or meta are used.
       derivation = hostPkgs.stdenv.mkDerivation {
         name = "vm-test-run-${config.name}";
@@ -84,5 +93,6 @@ in
         meta = config.meta;
       };
       inherit (config) passthru meta;
-  });
+    }
+  );
 }

checks/lib/container-driver/package.nix

@@ -1,8 +1,18 @@
-{ extraPythonPackages, python3Packages, buildPythonApplication, setuptools, util-linux, systemd }:
+{
+  extraPythonPackages,
+  python3Packages,
+  buildPythonApplication,
+  setuptools,
+  util-linux,
+  systemd,
+}:
 buildPythonApplication {
   pname = "test-driver";
   version = "0.0.1";
-  propagatedBuildInputs = [ util-linux systemd ] ++ extraPythonPackages python3Packages;
+  propagatedBuildInputs = [
+    util-linux
+    systemd
+  ] ++ extraPythonPackages python3Packages;
   nativeBuildInputs = [ setuptools ];
   format = "pyproject";
   src = ./.;

checks/lib/container-driver/pyproject.toml

@@ -19,8 +19,8 @@ test_driver = ["py.typed"]
 target-version = "py311"
 line-length = 88
 
-select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
-ignore = ["E501", "ANN101", "ANN401", "A003"]
+lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
+lint.ignore = ["E501", "ANN101", "ANN401", "A003"]
 
 [tool.mypy]
 python_version = "3.11"

checks/lib/container-driver/test_driver/__init__.py

@@ -258,7 +258,7 @@ class Driver:
 
         self.machines = []
         for container in containers:
-            name_match = re.match(r".*-nixos-system-(.+)-\d.+", container.name)
+            name_match = re.match(r".*-nixos-system-(.+)-(.+)", container.name)
             if not name_match:
                 raise ValueError(f"Unable to extract hostname from {container.name}")
             name = name_match.group(1)

checks/lib/container-test.nix

@@ -1,13 +1,12 @@
 test:
-{ pkgs
-, self
-, ...
-}:
+{ pkgs, self, ... }:
 let
   inherit (pkgs) lib;
   nixos-lib = import (pkgs.path + "/nixos/lib") { };
 in
-(nixos-lib.runTest ({ hostPkgs, ... }: {
+(nixos-lib.runTest (
+  { hostPkgs, ... }:
+  {
     hostPkgs = pkgs;
     # speed-up evaluation
     defaults = {
@@ -30,4 +29,5 @@ in
       test
       ./container-driver/module.nix
     ];
-})).config.result
+  }
+)).config.result

checks/lib/test-base.nix

@@ -1,8 +1,5 @@
 test:
-{ pkgs
-, self
-, ...
-}:
+{ pkgs, self, ... }:
 let
   inherit (pkgs) lib;
   nixos-lib = import (pkgs.path + "/nixos/lib") { };

checks/matrix-synapse/default.nix

@@ -0,0 +1,37 @@
+(import ../lib/container-test.nix) (
+  { pkgs, ... }:
+  {
+    name = "matrix-synapse";
+
+    nodes.machine =
+      { self, lib, ... }:
+      {
+        imports = [
+          self.clanModules.matrix-synapse
+          self.nixosModules.clanCore
+          {
+            clanCore.machineName = "machine";
+            clanCore.clanDir = ./.;
+            clan.matrix-synapse = {
+              enable = true;
+              domain = "clan.test";
+            };
+          }
+          {
+            # secret override
+            clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path = "${./synapse-registration_shared_secret}";
+            services.nginx.virtualHosts."matrix.clan.test" = {
+              enableACME = lib.mkForce false;
+              forceSSL = lib.mkForce false;
+            };
+          }
+        ];
+      };
+    testScript = ''
+      start_all()
+      machine.wait_for_unit("matrix-synapse")
+      machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 8008")
+      machine.succeed("${pkgs.curl}/bin/curl -Ssf -L http://localhost/_matrix/static/ -H 'Host: matrix.clan.test'")
+    '';
+  }
+)

checks/matrix-synapse/synapse-registration_shared_secret

@@ -0,0 +1 @@
+registration_shared_secret: supersecret

checks/meshnamed/default.nix

@@ -1,21 +0,0 @@
-(import ../lib/container-test.nix) ({ pkgs, ... }: {
-  name = "meshnamed";
-
-  nodes.machine = { self, ... }: {
-    imports = [
-      self.nixosModules.clanCore
-      {
-        clanCore.machineName = "machine";
-        clan.networking.meshnamed.networks.vpn.subnet = "fd43:7def:4b50:28d0:4e99:9347:3035:17ef/88";
-        clanCore.clanDir = ./.;
-      }
-    ];
-  };
-  testScript = ''
-    start_all()
-    machine.wait_for_unit("meshnamed")
-    out = machine.succeed("${pkgs.dnsutils}/bin/dig AAAA foo.7vbx332lkaunatuzsndtanix54.vpn @meshnamed +short")
-    print(out)
-    assert out.strip() == "fd43:7def:4b50:28d0:4e99:9347:3035:17ef"
-  '';
-})

checks/schemas.nix

@@ -1,35 +1,48 @@
-{ self, runCommand, check-jsonschema, pkgs, lib, ... }:
+{
+  self,
+  runCommand,
+  check-jsonschema,
+  pkgs,
+  lib,
+  ...
+}:
 let
   clanModules.clanCore = self.nixosModules.clanCore;
 
   baseModule = {
-    imports =
-      (import (pkgs.path + "/nixos/modules/module-list.nix"))
-      ++ [{
+    imports = (import (pkgs.path + "/nixos/modules/module-list.nix")) ++ [
+      {
         nixpkgs.hostPlatform = "x86_64-linux";
         clanCore.clanName = "dummy";
-      }];
+      }
+    ];
   };
 
-  optionsFromModule = module:
+  optionsFromModule =
+    module:
     let
       evaled = lib.evalModules {
-        modules = [ module baseModule ];
+        modules = [
+          module
+          baseModule
+        ];
       };
     in
     evaled.options.clan;
 
-  clanModuleSchemas = lib.mapAttrs (_: module: self.lib.jsonschema.parseOptions (optionsFromModule module)) clanModules;
+  clanModuleSchemas = lib.mapAttrs (
+    _: module: self.lib.jsonschema.parseOptions (optionsFromModule module)
+  ) clanModules;
 
-  mkTest = name: schema: runCommand "schema-${name}" { } ''
+  mkTest =
+    name: schema:
+    runCommand "schema-${name}" { } ''
       ${check-jsonschema}/bin/check-jsonschema \
         --check-metaschema ${builtins.toFile "schema-${name}" (builtins.toJSON schema)}
         touch $out
     '';
 in
-lib.mapAttrs'
-  (name: schema: {
+lib.mapAttrs' (name: schema: {
   name = "schema-${name}";
   value = mkTest name schema;
-  })
-  clanModuleSchemas
+}) clanModuleSchemas

checks/secrets/default.nix

@@ -1,10 +1,10 @@
 (import ../lib/test-base.nix) {
   name = "secrets";
 
-  nodes.machine = { self, config, ... }: {
-    imports = [
-      (self.nixosModules.clanCore)
-    ];
+  nodes.machine =
+    { self, config, ... }:
+    {
+      imports = [ (self.nixosModules.clanCore) ];
       environment.etc."secret".source = config.sops.secrets.secret.path;
       environment.etc."group-secret".source = config.sops.secrets.group-secret.path;
       sops.age.keyFile = ./key.age;

checks/syncthing/default.nix

@@ -19,7 +19,7 @@
               "syncthing.key".source = ./introducer/introducer_test_key;
               "syncthing.api".source = ./introducer/introducer_test_api;
             };
-            clanCore.secrets.syncthing.secrets."syncthing.api".path = "/etc/syncthing.api";
+            clanCore.facts.services.syncthing.secret."syncthing.api".path = "/etc/syncthing.api";
             services.syncthing.cert = "/etc/syncthing.pam";
             services.syncthing.key = "/etc/syncthing.key";
             # Doesn't test zerotier!

checks/wayland-proxy-virtwl/default.nix

@@ -1,7 +1,16 @@
-import ../lib/test-base.nix ({ config, pkgs, lib, ... }: {
+import ../lib/test-base.nix (
+  {
+    config,
+    pkgs,
+    lib,
+    ...
+  }:
+  {
     name = "wayland-proxy-virtwl";
 
-  nodes.machine = { self, ... }: {
+    nodes.machine =
+      { self, ... }:
+      {
         imports = [
           self.nixosModules.clanCore
           {
@@ -22,4 +31,5 @@ import ../lib/test-base.nix ({ config, pkgs, lib, ... }: {
       # use machinectl
       machine.succeed("machinectl shell .host ${config.nodes.machine.systemd.package}/bin/systemctl --user start wayland-proxy-virtwl >&2")
     '';
-})
+  }
+)

checks/zt-tcp-relay/default.nix

@@ -1,7 +1,11 @@
-(import ../lib/container-test.nix) ({ pkgs, ... }: {
+(import ../lib/container-test.nix) (
+  { pkgs, ... }:
+  {
     name = "zt-tcp-relay";
 
-  nodes.machine = { self, ... }: {
+    nodes.machine =
+      { self, ... }:
+      {
         imports = [
           self.nixosModules.clanCore
           self.clanModules.zt-tcp-relay
@@ -17,4 +21,5 @@
       out = machine.succeed("${pkgs.netcat}/bin/nc -z -v localhost 4443")
       print(out)
     '';
-})
+  }
+)

clanModules/borgbackup.nix

@@ -1,15 +1,21 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   cfg = config.clan.borgbackup;
 in
 {
-  options.clan.borgbackup = {
-    enable = lib.mkEnableOption "backups with borgbackup";
-    destinations = lib.mkOption {
-      type = lib.types.attrsOf (lib.types.submodule ({ name, ... }: {
+  options.clan.borgbackup.destinations = lib.mkOption {
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { name, ... }:
+        {
           options = {
             name = lib.mkOption {
-            type = lib.types.str;
+              type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
               default = name;
               description = "the name of the backup job";
             };
@@ -19,72 +25,107 @@ in
             };
             rsh = lib.mkOption {
               type = lib.types.str;
-            default = "ssh -i ${config.clanCore.secrets.borgbackup.secrets."borgbackup.ssh".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+              default = "ssh -i ${
+                config.clanCore.facts.services.borgbackup.secret."borgbackup.ssh".path
+              } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+              defaultText = "ssh -i \${config.clanCore.facts.services.borgbackup.secret.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
               description = "the rsh to use for the backup";
             };
-
           };
-      }));
+        }
+      )
+    );
+    default = { };
     description = ''
       destinations where the machine should be backuped to
     '';
   };
-  };
-  config = lib.mkIf cfg.enable {
-    services.borgbackup.jobs = lib.mapAttrs
-      (_: dest: {
-        paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
-        exclude = [
-          "*.pyc"
+
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "borgbackup"
+      "enable"
+    ] "Just define clan.borgbackup.destinations to enable it")
   ];
+
+  config = lib.mkIf (cfg.destinations != { }) {
+    services.borgbackup.jobs = lib.mapAttrs (_: dest: {
+      paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
+      exclude = [ "*.pyc" ];
       repo = dest.repo;
       environment.BORG_RSH = dest.rsh;
-        encryption.mode = "none";
       compression = "auto,zstd";
       startAt = "*-*-* 01:00:00";
+      persistentTimer = true;
       preHook = ''
         set -x
       '';
 
+      encryption = {
+        mode = "repokey";
+        passCommand = "cat ${config.clanCore.facts.services.borgbackup.secret."borgbackup.repokey".path}";
+      };
+
       prune.keep = {
         within = "1d"; # Keep all archives from the last day
         daily = 7;
         weekly = 4;
         monthly = 0;
       };
-      })
-      cfg.destinations;
+    }) cfg.destinations;
 
-    clanCore.secrets.borgbackup = {
-      facts."borgbackup.ssh.pub" = { };
-      secrets."borgbackup.ssh" = { };
-      generator.path = [ pkgs.openssh pkgs.coreutils ];
+    clanCore.facts.services.borgbackup = {
+      public."borgbackup.ssh.pub" = { };
+      secret."borgbackup.ssh" = { };
+      secret."borgbackup.repokey" = { };
+      generator.path = [
+        pkgs.openssh
+        pkgs.coreutils
+        pkgs.xkcdpass
+      ];
       generator.script = ''
         ssh-keygen -t ed25519 -N "" -f "$secrets"/borgbackup.ssh
         mv "$secrets"/borgbackup.ssh.pub "$facts"/borgbackup.ssh.pub
+        xkcdpass -n 4 -d - > "$secrets"/borgbackup.repokey
       '';
     };
 
-    clanCore.backups.providers.borgbackup = {
-      # TODO list needs to run locally or on the remote machine
-      list = ''
-        ${lib.concatMapStringsSep "\n" (dest: ''
-          # we need yes here to skip the changed url verification
-          yes y | borg-job-${dest.name} list --json | jq -r '. + {"job-name": "${dest.name}"}'
-        '') (lib.attrValues cfg.destinations)}
-      '';
-      create = ''
+    environment.systemPackages = [
+      (pkgs.writeShellScriptBin "borgbackup-create" ''
+        set -efu -o pipefail
         ${lib.concatMapStringsSep "\n" (dest: ''
           systemctl start borgbackup-job-${dest.name}
         '') (lib.attrValues cfg.destinations)}
-      '';
-
-      restore = ''
+      '')
+      (pkgs.writeShellScriptBin "borgbackup-list" ''
         set -efu
+        (${
+          lib.concatMapStringsSep "\n" (
+            dest:
+            # we need yes here to skip the changed url verification
+            ''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' ''
+          ) (lib.attrValues cfg.destinations)
+        }) | ${pkgs.jq}/bin/jq -s 'add'
+      '')
+      (pkgs.writeShellScriptBin "borgbackup-restore" ''
+        set -efux
         cd /
         IFS=';' read -ra FOLDER <<< "$FOLDERS"
-        yes y | borg-job-"$JOB" extract --list "$LOCATION"::"$ARCHIVE_ID" "''${FOLDER[@]}"
-      '';
+        job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}')
+        backup_name=''${NAME#"$job_name"::}
+        if ! command -v borg-job-"$job_name" &> /dev/null; then
+          echo "borg-job-$job_name not found: Backup name is invalid" >&2
+          exit 1
+        fi
+        yes y | borg-job-"$job_name" extract --list "$backup_name" "''${FOLDER[@]}"
+      '')
+    ];
+
+    clanCore.backups.providers.borgbackup = {
+      list = "borgbackup-list";
+      create = "borgbackup-create";
+      restore = "borgbackup-restore";
     };
   };
 }

clanModules/deltachat/README.md

@@ -0,0 +1,15 @@
+Email-based instant messaging for Desktop.
+
+!!! warning "Under construction"
+
+!!! info
+    This module will automatically configure an email server on the machine for handling the e-mail messaging seamlessly.
+
+## Features
+
+- [x] **Email-based**: Uses any email account as its backend.
+- [x] **End-to-End Encryption**: Supports Autocrypt to automatically encrypt messages.
+- [x] **No Phone Number Required**: Uses your email address instead of a phone number.
+- [x] **Cross-Platform**: Available on desktop and mobile platforms.
+- [x] **Automatic Server Setup**: Includes your own DeltaChat server for enhanced control and privacy.
+- [ ] **Bake a cake**: This module cannot cake a bake.

clanModules/deltachat/default.nix

@@ -1,12 +1,11 @@
-{ config, pkgs, ... }: {
+{ config, pkgs, ... }:
+{
   networking.firewall.interfaces."zt+".allowedTCPPorts = [ 25 ]; # smtp with other hosts
   environment.systemPackages = [ pkgs.deltachat-desktop ];
 
   services.maddy =
     let
-      # FIXME move this to public setting
-      meshname = config.clanCore.secrets.zerotier.facts.zerotier-meshname.value or null;
-      domain = if meshname == null then "${config.clanCore.machineName}.local" else "${meshname}.vpn";
+      domain = "${config.clanCore.machineName}.local";
     in
     {
       enable = true;
@@ -136,9 +135,7 @@
           storage &local_mailboxes
         }
       '';
-      ensureAccounts = [
-        "user@${domain}"
-      ];
+      ensureAccounts = [ "user@${domain}" ];
       ensureCredentials = {
         "user@${domain}".passwordFile = pkgs.writeText "dummy" "foobar";
       };

clanModules/diskLayouts.nix

@@ -6,7 +6,10 @@
       example = "/dev/disk/by-id/ata-Samsung_SSD_850_EVO_250GB_S21PNXAGB12345";
     };
   };
-  config.disko.devices = {
+  config = {
+    boot.loader.grub.efiSupport = lib.mkDefault true;
+    boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+    disko.devices = {
       disk = {
         main = {
           type = "disk";
@@ -40,5 +43,5 @@
         };
       };
     };
+  };
 }
-

clanModules/ergochat.nix

@@ -0,0 +1,14 @@
+_: {
+  services.ergochat = {
+    enable = true;
+
+    settings = {
+      datastore = {
+        autoupgrade = true;
+        path = "/var/lib/ergo/ircd.db";
+      };
+    };
+  };
+
+  clanCore.state.ergochat.folders = [ "/var/lib/ergo" ];
+}

clanModules/flake-module.nix

@@ -1,4 +1,5 @@
-{ inputs, ... }: {
+{ inputs, ... }:
+{
   flake.clanModules = {
     diskLayouts = {
       imports = [
@@ -7,13 +8,22 @@
       ];
     };
     borgbackup = ./borgbackup.nix;
-    deltachat = ./deltachat.nix;
-    moonlight = ./moonlight.nix;
-    sunshine = ./sunshine.nix;
-    syncthing = ./syncthing.nix;
-    xfce = ./xfce.nix;
-    zt-tcp-relay = ./zt-tcp-relay.nix;
+    ergochat = ./ergochat.nix;
+    deltachat = ./deltachat;
+    graphical = ./graphical.nix;
+    localbackup = ./localbackup.nix;
     localsend = ./localsend.nix;
+    matrix-synapse = ./matrix-synapse.nix;
+    moonlight = ./moonlight.nix;
+    sshd = ./sshd.nix;
+    sunshine = ./sunshine.nix;
+    syncthing = ./syncthing;
+    root-password = ./root-password;
+    thelounge = ./thelounge.nix;
+    vm-user = ./vm-user.nix;
     waypipe = ./waypipe.nix;
+    xfce = ./xfce.nix;
+    xfce-vm = ./xfce-vm.nix;
+    zt-tcp-relay = ./zt-tcp-relay.nix;
   };
 }

clanModules/graphical.nix

@@ -0,0 +1 @@
+_: { fonts.enableDefaultPackages = true; }

clanModules/localbackup.nix

@@ -0,0 +1,223 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.clan.localbackup;
+  rsnapshotConfig = target: states: ''
+    config_version	1.2
+    snapshot_root	${target.directory}
+    sync_first	1
+    cmd_cp	${pkgs.coreutils}/bin/cp
+    cmd_rm	${pkgs.coreutils}/bin/rm
+    cmd_rsync	${pkgs.rsync}/bin/rsync
+    cmd_ssh	${pkgs.openssh}/bin/ssh
+    cmd_logger	${pkgs.inetutils}/bin/logger
+    cmd_du	${pkgs.coreutils}/bin/du
+    cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
+    ${lib.optionalString (target.preBackupHook != null) ''
+      cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
+        set -efu -o pipefail
+        ${target.preBackupHook}
+      ''}
+    ''}
+
+    ${lib.optionalString (target.postBackupHook != null) ''
+      cmd_postexec	${pkgs.writeShellScript "postexec.sh" ''
+        set -efu -o pipefail
+        ${target.postBackupHook}
+      ''}
+    ''}
+    retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
+    ${lib.concatMapStringsSep "\n" (state: ''
+      ${lib.concatMapStringsSep "\n" (folder: ''
+        backup	${folder}	${config.networking.hostName}/
+      '') state.folders}
+    '') states}
+  '';
+in
+{
+  options.clan.localbackup = {
+    targets = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
+                default = name;
+                description = "the name of the backup job";
+              };
+              directory = lib.mkOption {
+                type = lib.types.str;
+                description = "the directory to backup";
+              };
+              mountpoint = lib.mkOption {
+                type = lib.types.nullOr lib.types.str;
+                default = null;
+                description = "mountpoint of the directory to backup. If set, the directory will be mounted before the backup and unmounted afterwards";
+              };
+              preMountHook = lib.mkOption {
+                type = lib.types.nullOr lib.types.lines;
+                default = null;
+                description = "Shell commands to run before the directory is mounted";
+              };
+              postMountHook = lib.mkOption {
+                type = lib.types.nullOr lib.types.lines;
+                default = null;
+                description = "Shell commands to run after the directory is mounted";
+              };
+              preUnmountHook = lib.mkOption {
+                type = lib.types.nullOr lib.types.lines;
+                default = null;
+                description = "Shell commands to run before the directory is unmounted";
+              };
+              postUnmountHook = lib.mkOption {
+                type = lib.types.nullOr lib.types.lines;
+                default = null;
+                description = "Shell commands to run after the directory is unmounted";
+              };
+              preBackupHook = lib.mkOption {
+                type = lib.types.nullOr lib.types.lines;
+                default = null;
+                description = "Shell commands to run before the backup";
+              };
+              postBackupHook = lib.mkOption {
+                type = lib.types.nullOr lib.types.lines;
+                default = null;
+                description = "Shell commands to run after the backup";
+              };
+            };
+          }
+        )
+      );
+      default = { };
+      description = "List of directories where backups are stored";
+    };
+
+    snapshots = lib.mkOption {
+      type = lib.types.int;
+      default = 20;
+      description = "Number of snapshots to keep";
+    };
+  };
+
+  config =
+    let
+      mountHook = target: ''
+        if [[ -x /run/current-system/sw/bin/localbackup-mount-${target.name} ]]; then
+          /run/current-system/sw/bin/localbackup-mount-${target.name}
+        fi
+        if [[ -x /run/current-system/sw/bin/localbackup-unmount-${target.name} ]]; then
+          trap "/run/current-system/sw/bin/localbackup-unmount-${target.name}" EXIT
+        fi
+      '';
+    in
+    lib.mkIf (cfg.targets != { }) {
+      environment.systemPackages =
+        [
+          (pkgs.writeShellScriptBin "localbackup-create" ''
+            set -efu -o pipefail
+            export PATH=${
+              lib.makeBinPath [
+                pkgs.rsnapshot
+                pkgs.coreutils
+                pkgs.util-linux
+              ]
+            }
+            ${lib.concatMapStringsSep "\n" (target: ''
+              (
+                ${mountHook target}
+                echo "Creating backup '${target.name}'"
+                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
+                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
+              )
+            '') (builtins.attrValues cfg.targets)}
+          '')
+          (pkgs.writeShellScriptBin "localbackup-list" ''
+            set -efu -o pipefail
+            export PATH=${
+              lib.makeBinPath [
+                pkgs.jq
+                pkgs.findutils
+                pkgs.coreutils
+                pkgs.util-linux
+              ]
+            }
+            (${
+              lib.concatMapStringsSep "\n" (target: ''
+                (
+                  ${mountHook target}
+                  find ${lib.escapeShellArg target.directory} -mindepth 1 -maxdepth 1 -name "snapshot.*" -print0 -type d \
+                    | jq -Rs 'split("\u0000") | .[] | select(. != "") | { "name": ("${target.name}::" + .)}'
+                )
+              '') (builtins.attrValues cfg.targets)
+            }) | jq -s .
+          '')
+          (pkgs.writeShellScriptBin "localbackup-restore" ''
+            set -efu -o pipefail
+            export PATH=${
+              lib.makeBinPath [
+                pkgs.rsync
+                pkgs.coreutils
+                pkgs.util-linux
+                pkgs.gawk
+              ]
+            }
+            name=$(awk -F'::' '{print $1}' <<< $NAME)
+            backupname=''${NAME#$name::}
+
+            if command -v localbackup-mount-$name; then
+              localbackup-mount-$name
+            fi
+            if command -v localbackup-unmount-$name; then
+              trap "localbackup-unmount-$name" EXIT
+            fi
+
+            if [[ ! -d $backupname ]]; then
+              echo "No backup found $backupname"
+              exit 1
+            fi
+
+            IFS=';' read -ra FOLDER <<< "$FOLDERS"
+            for folder in "''${FOLDER[@]}"; do
+              rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
+            done
+          '')
+        ]
+        ++ (lib.mapAttrsToList (
+          name: target:
+          pkgs.writeShellScriptBin ("localbackup-mount-" + name) ''
+            set -efu -o pipefail
+            ${lib.optionalString (target.preMountHook != null) target.preMountHook}
+            ${lib.optionalString (target.mountpoint != null) ''
+              if ! ${pkgs.util-linux}/bin/mountpoint -q ${lib.escapeShellArg target.mountpoint}; then
+                ${pkgs.util-linux}/bin/mount -o X-mount.mkdir ${lib.escapeShellArg target.mountpoint}
+              fi
+            ''}
+            ${lib.optionalString (target.postMountHook != null) target.postMountHook}
+          ''
+        ) cfg.targets)
+        ++ lib.mapAttrsToList (
+          name: target:
+          pkgs.writeShellScriptBin ("localbackup-unmount-" + name) ''
+            set -efu -o pipefail
+            ${lib.optionalString (target.preUnmountHook != null) target.preUnmountHook}
+            ${lib.optionalString (
+              target.mountpoint != null
+            ) "${pkgs.util-linux}/bin/umount ${lib.escapeShellArg target.mountpoint}"}
+            ${lib.optionalString (target.postUnmountHook != null) target.postUnmountHook}
+          ''
+        ) cfg.targets;
+
+      clanCore.backups.providers.localbackup = {
+        # TODO list needs to run locally or on the remote machine
+        list = "localbackup-list";
+        create = "localbackup-create";
+        restore = "localbackup-restore";
+      };
+    };
+}

clanModules/localsend.nix

@@ -1,7 +1,8 @@
-{ config
-, pkgs
-, lib
-, ...
+{
+  config,
+  pkgs,
+  lib,
+  ...
 }:
 {
   # Integration can be improved, if the following issues get implemented:
@@ -16,10 +17,7 @@
     package = lib.mkPackageOption pkgs "localsend" { };
   };
 
-  imports =
-    if config.clan.localsend.enable then
-      [
-        {
+  config = lib.mkIf config.clan.localsend.enable {
     clanCore.state.localsend.folders = [
       "/var/localsend"
       config.clan.localsend.defaultLocation
@@ -36,8 +34,5 @@
         Address = "192.168.56.2/24";
       };
     };
-        }
-      ]
-    else
-      [ ];
+  };
 }

clanModules/matrix-synapse.nix

@@ -0,0 +1,127 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.clan.matrix-synapse;
+in
+{
+  options.clan.matrix-synapse = {
+    enable = lib.mkEnableOption "Enable matrix-synapse";
+    domain = lib.mkOption {
+      type = lib.types.str;
+      description = "The domain name of the matrix server";
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    services.matrix-synapse = {
+      enable = true;
+      settings = {
+        server_name = cfg.domain;
+        database = {
+          args.user = "matrix-synapse";
+          args.database = "matrix-synapse";
+          name = "psycopg2";
+        };
+        turn_uris = [
+          "turn:turn.matrix.org?transport=udp"
+          "turn:turn.matrix.org?transport=tcp"
+        ];
+        listeners = [
+          {
+            port = 8008;
+            bind_addresses = [ "::1" ];
+            type = "http";
+            tls = false;
+            x_forwarded = true;
+            resources = [
+              {
+                names = [ "client" ];
+                compress = true;
+              }
+              {
+                names = [ "federation" ];
+                compress = false;
+              }
+            ];
+          }
+        ];
+      };
+      extraConfigFiles = [ "/var/lib/matrix-synapse/registration_shared_secret.yaml" ];
+    };
+    systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
+      "+${pkgs.writeScript "copy_registration_shared_secret" ''
+        #!/bin/sh
+        cp ${config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path} /var/lib/matrix-synapse/registration_shared_secret.yaml
+        chown matrix-synapse:matrix-synapse /var/lib/matrix-synapse/registration_shared_secret.yaml
+        chmod 600 /var/lib/matrix-synapse/registration_shared_secret.yaml
+      ''}"
+    ];
+
+    clanCore.facts.services."matrix-synapse" = {
+      secret."synapse-registration_shared_secret" = { };
+      generator.path = with pkgs; [
+        coreutils
+        pwgen
+      ];
+      generator.script = ''
+        echo "registration_shared_secret: $(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret
+      '';
+    };
+
+    services.postgresql.enable = true;
+    # we need to use both ensusureDatabases and initialScript, because the former runs everytime but with the wrong collation
+    services.postgresql = {
+      ensureDatabases = [ "matrix-synapse" ];
+      ensureUsers = [
+        {
+          name = "matrix-synapse";
+          ensureDBOwnership = true;
+        }
+      ];
+      initialScript = pkgs.writeText "synapse-init.sql" ''
+        CREATE DATABASE "matrix-synapse"
+          TEMPLATE template0
+          LC_COLLATE = "C"
+          LC_CTYPE = "C";
+      '';
+    };
+    services.nginx = {
+      enable = true;
+      virtualHosts = {
+        ${cfg.domain} = {
+          locations."= /.well-known/matrix/server".extraConfig = ''
+            add_header Content-Type application/json;
+            return 200 '${builtins.toJSON { "m.server" = "matrix.${cfg.domain}:443"; }}';
+          '';
+          locations."= /.well-known/matrix/client".extraConfig = ''
+            add_header Content-Type application/json;
+            add_header Access-Control-Allow-Origin *;
+            return 200 '${
+              builtins.toJSON {
+                "m.homeserver" = {
+                  "base_url" = "https://matrix.${cfg.domain}";
+                };
+                "m.identity_server" = {
+                  "base_url" = "https://vector.im";
+                };
+              }
+            }';
+          '';
+        };
+        "matrix.${cfg.domain}" = {
+          forceSSL = true;
+          enableACME = true;
+          locations."/_matrix" = {
+            proxyPass = "http://localhost:8008";
+          };
+          locations."/test".extraConfig = ''
+            return 200 "Hello, world!";
+          '';
+        };
+      };
+    };
+  };
+}

clanModules/moonlight.nix

@@ -1,4 +1,86 @@
-{ pkgs, ... }: {
+{ pkgs, config, ... }:
+let
+  ms-accept = pkgs.callPackage ../pkgs/moonlight-sunshine-accept { };
+  defaultPort = 48011;
+in
+{
   hardware.opengl.enable = true;
-  environment.systemPackages = [ pkgs.moonlight-qt ];
+  environment.systemPackages = [
+    pkgs.moonlight-qt
+    ms-accept
+  ];
+
+  systemd.tmpfiles.rules = [
+    "d '/var/lib/moonlight' 0770 'user' 'users' - -"
+    "C '/var/lib/moonlight/moonlight.cert' 0644 'user' 'users' - ${
+      config.clanCore.facts.services.moonlight.secret."moonlight.cert".path or ""
+    }"
+    "C '/var/lib/moonlight/moonlight.key' 0644 'user' 'users' - ${
+      config.clanCore.facts.services.moonlight.secret."moonlight.key".path or ""
+    }"
+  ];
+
+  systemd.user.services.init-moonlight = {
+    enable = false;
+    description = "Initializes moonlight";
+    wantedBy = [ "graphical-session.target" ];
+    script = ''
+      ${ms-accept}/bin/moonlight-sunshine-accept moonlight init-config --key /var/lib/moonlight/moonlight.key --cert /var/lib/moonlight/moonlight.cert
+    '';
+    serviceConfig = {
+      user = "user";
+      Type = "oneshot";
+      WorkingDirectory = "/home/user/";
+      RunTimeDirectory = "moonlight";
+      TimeoutSec = "infinity";
+      Restart = "on-failure";
+      RemainAfterExit = true;
+      ReadOnlyPaths = [
+        "/var/lib/moonlight/moonlight.key"
+        "/var/lib/moonlight/moonlight.cert"
+      ];
+    };
+  };
+
+  systemd.user.services.moonlight-join = {
+    description = "Join sunshine hosts";
+    script = ''${ms-accept}/bin/moonlight-sunshine-accept moonlight join --port ${builtins.toString defaultPort} --cert '${
+      config.clanCore.facts.services.moonlight.public."moonlight.cert".value or ""
+    }' --host fd2e:25da:6035:c98f:cd99:93e0:b9b8:9ca1'';
+    serviceConfig = {
+      Type = "oneshot";
+      TimeoutSec = "infinity";
+      Restart = "on-failure";
+      ReadOnlyPaths = [
+        "/var/lib/moonlight/moonlight.key"
+        "/var/lib/moonlight/moonlight.cert"
+      ];
+    };
+  };
+  systemd.user.timers.moonlight-join = {
+    description = "Join sunshine hosts";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnUnitActiveSec = "5min";
+      OnBootSec = "0min";
+      Persistent = true;
+      Unit = "moonlight-join.service";
+    };
+  };
+
+  clanCore.facts.services.moonlight = {
+    secret."moonlight.key" = { };
+    secret."moonlight.cert" = { };
+    public."moonlight.cert" = { };
+    generator.path = [
+      pkgs.coreutils
+      ms-accept
+    ];
+    generator.script = ''
+      moonlight-sunshine-accept moonlight init
+      mv credentials/cakey.pem "$secrets"/moonlight.key
+      cp credentials/cacert.pem "$secrets"/moonlight.cert
+      mv credentials/cacert.pem "$facts"/moonlight.cert
+    '';
+  };
 }

clanModules/root-password/README.md

@@ -0,0 +1,13 @@
+Creates a root-password
+
+!!! tip "This module sets the password for the root user (automatically)."
+
+After the system was installed/deployed the following command can be used to display the root-password:
+
+```bash
+clan secrets get {machine_name}-password
+```
+
+---
+
+See also: [Facts / Secrets](../../getting-started/secrets.md)

clanModules/root-password/default.nix

@@ -0,0 +1,20 @@
+{ pkgs, config, ... }:
+{
+  users.mutableUsers = false;
+  users.users.root.hashedPasswordFile =
+    config.clanCore.facts.services.root-password.secret.password-hash.path;
+  sops.secrets."${config.clanCore.machineName}-password-hash".neededForUsers = true;
+  clanCore.facts.services.root-password = {
+    secret.password = { };
+    secret.password-hash = { };
+    generator.path = with pkgs; [
+      coreutils
+      xkcdpass
+      mkpasswd
+    ];
+    generator.script = ''
+      xkcdpass --numwords 3 --delimiter - --count 1 > $secrets/password
+      cat $secrets/password | mkpasswd -s -m sha-512 > $secrets/password-hash
+    '';
+  };
+}

clanModules/sshd.nix

@@ -0,0 +1,24 @@
+{ config, pkgs, ... }:
+{
+  services.openssh.enable = true;
+
+  services.openssh.hostKeys = [
+    {
+      path = config.clanCore.facts.services.openssh.secret."ssh.id_ed25519".path;
+      type = "ed25519";
+    }
+  ];
+
+  clanCore.facts.services.openssh = {
+    secret."ssh.id_ed25519" = { };
+    public."ssh.id_ed25519.pub" = { };
+    generator.path = [
+      pkgs.coreutils
+      pkgs.openssh
+    ];
+    generator.script = ''
+      ssh-keygen -t ed25519 -N "" -f $secrets/ssh.id_ed25519
+      mv $secrets/ssh.id_ed25519.pub $facts/ssh.id_ed25519.pub
+    '';
+  };
+}

clanModules/sunshine.nix

@@ -1,4 +1,21 @@
-{ pkgs, config, ... }:
+{
+  pkgs,
+  config,
+  lib,
+  ...
+}:
+let
+  ms-accept = pkgs.callPackage ../pkgs/moonlight-sunshine-accept { };
+  sunshineConfiguration = pkgs.writeText "sunshine.conf" ''
+    address_family = both
+    channels = 5
+    pkey = /var/lib/sunshine/sunshine.key
+    cert = /var/lib/sunshine/sunshine.cert
+    file_state = /var/lib/sunshine/state.json
+    credentials_file = /var/lib/sunshine/credentials.json
+  '';
+  listenPort = 48011;
+in
 {
   networking.firewall = {
     allowedTCPPorts = [
@@ -6,6 +23,7 @@
       47989
       47990
       48010
+      48011
     ];
 
     allowedUDPPorts = [
@@ -29,81 +47,161 @@
       to = 48010;
     }
   ];
+  networking.firewall.interfaces."zt+".allowedTCPPorts = [
+    47984
+    47989
+    47990
+    48010
+    listenPort
+  ];
+  networking.firewall.interfaces."zt+".allowedUDPPortRanges = [
+    {
+      from = 47998;
+      to = 48010;
+    }
+  ];
 
   environment.systemPackages = [
+    ms-accept
     pkgs.sunshine
     pkgs.avahi
     # Convenience script, until we find a better UX
     (pkgs.writers.writeDashBin "sun" ''
-      ${pkgs.sunshine}/bin/sunshine -1 ${
-        pkgs.writeText "sunshine.conf" ''
-          address_family = both
-        ''
-      } "$@"
+      ${pkgs.sunshine}/bin/sunshine -0 ${sunshineConfiguration} "$@"
     '')
     # Create a dummy account, for easier setup,
     # don't use this account in actual production yet.
     (pkgs.writers.writeDashBin "init-sun" ''
       ${pkgs.sunshine}/bin/sunshine \
-      --creds "sun" "sun"
+      --creds "sunshine" "sunshine"
     '')
   ];
 
   # Required to simulate input
   boot.kernelModules = [ "uinput" ];
-  security.rtkit.enable = true;
-
-  # services.udev.extraRules = ''
-  #   KERNEL=="uinput", SUBSYSTEM=="misc", OPTIONS+="static_node=uinput", TAG+="uaccess"
-  # '';
 
   services.udev.extraRules = ''
-    KERNEL=="uinput", GROUP="input", MODE="0660" OPTIONS+="static_node=uinput"
+    KERNEL=="uinput", SUBSYSTEM=="misc", OPTIONS+="static_node=uinput", TAG+="uaccess"
   '';
 
-  security.wrappers.sunshine = {
+  security = {
+    rtkit.enable = true;
+    wrappers.sunshine = {
       owner = "root";
       group = "root";
       capabilities = "cap_sys_admin+p";
       source = "${pkgs.sunshine}/bin/sunshine";
     };
+  };
+
+  systemd.tmpfiles.rules = [
+    "d '/var/lib/sunshine' 0770 'user' 'users' - -"
+    "C '/var/lib/sunshine/sunshine.cert' 0644 'user' 'users' - ${
+      config.clanCore.facts.services.sunshine.secret."sunshine.cert".path or ""
+    }"
+    "C '/var/lib/sunshine/sunshine.key' 0644 'user' 'users' - ${
+      config.clanCore.facts.services.sunshine.secret."sunshine.key".path or ""
+    }"
+  ];
+
+  hardware.opengl.enable = true;
 
   systemd.user.services.sunshine = {
-    description = "sunshine";
-    wantedBy = [ "graphical-session.target" ];
-    environment = {
-      DISPLAY = ":0";
-    };
+    enable = true;
+    description = "Sunshine self-hosted game stream host for Moonlight";
+    startLimitBurst = 5;
+    startLimitIntervalSec = 500;
+    script = "/run/current-system/sw/bin/env /run/wrappers/bin/sunshine ${sunshineConfiguration}";
     serviceConfig = {
-      ExecStart = "${config.security.wrapperDir}/sunshine";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      ReadWritePaths = [ "/var/lib/sunshine" ];
+      ReadOnlyPaths = [
+        (config.clanCore.facts.services.sunshine.secret."sunshine.key".path or "")
+        (config.clanCore.facts.services.sunshine.secret."sunshine.cert".path or "")
+      ];
     };
+    wantedBy = [ "graphical-session.target" ];
+    partOf = [ "graphical-session.target" ];
+    wants = [ "graphical-session.target" ];
+    after = [
+      "sunshine-init-state.service"
+      "sunshine-init-credentials.service"
+    ];
   };
 
-  # xdg.configFile."sunshine/apps.json".text = builtins.toJSON {
-  #   env = "/run/current-system/sw/bin";
-  #   apps = [
-  #     {
-  #       name = "Steam";
-  #       output = "steam.txt";
-  #       detached = [
-  #         "${pkgs.util-linux}/bin/setsid ${pkgs.steam}/bin/steam steam://open/bigpicture"
-  #       ];
-  #       image-path = "steam.png";
-  #     }
-  #   ];
-  # };
+  systemd.user.services.sunshine-init-state = {
+    enable = true;
+    description = "Sunshine self-hosted game stream host for Moonlight";
+    startLimitBurst = 5;
+    startLimitIntervalSec = 500;
+    script = ''
+      ${ms-accept}/bin/moonlight-sunshine-accept sunshine init-state --uuid ${
+        config.clanCore.facts.services.sunshine.public.sunshine-uuid.value or null
+      } --state-file /var/lib/sunshine/state.json
+    '';
+    serviceConfig = {
+      Restart = "on-failure";
+      RestartSec = "5s";
+      Type = "oneshot";
+      ReadWritePaths = [ "/var/lib/sunshine" ];
+    };
+    wantedBy = [ "graphical-session.target" ];
+  };
 
-  services = {
-    avahi = {
+  systemd.user.services.sunshine-init-credentials = {
     enable = true;
-      reflector = true;
-      nssmdns = true;
-      publish = {
+    description = "Sunshine self-hosted game stream host for Moonlight";
+    startLimitBurst = 5;
+    startLimitIntervalSec = 500;
+    script = ''
+      ${lib.getExe pkgs.sunshine} ${sunshineConfiguration} --creds sunshine sunshine
+    '';
+    serviceConfig = {
+      Restart = "on-failure";
+      RestartSec = "5s";
+      Type = "oneshot";
+      ReadWritePaths = [ "/var/lib/sunshine" ];
+    };
+    wantedBy = [ "graphical-session.target" ];
+  };
+
+  systemd.user.services.sunshine-listener = {
     enable = true;
-        addresses = true;
-        userServices = true;
-        workstation = true;
+    description = "Sunshine self-hosted game stream host for Moonlight";
+    startLimitBurst = 5;
+    startLimitIntervalSec = 500;
+    script = ''
+      ${ms-accept}/bin/moonlight-sunshine-accept sunshine listen --port ${builtins.toString listenPort} --uuid ${
+        config.clanCore.facts.services.sunshine.public.sunshine-uuid.value or null
+      }  --state /var/lib/sunshine/state.json --cert '${
+        config.clanCore.facts.services.sunshine.public."sunshine.cert".value or null
+      }'
+    '';
+    serviceConfig = {
+      # );
+      Restart = "on-failure";
+      RestartSec = 5;
+      ReadWritePaths = [ "/var/lib/sunshine" ];
     };
+    wantedBy = [ "graphical-session.target" ];
   };
+
+  clanCore.facts.services.ergochat = {
+    secret."sunshine.key" = { };
+    secret."sunshine.cert" = { };
+    public."sunshine-uuid" = { };
+    public."sunshine.cert" = { };
+    generator.path = [
+      pkgs.coreutils
+      ms-accept
+    ];
+    generator.script = ''
+      moonlight-sunshine-accept sunshine init
+      mv credentials/cakey.pem "$secrets"/sunshine.key
+      cp credentials/cacert.pem "$secrets"/sunshine.cert
+      mv credentials/cacert.pem "$facts"/sunshine.cert
+      mv uuid "$facts"/sunshine-uuid
+    '';
   };
 }

clanModules/syncthing/README.md

@@ -0,0 +1,34 @@
+Syncthing is a free, open-source file synchronization application designed to allow users to synchronize files between multiple devices over the internet or local networks securely and privately. 
+
+It is an alternative to cloud-based file sharing services.
+
+## Usage
+
+We recommend configuring this module as an sync-service through the provided options. Although it provides a Web GUI through which more usage scenarios are supported.
+
+## Features
+
+- **Private and Secure**: Syncthing uses TLS encryption to secure data transfer between devices, ensuring that only the intended devices can read your data.
+- **Decentralized**: No central server is involved in the data transfer. Each device communicates directly with others.
+- **Open Source**: The source code is openly available for audit and contribution, fostering trust and continuous improvement.
+- **Cross-Platform**: Syncthing supports multiple platforms including Windows, macOS, Linux, BSD, and Android.
+- **Real-time Synchronization**: Changes made to files are synchronized in real-time across all connected devices.
+- **Web GUI**: It includes a user-friendly web interface for managing devices and configurations. (`127.0.0.1:8384`)
+
+## Configuration
+
+- **Share Folders**: Select folders to share with connected devices and configure permissions and synchronization parameters.
+
+!!! info
+    Clan automatically discovers other devices. Automatic discovery requires one machine to be an [introducer](#clan.syncthing.introducer)
+
+    If that is not the case you can add the other device by its Device ID manually.
+    You can find and share Device IDs under the "Add Device" button in the Web GUI. (`127.0.0.1:8384`)
+
+## Troubleshooting
+
+- **Sync Conflicts**: Resolve synchronization conflicts manually by reviewing file versions and modification times in the Web GUI (`127.0.0.1:8384`).
+
+## Support
+
+- **Documentation**: Extensive documentation is available on the [Syncthing website](https://docs.syncthing.net/).

clanModules/syncthing/default.nix

@@ -1,14 +1,16 @@
-{ config
-, pkgs
-, lib
-, ...
+{
+  config,
+  pkgs,
+  lib,
+  ...
 }:
 {
   options.clan.syncthing = {
     id = lib.mkOption {
       type = lib.types.nullOr lib.types.str;
       example = "BABNJY4-G2ICDLF-QQEG7DD-N3OBNGF-BCCOFK6-MV3K7QJ-2WUZHXS-7DTW4AS";
-      default = config.clanCore.secrets.syncthing.facts."syncthing.pub".value or null;
+      default = config.clanCore.facts.services.syncthing.public."syncthing.pub".value or null;
+      defaultText = "config.clanCore.facts.services.syncthing.public.\"syncthing.pub\".value";
     };
     introducer = lib.mkOption {
       description = ''
@@ -53,17 +55,19 @@
 
       assertions = [
         {
-          assertion =
-            lib.all (attr: builtins.hasAttr attr config.services.syncthing.settings.folders)
-              config.clan.syncthing.autoShares;
+          assertion = lib.all (
+            attr: builtins.hasAttr attr config.services.syncthing.settings.folders
+          ) config.clan.syncthing.autoShares;
           message = ''
             Syncthing: If you want to AutoShare a folder, you need to have it configured on the sharing device.
           '';
         }
       ];
 
-      # Activates inofify compatibilty on syncthing
-      boot.kernel.sysctl."fs.inotify.max_user_watches" = lib.mkDefault 524288;
+      # Activates inotify compatibility on syncthing
+      # use mkOverride 900 here as it otherwise would collide with the default of the
+      # upstream nixos xserver.nix
+      boot.kernel.sysctl."fs.inotify.max_user_watches" = lib.mkOverride 900 524288;
 
       services.syncthing = {
         enable = true;
@@ -80,12 +84,8 @@
 
         group = "syncthing";
 
-        key =
-          lib.mkDefault
-            config.clan.secrets.syncthing.secrets."syncthing.key".path or null;
-        cert =
-          lib.mkDefault
-            config.clan.secrets.syncthing.secrets."syncthing.cert".path or null;
+        key = lib.mkDefault config.clan.secrets.syncthing.secrets."syncthing.key".path or null;
+        cert = lib.mkDefault config.clan.secrets.syncthing.secrets."syncthing.cert".path or null;
 
         settings = {
           options = {
@@ -115,7 +115,7 @@
           getPendingDevices = "/rest/cluster/pending/devices";
           postNewDevice = "/rest/config/devices";
           SharedFolderById = "/rest/config/folders/";
-          apiKey = config.clanCore.secrets.syncthing.secrets."syncthing.api".path or null;
+          apiKey = config.clanCore.facts.services.syncthing.secret."syncthing.api".path or null;
         in
         lib.mkIf config.clan.syncthing.autoAcceptDevices {
           description = "Syncthing auto accept devices";
@@ -127,38 +127,24 @@
             set -x
             # query pending deviceID's
             APIKEY=$(cat ${apiKey})
-            PENDING=$(${
-              lib.getExe pkgs.curl
-            } -X GET -H "X-API-Key: $APIKEY" ${baseAddress}${getPendingDevices})
+            PENDING=$(${lib.getExe pkgs.curl} -X GET -H "X-API-Key: $APIKEY" ${baseAddress}${getPendingDevices})
             PENDING=$(echo $PENDING | ${lib.getExe pkgs.jq} keys[])
 
             # accept pending deviceID's
             for ID in $PENDING;do
-            ${
-              lib.getExe pkgs.curl
-            } -X POST -d "{\"deviceId\": $ID}" -H "Content-Type: application/json" -H "X-API-Key: $APIKEY" ${baseAddress}${postNewDevice}
+            ${lib.getExe pkgs.curl} -X POST -d "{\"deviceId\": $ID}" -H "Content-Type: application/json" -H "X-API-Key: $APIKEY" ${baseAddress}${postNewDevice}
 
             # get all shared folders by their ID
             for folder in ${builtins.toString config.clan.syncthing.autoShares}; do
-              SHARED_IDS=$(${
-                lib.getExe pkgs.curl
-              } -X GET -H "X-API-Key: $APIKEY" ${baseAddress}${SharedFolderById}"$folder" | ${
-                lib.getExe pkgs.jq
-              } ."devices")
-              PATCHED_IDS=$(echo $SHARED_IDS | ${
-                lib.getExe pkgs.jq
-              } ".+= [{\"deviceID\": $ID, \"introducedBy\": \"\", \"encryptionPassword\": \"\"}]")
-              ${
-                lib.getExe pkgs.curl
-              } -X PATCH -d "{\"devices\": $PATCHED_IDS}" -H "X-API-Key: $APIKEY" ${baseAddress}${SharedFolderById}"$folder"
+              SHARED_IDS=$(${lib.getExe pkgs.curl} -X GET -H "X-API-Key: $APIKEY" ${baseAddress}${SharedFolderById}"$folder" | ${lib.getExe pkgs.jq} ."devices")
+              PATCHED_IDS=$(echo $SHARED_IDS | ${lib.getExe pkgs.jq} ".+= [{\"deviceID\": $ID, \"introducedBy\": \"\", \"encryptionPassword\": \"\"}]")
+              ${lib.getExe pkgs.curl} -X PATCH -d "{\"devices\": $PATCHED_IDS}" -H "X-API-Key: $APIKEY" ${baseAddress}${SharedFolderById}"$folder"
               done
             done
           '';
         };
 
-      systemd.timers.syncthing-auto-accept =
-        lib.mkIf config.clan.syncthing.autoAcceptDevices
-          {
+      systemd.timers.syncthing-auto-accept = lib.mkIf config.clan.syncthing.autoAcceptDevices {
         description = "Syncthing Auto Accept";
 
         wantedBy = [ "syncthing-auto-accept.service" ];
@@ -171,7 +157,7 @@
 
       systemd.services.syncthing-init-api-key =
         let
-          apiKey = config.clanCore.secrets.syncthing.secrets."syncthing.api".path or null;
+          apiKey = config.clanCore.facts.services.syncthing.secret."syncthing.api".path or null;
         in
         lib.mkIf config.clan.syncthing.autoAcceptDevices {
           description = "Set the api key";
@@ -182,9 +168,7 @@
             set -efu pipefail
 
             APIKEY=$(cat ${apiKey})
-            ${
-              lib.getExe pkgs.gnused
-            } -i "s/<apikey>.*<\/apikey>/<apikey>$APIKEY<\/apikey>/" /var/lib/syncthing/config.xml
+            ${lib.getExe pkgs.gnused} -i "s/<apikey>.*<\/apikey>/<apikey>$APIKEY<\/apikey>/" /var/lib/syncthing/config.xml
             # sudo systemctl restart syncthing.service
             systemctl restart syncthing.service
           '';
@@ -195,11 +179,11 @@
           };
         };
 
-      clanCore.secrets.syncthing = {
-        secrets."syncthing.key" = { };
-        secrets."syncthing.cert" = { };
-        secrets."syncthing.api" = { };
-        facts."syncthing.pub" = { };
+      clanCore.facts.services.syncthing = {
+        secret."syncthing.key" = { };
+        secret."syncthing.cert" = { };
+        secret."syncthing.api" = { };
+        public."syncthing.pub" = { };
         generator.path = [
           pkgs.coreutils
           pkgs.gnugrep

clanModules/thelounge.nix

@@ -0,0 +1,15 @@
+_: {
+  services.thelounge = {
+    enable = true;
+    public = true;
+    extraConfig = {
+      prefetch = true;
+      defaults = {
+        port = 6667;
+        tls = false;
+      };
+    };
+  };
+
+  clanCore.state.thelounde.folders = [ "/var/lib/thelounge" ];
+}

clanModules/vm-user.nix

@@ -0,0 +1,20 @@
+{
+  security = {
+    sudo.wheelNeedsPassword = false;
+    polkit.enable = true;
+    rtkit.enable = true;
+  };
+
+  users.users.user = {
+    isNormalUser = true;
+    createHome = true;
+    uid = 1000;
+    initialHashedPassword = "";
+    extraGroups = [
+      "wheel"
+      "video"
+      "render"
+    ];
+    shell = "/run/current-system/sw/bin/bash";
+  };
+}

clanModules/waypipe.nix

@@ -1,7 +1,8 @@
-{ pkgs
-, lib
-, config
-, ...
+{
+  pkgs,
+  lib,
+  config,
+  ...
 }:
 {
   options.clan.services.waypipe = {
@@ -49,7 +50,10 @@
       isNormalUser = true;
       uid = 1000;
       password = "";
-      extraGroups = [ "wheel" "video" ];
+      extraGroups = [
+        "wheel"
+        "video"
+      ];
       shell = "/run/current-system/sw/bin/bash";
     };
 
@@ -64,7 +68,7 @@
         SDL_VIDEODRIVER=wayland
       '';
       script = ''
-        ${lib.getExe config.clanCore.clanPkgs.waypipe} \
+        ${lib.getExe pkgs.waypipe} \
         ${lib.escapeShellArgs config.clan.services.waypipe.flags} \
         ${lib.escapeShellArgs config.clan.services.waypipe.command}
       '';

clanModules/xfce-vm.nix

@@ -0,0 +1,15 @@
+{
+  imports = [
+    ./vm-user.nix
+    ./graphical.nix
+  ];
+
+  services.xserver = {
+    enable = true;
+    displayManager.autoLogin.enable = true;
+    displayManager.autoLogin.user = "user";
+    desktopManager.xfce.enable = true;
+    desktopManager.xfce.enableScreensaver = false;
+    xkb.layout = "us";
+  };
+}

clanModules/zt-tcp-relay.nix

@@ -1,4 +1,10 @@
-{ pkgs, lib, config, ... }: {
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+{
   options.clan.zt-tcp-relay = {
     port = lib.mkOption {
       type = lib.types.port;
@@ -13,7 +19,9 @@
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
       serviceConfig = {
-        ExecStart = "${pkgs.callPackage ../pkgs/zt-tcp-relay {}}/bin/zt-tcp-relay --listen [::]:${builtins.toString config.clan.zt-tcp-relay.port}";
+        ExecStart = "${
+          pkgs.callPackage ../pkgs/zt-tcp-relay { }
+        }/bin/zt-tcp-relay --listen [::]:${builtins.toString config.clan.zt-tcp-relay.port}";
         Restart = "always";
         RestartSec = "5";
         dynamicUsers = true;

devShell.nix

@@ -1,13 +1,32 @@
 {
   perSystem =
-    { pkgs
-    , self'
-    , config
-    , ...
-    }: {
+    {
+      pkgs,
+      self',
+      config,
+      ...
+    }:
+    let
+      writers = pkgs.callPackage ./pkgs/builders/script-writers.nix { };
+
+      ansiEscapes = {
+        reset = ''\033[0m'';
+        green = ''\033[32m'';
+      };
+
+      # A python program to switch between dev-shells
+      # usage: select-shell shell-name
+      # the currently enabled dev-shell gets stored in ./.direnv/selected-shell
+      select-shell = writers.writePython3Bin "select-shell" {
+        flakeIgnore = [ "E501" ];
+      } ./pkgs/scripts/select-shell.py;
+    in
+    {
       devShells.default = pkgs.mkShell {
         packages = [
+          select-shell
           pkgs.tea
+          pkgs.nix
           self'.packages.tea-create-pr
           self'.packages.merge-after-ci
           self'.packages.pending-reviews
@@ -17,6 +36,8 @@
         shellHook = ''
           # no longer used
           rm -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
+
+          echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}"
         '';
       };
     };

docs/.envrc

@@ -0,0 +1,6 @@
+source_up
+
+watch_file $(find ./nix -name "*.nix" -printf '"%p" ')
+
+# Because we depend on nixpkgs sources, uploading to builders takes a long time
+use flake .#docs --builders ''

docs/.gitignore

@@ -0,0 +1 @@
+/site/reference

docs/CONTRIBUTING.md

@@ -1,10 +1,6 @@
 # Contributing
 
-**Frontend**: Our frontend is powered by [React NextJS](https://nextjs.org/), a popular and versatile framework for building web applications.
-
-**Backend**: For the backend, we use Python along with the [FastAPI framework](https://fastapi.tiangolo.com/). To ensure seamless communication between the frontend and backend, we generate an `openapi.json` file from the Python code, which defines the REST API. This file is then used with [Orval](https://orval.dev/) to generate TypeScript bindings for the REST API. We're committed to code correctness, so we use [mypy](https://mypy-lang.org/) to ensure that our Python code is statically typed correctly. For backend testing, we rely on [pytest](https://docs.pytest.org/en/7.4.x/).
-
-**Continuous Integration (CI)**: We've set up a CI bot that rigorously checks your code using the quality assurance (QA) tools mentioned above. If any errors are detected, it will block pull requests until they're resolved.
+**Continuous Integration (CI)**: Each pull request gets automatically tested by gitea. If any errors are detected, it will block pull requests until they're resolved.
 
 **Dependency Management**: We use the [Nix package manager](https://nixos.org/) to manage dependencies and ensure reproducibility, making your development process more robust.
 
@@ -34,7 +30,7 @@ Let's get your development environment up and running:
 3. **Add direnv to your shell**:
 
    - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
-     You can do this by executing following command:
+     You can do this by executing following command. The example below will setup direnv for `zsh` and `bash`
 
    ```bash
    echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
@@ -52,39 +48,6 @@ Let's get your development environment up and running:
      ```
    - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
 
-6. **Build the Backend**:
-
-   - Go to the `pkgs/clan-cli` directory and execute:
-     ```bash
-     direnv allow
-     ```
-   - Wait for the backend to build.
-
-7. **Start the Backend Server**:
-
-   - To start the backend server, execute:
-     ```bash
-     clan webui --reload --no-open --log-level debug
-     ```
-   - The server will automatically restart if any Python files change.
-
-8. **Build the Frontend**:
-
-   - In a different shell, navigate to the `pkgs/ui` directory and execute:
-     ```bash
-     direnv allow
-     ```
-   - Wait for the frontend to build.
-
-   NOTE: If you have the error "@clan/colors.json" you executed `npm install`, please do not do that. `direnv reload` will handle dependency management. Please delete node_modules with `rm -rf node_modules`.
-
-9. **Start the Frontend**:
-   - To start the frontend, execute:
-     ```bash
-     npm run dev
-     ```
-   - Access the website by going to [http://localhost:3000](http://localhost:3000).
-
 # Setting Up Your Git Workflow
 
 Let's set up your Git workflow to collaborate effectively:
@@ -96,7 +59,7 @@ Let's set up your Git workflow to collaborate effectively:
      tea login add
      ```
    - Fill out the prompt as follows:
-     - URL of Gitea instance: `https://gitea.gchq.icu`
+     - URL of Gitea instance: `https://git.clan.lol`
      - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171`
      - Do you have an access token? No
      - Username: YourUsername
@@ -125,7 +88,7 @@ Let's set up your Git workflow to collaborate effectively:
 
 4. **Review Your Pull Request**:
 
-   - Visit https://gitea.gchq.icu and go to the project page. Check under "Pull Requests" for any issues with your pull request.
+   - Visit https://git.clan.lol and go to the project page. Check under "Pull Requests" for any issues with your pull request.
 
 5. **Push Your Changes**:
    - If there are issues, fix them and redo step 2. Afterward, execute:
@@ -136,21 +99,22 @@ Let's set up your Git workflow to collaborate effectively:
 
 # Debugging
 
-When working on the backend of your project, debugging is an essential part of the development process. Here are some methods for debugging and testing the backend of your application:
+Here are some methods for debugging and testing the clan-cli:
 
-## Test Backend Locally in Devshell with Breakpoints
+## Test Locally in Devshell with Breakpoints
 
-To test the backend locally in a development environment and set breakpoints for debugging, follow these steps:
+To test the cli locally in a development environment and set breakpoints for debugging, follow these steps:
 
 1. Run the following command to execute your tests and allow for debugging with breakpoints:
    ```bash
-   pytest -n0 -s --maxfail=1
+   cd ./pkgs/clan-cli
+   pytest -n0 -s --maxfail=1 ./tests/test_nameofthetest.py
    ```
    You can place `breakpoint()` in your Python code where you want to trigger a breakpoint for debugging.
 
-## Test Backend Locally in a Nix Sandbox
+## Test Locally in a Nix Sandbox
 
-To run your backend tests in a Nix sandbox, you have two options depending on whether your test functions have been marked as impure or not:
+To run tests in a Nix sandbox, you have two options depending on whether your test functions have been marked as impure or not:
 
 ### Running Tests Marked as Impure
 
@@ -190,28 +154,8 @@ If you need to inspect the Nix sandbox while running tests, follow these steps:
    psgrep -a -x your_python_process_name
    ```
 
-These debugging and testing methods will help you identify and fix issues in your backend code efficiently, ensuring the reliability and robustness of your application.
 
-For more information on testing read [property and contract based testing](testing.md)
+# Standards
 
-# Using this Template
-
-To make the most of this template:
-
-1. Set up a new Gitea account named `ui-asset-bot`. Generate an access token with all access permissions and set it under `settings/actions/secrets` as a secret called `BOT_ACCESS_TOKEN`.
-
-   - Also, edit the file `.gitea/workflows/ui_assets.yaml` and change the `BOT_EMAIL` variable to match the email you set for that account. Gitea matches commits to accounts by their email address, so this step is essential.
-
-2. Create a second Gitea account named `merge-bot`. Edit the file `pkgs/merge-after-ci/default.nix` if the name should be different. Under "Branches," set the main branch to be protected and add `merge-bot` to the whitelisted users for pushing. Set the unprotected file pattern to `**/ui-assets.nix`.
-
-   - Enable the status check for "build / test (pull_request)."
-
-3. Add both `merge-bot` and `ui-asset-bot` as collaborators.
-   - Set the option to "Delete pull request branch after merge by default."
-   - Also, set the default merge style to "Rebase then create merge commit."
-
-With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!.
-
-# API guidelines
-
-see [./api-guidelines](./api-guidelines)
+Every new module name should be in kebab-case.
+Every fact definition, where possible should be in kebab-case.

docs/admins/_index.md

@@ -1,10 +0,0 @@
-+++
-title = "Admin Documentation"
-description = "Documentation administrators creating or managing cLANs"
-date = 2025-05-01T19:00:00+00:00
-updated = 2021-05-01T19:00:00+00:00
-template = "docs/section.html"
-weight = 15
-sort_by = "title"
-draft = false
-+++

docs/admins/clan-config.md

@@ -1,69 +0,0 @@
-# cLAN config
-
-`clan config` allows you to manage your nixos configuration via the terminal.
-Similar as how `git config` reads and sets git options, `clan config` does the same with your nixos options
-It also supports auto completion making it easy to find the right options.
-
-## Set up clan-config
-
-Add the clan tool to your flake inputs:
-
-```
-clan.url = "git+https://git.clan.lol/clan/clan-core";
-```
-
-and inside the mkFlake:
-
-```
-imports = [
-  inputs.clan.flakeModules.clan-config
-];
-```
-
-Add an empty config file and add it to git
-
-```command
-echo "{}" > ./clan-settings.json
-git add ./clan-settings.json
-```
-
-Import the clan-config module into your nixos configuration:
-
-```nix
-{
-  imports = [
-    # clan-settings.json is located in the same directory as your flake.
-    # Adapt the path if necessary.
-    (builtins.fromJSON (builtins.readFile ./clan-settings.json))
-  ];
-}
-
-
-```
-
-Make sure your nixos configuration is set a default
-
-```nix
-{self, ...}: {
-  flake.nixosConfigurations.default = self.nixosConfigurations.my-machine;
-}
-```
-
-Use all inputs provided by the clan-config devShell in your own devShell:
-
-```nix
-{ ... }: {
-  perSystem = { pkgs, self', ... }: {
-    devShells.default = pkgs.mkShell {
-      inputsFrom = [ self'.devShells.clan-config ];
-      # ...
-    };
-  };
-}
-```
-
-re-load your dev-shell to make the clan tool available.
-
-```command
-clan config --help
-```

docs/admins/machines.md

@@ -1,138 +0,0 @@
-# Managing NixOS Machines
-
-## Add Your First Machine
-
-To start managing a new machine, use the following commands to create and then list your machines:
-
-```shellSession
-$ clan machines create my-machine
-$ clan machines list
-my-machine
-```
-
-## Configure Your Machine
-
-In the example below, we demonstrate how to add a new user named `my-user` and set a password. This user will be configured to log in to the machine `my-machine`.
-
-### Creating a New User
-
-```shellSession
-# Add a new user
-$ clan config --machine my-machine users.users.my-user.isNormalUser true
-
-# Set a password for the user
-$ clan config --machine my-machine users.users.my-user.hashedPassword $(mkpasswd)
-```
-
-_Note: The `$(mkpasswd)` command generates a hashed password. Ensure you have the `mkpasswd` utility installed or use an alternative method to generate a secure hashed password._
-
-## Test Your Machine Configuration Inside a VM
-
-Before deploying your configuration to a live environment, you can run a virtual machine (VM) to test the settings:
-
-```shellSession
-$ clan vms run my-machine
-```
-
-This command run a VM based on the configuration of `my-machine`, allowing you to verify changes in a controlled environment.
-
-## Installing a New Machine
-
-Clan CLI, in conjunction with [nixos-anywhere](https://github.com/nix-community/nixos-anywhere), provides a seamless method for installing NixOS on various machines.
-This process involves preparing a suitable hardware and disk partitioning configuration and ensuring the target machine is accessible via SSH.
-
-### Prerequisites
-
-- A running Linux system with SSH on the target machine is required. This is typically pre-configured for many server providers.
-- For installations on physical hardware, create a NixOS installer image and transfer it to a bootable USB drive as described below.
-
-## Creating a Bootable USB Drive on Linux
-
-To create a bootable USB flash drive with the NixOS installer:
-
-1. **Build the Installer Image**:
-
-   ```shellSession
-   $ nix build git+https://git.clan.lol/clan/clan-core.git#install-iso
-   ```
-
-2. **Prepare the USB Flash Drive**:
-
-   - Insert your USB flash drive into your computer.
-   - Identify your flash drive with `lsblk`. Look for the device with a matching size.
-   - Ensure all partitions on the drive are unmounted. Replace `sdX` in the command below with your device identifier (like `sdb`, etc.):
-
-     ```shellSession
-     sudo umount /dev/sdX*
-     ```
-
-3. **Write the Image to the USB Drive**:
-
-   - Use the `dd` utility to write the NixOS installer image to your USB drive:
-
-     ```shellSession
-     sudo dd bs=4M conv=fsync oflag=direct status=progress if=./result/stick.raw of=/dev/sdX
-     ```
-
-4. **Boot and Connect**:
-   - After writing the installer to the USB drive, use it to boot the target machine.
-   - The installer will display an IP address and a root password, which you can use to connect via SSH.
-
-### Finishing the installation
-
-With the target machine running Linux and accessible via SSH, execute the following command to install NixOS on the target machine, replacing `<target_host>` with the machine's hostname or IP address:
-
-```shellSession
-$ clan machines install my-machine <target_host>
-```
-
-## Update Your Machines
-
-Clan CLI enables you to remotely update your machines over SSH. This requires setting up a target address for each target machine.
-
-### Setting the Target Host
-
-Replace `host_or_ip` with the actual hostname or IP address of your target machine:
-
-```shellSession
-$ clan config --machine my-machine clan.networking.targetHost root@host_or_ip
-```
-
-_Note: The use of `root@` in the target address implies SSH access as the root user.
-Ensure that the root login is secured and only used when necessary._
-
-### Updating Machine Configurations
-
-Execute the following command to update the specified machine:
-
-```shellSession
-$ clan machines update my-machine
-```
-
-You can also update all configured machines simultaneously by omitting the machine name:
-
-```shellSession
-$ clan machines update
-```
-
-### Setting a Build Host
-
-If the machine does not have enough resources to run the NixOS evaluation or build itself,
-it is also possible to specify a build host instead.
-During an update, the cli will ssh into the build host and run `nixos-rebuild` from there.
-
-```shellSession
-$ clan config --machine my-machine clan.networking.buildHost root@host_or_ip
-```
-
-### Excluding a machine from `clan machine update`
-
-To exclude machines from beeing updated when running `clan machines update` without any machines specified,
-one can set the `clan.deployment.requireExplicitUpdate` option to true:
-
-
-```shellSession
-$ clan config --machine my-machine clan.deployment.requireExplicitUpdate true
-```
-
-This is useful for machines that are not always online or are not part of the regular update cycle.

docs/admins/quickstart.md

@@ -1,135 +0,0 @@
-# Initializing a New Clan Project
-
-## Create a new flake
-
-1. To start a new project, execute the following command to add the clan cli to your shell:
-
-```shellSession
-$ nix shell git+https://git.clan.lol/clan/clan-core
-```
-
-2. Then use the following commands to initialize a new clan-flake:
-
-```shellSession
-$ clan flake create my-clan
-```
-
-This action will generate two primary files: `flake.nix` and `.clan-flake`.
-
-```shellSession
-$ ls -la
-drwx------ joerg users   5 B  a minute ago   ./
-drwxrwxrwt root  root  139 B  12 seconds ago ../
-.rw-r--r-- joerg users  77 B  a minute ago   .clan-flake
-.rw-r--r-- joerg users 4.8 KB a minute ago   flake.lock
-.rw-r--r-- joerg users 242 B  a minute ago   flake.nix
-```
-
-### Understanding the .clan-flake Marker File
-
-The `.clan-flake` marker file serves an optional purpose: it helps the `clan-cli` utility locate the project's root directory.
-If `.clan-flake` is missing, `clan-cli` will instead search for other indicators like `.git`, `.hg`, `.svn`, or `flake.nix` to identify the project root.
-
-## What's next
-
-After creating your flake, you can check out how to add [new machines](./machines.md)
-
----
-
-# Migrating Existing NixOS Configuration Flake
-
-Absolutely, let's break down the migration step by step, explaining each action in detail:
-
-#### Before You Begin
-
-1. **Backup Your Current Configuration**: Always start by making a backup of your current NixOS configuration to ensure you can revert if needed.
-
-   ```shellSession
-   $ cp -r /etc/nixos ~/nixos-backup
-   ```
-
-2. **Update Flake Inputs**: Add a new input for the `clan-core` dependency:
-
-   ```nix
-   inputs.clan-core = {
-     url = "git+https://git.clan.lol/clan/clan-core";
-     # Don't do this if your machines are on nixpkgs stable.
-     inputs.nixpkgs.follows = "nixpkgs";
-   };
-   ```
-
-   - `url`: Specifies the Git repository URL for Clan Core.
-   - `inputs.nixpkgs.follows`: Tells Nix to use the same `nixpkgs` input as your main input (in this case, it follows `nixpkgs`).
-
-3. **Update Outputs**: Then modify the `outputs` section of your `flake.nix` to adapt to Clan Core's new provisioning method. The key changes are as follows:
-
-   Add `clan-core` to the output
-
-   ```diff
-   -  outputs = { self, nixpkgs,  }:
-   +  outputs = { self, nixpkgs, clan-core }:
-   ```
-
-   Previous configuration:
-
-   ```nix
-   {
-       nixosConfigurations.example-desktop = nixpkgs.lib.nixosSystem {
-           system = "x86_64-linux";
-           modules = [
-               ./configuration.nix
-           ];
-           [...]
-       };
-   }
-   ```
-
-   After change:
-
-   ```nix
-   let clan = clan-core.lib.buildClan {
-       # this needs to point at the repository root
-       directory = self;
-       specialArgs = {};
-       clanName = "NEEDS_TO_BE_UNIQUE"; # TODO: Changeme
-       machines = {
-           example-desktop = {
-               nixpkgs.hostPlatform = "x86_64-linux";
-               imports = [
-                   ./configuration.nix
-               ];
-           };
-       };
-   };
-   in { inherit (clan) nixosConfigurations clanInternals; }
-   ```
-
-   - `nixosConfigurations`: Defines NixOS configurations, using Clan Core’s `buildClan` function to manage the machines.
-   - Inside `machines`, a new machine configuration is defined (in this case, `example-desktop`).
-   - Inside `example-desktop` which is the target machine hostname, `nixpkgs.hostPlatform` specifies the host platform as `x86_64-linux`.
-   - `clanInternals`: Is required to enable evaluation of the secret generation/upload script on every architecture
-   - `clanName`: Is required and needs to be globally unique, as else we have a cLAN name clash
-
-4. **Rebuild and Switch**: Rebuild your NixOS configuration using the updated flake:
-
-   ```shellSession
-   $ sudo nixos-rebuild switch --flake .
-   ```
-
-   - This command rebuilds and switches to the new configuration. Make sure to include the `--flake .` argument to use the current directory as the flake source.
-
-5. **Test Configuration**: Before rebooting, verify that your new configuration builds without errors or warnings.
-
-6. **Reboot**: If everything is fine, you can reboot your system to apply the changes:
-
-   ```shellSession
-   $ sudo reboot
-   ```
-
-7. **Verify**: After the reboot, confirm that your system is running with the new configuration, and all services and applications are functioning as expected.
-
-By following these steps, you've successfully migrated your NixOS Flake configuration to include the `clan-core` input and adapted the `outputs` section to work with Clan Core's new machine provisioning method.
-
-## What's next
-
-After creating your flake, you can check out how to add [new machines](./machines.md)

docs/admins/zerotier.md

@@ -1,69 +0,0 @@
-# ZeroTier Configuration with NixOS in Clan
-
-This guide provides detailed instructions for configuring
-[ZeroTier VPN](https://zerotier.com) within Clan. Follow the
-outlined steps to set up a machine as a VPN controller (`<CONTROLLER>`) and to
-include a new machine into the VPN.
-
-## 1. Setting Up the VPN Controller
-
-The VPN controller is initially essential for providing configuration to new
-peers. Post the address allocation, the controller's continuous operation is not
-crucial.
-
-### Instructions:
-
-1. **Designate a Machine**: Label a machine as the VPN controller in the clan,
-   referred to as `<CONTROLLER>` henceforth in this guide.
-2. **Add Configuration**: Input the below configuration to the NixOS
-   configuration of the controller machine:
-   ```nix
-   clan.networking.zerotier.controller = {
-     enable = true;
-     public = true;
-   };
-   ```
-3. **Update the Controller Machine**: Execute the following:
-   ```console
-   $ clan machines update <CONTROLLER>
-   ```
-   Your machine is now operational as the VPN controller.
-
-## 2. Integrating a New Machine to the VPN
-
-To introduce a new machine to the VPN, adhere to the following steps:
-
-### Instructions:
-
-1. **Update Configuration**: On the new machine, incorporate the below to its
-   configuration, substituting `<CONTROLLER>` with the controller machine name:
-   ```nix
-   { config, ... }: {
-     clan.networking.zerotier.networkId = builtins.readFile (config.clanCore.clanDir + "/machines/<CONTROLLER>/facts/zerotier-network-id");
-   }
-   ```
-2. **Update the New Machine**: Execute:
-   ```console
-   $ clan machines update <NEW_MACHINE>
-   ```
-   Replace `<NEW_MACHINE>` with the designated new machine name.
-3. **Retrieve the ZeroTier ID**: On the `new_machine`, execute:
-   ```console
-   $ sudo zerotier-cli info
-   ```
-   Example Output: `200 info d2c71971db 1.12.1 OFFLINE`, where `d2c71971db` is
-   the ZeroTier ID.
-4. **Authorize the New Machine on Controller**: On the controller machine,
-   execute:
-   ```console
-   $ sudo zerotier-members allow <ID>
-   ```
-   Substitute `<ID>` with the ZeroTier ID obtained previously.
-5. **Verify Connection**: On the `new_machine`, re-execute:
-   ```console
-   $ sudo zerotier-cli info
-   ```
-   The status should now be "ONLINE" e.g., `200 info 47303517ef 1.12.1 ONLINE`.
-
-Congratulations! The new machine is now part of the VPN, and the ZeroTier
-configuration on NixOS within the Clan project is complete.

docs/contributing/api-guidelines.md

@@ -1,138 +0,0 @@
-# API Guidelines
-
-This issue serves to collect our common understanding how to design our API so that it is extensible and usable and understandable.
-
-## Resource oriented
-
-A resource-oriented API is generally modeled as a resource hierarchy, where each node is either a simple resource or a collection resource. For convenience, they are often called a resource and a collection, respectively.
-
-Examples of Resource Nouns:
-
-`machine`
-`user`
-`flake`
-
-Often resources have sub-resources. Even if it is not foreseen, it is recommended to use plural (trailing `s`) on resources to allow them to be collections of sub-resources.
-
-e.g,
-
-`users`
-->
-`users/*/profile`
-
-## Verbs
-
-Verbs should not be part of the URL
-
-Bad:
-`/api/create-products`
-
-Good:
-`/api/products`
-
-Only resources are part of the URL, verbs are described via the HTTP Method.
-
-Exception:
-
-If a different HTTP Method must be used for technical reasons it is okay to terminate the path with a (short) verb / action.
-
-Okay ish:
-`/api/products/create`
-
-## Usually the following HTTP Methods exist to interact with a resource
-
-- POST (create an order for a resource)
-- GET (retrieve the information)
-- PUT (update and replace information)
-- PATCH (update and modify information) **(Not used yet)**
-- DELETE (delete the item)
-
-## Every resource should be CRUD compatible
-
-All API resources MUST be designed in a way that allows the typical CRUD operations.
-
-Where crud stands for:
-
-C - Create
-R - Read
-U - Update
-D - Delete
-
-Resources should implement at least a "Read" operation.
-
-## Body
-
-Use JSON as an exchange format.
-
-All responses MUST be JSON parseable.
-
-Bad:
-`bare string`
-
-Better:
-`"quoted string"`
-
-Best: (Enveloped see next section)
-`{ name: "quoted string"}`
-
-Errors should have a consistent JSON format, such that it is clear in which field to look at for displaying error messages.
-
-## Envelop all Data collections
-
-Response data should be wrapped into an JSON Object `{}`
-Lists `[]` should also contain Objects `{}`.
-This allows everything, to be extensible, without breaking backwards compatibility. (Adding fields is trivial, since the schema doesn't change)
-
-Example:
-
-```
-{
-   "users": [{
-     first_name: "John",
-     last_name: "Doe",
-     …
-   }, {
-     first_name: "Jane",
-     last_name: "Doe",
-     …
-   }
-  ....
-  ],
-   "skip": 0,
-   "limit": 20,
-  ....
-}
-```
-
-Bad Example of a breaking change:
-`GET /api/flakes`
-`old`
-
-```
-[
-  "dream2nix"
-  "disko"
-]
-```
-
-`new`
-
-```
-[
- {
-    name: "dream2nix",
-    url: "github/...."
- },
- {
-    name: "disko",
-    url: "github/...."
- }
-]
-```
-
-Those kind of breaking changes can be avoided by using an object from the beginning.
-Even if the object only contains one key, it is extensible, without breaking.
-
-## More will follow.
-
-...maybe

docs/contributing/testing.md

@@ -1,111 +0,0 @@
-# Property vs Contract based testing
-
-In this section, we'll explore the importance of testing the backend of your FastAPI application, specifically focusing on the advantages of using contract-based testing with property-based testing frameworks.
-
-## Why Use Property-Based Testing?
-
-Property-based testing is a powerful approach to test your APIs, offering several key benefits:
-
-### 1. Scope
-
-Instead of having to write numerous test cases for various input arguments, property-based testing enables you to test a range of arguments for each parameter using a single test. This approach significantly enhances the robustness of your test suite while reducing redundancy in your testing code. In short, your test code becomes cleaner, more DRY (Don't Repeat Yourself), and more efficient. It also becomes more effective as you can easily test numerous edge cases.
-
-### 2. Reproducibility
-
-Property-based testing tools retain test cases and their results, allowing you to reproduce and replay tests in case of failure. This feature is invaluable for debugging and ensuring the stability of your application over time.
-
-## Frameworks for Property-Based Testing
-
-To implement property-based testing in FastAPI, you can use the following framework:
-
-- [Hypothesis: Property-Based Testing](https://hypothesis.readthedocs.io/en/latest/quickstart.html)
-- [Schemathesis](https://schemathesis.readthedocs.io/en/stable/#id2)
-
-## Example
-
-Running schemathesis fuzzer on GET requests
-
-```bash
-nix run .#runSchemaTests
-```
-
-If you want to test more request types edit the file [flake-module.nix](../checks/impure/flake-module.nix)
-
-After a run it will upload the results to `schemathesis.io` and give you a link to the report.
-The credentials to the account are `Username: schemathesis@qube.email` and `Password:6tv4eP96WXsarF`
-
-## Why Schemas Are Not Contracts
-
-A schema is a description of the data structure of your API, whereas a contract defines not only the structure but also the expected behavior and constraints. The following resource explains why schemas are not contracts in more detail:
-
-- [Why Schemas Are Not Contracts](https://pactflow.io/blog/schemas-are-not-contracts/)
-
-In a nutshell, schemas may define the data structure but often fail to capture complex constraints and the expected interactions between different API endpoints. Contracts fill this gap by specifying both the structure and behavior of your API.
-
-## Why Use Contract-Driven Testing?
-
-Contract-driven testing combines the benefits of type annotations and property-based testing, providing a robust approach to ensuring the correctness of your APIs.
-
-- Contracts become an integral part of the function signature and can be checked statically, ensuring that the API adheres to the defined contract.
-- Contracts, like property-based tests, allow you to specify conditions and constraints, with the testing framework automatically generating test cases and verifying call results.
-
-### Frameworks for Contract-Driven Testing
-
-To implement contract-driven testing in FastAPI, consider the following framework and extension:
-
-- [Deal: Contract Driven Development](https://deal.readthedocs.io/)
-  By adopting contract-driven testing, you can ensure that your FastAPI application not only has a well-defined structure but also behaves correctly, making it more robust and reliable.
-- [Whitepaper: Python by contract](https://users.ece.utexas.edu/~gligoric/papers/ZhangETAL22PythonByContractDataset.pdf) This paper goes more into detail how it works
-
-## Examples
-
-You can annotate functions with `@deal.raises(ClanError)` to say that they can _only_ raise a ClanError Exception.
-
-```python
-import deal
-
-@deal.raises(ClanError)
-def get_task(uuid: UUID) -> BaseTask:
-    global POOL
-    return POOL[uuid]
-```
-
-To say that it can raise multiple exceptions just add after one another separated with a `,`
-
-```python
-import deal
-
-@deal.raises(ClanError, IndexError, ZeroDivisionError)
-def get_task(uuid: UUID) -> BaseTask:
-    global POOL
-    return POOL[uuid]
-```
-
-### Adding deal annotated functions to pytest
-
-```python
-from clan_cli.task_manager import get_task
-import deal
-
-@deal.cases(get_task) # <--- Add function get_task to testing corpus
-def test_get_task(case: deal.TestCase) -> None:
-    case() # <--- Call testing framework with function
-```
-
-### Adding example input for deeper testing
-
-You can combine hypothesis annotations with deal annotations to add example inputs to the function so that the verifier can reach deeper parts of the function.
-
-```python
-import deal
-
-@deal.example(lambda: get_task(UUID("5c2061e0-4512-4b30-aa8e-7be4a75b8b45"))) # type: ignore
-@deal.example(lambda: get_task(UUID("7c2061e6-4512-4b30-aa8e-7be4a75b8b45"))) # type: ignore
-@deal.raises(ClanError)
-def get_task(uuid: UUID) -> BaseTask:
-    global POOL
-    return POOL[uuid]
-```
-
-You can also add `pre` and `post` conditions. A `pre` condition must be true before the function is executed. A `post` condition must be true after the function was executed. For more information read the [Writing Contracts Section](https://deal.readthedocs.io/basic/values.html).
-Or read the [API doc of Deal](https://deal.readthedocs.io/details/api.html)

docs/mkdocs.yml

@@ -0,0 +1,112 @@
+site_name: cLAN documentation
+site_url: https://docs.clan.lol
+repo_url: https://git.clan.lol/clan/clan-core/
+repo_name: clan-core
+edit_uri: _edit/main/docs/docs/
+
+validation:
+  omitted_files: warn
+  absolute_links: warn
+  unrecognized_links: warn
+
+markdown_extensions:
+  - attr_list
+  - pymdownx.emoji:
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
+
+  - pymdownx.tasklist:
+      custom_checkbox: true
+  - pymdownx.superfences
+  - pymdownx.tabbed:
+      alternate_style: true
+  - footnotes
+  - meta
+  - admonition
+  - pymdownx.details
+  - pymdownx.highlight:
+      use_pygments: true
+  - toc:
+      title: On this page
+
+exclude_docs: |
+  .*
+  !templates/
+  /drafts/
+
+nav:
+  - Getting started:
+      - index.md
+      - Configure: getting-started/configure.md
+      - Deploy Machine: getting-started/machines.md
+      - Installer: getting-started/installer.md
+      - Setup Networking: getting-started/networking.md
+      - Provision Secrets & Passwords: getting-started/secrets.md
+      - Backup & Restore: getting-started/backups.md
+      - Flake-parts: getting-started/flake-parts.md
+  - Templates: templates/index.md
+  - Reference:
+      - clan-core:
+          - reference/clan-core/index.md
+          - reference/clan-core/backups.md
+          - reference/clan-core/facts.md
+          - reference/clan-core/sops.md
+          - reference/clan-core/state.md
+      - clanModules:
+          - reference/clanModules/borgbackup.md
+          - reference/clanModules/deltachat.md
+          - reference/clanModules/diskLayouts.md
+          - reference/clanModules/ergochat.md
+          - reference/clanModules/graphical.md
+          - reference/clanModules/localbackup.md
+          - reference/clanModules/localsend.md
+          - reference/clanModules/matrix-synapse.md
+          - reference/clanModules/moonlight.md
+          - reference/clanModules/root-password.md
+          - reference/clanModules/sshd.md
+          - reference/clanModules/sunshine.md
+          - reference/clanModules/syncthing.md
+          - reference/clanModules/thelounge.md
+          - reference/clanModules/vm-user.md
+          - reference/clanModules/waypipe.md
+          - reference/clanModules/xfce-vm.md
+          - reference/clanModules/xfce.md
+          - reference/clanModules/zt-tcp-relay.md
+  - Contributing: contributing/contributing.md
+
+docs_dir: site
+site_dir: out
+
+theme:
+  logo: static/logo.png
+  name: material
+  features:
+    - navigation.instant
+    - navigation.tabs
+    - content.code.annotate
+    - content.code.copy
+    - content.tabs.link
+  icon:
+    repo: fontawesome/brands/git
+
+  palette:
+    # Palette toggle for light mode
+    - media: "(prefers-color-scheme: light)"
+      scheme: default
+      primary: teal
+      accent: deep purple
+      toggle:
+        icon: material/weather-night
+        name: Switch to dark mode
+
+    # Palette toggle for dark mode
+    - media: "(prefers-color-scheme: dark)"
+      primary: teal
+      accent: deep purple
+      scheme: slate
+      toggle:
+        icon: material/weather-sunny
+        name: Switch to light mode
+
+plugins:
+  - search

docs/nix/default.nix

@@ -0,0 +1,27 @@
+{ pkgs, module-docs, ... }:
+pkgs.stdenv.mkDerivation {
+  name = "clan-documentation";
+
+  src = ../.;
+
+  nativeBuildInputs =
+    [ pkgs.python3 ]
+    ++ (with pkgs.python3Packages; [
+      mkdocs
+      mkdocs-material
+    ]);
+  configurePhase = ''
+    mkdir -p ./site/reference
+    cp -af ${module-docs}/* ./site/reference/
+
+  '';
+
+  buildPhase = ''
+    mkdocs build --strict
+    ls -la .
+  '';
+
+  installPhase = ''
+    cp -a out/ $out/
+  '';
+}

docs/nix/deploy-docs.nix

@@ -0,0 +1,36 @@
+{
+  writeShellScriptBin,
+  coreutils,
+  openssh,
+  rsync,
+  lib,
+  docs,
+}:
+
+writeShellScriptBin "deploy-docs" ''
+  set -eux -o pipefail
+  export PATH="${
+    lib.makeBinPath [
+      coreutils
+      openssh
+      rsync
+    ]
+  }"
+
+  if [ -n "''${SSH_HOMEPAGE_KEY:-}" ]; then
+    echo "$SSH_HOMEPAGE_KEY" > ./ssh_key
+    chmod 600 ./ssh_key
+    sshExtraArgs="-i ./ssh_key"
+  else
+    sshExtraArgs=
+  fi
+
+  rsync \
+    -e "ssh -o StrictHostKeyChecking=no $sshExtraArgs" \
+    -a ${docs}/ \
+    www@clan.lol:/var/www/docs.clan.lol
+
+  if [ -e ./ssh_key ]; then
+    rm ./ssh_key
+  fi
+''

docs/nix/flake-module.nix

@@ -0,0 +1,72 @@
+{ inputs, self, ... }:
+{
+  perSystem =
+    {
+      config,
+      self',
+      pkgs,
+      ...
+    }:
+    let
+      # Simply evaluated options (JSON)
+      # { clanCore = «derivation JSON»; clanModules = { ${name} = «derivation JSON» }; }
+      jsonDocs = import ./get-module-docs.nix {
+        inherit (inputs) nixpkgs;
+        inherit pkgs self;
+        inherit (self.nixosModules) clanCore;
+        inherit (self) clanModules;
+      };
+
+      clanModulesFileInfo = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModules);
+      clanModulesReadmes = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesReadmes);
+
+      # Simply evaluated options (JSON)
+      renderOptions =
+        pkgs.runCommand "renderOptions.py"
+          {
+            # TODO: ruff does not splice properly in nativeBuildInputs
+            depsBuildBuild = [ pkgs.ruff ];
+            nativeBuildInputs = [
+              pkgs.python3
+              pkgs.mypy
+            ];
+          }
+          ''
+            install ${./scripts/renderOptions.py} $out
+            patchShebangs --build $out
+
+            ruff format --check --diff $out
+            ruff --line-length 88 $out
+            mypy --strict $out
+          '';
+
+      module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
+        export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json 
+        # A file that contains the links to all clanModule docs
+        export CLAN_MODULES=${clanModulesFileInfo}
+        export CLAN_MODULES_READMES=${clanModulesReadmes}
+
+        mkdir $out
+
+        # The python script will place mkDocs files in the output directory
+        python3 ${renderOptions}
+      '';
+    in
+    {
+      devShells.docs = pkgs.callPackage ./shell.nix {
+        inherit (self'.packages) docs;
+        inherit module-docs;
+      };
+      packages = {
+        docs = pkgs.python3.pkgs.callPackage ./default.nix {
+          inherit (inputs) nixpkgs;
+          inherit module-docs;
+        };
+        deploy-docs = pkgs.callPackage ./deploy-docs.nix { inherit (config.packages) docs; };
+        inherit module-docs;
+      };
+      legacyPackages = {
+        foo = jsonDocs;
+      };
+    };
+}

docs/nix/get-module-docs.nix

@@ -0,0 +1,64 @@
+{
+  nixpkgs,
+  pkgs,
+  clanCore,
+  clanModules,
+  self,
+}:
+let
+  allNixosModules = (import "${nixpkgs}/nixos/modules/module-list.nix") ++ [
+    "${nixpkgs}/nixos/modules/misc/assertions.nix"
+    { nixpkgs.hostPlatform = "x86_64-linux"; }
+  ];
+
+  clanCoreNixosModules = [
+    clanCore
+    { clanCore.clanDir = ./.; }
+  ] ++ allNixosModules;
+
+  # TODO: optimally we would not have to evaluate all nixos modules for every page
+  #   but some of our module options secretly depend on nixos modules.
+  #   We would have to get rid of these implicit dependencies and make them explicit
+  clanCoreNixos = pkgs.nixos { imports = clanCoreNixosModules; };
+
+  # using extendModules here instead of re-evaluating nixos every time
+  #   improves eval performance slightly (10%)
+  getOptions = modules: (clanCoreNixos.extendModules { inherit modules; }).options;
+
+  evalDocs =
+    options:
+    pkgs.nixosOptionsDoc {
+      options = options;
+      warningsAreErrors = false;
+    };
+
+  # clanModules docs
+  clanModulesDocs = builtins.mapAttrs (
+    name: module: (evalDocs ((getOptions [ module ]).clan.${name} or { })).optionsJSON
+  ) clanModules;
+
+  clanModulesReadmes = builtins.mapAttrs (
+    module_name: _module:
+    let
+      readme = "${self}/clanModules/${module_name}/README.md";
+      readmeContents =
+        if
+          builtins.trace "Trying to get Module README.md for ${module_name} from ${readme}"
+            # TODO: Edge cases
+            (builtins.pathExists readme)
+        then
+          (builtins.readFile readme)
+        else
+          null;
+    in
+    readmeContents
+  ) clanModules;
+
+  # clanCore docs
+  clanCoreDocs = (evalDocs (getOptions [ ]).clanCore).optionsJSON;
+in
+{
+  inherit clanModulesReadmes;
+  clanCore = clanCoreDocs;
+  clanModules = clanModulesDocs;
+}

docs/nix/scripts/renderOptions.py

@@ -0,0 +1,235 @@
+# Options are available in the following format:
+# https://github.com/nixos/nixpkgs/blob/master/nixos/lib/make-options-doc/default.nix
+#
+# ```json
+# {
+# ...
+# "fileSystems.<name>.options": {
+#     "declarations": ["nixos/modules/tasks/filesystems.nix"],
+#     "default": {
+#       "_type": "literalExpression",
+#       "text": "[\n  \"defaults\"\n]"
+#     },
+#     "description": "Options used to mount the file system.",
+#     "example": {
+#       "_type": "literalExpression",
+#       "text": "[\n  \"data=journal\"\n]"
+#     },
+#     "loc": ["fileSystems", "<name>", "options"],
+#     "readOnly": false,
+#     "type": "non-empty (list of string (with check: non-empty))"
+#     "relatedPackages": "- [`pkgs.tmux`](\n    https://search.nixos.org/packages?show=tmux&sort=relevance&query=tmux\n  )\n",
+# }
+# }
+# ```
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+# Get environment variables
+CLAN_CORE = os.getenv("CLAN_CORE")
+CLAN_MODULES = os.environ.get("CLAN_MODULES")
+CLAN_MODULES_READMES = os.environ.get("CLAN_MODULES_READMES")
+
+OUT = os.environ.get("out")
+
+
+def sanitize(text: str) -> str:
+    return text.replace(">", "\\>")
+
+
+def replace_store_path(text: str) -> Path:
+    res = text
+    if text.startswith("/nix/store/"):
+        res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str(
+            Path(*Path(text).parts[4:])
+        )
+    return Path(res)
+
+
+def render_option_header(name: str) -> str:
+    return f"# {name}\n"
+
+
+def join_lines_with_indentation(lines: list[str], indent: int = 4) -> str:
+    """
+    Joins multiple lines with a specified number of whitespace characters as indentation.
+
+    Args:
+    lines (list of str): The lines of text to join.
+    indent (int): The number of whitespace characters to use as indentation for each line.
+
+    Returns:
+    str: The indented and concatenated string.
+    """
+    # Create the indentation string (e.g., four spaces)
+    indent_str = " " * indent
+    # Join each line with the indentation added at the beginning
+    return "\n".join(indent_str + line for line in lines)
+
+
+def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
+    read_only = option.get("readOnly")
+
+    res = f"""
+{"#" * level} {sanitize(name)} {{#{sanitize(name)}}}
+{"Readonly" if read_only else ""}
+{option.get("description", "No description available.")}
+
+**Type**: `{option["type"]}`
+
+"""
+    if option.get("default"):
+        res += f"""
+**Default**:
+
+```nix
+{option["default"]["text"] if option.get("default") else "No default set."}
+```
+        """
+    example = option.get("example", {}).get("text")
+    if example:
+        example_indented = join_lines_with_indentation(example.split("\n"))
+        res += f"""
+
+???+ example
+
+    ```nix
+{example_indented}
+    ```
+"""
+    if option.get("relatedPackages"):
+        res += f"""
+### Related Packages
+
+{option["relatedPackages"]}
+"""
+
+    decls = option.get("declarations", [])
+    source_path = replace_store_path(decls[0])
+    res += f"""
+:simple-git: [{source_path.name}]({source_path})
+"""
+    res += "\n"
+
+    return res
+
+
+def module_header(module_name: str) -> str:
+    return f"# {module_name}\n"
+
+
+def module_usage(module_name: str) -> str:
+    return f"""## Usage
+
+To use this module, import it like this:
+
+```nix
+{{config, lib, inputs, ...}}: {{
+    imports = [ inputs.clan-core.clanModules.{module_name} ];
+    # ...
+}}
+```
+"""
+
+
+clan_core_descr = """ClanCore delivers all the essential features for every clan. 
+It's always included in your setup, and you can customize your clan's behavior with the configuration [options](#module-options) provided below.
+
+"""
+
+options_head = "\n## Module Options\n"
+
+
+def produce_clan_core_docs() -> None:
+    if not CLAN_CORE:
+        raise ValueError(
+            f"Environment variables are not set correctly: $CLAN_CORE={CLAN_CORE}"
+        )
+
+    if not OUT:
+        raise ValueError(f"Environment variables are not set correctly: $out={OUT}")
+
+    # A mapping of output file to content
+    core_outputs: dict[str, str] = {}
+    with open(CLAN_CORE) as f:
+        options: dict[str, dict[str, Any]] = json.load(f)
+        module_name = "clan-core"
+        for option_name, info in options.items():
+            outfile = f"{module_name}/index.md"
+
+            # Create seperate files for nested options
+            if len(option_name.split(".")) <= 2:
+                # i.e. clan-core.clanDir
+                output = core_outputs.get(
+                    outfile,
+                    module_header(module_name) + clan_core_descr + options_head,
+                )
+                output += render_option(option_name, info)
+                # Update the content
+                core_outputs[outfile] = output
+            else:
+                # Clan sub-options
+                [_, sub] = option_name.split(".")[0:2]
+                outfile = f"{module_name}/{sub}.md"
+                # Get the content or write the header
+                output = core_outputs.get(outfile, render_option_header(sub))
+                output += render_option(option_name, info)
+                # Update the content
+                core_outputs[outfile] = output
+
+        for outfile, output in core_outputs.items():
+            (Path(OUT) / outfile).parent.mkdir(parents=True, exist_ok=True)
+            with open(Path(OUT) / outfile, "w") as of:
+                of.write(output)
+
+
+def produce_clan_modules_docs() -> None:
+    if not CLAN_MODULES:
+        raise ValueError(
+            f"Environment variables are not set correctly: $CLAN_MODULES={CLAN_MODULES}"
+        )
+    if not CLAN_MODULES_READMES:
+        raise ValueError(
+            f"Environment variables are not set correctly: $CLAN_MODULES_READMES={CLAN_MODULES_READMES}"
+        )
+
+    if not OUT:
+        raise ValueError(f"Environment variables are not set correctly: $out={OUT}")
+
+    with open(CLAN_MODULES) as f:
+        links: dict[str, str] = json.load(f)
+
+    with open(CLAN_MODULES_READMES) as readme:
+        readme_map: dict[str, str] = json.load(readme)
+
+    # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... }
+    for module_name, options_file in links.items():
+        with open(Path(options_file) / "share/doc/nixos/options.json") as f:
+            options: dict[str, dict[str, Any]] = json.load(f)
+            print(f"Rendering options for {module_name}...")
+            output = module_header(module_name)
+
+            if readme_map.get(module_name, None):
+                output += f"{readme_map[module_name]}\n"
+
+            output += module_usage(module_name)
+
+            output += options_head if len(options.items()) else ""
+            for option_name, info in options.items():
+                output += render_option(option_name, info)
+
+            outfile = Path(OUT) / f"clanModules/{module_name}.md"
+            outfile.parent.mkdir(
+                parents=True,
+                exist_ok=True,
+            )
+            with open(outfile, "w") as of:
+                of.write(output)
+
+
+if __name__ == "__main__":
+    produce_clan_core_docs()
+    produce_clan_modules_docs()

docs/nix/shell.nix

@@ -0,0 +1,16 @@
+{
+  docs,
+  pkgs,
+  module-docs,
+  ...
+}:
+pkgs.mkShell {
+  inputsFrom = [ docs ];
+  shellHook = ''
+    mkdir -p ./site/reference
+    cp -af ${module-docs}/* ./site/reference/
+    chmod +w ./site/reference/*
+
+    echo "Generated API documentation in './site/reference/' "
+  '';
+}

docs/site/contributing/contributing.md

@@ -0,0 +1 @@
+../../CONTRIBUTING.md

docs/site/drafts/install-iso.md

@@ -0,0 +1,146 @@
+# Hardware Installation
+
+For installations on physical hardware, create a NixOS installer image and transfer it to a bootable USB drive as described below.
+
+## Creating a Bootable USB Drive on Linux
+
+To create a bootable USB flash drive with the NixOS installer:
+
+### Download the install iso
+
+Either with wget:
+
+```shellSession
+wget https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-installer-x86_64-linux.iso
+```
+
+or with curl:
+
+```shellSession
+curl -L https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-installer-x86_64-linux.iso -o nixos-installer-x86_64-linux.iso
+```
+
+### Prepare the USB Flash Drive
+
+1. Insert your USB flash drive into your computer.
+
+2. Identify your flash drive with `lsblk`.
+
+    ```shellSession
+    lsblk
+    NAME                                          MAJ:MIN RM   SIZE RO TYPE  MOUNTPOINTS
+    sdb                                             8:0    1 117,2G  0 disk
+    └─sdb1                                          8:1    1 117,2G  0 part  /run/media/qubasa/INTENSO
+    nvme0n1                                       259:0    0   1,8T  0 disk
+    ├─nvme0n1p1                                   259:1    0   512M  0 part  /boot
+    └─nvme0n1p2                                   259:2    0   1,8T  0 part
+      └─luks-f7600028-9d83-4967-84bc-dd2f498bc486 254:0    0   1,8T  0 crypt /nix/store
+    ```
+
+    In this case it's `sdb`
+
+3. Ensure all partitions on the drive are unmounted. Replace `sdX` in the command below with your device identifier (like `sdb`, etc.):
+
+    ```shellSession
+    sudo umount /dev/sdb1
+    ```
+
+### Write the Image to the USB Drive
+
+Use the `dd` utility to write the NixOS installer image to your USB drive:
+
+  ```shellSession
+  sudo dd bs=4M conv=fsync oflag=direct status=progress if=./nixos-installer-x86_64-linux.iso of=/dev/sd<X>
+  ```
+
+  In this case, the USB device is `sdb` use `of=/dev/sdb`
+
+### Boot and Connect
+
+After writing the installer to the USB drive, use it to boot the target machine.
+
+1. For this secure boot needs to be disabled. Go into your UEFI / Bios settings by pressing one of the keys outlined below while booting:
+
+   - **Dell**: F2/Del (BIOS Setup)
+   - **HP**: Esc (Startup Menu)
+   - **Lenovo**: F2/Fn+F2/Novo Button (IdeaPad Boot Menu/BIOS Setup)
+   - **Acer**: F2/Del (BIOS Setup)
+   - **Asus**: F2/Del (BIOS Setup)
+   - **Toshiba**: Esc then F12 (Alternate Method)
+   - **Sony**: F11
+   - **Samsung**: F2 (BIOS Setup)
+   - **MSI**: Del (BIOS Setup)
+   - **Apple**: Option (Alt) Key (Boot Menu for Mac)
+   - If your hardware was not listed read the manufacturers instructions how to enter the boot Menu/BIOS Setup.
+
+2. Inside the UEFI/Bios Menu go to `Security->Secure Boot` and disable secure boot
+
+3. Save your settings. Put in the USB stick and reboot.
+
+4. Press one of keys outlined below to go into the Boot Menu
+
+       - **Dell**: F12 (Boot Menu)
+       - **HP**: F9 (Boot Menu)
+       - **Lenovo**: F12 (ThinkPad Boot Menu)
+       - **Acer**: F12 (Boot Menu)
+       - **Asus**: F8/Esc (Boot Menu)
+       - **Toshiba**: F12/F2 (Boot Menu)
+       - **Sony**: F11
+       - **Samsung**: F2/F12/Esc (Boot Menu)
+       - **MSI**: F11
+       - **Apple**: Option (Alt) Key (Boot Menu for Mac)
+       - If your hardware was not listed read the manufacturers instructions how to enter the boot Menu/BIOS Setup.
+
+
+5. Select `NixOS` to boot into the clan installer
+
+6. The installer will display an IP address and a root password, which you can use to connect via SSH.  
+    Alternatively you can also use the displayed QR code.
+
+7. Set your keyboard language (i.e. `de` for German keyboards, default is English). Important for writing passwords correctly.
+
+    ```shellSession
+    loadkeys de
+    ```
+
+8. If you only have Wifi available, execute:
+
+    1. Bring up the `iwd` shell
+
+        ```shellSession
+        iwctl
+        ```
+
+    2. List available networks. Double press tab after station for autocompleting your wlan device. In this case `wlan0`
+
+        ```shellSession
+        [iwd] station wlan0 get-networks
+        ```
+
+    3. Connect to a Wifi network. Replace `SSID` with the wlan network name.
+
+        ```shellSession
+        [iwd] station wlan0 connect SSID
+        ```
+
+9. Now that you have internet re-execute the init script by pressing `Ctrl+D` or by executing:
+
+    ```shellSession
+    bash
+    ```
+
+10. Connect to the machine over ssh
+
+    ```shellSession
+    ssh-copy-id -o PreferredAuthentications=password root@<ip>
+    ```
+
+    Use the root password displayed on your screen as login.
+
+---
+
+# Whats next?
+
+- Deploy a clan machine-configuration on your prepared machine
+
+---

docs/site/getting-started/backups.md

@@ -0,0 +1,149 @@
+# Backups
+
+## Introduction to Backups
+
+When you're managing your own services, creating regular backups is crucial to ensure your data's safety.
+This guide introduces you to Clan's built-in backup functionalities.
+Clan supports backing up your data to both local storage devices (like USB drives) and remote servers, using well-known tools like borgbackup and rsnapshot.
+We might add more options in the future, but for now, let's dive into how you can secure your data.
+
+## Backing Up Locally with Localbackup
+
+### What is Localbackup?
+
+Localbackup lets you backup your data onto physical storage devices connected to your computer,
+such as USB hard drives or network-attached storage. It uses a tool called rsnapshot for this purpose.
+
+### Setting Up Localbackup
+
+1. **Identify Your Backup Device:**
+
+First, figure out which device you'll use for backups. You can see all connected devices by running this command in your terminal:
+
+```bash
+lsblk --output NAME,PTUUID,FSTYPE,SIZE,MOUNTPOINT
+```
+
+Look for the device you intend to use for backups and note its details.
+
+2. **Configure Your Backup Device:**
+
+Once you've identified your device, you'll need to add it to your configuration.
+Here's an example NixOS configuration for a device located at `/dev/sda2` with an `ext4` filesystem:
+
+```nix
+{
+  fileSystems."/mnt/hdd" = {
+    device = "/dev/sda2";
+    fsType = "ext4";
+    options = [ "defaults" "noauto" ];
+  };
+}
+```
+
+Replace `/dev/sda2` with your device and `/mnt/hdd` with your preferred mount point.
+
+3. **Set Backup Targets:** Next, define where on your device you'd like the backups to be stored:
+
+   ```nix
+   {
+     clan.localbackup.targets.hdd = {
+       directory = "/mnt/hdd/backup";
+       mountpoint = "/mnt/hdd";
+     };
+   }
+   ```
+
+   Change `/mnt/hdd` to the actual mount point you're using.
+
+4. **Create Backups:** To create a backup, run:
+
+   ```bash
+   clan backups create mymachine
+   ```
+
+   This command saves snapshots of your data onto the backup device.
+
+5. **Listing Backups:** To see available backups, run:
+
+  ```bash
+  clan backups list mymachine
+  ```
+
+## Remote Backups with Borgbackup
+
+### Overview of Borgbackup
+
+Borgbackup splits the backup process into two parts: a backup client that sends data to a backup server.
+The server stores the backups.
+
+### Setting Up the Borgbackup Client
+
+1. **Specify Backup Server:**
+
+Start by indicating where your backup data should be sent. Replace `hostname` with your server's address:
+
+```nix
+{
+  clan.borgbackup.destinations = {
+    myhostname = {
+      repo = "borg@backuphost:/var/lib/borgbackup/myhostname";
+    };
+  };
+}
+```
+
+2. **Select Folders to Backup:**
+
+Decide which folders you want to back up. For example, to backup your home and root directories:
+
+```nix
+{ clanCore.state.userdata.folders = [ "/home" "/root" ]; }
+```
+
+3. **Generate Backup Credentials:**
+
+Run `clan facts generate <yourmachine>` to prepare your machine for backup, creating necessary SSH keys and credentials.
+
+### Setting Up the Borgbackup Server
+
+1. **Configure Backup Repository:**
+
+On the server where backups will be stored, enable the SSH daemon and set up a repository for each client:
+
+```nix
+{
+  services.borgbackup.repos.myhostname = {
+    path = "/var/lib/borgbackup/myhostname";
+    authorizedKeys = [
+      (builtins.readFile ./machines/myhostname/facts/borgbackup.ssh.pub)
+    ];
+  };
+}
+```
+
+Ensure the path to the public key is correct.
+
+2. **Update Your Systems:** Apply your changes by running `clan machines update` to both the server and your client
+
+### Managing Backups
+
+- **Scheduled Backups:**
+
+  Backups are automatically performed nightly. To check the next scheduled backup, use:
+
+  ```bash
+  systemctl list-timers | grep -E 'NEXT|borg'
+  ```
+
+- **Listing Backups:** To see available backups, run:
+
+  ```bash
+  clan backups list mymachine
+  ```
+
+- **Manual Backups:** You can also initiate a backup manually:
+
+  ```bash
+  clan backups create mymachine
+  ```

docs/site/getting-started/configure.md

@@ -0,0 +1,150 @@
+# Configuration - How to configure clan with your own machines
+
+## Global configuration
+
+In the `flake.nix` file:
+
+- [x] set a unique `clanName`.
+- [ ] set `clanIcon` (optional)
+- [ ] Set `machineIcon` per machine (optional)
+
+These icons will be used by our future GUI.
+
+=== "**buildClan**"
+
+    ```nix title="clan-core.lib.buildClan"
+    buildClan {
+        # Set a unique name 
+        clanName = "Lobsters";
+        # Optional, a path to an image file
+        clanIcon = ./path/to/file; 
+        # Should usually point to the directory of flake.nix
+        directory = ./.;
+
+        machines = {
+            jon = {
+                # ...
+                # Optional, a path to an image file
+                clanCore.machineIcon = ./path/to/file; 
+            };
+            # ...
+        }
+    }
+    ```
+
+=== "**flakeParts**"
+
+    !!! info "See [Clan with flake-parts](./flake-parts.md) for help migrating to flake-parts."
+
+    ```nix title="clan-core.flakeModules.default"
+    clan = {
+        # Set a unique name 
+        clanName = "Lobsters";
+        # Optional, a path to an image file
+        clanIcon = ./path/to/file;
+
+        machines = {
+            jon = {
+                # ...
+                # Optional, a path to an image file
+                clanCore.machineIcon = ./path/to/file; 
+            };
+            # ...
+        }
+    };
+    ```
+
+## Machine configuration
+
+Adding or configuring a new machine requires two simple steps:
+
+### Step 1. Identify Target Disk-ID
+
+1. Find the remote disk id by executing:
+
+    ```bash title="setup computer"
+    ssh root@<target-computer> lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
+    ```
+
+    Which should show something like:
+
+    ```bash
+    NAME        ID-LINK                                         FSTYPE   SIZE MOUNTPOINT
+    sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G 
+    ├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M 
+    ├─sda2      usb-ST_16GB_AA6271026J1000000509-0:0-part2      vfat     100M /boot
+    └─sda3      usb-ST_16GB_AA6271026J1000000509-0:0-part3      ext4     2.9G /
+    nvme0n1     nvme-eui.e8238fa6bf530001001b448b4aec2929              476.9G 
+    ├─nvme0n1p1 nvme-eui.e8238fa6bf530001001b448b4aec2929-part1 vfat     512M 
+    ├─nvme0n1p2 nvme-eui.e8238fa6bf530001001b448b4aec2929-part2 ext4   459.6G 
+    └─nvme0n1p3 nvme-eui.e8238fa6bf530001001b448b4aec2929-part3 swap    16.8G
+    ```
+
+1. Edit the following fields inside the `flake.nix`
+
+    === "**buildClan**"
+
+        ```nix title="clan-core.lib.buildClan"
+        buildClan {
+          # ...
+          machines = {
+            "jon" = {
+              # ...
+
+              # Change this to the correct ip-address or hostname
+              # The hostname is the machine name by default
+              clan.networking.targetHost = pkgs.lib.mkDefault "root@<hostname>"
+              
+              # Change this to the ID-LINK of the desired disk shown by 'lsblk'
+              clan.diskLayouts.singleDiskExt4 = {
+                device = "/dev/disk/by-id/__CHANGE_ME__";
+              }
+
+              # ...
+            };
+          };     
+        }
+        ```
+
+    === "**flakeParts**"
+
+
+
+        ```nix title="clan-core.flakeModules.default"
+        clan = {
+          # ...
+          machines = {
+            "jon" = {
+              # ...
+
+              # Change this to the correct ip-address or hostname
+              # The hostname is the machine name by default
+              clan.networking.targetHost = pkgs.lib.mkDefault "root@<hostname>"
+              
+              # Change this to the ID-LINK of the desired disk shown by 'lsblk'
+              clan.diskLayouts.singleDiskExt4 = {
+                device = "/dev/disk/by-id/__CHANGE_ME__";
+              }
+
+              # ...
+            };
+          };     
+        };
+        ```
+
+### Step 2. Detect hardware specific drivers
+
+1. Generate a `hardware-configuration.nix` for your target computer
+
+    ```bash
+    ssh root@<target-computer> nixos-generate-config --no-filesystems --show-hardware-config > hardware-configuration.nix
+    ```
+
+2. Move the generated file to `machines/jon/hardware-configuration.nix`.
+
+### Initialize the facts
+
+!!! Info
+    **All facts are automatically initialized.**
+
+If you need additional help see our [facts chapter](./secrets.md)

docs/site/getting-started/flake-parts.md

@@ -0,0 +1,102 @@
+# Clan with `flake-parts`
+
+Clan supports integration with [flake.parts](https://flake.parts/) a tool which allows composing nixos modules in a modular way.
+
+Here's how to set up Clan using `nix flakes` and `flake-parts`.
+
+## 1. Update Your Flake Inputs
+
+To begin, you'll need to add `flake-parts` as a new dependency in your flake's inputs. This is alongside the already existing dependencies, such as `clan-core` and `nixpkgs`. Here's how you can update your `flake.nix` file:
+
+```nix
+# flake.nix
+inputs = {
+  nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
+
+  # New flake-parts input
+  flake-parts.url = "github:hercules-ci/flake-parts";
+  flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
+  
+  clan-core = {
+    url = "git+https://git.clan.lol/clan/clan-core";
+    inputs.nixpkgs.follows = "nixpkgs"; # Needed if your configuration uses nixpkgs unstable.
+    # New
+    inputs.flake-parts.follows = "flake-parts";
+  };
+}
+```
+
+## 2. Import Clan-Core Flake Module
+
+After updating your flake inputs, the next step is to import the `clan-core` flake module. This will make the [clan options](https://git.clan.lol/clan/clan-core/src/branch/main/flakeModules/clan.nix) available within `mkFlake`.
+
+```nix
+  outputs =
+    inputs@{ flake-parts, ... }:
+    flake-parts.lib.mkFlake { inherit inputs; } (
+      {
+        # 
+        imports = [
+          inputs.clan-core.flakeModules.default
+        ];
+      }
+    );
+```
+
+### 3. Configure Clan Settings and Define Machines
+
+Configure your clan settings and define machine configurations.
+
+Below is a guide on how to structure this in your flake.nix:
+
+```nix
+  outputs = inputs@{ flake-parts, clan-core, ... }:
+    flake-parts.lib.mkFlake { inherit inputs; } ({self, pkgs, ...}: {
+      # We define our own systems below. you can still use this to add system specific outputs to your flake.
+      # See: https://flake.parts/getting-started
+      systems = [];
+
+      # import clan-core modules
+      imports = [
+        clan-core.flakeModules.default
+      ];
+      # Define your clan
+      clan = {
+        # Clan wide settings. (Required)
+        clanName = ""; # Ensure to choose a unique name.
+
+        machines = {
+          jon = {
+            imports = [
+              ./machines/jon/configuration.nix
+              # ... more modules
+            ];
+            nixpkgs.hostPlatform = "x86_64-linux";
+            clanCore.machineIcon = null; # Optional, a path to an image file
+
+            # Set this for clan commands use ssh i.e. `clan machines update`
+            clan.networking.targetHost = pkgs.lib.mkDefault "root@jon";
+
+            # remote> lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
+            clan.diskLayouts.singleDiskExt4 = {
+              device = "/dev/disk/by-id/nvme-eui.e8238fa6bf530001001b448b4aec2929";
+            };
+
+            # There needs to be exactly one controller per clan
+            clan.networking.zerotier.controller.enable = true;
+            
+          };
+        };
+      };
+    });
+```
+
+For detailed information about configuring `flake-parts` and the available options within Clan,
+refer to the Clan module documentation located [here](https://git.clan.lol/clan/clan-core/src/branch/main/flakeModules/clan.nix).
+
+## Whats next?
+
+- [Configure Machines](configure.md): Customize machine configuration
+- [Deploying](machines.md): Deploying a Machine configuration
+
+---

docs/site/getting-started/installer.md

@@ -0,0 +1,149 @@
+# Installer
+
+We offer a dedicated installer to assist remote installations.
+
+In this tutorial we will guide you through building and flashing it to a bootable USB drive.
+
+## Creating and Using the **Clan Installer**
+
+### Step 0. Prerequisites
+
+- [x] A free USB Drive with at least 1.5GB (All data on it will be lost)
+- [x] Linux/NixOS Machine with Internet
+
+### Step 1. Identify the USB Flash Drive
+
+1. Insert your USB flash drive into your computer.
+
+2. Identify your flash drive with `lsblk`:
+
+    ```shellSession
+    lsblk
+    ```
+
+    ```{.console, .no-copy}
+    NAME                                          MAJ:MIN RM   SIZE RO TYPE  MOUNTPOINTS
+    sdb                                             8:0    1 117,2G  0 disk
+    └─sdb1                                          8:1    1 117,2G  0 part  /run/media/qubasa/INTENSO
+    nvme0n1                                       259:0    0   1,8T  0 disk
+    ├─nvme0n1p1                                   259:1    0   512M  0 part  /boot
+    └─nvme0n1p2                                   259:2    0   1,8T  0 part
+      └─luks-f7600028-9d83-4967-84bc-dd2f498bc486 254:0    0   1,8T  0 crypt /nix/store
+    ```
+
+    !!! Info "In this case the USB device is `sdb`"
+
+3. Ensure all partitions on the drive are unmounted. Replace `sdb1` in the command below with your device identifier (like `sdc1`, etc.):
+
+```shellSession
+sudo umount /dev/sdb1
+```
+
+### Step 2. Download the Installer
+
+```shellSession
+wget https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-installer-x86_64-linux.iso
+```
+
+### Step 3. Flash the Installer to the USB Drive
+
+!!! Danger "Specifying the wrong device can lead to unrecoverable data loss."
+
+    The `dd` utility will erase the disk. Make sure to specify the correct device (`of=...`)
+
+    For example if the USB device is `sdb` use `of=/dev/sdb`.
+
+    
+
+Use the `dd` utility to write the NixOS installer image to your USB drive:
+
+```shellSession
+sudo dd bs=4M conv=fsync oflag=direct status=progress if=./nixos-installer-x86_64-linux.iso of=/dev/sd<X>
+```
+
+### Step 4. Boot and Connect to your network
+
+After writing the installer to the USB drive, use it to boot the target machine.
+
+!!! info 
+    Plug it into the target machine and select the USB drive as a temporary boot device.
+
+??? tip "Here you can find the key combinations for selection used by most vendors."
+    - **Dell**: F12 (Boot Menu), F2/Del (BIOS Setup)
+    - **HP**: F9 (Boot Menu), Esc (Startup Menu)
+    - **Lenovo**: F12 (ThinkPad Boot Menu), F2/Fn+F2/Novo Button (IdeaPad Boot Menu/BIOS Setup)
+    - **Acer**: F12 (Boot Menu), F2/Del (BIOS Setup)
+    - **Asus**: F8/Esc (Boot Menu), F2/Del (BIOS Setup)
+    - **Toshiba**: F12/F2 (Boot Menu), Esc then F12 (Alternate Method)
+    - **Sony**: F11/Assist Button (Boot Menu/Recovery Options)
+    - **Samsung**: F2/F12/Esc (Boot Menu), F2 (BIOS Setup)
+    - **MSI**: F11 (Boot Menu), Del (BIOS Setup)
+    - **Apple**: Option (Alt) Key (Boot Menu for Mac)
+    - If your hardware was not listed read the manufacturers instructions how to enter the boot Menu/BIOS Setup.
+
+**During Boot**
+
+Select `NixOS` to boot into the clan installer.
+
+**After Booting**
+
+For deploying your configuration the machine needs to be connected via LAN (recommended).
+
+For connecting via Wifi, please consult the [guide below](#optional-connect-to-wifi).
+
+---
+
+## Whats next?
+
+- [Configure Machines](configure.md): Customize machine configuration
+- [Deploying](machines.md): Deploying a Machine configuration
+- [WiFi](#optional-connect-to-wifi): Guide for connecting to Wifi.
+
+---
+
+## (Optional) Connect to Wifi
+
+If you don't have access via LAN the Installer offers support for connecting via Wifi.
+
+```shellSession
+iwctl
+```
+
+This will enter `iwd`
+
+```{.console, .no-copy}
+[iwd]#
+```
+
+Now run the following command to connect to your Wifi:
+
+```shellSession
+# Identify your network device.
+device list
+# Replace 'wlan0' with your wireless device name
+# Find your Wifi SSID.
+station wlan0 scan
+station wlan0 get-networks
+
+# Replace your_ssid with the Wifi SSID
+# Connect to your network.
+station wlan0 connect your_ssid
+
+# Verify you are connected
+station wlan0 show
+```
+
+If the connection was successful you should see something like this:
+
+```{.console, .no-copy}
+State                 connected
+Connected network     FRITZ!Box (Your router device)
+IPv4 address          192.168.188.50 (Your new local ip)
+```
+
+Press `ctrl-d` to exit `IWD`.
+
+!!! Important
+    Press `ctrl-d` **again** to update the displayed QR code and connection information.
+
+You're all set up

docs/site/getting-started/machines.md

@@ -0,0 +1,213 @@
+# Deploy Machine
+
+Integrating a new machine into your Clan environment is an easy yet flexible process, allowing for a straight forward management of multiple NixOS configurations.
+
+We'll walk you through adding a new computer to your Clan.
+
+## Installing a New Machine
+
+Clan CLI, in conjunction with [nixos-anywhere](https://github.com/nix-community/nixos-anywhere), provides a seamless method for installing NixOS on various machines.
+
+This process involves preparing a suitable hardware and disk partitioning configuration and ensuring the target machine is accessible via SSH.
+
+### Step 0. Prerequisites
+
+=== "**Physical Hardware**"
+
+    - [x] **Two Computers**: You need one computer that you're getting ready (we'll call this the Target Computer) and another one to set it up from (we'll call this the Setup Computer). Make sure both can talk to each other over the network using SSH.
+    - [x] **Machine configuration**: See our basic [configuration guide](./configure.md)
+    - [x] **Initialized secrets**: See [secrets](secrets.md) for how to initialize your secrets.
+    - [x] **USB Flash Drive**: See [Clan Installer](installer.md)
+
+    !!! Steps
+    
+        1. Create a NixOS installer image and transfer it to a bootable USB drive as described in the [installer](./installer.md).
+         
+        2. Boot the target machine and connect it to a network that makes it reachable from your setup computer.
+
+=== "**Baremetal Machines**"
+
+    - [x] **Two Computers**: You need one computer that you're getting ready (we'll call this the Target Computer) and another one to set it up from (we'll call this the Setup Computer). Make sure both can talk to each other over the network using SSH.
+    - [x] **Machine configuration**: See our basic [configuration guide](./configure.md)
+    - [x] **Initialized secrets**: See [secrets](secrets.md) for how to initialize your secrets.
+
+    !!! Steps
+
+        - Any cloud machine if it is reachable via SSH and supports `kexec`.
+
+Confirm the machine is reachable via SSH from your setup computer.
+
+```bash
+ssh root@<your_target_machine_ip>
+```
+
+### Step 1. Deploy the machine
+
+**Finally deployment time!** Use the following command to build and deploy the image via SSH onto your machine.
+
+=== "**SSH access**"
+
+
+
+    Replace `<target_host>` with the **target computers' ip address**:
+
+    ```bash
+    clan machines install my-machine <target_host>
+    ```
+
+    !!!note 
+        Building and deploying time will depend on hardware and connection speed.
+
+=== "**Image Installer**"
+
+    This method makes use of the image installers of [nixos-images](https://github.com/nix-community/nixos-images).
+    See how to prepare the installer for use [here](./installer.md).
+
+    The installer will randomly generate a password and local addresses on boot, then run ssh with these preconfigured.
+    The installer shows it's deployment relevant information in two formats, a text form, as well as a QR code.
+
+    ???example "An example view of a booted installer." 
+        This is an example of the booted installer.
+
+        ```{ .bash .annotate }
+        ┌─────────────────────────────────────────────────────────────────────────────────────┐
+        │ ┌───────────────────────────┐                                                       │
+        │ │███████████████████████████│ # This is the QR Code (1)                             │
+        │ │██ ▄▄▄▄▄ █▀▄█▀█▀▄█ ▄▄▄▄▄ ██│                                                       │
+        │ │██ █   █ █▀▄▄▄█ ▀█ █   █ ██│                                                       │
+        │ │██ █▄▄▄█ █▀▄ ▀▄▄▄█ █▄▄▄█ ██│                                                       │
+        │ │██▄▄▄▄▄▄▄█▄▀ ▀▄▀▄█▄▄▄▄▄▄▄██│                                                       │
+        │ │███▀▀▀ █▄▄█ ▀▄   ▄▀▄█   ███│                                                       │
+        │ │██▄██▄▄█▄▄▀▀██▄▀ ▄▄▄ ▄▀█▀██│                                                       │
+        │ │██ ▄▄▄▄▄ █▄▄▄▄ █ █▄█ █▀ ███│                                                       │
+        │ │██ █   █ █ █  █ ▄▄▄  ▄▀▀ ██│                                                       │
+        │ │██ █▄▄▄█ █ ▄ ▄    ▄ ▀█ ▄███│                                                       │
+        │ │██▄▄▄▄▄▄▄█▄▄▄▄▄▄█▄▄▄▄▄█▄███│                                                       │
+        │ │███████████████████████████│                                                       │
+        │ └───────────────────────────┘                                                       │
+        │ ┌─────────────────────────────────────────────────────────────────────────────────┐ │
+        │ │Root password: cheesy-capital-unwell  # password (2)                             │ │
+        │ │Local network addresses:                                                         │ │
+        │ │enp1s0           UP    192.168.178.169/24 metric 1024 fe80::21e:6ff:fe45:3c92/64 │ │
+        │ │enp2s0           DOWN                                                            │ │
+        │ │wlan0            DOWN # connect to wlan (3)                                      │ │
+        │ │Onion address: 6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion    │ │
+        │ │Multicast DNS: nixos-installer.local                                             │ │
+        │ └─────────────────────────────────────────────────────────────────────────────────┘ │
+        │ Press 'Ctrl-C' for console access                                                   │ 
+        │                                                                                     │
+        └─────────────────────────────────────────────────────────────────────────────────────┘
+        ```
+ 
+        1.  This is not an actual QR code, because it is displayed rather poorly on text sites.
+            This would be the actual content of this specific QR code prettified:
+            ```json
+            {
+              "pass": "cheesy-capital-unwell",
+              "tor": "6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion",
+              "addrs": [
+                "2001:9e8:347:ca00:21e:6ff:fe45:3c92"
+              ]
+            }
+            ```
+            
+            To generate the actual QR code, that would be displayed use:
+            ```shellSession
+            echo '{"pass":"cheesy-capital-unwell","tor":"6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion","addrs":["2001:9e8:347:ca00:21e:6ff:fe45:3c92"]}' | nix run nixpkgs#qrencode -- -s 2 -m 2 -t utf8
+            ```
+        2.  The root password for the installer medium.
+            This password is autogenerated and meant to be easily typeable.
+        3.  See how to connect the installer medium to wlan [here](./installer.md#optional-connect-to-wifi).
+        4.  :man_raising_hand: I'm a code annotation! I can contain `code`, __formatted
+            text__, images, ... basically anything that can be written in Markdown.
+
+    
+    !!!tip
+        We recommend using KDE Connect for sharing the deployment information from the QR code with the deploying machine.
+
+    
+    The QR code can be used to deploy either with an image, that is decoded on the fly, or it's contained json information.
+
+    With the path to a `json` string, or the string itself:
+    ```terminal
+    clan machines install [MACHINE] --json [JSON]
+    ```
+    With the path to an image containing the relevant QR code:
+    ```terminal
+    clan machines install [MACHINE] --png [PATH]
+    ```
+
+
+!!! success
+
+    Your machine is all set up. 🎉 🚀
+
+---
+
+## What's next ?
+
+- [**Update a Machine**](#update-your-machines): Learn how to update an existing machine?
+
+Coming Soon:
+
+- **Join Your Machines in a Private Network:**: Stay tuned for steps on linking all your machines into a secure mesh network with Clan.
+
+---
+
+## Update Your Machines
+
+Clan CLI enables you to remotely update your machines over SSH. This requires setting up a target address for each target machine.
+
+### Setting the Target Host
+
+Replace `host_or_ip` with the actual hostname or IP address of your target machine:
+
+```bash
+clan config --machine my-machine clan.networking.targetHost root@host_or_ip
+```
+
+!!! warning
+    The use of `root@` in the target address implies SSH access as the `root` user.
+    Ensure that the root login is secured and only used when necessary.
+
+### Updating Machine Configurations
+
+Execute the following command to update the specified machine:
+
+```bash
+clan machines update my-machine
+```
+
+You can also update all configured machines simultaneously by omitting the machine name:
+
+```bash
+clan machines update
+```
+
+### Setting a Build Host
+
+If the machine does not have enough resources to run the NixOS evaluation or build itself,
+it is also possible to specify a build host instead.
+During an update, the cli will ssh into the build host and run `nixos-rebuild` from there.
+
+```bash
+clan config --machine my-machine clan.networking.buildHost root@host_or_ip
+```
+
+### Excluding a machine from `clan machine update`
+
+To exclude machines from being updated when running `clan machines update` without any machines specified,
+one can set the `clan.deployment.requireExplicitUpdate` option to true:
+
+```bash
+clan config --machine my-machine clan.deployment.requireExplicitUpdate true
+```
+
+This is useful for machines that are not always online or are not part of the regular update cycle.
+
+---
+
+# TODO:
+* TODO: How to join others people zerotier
+  * `services.zerotier.joinNetworks = [ "network-id" ]`
+* Controller needs to approve over webinterface or cli

docs/site/getting-started/networking.md

@@ -0,0 +1,95 @@
+# Overlay Networks
+
+This guide provides detailed instructions for configuring
+[ZeroTier VPN](https://zerotier.com) within Clan. Follow the
+outlined steps to set up a machine as a VPN controller (`<CONTROLLER>`) and to
+include a new machine into the VPN.
+
+## 1. Setting Up the VPN Controller
+
+The VPN controller is initially essential for providing configuration to new
+peers. Once addresses are allocated, the controller's continuous operation is not essential.
+
+### Instructions
+
+1. **Designate a Machine**: Label a machine as the VPN controller in the clan,
+   referred to as `<CONTROLLER>` henceforth in this guide.
+1. **Add Configuration**: Input the following configuration to the NixOS
+   configuration of the controller machine:
+   ```nix
+   clan.networking.zerotier.controller = {
+     enable = true;
+     public = true;
+   };
+   ```
+1. **Update the Controller Machine**: Execute the following:
+   ```bash
+   $ clan machines update <CONTROLLER>
+   ```
+   Your machine is now operational as the VPN controller.
+
+## 2. Integrating a New Machine to the VPN
+
+To introduce a new machine to the VPN, adhere to the following steps:
+
+### Instructions:
+
+1. **Update Configuration**: On the new machine, incorporate the following to its
+   configuration, substituting `<CONTROLLER>` with the controller machine name:
+   ```nix
+   { config, ... }: {
+     clan.networking.zerotier.networkId = builtins.readFile (config.clanCore.clanDir + "/machines/<CONTROLLER>/facts/zerotier-network-id");
+   }
+   ```
+1. **Update the New Machine**: Execute:
+   ```bash
+   $ clan machines update <NEW_MACHINE>
+   ```
+   Replace `<NEW_MACHINE>` with the designated new machine name.
+1. **Retrieve the ZeroTier ID**: On the `new_machine`, execute:
+   ```bash
+   $ sudo zerotier-cli info
+   ```
+   Example Output: 
+   ```{.console, .no-copy}
+   200 info d2c71971db 1.12.1 OFFLINE
+   ```
+   , where `d2c71971db` is the ZeroTier ID.
+1. **Authorize the New Machine on the Controller**: On the controller machine,
+   execute:
+   ```bash
+   $ sudo zerotier-members allow <ID>
+   ```
+   Substitute `<ID>` with the ZeroTier ID obtained previously.
+1. **Verify Connection**: On the `new_machine`, re-execute:
+   ```bash
+   $ sudo zerotier-cli info
+   ```
+   The status should now be "ONLINE":
+   ```{.console, .no-copy}
+   200 info d2c71971db 1.12.1 ONLINE
+   ```
+
+!!! success "Congratulations!"
+    The new machine is now part of the VPN, and the ZeroTier
+    configuration on NixOS within the Clan project is complete.
+
+## Decision
+
+We chose zerotier because in our tests it was the easiest solution to bootstrap. You can selfhost a controller and the controller doesn't need to be globally reachable.
+
+In the future we plan to add additional network technologies like tinc, head/tailscale, yggdrassil and mycelium.
+
+## Specification
+
+By default all machines within one clan are connected via the chosen network technology.
+
+```
+Clan 
+    Node A
+    <-> (zerotier / mycelium / ...) 
+    Node B
+```
+
+If you select multiple network technologies at the same time. e.g. (zerotier + yggdrassil)
+One of them is the primary network and the above statement holds for the primary network.

docs/site/getting-started/secrets.md

@@ -1,69 +1,151 @@
-# Managing Secrets with Clan
+# Secrets / Facts
 
-Clan enables encryption of secrets within a Clan flake, ensuring secure sharing among users.
-This documentation will guide you through managing secrets with the Clan CLI,
-which utilizes the [sops](https://github.com/getsops/sops) format and
-integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
+Clan enables encryption of secrets (such as passwords & keys) ensuring security and ease-of-use among users.
 
-## 1. Generating Keys and Creating Secrets
+Clan utilizes the [sops](https://github.com/getsops/sops) format and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
 
-To begin, generate a key pair:
+This documentation will guide you through managing secrets with the Clan CLI
 
-```shellSession
-$ clan secrets key generate
+## 1. Initializing Secrets
+
+### Create Your Master Keypair
+
+To get started, you'll need to create **Your master keypair**.
+
+!!! info
+    Don't worry — if you've already made one before, this step won't change or overwrite it.
+
+```bash
+clan secrets key generate
 ```
 
 **Output**:
 
-```
+```{.console, .no-copy}
 Public key: age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7
-Generated age private key at '/home/joerg/.config/sops/age/keys.txt' for your user.
+
 Generated age private key at '/home/joerg/.config/sops/age/keys.txt' for your user. Please back it up on a secure location or you will lose access to your secrets.
-Also add your age public key to the repository with 'clan secrets users add youruser age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7' (replace you
-user with your user name)
+Also add your age public key to the repository with 'clan secrets users add YOUR_USER age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7' (replace YOUR_USER with your actual username)
 ```
 
-⚠️ **Important**: Backup the generated private key securely, or risk losing access to your secrets.
+!!! warning 
+    Make sure to keep a safe backup of the private key you've just created.
+    If it's lost, you won't be able to get to your secrets anymore because they all need the master key to be unlocked.
 
-Next, add your public key to the Clan flake repository:
+!!! note 
+    It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
 
-```shellSession
-$ clan secrets users add <your_username> <your_public_key>
+### Add Your Public Key
+
+```bash
+clan secrets users add <your_username> <your_public_key>
 ```
 
-Doing so creates this structure in your Clan flake:
+!!! note 
+    Choose the same username as on your Setup/Source Machine that you use to control the deployment with.
 
-```
+Once run this will create the following files:
+
+```{.console, .no-copy}
 sops/
 └── users/
     └── <your_username>/
         └── key.json
 ```
 
-Now, to set your first secret:
+## 2. Adding Machine Keys
 
-```shellSession
+New machines in Clan come with age keys stored in `./sops/machines/<machine_name>`. To list these machines:
+
+```bash
+$ clan secrets machines list
+```
+
+For existing machines, add their keys:
+
+```bash
+$ clan secrets machines add <machine_name> <age_key>
+```
+
+### Advanced
+
+To fetch an age key from an SSH host key:
+
+```bash
+$ ssh-keyscan <domain_name> | nix shell nixpkgs#ssh-to-age -c ssh-to-age
+```
+
+## 3. Assigning Access
+
+By default, secrets are encrypted for your key. To specify which users and machines can access a secret:
+
+```bash
+$ clan secrets set --machine <machine1> --machine <machine2> --user <user1> --user <user2> <secret_name>
+```
+
+You can add machines/users to existing secrets without modifying the secret:
+
+```bash
+$ clan secrets machines add-secret <machine_name> <secret_name>
+```
+
+## 4. Adding Secrets
+
+```bash
 $ clan secrets set mysecret
 Paste your secret: 
 ```
 
-Note: As you type your secret, keypresses won't be displayed. Press Enter to save the secret.
+!!! note 
+    As you type your secret won't be displayed. Press Enter to save the secret.
 
-Retrieve the stored secret:
+## 5. Retrieving Stored Secrets
 
-```shellSession
+```bash
 $ clan secrets get mysecret
 ```
 
-And list all secrets like this:
+### List all Secrets
 
-```shellSession
+```bash
 $ clan secrets list
 ```
 
+## 6. Groups
+
+Clan CLI makes it easy to manage access by allowing you to create groups.
+
+All users within a group inherit access to all secrets of the group.
+
+This feature eases the process of handling permissions for multiple users.
+
+Here's how to get started:
+
+1. **Creating Groups**:
+
+   Assign users to a new group, e.g., `admins`:
+
+   ```bash
+   $ clan secrets groups add admins <username>
+   ```
+
+2. **Listing Groups**:
+
+   ```bash
+   $ clan secrets groups list
+   ```
+
+3. **Assigning Secrets to Groups**:
+
+   ```bash
+   $ clan secrets groups add-secret <group_name> <secret_name>
+   ```
+
+## Further
+
 Secrets in the repository follow this structure:
 
-```
+```{.console, .no-copy}
 sops/
 ├── secrets/
 │   └── <secret_name>/
@@ -73,73 +155,15 @@ sops/
 ```
 
 The content of the secret is stored encrypted inside the `secret` file under `mysecret`.
+
 By default, secrets are encrypted with your key to ensure readability.
 
-## 2. Adding Machine Keys
-
-New machines in Clan come with age keys stored in `./sops/machines/<machine_name>`. To list these machines:
-
-```shellSession
-$ clan secrets machines list
-```
-
-For existing machines, add their keys:
-
-```shellSession
-$ clan secrets machines add <machine_name> <age_key>
-```
-
-To fetch an age key from an SSH host key:
-
-```shellSession
-$ ssh-keyscan <domain_name> | nix shell nixpkgs#ssh-to-age -c ssh-to-age
-```
-
-## 3. Assigning Access
-
-By default, secrets are encrypted for your key. To specify which users and machines can access a secret:
-
-```shellSession
-$ clan secrets set --machine <machine1> --machine <machine2> --user <user1> --user <user2> <secret_name>
-```
-
-You can add machines/users to existing secrets without modifying the secret:
-
-```shellSession
-$ clan secrets machines add-secret <machine_name> <secret_name>
-```
-
-## 4. Utilizing Groups
-
-For convenience, Clan CLI allows group creation to simplify access management. Here's how:
-
-1. **Creating Groups**:
-
-   Assign users to a new group, e.g., `admins`:
-
-   ```shellSession
-   $ clan secrets groups add admins <username>
-   ```
-
-2. **Listing Groups**:
-
-   ```shellSession
-   $ clan secrets groups list
-   ```
-
-3. **Assigning Secrets to Groups**:
-
-   ```shellSession
-   $ clan secrets groups add-secret <group_name> <secret_name>
-   ```
-
-# NixOS integration
+### NixOS integration
 
 A NixOS machine will automatically import all secrets that are encrypted for the
 current machine. At runtime it will use the host key to decrypt all secrets into
-a in-memory, non-persistent filesystem using
-[sops-nix](https://github.com/Mic92/sops-nix). In your nixos configuration you
-can get a path to secrets like this `config.sops.secrets.<name>.path`. Example:
+an in-memory, non-persistent filesystem using [sops-nix](https://github.com/Mic92/sops-nix). 
+In your nixos configuration you can get a path to secrets like this `config.sops.secrets.<name>.path`. For example:
 
 ```nix
 { config, ...}: {
@@ -155,19 +179,18 @@ can get a path to secrets like this `config.sops.secrets.<name>.path`. Example:
 See the [readme](https://github.com/Mic92/sops-nix) of sops-nix for more
 examples.
 
-# Importing existing sops-based keys / sops-nix
+### Migration: Importing existing sops-based keys / sops-nix
 
-`clan secrets` stores each secrets in a single file, whereas [sops](https://github.com/Mic92/sops-nix)
-commonly allows to put all secrets in a yaml or json documents.
+`clan secrets` stores each secret in a single file, whereas [sops](https://github.com/Mic92/sops-nix) commonly allows to put all secrets in a yaml or json document.
 
-If you already happend to use sops-nix, you can migrate by using the `clan secrets import-sops` command by importing these documents:
+If you already happened to use sops-nix, you can migrate by using the `clan secrets import-sops` command by importing these files:
 
-```shellSession
+```bash
 % clan secrets import-sops --prefix matchbox- --group admins --machine matchbox nixos/matchbox/secrets/secrets.yaml
 ```
 
-This will create secrets for each secret found in `nixos/matchbox/secrets/secrets.yaml` in a ./sops folder of your repository.
-Each member of the group `admins` will be able
+This will create secrets for each secret found in `nixos/matchbox/secrets/secrets.yaml` in a `./sops` folder of your repository.
+Each member of the group `admins` in this case will be able to decrypt the secrets with their respective key.
 
 Since our clan secret module will auto-import secrets that are encrypted for a particular nixos machine,
 you can now remove `sops.secrets.<secrets> = { };` unless you need to specify more options for the secret like owner/group of the secret file.

docs/site/index.md

@@ -0,0 +1,113 @@
+# Getting Started
+
+Welcome to your simple guide on starting a new Clan project.
+
+## What's Inside
+
+We've put together a straightforward guide to help you out:
+
+- [**Starting with a New Clan Project**](#starting-with-a-new-clan-project): Create a new Clan from scratch.
+- [**Integrating Clan using Flake-Parts**](getting-started/flake-parts.md)
+
+---
+
+## **Starting with a New Clan Project**
+
+Create your own clan with these initial steps.
+
+### Prerequisites
+
+#### Linux
+
+Clan depends on nix installed on your system. Run the following command to install nix.
+
+```bash
+curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
+```
+
+#### NixOS
+
+If you run NixOS the `nix` binary is already installed.
+
+You will also need to enable the `flakes` and `nix-commands` experimental features.
+
+```bash
+# /etc/nix/nix.conf or ~/.config/nix/nix.conf
+experimental-features = nix-command flakes
+```
+
+#### Other
+
+Clan doesn't offer dedicated support for other operating systems yet.
+
+### Step 1: Add Clan CLI to Your Shell
+
+Add the Clan CLI into your development workflow:
+
+```bash
+nix shell git+https://git.clan.lol/clan/clan-core#clan-cli
+```
+
+### Step 2: Initialize Your Project
+
+Set the foundation of your Clan project by initializing it as follows:
+
+```bash
+clan flakes create my-clan
+```
+
+This command creates the `flake.nix` and `.clan-flake` files for your project.
+
+### Step 3: Verify the Project Structure
+
+Ensure that all project files exist by running:
+
+```bash
+cd my-clan
+tree
+```
+
+This should yield the following:
+
+``` { .console .no-copy }
+.
+├── flake.nix
+├── machines
+│   ├── jon
+│   │   ├── configuration.nix
+│   │   └── hardware-configuration.nix
+│   └── sara
+│       ├── configuration.nix
+│       └── hardware-configuration.nix
+└── modules
+    └── shared.nix
+
+5 directories, 6 files
+```
+
+```bash
+clan machines list
+```
+
+``` { .console .no-copy }
+jon
+sara
+```
+
+!!! success
+
+    You just successfully bootstrapped your first clan directory.
+
+---
+
+### What's Next?
+
+- [**Machine Configuration**](getting-started/configure.md): Declare behavior and configuration of machines.
+
+- [**Deploy Machines**](getting-started/machines.md): Learn how to deploy to any remote machine.
+
+- [**Installer**](getting-started/installer.md): Setting up new computers remotely is easy with an USB stick.
+
+- [**Check out our Templates**](templates/index.md)
+
+---

docs/site/templates/index.md

@@ -0,0 +1,24 @@
+# Templates
+
+We provide some starting templates you can easily use one of those via `nix flakes`.
+
+They showcase best practices and guide you through setting up and using Clan's modules
+
+I.e. To use the `new-clan` template run the following command:
+
+```bash
+nix flake init -t git+https://git.clan.lol/clan/clan-core#new-clan
+```
+
+## Available Templates
+
+We offer the following templates:
+
+To initialize a clan with one of those run:
+```bash
+nix flake init -t git+https://git.clan.lol/clan/clan-core#[TEMPLATE_NAME]
+```
+
+Substitute `[TEMPLATE_NAME]` with the name of the template.
+
+- **new-clan**:  Perfect for beginners, this template shows you how to link two machines in a basic setup.

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1707524024,
-        "narHash": "sha256-HmumZ8FuWAAYZrWUKm3N4G4h8nmZ5VUVX+vXLmCJNKM=",
+        "lastModified": 1712356478,
+        "narHash": "sha256-kTcEtrQIRnexu5lAbLsmUcfR2CrmsACF1s3ZFw1NEVA=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "d07de570ba05cec2807d058daaa044f6955720c7",
+        "rev": "0a17298c0d96190ef3be729d594ba202b9c53beb",
         "type": "github"
       },
       "original": {
@@ -27,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1706830856,
-        "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=",
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
         "type": "github"
       },
       "original": {
@@ -42,11 +42,11 @@
     },
     "nixlib": {
       "locked": {
-        "lastModified": 1693701915,
-        "narHash": "sha256-waHPLdDYUOHSEtMKKabcKIMhlUOHPOOPQ9UyFeEoovs=",
+        "lastModified": 1711846064,
+        "narHash": "sha256-cqfX0QJNEnge3a77VnytM0Q6QZZ0DziFXt6tSCV8ZSc=",
         "owner": "nix-community",
         "repo": "nixpkgs.lib",
-        "rev": "f5af57d3ef9947a70ac86e42695231ac1ad00c25",
+        "rev": "90b1a963ff84dc532db92f678296ff2499a60a87",
         "type": "github"
       },
       "original": {
@@ -63,11 +63,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1707405218,
-        "narHash": "sha256-ZQ366Oo8WJbCqXAZET7N0Sz6RQ3G2IbqVtxQRSa3SXc=",
+        "lastModified": 1712191720,
+        "narHash": "sha256-xXtSSnVHURHsxLQO30dzCKW5NJVGV/umdQPmFjPFMVA=",
         "owner": "nix-community",
         "repo": "nixos-generators",
-        "rev": "843e2f04c716092797ffa4ce14c446adce2f09ef",
+        "rev": "0c15e76bed5432d7775a22e8d22059511f59d23a",
         "type": "github"
       },
       "original": {
@@ -78,11 +78,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1707639604,
-        "narHash": "sha256-J5ipSdfkbYcYaH3Js2dUf3Of94BWStapdmxpW5wwH1U=",
+        "lastModified": 1712468661,
+        "narHash": "sha256-n2gVVBs+rV+HzPv/N3QQv5cdAXqSkjmaObvfeMqnw2c=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "bdc57436da855500d44e9c1ce7450c0772e1cfa1",
+        "rev": "298edc8f1e0dfffce67f50375c9f5952e04a6d02",
         "type": "github"
       },
       "original": {
@@ -110,11 +110,11 @@
         "nixpkgs-stable": []
       },
       "locked": {
-        "lastModified": 1707620614,
-        "narHash": "sha256-gfAoB9dGzBu62NoAoM945aok7+6M+LFu+nvnGwAsTp4=",
+        "lastModified": 1712458908,
+        "narHash": "sha256-DMgBS+jNHDg8z3g9GkwqL8xTKXCRQ/0FGsAyrniVonc=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "2eb7c4ba3aa75e2660fd217eb1ab64d5b793608e",
+        "rev": "39191e8e6265b106c9a2ba0cfd3a4dafe98a31c6",
         "type": "github"
       },
       "original": {
@@ -130,11 +130,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1707300477,
-        "narHash": "sha256-qQF0fEkHlnxHcrKIMRzOETnRBksUK048MXkX0SOmxvA=",
+        "lastModified": 1711963903,
+        "narHash": "sha256-N3QDhoaX+paWXHbEXZapqd1r95mdshxToGowtjtYkGI=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "ac599dab59a66304eb511af07b3883114f061b9d",
+        "rev": "49dc4a92b02b8e68798abd99184f228243b6e3ac",
         "type": "github"
       },
       "original": {

flake.nix

@@ -2,7 +2,9 @@
   description = "clan.lol base operating system";
 
   nixConfig.extra-substituters = [ "https://cache.clan.lol" ];
-  nixConfig.extra-trusted-public-keys = [ "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28=" ];
+  nixConfig.extra-trusted-public-keys = [
+    "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28="
+  ];
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
@@ -20,8 +22,11 @@
     treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
   };
 
-  outputs = inputs @ { flake-parts, ... }:
-    flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }: {
+  outputs =
+    inputs@{ flake-parts, ... }:
+    flake-parts.lib.mkFlake { inherit inputs; } (
+      { ... }:
+      {
         systems = [
           "x86_64-linux"
           "aarch64-linux"
@@ -29,34 +34,19 @@
         ];
         imports = [
           ./checks/flake-module.nix
-        ./devShell.nix
-        ./formatter.nix
-        ./templates/flake-module.nix
           ./clanModules/flake-module.nix
-
-        ./pkgs/flake-module.nix
-
+          ./flakeModules/flake-module.nix
+          (import ./flakeModules/clan.nix inputs.self)
+          ./devShell.nix
+          # TODO: migrate this @davHau
+          # ./docs/flake-module
+          ./docs/nix/flake-module.nix
+          ./formatter.nix
           ./lib/flake-module.nix
           ./nixosModules/flake-module.nix
-        {
-          options.flake = flake-parts.lib.mkSubmoduleOptions {
-            clanInternals = lib.mkOption {
-              type = lib.types.submodule {
-                options = {
-                  all-machines-json = lib.mkOption {
-                    type = lib.types.attrsOf lib.types.str;
-                  };
-                  machines = lib.mkOption {
-                    type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified);
-                  };
-                  machinesFunc = lib.mkOption {
-                    type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified);
-                  };
-                };
-              };
-            };
-          };
-        }
+          ./pkgs/flake-module.nix
+          ./templates/flake-module.nix
         ];
-    });
+      }
+    );
 }

flakeModules/clan.nix

@@ -0,0 +1,75 @@
+clan-core:
+{
+  config,
+  lib,
+  flake-parts-lib,
+  inputs,
+  self,
+  ...
+}:
+let
+  inherit (lib) mkOption types;
+  buildClan = import ../lib/build-clan {
+    inherit lib clan-core;
+    inherit (inputs) nixpkgs;
+  };
+
+  cfg = config.clan;
+in
+{
+  options.clan = {
+    directory = mkOption {
+      type = types.path;
+      description = "The directory containing the clan subdirectory";
+      default = self; # default to the directory of the flake
+    };
+    specialArgs = mkOption {
+      type = types.attrsOf types.raw;
+      default = { };
+      description = "Extra arguments to pass to nixosSystem i.e. useful to make self available";
+    };
+    machines = mkOption {
+      type = types.attrsOf types.raw;
+      default = { };
+      description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }";
+    };
+    clanName = mkOption {
+      type = types.str;
+      description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
+    };
+    clanIcon = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = "A path to an icon to be used for the clan, should be the same for all machines";
+    };
+    pkgsForSystem = mkOption {
+      type = types.functionTo types.raw;
+      default = _system: null;
+      description = "A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.";
+    };
+  };
+  options.flake = flake-parts-lib.mkSubmoduleOptions {
+    clanInternals = lib.mkOption {
+      type = lib.types.submodule {
+        options = {
+          all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
+          machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
+          machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
+        };
+      };
+    };
+  };
+  config = {
+    flake = buildClan {
+      inherit (cfg)
+        directory
+        specialArgs
+        machines
+        clanName
+        clanIcon
+        pkgsForSystem
+        ;
+    };
+  };
+  _file = __curPos.file;
+}

flakeModules/flake-module.nix

@@ -0,0 +1,7 @@
+{ self, config, ... }:
+{
+  flake.flakeModules = {
+    clan = import ./clan.nix self;
+    default = config.flake.flakeModules.clan;
+  };
+}

formatter.nix

@@ -1,20 +1,17 @@
-{ lib
-, inputs
-, ...
-}: {
-  imports = [
-    inputs.treefmt-nix.flakeModule
-  ];
-  perSystem = { self', pkgs, ... }: {
+{ lib, inputs, ... }:
+{
+  imports = [ inputs.treefmt-nix.flakeModule ];
+  perSystem =
+    { self', pkgs, ... }:
+    {
       treefmt.projectRootFile = "flake.nix";
-    treefmt.flakeCheck = true;
-    treefmt.flakeFormatter = true;
       treefmt.programs.shellcheck.enable = true;
 
       treefmt.programs.mypy.enable = true;
       treefmt.programs.mypy.directories = {
-      "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.pytestDependencies;
-      "pkgs/clan-vm-manager".extraPythonPackages = self'.packages.clan-vm-manager.propagatedBuildInputs;
+        "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
+        "pkgs/clan-vm-manager".extraPythonPackages =
+          self'.packages.clan-vm-manager.externalTestDeps ++ self'.packages.clan-cli.testDependencies;
       };
 
       treefmt.settings.formatter.nix = {
@@ -25,18 +22,22 @@
             # First deadnix
             ${lib.getExe pkgs.deadnix} --edit "$@"
             # Then nixpkgs-fmt
-          ${lib.getExe pkgs.nixpkgs-fmt} "$@"
+            ${lib.getExe pkgs.nixfmt-rfc-style} "$@"
           ''
           "--" # this argument is ignored by bash
         ];
         includes = [ "*.nix" ];
+        excludes = [
+          # Was copied from nixpkgs. Keep diff minimal to simplify upstreaming.
+          "pkgs/builders/script-writers.nix"
+        ];
       };
       treefmt.settings.formatter.python = {
         command = "sh";
         options = [
           "-eucx"
           ''
-          ${lib.getExe pkgs.ruff} --fix "$@"
+            ${lib.getExe pkgs.ruff} check --fix "$@"
             ${lib.getExe pkgs.ruff} format "$@"
           ''
           "--" # this argument is ignored by bash

lib/build-clan/default.nix

@@ -1,36 +1,51 @@
-{ clan-core, nixpkgs, lib }:
-{ directory  # The directory containing the machines subdirectory
-, specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available
-, machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... }
-, clanName # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
-, clanIcon ? null # A path to an icon to be used for the clan, should be the same for all machines
+{
+  clan-core,
+  nixpkgs,
+  lib,
+}:
+{
+  directory, # The directory containing the machines subdirectory
+  specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available
+  machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... }
+  clanName, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
+  clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines
+  pkgsForSystem ? (_system: null), # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
+# This improves performance, but all nipxkgs.* options will be ignored.
 }:
 let
-  machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (builtins.readDir (directory + /machines));
+  machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
+    builtins.readDir (directory + /machines)
+  );
 
-  machineSettings = machineName:
+  machineSettings =
+    machineName:
     # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily
     # This is useful for doing a dry-run before writing changes into the settings.json
     # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval
-    if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != ""
-    then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))
+    if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then
+      builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))
     else
-      lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json")
-        (builtins.fromJSON
-          (builtins.readFile (directory + /machines/${machineName}/settings.json)));
+      lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") (
+        builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))
+      );
 
   # Read additional imports specified via a config option in settings.json
   # This is not an infinite recursion, because the imports are discovered here
   #   before calling evalModules.
   # It is still useful to have the imports as an option, as this allows for type
   #   checking and easy integration with the config frontend(s)
-  machineImports = machineSettings:
-    map
-      (module: clan-core.clanModules.${module})
-      (machineSettings.clanImports or [ ]);
+  machineImports =
+    machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]);
 
   # TODO: remove default system once we have a hardware-config mechanism
-  nixosConfiguration = { system ? "x86_64-linux", name, pkgs ? null, extraConfig ? { } }: nixpkgs.lib.nixosSystem {
+  nixosConfiguration =
+    {
+      system ? "x86_64-linux",
+      name,
+      pkgs ? null,
+      extraConfig ? { },
+    }:
+    nixpkgs.lib.nixosSystem {
       modules =
         let
           settings = machineSettings name;
@@ -41,7 +56,9 @@ let
           clan-core.nixosModules.clanCore
           extraConfig
           (machines.${name} or { })
-        ({
+          (
+            {
+              networking.hostName = lib.mkDefault name;
               clanCore.clanName = clanName;
               clanCore.clanIcon = clanIcon;
               clanCore.clanDir = directory;
@@ -53,11 +70,13 @@ let
                 type = "path";
                 path = lib.mkDefault nixpkgs;
               };
-        } // lib.optionalAttrs (pkgs != null) {
-          nixpkgs.pkgs = lib.mkForce pkgs;
-        })
+            }
+            // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; }
+          )
         ];
-    inherit specialArgs;
+      specialArgs = {
+        inherit clan-core;
+      } // specialArgs;
     };
 
   allMachines = machinesDirs // machines;
@@ -75,17 +94,38 @@ let
   # This instantiates nixos for each system that we support:
   # configPerSystem = <system>.<machine>.nixosConfiguration
   # We need this to build nixos secret generators for each system
-  configsPerSystem = builtins.listToAttrs
-    (builtins.map
-      (system: lib.nameValuePair system
-        (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; }) allMachines))
-      supportedSystems);
+  configsPerSystem = builtins.listToAttrs (
+    builtins.map (
+      system:
+      lib.nameValuePair system (
+        lib.mapAttrs (
+          name: _:
+          nixosConfiguration {
+            inherit name system;
+            pkgs = pkgsForSystem system;
+          }
+        ) allMachines
+      )
+    ) supportedSystems
+  );
 
-  configsFuncPerSystem = builtins.listToAttrs
-    (builtins.map
-      (system: lib.nameValuePair system
-        (lib.mapAttrs (name: _: args: nixosConfiguration (args // { inherit name system; })) allMachines))
-      supportedSystems);
+  configsFuncPerSystem = builtins.listToAttrs (
+    builtins.map (
+      system:
+      lib.nameValuePair system (
+        lib.mapAttrs (
+          name: _: args:
+          nixosConfiguration (
+            args
+            // {
+              inherit name system;
+              pkgs = pkgsForSystem system;
+            }
+          )
+        ) allMachines
+      )
+    ) supportedSystems
+  );
 in
 {
   inherit nixosConfigurations;
@@ -93,8 +133,11 @@ in
   clanInternals = {
     machines = configsPerSystem;
     machinesFunc = configsFuncPerSystem;
-    all-machines-json = lib.mapAttrs
-      (system: configs: nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs))
-      configsPerSystem;
+    all-machines-json = lib.mapAttrs (
+      system: configs:
+      nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (
+        lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs
+      )
+    ) configsPerSystem;
   };
 }

lib/default.nix

@@ -1,4 +1,9 @@
-{ lib, clan-core, nixpkgs, ... }:
+{
+  lib,
+  clan-core,
+  nixpkgs,
+  ...
+}:
 {
   jsonschema = import ./jsonschema { inherit lib; };
 

lib/flake-module.nix

@@ -1,11 +1,11 @@
-{ lib
-, inputs
-, self
-, ...
-}: {
-  imports = [
-    ./jsonschema/flake-module.nix
-  ];
+{
+  lib,
+  inputs,
+  self,
+  ...
+}:
+{
+  imports = [ ./jsonschema/flake-module.nix ];
   flake.lib = import ./default.nix {
     inherit lib;
     inherit (inputs) nixpkgs;

lib/jsonschema/default.nix

@@ -1,15 +1,18 @@
-{ lib ? import <nixpkgs/lib>
-, excludedTypes ? [
+{
+  lib ? import <nixpkgs/lib>,
+  excludedTypes ? [
     "functionTo"
     "package"
-  ]
+  ],
 }:
 let
   # remove _module attribute from options
   clean = opts: builtins.removeAttrs opts [ "_module" ];
 
   # throw error if option type is not supported
-  notSupported = option: lib.trace option throw ''
+  notSupported =
+    option:
+    lib.trace option throw ''
       option type '${option.type.name}' ('${option.type.description}') not supported by jsonschema converter
       location: ${lib.concatStringsSep "." option.loc}
     '';
@@ -20,23 +23,29 @@ let
 
   filterExcludedAttrs = lib.filterAttrs (_name: opt: !isExcludedOption opt);
 
-  allBasicTypes =
-    [ "boolean" "integer" "number" "string" "array" "object" "null" ];
-
+  allBasicTypes = [
+    "boolean"
+    "integer"
+    "number"
+    "string"
+    "array"
+    "object"
+    "null"
+  ];
 in
 rec {
 
   # parses a nixos module to a jsonschema
-  parseModule = module:
+  parseModule =
+    module:
     let
-      evaled = lib.evalModules {
-        modules = [ module ];
-      };
+      evaled = lib.evalModules { modules = [ module ]; };
     in
     parseOptions evaled.options;
 
   # parses a set of evaluated nixos options to a jsonschema
-  parseOptions = options':
+  parseOptions =
+    options':
     let
       options = filterExcludedAttrs (clean options');
       # parse options to jsonschema properties
@@ -44,22 +53,20 @@ rec {
       # TODO: figure out how to handle if prop.anyOf is used
       isRequired = prop: !(prop ? default || prop.type or null == "object");
       requiredProps = lib.filterAttrs (_: prop: isRequired prop) properties;
-      required = lib.optionalAttrs (requiredProps != { }) {
-        required = lib.attrNames requiredProps;
-      };
+      required = lib.optionalAttrs (requiredProps != { }) { required = lib.attrNames requiredProps; };
     in
     # return jsonschema
-    required // {
+    required
+    // {
       type = "object";
       inherit properties;
     };
 
   # parses and evaluated nixos option to a jsonschema property definition
-  parseOption = option:
+  parseOption =
+    option:
     let
-      default = lib.optionalAttrs (option ? default) {
-        inherit (option) default;
-      };
+      default = lib.optionalAttrs (option ? default) { inherit (option) default; };
       description = lib.optionalAttrs (option ? description) {
         description = option.description.text or option.description;
       };
@@ -67,177 +74,217 @@ rec {
 
     # either type
     # TODO: if all nested optiosn are excluded, the parent sould be excluded too
-    if option.type.name or null == "either"
+    if
+      option.type.name or null == "either"
     # return jsonschema property definition for either
     then
       let
         optionsList' = [
-          { type = option.type.nestedTypes.left; _type = "option"; loc = option.loc; }
-          { type = option.type.nestedTypes.right; _type = "option"; loc = option.loc; }
+          {
+            type = option.type.nestedTypes.left;
+            _type = "option";
+            loc = option.loc;
+          }
+          {
+            type = option.type.nestedTypes.right;
+            _type = "option";
+            loc = option.loc;
+          }
         ];
         optionsList = filterExcluded optionsList';
       in
-      default // description // {
-        anyOf = map parseOption optionsList;
-      }
+      default // description // { anyOf = map parseOption optionsList; }
 
     # handle nested options (not a submodule)
-    else if ! option ? _type
-    then parseOptions option
+    else if !option ? _type then
+      parseOptions option
 
     # throw if not an option
-    else if option._type != "option" && option._type != "option-type"
-    then throw "parseOption: not an option"
+    else if option._type != "option" && option._type != "option-type" then
+      throw "parseOption: not an option"
 
     # parse nullOr
-    else if option.type.name == "nullOr"
+    else if
+      option.type.name == "nullOr"
     # return jsonschema property definition for nullOr
     then
       let
-        nestedOption =
-          { type = option.type.nestedTypes.elemType; _type = "option"; loc = option.loc; };
+        nestedOption = {
+          type = option.type.nestedTypes.elemType;
+          _type = "option";
+          loc = option.loc;
+        };
       in
-      default // description // {
-        anyOf =
-          [{ type = "null"; }]
-          ++ (
-            lib.optional (! isExcludedOption nestedOption)
-              (parseOption nestedOption)
-          );
+      default
+      // description
+      // {
+        anyOf = [
+          { type = "null"; }
+        ] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption));
       }
 
     # parse bool
-    else if option.type.name == "bool"
+    else if
+      option.type.name == "bool"
     # return jsonschema property definition for bool
-    then default // description // {
-      type = "boolean";
-    }
+    then
+      default // description // { type = "boolean"; }
 
     # parse float
-    else if option.type.name == "float"
+    else if
+      option.type.name == "float"
     # return jsonschema property definition for float
-    then default // description // {
-      type = "number";
-    }
+    then
+      default // description // { type = "number"; }
 
     # parse int
-    else if (option.type.name == "int" || option.type.name == "positiveInt")
+    else if
+      (option.type.name == "int" || option.type.name == "positiveInt")
     # return jsonschema property definition for int
-    then default // description // {
-      type = "integer";
-    }
+    then
+      default // description // { type = "integer"; }
 
     # parse string
-    else if option.type.name == "str"
+    else if
+      option.type.name == "str"
     # return jsonschema property definition for string
-    then default // description // {
-      type = "string";
-    }
+    then
+      default // description // { type = "string"; }
 
     # parse string
-    else if option.type.name == "path"
+    else if
+      option.type.name == "path"
     # return jsonschema property definition for path
-    then default // description // {
-      type = "string";
-    }
+    then
+      default // description // { type = "string"; }
 
     # parse anything
-    else if option.type.name == "anything"
+    else if
+      option.type.name == "anything"
     # return jsonschema property definition for anything
-    then default // description // {
-      type = allBasicTypes;
-    }
+    then
+      default // description // { type = allBasicTypes; }
 
     # parse unspecified
-    else if option.type.name == "unspecified"
+    else if
+      option.type.name == "unspecified"
     # return jsonschema property definition for unspecified
-    then default // description // {
-      type = allBasicTypes;
-    }
+    then
+      default // description // { type = allBasicTypes; }
 
     # parse raw
-    else if option.type.name == "raw"
+    else if
+      option.type.name == "raw"
     # return jsonschema property definition for raw
-    then default // description // {
-      type = allBasicTypes;
-    }
+    then
+      default // description // { type = allBasicTypes; }
 
     # parse enum
-    else if option.type.name == "enum"
+    else if
+      option.type.name == "enum"
     # return jsonschema property definition for enum
-    then default // description // {
-      enum = option.type.functor.payload;
-    }
+    then
+      default // description // { enum = option.type.functor.payload; }
 
     # parse listOf submodule
-    else if option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule"
+    else if
+      option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule"
     # return jsonschema property definition for listOf submodule
-    then default // description // {
+    then
+      default
+      // description
+      // {
         type = "array";
         items = parseOptions (option.type.functor.wrapped.getSubOptions option.loc);
       }
 
     # parse list
-    else if (option.type.name == "listOf")
+    else if
+      (option.type.name == "listOf")
     # return jsonschema property definition for list
     then
       let
-        nestedOption = { type = option.type.functor.wrapped; _type = "option"; loc = option.loc; };
+        nestedOption = {
+          type = option.type.functor.wrapped;
+          _type = "option";
+          loc = option.loc;
+        };
       in
-      default // description // {
+      default
+      // description
+      // {
         type = "array";
       }
-      // (lib.optionalAttrs (! isExcludedOption nestedOption) {
-        items = parseOption nestedOption;
-      })
+      // (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; })
 
     # parse list of unspecified
     else if
-      (option.type.name == "listOf")
-      && (option.type.functor.wrapped.name == "unspecified")
+      (option.type.name == "listOf") && (option.type.functor.wrapped.name == "unspecified")
     # return jsonschema property definition for list
-    then default // description // {
-      type = "array";
-    }
+    then
+      default // description // { type = "array"; }
 
     # parse attrsOf submodule
-    else if option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule"
+    else if
+      option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule"
     # return jsonschema property definition for attrsOf submodule
-    then default // description // {
+    then
+      default
+      // description
+      // {
         type = "object";
         additionalProperties = parseOptions (option.type.nestedTypes.elemType.getSubOptions option.loc);
       }
 
     # parse attrs
-    else if option.type.name == "attrs"
+    else if
+      option.type.name == "attrs"
     # return jsonschema property definition for attrs
-    then default // description // {
+    then
+      default
+      // description
+      // {
         type = "object";
         additionalProperties = true;
       }
 
     # parse attrsOf
     # TODO: if nested option is excluded, the parent sould be excluded too
-    else if option.type.name == "attrsOf" || option.type.name == "lazyAttrsOf"
+    else if
+      option.type.name == "attrsOf" || option.type.name == "lazyAttrsOf"
     # return jsonschema property definition for attrs
     then
       let
-        nestedOption = { type = option.type.nestedTypes.elemType; _type = "option"; loc = option.loc; };
+        nestedOption = {
+          type = option.type.nestedTypes.elemType;
+          _type = "option";
+          loc = option.loc;
+        };
       in
-      default // description // {
+      default
+      // description
+      // {
         type = "object";
         additionalProperties =
-          if ! isExcludedOption nestedOption
-          then parseOption { type = option.type.nestedTypes.elemType; _type = "option"; loc = option.loc; }
-          else false;
+          if !isExcludedOption nestedOption then
+            parseOption {
+              type = option.type.nestedTypes.elemType;
+              _type = "option";
+              loc = option.loc;
+            }
+          else
+            false;
       }
 
     # parse submodule
-    else if option.type.name == "submodule"
+    else if
+      option.type.name == "submodule"
     # return jsonschema property definition for submodule
     # then (lib.attrNames (option.type.getSubOptions option.loc).opt)
-    then parseOptions (option.type.getSubOptions option.loc)
+    then
+      parseOptions (option.type.getSubOptions option.loc)
 
     # throw error if option type is not supported
-    else notSupported option;
+    else
+      notSupported option;
 }

lib/jsonschema/example-interface.nix

@@ -1,7 +1,6 @@
-/*
-  An example nixos module declaring an interface.
-*/
-{ lib, ... }: {
+# An example nixos module declaring an interface.
+{ lib, ... }:
+{
   options = {
     # str
     name = lib.mkOption {
@@ -44,7 +43,11 @@
     # list of str
     kernelModules = lib.mkOption {
       type = lib.types.listOf lib.types.str;
-      default = [ "nvme" "xhci_pci" "ahci" ];
+      default = [
+        "nvme"
+        "xhci_pci"
+        "ahci"
+      ];
       description = "A list of enabled kernel modules";
     };
   };

lib/jsonschema/flake-module.nix

@@ -1,5 +1,7 @@
 {
-  perSystem = { pkgs, ... }: {
+  perSystem =
+    { pkgs, ... }:
+    {
       checks = {
 
         # check if the `clan config` example jsonschema and data is valid

lib/jsonschema/test.nix

@@ -1,6 +1,7 @@
 # run these tests via `nix-unit ./test.nix`
-{ lib ? (import <nixpkgs> { }).lib
-, slib ? import ./. { inherit lib; }
+{
+  lib ? (import <nixpkgs> { }).lib,
+  slib ? import ./. { inherit lib; },
 }:
 {
   parseOption = import ./test_parseOption.nix { inherit lib slib; };

lib/jsonschema/test_parseOption.nix

@@ -1,21 +1,25 @@
 # tests for the nixos options to jsonschema converter
 # run these tests via `nix-unit ./test.nix`
-{ lib ? (import <nixpkgs> { }).lib
-, slib ? import ./. { inherit lib; }
+{
+  lib ? (import <nixpkgs> { }).lib,
+  slib ? import ./. { inherit lib; },
 }:
 let
   description = "Test Description";
 
-  evalType = type: default:
+  evalType =
+    type: default:
     let
       evaledConfig = lib.evalModules {
-        modules = [{
+        modules = [
+          {
             options.opt = lib.mkOption {
               inherit type;
               inherit default;
               inherit description;
             };
-        }];
+          }
+        ];
       };
     in
     evaledConfig.options.opt;
@@ -25,11 +29,7 @@ in
   testNoDefaultNoDescription =
     let
       evaledConfig = lib.evalModules {
-        modules = [{
-          options.opt = lib.mkOption {
-            type = lib.types.bool;
-          };
-        }];
+        modules = [ { options.opt = lib.mkOption { type = lib.types.bool; }; } ];
       };
     in
     {
@@ -42,7 +42,8 @@ in
   testDescriptionIsAttrs =
     let
       evaledConfig = lib.evalModules {
-        modules = [{
+        modules = [
+          {
             options.opt = lib.mkOption {
               type = lib.types.bool;
               description = {
@@ -50,7 +51,8 @@ in
                 text = description;
               };
             };
-        }];
+          }
+        ];
       };
     in
     {
@@ -112,7 +114,11 @@ in
   testEnum =
     let
       default = "foo";
-      values = [ "foo" "bar" "baz" ];
+      values = [
+        "foo"
+        "bar"
+        "baz"
+      ];
     in
     {
       expr = slib.parseOption (evalType (lib.types.enum values) default);
@@ -124,7 +130,11 @@ in
 
   testListOfInt =
     let
-      default = [ 1 2 3 ];
+      default = [
+        1
+        2
+        3
+      ];
     in
     {
       expr = slib.parseOption (evalType (lib.types.listOf lib.types.int) default);
@@ -139,14 +149,26 @@ in
 
   testListOfUnspecified =
     let
-      default = [ 1 2 3 ];
+      default = [
+        1
+        2
+        3
+      ];
     in
     {
       expr = slib.parseOption (evalType (lib.types.listOf lib.types.unspecified) default);
       expected = {
         type = "array";
         items = {
-          type = [ "boolean" "integer" "number" "string" "array" "object" "null" ];
+          type = [
+            "boolean"
+            "integer"
+            "number"
+            "string"
+            "array"
+            "object"
+            "null"
+          ];
         };
         inherit default description;
       };
@@ -154,7 +176,11 @@ in
 
   testAttrs =
     let
-      default = { foo = 1; bar = 2; baz = 3; };
+      default = {
+        foo = 1;
+        bar = 2;
+        baz = 3;
+      };
     in
     {
       expr = slib.parseOption (evalType (lib.types.attrs) default);
@@ -167,7 +193,11 @@ in
 
   testAttrsOfInt =
     let
-      default = { foo = 1; bar = 2; baz = 3; };
+      default = {
+        foo = 1;
+        bar = 2;
+        baz = 3;
+      };
     in
     {
       expr = slib.parseOption (evalType (lib.types.attrsOf lib.types.int) default);
@@ -182,7 +212,11 @@ in
 
   testLazyAttrsOfInt =
     let
-      default = { foo = 1; bar = 2; baz = 3; };
+      default = {
+        foo = 1;
+        bar = 2;
+        baz = 3;
+      };
     in
     {
       expr = slib.parseOption (evalType (lib.types.lazyAttrsOf lib.types.int) default);
@@ -286,7 +320,10 @@ in
           inherit description;
         };
       };
-      default = { foo.opt = false; bar.opt = true; };
+      default = {
+        foo.opt = false;
+        bar.opt = true;
+      };
     in
     {
       expr = slib.parseOption (evalType (lib.types.attrsOf (lib.types.submodule subModule)) default);
@@ -315,7 +352,10 @@ in
           inherit description;
         };
       };
-      default = [{ opt = false; } { opt = true; }];
+      default = [
+        { opt = false; }
+        { opt = true; }
+      ];
     in
     {
       expr = slib.parseOption (evalType (lib.types.listOf (lib.types.submodule subModule)) default);
@@ -358,7 +398,15 @@ in
       expr = slib.parseOption (evalType lib.types.anything default);
       expected = {
         inherit default description;
-        type = [ "boolean" "integer" "number" "string" "array" "object" "null" ];
+        type = [
+          "boolean"
+          "integer"
+          "number"
+          "string"
+          "array"
+          "object"
+          "null"
+        ];
       };
     };
 
@@ -370,7 +418,15 @@ in
       expr = slib.parseOption (evalType lib.types.unspecified default);
       expected = {
         inherit default description;
-        type = [ "boolean" "integer" "number" "string" "array" "object" "null" ];
+        type = [
+          "boolean"
+          "integer"
+          "number"
+          "string"
+          "array"
+          "object"
+          "null"
+        ];
       };
     };
 
@@ -382,7 +438,15 @@ in
       expr = slib.parseOption (evalType lib.types.raw default);
       expected = {
         inherit default description;
-        type = [ "boolean" "integer" "number" "string" "array" "object" "null" ];
+        type = [
+          "boolean"
+          "integer"
+          "number"
+          "string"
+          "array"
+          "object"
+          "null"
+        ];
       };
     };
 }

lib/jsonschema/test_parseOptions.nix

@@ -1,14 +1,13 @@
 # tests for the nixos options to jsonschema converter
 # run these tests via `nix-unit ./test.nix`
-{ lib ? (import <nixpkgs> { }).lib
-, slib ? import ./. { inherit lib; }
+{
+  lib ? (import <nixpkgs> { }).lib,
+  slib ? import ./. { inherit lib; },
 }:
 let
   evaledOptions =
     let
-      evaledConfig = lib.evalModules {
-        modules = [ ./example-interface.nix ];
-      };
+      evaledConfig = lib.evalModules { modules = [ ./example-interface.nix ]; };
     in
     evaledConfig.options;
 in
@@ -21,11 +20,7 @@ in
   testParseNestedOptions =
     let
       evaled = lib.evalModules {
-        modules = [{
-          options.foo.bar = lib.mkOption {
-            type = lib.types.bool;
-          };
-        }];
+        modules = [ { options.foo.bar = lib.mkOption { type = lib.types.bool; }; } ];
       };
     in
     {
@@ -34,7 +29,9 @@ in
         properties = {
           foo = {
             properties = {
-              bar = { type = "boolean"; };
+              bar = {
+                type = "boolean";
+              };
             };
             required = [ "bar" ];
             type = "object";

machines/test-backup/facts/borgbackup.ssh.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBIbwIVnLy+uoDZ6uK/OCc1QK46SIGeC3mVc85dqLYQw lass@ignavia

machines/test-backup/facts/ssh.id_ed25519.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBIbwIVnLy+uoDZ6uK/OCc1QK46SIGeC3mVc85dqLYQw lass@ignavia

nixosModules/clanCore/backups.nix

@@ -1,11 +1,12 @@
 { lib, ... }:
 {
-  imports = [
-    ./state.nix
-  ];
+  imports = [ ./state.nix ];
   options.clanCore.backups = {
     providers = lib.mkOption {
-      type = lib.types.attrsOf (lib.types.submodule ({ name, ... }: {
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
             options = {
               name = lib.mkOption {
                 type = lib.types.str;
@@ -17,29 +18,27 @@
               list = lib.mkOption {
                 type = lib.types.str;
                 description = ''
-              script to list backups
+                  Command to list backups.
                 '';
               };
               restore = lib.mkOption {
                 type = lib.types.str;
                 description = ''
-              script to restore a backup
-              should take an optional service name as argument
-              gets ARCHIVE_ID, LOCATION, JOB and FOLDERS as environment variables
-              ARCHIVE_ID is the id of the backup
-              LOCATION is the remote identifier of the backup
-              JOB is the job name of the backup
-              FOLDERS is a colon separated list of folders to restore
+                  Command to restore a backup.
+                  The name of the backup and the folders to restore will be
+                  set as environment variables NAME and FOLDERS respectively.
                 '';
               };
               create = lib.mkOption {
                 type = lib.types.str;
                 description = ''
-              script to start a backup
+                  Command to start a backup
                 '';
               };
             };
-      }));
+          }
+        )
+      );
       default = { };
       description = ''
         Configured backup providers which are used by this machine

nixosModules/clanCore/default.nix

@@ -1,9 +1,9 @@
 {
   imports = [
     ./backups.nix
+    ./facts
     ./manual.nix
     ./imports.nix
-    ./meshnamed
     ./metadata.nix
     ./networking.nix
     ./nix-settings.nix
@@ -11,7 +11,6 @@
     ./outputs.nix
     ./packages.nix
     ./schema.nix
-    ./secrets
     ./vm.nix
     ./wayland-proxy-virtwl.nix
     ./zerotier

nixosModules/clanCore/facts/compat.nix

@@ -0,0 +1,167 @@
+{ config, lib, ... }:
+{
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "clanCore"
+      "secretsPrefix"
+    ] "secretsPrefix was only used by the sops module and the code is now integrated in there")
+    (lib.mkRenamedOptionModule
+      [
+        "clanCore"
+        "secretStore"
+      ]
+      [
+        "clanCore"
+        "facts"
+        "secretStore"
+      ]
+    )
+    (lib.mkRemovedOptionModule [
+      "clanCore"
+      "secretsDirectory"
+    ] "clancore.secretsDirectory was removed. Use clanCore.facts.secretPathFunction instead")
+    (lib.mkRenamedOptionModule
+      [
+        "clanCore"
+        "secretsUploadDirectory"
+      ]
+      [
+        "clanCore"
+        "facts"
+        "secretUploadDirectory"
+      ]
+    )
+  ];
+  options.clanCore.secrets = lib.mkOption {
+    visible = false;
+    default = { };
+    type = lib.types.attrsOf (
+      lib.types.submodule (service: {
+        options = {
+          name = lib.mkOption {
+            type = lib.types.str;
+            default = service.config._module.args.name;
+            description = ''
+              Namespace of the service
+            '';
+          };
+          generator = lib.mkOption {
+            type = lib.types.submodule (
+              { ... }:
+              {
+                options = {
+                  path = lib.mkOption {
+                    type = lib.types.listOf (lib.types.either lib.types.path lib.types.package);
+                    default = [ ];
+                    description = ''
+                      Extra paths to add to the PATH environment variable when running the generator.
+                    '';
+                  };
+                  prompt = lib.mkOption {
+                    type = lib.types.nullOr lib.types.str;
+                    default = null;
+                    description = ''
+                      prompt text to ask for a value.
+                      This value will be passed to the script as the environment variable $prompt_value.
+                    '';
+                  };
+                  script = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      Script to generate the secret.
+                      The script will be called with the following variables:
+                      - facts: path to a directory where facts can be stored
+                      - secrets: path to a directory where secrets can be stored
+                      The script is expected to generate all secrets and facts defined in the module.
+                    '';
+                  };
+                };
+              }
+            );
+          };
+          secrets = lib.mkOption {
+            default = { };
+            type = lib.types.attrsOf (
+              lib.types.submodule (secret: {
+                options =
+                  {
+                    name = lib.mkOption {
+                      type = lib.types.str;
+                      description = ''
+                        name of the secret
+                      '';
+                      default = secret.config._module.args.name;
+                    };
+                    path = lib.mkOption {
+                      type = lib.types.path;
+                      description = ''
+                        path to a secret which is generated by the generator
+                      '';
+                      default = config.clanCore.facts.secretPathFunction secret;
+                      defaultText = lib.literalExpression "config.clanCore.facts.secretPathFunction secret";
+                    };
+                  }
+                  // lib.optionalAttrs (config.clanCore.facts.secretStore == "sops") {
+                    groups = lib.mkOption {
+                      type = lib.types.listOf lib.types.str;
+                      default = config.clanCore.sops.defaultGroups;
+                      description = ''
+                        Groups to decrypt the secret for. By default we always use the user's key.
+                      '';
+                    };
+                  };
+              })
+            );
+            description = ''
+              path where the secret is located in the filesystem
+            '';
+          };
+          facts = lib.mkOption {
+            default = { };
+            type = lib.types.attrsOf (
+              lib.types.submodule (fact: {
+                options = {
+                  name = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      name of the fact
+                    '';
+                    default = fact.config._module.args.name;
+                  };
+                  path = lib.mkOption {
+                    type = lib.types.path;
+                    description = ''
+                      path to a fact which is generated by the generator
+                    '';
+                    default =
+                      config.clanCore.clanDir
+                      + "/machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}";
+                    defaultText = lib.literalExpression "\${config.clanCore.clanDir}/machines/\${config.clanCore.machineName}/facts/\${fact.config._module.args.name}";
+                  };
+                  value = lib.mkOption {
+                    defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
+                    type = lib.types.nullOr lib.types.str;
+                    default =
+                      if builtins.pathExists fact.config.path then lib.strings.fileContents fact.config.path else null;
+                  };
+                };
+              })
+            );
+          };
+        };
+      })
+    );
+  };
+  config = lib.mkIf (config.clanCore.secrets != { }) {
+    clanCore.facts.services = lib.mapAttrs' (
+      name: service:
+      lib.warn "clanCore.secrets.${name} is deprecated, use clanCore.facts.services.${name} instead" (
+        lib.nameValuePair name ({
+          secret = service.secrets;
+          public = service.facts;
+          generator = service.generator;
+        })
+      )
+    ) config.clanCore.secrets;
+  };
+}

nixosModules/clanCore/facts/default.nix

@@ -0,0 +1,222 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  options.clanCore.facts = {
+    secretStore = lib.mkOption {
+      type = lib.types.enum [
+        "sops"
+        "password-store"
+        "vm"
+        "custom"
+      ];
+      default = "sops";
+      description = ''
+        method to store secret facts
+        custom can be used to define a custom secret fact store.
+      '';
+    };
+
+    secretModule = lib.mkOption {
+      type = lib.types.str;
+      internal = true;
+      description = ''
+        the python import path to the secret module
+      '';
+    };
+
+    secretUploadDirectory = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        The directory where secrets are uploaded into, This is backend specific.
+      '';
+    };
+
+    secretPathFunction = lib.mkOption {
+      type = lib.types.raw;
+      description = ''
+        The function to use to generate the path for a secret.
+        The default function will use the path attribute of the secret.
+        The function will be called with the secret submodule as an argument.
+      '';
+    };
+
+    publicStore = lib.mkOption {
+      type = lib.types.enum [
+        "in_repo"
+        "vm"
+        "custom"
+      ];
+      default = "in_repo";
+      description = ''
+        method to store public facts.
+        custom can be used to define a custom public fact store.
+      '';
+    };
+    publicModule = lib.mkOption {
+      type = lib.types.str;
+      internal = true;
+      description = ''
+        the python import path to the public module
+      '';
+    };
+    publicDirectory = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+    };
+
+    services = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (service: {
+          options = {
+            name = lib.mkOption {
+              type = lib.types.str;
+              default = service.config._module.args.name;
+              description = ''
+                Namespace of the service
+              '';
+            };
+            generator = lib.mkOption {
+              type = lib.types.submodule (
+                { config, ... }:
+                {
+                  options = {
+                    path = lib.mkOption {
+                      type = lib.types.listOf (lib.types.either lib.types.path lib.types.package);
+                      default = [ ];
+                      description = ''
+                        Extra paths to add to the PATH environment variable when running the generator.
+                      '';
+                    };
+                    prompt = lib.mkOption {
+                      type = lib.types.nullOr lib.types.str;
+                      default = null;
+                      description = ''
+                        prompt text to ask for a value.
+                        This value will be passed to the script as the environment variable $prompt_value.
+                      '';
+                    };
+                    script = lib.mkOption {
+                      type = lib.types.str;
+                      description = ''
+                        Shell script snippet to generate the secrets and facts.
+                        The script has access to the following environment variables:
+                        - facts: path to a directory where facts can be stored
+                        - secrets: path to a directory where secrets can be stored
+                        The script is expected to generate all secrets and facts defined for this service.
+                      '';
+                    };
+                    finalScript = lib.mkOption {
+                      type = lib.types.str;
+                      readOnly = true;
+                      internal = true;
+                      default = ''
+                        set -eu -o pipefail
+
+                        export PATH="${lib.makeBinPath config.path}:${pkgs.coreutils}/bin"
+
+                        # prepare sandbox user
+                        mkdir -p /etc
+                        cp ${
+                          pkgs.runCommand "fake-etc" { } ''
+                            export PATH="${pkgs.coreutils}/bin"
+                            mkdir -p $out
+                            cp /etc/* $out/
+                          ''
+                        }/* /etc/
+
+                        ${config.script}
+                      '';
+                    };
+                  };
+                }
+              );
+            };
+            secret = lib.mkOption {
+              default = { };
+              type = lib.types.attrsOf (
+                lib.types.submodule (secret: {
+                  options =
+                    {
+                      name = lib.mkOption {
+                        type = lib.types.str;
+                        description = ''
+                          name of the secret
+                        '';
+                        default = secret.config._module.args.name;
+                      };
+                      path = lib.mkOption {
+                        type = lib.types.str;
+                        description = ''
+                          path to a secret which is generated by the generator
+                        '';
+                        default = config.clanCore.facts.secretPathFunction secret;
+                      };
+                    }
+                    // lib.optionalAttrs (config.clanCore.facts.secretModule == "clan_cli.facts.secret_modules.sops") {
+                      groups = lib.mkOption {
+                        type = lib.types.listOf lib.types.str;
+                        default = config.clanCore.sops.defaultGroups;
+                        description = ''
+                          Groups to decrypt the secret for. By default we always use the user's key.
+                        '';
+                      };
+                    };
+                })
+              );
+              description = ''
+                path where the secret is located in the filesystem
+              '';
+            };
+            public = lib.mkOption {
+              default = { };
+              type = lib.types.attrsOf (
+                lib.types.submodule (fact: {
+                  options = {
+                    name = lib.mkOption {
+                      type = lib.types.str;
+                      description = ''
+                        name of the public fact
+                      '';
+                      default = fact.config._module.args.name;
+                    };
+                    path = lib.mkOption {
+                      type = lib.types.path;
+                      description = ''
+                        path to a fact which is generated by the generator
+                      '';
+                      defaultText = lib.literalExpression "\${config.clanCore.clanDir}/machines/\${config.clanCore.machineName}/facts/\${fact.config.name}";
+                      default =
+                        config.clanCore.clanDir + "/machines/${config.clanCore.machineName}/facts/${fact.config.name}";
+                    };
+                    value = lib.mkOption {
+                      defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
+                      type = lib.types.nullOr lib.types.str;
+                      default =
+                        if builtins.pathExists fact.config.path then lib.strings.fileContents fact.config.path else null;
+                    };
+                  };
+                })
+              );
+            };
+          };
+        })
+      );
+    };
+  };
+  imports = [
+    ./compat.nix
+
+    ./secret/sops.nix
+    ./secret/password-store.nix
+    ./secret/vm.nix
+
+    ./public/in_repo.nix
+    ./public/vm.nix
+  ];
+}