diff --git a/.gitea/workflows/checks.yaml b/.gitea/workflows/checks.yaml index 7804a1659..1ad904868 100644 --- a/.gitea/workflows/checks.yaml +++ b/.gitea/workflows/checks.yaml @@ -9,7 +9,7 @@ jobs: runs-on: nix steps: - uses: actions/checkout@v3 - - run: nix run --refresh github:Mic92/nix-fast-build -- --no-nom --eval-workers 20 + - run: nix run --refresh github:Mic92/nix-fast-build -- --no-nom --eval-workers 10 check-links: runs-on: nix steps: diff --git a/.gitignore b/.gitignore index 45b863ff6..fed75a618 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ example_clan result* /pkgs/clan-cli/clan_cli/nixpkgs /pkgs/clan-cli/clan_cli/webui/assets -/machines nixos.qcow2 **/*.glade~ diff --git a/checks/backups/flake-module.nix b/checks/backups/flake-module.nix index 4ebecbfdd..8ff597aff 100644 --- a/checks/backups/flake-module.nix +++ b/checks/backups/flake-module.nix @@ -14,21 +14,27 @@ let }; in { - flake.nixosConfigurations = { inherit (clan.nixosConfigurations) test_backup_client; }; + flake.nixosConfigurations = { + inherit (clan.nixosConfigurations) test_backup_client; + }; flake.clanInternals = clan.clanInternals; flake.nixosModules = { - test_backup_server = { ... }: { - imports = [ - self.clanModules.borgbackup - ]; - services.sshd.enable = true; - services.borgbackup.repos.testrepo = { - authorizedKeys = [ - (builtins.readFile ../lib/ssh/pubkey) - ]; + test_backup_server = + { ... }: + { + imports = [ self.clanModules.borgbackup ]; + services.sshd.enable = true; + services.borgbackup.repos.testrepo = { + authorizedKeys = [ (builtins.readFile ../lib/ssh/pubkey) ]; + }; }; - }; - test_backup_client = { pkgs, lib, config, ... }: + test_backup_client = + { + pkgs, + lib, + config, + ... + }: let dependencies = [ self @@ -38,14 +44,10 @@ in closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; in { - imports = [ - self.clanModules.borgbackup - ]; + imports = [ self.clanModules.borgbackup ]; networking.hostName = "client"; services.sshd.enable = true; - users.users.root.openssh.authorizedKeys.keyFiles = [ - ../lib/ssh/pubkey - ]; + users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ]; systemd.tmpfiles.settings."vmsecrets" = { "/etc/secrets/borgbackup.ssh" = { @@ -75,71 +77,67 @@ in }; system.extraDependencies = dependencies; clanCore.state.test-backups.folders = [ "/var/test-backups" ]; - clan.borgbackup = { - enable = true; - destinations.test_backup_server.repo = "borg@server:."; - }; + clan.borgbackup.destinations.test_backup_server.repo = "borg@server:."; }; }; - perSystem = { nodes, pkgs, ... }: { - checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) { - test-backups = - (import ../lib/test-base.nix) - { - name = "test-backups"; - nodes.server = { - imports = [ - self.nixosModules.test_backup_server - self.nixosModules.clanCore - { - clanCore.machineName = "server"; - clanCore.clanDir = ../..; - } - ]; - }; - nodes.client = { - imports = [ - self.nixosModules.test_backup_client - self.nixosModules.clanCore - { - clanCore.machineName = "client"; - clanCore.clanDir = ../..; - } - ]; - }; + perSystem = + { nodes, pkgs, ... }: + { + checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) { + test-backups = (import ../lib/test-base.nix) { + name = "test-backups"; + nodes.server = { + imports = [ + self.nixosModules.test_backup_server + self.nixosModules.clanCore + { + clanCore.machineName = "server"; + clanCore.clanDir = ../..; + } + ]; + }; + nodes.client = { + imports = [ + self.nixosModules.test_backup_client + self.nixosModules.clanCore + { + clanCore.machineName = "client"; + clanCore.clanDir = ../..; + } + ]; + }; - testScript = '' - import json - start_all() + testScript = '' + import json + start_all() - # setup - client.succeed("mkdir -m 700 /root/.ssh") - client.succeed( - "cat ${../lib/ssh/privkey} > /root/.ssh/id_ed25519" - ) - client.succeed("chmod 600 /root/.ssh/id_ed25519") - client.wait_for_unit("sshd", timeout=30) - client.succeed("ssh -o StrictHostKeyChecking=accept-new root@client hostname") + # setup + client.succeed("mkdir -m 700 /root/.ssh") + client.succeed( + "cat ${../lib/ssh/privkey} > /root/.ssh/id_ed25519" + ) + client.succeed("chmod 600 /root/.ssh/id_ed25519") + client.wait_for_unit("sshd", timeout=30) + client.succeed("ssh -o StrictHostKeyChecking=accept-new root@client hostname") - # dummy data - client.succeed("mkdir /var/test-backups") - client.succeed("echo testing > /var/test-backups/somefile") + # dummy data + client.succeed("mkdir /var/test-backups") + client.succeed("echo testing > /var/test-backups/somefile") - # create - client.succeed("clan --debug --flake ${../..} backups create test_backup_client") - client.wait_until_succeeds("! systemctl is-active borgbackup-job-test_backup_server") + # create + client.succeed("clan --debug --flake ${../..} backups create test_backup_client") + client.wait_until_succeeds("! systemctl is-active borgbackup-job-test_backup_server") - # list - backup_id = json.loads(client.succeed("borg-job-test_backup_server list --json"))["archives"][0]["archive"] - assert(backup_id in client.succeed("clan --debug --flake ${../..} backups list test_backup_client")) + # list + backup_id = json.loads(client.succeed("borg-job-test_backup_server list --json"))["archives"][0]["archive"] + assert(backup_id in client.succeed("clan --debug --flake ${../..} backups list test_backup_client")) - # restore - client.succeed("rm -f /var/test-backups/somefile") - client.succeed(f"clan --debug --flake ${../..} backups restore test_backup_client borgbackup {backup_id}") - assert(client.succeed("cat /var/test-backups/somefile").strip() == "testing") - ''; - } - { inherit pkgs self; }; + # restore + client.succeed("rm -f /var/test-backups/somefile") + client.succeed(f"clan --debug --flake ${../..} backups restore test_backup_client borgbackup {backup_id}") + assert(client.succeed("cat /var/test-backups/somefile").strip() == "testing") + ''; + } { inherit pkgs self; }; + }; }; - }; } diff --git a/checks/borgbackup/default.nix b/checks/borgbackup/default.nix index 5c466cc17..ba0008adc 100644 --- a/checks/borgbackup/default.nix +++ b/checks/borgbackup/default.nix @@ -1,51 +1,51 @@ -(import ../lib/test-base.nix) ({ ... }: { - name = "borgbackup"; +(import ../lib/test-base.nix) ( + { ... }: + { + name = "borgbackup"; - nodes.machine = { self, pkgs, ... }: { - imports = [ - self.clanModules.borgbackup - self.nixosModules.clanCore + nodes.machine = + { self, pkgs, ... }: { - services.openssh.enable = true; - services.borgbackup.repos.testrepo = { - authorizedKeys = [ - (builtins.readFile ../lib/ssh/pubkey) - ]; - }; - } - { - clanCore.machineName = "machine"; - clanCore.clanDir = ./.; - clanCore.state.testState.folders = [ "/etc/state" ]; - environment.etc.state.text = "hello world"; - systemd.tmpfiles.settings."vmsecrets" = { - "/etc/secrets/borgbackup.ssh" = { - C.argument = "${../lib/ssh/privkey}"; - z = { - mode = "0400"; - user = "root"; + imports = [ + self.clanModules.borgbackup + self.nixosModules.clanCore + { + services.openssh.enable = true; + services.borgbackup.repos.testrepo = { + authorizedKeys = [ (builtins.readFile ../lib/ssh/pubkey) ]; }; - }; - "/etc/secrets/borgbackup.repokey" = { - C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345"); - z = { - mode = "0400"; - user = "root"; + } + { + clanCore.machineName = "machine"; + clanCore.clanDir = ./.; + clanCore.state.testState.folders = [ "/etc/state" ]; + environment.etc.state.text = "hello world"; + systemd.tmpfiles.settings."vmsecrets" = { + "/etc/secrets/borgbackup.ssh" = { + C.argument = "${../lib/ssh/privkey}"; + z = { + mode = "0400"; + user = "root"; + }; + }; + "/etc/secrets/borgbackup.repokey" = { + C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345"); + z = { + mode = "0400"; + user = "root"; + }; + }; }; - }; - }; - clanCore.secretStore = "vm"; + clanCore.secretStore = "vm"; - clan.borgbackup = { - enable = true; - destinations.test.repo = "borg@localhost:."; - }; - } - ]; - }; - testScript = '' - start_all() - machine.systemctl("start --wait borgbackup-job-test.service") - assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list") - ''; -}) + clan.borgbackup.destinations.test.repo = "borg@localhost:."; + } + ]; + }; + testScript = '' + start_all() + machine.systemctl("start --wait borgbackup-job-test.service") + assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list") + ''; + } +) diff --git a/checks/container/default.nix b/checks/container/default.nix index 37128b221..61c75a0e9 100644 --- a/checks/container/default.nix +++ b/checks/container/default.nix @@ -1,14 +1,19 @@ -(import ../lib/container-test.nix) ({ ... }: { - name = "secrets"; +(import ../lib/container-test.nix) ( + { ... }: + { + name = "secrets"; - nodes.machine = { ... }: { - networking.hostName = "machine"; - services.openssh.enable = true; - services.openssh.startWhenNeeded = false; - }; - testScript = '' - start_all() - machine.succeed("systemctl status sshd") - machine.wait_for_unit("sshd") - ''; -}) + nodes.machine = + { ... }: + { + networking.hostName = "machine"; + services.openssh.enable = true; + services.openssh.startWhenNeeded = false; + }; + testScript = '' + start_all() + machine.succeed("systemctl status sshd") + machine.wait_for_unit("sshd") + ''; + } +) diff --git a/checks/deltachat/default.nix b/checks/deltachat/default.nix index 0fcf20b13..aa6e43ffd 100644 --- a/checks/deltachat/default.nix +++ b/checks/deltachat/default.nix @@ -1,24 +1,29 @@ -(import ../lib/container-test.nix) ({ pkgs, ... }: { - name = "secrets"; +(import ../lib/container-test.nix) ( + { pkgs, ... }: + { + name = "secrets"; - nodes.machine = { self, ... }: { - imports = [ - self.clanModules.deltachat - self.nixosModules.clanCore + nodes.machine = + { self, ... }: { - clanCore.machineName = "machine"; - clanCore.clanDir = ./.; - } - ]; - }; - testScript = '' - start_all() - machine.wait_for_unit("maddy") - # imap - machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 143") - # smtp submission - machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 587") - # smtp - machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 25") - ''; -}) + imports = [ + self.clanModules.deltachat + self.nixosModules.clanCore + { + clanCore.machineName = "machine"; + clanCore.clanDir = ./.; + } + ]; + }; + testScript = '' + start_all() + machine.wait_for_unit("maddy") + # imap + machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 143") + # smtp submission + machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 587") + # smtp + machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 25") + ''; + } +) diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 22a5e66d6..b792a1f9a 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -1,40 +1,20 @@ -{ self, ... }: { +{ self, ... }: +{ imports = [ ./impure/flake-module.nix ./backups/flake-module.nix ./installation/flake-module.nix + ./flash/flake-module.nix ]; - perSystem = { pkgs, lib, self', ... }: { - checks = - let - nixosTestArgs = { - # reference to nixpkgs for the current system - inherit pkgs; - # this gives us a reference to our flake but also all flake inputs - inherit self; - }; - nixosTests = lib.optionalAttrs (pkgs.stdenv.isLinux) { - # import our test - secrets = import ./secrets nixosTestArgs; - container = import ./container nixosTestArgs; - deltachat = import ./deltachat nixosTestArgs; - zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs; - borgbackup = import ./borgbackup nixosTestArgs; - syncthing = import ./syncthing nixosTestArgs; - wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs; - }; - schemaTests = pkgs.callPackages ./schemas.nix { - inherit self; - }; - - flakeOutputs = lib.mapAttrs' (name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel) self.nixosConfigurations - // lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages - // lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells - // lib.mapAttrs' (name: config: lib.nameValuePair "home-manager-${name}" config.activation-script) (self'.legacyPackages.homeConfigurations or { }); - in - nixosTests // schemaTests // flakeOutputs; - legacyPackages = { - nixosTests = + perSystem = + { + pkgs, + lib, + self', + ... + }: + { + checks = let nixosTestArgs = { # reference to nixpkgs for the current system @@ -42,12 +22,44 @@ # this gives us a reference to our flake but also all flake inputs inherit self; }; + nixosTests = lib.optionalAttrs (pkgs.stdenv.isLinux) { + # import our test + secrets = import ./secrets nixosTestArgs; + container = import ./container nixosTestArgs; + deltachat = import ./deltachat nixosTestArgs; + zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs; + borgbackup = import ./borgbackup nixosTestArgs; + syncthing = import ./syncthing nixosTestArgs; + wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs; + }; + schemaTests = pkgs.callPackages ./schemas.nix { inherit self; }; + + flakeOutputs = + lib.mapAttrs' ( + name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel + ) self.nixosConfigurations + // lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages + // lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells + // lib.mapAttrs' (name: config: lib.nameValuePair "home-manager-${name}" config.activation-script) ( + self'.legacyPackages.homeConfigurations or { } + ); in - lib.optionalAttrs (pkgs.stdenv.isLinux) { - # import our test - secrets = import ./secrets nixosTestArgs; - container = import ./container nixosTestArgs; - }; + nixosTests // schemaTests // flakeOutputs; + legacyPackages = { + nixosTests = + let + nixosTestArgs = { + # reference to nixpkgs for the current system + inherit pkgs; + # this gives us a reference to our flake but also all flake inputs + inherit self; + }; + in + lib.optionalAttrs (pkgs.stdenv.isLinux) { + # import our test + secrets = import ./secrets nixosTestArgs; + container = import ./container nixosTestArgs; + }; + }; }; - }; } diff --git a/checks/flash/flake-module.nix b/checks/flash/flake-module.nix new file mode 100644 index 000000000..f43c4357c --- /dev/null +++ b/checks/flash/flake-module.nix @@ -0,0 +1,49 @@ +{ self, ... }: +{ + perSystem = + { + nodes, + pkgs, + lib, + ... + }: + let + dependencies = [ + self + pkgs.stdenv.drvPath + self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.toplevel + self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.diskoScript + self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.clan.deployment.file + self.inputs.nixpkgs.legacyPackages.${pkgs.hostPlatform.system}.disko + ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); + closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; + in + { + checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) { + flash = (import ../lib/test-base.nix) { + name = "flash"; + nodes.target = { + virtualisation.emptyDiskImages = [ 4096 ]; + virtualisation.memorySize = 3000; + environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ]; + environment.etc."install-closure".source = "${closureInfo}/store-paths"; + + nix.settings = { + substituters = lib.mkForce [ ]; + hashed-mirrors = null; + connect-timeout = lib.mkForce 3; + flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}''; + experimental-features = [ + "nix-command" + "flakes" + ]; + }; + }; + testScript = '' + start_all() + machine.succeed("clan --flake ${../..} flash --debug --yes --disk main /dev/vdb test_install_machine") + ''; + } { inherit pkgs self; }; + }; + }; +} diff --git a/checks/impure/flake-module.nix b/checks/impure/flake-module.nix index faeca31e9..a918c6c24 100644 --- a/checks/impure/flake-module.nix +++ b/checks/impure/flake-module.nix @@ -1,18 +1,22 @@ { - perSystem = { pkgs, lib, ... }: { - # a script that executes all other checks - packages.impure-checks = pkgs.writeShellScriptBin "impure-checks" '' - #!${pkgs.bash}/bin/bash - set -euo pipefail + perSystem = + { pkgs, lib, ... }: + { + # a script that executes all other checks + packages.impure-checks = pkgs.writeShellScriptBin "impure-checks" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail - export PATH="${lib.makeBinPath [ - pkgs.gitMinimal - pkgs.nix - pkgs.rsync # needed to have rsync installed on the dummy ssh server - ]}" - ROOT=$(git rev-parse --show-toplevel) - cd "$ROOT/pkgs/clan-cli" - nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -s -m impure ./tests $@" - ''; - }; + export PATH="${ + lib.makeBinPath [ + pkgs.gitMinimal + pkgs.nix + pkgs.rsync # needed to have rsync installed on the dummy ssh server + ] + }" + ROOT=$(git rev-parse --show-toplevel) + cd "$ROOT/pkgs/clan-cli" + nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -s -m impure ./tests $@" + ''; + }; } diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index 0768ad4f4..522da5662 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -12,32 +12,34 @@ let }; in { - flake.nixosConfigurations = { inherit (clan.nixosConfigurations) test_install_machine; }; + flake.nixosConfigurations = { + inherit (clan.nixosConfigurations) test_install_machine; + }; flake.clanInternals = clan.clanInternals; flake.nixosModules = { - test_install_machine = { lib, modulesPath, ... }: { - imports = [ - self.clanModules.diskLayouts - (modulesPath + "/testing/test-instrumentation.nix") # we need these 2 modules always to be able to run the tests - (modulesPath + "/profiles/qemu-guest.nix") - ]; - fileSystems."/nix/store" = lib.mkForce { - device = "nix-store"; - fsType = "9p"; - neededForBoot = true; - options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ]; + test_install_machine = + { lib, modulesPath, ... }: + { + imports = [ + self.clanModules.diskLayouts + (modulesPath + "/testing/test-instrumentation.nix") # we need these 2 modules always to be able to run the tests + (modulesPath + "/profiles/qemu-guest.nix") + ]; + clan.diskLayouts.singleDiskExt4.device = "/dev/vdb"; + + environment.etc."install-successful".text = "ok"; + + boot.consoleLogLevel = lib.mkForce 100; + boot.kernelParams = [ "boot.shell_on_fail" ]; }; - clan.diskLayouts.singleDiskExt4.device = "/dev/vdb"; - - environment.etc."install-successful".text = "ok"; - - boot.consoleLogLevel = lib.mkForce 100; - boot.kernelParams = [ - "boot.shell_on_fail" - ]; - }; }; - perSystem = { nodes, pkgs, lib, ... }: + perSystem = + { + nodes, + pkgs, + lib, + ... + }: let dependencies = [ self @@ -51,74 +53,69 @@ in in { checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) { - test-installation = - (import ../lib/test-base.nix) - { - name = "test-installation"; - nodes.target = { - services.openssh.enable = true; - users.users.root.openssh.authorizedKeys.keyFiles = [ - ../lib/ssh/pubkey - ]; - system.nixos.variant_id = "installer"; - virtualisation.emptyDiskImages = [ 4096 ]; - nix.settings = { - substituters = lib.mkForce [ ]; - hashed-mirrors = null; - connect-timeout = lib.mkForce 3; - flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}''; - experimental-features = [ - "nix-command" - "flakes" - ]; - }; - }; - nodes.client = { - environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ]; - environment.etc."install-closure".source = "${closureInfo}/store-paths"; - virtualisation.memorySize = 2048; - nix.settings = { - substituters = lib.mkForce [ ]; - hashed-mirrors = null; - connect-timeout = lib.mkForce 3; - flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}''; - experimental-features = [ - "nix-command" - "flakes" - ]; - }; - system.extraDependencies = dependencies; - }; + test-installation = (import ../lib/test-base.nix) { + name = "test-installation"; + nodes.target = { + services.openssh.enable = true; + users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ]; + system.nixos.variant_id = "installer"; + virtualisation.emptyDiskImages = [ 4096 ]; + nix.settings = { + substituters = lib.mkForce [ ]; + hashed-mirrors = null; + connect-timeout = lib.mkForce 3; + flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}''; + experimental-features = [ + "nix-command" + "flakes" + ]; + }; + }; + nodes.client = { + environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ]; + environment.etc."install-closure".source = "${closureInfo}/store-paths"; + virtualisation.memorySize = 2048; + nix.settings = { + substituters = lib.mkForce [ ]; + hashed-mirrors = null; + connect-timeout = lib.mkForce 3; + flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}''; + experimental-features = [ + "nix-command" + "flakes" + ]; + }; + system.extraDependencies = dependencies; + }; - testScript = '' - def create_test_machine(oldmachine=None, args={}): # taken from - machine = create_machine({ - "qemuFlags": - '-cpu max -m 1024 -virtfs local,path=/nix/store,security_model=none,mount_tag=nix-store,' - f' -drive file={oldmachine.state_dir}/empty0.qcow2,id=drive1,if=none,index=1,werror=report' - f' -device virtio-blk-pci,drive=drive1', - } | args) - driver.machines.append(machine) - return machine + testScript = '' + def create_test_machine(oldmachine=None, args={}): # taken from + startCommand = "${pkgs.qemu_test}/bin/qemu-kvm" + startCommand += " -cpu max -m 1024 -virtfs local,path=/nix/store,security_model=none,mount_tag=nix-store" + startCommand += f' -drive file={oldmachine.state_dir}/empty0.qcow2,id=drive1,if=none,index=1,werror=report' + startCommand += ' -device virtio-blk-pci,drive=drive1' + machine = create_machine({ + "startCommand": startCommand, + } | args) + driver.machines.append(machine) + return machine + start_all() - start_all() + client.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../lib/ssh/privkey} /root/.ssh/id_ed25519") + client.wait_until_succeeds("ssh -o StrictHostKeyChecking=accept-new -v root@target hostname") - client.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../lib/ssh/privkey} /root/.ssh/id_ed25519") - client.wait_until_succeeds("ssh -o StrictHostKeyChecking=accept-new -v root@target hostname") + client.succeed("clan --debug --flake ${../..} machines install --yes test_install_machine root@target >&2") + try: + target.shutdown() + except BrokenPipeError: + # qemu has already exited + pass - client.succeed("clan --debug --flake ${../..} machines install test_install_machine root@target >&2") - try: - target.shutdown() - except BrokenPipeError: - # qemu has already exited - pass - - new_machine = create_test_machine(oldmachine=target, args={ "name": "new_machine" }) - assert(new_machine.succeed("cat /etc/install-successful").strip() == "ok") - ''; - } - { inherit pkgs self; }; + new_machine = create_test_machine(oldmachine=target, args={ "name": "new_machine" }) + assert(new_machine.succeed("cat /etc/install-successful").strip() == "ok") + ''; + } { inherit pkgs self; }; }; }; } diff --git a/checks/lib/container-driver/module.nix b/checks/lib/container-driver/module.nix index 10adac320..ca4e72e7c 100644 --- a/checks/lib/container-driver/module.nix +++ b/checks/lib/container-driver/module.nix @@ -1,17 +1,23 @@ -{ hostPkgs, lib, config, ... }: +{ + hostPkgs, + lib, + config, + ... +}: let testDriver = hostPkgs.python3.pkgs.callPackage ./package.nix { inherit (config) extraPythonPackages; inherit (hostPkgs.pkgs) util-linux systemd; }; containers = map (m: m.system.build.toplevel) (lib.attrValues config.nodes); - pythonizeName = name: + pythonizeName = + name: let head = lib.substring 0 1 name; tail = lib.substring 1 (-1) name; in - (if builtins.match "[A-z_]" head == null then "_" else head) + - lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail; + (if builtins.match "[A-z_]" head == null then "_" else head) + + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail; nodeHostNames = let nodesList = map (c: c.system.name) (lib.attrValues config.nodes); @@ -21,68 +27,72 @@ let pythonizedNames = map pythonizeName nodeHostNames; in { - driver = lib.mkForce (hostPkgs.runCommand "nixos-test-driver-${config.name}" - { - nativeBuildInputs = [ - hostPkgs.makeWrapper - ] ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ]; - buildInputs = [ testDriver ]; - testScript = config.testScriptString; - preferLocalBuild = true; - passthru = config.passthru; - meta = config.meta // { - mainProgram = "nixos-test-driver"; + driver = lib.mkForce ( + hostPkgs.runCommand "nixos-test-driver-${config.name}" + { + nativeBuildInputs = [ + hostPkgs.makeWrapper + ] ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ]; + buildInputs = [ testDriver ]; + testScript = config.testScriptString; + preferLocalBuild = true; + passthru = config.passthru; + meta = config.meta // { + mainProgram = "nixos-test-driver"; + }; + } + '' + mkdir -p $out/bin + + containers=(${toString containers}) + + ${lib.optionalString (!config.skipTypeCheck) '' + # prepend type hints so the test script can be type checked with mypy + cat "${./test-script-prepend.py}" >> testScriptWithTypes + echo "${builtins.toString machineNames}" >> testScriptWithTypes + echo -n "$testScript" >> testScriptWithTypes + + echo "Running type check (enable/disable: config.skipTypeCheck)" + echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipTypeCheck" + + mypy --no-implicit-optional \ + --pretty \ + --no-color-output \ + testScriptWithTypes + ''} + + echo -n "$testScript" >> $out/test-script + + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver + + wrapProgram $out/bin/nixos-test-driver \ + ${lib.concatStringsSep " " (map (name: "--add-flags '--container ${name}'") containers)} \ + --add-flags "--test-script '$out/test-script'" + '' + ); + + test = lib.mkForce ( + lib.lazyDerivation { + # lazyDerivation improves performance when only passthru items and/or meta are used. + derivation = hostPkgs.stdenv.mkDerivation { + name = "vm-test-run-${config.name}"; + + requiredSystemFeatures = [ "uid-range" ]; + + buildCommand = '' + mkdir -p $out + + # effectively mute the XMLLogger + export LOGFILE=/dev/null + + ${config.driver}/bin/nixos-test-driver -o $out + ''; + + passthru = config.passthru; + + meta = config.meta; }; + inherit (config) passthru meta; } - '' - mkdir -p $out/bin - - containers=(${toString containers}) - - ${lib.optionalString (!config.skipTypeCheck) '' - # prepend type hints so the test script can be type checked with mypy - cat "${./test-script-prepend.py}" >> testScriptWithTypes - echo "${builtins.toString machineNames}" >> testScriptWithTypes - echo -n "$testScript" >> testScriptWithTypes - - echo "Running type check (enable/disable: config.skipTypeCheck)" - echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipTypeCheck" - - mypy --no-implicit-optional \ - --pretty \ - --no-color-output \ - testScriptWithTypes - ''} - - echo -n "$testScript" >> $out/test-script - - ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver - - wrapProgram $out/bin/nixos-test-driver \ - ${lib.concatStringsSep " " (map (name: "--add-flags '--container ${name}'") containers)} \ - --add-flags "--test-script '$out/test-script'" - ''); - - test = lib.mkForce (lib.lazyDerivation { - # lazyDerivation improves performance when only passthru items and/or meta are used. - derivation = hostPkgs.stdenv.mkDerivation { - name = "vm-test-run-${config.name}"; - - requiredSystemFeatures = [ "uid-range" ]; - - buildCommand = '' - mkdir -p $out - - # effectively mute the XMLLogger - export LOGFILE=/dev/null - - ${config.driver}/bin/nixos-test-driver -o $out - ''; - - passthru = config.passthru; - - meta = config.meta; - }; - inherit (config) passthru meta; - }); + ); } diff --git a/checks/lib/container-driver/package.nix b/checks/lib/container-driver/package.nix index 872ec1545..cbb47e2ee 100644 --- a/checks/lib/container-driver/package.nix +++ b/checks/lib/container-driver/package.nix @@ -1,8 +1,18 @@ -{ extraPythonPackages, python3Packages, buildPythonApplication, setuptools, util-linux, systemd }: +{ + extraPythonPackages, + python3Packages, + buildPythonApplication, + setuptools, + util-linux, + systemd, +}: buildPythonApplication { pname = "test-driver"; version = "0.0.1"; - propagatedBuildInputs = [ util-linux systemd ] ++ extraPythonPackages python3Packages; + propagatedBuildInputs = [ + util-linux + systemd + ] ++ extraPythonPackages python3Packages; nativeBuildInputs = [ setuptools ]; format = "pyproject"; src = ./.; diff --git a/checks/lib/container-driver/test_driver/__init__.py b/checks/lib/container-driver/test_driver/__init__.py index 95c16eb08..be9673a38 100644 --- a/checks/lib/container-driver/test_driver/__init__.py +++ b/checks/lib/container-driver/test_driver/__init__.py @@ -258,7 +258,7 @@ class Driver: self.machines = [] for container in containers: - name_match = re.match(r".*-nixos-system-(.+)-\d.+", container.name) + name_match = re.match(r".*-nixos-system-(.+)-(.+)", container.name) if not name_match: raise ValueError(f"Unable to extract hostname from {container.name}") name = name_match.group(1) diff --git a/checks/lib/container-test.nix b/checks/lib/container-test.nix index 3753167a0..2c0b19b29 100644 --- a/checks/lib/container-test.nix +++ b/checks/lib/container-test.nix @@ -1,33 +1,33 @@ test: -{ pkgs -, self -, ... -}: +{ pkgs, self, ... }: let inherit (pkgs) lib; nixos-lib = import (pkgs.path + "/nixos/lib") { }; in -(nixos-lib.runTest ({ hostPkgs, ... }: { - hostPkgs = pkgs; - # speed-up evaluation - defaults = { - documentation.enable = lib.mkDefault false; - boot.isContainer = true; +(nixos-lib.runTest ( + { hostPkgs, ... }: + { + hostPkgs = pkgs; + # speed-up evaluation + defaults = { + documentation.enable = lib.mkDefault false; + boot.isContainer = true; - # undo qemu stuff - system.build.initialRamdisk = ""; - virtualisation.sharedDirectories = lib.mkForce { }; - networking.useDHCP = false; + # undo qemu stuff + system.build.initialRamdisk = ""; + virtualisation.sharedDirectories = lib.mkForce { }; + networking.useDHCP = false; - # we have not private networking so far - networking.interfaces = lib.mkForce { }; - #networking.primaryIPAddress = lib.mkForce null; - systemd.services.backdoor.enable = false; - }; - # to accept external dependencies such as disko - node.specialArgs.self = self; - imports = [ - test - ./container-driver/module.nix - ]; -})).config.result + # we have not private networking so far + networking.interfaces = lib.mkForce { }; + #networking.primaryIPAddress = lib.mkForce null; + systemd.services.backdoor.enable = false; + }; + # to accept external dependencies such as disko + node.specialArgs.self = self; + imports = [ + test + ./container-driver/module.nix + ]; + } +)).config.result diff --git a/checks/lib/test-base.nix b/checks/lib/test-base.nix index 998ffba99..1c88b9ea8 100644 --- a/checks/lib/test-base.nix +++ b/checks/lib/test-base.nix @@ -1,8 +1,5 @@ test: -{ pkgs -, self -, ... -}: +{ pkgs, self, ... }: let inherit (pkgs) lib; nixos-lib = import (pkgs.path + "/nixos/lib") { }; diff --git a/checks/schemas.nix b/checks/schemas.nix index c12e1d8f1..5307cc35b 100644 --- a/checks/schemas.nix +++ b/checks/schemas.nix @@ -1,35 +1,48 @@ -{ self, runCommand, check-jsonschema, pkgs, lib, ... }: +{ + self, + runCommand, + check-jsonschema, + pkgs, + lib, + ... +}: let clanModules.clanCore = self.nixosModules.clanCore; baseModule = { - imports = - (import (pkgs.path + "/nixos/modules/module-list.nix")) - ++ [{ + imports = (import (pkgs.path + "/nixos/modules/module-list.nix")) ++ [ + { nixpkgs.hostPlatform = "x86_64-linux"; clanCore.clanName = "dummy"; - }]; + } + ]; }; - optionsFromModule = module: + optionsFromModule = + module: let evaled = lib.evalModules { - modules = [ module baseModule ]; + modules = [ + module + baseModule + ]; }; in evaled.options.clan; - clanModuleSchemas = lib.mapAttrs (_: module: self.lib.jsonschema.parseOptions (optionsFromModule module)) clanModules; + clanModuleSchemas = lib.mapAttrs ( + _: module: self.lib.jsonschema.parseOptions (optionsFromModule module) + ) clanModules; - mkTest = name: schema: runCommand "schema-${name}" { } '' - ${check-jsonschema}/bin/check-jsonschema \ - --check-metaschema ${builtins.toFile "schema-${name}" (builtins.toJSON schema)} - touch $out - ''; + mkTest = + name: schema: + runCommand "schema-${name}" { } '' + ${check-jsonschema}/bin/check-jsonschema \ + --check-metaschema ${builtins.toFile "schema-${name}" (builtins.toJSON schema)} + touch $out + ''; in -lib.mapAttrs' - (name: schema: { - name = "schema-${name}"; - value = mkTest name schema; - }) - clanModuleSchemas +lib.mapAttrs' (name: schema: { + name = "schema-${name}"; + value = mkTest name schema; +}) clanModuleSchemas diff --git a/checks/secrets/default.nix b/checks/secrets/default.nix index 8f050bf7b..97b9a7e45 100644 --- a/checks/secrets/default.nix +++ b/checks/secrets/default.nix @@ -1,19 +1,19 @@ (import ../lib/test-base.nix) { name = "secrets"; - nodes.machine = { self, config, ... }: { - imports = [ - (self.nixosModules.clanCore) - ]; - environment.etc."secret".source = config.sops.secrets.secret.path; - environment.etc."group-secret".source = config.sops.secrets.group-secret.path; - sops.age.keyFile = ./key.age; + nodes.machine = + { self, config, ... }: + { + imports = [ (self.nixosModules.clanCore) ]; + environment.etc."secret".source = config.sops.secrets.secret.path; + environment.etc."group-secret".source = config.sops.secrets.group-secret.path; + sops.age.keyFile = ./key.age; - clanCore.clanDir = "${./.}"; - clanCore.machineName = "machine"; + clanCore.clanDir = "${./.}"; + clanCore.machineName = "machine"; - networking.hostName = "machine"; - }; + networking.hostName = "machine"; + }; testScript = '' machine.succeed("cat /etc/secret >&2") machine.succeed("cat /etc/group-secret >&2") diff --git a/checks/wayland-proxy-virtwl/default.nix b/checks/wayland-proxy-virtwl/default.nix index d678bcb81..4bfa2df78 100644 --- a/checks/wayland-proxy-virtwl/default.nix +++ b/checks/wayland-proxy-virtwl/default.nix @@ -1,25 +1,35 @@ -import ../lib/test-base.nix ({ config, pkgs, lib, ... }: { - name = "wayland-proxy-virtwl"; +import ../lib/test-base.nix ( + { + config, + pkgs, + lib, + ... + }: + { + name = "wayland-proxy-virtwl"; - nodes.machine = { self, ... }: { - imports = [ - self.nixosModules.clanCore + nodes.machine = + { self, ... }: { - clanCore.machineName = "machine"; - clanCore.clanDir = ./.; - } - ]; - services.wayland-proxy-virtwl.enable = true; + imports = [ + self.nixosModules.clanCore + { + clanCore.machineName = "machine"; + clanCore.clanDir = ./.; + } + ]; + services.wayland-proxy-virtwl.enable = true; - virtualisation.qemu.options = [ - "-vga none -device virtio-gpu-rutabaga,cross-domain=on,hostmem=4G,wsi=headless" - ]; + virtualisation.qemu.options = [ + "-vga none -device virtio-gpu-rutabaga,cross-domain=on,hostmem=4G,wsi=headless" + ]; - virtualisation.qemu.package = lib.mkForce pkgs.qemu_kvm; - }; - testScript = '' - start_all() - # use machinectl - machine.succeed("machinectl shell .host ${config.nodes.machine.systemd.package}/bin/systemctl --user start wayland-proxy-virtwl >&2") - ''; -}) + virtualisation.qemu.package = lib.mkForce pkgs.qemu_kvm; + }; + testScript = '' + start_all() + # use machinectl + machine.succeed("machinectl shell .host ${config.nodes.machine.systemd.package}/bin/systemctl --user start wayland-proxy-virtwl >&2") + ''; + } +) diff --git a/checks/zt-tcp-relay/default.nix b/checks/zt-tcp-relay/default.nix index a4fcea4ab..b30e4c5f4 100644 --- a/checks/zt-tcp-relay/default.nix +++ b/checks/zt-tcp-relay/default.nix @@ -1,20 +1,25 @@ -(import ../lib/container-test.nix) ({ pkgs, ... }: { - name = "zt-tcp-relay"; +(import ../lib/container-test.nix) ( + { pkgs, ... }: + { + name = "zt-tcp-relay"; - nodes.machine = { self, ... }: { - imports = [ - self.nixosModules.clanCore - self.clanModules.zt-tcp-relay + nodes.machine = + { self, ... }: { - clanCore.machineName = "machine"; - clanCore.clanDir = ./.; - } - ]; - }; - testScript = '' - start_all() - machine.wait_for_unit("zt-tcp-relay.service") - out = machine.succeed("${pkgs.netcat}/bin/nc -z -v localhost 4443") - print(out) - ''; -}) + imports = [ + self.nixosModules.clanCore + self.clanModules.zt-tcp-relay + { + clanCore.machineName = "machine"; + clanCore.clanDir = ./.; + } + ]; + }; + testScript = '' + start_all() + machine.wait_for_unit("zt-tcp-relay.service") + out = machine.succeed("${pkgs.netcat}/bin/nc -z -v localhost 4443") + print(out) + ''; + } +) diff --git a/clanModules/borgbackup.nix b/clanModules/borgbackup.nix index 06358e6f9..b6d4d99da 100644 --- a/clanModules/borgbackup.nix +++ b/clanModules/borgbackup.nix @@ -1,68 +1,88 @@ -{ config, lib, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: let cfg = config.clan.borgbackup; in { - options.clan.borgbackup = { - enable = lib.mkEnableOption "backups with borgbackup"; - destinations = lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule ({ name, ... }: { - options = { - name = lib.mkOption { - type = lib.types.str; - default = name; - description = "the name of the backup job"; + options.clan.borgbackup.destinations = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + type = lib.types.str; + default = name; + description = "the name of the backup job"; + }; + repo = lib.mkOption { + type = lib.types.str; + description = "the borgbackup repository to backup to"; + }; + rsh = lib.mkOption { + type = lib.types.str; + default = "ssh -i ${ + config.clanCore.secrets.borgbackup.secrets."borgbackup.ssh".path + } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"; + description = "the rsh to use for the backup"; + }; }; - repo = lib.mkOption { - type = lib.types.str; - description = "the borgbackup repository to backup to"; - }; - rsh = lib.mkOption { - type = lib.types.str; - default = "ssh -i ${config.clanCore.secrets.borgbackup.secrets."borgbackup.ssh".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"; - description = "the rsh to use for the backup"; - }; - - }; - })); - description = '' - destinations where the machine should be backuped to - ''; - }; + } + ) + ); + default = { }; + description = '' + destinations where the machine should be backuped to + ''; }; - config = lib.mkIf cfg.enable { - services.borgbackup.jobs = lib.mapAttrs - (_: dest: { - paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state)); - exclude = [ "*.pyc" ]; - repo = dest.repo; - environment.BORG_RSH = dest.rsh; - compression = "auto,zstd"; - startAt = "*-*-* 01:00:00"; - persistentTimer = true; - preHook = '' - set -x - ''; - encryption = { - mode = "repokey"; - passCommand = "cat ${config.clanCore.secrets.borgbackup.secrets."borgbackup.repokey".path}"; - }; + imports = [ + (lib.mkRemovedOptionModule [ + "clan" + "borgbackup" + "enable" + ] "Just define clan.borgbackup.destinations to enable it") + ]; - prune.keep = { - within = "1d"; # Keep all archives from the last day - daily = 7; - weekly = 4; - monthly = 0; - }; - }) - cfg.destinations; + config = lib.mkIf (cfg.destinations != { }) { + services.borgbackup.jobs = lib.mapAttrs (_: dest: { + paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state)); + exclude = [ "*.pyc" ]; + repo = dest.repo; + environment.BORG_RSH = dest.rsh; + compression = "auto,zstd"; + startAt = "*-*-* 01:00:00"; + persistentTimer = true; + preHook = '' + set -x + ''; + + encryption = { + mode = "repokey"; + passCommand = "cat ${config.clanCore.secrets.borgbackup.secrets."borgbackup.repokey".path}"; + }; + + prune.keep = { + within = "1d"; # Keep all archives from the last day + daily = 7; + weekly = 4; + monthly = 0; + }; + }) cfg.destinations; clanCore.secrets.borgbackup = { facts."borgbackup.ssh.pub" = { }; secrets."borgbackup.ssh" = { }; secrets."borgbackup.repokey" = { }; - generator.path = [ pkgs.openssh pkgs.coreutils pkgs.xkcdpass ]; + generator.path = [ + pkgs.openssh + pkgs.coreutils + pkgs.xkcdpass + ]; generator.script = '' ssh-keygen -t ed25519 -N "" -f "$secrets"/borgbackup.ssh mv "$secrets"/borgbackup.ssh.pub "$facts"/borgbackup.ssh.pub @@ -74,8 +94,9 @@ in # TODO list needs to run locally or on the remote machine list = '' # we need yes here to skip the changed url verification - ${lib.concatMapStringsSep "\n" (dest: ''yes y | borg-job-${dest.name} list --json | jq -r '. + {"job-name": "${dest.name}"}' '') - (lib.attrValues cfg.destinations)} + ${lib.concatMapStringsSep "\n" ( + dest: ''yes y | borg-job-${dest.name} list --json | jq -r '. + {"job-name": "${dest.name}"}' '' + ) (lib.attrValues cfg.destinations)} ''; create = '' ${lib.concatMapStringsSep "\n" (dest: '' diff --git a/clanModules/deltachat.nix b/clanModules/deltachat.nix index 771068a11..489ffaa33 100644 --- a/clanModules/deltachat.nix +++ b/clanModules/deltachat.nix @@ -1,4 +1,5 @@ -{ config, pkgs, ... }: { +{ config, pkgs, ... }: +{ networking.firewall.interfaces."zt+".allowedTCPPorts = [ 25 ]; # smtp with other hosts environment.systemPackages = [ pkgs.deltachat-desktop ]; @@ -134,9 +135,7 @@ storage &local_mailboxes } ''; - ensureAccounts = [ - "user@${domain}" - ]; + ensureAccounts = [ "user@${domain}" ]; ensureCredentials = { "user@${domain}".passwordFile = pkgs.writeText "dummy" "foobar"; }; diff --git a/clanModules/diskLayouts.nix b/clanModules/diskLayouts.nix index 0bbc45e00..097ff2fa6 100644 --- a/clanModules/diskLayouts.nix +++ b/clanModules/diskLayouts.nix @@ -41,4 +41,3 @@ }; }; } - diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix index 00cf9d3b9..d2895e704 100644 --- a/clanModules/flake-module.nix +++ b/clanModules/flake-module.nix @@ -12,7 +12,11 @@ moonlight = ./moonlight.nix; sunshine = ./sunshine.nix; syncthing = ./syncthing.nix; + sshd = ./sshd.nix; + vm-user = ./vm-user.nix; + graphical = ./graphical.nix; xfce = ./xfce.nix; + xfce-vm = ./xfce-vm.nix; zt-tcp-relay = ./zt-tcp-relay.nix; localsend = ./localsend.nix; waypipe = ./waypipe.nix; diff --git a/clanModules/graphical.nix b/clanModules/graphical.nix new file mode 100644 index 000000000..e272d8b80 --- /dev/null +++ b/clanModules/graphical.nix @@ -0,0 +1 @@ +_: { fonts.enableDefaultPackages = true; } diff --git a/clanModules/localsend.nix b/clanModules/localsend.nix index b5b6210e8..3172b8dd0 100644 --- a/clanModules/localsend.nix +++ b/clanModules/localsend.nix @@ -1,7 +1,8 @@ -{ config -, pkgs -, lib -, ... +{ + config, + pkgs, + lib, + ... }: { # Integration can be improved, if the following issues get implemented: diff --git a/clanModules/sshd.nix b/clanModules/sshd.nix new file mode 100644 index 000000000..df38078d9 --- /dev/null +++ b/clanModules/sshd.nix @@ -0,0 +1,24 @@ +{ config, pkgs, ... }: +{ + services.openssh.enable = true; + + services.openssh.hostKeys = [ + { + path = config.clanCore.secrets.openssh.secrets."ssh.id_ed25519".path; + type = "ed25519"; + } + ]; + + clanCore.secrets.openssh = { + secrets."ssh.id_ed25519" = { }; + facts."ssh.id_ed25519.pub" = { }; + generator.path = [ + pkgs.coreutils + pkgs.openssh + ]; + generator.script = '' + ssh-keygen -t ed25519 -N "" -f $secrets/ssh.id_ed25519 + mv $secrets/ssh.id_ed25519.pub $facts/ssh.id_ed25519.pub + ''; + }; +} diff --git a/clanModules/syncthing.nix b/clanModules/syncthing.nix index 8b70bc064..bf73dcf0f 100644 --- a/clanModules/syncthing.nix +++ b/clanModules/syncthing.nix @@ -1,7 +1,8 @@ -{ config -, pkgs -, lib -, ... +{ + config, + pkgs, + lib, + ... }: { options.clan.syncthing = { @@ -53,16 +54,16 @@ assertions = [ { - assertion = - lib.all (attr: builtins.hasAttr attr config.services.syncthing.settings.folders) - config.clan.syncthing.autoShares; + assertion = lib.all ( + attr: builtins.hasAttr attr config.services.syncthing.settings.folders + ) config.clan.syncthing.autoShares; message = '' Syncthing: If you want to AutoShare a folder, you need to have it configured on the sharing device. ''; } ]; - # Activates inofify compatibilty on syncthing + # Activates inofify compatibility on syncthing boot.kernel.sysctl."fs.inotify.max_user_watches" = lib.mkDefault 524288; services.syncthing = { @@ -80,12 +81,8 @@ group = "syncthing"; - key = - lib.mkDefault - config.clan.secrets.syncthing.secrets."syncthing.key".path or null; - cert = - lib.mkDefault - config.clan.secrets.syncthing.secrets."syncthing.cert".path or null; + key = lib.mkDefault config.clan.secrets.syncthing.secrets."syncthing.key".path or null; + cert = lib.mkDefault config.clan.secrets.syncthing.secrets."syncthing.cert".path or null; settings = { options = { @@ -127,47 +124,33 @@ set -x # query pending deviceID's APIKEY=$(cat ${apiKey}) - PENDING=$(${ - lib.getExe pkgs.curl - } -X GET -H "X-API-Key: $APIKEY" ${baseAddress}${getPendingDevices}) + PENDING=$(${lib.getExe pkgs.curl} -X GET -H "X-API-Key: $APIKEY" ${baseAddress}${getPendingDevices}) PENDING=$(echo $PENDING | ${lib.getExe pkgs.jq} keys[]) # accept pending deviceID's for ID in $PENDING;do - ${ - lib.getExe pkgs.curl - } -X POST -d "{\"deviceId\": $ID}" -H "Content-Type: application/json" -H "X-API-Key: $APIKEY" ${baseAddress}${postNewDevice} + ${lib.getExe pkgs.curl} -X POST -d "{\"deviceId\": $ID}" -H "Content-Type: application/json" -H "X-API-Key: $APIKEY" ${baseAddress}${postNewDevice} # get all shared folders by their ID for folder in ${builtins.toString config.clan.syncthing.autoShares}; do - SHARED_IDS=$(${ - lib.getExe pkgs.curl - } -X GET -H "X-API-Key: $APIKEY" ${baseAddress}${SharedFolderById}"$folder" | ${ - lib.getExe pkgs.jq - } ."devices") - PATCHED_IDS=$(echo $SHARED_IDS | ${ - lib.getExe pkgs.jq - } ".+= [{\"deviceID\": $ID, \"introducedBy\": \"\", \"encryptionPassword\": \"\"}]") - ${ - lib.getExe pkgs.curl - } -X PATCH -d "{\"devices\": $PATCHED_IDS}" -H "X-API-Key: $APIKEY" ${baseAddress}${SharedFolderById}"$folder" + SHARED_IDS=$(${lib.getExe pkgs.curl} -X GET -H "X-API-Key: $APIKEY" ${baseAddress}${SharedFolderById}"$folder" | ${lib.getExe pkgs.jq} ."devices") + PATCHED_IDS=$(echo $SHARED_IDS | ${lib.getExe pkgs.jq} ".+= [{\"deviceID\": $ID, \"introducedBy\": \"\", \"encryptionPassword\": \"\"}]") + ${lib.getExe pkgs.curl} -X PATCH -d "{\"devices\": $PATCHED_IDS}" -H "X-API-Key: $APIKEY" ${baseAddress}${SharedFolderById}"$folder" done done ''; }; - systemd.timers.syncthing-auto-accept = - lib.mkIf config.clan.syncthing.autoAcceptDevices - { - description = "Syncthing Auto Accept"; + systemd.timers.syncthing-auto-accept = lib.mkIf config.clan.syncthing.autoAcceptDevices { + description = "Syncthing Auto Accept"; - wantedBy = [ "syncthing-auto-accept.service" ]; + wantedBy = [ "syncthing-auto-accept.service" ]; - timerConfig = { - OnActiveSec = lib.mkDefault 60; - OnUnitActiveSec = lib.mkDefault 60; - }; - }; + timerConfig = { + OnActiveSec = lib.mkDefault 60; + OnUnitActiveSec = lib.mkDefault 60; + }; + }; systemd.services.syncthing-init-api-key = let @@ -182,9 +165,7 @@ set -efu pipefail APIKEY=$(cat ${apiKey}) - ${ - lib.getExe pkgs.gnused - } -i "s/.*<\/apikey>/$APIKEY<\/apikey>/" /var/lib/syncthing/config.xml + ${lib.getExe pkgs.gnused} -i "s/.*<\/apikey>/$APIKEY<\/apikey>/" /var/lib/syncthing/config.xml # sudo systemctl restart syncthing.service systemctl restart syncthing.service ''; diff --git a/clanModules/vm-user.nix b/clanModules/vm-user.nix new file mode 100644 index 000000000..28e93535e --- /dev/null +++ b/clanModules/vm-user.nix @@ -0,0 +1,20 @@ +{ + security = { + sudo.wheelNeedsPassword = false; + polkit.enable = true; + rtkit.enable = true; + }; + + users.users.user = { + isNormalUser = true; + createHome = true; + uid = 1000; + initialHashedPassword = ""; + extraGroups = [ + "wheel" + "video" + "render" + ]; + shell = "/run/current-system/sw/bin/bash"; + }; +} diff --git a/clanModules/waypipe.nix b/clanModules/waypipe.nix index b9d7d76e8..322e8d612 100644 --- a/clanModules/waypipe.nix +++ b/clanModules/waypipe.nix @@ -1,7 +1,8 @@ -{ pkgs -, lib -, config -, ... +{ + pkgs, + lib, + config, + ... }: { options.clan.services.waypipe = { @@ -49,7 +50,10 @@ isNormalUser = true; uid = 1000; password = ""; - extraGroups = [ "wheel" "video" ]; + extraGroups = [ + "wheel" + "video" + ]; shell = "/run/current-system/sw/bin/bash"; }; diff --git a/clanModules/xfce-vm.nix b/clanModules/xfce-vm.nix new file mode 100644 index 000000000..7eadd7f42 --- /dev/null +++ b/clanModules/xfce-vm.nix @@ -0,0 +1,15 @@ +{ + imports = [ + ./vm-user.nix + ./graphical.nix + ]; + + services.xserver = { + enable = true; + displayManager.autoLogin.enable = true; + displayManager.autoLogin.user = "user"; + desktopManager.xfce.enable = true; + desktopManager.xfce.enableScreensaver = false; + xkb.layout = "us"; + }; +} diff --git a/clanModules/zt-tcp-relay.nix b/clanModules/zt-tcp-relay.nix index d48d550fc..7ad8f438f 100644 --- a/clanModules/zt-tcp-relay.nix +++ b/clanModules/zt-tcp-relay.nix @@ -1,4 +1,10 @@ -{ pkgs, lib, config, ... }: { +{ + pkgs, + lib, + config, + ... +}: +{ options.clan.zt-tcp-relay = { port = lib.mkOption { type = lib.types.port; @@ -13,7 +19,9 @@ wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; serviceConfig = { - ExecStart = "${pkgs.callPackage ../pkgs/zt-tcp-relay {}}/bin/zt-tcp-relay --listen [::]:${builtins.toString config.clan.zt-tcp-relay.port}"; + ExecStart = "${ + pkgs.callPackage ../pkgs/zt-tcp-relay { } + }/bin/zt-tcp-relay --listen [::]:${builtins.toString config.clan.zt-tcp-relay.port}"; Restart = "always"; RestartSec = "5"; dynamicUsers = true; diff --git a/devShell-python.nix b/devShell-python.nix index d5ac4680e..066c44731 100644 --- a/devShell-python.nix +++ b/devShell-python.nix @@ -1,9 +1,10 @@ { perSystem = - { pkgs - , self' - , lib - , ... + { + pkgs, + self', + lib, + ... }: let python3 = pkgs.python3; @@ -14,15 +15,11 @@ ps: clan-cli.propagatedBuildInputs ++ clan-cli.devDependencies - ++ [ - ps.pip - # clan-vm-manager deps - ps.pygobject3 - ] + ++ [ ps.pip ] + ++ [ clan-vm-manager.externalPythonDeps ] + # clan-vm-manager deps ); - linuxOnlyPackages = lib.optionals pkgs.stdenv.isLinux [ - pkgs.xdg-utils - ]; + linuxOnlyPackages = lib.optionals pkgs.stdenv.isLinux [ pkgs.xdg-utils ]; in { devShells.python = pkgs.mkShell { @@ -49,9 +46,9 @@ ## PYTHON - tmp_path=$(realpath ./.direnv) + tmp_path="$(realpath ./.direnv/python)" repo_root=$(realpath .) - mkdir -p "$tmp_path/python/${pythonWithDeps.sitePackages}" + mkdir -p "$tmp_path/${pythonWithDeps.sitePackages}" # local dependencies localPackages=( @@ -59,28 +56,41 @@ $repo_root/pkgs/clan-vm-manager ) - # Install the package in editable mode - # This allows executing `clan` from within the dev-shell using the current - # version of the code and its dependencies. - # TODO: this is slow. get rid of pip or add better caching - echo "==== Installing local python packages in editable mode ====" + # Install executable wrappers for local python packages scripts + # This is done by utilizing `pip install --editable` + # As a result, executables like `clan` can be executed from within the dev-shell + # while using the current version of the code and its dependencies. for package in "''${localPackages[@]}"; do - ${pythonWithDeps}/bin/pip install \ - --quiet \ - --disable-pip-version-check \ - --no-index \ - --no-build-isolation \ - --prefix "$tmp_path/python" \ - --editable "$package" + pname=$(basename "$package") + if + [ ! -e "$tmp_path/meta/$pname/pyproject.toml" ] \ + || [ ! -e "$package/pyproject.toml" ] \ + || ! cmp -s "$tmp_path/meta/$pname/pyproject.toml" "$package/pyproject.toml" + then + echo "==== Installing local python package $pname in editable mode ====" + mkdir -p "$tmp_path/meta/$pname" + cp $package/pyproject.toml $tmp_path/meta/$pname/pyproject.toml + ${python3.pkgs.pip}/bin/pip install \ + --quiet \ + --disable-pip-version-check \ + --no-index \ + --no-build-isolation \ + --prefix "$tmp_path" \ + --editable "$package" + fi done - export PATH="$tmp_path/python/bin:$PATH" - export PYTHONPATH="''${PYTHONPATH:+$PYTHONPATH:}$tmp_path/python/${pythonWithDeps.sitePackages}" + export PATH="$tmp_path/bin:$PATH" + export PYTHONPATH="''${PYTHONPATH:+$PYTHONPATH:}$tmp_path/${pythonWithDeps.sitePackages}" for package in "''${localPackages[@]}"; do export PYTHONPATH="$package:$PYTHONPATH" done + + + ## GUI + if ! command -v xdg-mime &> /dev/null; then echo "Warning: 'xdg-mime' is not available. The desktop file cannot be installed." fi @@ -93,7 +103,6 @@ UI_BIN="clan-vm-manager" cp -f $DESKTOP_SRC $DESKTOP_DST - sleep 2 sed -i "s|Exec=.*clan-vm-manager|Exec=$UI_BIN|" $DESKTOP_DST xdg-mime default $DESKTOP_FILE_NAME x-scheme-handler/clan echo "==== Validating desktop file installation ====" diff --git a/devShell.nix b/devShell.nix index c3f6eba7d..1815a4f7b 100644 --- a/devShell.nix +++ b/devShell.nix @@ -1,9 +1,10 @@ { perSystem = - { pkgs - , self' - , config - , ... + { + pkgs, + self', + config, + ... }: let writers = pkgs.callPackage ./pkgs/builders/script-writers.nix { }; @@ -16,16 +17,16 @@ # A python program to switch between dev-shells # usage: select-shell shell-name # the currently enabled dev-shell gets stored in ./.direnv/selected-shell - select-shell = writers.writePython3Bin "select-shell" - { - flakeIgnore = [ "E501" ]; - } ./pkgs/scripts/select-shell.py; + select-shell = writers.writePython3Bin "select-shell" { + flakeIgnore = [ "E501" ]; + } ./pkgs/scripts/select-shell.py; in { devShells.default = pkgs.mkShell { packages = [ select-shell pkgs.tea + pkgs.nix self'.packages.tea-create-pr self'.packages.merge-after-ci self'.packages.pending-reviews diff --git a/docs/admins/secrets-management.md b/docs/admins/secrets-management.md index 658abb3e4..dad7aa5fa 100644 --- a/docs/admins/secrets-management.md +++ b/docs/admins/secrets-management.md @@ -160,7 +160,7 @@ examples. `clan secrets` stores each secrets in a single file, whereas [sops](https://github.com/Mic92/sops-nix) commonly allows to put all secrets in a yaml or json documents. -If you already happend to use sops-nix, you can migrate by using the `clan secrets import-sops` command by importing these documents: +If you already happened to use sops-nix, you can migrate by using the `clan secrets import-sops` command by importing these documents: ```shellSession % clan secrets import-sops --prefix matchbox- --group admins --machine matchbox nixos/matchbox/secrets/secrets.yaml diff --git a/flake.lock b/flake.lock index a34c6dd21..773c4c17a 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1709632354, - "narHash": "sha256-jxRHwqrtNze51WKFKvxlQ8Inf62UNRl5cFqEQ2V96vE=", + "lastModified": 1710427903, + "narHash": "sha256-sV0Q5ndvfjK9JfCg/QM/HX/fcittohvtq8dD62isxdM=", "owner": "nix-community", "repo": "disko", - "rev": "0d11aa8d6431326e10b8656420f91085c3bd0b12", + "rev": "21d89b333ca300bef82c928c856d48b94a9f997c", "type": "github" }, "original": { @@ -27,11 +27,11 @@ ] }, "locked": { - "lastModified": 1706830856, - "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=", + "lastModified": 1709336216, + "narHash": "sha256-Dt/wOWeW6Sqm11Yh+2+t0dfEWxoMxGBvv3JpIocFl9E=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f", + "rev": "f7b3c975cf067e56e7cda6cb098ebe3fb4d74ca2", "type": "github" }, "original": { @@ -42,11 +42,11 @@ }, "nixlib": { "locked": { - "lastModified": 1708217146, - "narHash": "sha256-nGfEv7k78slqIR5E0zzWSx214d/4/ZPKDkObLJqVLVw=", + "lastModified": 1710031547, + "narHash": "sha256-pkUg3hOKuGWMGF9WEMPPN/G4pqqdbNGJQ54yhyQYDVY=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "e623008d8a46517470e6365505f1a3ce171fa46a", + "rev": "630ebdc047ca96d8126e16bb664c7730dc52f6e6", "type": "github" }, "original": { @@ -63,11 +63,11 @@ ] }, "locked": { - "lastModified": 1708563055, - "narHash": "sha256-FaojUZNu+YPFi3eCI7mL4kxPKQ51DoySa7mqmllUOuc=", + "lastModified": 1710398463, + "narHash": "sha256-fQlYanU84E8uwBpcoTCcLCwU8cqn0eQ7nwTcrWfSngc=", "owner": "nix-community", "repo": "nixos-generators", - "rev": "f4631dee1a0fd56c0db89860e83e3588a28c7631", + "rev": "efd4e38532b5abfaa5c9fc95c5a913157dc20ccb", "type": "github" }, "original": { @@ -78,11 +78,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1708847675, - "narHash": "sha256-RUZ7KEs/a4EzRELYDGnRB6i7M1Izii3JD/LyzH0c6Tg=", + "lastModified": 1710672219, + "narHash": "sha256-Bp3Jsq1Jn8q4EesBlcOVNwnEipNpzYs73kvR3+3EUC4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2a34566b67bef34c551f204063faeecc444ae9da", + "rev": "f471be9644f3ab2f3cb868de1787ab70a537b0e7", "type": "github" }, "original": { @@ -110,11 +110,11 @@ "nixpkgs-stable": [] }, "locked": { - "lastModified": 1708830076, - "narHash": "sha256-Cjh2xdjxC6S6nW6Whr2dxSeh8vjodzhTmQdI4zPJ4RA=", + "lastModified": 1710644594, + "narHash": "sha256-RquCuzxfy4Nr8DPbdp3D/AsbYep21JgQzG8aMH9jJ4A=", "owner": "Mic92", "repo": "sops-nix", - "rev": "2874fbbe4a65bd2484b0ad757d27a16107f6bc17", + "rev": "83b68a0e8c94b72cdd0a6e547a14ca7eb1c03616", "type": "github" }, "original": { @@ -130,11 +130,11 @@ ] }, "locked": { - "lastModified": 1708897213, - "narHash": "sha256-QECZB+Hgz/2F/8lWvHNk05N6NU/rD9bWzuNn6Cv8oUk=", + "lastModified": 1710278050, + "narHash": "sha256-Oc6BP7soXqb8itlHI8UKkdf3V9GeJpa1S39SR5+HJys=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e497a9ddecff769c2a7cbab51e1ed7a8501e7a3a", + "rev": "35791f76524086ab4b785a33e4abbedfda64bd22", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 14c799a7b..eb482c23f 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,9 @@ description = "clan.lol base operating system"; nixConfig.extra-substituters = [ "https://cache.clan.lol" ]; - nixConfig.extra-trusted-public-keys = [ "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28=" ]; + nixConfig.extra-trusted-public-keys = [ + "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28=" + ]; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small"; @@ -20,44 +22,42 @@ treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; }; - outputs = inputs @ { flake-parts, ... }: - flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }: { - systems = [ - "x86_64-linux" - "aarch64-linux" - "aarch64-darwin" - ]; - imports = [ - ./checks/flake-module.nix - ./devShell.nix - ./devShell-python.nix - ./formatter.nix - ./templates/flake-module.nix - ./clanModules/flake-module.nix + outputs = + inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } ( + { lib, ... }: + { + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + ]; + imports = [ + ./checks/flake-module.nix + ./devShell.nix + ./devShell-python.nix + ./formatter.nix + ./templates/flake-module.nix + ./clanModules/flake-module.nix - ./pkgs/flake-module.nix + ./pkgs/flake-module.nix - ./lib/flake-module.nix - ./nixosModules/flake-module.nix - { - options.flake = flake-parts.lib.mkSubmoduleOptions { - clanInternals = lib.mkOption { - type = lib.types.submodule { - options = { - all-machines-json = lib.mkOption { - type = lib.types.attrsOf lib.types.str; - }; - machines = lib.mkOption { - type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); - }; - machinesFunc = lib.mkOption { - type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); + ./lib/flake-module.nix + ./nixosModules/flake-module.nix + { + options.flake = flake-parts.lib.mkSubmoduleOptions { + clanInternals = lib.mkOption { + type = lib.types.submodule { + options = { + all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.str; }; + machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; + machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; }; }; }; }; - }; - } - ]; - }); + } + ]; + } + ); } diff --git a/formatter.nix b/formatter.nix index 2b7b6d0a6..4aeacde3b 100644 --- a/formatter.nix +++ b/formatter.nix @@ -1,51 +1,48 @@ -{ lib -, inputs -, ... -}: { - imports = [ - inputs.treefmt-nix.flakeModule - ]; - perSystem = { self', pkgs, ... }: { - treefmt.projectRootFile = "flake.nix"; - treefmt.flakeCheck = true; - treefmt.flakeFormatter = true; - treefmt.programs.shellcheck.enable = true; +{ lib, inputs, ... }: +{ + imports = [ inputs.treefmt-nix.flakeModule ]; + perSystem = + { self', pkgs, ... }: + { + treefmt.projectRootFile = "flake.nix"; + treefmt.programs.shellcheck.enable = true; - treefmt.programs.mypy.enable = true; - treefmt.programs.mypy.directories = { - "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.pytestDependencies; - "pkgs/clan-vm-manager".extraPythonPackages = self'.packages.clan-vm-manager.propagatedBuildInputs; - }; + treefmt.programs.mypy.enable = true; + treefmt.programs.mypy.directories = { + "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.pytestDependencies; + "pkgs/clan-vm-manager".extraPythonPackages = + self'.packages.clan-vm-manager.externalPythonDeps ++ self'.packages.clan-cli.pytestDependencies; + }; - treefmt.settings.formatter.nix = { - command = "sh"; - options = [ - "-eucx" - '' - # First deadnix - ${lib.getExe pkgs.deadnix} --edit "$@" - # Then nixpkgs-fmt - ${lib.getExe pkgs.nixpkgs-fmt} "$@" - '' - "--" # this argument is ignored by bash - ]; - includes = [ "*.nix" ]; - excludes = [ - # Was copied from nixpkgs. Keep diff minimal to simplify upstreaming. - "pkgs/builders/script-writers.nix" - ]; + treefmt.settings.formatter.nix = { + command = "sh"; + options = [ + "-eucx" + '' + # First deadnix + ${lib.getExe pkgs.deadnix} --edit "$@" + # Then nixpkgs-fmt + ${lib.getExe pkgs.nixfmt-rfc-style} "$@" + '' + "--" # this argument is ignored by bash + ]; + includes = [ "*.nix" ]; + excludes = [ + # Was copied from nixpkgs. Keep diff minimal to simplify upstreaming. + "pkgs/builders/script-writers.nix" + ]; + }; + treefmt.settings.formatter.python = { + command = "sh"; + options = [ + "-eucx" + '' + ${lib.getExe pkgs.ruff} --fix "$@" + ${lib.getExe pkgs.ruff} format "$@" + '' + "--" # this argument is ignored by bash + ]; + includes = [ "*.py" ]; + }; }; - treefmt.settings.formatter.python = { - command = "sh"; - options = [ - "-eucx" - '' - ${lib.getExe pkgs.ruff} --fix "$@" - ${lib.getExe pkgs.ruff} format "$@" - '' - "--" # this argument is ignored by bash - ]; - includes = [ "*.py" ]; - }; - }; } diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index eaccc25ef..aa8f2ed28 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -1,66 +1,80 @@ -{ clan-core, nixpkgs, lib }: -{ directory # The directory containing the machines subdirectory -, specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available -, machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... } -, clanName # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to. -, clanIcon ? null # A path to an icon to be used for the clan, should be the same for all machines -, pkgsForSystem ? (_system: null) # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. - # This improves performance, but all nipxkgs.* options will be ignored. +{ + clan-core, + nixpkgs, + lib, +}: +{ + directory, # The directory containing the machines subdirectory + specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available + machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... } + clanName, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to. + clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines + pkgsForSystem ? (_system: null), # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. +# This improves performance, but all nipxkgs.* options will be ignored. }: let - machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (builtins.readDir (directory + /machines)); + machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( + builtins.readDir (directory + /machines) + ); - machineSettings = machineName: + machineSettings = + machineName: # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily # This is useful for doing a dry-run before writing changes into the settings.json # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval - if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" - then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then + builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) else - lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") - (builtins.fromJSON - (builtins.readFile (directory + /machines/${machineName}/settings.json))); + lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( + builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json)) + ); # Read additional imports specified via a config option in settings.json # This is not an infinite recursion, because the imports are discovered here # before calling evalModules. # It is still useful to have the imports as an option, as this allows for type # checking and easy integration with the config frontend(s) - machineImports = machineSettings: - map - (module: clan-core.clanModules.${module}) - (machineSettings.clanImports or [ ]); + machineImports = + machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); # TODO: remove default system once we have a hardware-config mechanism - nixosConfiguration = { system ? "x86_64-linux", name, pkgs ? null, extraConfig ? { } }: nixpkgs.lib.nixosSystem { - modules = - let - settings = machineSettings name; - in - (machineImports settings) - ++ [ - settings - clan-core.nixosModules.clanCore - extraConfig - (machines.${name} or { }) - ({ - clanCore.clanName = clanName; - clanCore.clanIcon = clanIcon; - clanCore.clanDir = directory; - clanCore.machineName = name; - nixpkgs.hostPlatform = lib.mkDefault system; + nixosConfiguration = + { + system ? "x86_64-linux", + name, + pkgs ? null, + extraConfig ? { }, + }: + nixpkgs.lib.nixosSystem { + modules = + let + settings = machineSettings name; + in + (machineImports settings) + ++ [ + settings + clan-core.nixosModules.clanCore + extraConfig + (machines.${name} or { }) + ( + { + clanCore.clanName = clanName; + clanCore.clanIcon = clanIcon; + clanCore.clanDir = directory; + clanCore.machineName = name; + nixpkgs.hostPlatform = lib.mkDefault system; - # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) - nix.registry.nixpkgs.to = { - type = "path"; - path = lib.mkDefault nixpkgs; - }; - } // lib.optionalAttrs (pkgs != null) { - nixpkgs.pkgs = lib.mkForce pkgs; - }) - ]; - inherit specialArgs; - }; + # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) + nix.registry.nixpkgs.to = { + type = "path"; + path = lib.mkDefault nixpkgs; + }; + } + // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } + ) + ]; + inherit specialArgs; + }; allMachines = machinesDirs // machines; @@ -77,27 +91,38 @@ let # This instantiates nixos for each system that we support: # configPerSystem = ..nixosConfiguration # We need this to build nixos secret generators for each system - configsPerSystem = builtins.listToAttrs - (builtins.map - (system: lib.nameValuePair system - (lib.mapAttrs - (name: _: nixosConfiguration { + configsPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: + nixosConfiguration { inherit name system; pkgs = pkgsForSystem system; - }) - allMachines)) - supportedSystems); + } + ) allMachines + ) + ) supportedSystems + ); - configsFuncPerSystem = builtins.listToAttrs - (builtins.map - (system: lib.nameValuePair system - (lib.mapAttrs - (name: _: args: nixosConfiguration (args // { - inherit name system; - pkgs = pkgsForSystem system; - })) - allMachines)) - supportedSystems); + configsFuncPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: args: + nixosConfiguration ( + args + // { + inherit name system; + pkgs = pkgsForSystem system; + } + ) + ) allMachines + ) + ) supportedSystems + ); in { inherit nixosConfigurations; @@ -105,8 +130,11 @@ in clanInternals = { machines = configsPerSystem; machinesFunc = configsFuncPerSystem; - all-machines-json = lib.mapAttrs - (system: configs: nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs)) - configsPerSystem; + all-machines-json = lib.mapAttrs ( + system: configs: + nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( + lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs + ) + ) configsPerSystem; }; } diff --git a/lib/default.nix b/lib/default.nix index 856a4dff6..58e95e79d 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -1,4 +1,9 @@ -{ lib, clan-core, nixpkgs, ... }: +{ + lib, + clan-core, + nixpkgs, + ... +}: { jsonschema = import ./jsonschema { inherit lib; }; diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 089da6c90..48437bb14 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -1,11 +1,11 @@ -{ lib -, inputs -, self -, ... -}: { - imports = [ - ./jsonschema/flake-module.nix - ]; +{ + lib, + inputs, + self, + ... +}: +{ + imports = [ ./jsonschema/flake-module.nix ]; flake.lib = import ./default.nix { inherit lib; inherit (inputs) nixpkgs; diff --git a/lib/jsonschema/default.nix b/lib/jsonschema/default.nix index 53dbb9e05..5ecda439f 100644 --- a/lib/jsonschema/default.nix +++ b/lib/jsonschema/default.nix @@ -1,243 +1,290 @@ -{ lib ? import -, excludedTypes ? [ +{ + lib ? import , + excludedTypes ? [ "functionTo" "package" - ] + ], }: let # remove _module attribute from options clean = opts: builtins.removeAttrs opts [ "_module" ]; # throw error if option type is not supported - notSupported = option: lib.trace option throw '' - option type '${option.type.name}' ('${option.type.description}') not supported by jsonschema converter - location: ${lib.concatStringsSep "." option.loc} - ''; + notSupported = + option: + lib.trace option throw '' + option type '${option.type.name}' ('${option.type.description}') not supported by jsonschema converter + location: ${lib.concatStringsSep "." option.loc} + ''; isExcludedOption = option: (lib.elem (option.type.name or null) excludedTypes); - filterExcluded = lib.filter (opt: ! isExcludedOption opt); + filterExcluded = lib.filter (opt: !isExcludedOption opt); - filterExcludedAttrs = lib.filterAttrs (_name: opt: ! isExcludedOption opt); - - allBasicTypes = - [ "boolean" "integer" "number" "string" "array" "object" "null" ]; + filterExcludedAttrs = lib.filterAttrs (_name: opt: !isExcludedOption opt); + allBasicTypes = [ + "boolean" + "integer" + "number" + "string" + "array" + "object" + "null" + ]; in rec { # parses a nixos module to a jsonschema - parseModule = module: + parseModule = + module: let - evaled = lib.evalModules { - modules = [ module ]; - }; + evaled = lib.evalModules { modules = [ module ]; }; in parseOptions evaled.options; # parses a set of evaluated nixos options to a jsonschema - parseOptions = options': + parseOptions = + options': let options = filterExcludedAttrs (clean options'); # parse options to jsonschema properties properties = lib.mapAttrs (_name: option: parseOption option) options; # TODO: figure out how to handle if prop.anyOf is used - isRequired = prop: ! (prop ? default || prop.type or null == "object"); + isRequired = prop: !(prop ? default || prop.type or null == "object"); requiredProps = lib.filterAttrs (_: prop: isRequired prop) properties; - required = lib.optionalAttrs (requiredProps != { }) { - required = lib.attrNames requiredProps; - }; + required = lib.optionalAttrs (requiredProps != { }) { required = lib.attrNames requiredProps; }; in # return jsonschema - required // { + required + // { type = "object"; inherit properties; }; # parses and evaluated nixos option to a jsonschema property definition - parseOption = option: + parseOption = + option: let - default = lib.optionalAttrs (option ? default) { - inherit (option) default; - }; + default = lib.optionalAttrs (option ? default) { inherit (option) default; }; description = lib.optionalAttrs (option ? description) { description = option.description.text or option.description; }; in # either type - # TODO: if all nested optiosn are excluded, the parent sould be excluded too - if option.type.name or null == "either" + # TODO: if all nested optiosn are excluded, the parent sould be excluded too + if + option.type.name or null == "either" # return jsonschema property definition for either then let optionsList' = [ - { type = option.type.nestedTypes.left; _type = "option"; loc = option.loc; } - { type = option.type.nestedTypes.right; _type = "option"; loc = option.loc; } + { + type = option.type.nestedTypes.left; + _type = "option"; + loc = option.loc; + } + { + type = option.type.nestedTypes.right; + _type = "option"; + loc = option.loc; + } ]; optionsList = filterExcluded optionsList'; in - default // description // { - anyOf = map parseOption optionsList; - } + default // description // { anyOf = map parseOption optionsList; } # handle nested options (not a submodule) - else if ! option ? _type - then parseOptions option + else if !option ? _type then + parseOptions option # throw if not an option - else if option._type != "option" && option._type != "option-type" - then throw "parseOption: not an option" + else if option._type != "option" && option._type != "option-type" then + throw "parseOption: not an option" # parse nullOr - else if option.type.name == "nullOr" + else if + option.type.name == "nullOr" # return jsonschema property definition for nullOr then let - nestedOption = - { type = option.type.nestedTypes.elemType; _type = "option"; loc = option.loc; }; + nestedOption = { + type = option.type.nestedTypes.elemType; + _type = "option"; + loc = option.loc; + }; in - default // description // { - anyOf = - [{ type = "null"; }] - ++ ( - lib.optional (! isExcludedOption nestedOption) - (parseOption nestedOption) - ); + default + // description + // { + anyOf = [ + { type = "null"; } + ] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption)); } # parse bool - else if option.type.name == "bool" + else if + option.type.name == "bool" # return jsonschema property definition for bool - then default // description // { - type = "boolean"; - } + then + default // description // { type = "boolean"; } # parse float - else if option.type.name == "float" + else if + option.type.name == "float" # return jsonschema property definition for float - then default // description // { - type = "number"; - } + then + default // description // { type = "number"; } # parse int - else if (option.type.name == "int" || option.type.name == "positiveInt") + else if + (option.type.name == "int" || option.type.name == "positiveInt") # return jsonschema property definition for int - then default // description // { - type = "integer"; - } + then + default // description // { type = "integer"; } # parse string - else if option.type.name == "str" + else if + option.type.name == "str" # return jsonschema property definition for string - then default // description // { - type = "string"; - } + then + default // description // { type = "string"; } # parse string - else if option.type.name == "path" + else if + option.type.name == "path" # return jsonschema property definition for path - then default // description // { - type = "string"; - } + then + default // description // { type = "string"; } # parse anything - else if option.type.name == "anything" + else if + option.type.name == "anything" # return jsonschema property definition for anything - then default // description // { - type = allBasicTypes; - } + then + default // description // { type = allBasicTypes; } # parse unspecified - else if option.type.name == "unspecified" + else if + option.type.name == "unspecified" # return jsonschema property definition for unspecified - then default // description // { - type = allBasicTypes; - } + then + default // description // { type = allBasicTypes; } # parse raw - else if option.type.name == "raw" + else if + option.type.name == "raw" # return jsonschema property definition for raw - then default // description // { - type = allBasicTypes; - } + then + default // description // { type = allBasicTypes; } # parse enum - else if option.type.name == "enum" + else if + option.type.name == "enum" # return jsonschema property definition for enum - then default // description // { - enum = option.type.functor.payload; - } + then + default // description // { enum = option.type.functor.payload; } # parse listOf submodule - else if option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule" + else if + option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule" # return jsonschema property definition for listOf submodule - then default // description // { - type = "array"; - items = parseOptions (option.type.functor.wrapped.getSubOptions option.loc); - } + then + default + // description + // { + type = "array"; + items = parseOptions (option.type.functor.wrapped.getSubOptions option.loc); + } # parse list - else if (option.type.name == "listOf") + else if + (option.type.name == "listOf") # return jsonschema property definition for list then let - nestedOption = { type = option.type.functor.wrapped; _type = "option"; loc = option.loc; }; + nestedOption = { + type = option.type.functor.wrapped; + _type = "option"; + loc = option.loc; + }; in - default // description // { + default + // description + // { type = "array"; } - // (lib.optionalAttrs (! isExcludedOption nestedOption) { - items = parseOption nestedOption; - }) + // (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; }) # parse list of unspecified else if - (option.type.name == "listOf") - && (option.type.functor.wrapped.name == "unspecified") + (option.type.name == "listOf") && (option.type.functor.wrapped.name == "unspecified") # return jsonschema property definition for list - then default // description // { - type = "array"; - } + then + default // description // { type = "array"; } # parse attrsOf submodule - else if option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule" + else if + option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule" # return jsonschema property definition for attrsOf submodule - then default // description // { - type = "object"; - additionalProperties = parseOptions (option.type.nestedTypes.elemType.getSubOptions option.loc); - } + then + default + // description + // { + type = "object"; + additionalProperties = parseOptions (option.type.nestedTypes.elemType.getSubOptions option.loc); + } # parse attrs - else if option.type.name == "attrs" + else if + option.type.name == "attrs" # return jsonschema property definition for attrs - then default // description // { - type = "object"; - additionalProperties = true; - } + then + default + // description + // { + type = "object"; + additionalProperties = true; + } # parse attrsOf # TODO: if nested option is excluded, the parent sould be excluded too - else if option.type.name == "attrsOf" || option.type.name == "lazyAttrsOf" + else if + option.type.name == "attrsOf" || option.type.name == "lazyAttrsOf" # return jsonschema property definition for attrs then let - nestedOption = { type = option.type.nestedTypes.elemType; _type = "option"; loc = option.loc; }; + nestedOption = { + type = option.type.nestedTypes.elemType; + _type = "option"; + loc = option.loc; + }; in - default // description // { + default + // description + // { type = "object"; additionalProperties = - if ! isExcludedOption nestedOption - then parseOption { type = option.type.nestedTypes.elemType; _type = "option"; loc = option.loc; } - else false; + if !isExcludedOption nestedOption then + parseOption { + type = option.type.nestedTypes.elemType; + _type = "option"; + loc = option.loc; + } + else + false; } # parse submodule - else if option.type.name == "submodule" + else if + option.type.name == "submodule" # return jsonschema property definition for submodule # then (lib.attrNames (option.type.getSubOptions option.loc).opt) - then parseOptions (option.type.getSubOptions option.loc) + then + parseOptions (option.type.getSubOptions option.loc) # throw error if option type is not supported - else notSupported option; + else + notSupported option; } diff --git a/lib/jsonschema/example-interface.nix b/lib/jsonschema/example-interface.nix index 4a8cfe702..97fe4145d 100644 --- a/lib/jsonschema/example-interface.nix +++ b/lib/jsonschema/example-interface.nix @@ -1,7 +1,6 @@ -/* - An example nixos module declaring an interface. -*/ -{ lib, ... }: { +# An example nixos module declaring an interface. +{ lib, ... }: +{ options = { # str name = lib.mkOption { @@ -44,7 +43,11 @@ # list of str kernelModules = lib.mkOption { type = lib.types.listOf lib.types.str; - default = [ "nvme" "xhci_pci" "ahci" ]; + default = [ + "nvme" + "xhci_pci" + "ahci" + ]; description = "A list of enabled kernel modules"; }; }; diff --git a/lib/jsonschema/flake-module.nix b/lib/jsonschema/flake-module.nix index 09e792dd4..db133d602 100644 --- a/lib/jsonschema/flake-module.nix +++ b/lib/jsonschema/flake-module.nix @@ -1,29 +1,31 @@ { - perSystem = { pkgs, ... }: { - checks = { + perSystem = + { pkgs, ... }: + { + checks = { - # check if the `clan config` example jsonschema and data is valid - lib-jsonschema-example-valid = pkgs.runCommand "lib-jsonschema-example-valid" { } '' - echo "Checking that example-schema.json is valid" - ${pkgs.check-jsonschema}/bin/check-jsonschema \ - --check-metaschema ${./.}/example-schema.json + # check if the `clan config` example jsonschema and data is valid + lib-jsonschema-example-valid = pkgs.runCommand "lib-jsonschema-example-valid" { } '' + echo "Checking that example-schema.json is valid" + ${pkgs.check-jsonschema}/bin/check-jsonschema \ + --check-metaschema ${./.}/example-schema.json - echo "Checking that example-data.json is valid according to example-schema.json" - ${pkgs.check-jsonschema}/bin/check-jsonschema \ - --schemafile ${./.}/example-schema.json \ - ${./.}/example-data.json + echo "Checking that example-data.json is valid according to example-schema.json" + ${pkgs.check-jsonschema}/bin/check-jsonschema \ + --schemafile ${./.}/example-schema.json \ + ${./.}/example-data.json - touch $out - ''; + touch $out + ''; - # check if the `clan config` nix jsonschema converter unit tests succeed - lib-jsonschema-nix-unit-tests = pkgs.runCommand "lib-jsonschema-nix-unit-tests" { } '' - export NIX_PATH=nixpkgs=${pkgs.path} - ${pkgs.nix-unit}/bin/nix-unit \ - ${./.}/test.nix \ - --eval-store $(realpath .) - touch $out - ''; + # check if the `clan config` nix jsonschema converter unit tests succeed + lib-jsonschema-nix-unit-tests = pkgs.runCommand "lib-jsonschema-nix-unit-tests" { } '' + export NIX_PATH=nixpkgs=${pkgs.path} + ${pkgs.nix-unit}/bin/nix-unit \ + ${./.}/test.nix \ + --eval-store $(realpath .) + touch $out + ''; + }; }; - }; } diff --git a/lib/jsonschema/test.nix b/lib/jsonschema/test.nix index 34e05274b..adf8ab00a 100644 --- a/lib/jsonschema/test.nix +++ b/lib/jsonschema/test.nix @@ -1,6 +1,7 @@ # run these tests via `nix-unit ./test.nix` -{ lib ? (import { }).lib -, slib ? import ./. { inherit lib; } +{ + lib ? (import { }).lib, + slib ? import ./. { inherit lib; }, }: { parseOption = import ./test_parseOption.nix { inherit lib slib; }; diff --git a/lib/jsonschema/test_parseOption.nix b/lib/jsonschema/test_parseOption.nix index a20b329eb..8471ed432 100644 --- a/lib/jsonschema/test_parseOption.nix +++ b/lib/jsonschema/test_parseOption.nix @@ -1,21 +1,25 @@ # tests for the nixos options to jsonschema converter # run these tests via `nix-unit ./test.nix` -{ lib ? (import { }).lib -, slib ? import ./. { inherit lib; } +{ + lib ? (import { }).lib, + slib ? import ./. { inherit lib; }, }: let description = "Test Description"; - evalType = type: default: + evalType = + type: default: let evaledConfig = lib.evalModules { - modules = [{ - options.opt = lib.mkOption { - inherit type; - inherit default; - inherit description; - }; - }]; + modules = [ + { + options.opt = lib.mkOption { + inherit type; + inherit default; + inherit description; + }; + } + ]; }; in evaledConfig.options.opt; @@ -25,11 +29,7 @@ in testNoDefaultNoDescription = let evaledConfig = lib.evalModules { - modules = [{ - options.opt = lib.mkOption { - type = lib.types.bool; - }; - }]; + modules = [ { options.opt = lib.mkOption { type = lib.types.bool; }; } ]; }; in { @@ -42,15 +42,17 @@ in testDescriptionIsAttrs = let evaledConfig = lib.evalModules { - modules = [{ - options.opt = lib.mkOption { - type = lib.types.bool; - description = { - _type = "mdDoc"; - text = description; + modules = [ + { + options.opt = lib.mkOption { + type = lib.types.bool; + description = { + _type = "mdDoc"; + text = description; + }; }; - }; - }]; + } + ]; }; in { @@ -112,7 +114,11 @@ in testEnum = let default = "foo"; - values = [ "foo" "bar" "baz" ]; + values = [ + "foo" + "bar" + "baz" + ]; in { expr = slib.parseOption (evalType (lib.types.enum values) default); @@ -124,7 +130,11 @@ in testListOfInt = let - default = [ 1 2 3 ]; + default = [ + 1 + 2 + 3 + ]; in { expr = slib.parseOption (evalType (lib.types.listOf lib.types.int) default); @@ -139,14 +149,26 @@ in testListOfUnspecified = let - default = [ 1 2 3 ]; + default = [ + 1 + 2 + 3 + ]; in { expr = slib.parseOption (evalType (lib.types.listOf lib.types.unspecified) default); expected = { type = "array"; items = { - type = [ "boolean" "integer" "number" "string" "array" "object" "null" ]; + type = [ + "boolean" + "integer" + "number" + "string" + "array" + "object" + "null" + ]; }; inherit default description; }; @@ -154,7 +176,11 @@ in testAttrs = let - default = { foo = 1; bar = 2; baz = 3; }; + default = { + foo = 1; + bar = 2; + baz = 3; + }; in { expr = slib.parseOption (evalType (lib.types.attrs) default); @@ -167,7 +193,11 @@ in testAttrsOfInt = let - default = { foo = 1; bar = 2; baz = 3; }; + default = { + foo = 1; + bar = 2; + baz = 3; + }; in { expr = slib.parseOption (evalType (lib.types.attrsOf lib.types.int) default); @@ -182,7 +212,11 @@ in testLazyAttrsOfInt = let - default = { foo = 1; bar = 2; baz = 3; }; + default = { + foo = 1; + bar = 2; + baz = 3; + }; in { expr = slib.parseOption (evalType (lib.types.lazyAttrsOf lib.types.int) default); @@ -286,7 +320,10 @@ in inherit description; }; }; - default = { foo.opt = false; bar.opt = true; }; + default = { + foo.opt = false; + bar.opt = true; + }; in { expr = slib.parseOption (evalType (lib.types.attrsOf (lib.types.submodule subModule)) default); @@ -315,7 +352,10 @@ in inherit description; }; }; - default = [{ opt = false; } { opt = true; }]; + default = [ + { opt = false; } + { opt = true; } + ]; in { expr = slib.parseOption (evalType (lib.types.listOf (lib.types.submodule subModule)) default); @@ -358,7 +398,15 @@ in expr = slib.parseOption (evalType lib.types.anything default); expected = { inherit default description; - type = [ "boolean" "integer" "number" "string" "array" "object" "null" ]; + type = [ + "boolean" + "integer" + "number" + "string" + "array" + "object" + "null" + ]; }; }; @@ -370,7 +418,15 @@ in expr = slib.parseOption (evalType lib.types.unspecified default); expected = { inherit default description; - type = [ "boolean" "integer" "number" "string" "array" "object" "null" ]; + type = [ + "boolean" + "integer" + "number" + "string" + "array" + "object" + "null" + ]; }; }; @@ -382,7 +438,15 @@ in expr = slib.parseOption (evalType lib.types.raw default); expected = { inherit default description; - type = [ "boolean" "integer" "number" "string" "array" "object" "null" ]; + type = [ + "boolean" + "integer" + "number" + "string" + "array" + "object" + "null" + ]; }; }; } diff --git a/lib/jsonschema/test_parseOptions.nix b/lib/jsonschema/test_parseOptions.nix index c4564a7ea..1faf6e3b7 100644 --- a/lib/jsonschema/test_parseOptions.nix +++ b/lib/jsonschema/test_parseOptions.nix @@ -1,14 +1,13 @@ # tests for the nixos options to jsonschema converter # run these tests via `nix-unit ./test.nix` -{ lib ? (import { }).lib -, slib ? import ./. { inherit lib; } +{ + lib ? (import { }).lib, + slib ? import ./. { inherit lib; }, }: let evaledOptions = let - evaledConfig = lib.evalModules { - modules = [ ./example-interface.nix ]; - }; + evaledConfig = lib.evalModules { modules = [ ./example-interface.nix ]; }; in evaledConfig.options; in @@ -21,11 +20,7 @@ in testParseNestedOptions = let evaled = lib.evalModules { - modules = [{ - options.foo.bar = lib.mkOption { - type = lib.types.bool; - }; - }]; + modules = [ { options.foo.bar = lib.mkOption { type = lib.types.bool; }; } ]; }; in { @@ -34,7 +29,9 @@ in properties = { foo = { properties = { - bar = { type = "boolean"; }; + bar = { + type = "boolean"; + }; }; required = [ "bar" ]; type = "object"; diff --git a/machines/test_backup_client/facts/borgbackup.ssh.pub b/machines/test_backup_client/facts/borgbackup.ssh.pub new file mode 100644 index 000000000..c305404cd --- /dev/null +++ b/machines/test_backup_client/facts/borgbackup.ssh.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBIbwIVnLy+uoDZ6uK/OCc1QK46SIGeC3mVc85dqLYQw lass@ignavia diff --git a/nixosModules/clanCore/backups.nix b/nixosModules/clanCore/backups.nix index 44d6f4fe2..06a43744a 100644 --- a/nixosModules/clanCore/backups.nix +++ b/nixosModules/clanCore/backups.nix @@ -1,45 +1,48 @@ { lib, ... }: { - imports = [ - ./state.nix - ]; + imports = [ ./state.nix ]; options.clanCore.backups = { providers = lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule ({ name, ... }: { - options = { - name = lib.mkOption { - type = lib.types.str; - default = name; - description = '' - Name of the backup provider - ''; - }; - list = lib.mkOption { - type = lib.types.str; - description = '' - script to list backups - ''; - }; - restore = lib.mkOption { - type = lib.types.str; - description = '' - script to restore a backup - should take an optional service name as argument - gets ARCHIVE_ID, LOCATION, JOB and FOLDERS as environment variables - ARCHIVE_ID is the id of the backup - LOCATION is the remote identifier of the backup - JOB is the job name of the backup - FOLDERS is a colon separated list of folders to restore - ''; - }; - create = lib.mkOption { - type = lib.types.str; - description = '' - script to start a backup - ''; - }; - }; - })); + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + type = lib.types.str; + default = name; + description = '' + Name of the backup provider + ''; + }; + list = lib.mkOption { + type = lib.types.str; + description = '' + script to list backups + ''; + }; + restore = lib.mkOption { + type = lib.types.str; + description = '' + script to restore a backup + should take an optional service name as argument + gets ARCHIVE_ID, LOCATION, JOB and FOLDERS as environment variables + ARCHIVE_ID is the id of the backup + LOCATION is the remote identifier of the backup + JOB is the job name of the backup + FOLDERS is a colon separated list of folders to restore + ''; + }; + create = lib.mkOption { + type = lib.types.str; + description = '' + script to start a backup + ''; + }; + }; + } + ) + ); default = { }; description = '' Configured backup providers which are used by this machine diff --git a/nixosModules/clanCore/imports.nix b/nixosModules/clanCore/imports.nix index e89eaf0fc..959378eb4 100644 --- a/nixosModules/clanCore/imports.nix +++ b/nixosModules/clanCore/imports.nix @@ -1,6 +1,5 @@ -{ lib -, ... -}: { +{ lib, ... }: +{ /* Declaring imports inside the module system does not trigger an infinite recursion in this case because buildClan generates the imports from the diff --git a/nixosModules/clanCore/manual.nix b/nixosModules/clanCore/manual.nix index f63121385..e5b4f9a2e 100644 --- a/nixosModules/clanCore/manual.nix +++ b/nixosModules/clanCore/manual.nix @@ -1 +1,4 @@ -{ pkgs, ... }: { documentation.nixos.enable = pkgs.lib.mkDefault false; } +{ pkgs, ... }: +{ + documentation.nixos.enable = pkgs.lib.mkDefault false; +} diff --git a/nixosModules/clanCore/metadata.nix b/nixosModules/clanCore/metadata.nix index 391f32b86..d0bdf1455 100644 --- a/nixosModules/clanCore/metadata.nix +++ b/nixosModules/clanCore/metadata.nix @@ -1,4 +1,5 @@ -{ lib, pkgs, ... }: { +{ lib, pkgs, ... }: +{ options.clanCore = { clanName = lib.mkOption { type = lib.types.str; diff --git a/nixosModules/clanCore/networking.nix b/nixosModules/clanCore/networking.nix index affed63f8..a82a7469f 100644 --- a/nixosModules/clanCore/networking.nix +++ b/nixosModules/clanCore/networking.nix @@ -49,7 +49,18 @@ }; imports = [ - (lib.mkRenamedOptionModule [ "clan" "networking" "deploymentAddress" ] [ "clan" "networking" "targetHost" ]) + (lib.mkRenamedOptionModule + [ + "clan" + "networking" + "deploymentAddress" + ] + [ + "clan" + "networking" + "targetHost" + ] + ) ]; config = { # conflicts with systemd-resolved @@ -64,16 +75,18 @@ systemd.network.wait-online.enable = false; # Provide a default network configuration but don't compete with network-manager or dhcpcd - systemd.network.networks."50-uplink" = lib.mkIf (!(config.networking.networkmanager.enable || config.networking.dhcpcd.enable)) { - matchConfig.Type = "ether"; - networkConfig = { - DHCP = "yes"; - LLDP = "yes"; - LLMNR = "yes"; - MulticastDNS = "yes"; - IPv6AcceptRA = "yes"; - }; - }; + systemd.network.networks."50-uplink" = + lib.mkIf (!(config.networking.networkmanager.enable || config.networking.dhcpcd.enable)) + { + matchConfig.Type = "ether"; + networkConfig = { + DHCP = "yes"; + LLDP = "yes"; + LLMNR = "yes"; + MulticastDNS = "yes"; + IPv6AcceptRA = "yes"; + }; + }; # Use networkd instead of the pile of shell scripts networking.useNetworkd = lib.mkDefault true; diff --git a/nixosModules/clanCore/options.nix b/nixosModules/clanCore/options.nix index b20779995..68824b4b4 100644 --- a/nixosModules/clanCore/options.nix +++ b/nixosModules/clanCore/options.nix @@ -1,4 +1,10 @@ -{ pkgs, options, lib, ... }: { +{ + pkgs, + options, + lib, + ... +}: +{ options.clanCore.optionsNix = lib.mkOption { type = lib.types.raw; internal = true; diff --git a/nixosModules/clanCore/outputs.nix b/nixosModules/clanCore/outputs.nix index 75d2c063f..634357248 100644 --- a/nixosModules/clanCore/outputs.nix +++ b/nixosModules/clanCore/outputs.nix @@ -1,4 +1,10 @@ -{ config, lib, pkgs, ... }: { +{ + config, + lib, + pkgs, + ... +}: +{ # TODO: factor these out into a separate interface.nix. # Also think about moving these options out of `system.clan`. # Maybe we should not re-use the already polluted confg.system namespace @@ -90,6 +96,8 @@ inherit (config.clan.deployment) requireExplicitUpdate; inherit (config.clanCore) secretsUploadDirectory; }; - system.clan.deployment.file = pkgs.writeText "deployment.json" (builtins.toJSON config.system.clan.deployment.data); + system.clan.deployment.file = pkgs.writeText "deployment.json" ( + builtins.toJSON config.system.clan.deployment.data + ); }; } diff --git a/nixosModules/clanCore/packages.nix b/nixosModules/clanCore/packages.nix index 3481b3ecc..7771874ec 100644 --- a/nixosModules/clanCore/packages.nix +++ b/nixosModules/clanCore/packages.nix @@ -1,4 +1,5 @@ -{ pkgs, ... }: { +{ pkgs, ... }: +{ # essential debugging tools for networked services environment.systemPackages = [ pkgs.dnsutils diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index fae55c3a1..9f896b8c4 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -1,7 +1,17 @@ -{ config, lib, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: { options.clanCore.secretStore = lib.mkOption { - type = lib.types.enum [ "sops" "password-store" "vm" "custom" ]; + type = lib.types.enum [ + "sops" + "password-store" + "vm" + "custom" + ]; default = "sops"; description = '' method to store secrets @@ -34,8 +44,8 @@ options.clanCore.secrets = lib.mkOption { default = { }; - type = lib.types.attrsOf - (lib.types.submodule (service: { + type = lib.types.attrsOf ( + lib.types.submodule (service: { options = { name = lib.mkOption { type = lib.types.str; @@ -45,123 +55,138 @@ ''; }; generator = lib.mkOption { - type = lib.types.submodule ({ config, ... }: { - options = { - path = lib.mkOption { - type = lib.types.listOf (lib.types.either lib.types.path lib.types.package); - default = [ ]; - description = '' - Extra paths to add to the PATH environment variable when running the generator. - ''; - }; - prompt = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - description = '' - prompt text to ask for a value. - This value will be passed to the script as the environment variable $prompt_value. - ''; - }; - script = lib.mkOption { - type = lib.types.str; - description = '' - Script to generate the secret. - The script will be called with the following variables: - - facts: path to a directory where facts can be stored - - secrets: path to a directory where secrets can be stored - The script is expected to generate all secrets and facts defined in the module. - ''; - }; - finalScript = lib.mkOption { - type = lib.types.str; - readOnly = true; - internal = true; - default = '' - set -eu -o pipefail + type = lib.types.submodule ( + { config, ... }: + { + options = { + path = lib.mkOption { + type = lib.types.listOf (lib.types.either lib.types.path lib.types.package); + default = [ ]; + description = '' + Extra paths to add to the PATH environment variable when running the generator. + ''; + }; + prompt = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + prompt text to ask for a value. + This value will be passed to the script as the environment variable $prompt_value. + ''; + }; + script = lib.mkOption { + type = lib.types.str; + description = '' + Script to generate the secret. + The script will be called with the following variables: + - facts: path to a directory where facts can be stored + - secrets: path to a directory where secrets can be stored + The script is expected to generate all secrets and facts defined in the module. + ''; + }; + finalScript = lib.mkOption { + type = lib.types.str; + readOnly = true; + internal = true; + default = '' + set -eu -o pipefail - export PATH="${lib.makeBinPath config.path}:${pkgs.coreutils}/bin" + export PATH="${lib.makeBinPath config.path}:${pkgs.coreutils}/bin" - # prepare sandbox user - mkdir -p /etc - cp ${pkgs.runCommand "fake-etc" {} '' - export PATH="${pkgs.coreutils}/bin" - mkdir -p $out - cp /etc/* $out/ - ''}/* /etc/ + # prepare sandbox user + mkdir -p /etc + cp ${ + pkgs.runCommand "fake-etc" { } '' + export PATH="${pkgs.coreutils}/bin" + mkdir -p $out + cp /etc/* $out/ + '' + }/* /etc/ - ${config.script} - ''; + ${config.script} + ''; + }; }; - }; - }); + } + ); }; secrets = let config' = config; in lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule ({ config, name, ... }: { - options = { - name = lib.mkOption { - type = lib.types.str; - description = '' - name of the secret - ''; - default = name; - }; - path = lib.mkOption { - type = lib.types.str; - description = '' - path to a secret which is generated by the generator - ''; - default = "${config'.clanCore.secretsDirectory}/${config'.clanCore.secretsPrefix}${config.name}"; - }; - } // lib.optionalAttrs (config'.clanCore.secretStore == "sops") { - groups = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = config'.clanCore.sops.defaultGroups; - description = '' - Groups to decrypt the secret for. By default we always use the user's key. - ''; - }; - }; - })); + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule ( + { config, name, ... }: + { + options = + { + name = lib.mkOption { + type = lib.types.str; + description = '' + name of the secret + ''; + default = name; + }; + path = lib.mkOption { + type = lib.types.str; + description = '' + path to a secret which is generated by the generator + ''; + default = "${config'.clanCore.secretsDirectory}/${config'.clanCore.secretsPrefix}${config.name}"; + }; + } + // lib.optionalAttrs (config'.clanCore.secretStore == "sops") { + groups = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = config'.clanCore.sops.defaultGroups; + description = '' + Groups to decrypt the secret for. By default we always use the user's key. + ''; + }; + }; + } + ) + ); description = '' path where the secret is located in the filesystem ''; }; facts = lib.mkOption { default = { }; - type = lib.types.attrsOf (lib.types.submodule (fact: { - options = { - name = lib.mkOption { - type = lib.types.str; - description = '' - name of the fact - ''; - default = fact.config._module.args.name; + type = lib.types.attrsOf ( + lib.types.submodule (fact: { + options = { + name = lib.mkOption { + type = lib.types.str; + description = '' + name of the fact + ''; + default = fact.config._module.args.name; + }; + path = lib.mkOption { + type = lib.types.path; + description = '' + path to a fact which is generated by the generator + ''; + default = + config.clanCore.clanDir + + "/machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}"; + }; + value = lib.mkOption { + defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}"; + type = lib.types.nullOr lib.types.str; + default = + if builtins.pathExists fact.config.path then lib.strings.fileContents fact.config.path else null; + }; }; - path = lib.mkOption { - type = lib.types.str; - description = '' - path to a fact which is generated by the generator - ''; - default = "machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}"; - }; - value = lib.mkOption { - defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}"; - type = lib.types.nullOr lib.types.str; - default = - if builtins.pathExists "${config.clanCore.clanDir}/${fact.config.path}" then - lib.strings.removeSuffix "\n" (builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}") - else - null; - }; - }; - })); + }) + ); }; }; - })); + }) + ); }; imports = [ ./sops.nix diff --git a/nixosModules/clanCore/secrets/password-store.nix b/nixosModules/clanCore/secrets/password-store.nix index 529a6c24f..b3bf615d3 100644 --- a/nixosModules/clanCore/secrets/password-store.nix +++ b/nixosModules/clanCore/secrets/password-store.nix @@ -13,4 +13,3 @@ system.clan.secretsModule = "clan_cli.secrets.modules.password_store"; }; } - diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/clanCore/secrets/sops.nix index a5f627a57..d242c83ab 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/clanCore/secrets/sops.nix @@ -1,22 +1,33 @@ -{ config, lib, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: let secretsDir = config.clanCore.clanDir + "/sops/secrets"; groupsDir = config.clanCore.clanDir + "/sops/groups"; - # My symlink is in the nixos module detected as a directory also it works in the repl. Is this because of pure evaluation? - containsSymlink = path: - builtins.pathExists path && (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink"); + containsSymlink = + path: + builtins.pathExists path + && (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink"); - containsMachine = parent: name: type: + containsMachine = + parent: name: type: type == "directory" && containsSymlink "${parent}/${name}/machines/${config.clanCore.machineName}"; - containsMachineOrGroups = name: type: - (containsMachine secretsDir name type) || lib.any (group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}") groups; + containsMachineOrGroups = + name: type: + (containsMachine secretsDir name type) + || lib.any ( + group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}" + ) groups; - filterDir = filter: dir: - lib.optionalAttrs (builtins.pathExists dir) - (lib.filterAttrs filter (builtins.readDir dir)); + filterDir = + filter: dir: + lib.optionalAttrs (builtins.pathExists dir) (lib.filterAttrs filter (builtins.readDir dir)); groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir); secrets = filterDir containsMachineOrGroups secretsDir; @@ -34,17 +45,18 @@ in clanCore.secretsDirectory = "/run/secrets"; clanCore.secretsPrefix = config.clanCore.machineName + "-"; system.clan.secretsModule = "clan_cli.secrets.modules.sops"; - sops.secrets = builtins.mapAttrs - (name: _: { - sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret"; - format = "binary"; - }) - secrets; + sops.secrets = builtins.mapAttrs (name: _: { + sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret"; + format = "binary"; + }) secrets; # To get proper error messages about missing secrets we need a dummy secret file that is always present - sops.defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" ""))); + sops.defaultSopsFile = lib.mkIf config.sops.validateSopsFiles ( + lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" "")) + ); - sops.age.keyFile = lib.mkIf (builtins.pathExists (config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret")) - (lib.mkDefault "/var/lib/sops-nix/key.txt"); + sops.age.keyFile = lib.mkIf (builtins.pathExists ( + config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret" + )) (lib.mkDefault "/var/lib/sops-nix/key.txt"); clanCore.secretsUploadDirectory = lib.mkDefault "/var/lib/sops-nix"; }; } diff --git a/nixosModules/clanCore/secrets/vm.nix b/nixosModules/clanCore/secrets/vm.nix index ce071dd29..1622c5cf2 100644 --- a/nixosModules/clanCore/secrets/vm.nix +++ b/nixosModules/clanCore/secrets/vm.nix @@ -7,4 +7,3 @@ system.clan.factsModule = "clan_cli.facts.modules.vm"; }; } - diff --git a/nixosModules/clanCore/state.nix b/nixosModules/clanCore/state.nix index f2ecf6591..50bc80a13 100644 --- a/nixosModules/clanCore/state.nix +++ b/nixosModules/clanCore/state.nix @@ -1,41 +1,43 @@ { lib, ... }: { # defaults - config.clanCore.state.HOME.folders = [ - "/home" - ]; + config.clanCore.state.HOME.folders = [ "/home" ]; # interface options.clanCore.state = lib.mkOption { default = { }; - type = lib.types.attrsOf - (lib.types.submodule ({ ... }: { - options = { - folders = lib.mkOption { - type = lib.types.listOf lib.types.str; - description = '' - Folder where state resides in - ''; - }; - preRestoreScript = lib.mkOption { - type = lib.types.str; - default = ":"; - description = '' - script to run before restoring the state dir from a backup + type = lib.types.attrsOf ( + lib.types.submodule ( + { ... }: + { + options = { + folders = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + Folder where state resides in + ''; + }; + preRestoreScript = lib.mkOption { + type = lib.types.str; + default = ":"; + description = '' + script to run before restoring the state dir from a backup - Utilize this to stop services which currently access these folders - ''; - }; - postRestoreScript = lib.mkOption { - type = lib.types.str; - default = ":"; - description = '' - script to restore the service after the state dir was restored from a backup + Utilize this to stop services which currently access these folders + ''; + }; + postRestoreScript = lib.mkOption { + type = lib.types.str; + default = ":"; + description = '' + script to restore the service after the state dir was restored from a backup - Utilize this to start services which were previously stopped - ''; + Utilize this to start services which were previously stopped + ''; + }; }; - }; - })); + } + ) + ); }; } diff --git a/nixosModules/clanCore/vm.nix b/nixosModules/clanCore/vm.nix index 89a23e9f7..62d98d210 100644 --- a/nixosModules/clanCore/vm.nix +++ b/nixosModules/clanCore/vm.nix @@ -1,12 +1,15 @@ -{ lib, config, pkgs, options, extendModules, modulesPath, ... }: +{ + lib, + config, + pkgs, + options, + extendModules, + modulesPath, + ... +}: let # Flatten the list of state folders into a single list - stateFolders = lib.flatten ( - lib.mapAttrsToList - (_item: attrs: attrs.folders) - config.clanCore.state - ); - + stateFolders = lib.flatten (lib.mapAttrsToList (_item: attrs: attrs.folders) config.clanCore.state); vmModule = { imports = [ @@ -24,12 +27,18 @@ let services.acpid.handlers.power.event = "button/power.*"; services.acpid.handlers.power.action = "poweroff"; + # only works on x11 + services.spice-vdagentd.enable = config.services.xserver.enable; + boot.initrd.systemd.enable = true; # currently needed for system.etc.overlay.enable boot.kernelPackages = pkgs.linuxPackages_latest; - boot.initrd.systemd.storePaths = [ pkgs.util-linux pkgs.e2fsprogs ]; + boot.initrd.systemd.storePaths = [ + pkgs.util-linux + pkgs.e2fsprogs + ]; boot.initrd.systemd.emergencyAccess = true; # sysusers is faster than nixos's perl scripts @@ -40,50 +49,72 @@ let boot.initrd.kernelModules = [ "virtiofs" ]; virtualisation.writableStore = false; - virtualisation.fileSystems = lib.mkForce ({ - "/nix/store" = { - device = "nix-store"; - options = [ "x-systemd.requires=systemd-modules-load.service" "ro" ]; - fsType = "virtiofs"; - }; + virtualisation.fileSystems = lib.mkForce ( + { + "/nix/store" = { + device = "nix-store"; + options = [ + "x-systemd.requires=systemd-modules-load.service" + "ro" + ]; + fsType = "virtiofs"; + }; - "/" = { - device = "/dev/vda"; - fsType = "ext4"; - options = [ "defaults" "x-systemd.makefs" "nobarrier" "noatime" "nodiratime" "data=writeback" "discard" ]; - }; + "/" = { + device = "/dev/vda"; + fsType = "ext4"; + options = [ + "defaults" + "x-systemd.makefs" + "nobarrier" + "noatime" + "nodiratime" + "data=writeback" + "discard" + ]; + }; - "/vmstate" = { - device = "/dev/vdb"; - options = [ "x-systemd.makefs" "noatime" "nodiratime" "discard" ]; - noCheck = true; - fsType = "ext4"; - }; + "/vmstate" = { + device = "/dev/vdb"; + options = [ + "x-systemd.makefs" + "noatime" + "nodiratime" + "discard" + ]; + noCheck = true; + fsType = "ext4"; + }; - ${config.clanCore.secretsUploadDirectory} = { - device = "secrets"; - fsType = "9p"; - neededForBoot = true; - options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ]; - }; - - } // lib.listToAttrs (map - (folder: - lib.nameValuePair folder { - device = "/vmstate${folder}"; - fsType = "none"; - options = [ "bind" ]; - }) - stateFolders)); + ${config.clanCore.secretsUploadDirectory} = { + device = "secrets"; + fsType = "9p"; + neededForBoot = true; + options = [ + "trans=virtio" + "version=9p2000.L" + "cache=loose" + ]; + }; + } + // lib.listToAttrs ( + map ( + folder: + lib.nameValuePair folder { + device = "/vmstate${folder}"; + fsType = "none"; + options = [ "bind" ]; + } + ) stateFolders + ) + ); }; # We cannot simply merge the VM config into the current system config, because # it is not necessarily a VM. # Instead we use extendModules to create a second instance of the current # system configuration, and then merge the VM config into that. - vmConfig = extendModules { - modules = [ vmModule ]; - }; + vmConfig = extendModules { modules = [ vmModule ]; }; in { options = { @@ -207,12 +238,14 @@ in }; # for clan vm create system.clan.vm = { - create = pkgs.writeText "vm.json" (builtins.toJSON { - initrd = "${vmConfig.config.system.build.initialRamdisk}/${vmConfig.config.system.boot.loader.initrdFile}"; - toplevel = vmConfig.config.system.build.toplevel; - regInfo = (pkgs.closureInfo { rootPaths = vmConfig.config.virtualisation.additionalPaths; }); - inherit (config.clan.virtualisation) memorySize cores graphics; - }); + create = pkgs.writeText "vm.json" ( + builtins.toJSON { + initrd = "${vmConfig.config.system.build.initialRamdisk}/${vmConfig.config.system.boot.loader.initrdFile}"; + toplevel = vmConfig.config.system.build.toplevel; + regInfo = (pkgs.closureInfo { rootPaths = vmConfig.config.virtualisation.additionalPaths; }); + inherit (config.clan.virtualisation) memorySize cores graphics; + } + ); }; virtualisation = lib.optionalAttrs (options.virtualisation ? cores) { diff --git a/nixosModules/clanCore/wayland-proxy-virtwl.nix b/nixosModules/clanCore/wayland-proxy-virtwl.nix index d44d4753a..0c52fa6a5 100644 --- a/nixosModules/clanCore/wayland-proxy-virtwl.nix +++ b/nixosModules/clanCore/wayland-proxy-virtwl.nix @@ -1,4 +1,9 @@ -{ pkgs, config, lib, ... }: +{ + pkgs, + config, + lib, + ... +}: { options = { # maybe upstream this? diff --git a/nixosModules/clanCore/zerotier/default.nix b/nixosModules/clanCore/zerotier/default.nix index 719635aac..495394d2b 100644 --- a/nixosModules/clanCore/zerotier/default.nix +++ b/nixosModules/clanCore/zerotier/default.nix @@ -1,4 +1,9 @@ -{ config, lib, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: let cfg = config.clan.networking.zerotier; facts = config.clanCore.secrets.zerotier.facts or { }; @@ -76,16 +81,18 @@ in }; settings = lib.mkOption { description = lib.mdDoc "override the network config in /var/lib/zerotier/bla/$network.json"; - type = lib.types.submodule { - freeformType = (pkgs.formats.json { }).type; - }; + type = lib.types.submodule { freeformType = (pkgs.formats.json { }).type; }; }; }; config = lib.mkMerge [ ({ # Override license so that we can build zerotierone without # having to re-import nixpkgs. - services.zerotierone.package = lib.mkDefault (pkgs.zerotierone.overrideAttrs (_old: { meta = { }; })); + services.zerotierone.package = lib.mkDefault ( + pkgs.zerotierone.overrideAttrs (_old: { + meta = { }; + }) + ); }) (lib.mkIf ((facts.zerotier-ip.value or null) != null) { environment.etc."zerotier/ip".text = facts.zerotier-ip.value; @@ -104,29 +111,33 @@ in systemd.services.zerotierone.serviceConfig.ExecStartPre = [ "+${pkgs.writeShellScript "init-zerotier" '' - cp ${config.clanCore.secrets.zerotier.secrets.zerotier-identity-secret.path} /var/lib/zerotier-one/identity.secret - zerotier-idtool getpublic /var/lib/zerotier-one/identity.secret > /var/lib/zerotier-one/identity.public + cp ${config.clanCore.secrets.zerotier.secrets.zerotier-identity-secret.path} /var/lib/zerotier-one/identity.secret + zerotier-idtool getpublic /var/lib/zerotier-one/identity.secret > /var/lib/zerotier-one/identity.public - ${lib.optionalString (cfg.controller.enable) '' - mkdir -p /var/lib/zerotier-one/controller.d/network - ln -sfT ${pkgs.writeText "net.json" (builtins.toJSON cfg.settings)} /var/lib/zerotier-one/controller.d/network/${cfg.networkId}.json - ''} - ${lib.optionalString (cfg.moon.stableEndpoints != []) '' - if [[ ! -f /var/lib/zerotier-one/moon.json ]]; then - zerotier-idtool initmoon /var/lib/zerotier-one/identity.public > /var/lib/zerotier-one/moon.json - fi - ${genMoonScript}/bin/genmoon /var/lib/zerotier-one/moon.json ${builtins.toFile "moon.json" (builtins.toJSON cfg.moon.stableEndpoints)} /var/lib/zerotier-one/moons.d - ''} + ${lib.optionalString (cfg.controller.enable) '' + mkdir -p /var/lib/zerotier-one/controller.d/network + ln -sfT ${pkgs.writeText "net.json" (builtins.toJSON cfg.settings)} /var/lib/zerotier-one/controller.d/network/${cfg.networkId}.json + ''} + ${lib.optionalString (cfg.moon.stableEndpoints != [ ]) '' + if [[ ! -f /var/lib/zerotier-one/moon.json ]]; then + zerotier-idtool initmoon /var/lib/zerotier-one/identity.public > /var/lib/zerotier-one/moon.json + fi + ${genMoonScript}/bin/genmoon /var/lib/zerotier-one/moon.json ${builtins.toFile "moon.json" (builtins.toJSON cfg.moon.stableEndpoints)} /var/lib/zerotier-one/moons.d + ''} - # cleanup old networks - if [[ -d /var/lib/zerotier-one/networks.d ]]; then - find /var/lib/zerotier-one/networks.d \ - -type f \ - -name "*.conf" \ - -not \( ${lib.concatMapStringsSep " -o " (netId: ''-name "${netId}.conf"'') config.services.zerotierone.joinNetworks} \) \ - -delete - fi - ''}" + # cleanup old networks + if [[ -d /var/lib/zerotier-one/networks.d ]]; then + find /var/lib/zerotier-one/networks.d \ + -type f \ + -name "*.conf" \ + -not \( ${ + lib.concatMapStringsSep " -o " ( + netId: ''-name "${netId}.conf"'' + ) config.services.zerotierone.joinNetworks + } \) \ + -delete + fi + ''}" ]; systemd.services.zerotierone.serviceConfig.ExecStartPost = [ "+${pkgs.writeShellScript "configure-interface" '' @@ -145,7 +156,7 @@ in ${lib.concatMapStringsSep "\n" (moon: '' zerotier-cli orbit ${moon} ${moon} '') cfg.moon.orbitMoons} - ''}" + ''}" ]; networking.firewall.interfaces."zt+".allowedTCPPorts = [ 5353 ]; # mdns @@ -172,7 +183,11 @@ in facts.zerotier-ip = { }; facts.zerotier-network-id = { }; secrets.zerotier-identity-secret = { }; - generator.path = [ config.services.zerotierone.package pkgs.fakeroot pkgs.python3 ]; + generator.path = [ + config.services.zerotierone.package + pkgs.fakeroot + pkgs.python3 + ]; generator.script = '' python3 ${./generate.py} --mode network \ --ip "$facts/zerotier-ip" \ @@ -188,7 +203,10 @@ in clanCore.secrets.zerotier = { facts.zerotier-ip = { }; secrets.zerotier-identity-secret = { }; - generator.path = [ config.services.zerotierone.package pkgs.python3 ]; + generator.path = [ + config.services.zerotierone.package + pkgs.python3 + ]; generator.script = '' python3 ${./generate.py} --mode identity \ --ip "$facts/zerotier-ip" \ @@ -200,9 +218,7 @@ in (lib.mkIf (cfg.controller.enable && (facts.zerotier-network-id.value or null) != null) { clan.networking.zerotier.networkId = facts.zerotier-network-id.value; clan.networking.zerotier.settings = { - authTokens = [ - null - ]; + authTokens = [ null ]; authorizationEndpoint = ""; capabilities = [ ]; clientId = ""; @@ -242,7 +258,9 @@ in environment.etc."zerotier/network-id".text = facts.zerotier-network-id.value; systemd.services.zerotierone.serviceConfig.ExecStartPost = [ "+${pkgs.writeShellScript "whitelist-controller" '' - ${config.clanCore.clanPkgs.zerotier-members}/bin/zerotier-members allow ${builtins.substring 0 10 cfg.networkId} + ${config.clanCore.clanPkgs.zerotier-members}/bin/zerotier-members allow ${ + builtins.substring 0 10 cfg.networkId + } ''}" ]; }) diff --git a/nixosModules/flake-module.nix b/nixosModules/flake-module.nix index 4fb480ea9..e60126947 100644 --- a/nixosModules/flake-module.nix +++ b/nixosModules/flake-module.nix @@ -1,14 +1,22 @@ -{ inputs, self, ... }: { +{ inputs, self, ... }: +{ flake.nixosModules = { hidden-ssh-announce.imports = [ ./hidden-ssh-announce.nix ]; - installer.imports = [ ./installer ]; + installer.imports = [ + ./installer + self.nixosModules.hidden-ssh-announce + inputs.disko.nixosModules.disko + ]; clanCore.imports = [ inputs.sops-nix.nixosModules.sops ./clanCore ./iso - ({ pkgs, lib, ... }: { - clanCore.clanPkgs = lib.mkDefault self.packages.${pkgs.hostPlatform.system}; - }) + ( + { pkgs, lib, ... }: + { + clanCore.clanPkgs = lib.mkDefault self.packages.${pkgs.hostPlatform.system}; + } + ) ]; }; } diff --git a/nixosModules/hidden-ssh-announce.nix b/nixosModules/hidden-ssh-announce.nix index cca89a779..41ff88c84 100644 --- a/nixosModules/hidden-ssh-announce.nix +++ b/nixosModules/hidden-ssh-announce.nix @@ -1,8 +1,10 @@ -{ config -, lib -, pkgs -, ... -}: { +{ + config, + lib, + pkgs, + ... +}: +{ options.hidden-ssh-announce = { enable = lib.mkEnableOption "hidden-ssh-announce"; script = lib.mkOption { @@ -32,8 +34,14 @@ }; systemd.services.hidden-ssh-announce = { description = "announce hidden ssh"; - after = [ "tor.service" "network-online.target" ]; - wants = [ "tor.service" "network-online.target" ]; + after = [ + "tor.service" + "network-online.target" + ]; + wants = [ + "tor.service" + "network-online.target" + ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { # ${pkgs.tor}/bin/torify diff --git a/nixosModules/installer/default.nix b/nixosModules/installer/default.nix index 379fbcd57..842e50728 100644 --- a/nixosModules/installer/default.nix +++ b/nixosModules/installer/default.nix @@ -1,11 +1,11 @@ -{ lib -, pkgs -, modulesPath -, ... -}: { - systemd.tmpfiles.rules = [ - "d /var/shared 0777 root root - -" - ]; +{ + lib, + pkgs, + modulesPath, + ... +}: +{ + systemd.tmpfiles.rules = [ "d /var/shared 0777 root root - -" ]; imports = [ (modulesPath + "/profiles/installation-device.nix") (modulesPath + "/profiles/all-hardware.nix") @@ -21,7 +21,17 @@ enable = true; script = pkgs.writeShellScript "write-hostname" '' set -efu - export PATH=${lib.makeBinPath (with pkgs; [ iproute2 coreutils jq qrencode ])} + export PATH=${ + lib.makeBinPath ( + with pkgs; + [ + iproute2 + coreutils + jq + qrencode + ] + ) + } mkdir -p /var/shared echo "$1" > /var/shared/onion-hostname @@ -37,7 +47,7 @@ }; services.getty.autologinUser = lib.mkForce "root"; programs.bash.interactiveShellInit = '' - if [ "$(tty)" = "/dev/tty1" ]; then + if [[ "$(tty)" =~ /dev/(tty1|hvc0|ttyS0)$ ]]; then echo -n 'waiting for tor to generate the hidden service' until test -e /var/shared/qrcode.utf8; do echo -n .; sleep 1; done echo @@ -48,8 +58,13 @@ cat /var/shared/qrcode.utf8 fi ''; - boot.loader.grub.efiInstallAsRemovable = true; - boot.loader.grub.efiSupport = true; + + boot.loader.systemd-boot.enable = true; + + # Grub doesn't find devices for both BIOS and UEFI? + + #boot.loader.grub.efiInstallAsRemovable = true; + #boot.loader.grub.efiSupport = true; disko.devices = { disk = { stick = { @@ -59,10 +74,10 @@ content = { type = "gpt"; partitions = { - boot = { - size = "1M"; - type = "EF02"; # for grub MBR - }; + #boot = { + # size = "1M"; + # type = "EF02"; # for grub MBR + #}; ESP = { size = "100M"; type = "EF00"; diff --git a/nixosModules/iso/default.nix b/nixosModules/iso/default.nix index 85f48102c..db3afe562 100644 --- a/nixosModules/iso/default.nix +++ b/nixosModules/iso/default.nix @@ -1,4 +1,10 @@ -{ config, extendModules, lib, pkgs, ... }: +{ + config, + extendModules, + lib, + pkgs, + ... +}: let # Generates a fileSystems entry for bind mounting a given state folder path # It binds directories from /var/clanstate/{some-path} to /{some-path}. @@ -13,54 +19,47 @@ let }; # Flatten the list of state folders into a single list - stateFolders = lib.flatten ( - lib.mapAttrsToList - (_item: attrs: attrs.folders) - config.clanCore.state - ); + stateFolders = lib.flatten (lib.mapAttrsToList (_item: attrs: attrs.folders) config.clanCore.state); # A module setting up bind mounts for all state folders stateMounts = { - fileSystems = - lib.listToAttrs - (map mkBindMount stateFolders); + fileSystems = lib.listToAttrs (map mkBindMount stateFolders); }; - isoModule = { config, ... }: { - imports = [ - stateMounts - ]; - options.clan.iso.disko = lib.mkOption { - type = lib.types.submodule { - freeformType = (pkgs.formats.json { }).type; - }; - default = { - disk = { - iso = { - type = "disk"; - imageSize = "10G"; # TODO add auto image size in disko - content = { - type = "gpt"; - partitions = { - boot = { - size = "1M"; - type = "EF02"; # for grub MBR - }; - ESP = { - size = "100M"; - type = "EF00"; - content = { - type = "filesystem"; - format = "vfat"; - mountpoint = "/boot"; + isoModule = + { config, ... }: + { + imports = [ stateMounts ]; + options.clan.iso.disko = lib.mkOption { + type = lib.types.submodule { freeformType = (pkgs.formats.json { }).type; }; + default = { + disk = { + iso = { + type = "disk"; + imageSize = "10G"; # TODO add auto image size in disko + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; # for grub MBR }; - }; - root = { - size = "100%"; - content = { - type = "filesystem"; - format = "ext4"; - mountpoint = "/"; + ESP = { + size = "100M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + }; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; }; }; }; @@ -68,19 +67,16 @@ let }; }; }; + config = { + disko.devices = lib.mkOverride 51 config.clan.iso.disko; + boot.loader.grub.enable = true; + boot.loader.grub.efiSupport = true; + boot.loader.grub.device = lib.mkForce "/dev/vda"; + boot.loader.grub.efiInstallAsRemovable = true; + }; }; - config = { - disko.devices = lib.mkOverride 51 config.clan.iso.disko; - boot.loader.grub.enable = true; - boot.loader.grub.efiSupport = true; - boot.loader.grub.device = lib.mkForce "/dev/vda"; - boot.loader.grub.efiInstallAsRemovable = true; - }; - }; - isoConfig = extendModules { - modules = [ isoModule ]; - }; + isoConfig = extendModules { modules = [ isoModule ]; }; in { config = { diff --git a/pkgs/clan-cli/clan_cli/clan_uri.py b/pkgs/clan-cli/clan_cli/clan_uri.py index 184f9c9fc..9cb387cb8 100644 --- a/pkgs/clan-cli/clan_cli/clan_uri.py +++ b/pkgs/clan-cli/clan_cli/clan_uri.py @@ -3,167 +3,171 @@ import dataclasses import urllib.parse import urllib.request from dataclasses import dataclass -from enum import Enum, member from pathlib import Path -from typing import Any, Self +from typing import Any from .errors import ClanError -# Define an enum with different members that have different values -class ClanScheme(Enum): - # Use the dataclass decorator to add fields and methods to the members - @member - @dataclass - class REMOTE: - url: str # The url field holds the HTTP URL +@dataclass +class FlakeId: + _value: str | Path - def __str__(self) -> str: - return f"REMOTE({self.url})" # The __str__ method returns a custom string representation + def __str__(self) -> str: + return f"{self._value}" # The __str__ method returns a custom string representation - @member - @dataclass - class LOCAL: - path: Path # The path field holds the local path + @property + def path(self) -> Path: + assert isinstance(self._value, Path) + return self._value - def __str__(self) -> str: - return f"LOCAL({self.path})" # The __str__ method returns a custom string representation + @property + def url(self) -> str: + assert isinstance(self._value, str) + return self._value + + def __repr__(self) -> str: + return f"ClanUrl({self._value})" + + def is_local(self) -> bool: + return isinstance(self._value, Path) + + def is_remote(self) -> bool: + return isinstance(self._value, str) # Parameters defined here will be DELETED from the nested uri # so make sure there are no conflicts with other webservices @dataclass -class ClanParameters: - flake_attr: str = "defaultVM" +class MachineParams: + dummy_opt: str = "dummy" + + +@dataclass +class MachineData: + flake_id: FlakeId + name: str = "defaultVM" + params: MachineParams = dataclasses.field(default_factory=MachineParams) + + def get_id(self) -> str: + return f"{self.flake_id}#{self.name}" # Define the ClanURI class class ClanURI: + _orig_uri: str + _components: urllib.parse.ParseResult + flake_id: FlakeId + _machines: list[MachineData] + # Initialize the class with a clan:// URI def __init__(self, uri: str) -> None: + self._machines = [] + # users might copy whitespace along with the uri uri = uri.strip() - self._full_uri = uri + self._orig_uri = uri # Check if the URI starts with clan:// # If it does, remove the clan:// prefix if uri.startswith("clan://"): - self._nested_uri = uri[7:] + nested_uri = uri[7:] else: - raise ClanError(f"Invalid scheme: expected clan://, got {uri}") + raise ClanError(f"Invalid uri: expected clan://, got {uri}") # Parse the URI into components - # scheme://netloc/path;parameters?query#fragment - self._components = urllib.parse.urlparse(self._nested_uri) + # url://netloc/path;parameters?query#fragment + self._components = urllib.parse.urlparse(nested_uri) - # Parse the query string into a dictionary - query = urllib.parse.parse_qs(self._components.query) + # Replace the query string in the components with the new query string + clean_comps = self._components._replace( + query=self._components.query, fragment="" + ) - # Create a new dictionary with only the parameters we want - # example: https://example.com?flake_attr=myVM&password=1234 - # becomes: https://example.com?password=1234 - # clan_params = {"flake_attr": "myVM"} - # query = {"password": ["1234"]} - clan_params: dict[str, str] = {} - for field in dataclasses.fields(ClanParameters): - if field.name in query: - values = query[field.name] + # Parse the URL into a ClanUrl object + self.flake_id = self._parse_url(clean_comps) + + # Parse the fragment into a list of machine queries + # Then parse every machine query into a MachineParameters object + machine_frags = list( + filter(lambda x: len(x) > 0, self._components.fragment.split("#")) + ) + for machine_frag in machine_frags: + machine = self._parse_machine_query(machine_frag) + self._machines.append(machine) + + # If there are no machine fragments, add a default machine + if len(machine_frags) == 0: + default_machine = MachineData(flake_id=self.flake_id) + self._machines.append(default_machine) + + def _parse_url(self, comps: urllib.parse.ParseResult) -> FlakeId: + comb = ( + comps.scheme, + comps.netloc, + comps.path, + comps.params, + comps.query, + comps.fragment, + ) + match comb: + case ("file", "", path, "", "", _) | ("", "", path, "", "", _): # type: ignore + flake_id = FlakeId(Path(path).expanduser().resolve()) + case _: + flake_id = FlakeId(comps.geturl()) + + return flake_id + + def _parse_machine_query(self, machine_frag: str) -> MachineData: + comp = urllib.parse.urlparse(machine_frag) + query = urllib.parse.parse_qs(comp.query) + machine_name = comp.path + + machine_params: dict[str, Any] = {} + for dfield in dataclasses.fields(MachineParams): + if dfield.name in query: + values = query[dfield.name] if len(values) > 1: - raise ClanError(f"Multiple values for parameter: {field.name}") - clan_params[field.name] = values[0] + raise ClanError(f"Multiple values for parameter: {dfield.name}") + machine_params[dfield.name] = values[0] # Remove the field from the query dictionary # clan uri and nested uri share one namespace for query parameters # we need to make sure there are no conflicts - del query[field.name] - # Reencode the query dictionary into a query string - real_query = urllib.parse.urlencode(query, doseq=True) + del query[dfield.name] + params = MachineParams(**machine_params) + machine = MachineData(flake_id=self.flake_id, name=machine_name, params=params) + return machine - # If the fragment contains a #, use the part after the # as the flake_attr - # on multiple #, use the first one - if self._components.fragment != "": - clan_params["flake_attr"] = self._components.fragment.split("#")[0] + @property + def machine(self) -> MachineData: + return self._machines[0] - # Replace the query string in the components with the new query string - self._components = self._components._replace(query=real_query, fragment="") + def get_orig_uri(self) -> str: + return self._orig_uri - # Create a ClanParameters object from the clan_params dictionary - self.params = ClanParameters(**clan_params) - - comb = ( - self._components.scheme, - self._components.netloc, - self._components.path, - self._components.params, - self._components.query, - self._components.fragment, - ) - match comb: - case ("file", "", path, "", "", "") | ("", "", path, "", "", _): # type: ignore - self.scheme = ClanScheme.LOCAL.value(Path(path).expanduser().resolve()) # type: ignore - case _: - self.scheme = ClanScheme.REMOTE.value(self._components.geturl()) # type: ignore - - def get_internal(self) -> str: - match self.scheme: - case ClanScheme.LOCAL.value(path): - return str(path) - case ClanScheme.REMOTE.value(url): - return url - case _: - raise ClanError(f"Unsupported uri components: {self.scheme}") - - def get_full_uri(self) -> str: - return self._full_uri - - def get_id(self) -> str: - return f"{self.get_internal()}#{self.params.flake_attr}" - - @classmethod - def from_path( - cls, # noqa - path: Path, - flake_attr: str | None = None, - params: dict[str, Any] | ClanParameters | None = None, - ) -> Self: - return cls.from_str(str(path), flake_attr=flake_attr, params=params) + def get_url(self) -> str: + return str(self.flake_id) @classmethod def from_str( cls, # noqa url: str, - flake_attr: str | None = None, - params: dict[str, Any] | ClanParameters | None = None, - ) -> Self: - if flake_attr is not None and params is not None: - raise ClanError("flake_attr and params are mutually exclusive") + machine_name: str | None = None, + ) -> "ClanURI": + clan_uri = "" + if not url.startswith("clan://"): + clan_uri += "clan://" - prefix = "clan://" - if url.startswith(prefix): - url = url[len(prefix) :] + clan_uri += url - if params is None and flake_attr is None: - return cls(f"clan://{url}") + if machine_name: + clan_uri += f"#{machine_name}" - comp = urllib.parse.urlparse(url) - query = urllib.parse.parse_qs(comp.query) - - if isinstance(params, dict): - query.update(params) - elif isinstance(params, ClanParameters): - query.update(params.__dict__) - elif flake_attr is not None: - query["flake_attr"] = [flake_attr] - else: - raise ClanError(f"Unsupported params type: {type(params)}") - - new_query = urllib.parse.urlencode(query, doseq=True) - comp = comp._replace(query=new_query) - new_url = urllib.parse.urlunparse(comp) - return cls(f"clan://{new_url}") + return cls(clan_uri) def __str__(self) -> str: - return self.get_full_uri() + return self.get_orig_uri() def __repr__(self) -> str: - return f"ClanURI({self.get_full_uri()})" + return f"ClanURI({self})" diff --git a/pkgs/clan-cli/clan_cli/cmd.py b/pkgs/clan-cli/clan_cli/cmd.py index 932d8477d..09f71ffb4 100644 --- a/pkgs/clan-cli/clan_cli/cmd.py +++ b/pkgs/clan-cli/clan_cli/cmd.py @@ -4,7 +4,8 @@ import select import shlex import subprocess import sys -from datetime import datetime +import weakref +from datetime import datetime, timedelta from enum import Enum from pathlib import Path from typing import IO, Any @@ -58,6 +59,45 @@ def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]: return stdout_buf.decode("utf-8"), stderr_buf.decode("utf-8") +class TimeTable: + """ + This class is used to store the time taken by each command + and print it at the end of the program if env PERF=1 is set. + """ + + def __init__(self) -> None: + self.table: dict[str, timedelta] = {} + weakref.finalize(self, self.table_print) + + def table_print(self) -> None: + if os.getenv("PERF") != "1": + return + print("======== CMD TIMETABLE ========") + + # Sort the table by time in descending order + sorted_table = sorted( + self.table.items(), key=lambda item: item[1], reverse=True + ) + + for k, v in sorted_table: + # Check if timedelta is greater than 1 second + if v.total_seconds() > 1: + # Print in red + print(f"\033[91mTook {v}s\033[0m for command: '{k}'") + else: + # Print in default color + print(f"Took {v} for command: '{k}'") + + def add(self, cmd: str, time: timedelta) -> None: + if cmd in self.table: + self.table[cmd] += time + else: + self.table[cmd] = time + + +TIME_TABLE = TimeTable() + + def run( cmd: list[str], *, @@ -83,7 +123,8 @@ def run( rc = process.wait() tend = datetime.now() - glog.debug(f"Command took {tend - tstart}s to run") + global TIME_TABLE + TIME_TABLE.add(shlex.join(cmd), tend - tstart) # Wait for the subprocess to finish cmd_out = CmdOut( diff --git a/pkgs/clan-cli/clan_cli/flakes/inspect.py b/pkgs/clan-cli/clan_cli/flakes/inspect.py index 9897ed82c..8d9f4e2e0 100644 --- a/pkgs/clan-cli/clan_cli/flakes/inspect.py +++ b/pkgs/clan-cli/clan_cli/flakes/inspect.py @@ -86,7 +86,6 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig: # Get the flake metadata meta = nix_metadata(flake_url) - return FlakeConfig( vm=vm, flake_url=flake_url, diff --git a/pkgs/clan-cli/clan_cli/flash.py b/pkgs/clan-cli/clan_cli/flash.py index e99edb769..d15df36bc 100644 --- a/pkgs/clan-cli/clan_cli/flash.py +++ b/pkgs/clan-cli/clan_cli/flash.py @@ -1,51 +1,110 @@ import argparse import importlib import logging +import os +import shlex +import shutil +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory +from typing import Any +from .cmd import Log, run +from .errors import ClanError from .machines.machines import Machine -from .secrets.generate import generate_secrets +from .nix import nix_shell +from .secrets.modules import SecretStoreBase log = logging.getLogger(__name__) -def flash_machine(machine: Machine, device: str | None = None) -> None: +def flash_machine( + machine: Machine, disks: dict[str, str], dry_run: bool, debug: bool +) -> None: secrets_module = importlib.import_module(machine.secrets_module) - secret_store = secrets_module.SecretStore(machine=machine) - - generate_secrets(machine) - + secret_store: SecretStoreBase = secrets_module.SecretStore(machine=machine) with TemporaryDirectory() as tmpdir_: tmpdir = Path(tmpdir_) - upload_dir_ = machine.secrets_upload_directory + upload_dir = machine.secrets_upload_directory - if upload_dir_.startswith("/"): - upload_dir_ = upload_dir_[1:] - upload_dir = tmpdir / upload_dir_ - upload_dir.mkdir(parents=True) - secret_store.upload(upload_dir) + if upload_dir.startswith("/"): + local_dir = tmpdir / upload_dir[1:] + else: + local_dir = tmpdir / upload_dir - fs_image = machine.build_nix("config.system.clan.iso") - print(fs_image) + local_dir.mkdir(parents=True) + secret_store.upload(local_dir) + disko_install = [] + + if os.geteuid() != 0: + if shutil.which("sudo") is None: + raise ClanError( + "sudo is required to run disko-install as a non-root user" + ) + disko_install.append("sudo") + + disko_install.append("disko-install") + if dry_run: + disko_install.append("--dry-run") + if debug: + disko_install.append("--debug") + for name, device in disks.items(): + disko_install.extend(["--disk", name, device]) + + disko_install.extend(["--extra-files", str(local_dir), upload_dir]) + disko_install.extend(["--flake", str(machine.flake) + "#" + machine.name]) + + cmd = nix_shell( + ["nixpkgs#disko"], + disko_install, + ) + print("$", " ".join(map(shlex.quote, cmd))) + run(cmd, log=Log.BOTH, error_msg=f"Failed to flash {machine}") @dataclass class FlashOptions: flake: Path machine: str - device: str | None + disks: dict[str, str] + dry_run: bool + confirm: bool + debug: bool + + +class AppendDiskAction(argparse.Action): + def __init__(self, option_strings: str, dest: str, **kwargs: Any) -> None: + super().__init__(option_strings, dest, **kwargs) + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: str | Sequence[str] | None, + option_string: str | None = None, + ) -> None: + disks = getattr(namespace, self.dest) + assert isinstance(values, list), "values must be a list" + disks[values[0]] = values[1] def flash_command(args: argparse.Namespace) -> None: opts = FlashOptions( flake=args.flake, machine=args.machine, - device=args.device, + disks=args.disk, + dry_run=args.dry_run, + confirm=not args.yes, + debug=args.debug, ) machine = Machine(opts.machine, flake=opts.flake) - flash_machine(machine, device=opts.device) + if opts.confirm and not opts.dry_run: + disk_str = ", ".join(f"{name}={device}" for name, device in opts.disks.items()) + ask = input(f"Install {machine.name} to {disk_str}? [y/N] ") + if ask != "y": + return + flash_machine(machine, disks=opts.disks, dry_run=opts.dry_run, debug=opts.debug) def register_parser(parser: argparse.ArgumentParser) -> None: @@ -55,8 +114,30 @@ def register_parser(parser: argparse.ArgumentParser) -> None: help="machine to install", ) parser.add_argument( - "--device", + "--disk", type=str, - help="device to flash the system to", + nargs=2, + metavar=("name", "value"), + action=AppendDiskAction, + help="device to flash to", + default={}, + ) + parser.add_argument( + "--yes", + action="store_true", + help="do not ask for confirmation", + default=False, + ) + parser.add_argument( + "--dry-run", + help="Only build the system, don't flash it", + default=False, + action="store_true", + ) + parser.add_argument( + "--debug", + help="Print debug information", + default=False, + action="store_true", ) parser.set_defaults(func=flash_command) diff --git a/pkgs/clan-cli/clan_cli/history/add.py b/pkgs/clan-cli/clan_cli/history/add.py index 16bb96987..f3b5ab7aa 100644 --- a/pkgs/clan-cli/clan_cli/history/add.py +++ b/pkgs/clan-cli/clan_cli/history/add.py @@ -17,13 +17,6 @@ from ..locked_open import read_history_file, write_history_file log = logging.getLogger(__name__) -class EnhancedJSONEncoder(json.JSONEncoder): - def default(self, o: Any) -> Any: - if dataclasses.is_dataclass(o): - return dataclasses.asdict(o) - return super().default(o) - - @dataclasses.dataclass class HistoryEntry: last_used: str @@ -79,8 +72,8 @@ def new_history_entry(url: str, machine: str) -> HistoryEntry: def add_all_to_history(uri: ClanURI) -> list[HistoryEntry]: history = list_history() new_entries: list[HistoryEntry] = [] - for machine in list_machines(uri.get_internal()): - new_entry = _add_maschine_to_history_list(uri.get_internal(), machine, history) + for machine in list_machines(uri.get_url()): + new_entry = _add_maschine_to_history_list(uri.get_url(), machine, history) new_entries.append(new_entry) write_history_file(history) return new_entries @@ -89,9 +82,7 @@ def add_all_to_history(uri: ClanURI) -> list[HistoryEntry]: def add_history(uri: ClanURI) -> HistoryEntry: user_history_file().parent.mkdir(parents=True, exist_ok=True) history = list_history() - new_entry = _add_maschine_to_history_list( - uri.get_internal(), uri.params.flake_attr, history - ) + new_entry = _add_maschine_to_history_list(uri.get_url(), uri.machine.name, history) write_history_file(history) return new_entry @@ -121,9 +112,7 @@ def add_history_command(args: argparse.Namespace) -> None: # takes a (sub)parser and configures it def register_add_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "uri", type=ClanURI.from_str, help="Path to the flake", default="." - ) + parser.add_argument("uri", type=ClanURI, help="Path to the flake", default=".") parser.add_argument( "--all", help="Add all machines", default=False, action="store_true" ) diff --git a/pkgs/clan-cli/clan_cli/history/update.py b/pkgs/clan-cli/clan_cli/history/update.py index 9b176dea2..12ecf24e0 100644 --- a/pkgs/clan-cli/clan_cli/history/update.py +++ b/pkgs/clan-cli/clan_cli/history/update.py @@ -4,7 +4,7 @@ import datetime from clan_cli.flakes.inspect import inspect_flake -from ..clan_uri import ClanParameters, ClanURI +from ..clan_uri import ClanURI from ..errors import ClanCmdError from ..locked_open import write_history_file from ..nix import nix_metadata @@ -28,9 +28,9 @@ def update_history() -> list[HistoryEntry]: ) uri = ClanURI.from_str( url=str(entry.flake.flake_url), - params=ClanParameters(entry.flake.flake_attr), + machine_name=entry.flake.flake_attr, ) - flake = inspect_flake(uri.get_internal(), uri.params.flake_attr) + flake = inspect_flake(uri.get_url(), uri.machine.name) flake.flake_url = str(flake.flake_url) entry = HistoryEntry( flake=flake, last_used=datetime.datetime.now().isoformat() diff --git a/pkgs/clan-cli/clan_cli/jsonrpc.py b/pkgs/clan-cli/clan_cli/jsonrpc.py new file mode 100644 index 000000000..0d779ee45 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/jsonrpc.py @@ -0,0 +1,15 @@ +import dataclasses +import json +from typing import Any + + +class ClanJSONEncoder(json.JSONEncoder): + def default(self, o: Any) -> Any: + # Check if the object has a to_json method + if hasattr(o, "to_json") and callable(o.to_json): + return o.to_json() + # Check if the object is a dataclass + elif dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + # Otherwise, use the default serialization + return super().default(o) diff --git a/pkgs/clan-cli/clan_cli/locked_open.py b/pkgs/clan-cli/clan_cli/locked_open.py index b8ea03b9f..a049d0ca3 100644 --- a/pkgs/clan-cli/clan_cli/locked_open.py +++ b/pkgs/clan-cli/clan_cli/locked_open.py @@ -1,4 +1,3 @@ -import dataclasses import fcntl import json from collections.abc import Generator @@ -6,16 +5,11 @@ from contextlib import contextmanager from pathlib import Path from typing import Any +from clan_cli.jsonrpc import ClanJSONEncoder + from .dirs import user_history_file -class EnhancedJSONEncoder(json.JSONEncoder): - def default(self, o: Any) -> Any: - if dataclasses.is_dataclass(o): - return dataclasses.asdict(o) - return super().default(o) - - @contextmanager def _locked_open(filename: str | Path, mode: str = "r") -> Generator: """ @@ -29,7 +23,7 @@ def _locked_open(filename: str | Path, mode: str = "r") -> Generator: def write_history_file(data: Any) -> None: with _locked_open(user_history_file(), "w+") as f: - f.write(json.dumps(data, cls=EnhancedJSONEncoder, indent=4)) + f.write(json.dumps(data, cls=ClanJSONEncoder, indent=4)) def read_history_file() -> list[dict]: diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 701303fed..44b64dcf8 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -13,10 +13,11 @@ from ..secrets.generate import generate_secrets log = logging.getLogger(__name__) -def install_nixos(machine: Machine, kexec: str | None = None) -> None: +def install_nixos( + machine: Machine, kexec: str | None = None, debug: bool = False +) -> None: secrets_module = importlib.import_module(machine.secrets_module) log.info(f"installing {machine.name}") - log.info(f"using secret store: {secrets_module.SecretStore}") secret_store = secrets_module.SecretStore(machine=machine) h = machine.target_host @@ -44,8 +45,12 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None: "--extra-files", str(tmpdir), ] + if machine.target_host.port: + cmd += ["--ssh-port", str(machine.target_host.port)] if kexec: cmd += ["--kexec", kexec] + if debug: + cmd.append("--debug") cmd.append(target_host) run( @@ -63,6 +68,8 @@ class InstallOptions: machine: str target_host: str kexec: str | None + confirm: bool + debug: bool def install_command(args: argparse.Namespace) -> None: @@ -71,11 +78,18 @@ def install_command(args: argparse.Namespace) -> None: machine=args.machine, target_host=args.target_host, kexec=args.kexec, + confirm=not args.yes, + debug=args.debug, ) machine = Machine(opts.machine, flake=opts.flake) machine.target_host_address = opts.target_host - install_nixos(machine, kexec=opts.kexec) + if opts.confirm: + ask = input(f"Install {machine.name} to {opts.target_host}? [y/N] ") + if ask != "y": + return + + install_nixos(machine, kexec=opts.kexec, debug=opts.debug) def register_install_parser(parser: argparse.ArgumentParser) -> None: @@ -84,6 +98,18 @@ def register_install_parser(parser: argparse.ArgumentParser) -> None: type=str, help="use another kexec tarball to bootstrap NixOS", ) + parser.add_argument( + "--yes", + action="store_true", + help="do not ask for confirmation", + default=False, + ) + parser.add_argument( + "--debug", + action="store_true", + help="print debug information", + default=False, + ) parser.add_argument( "machine", type=str, diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 8acc5b4ae..72e4b9f99 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -6,6 +6,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile from typing import Any +from clan_cli.clan_uri import ClanURI, MachineData from clan_cli.dirs import vm_state_dir from qemu.qmp import QEMUMonitorProtocol @@ -17,7 +18,7 @@ from ..ssh import Host, parse_deployment_address log = logging.getLogger(__name__) -class VMAttr: +class QMPWrapper: def __init__(self, state_dir: Path) -> None: # These sockets here are just symlinks to the real sockets which # are created by the run.py file. The reason being that we run into @@ -40,11 +41,21 @@ class VMAttr: class Machine: + flake: str | Path + name: str + data: MachineData + eval_cache: dict[str, str] + build_cache: dict[str, Path] + _flake_path: Path | None + _deployment_info: None | dict[str, str] + vm: QMPWrapper + def __init__( self, name: str, flake: Path | str, deployment_info: dict | None = None, + machine: MachineData | None = None, ) -> None: """ Creates a Machine @@ -52,20 +63,26 @@ class Machine: @clan_dir: the directory of the clan, optional, if not set it will be determined from the current working directory @machine_json: can be optionally used to skip evaluation of the machine, location of the json file with machine data """ - self.name: str = name - self.flake: str | Path = flake + if machine is None: + uri = ClanURI.from_str(str(flake), name) + machine = uri.machine + self.flake: str | Path = machine.flake_id._value + self.name: str = machine.name + self.data: MachineData = machine + else: + self.data: MachineData = machine self.eval_cache: dict[str, str] = {} self.build_cache: dict[str, Path] = {} - + self._flake_path: Path | None = None self._deployment_info: None | dict[str, str] = deployment_info - state_dir = vm_state_dir(flake_url=str(self.flake), vm_name=self.name) + state_dir = vm_state_dir(flake_url=str(self.flake), vm_name=self.data.name) - self.vm: VMAttr = VMAttr(state_dir) + self.vm: QMPWrapper = QMPWrapper(state_dir) def __str__(self) -> str: - return f"Machine(name={self.name}, flake={self.flake})" + return f"Machine(name={self.data.name}, flake={self.data.flake_id})" def __repr__(self) -> str: return str(self) @@ -86,7 +103,7 @@ class Machine: "deploymentAddress" ) if val is None: - msg = f"the 'clan.networking.targetHost' nixos option is not set for machine '{self.name}'" + msg = f"the 'clan.networking.targetHost' nixos option is not set for machine '{self.data.name}'" raise ClanError(msg) return val @@ -109,7 +126,7 @@ class Machine: return json.loads(Path(self.deployment_info["secretsData"]).read_text()) except json.JSONDecodeError as e: raise ClanError( - f"Failed to parse secretsData for machine {self.name} as json" + f"Failed to parse secretsData for machine {self.data.name} as json" ) from e return {} @@ -119,19 +136,23 @@ class Machine: @property def flake_dir(self) -> Path: - if isinstance(self.flake, Path): - return self.flake + if self._flake_path: + return self._flake_path - if hasattr(self, "flake_path"): - return Path(self.flake_path) + if self.data.flake_id.is_local(): + self._flake_path = self.data.flake_id.path + elif self.data.flake_id.is_remote(): + self._flake_path = Path(nix_metadata(self.data.flake_id.url)["path"]) + else: + raise ClanError(f"Unsupported flake url: {self.data.flake_id}") - self.flake_path: str = nix_metadata(self.flake)["path"] - return Path(self.flake_path) + assert self._flake_path is not None + return self._flake_path @property def target_host(self) -> Host: return parse_deployment_address( - self.name, self.target_host_address, meta={"machine": self} + self.data.name, self.target_host_address, meta={"machine": self} ) @property @@ -145,7 +166,7 @@ class Machine: return self.target_host # enable ssh agent forwarding to allow the build host to access the target host return parse_deployment_address( - self.name, + self.data.name, build_host, forward_agent=True, meta={"machine": self, "target_host": self.target_host}, @@ -204,7 +225,7 @@ class Machine: args += [ "--expr", f""" - ((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{ + ((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.data.name}" {{ extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{ type = "file"; url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{file_info["path"]}" else "file:{file_info["path"]}"; @@ -214,15 +235,13 @@ class Machine: """, ] else: - if isinstance(self.flake, Path): - if (self.flake / ".git").exists(): - flake = f"git+file://{self.flake}" - else: - flake = f"path:{self.flake}" + if (self.flake_dir / ".git").exists(): + flake = f"git+file://{self.flake_dir}" else: - flake = self.flake + flake = f"path:{self.flake_dir}" + args += [ - f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}', + f'{flake}#clanInternals.machines."{system}".{self.data.name}.{attr}', *nix_options, ] diff --git a/pkgs/clan-cli/clan_cli/py.typed b/pkgs/clan-cli/clan_cli/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index d87f8d99b..13721559e 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -2,6 +2,7 @@ import argparse import importlib import logging import os +import subprocess from collections.abc import Callable from pathlib import Path from tempfile import TemporaryDirectory @@ -19,6 +20,15 @@ from .modules import SecretStoreBase log = logging.getLogger(__name__) +def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str: + """ + Read multi-line input from stdin. + """ + print(prompt, flush=True) + proc = subprocess.run(["cat"], stdout=subprocess.PIPE, text=True) + return proc.stdout + + def generate_service_secrets( machine: Machine, service: str, @@ -128,7 +138,12 @@ def generate_secrets( fact_store = facts_module.FactStore(machine=machine) if prompt is None: - prompt = lambda text: input(f"{text}: ") + + def prompt_func(text: str) -> str: + print(f"{text}: ") + return read_multiline_input() + + prompt = prompt_func with TemporaryDirectory() as tmp: tmpdir = Path(tmp) diff --git a/pkgs/clan-cli/clan_cli/vms/inspect.py b/pkgs/clan-cli/clan_cli/vms/inspect.py index cb70b7588..a09b4361a 100644 --- a/pkgs/clan-cli/clan_cli/vms/inspect.py +++ b/pkgs/clan-cli/clan_cli/vms/inspect.py @@ -22,7 +22,7 @@ class VmConfig: def inspect_vm(machine: Machine) -> VmConfig: data = json.loads(machine.eval_nix("config.clanCore.vm.inspect")) - return VmConfig(flake_url=machine.flake, **data) + return VmConfig(flake_url=str(machine.flake), **data) @dataclass diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index b4751cb46..6b35f98d8 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -174,12 +174,13 @@ def run_vm( if vm.graphics and not vm.waypipe: packages.append("nixpkgs#virt-viewer") remote_viewer_mimetypes = module_root() / "vms" / "mimetypes" - env[ - "XDG_DATA_DIRS" - ] = f"{remote_viewer_mimetypes}:{env.get('XDG_DATA_DIRS', '')}" + env["XDG_DATA_DIRS"] = ( + f"{remote_viewer_mimetypes}:{env.get('XDG_DATA_DIRS', '')}" + ) - with start_waypipe(qemu_cmd.vsock_cid, f"[{vm.machine_name}] "), start_virtiofsd( - virtiofsd_socket + with ( + start_waypipe(qemu_cmd.vsock_cid, f"[{vm.machine_name}] "), + start_virtiofsd(virtiofsd_socket), ): run( nix_shell(packages, qemu_cmd.args), diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 2b256e8c5..fa5252f13 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -1,36 +1,37 @@ -{ age -, lib -, argcomplete -, installShellFiles -, nix -, openssh -, pytest -, pytest-cov -, pytest-xdist -, pytest-subprocess -, pytest-timeout -, remote-pdb -, ipdb -, python3 -, runCommand -, setuptools -, sops -, stdenv -, wheel -, fakeroot -, rsync -, bash -, sshpass -, zbar -, tor -, git -, nixpkgs -, qemu -, gnupg -, e2fsprogs -, mypy -, rope -, clan-core-path +{ + age, + lib, + argcomplete, + installShellFiles, + nix, + openssh, + pytest, + pytest-cov, + pytest-xdist, + pytest-subprocess, + pytest-timeout, + remote-pdb, + ipdb, + python3, + runCommand, + setuptools, + sops, + stdenv, + wheel, + fakeroot, + rsync, + bash, + sshpass, + zbar, + tor, + git, + nixpkgs, + qemu, + gnupg, + e2fsprogs, + mypy, + rope, + clan-core-path, }: let @@ -38,19 +39,22 @@ let argcomplete # optional dependency: if not enabled, shell completion will not work ]; - pytestDependencies = runtimeDependencies ++ dependencies ++ [ - pytest - pytest-cov - pytest-subprocess - pytest-xdist - pytest-timeout - remote-pdb - ipdb - openssh - git - gnupg - stdenv.cc - ]; + pytestDependencies = + runtimeDependencies + ++ dependencies + ++ [ + pytest + pytest-cov + pytest-subprocess + pytest-xdist + pytest-timeout + remote-pdb + ipdb + openssh + git + gnupg + stdenv.cc + ]; # Optional dependencies for clan cli, we re-expose them here to make sure they all build. runtimeDependencies = [ @@ -70,7 +74,9 @@ let e2fsprogs ]; - runtimeDependenciesAsSet = builtins.listToAttrs (builtins.map (p: lib.nameValuePair (lib.getName p.name) p) runtimeDependencies); + runtimeDependenciesAsSet = builtins.listToAttrs ( + builtins.map (p: lib.nameValuePair (lib.getName p.name) p) runtimeDependencies + ); checkPython = python3.withPackages (_ps: pytestDependencies); @@ -121,42 +127,48 @@ python3.pkgs.buildPythonApplication { propagatedBuildInputs = dependencies; # also re-expose dependencies so we test them in CI - passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesAsSet) // rec { - clan-pytest-without-core = runCommand "clan-pytest-without-core" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } '' - cp -r ${source} ./src - chmod +w -R ./src - cd ./src + passthru.tests = + (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesAsSet) + // rec { + clan-pytest-without-core = + runCommand "clan-pytest-without-core" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } + '' + cp -r ${source} ./src + chmod +w -R ./src + cd ./src - export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 - ${checkPython}/bin/python -m pytest -m "not impure and not with_core" ./tests - touch $out - ''; - # separate the tests that can never be cached - clan-pytest-with-core = runCommand "clan-pytest-with-core" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } '' - cp -r ${source} ./src - chmod +w -R ./src - cd ./src + export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 + ${checkPython}/bin/python -m pytest -m "not impure and not with_core" ./tests + touch $out + ''; + # separate the tests that can never be cached + clan-pytest-with-core = + runCommand "clan-pytest-with-core" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } + '' + cp -r ${source} ./src + chmod +w -R ./src + cd ./src - export CLAN_CORE=${clan-core-path} - export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 - ${checkPython}/bin/python -m pytest -m "not impure and with_core" ./tests - touch $out - ''; + export CLAN_CORE=${clan-core-path} + export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 + ${checkPython}/bin/python -m pytest -m "not impure and with_core" ./tests + touch $out + ''; - clan-pytest = runCommand "clan-pytest" { } '' - echo ${clan-pytest-without-core} - echo ${clan-pytest-with-core} - touch $out - ''; - check-for-breakpoints = runCommand "breakpoints" { } '' - if grep --include \*.py -Rq "breakpoint()" ${source}; then - echo "breakpoint() found in ${source}:" - grep --include \*.py -Rn "breakpoint()" ${source} - exit 1 - fi - touch $out - ''; - }; + clan-pytest = runCommand "clan-pytest" { } '' + echo ${clan-pytest-without-core} + echo ${clan-pytest-with-core} + touch $out + ''; + check-for-breakpoints = runCommand "breakpoints" { } '' + if grep --include \*.py -Rq "breakpoint()" ${source}; then + echo "breakpoint() found in ${source}:" + grep --include \*.py -Rn "breakpoint()" ${source} + exit 1 + fi + touch $out + ''; + }; passthru.nixpkgs = nixpkgs'; passthru.checkPython = checkPython; @@ -178,7 +190,7 @@ python3.pkgs.buildPythonApplication { <(${argcomplete}/bin/register-python-argcomplete --shell fish clan) ''; # Don't leak python packages into a devshell. - # It can be very confusing if you `nix run` than load the cli from the devshell instead. + # It can be very confusing if you `nix run` then load the cli from the devshell instead. postFixup = '' rm $out/nix-support/propagated-build-inputs ''; diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index cd1ae1bba..ba8cb7f09 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -1,37 +1,44 @@ -{ inputs, self, lib, ... }: { - perSystem = { self', pkgs, ... }: + inputs, + self, + lib, + ... +}: +{ + perSystem = + { self', pkgs, ... }: let flakeLock = lib.importJSON (self + /flake.lock); flakeInputs = (builtins.removeAttrs inputs [ "self" ]); flakeLockVendoredDeps = flakeLock // { - nodes = flakeLock.nodes // ( - lib.flip lib.mapAttrs flakeInputs (name: _: flakeLock.nodes.${name} // { - locked = { - inherit (flakeLock.nodes.${name}.locked) narHash; - lastModified = - # lol, nixpkgs has a different timestamp on the fs??? - if name == "nixpkgs" - then 0 - else 1; - path = "${inputs.${name}}"; - type = "path"; - }; - }) - ); + nodes = + flakeLock.nodes + // (lib.flip lib.mapAttrs flakeInputs ( + name: _: + flakeLock.nodes.${name} + // { + locked = { + inherit (flakeLock.nodes.${name}.locked) narHash; + lastModified = + # lol, nixpkgs has a different timestamp on the fs??? + if name == "nixpkgs" then 0 else 1; + path = "${inputs.${name}}"; + type = "path"; + }; + } + )); }; - flakeLockFile = builtins.toFile "clan-core-flake.lock" - (builtins.toJSON flakeLockVendoredDeps); - clanCoreWithVendoredDeps = lib.trace flakeLockFile pkgs.runCommand "clan-core-with-vendored-deps" { } '' - cp -r ${self} $out - chmod +w -R $out - cp ${flakeLockFile} $out/flake.lock - ''; + flakeLockFile = builtins.toFile "clan-core-flake.lock" (builtins.toJSON flakeLockVendoredDeps); + clanCoreWithVendoredDeps = + lib.trace flakeLockFile pkgs.runCommand "clan-core-with-vendored-deps" { } + '' + cp -r ${self} $out + chmod +w -R $out + cp ${flakeLockFile} $out/flake.lock + ''; in { - devShells.clan-cli = pkgs.callPackage ./shell.nix { - inherit (self'.packages) clan-cli; - }; + devShells.clan-cli = pkgs.callPackage ./shell.nix { inherit (self'.packages) clan-cli; }; packages = { clan-cli = pkgs.python3.pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs; @@ -42,5 +49,4 @@ checks = self'.packages.clan-cli.tests; }; - } diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 7800c659c..9193d90a5 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -12,7 +12,7 @@ scripts = { clan = "clan_cli:main" } exclude = ["clan_cli.nixpkgs*", "result"] [tool.setuptools.package-data] -clan_cli = ["config/jsonschema/*", "webui/assets/**/*", "vms/mimetypes/**/*"] +clan_cli = ["py.typed", "config/jsonschema/*", "webui/assets/**/*", "vms/mimetypes/**/*"] [tool.pytest.ini_options] testpaths = "tests" @@ -29,7 +29,6 @@ warn_redundant_casts = true disallow_untyped_calls = true disallow_untyped_defs = true no_implicit_optional = true -disable_error_code = ["has-type"] exclude = "clan_cli.nixpkgs" [[tool.mypy.overrides]] diff --git a/pkgs/clan-cli/qemu/qmp.py b/pkgs/clan-cli/qemu/qmp.py index 6dbef7c68..0e878f6c6 100644 --- a/pkgs/clan-cli/qemu/qmp.py +++ b/pkgs/clan-cli/qemu/qmp.py @@ -1,6 +1,6 @@ # mypy: ignore-errors -""" QEMU Monitor Protocol Python class """ +"""QEMU Monitor Protocol Python class""" # Copyright (C) 2009, 2010 Red Hat Inc. # # Authors: diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index ecc335746..e239355c1 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -1,16 +1,20 @@ -{ nix-unit, clan-cli, system, mkShell, writeScriptBin, openssh, ruff, python3 }: +{ + nix-unit, + clan-cli, + system, + mkShell, + writeScriptBin, + openssh, + ruff, + python3, +}: let checkScript = writeScriptBin "check" '' nix build .#checks.${system}.{treefmt,clan-pytest} -L "$@" ''; pythonWithDeps = python3.withPackages ( - ps: - clan-cli.propagatedBuildInputs - ++ clan-cli.devDependencies - ++ [ - ps.pip - ] + ps: clan-cli.propagatedBuildInputs ++ clan-cli.devDependencies ++ [ ps.pip ] ); in mkShell { diff --git a/pkgs/clan-cli/tests/machines/vm1/default.nix b/pkgs/clan-cli/tests/machines/vm1/default.nix index 8ae4d04fb..2f0f4ceee 100644 --- a/pkgs/clan-cli/tests/machines/vm1/default.nix +++ b/pkgs/clan-cli/tests/machines/vm1/default.nix @@ -1,4 +1,5 @@ -{ lib, ... }: { +{ lib, ... }: +{ clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; system.stateVersion = lib.version; sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__"; diff --git a/pkgs/clan-cli/tests/machines/vm_with_secrets/default.nix b/pkgs/clan-cli/tests/machines/vm_with_secrets/default.nix index 8ae4d04fb..2f0f4ceee 100644 --- a/pkgs/clan-cli/tests/machines/vm_with_secrets/default.nix +++ b/pkgs/clan-cli/tests/machines/vm_with_secrets/default.nix @@ -1,4 +1,5 @@ -{ lib, ... }: { +{ lib, ... }: +{ clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; system.stateVersion = lib.version; sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__"; diff --git a/pkgs/clan-cli/tests/machines/vm_without_secrets/default.nix b/pkgs/clan-cli/tests/machines/vm_without_secrets/default.nix index db5b9eed3..2d02b56af 100644 --- a/pkgs/clan-cli/tests/machines/vm_without_secrets/default.nix +++ b/pkgs/clan-cli/tests/machines/vm_without_secrets/default.nix @@ -1,4 +1,5 @@ -{ lib, ... }: { +{ lib, ... }: +{ clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; system.stateVersion = lib.version; clan.virtualisation.graphics = false; diff --git a/pkgs/clan-cli/tests/test_clan_uri.py b/pkgs/clan-cli/tests/test_clan_uri.py index 64cc46326..efddafa64 100644 --- a/pkgs/clan-cli/tests/test_clan_uri.py +++ b/pkgs/clan-cli/tests/test_clan_uri.py @@ -1,145 +1,97 @@ from pathlib import Path -from clan_cli.clan_uri import ClanParameters, ClanScheme, ClanURI +from clan_cli.clan_uri import ClanURI -def test_get_internal() -> None: +def test_get_url() -> None: # Create a ClanURI object from a remote URI with parameters - uri = ClanURI("clan://https://example.com?flake_attr=myVM&password=1234") - assert uri.get_internal() == "https://example.com?password=1234" + uri = ClanURI("clan://https://example.com?password=1234#myVM") + assert uri.get_url() == "https://example.com?password=1234" uri = ClanURI("clan://~/Downloads") - assert uri.get_internal().endswith("/Downloads") + assert uri.get_url().endswith("/Downloads") uri = ClanURI("clan:///home/user/Downloads") - assert uri.get_internal() == "/home/user/Downloads" + assert uri.get_url() == "/home/user/Downloads" uri = ClanURI("clan://file:///home/user/Downloads") - assert uri.get_internal() == "/home/user/Downloads" + assert uri.get_url() == "/home/user/Downloads" def test_local_uri() -> None: # Create a ClanURI object from a local URI uri = ClanURI("clan://file:///home/user/Downloads") - match uri.scheme: - case ClanScheme.LOCAL.value(path): - assert path == Path("/home/user/Downloads") # type: ignore - case _: - assert False + assert uri.flake_id.path == Path("/home/user/Downloads") def test_is_remote() -> None: # Create a ClanURI object from a remote URI uri = ClanURI("clan://https://example.com") - - match uri.scheme: - case ClanScheme.REMOTE.value(url): - assert url == "https://example.com" # type: ignore - case _: - assert False + assert uri.flake_id.url == "https://example.com" def test_direct_local_path() -> None: # Create a ClanURI object from a remote URI uri = ClanURI("clan://~/Downloads") - assert uri.get_internal().endswith("/Downloads") + assert uri.get_url().endswith("/Downloads") def test_direct_local_path2() -> None: # Create a ClanURI object from a remote URI uri = ClanURI("clan:///home/user/Downloads") - assert uri.get_internal() == "/home/user/Downloads" + assert uri.get_url() == "/home/user/Downloads" def test_remote_with_clanparams() -> None: # Create a ClanURI object from a remote URI with parameters uri = ClanURI("clan://https://example.com") - assert uri.params.flake_attr == "defaultVM" - - match uri.scheme: - case ClanScheme.REMOTE.value(url): - assert url == "https://example.com" # type: ignore - case _: - assert False - - -def test_from_path_with_custom() -> None: - # Create a ClanURI object from a remote URI with parameters - uri_str = Path("/home/user/Downloads") - params = ClanParameters(flake_attr="myVM") - uri = ClanURI.from_path(uri_str, params=params) - assert uri.params.flake_attr == "myVM" - - match uri.scheme: - case ClanScheme.LOCAL.value(path): - assert path == Path("/home/user/Downloads") # type: ignore - case _: - assert False - - -def test_from_path_with_default() -> None: - # Create a ClanURI object from a remote URI with parameters - uri_str = Path("/home/user/Downloads") - params = ClanParameters() - uri = ClanURI.from_path(uri_str, params=params) - assert uri.params.flake_attr == "defaultVM" - - match uri.scheme: - case ClanScheme.LOCAL.value(path): - assert path == Path("/home/user/Downloads") # type: ignore - case _: - assert False - - -def test_from_str() -> None: - # Create a ClanURI object from a remote URI with parameters - uri_str = "https://example.com?password=asdasd&test=1234" - params = ClanParameters(flake_attr="myVM") - uri = ClanURI.from_str(url=uri_str, params=params) - assert uri.params.flake_attr == "myVM" - - match uri.scheme: - case ClanScheme.REMOTE.value(url): - assert url == "https://example.com?password=asdasd&test=1234" # type: ignore - case _: - assert False - - uri = ClanURI.from_str(url=uri_str, params={"flake_attr": "myVM"}) - assert uri.params.flake_attr == "myVM" - - uri = ClanURI.from_str(uri_str, "myVM") - assert uri.params.flake_attr == "myVM" - - uri_str = "~/Downloads/democlan" - params = ClanParameters(flake_attr="myVM") - uri = ClanURI.from_str(url=uri_str, params=params) - assert uri.params.flake_attr == "myVM" - assert uri.get_internal().endswith("/Downloads/democlan") - - uri_str = "~/Downloads/democlan" - uri = ClanURI.from_str(url=uri_str) - assert uri.params.flake_attr == "defaultVM" - assert uri.get_internal().endswith("/Downloads/democlan") - - uri_str = "clan://~/Downloads/democlan" - uri = ClanURI.from_str(url=uri_str) - assert uri.params.flake_attr == "defaultVM" - assert uri.get_internal().endswith("/Downloads/democlan") + assert uri.machine.name == "defaultVM" + assert uri.flake_id.url == "https://example.com" def test_remote_with_all_params() -> None: - # Create a ClanURI object from a remote URI with parameters - uri = ClanURI("clan://https://example.com?flake_attr=myVM&password=1234") - assert uri.params.flake_attr == "myVM" - - match uri.scheme: - case ClanScheme.REMOTE.value(url): - assert url == "https://example.com?password=1234" # type: ignore - case _: - assert False + uri = ClanURI("clan://https://example.com?password=12345#myVM#secondVM?dummy_opt=1") + assert uri.machine.name == "myVM" + assert uri._machines[1].name == "secondVM" + assert uri._machines[1].params.dummy_opt == "1" + assert uri.flake_id.url == "https://example.com?password=12345" -def test_with_hashtag() -> None: - uri = ClanURI("clan://https://example.com?flake_attr=thirdVM#myVM#secondVM") - assert uri.params.flake_attr == "myVM" +def test_from_str_remote() -> None: + uri = ClanURI.from_str(url="https://example.com", machine_name="myVM") + assert uri.get_url() == "https://example.com" + assert uri.get_orig_uri() == "clan://https://example.com#myVM" + assert uri.machine.name == "myVM" + assert len(uri._machines) == 1 + assert uri.flake_id.url == "https://example.com" + + +def test_from_str_local() -> None: + uri = ClanURI.from_str(url="~/Projects/democlan", machine_name="myVM") + assert uri.get_url().endswith("/Projects/democlan") + assert uri.get_orig_uri() == "clan://~/Projects/democlan#myVM" + assert uri.machine.name == "myVM" + assert len(uri._machines) == 1 + assert uri.flake_id.is_local() + assert str(uri.flake_id).endswith("/Projects/democlan") # type: ignore + + +def test_from_str_local_no_machine() -> None: + uri = ClanURI.from_str("~/Projects/democlan") + assert uri.get_url().endswith("/Projects/democlan") + assert uri.get_orig_uri() == "clan://~/Projects/democlan" + assert uri.machine.name == "defaultVM" + assert len(uri._machines) == 1 + assert uri.flake_id.is_local() + assert str(uri.flake_id).endswith("/Projects/democlan") # type: ignore + + +def test_from_str_local_no_machine2() -> None: + uri = ClanURI.from_str("~/Projects/democlan#syncthing-peer1") + assert uri.get_url().endswith("/Projects/democlan") + assert uri.get_orig_uri() == "clan://~/Projects/democlan#syncthing-peer1" + assert uri.machine.name == "syncthing-peer1" + assert len(uri._machines) == 1 + assert uri.flake_id.is_local() + assert str(uri.flake_id).endswith("/Projects/democlan") # type: ignore diff --git a/pkgs/clan-cli/tests/test_flake/fake-module.nix b/pkgs/clan-cli/tests/test_flake/fake-module.nix index b3c6580df..d00d06624 100644 --- a/pkgs/clan-cli/tests/test_flake/fake-module.nix +++ b/pkgs/clan-cli/tests/test_flake/fake-module.nix @@ -1,6 +1,5 @@ -{ lib -, ... -}: { +{ lib, ... }: +{ options.clan.fake-module.fake-flag = lib.mkOption { type = lib.types.bool; default = false; diff --git a/pkgs/clan-cli/tests/test_flake/flake.nix b/pkgs/clan-cli/tests/test_flake/flake.nix index 3f335a4eb..ec93e1455 100644 --- a/pkgs/clan-cli/tests/test_flake/flake.nix +++ b/pkgs/clan-cli/tests/test_flake/flake.nix @@ -2,32 +2,41 @@ # this placeholder is replaced by the path to nixpkgs inputs.nixpkgs.url = "__NIXPKGS__"; - outputs = inputs': + outputs = + inputs': let # fake clan-core input fake-clan-core = { clanModules.fake-module = ./fake-module.nix; }; - inputs = inputs' // { clan-core = fake-clan-core; }; + inputs = inputs' // { + clan-core = fake-clan-core; + }; machineSettings = ( - if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" - then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) - else if builtins.pathExists ./machines/machine1/settings.json - then builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json) - else { } + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then + builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) + else if builtins.pathExists ./machines/machine1/settings.json then + builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json) + else + { } + ); + machineImports = map (module: fake-clan-core.clanModules.${module}) ( + machineSettings.clanImports or [ ] ); - machineImports = - map - (module: fake-clan-core.clanModules.${module}) - (machineSettings.clanImports or [ ]); in { nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem { - modules = - machineImports ++ [ - ./nixosModules/machine1.nix - machineSettings - ({ lib, options, pkgs, ... }: { + modules = machineImports ++ [ + ./nixosModules/machine1.nix + machineSettings + ( + { + lib, + options, + pkgs, + ... + }: + { config = { nixpkgs.hostPlatform = "x86_64-linux"; # speed up by not instantiating nixpkgs twice and disable documentation @@ -51,8 +60,9 @@ The buildClan function will automatically import these modules for the current machine. ''; }; - }) - ]; + } + ) + ]; }; }; } diff --git a/pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix b/pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix index 8bf312e38..26371f09b 100644 --- a/pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix +++ b/pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix @@ -1,4 +1,5 @@ -{ lib, ... }: { +{ lib, ... }: +{ options.clan.jitsi.enable = lib.mkOption { type = lib.types.bool; default = false; diff --git a/pkgs/clan-cli/tests/test_flake_with_core/flake.nix b/pkgs/clan-cli/tests/test_flake_with_core/flake.nix index 3870cb6c9..2b1f4d0ea 100644 --- a/pkgs/clan-cli/tests/test_flake_with_core/flake.nix +++ b/pkgs/clan-cli/tests/test_flake_with_core/flake.nix @@ -5,40 +5,45 @@ # this placeholder is replaced by the path to nixpkgs inputs.clan-core.url = "__CLAN_CORE__"; - outputs = { self, clan-core }: + outputs = + { self, clan-core }: let clan = clan-core.lib.buildClan { directory = self; clanName = "test_flake_with_core"; machines = { - vm1 = { lib, ... }: { - clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; - system.stateVersion = lib.version; - sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__"; - clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__"; - clanCore.sops.defaultGroups = [ "admins" ]; - clan.virtualisation.graphics = false; + vm1 = + { lib, ... }: + { + clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; + system.stateVersion = lib.version; + sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__"; + clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__"; + clanCore.sops.defaultGroups = [ "admins" ]; + clan.virtualisation.graphics = false; - clan.networking.zerotier.controller.enable = true; - networking.useDHCP = false; + clan.networking.zerotier.controller.enable = true; + networking.useDHCP = false; - systemd.services.shutdown-after-boot = { - enable = true; - wantedBy = [ "multi-user.target" ]; - after = [ "multi-user.target" ]; - script = '' - #!/usr/bin/env bash - shutdown -h now - ''; + systemd.services.shutdown-after-boot = { + enable = true; + wantedBy = [ "multi-user.target" ]; + after = [ "multi-user.target" ]; + script = '' + #!/usr/bin/env bash + shutdown -h now + ''; + }; + }; + vm2 = + { lib, ... }: + { + clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; + system.stateVersion = lib.version; + sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__"; + clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__"; + clan.networking.zerotier.networkId = "82b44b162ec6c013"; }; - }; - vm2 = { lib, ... }: { - clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; - system.stateVersion = lib.version; - sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__"; - clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__"; - clan.networking.zerotier.networkId = "82b44b162ec6c013"; - }; }; }; in diff --git a/pkgs/clan-cli/tests/test_flake_with_core_and_pass/flake.nix b/pkgs/clan-cli/tests/test_flake_with_core_and_pass/flake.nix index a7187d540..d9ca403bf 100644 --- a/pkgs/clan-cli/tests/test_flake_with_core_and_pass/flake.nix +++ b/pkgs/clan-cli/tests/test_flake_with_core_and_pass/flake.nix @@ -5,30 +5,33 @@ # this placeholder is replaced by the path to clan-core inputs.clan-core.url = "__CLAN_CORE__"; - outputs = { self, clan-core }: + outputs = + { self, clan-core }: let clan = clan-core.lib.buildClan { directory = self; clanName = "test_flake_with_core_and_pass"; machines = { - vm1 = { lib, ... }: { - clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; - system.stateVersion = lib.version; - clanCore.secretStore = "password-store"; - clanCore.secretsUploadDirectory = lib.mkForce "__CLAN_SOPS_KEY_DIR__/secrets"; + vm1 = + { lib, ... }: + { + clan.networking.targetHost = "__CLAN_TARGET_ADDRESS__"; + system.stateVersion = lib.version; + clanCore.secretStore = "password-store"; + clanCore.secretsUploadDirectory = lib.mkForce "__CLAN_SOPS_KEY_DIR__/secrets"; - clan.networking.zerotier.controller.enable = true; + clan.networking.zerotier.controller.enable = true; - systemd.services.shutdown-after-boot = { - enable = true; - wantedBy = [ "multi-user.target" ]; - after = [ "multi-user.target" ]; - script = '' - #!/usr/bin/env bash - shutdown -h now - ''; + systemd.services.shutdown-after-boot = { + enable = true; + wantedBy = [ "multi-user.target" ]; + after = [ "multi-user.target" ]; + script = '' + #!/usr/bin/env bash + shutdown -h now + ''; + }; }; - }; }; }; in diff --git a/pkgs/clan-cli/tests/test_flake_with_core_dynamic_machines/flake.nix b/pkgs/clan-cli/tests/test_flake_with_core_dynamic_machines/flake.nix index 01cc5746f..e10e244db 100644 --- a/pkgs/clan-cli/tests/test_flake_with_core_dynamic_machines/flake.nix +++ b/pkgs/clan-cli/tests/test_flake_with_core_dynamic_machines/flake.nix @@ -5,7 +5,8 @@ # this placeholder is replaced by the path to nixpkgs inputs.clan-core.url = "__CLAN_CORE__"; - outputs = { self, clan-core }: + outputs = + { self, clan-core }: let clan = clan-core.lib.buildClan { directory = self; @@ -14,9 +15,7 @@ let machineModules = builtins.readDir (self + "/machines"); in - builtins.mapAttrs - (name: _type: import (self + "/machines/${name}")) - machineModules; + builtins.mapAttrs (name: _type: import (self + "/machines/${name}")) machineModules; }; in { diff --git a/pkgs/clan-cli/tests/test_history_cli.py b/pkgs/clan-cli/tests/test_history_cli.py index d5eb34bff..7da808ed2 100644 --- a/pkgs/clan-cli/tests/test_history_cli.py +++ b/pkgs/clan-cli/tests/test_history_cli.py @@ -6,7 +6,7 @@ from cli import Cli from fixtures_flakes import FlakeForTest from pytest import CaptureFixture -from clan_cli.clan_uri import ClanParameters, ClanURI +from clan_cli.clan_uri import ClanURI from clan_cli.dirs import user_history_file from clan_cli.history.add import HistoryEntry @@ -19,8 +19,7 @@ def test_history_add( test_flake_with_core: FlakeForTest, ) -> None: cli = Cli() - params = ClanParameters(flake_attr="vm1") - uri = ClanURI.from_path(test_flake_with_core.path, params=params) + uri = ClanURI.from_str(str(test_flake_with_core.path), "vm1") cmd = [ "history", "add", @@ -40,8 +39,7 @@ def test_history_list( test_flake_with_core: FlakeForTest, ) -> None: cli = Cli() - params = ClanParameters(flake_attr="vm1") - uri = ClanURI.from_path(test_flake_with_core.path, params=params) + uri = ClanURI.from_str(str(test_flake_with_core.path), "vm1") cmd = [ "history", "list", diff --git a/pkgs/clan-vm-manager/README.md b/pkgs/clan-vm-manager/README.md index df504a6a2..8a31c44eb 100644 --- a/pkgs/clan-vm-manager/README.md +++ b/pkgs/clan-vm-manager/README.md @@ -1,34 +1,34 @@ -## Developing GTK4 Applications +# Clan VM Manager +Provides users with the simple functionality to manage their locally registered clans. + +![app-preview](screenshots/image.png) + +## Available commands + +Run this application -## Demos -Adw has a demo application showing all widgets. You can run it by executing: ```bash -adwaita-1-demo -``` -GTK4 has a demo application showing all widgets. You can run it by executing: -```bash -gtk4-widget-factory +./bin/clan-vm-manager ``` -To find available icons execute: +Join the default machine of a clan + ```bash -gtk4-icon-browser +./bin/clan-vm-manager [clan-uri] ``` +Join a specific machine of a clan +```bash +./bin/clan-vm-manager [clan-uri]#[machine] +``` -## Links -- [Adw PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Adw-1) -- [GTK4 PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Gtk-4.0) -- [Adw Widget Gallery](https://gnome.pages.gitlab.gnome.org/libadwaita/doc/main/widget-gallery.html) -- [Python + GTK3 Tutorial](https://python-gtk-3-tutorial.readthedocs.io/en/latest/textview.html) +For more available commands see the developer section below. +## Developing this Application - -## Debugging Style and Layout - -You can append `--debug` flag to enable debug logging printed into the console. +### Debugging Style and Layout ```bash # Enable the GTK debugger @@ -38,8 +38,57 @@ gsettings set org.gtk.Settings.Debug enable-inspector-keybinding true GTK_DEBUG=interactive ./bin/clan-vm-manager --debug ``` -## Profiling -To activate profiling execute: -``` +Appending `--debug` flag enables debug logging printed into the console. + +### Profiling + +To activate profiling you can run + +```bash PERF=1 ./bin/clan-vm-manager -``` \ No newline at end of file +``` + +### Library Components + +> Note: +> +> we recognized bugs when starting some cli-commands through the integrated vs-code terminal. +> If encountering issues make sure to run commands in a regular os-shell. + +lib-Adw has a demo application showing all widgets. You can run it by executing + +```bash +adwaita-1-demo +``` + +GTK4 has a demo application showing all widgets. You can run it by executing + +```bash +gtk4-widget-factory +``` + +To find available icons execute + +```bash +gtk4-icon-browser +``` + +### Links + +Here are some important documentation links related to the Clan VM Manager: + +- [Adw PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Adw-1): This link provides the PyGObject reference documentation for the Adw library, which is used in the Clan VM Manager. It contains detailed information about the Adw widgets and their usage. + +- [GTK4 PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Gtk-4.0): This link provides the PyGObject reference documentation for GTK4, the toolkit used for building the user interface of the Clan VM Manager. It includes information about GTK4 widgets, signals, and other features. + +- [Adw Widget Gallery](https://gnome.pages.gitlab.gnome.org/libadwaita/doc/main/widget-gallery.html): This link showcases a widget gallery for Adw, allowing you to see the available widgets and their visual appearance. It can be helpful for designing the user interface of the Clan VM Manager. + +- [Python + GTK3 Tutorial](https://python-gtk-3-tutorial.readthedocs.io/en/latest/textview.html): Although the Clan VM Manager uses GTK4, this tutorial for GTK3 can still be useful as it covers the basics of building GTK-based applications with Python. It includes examples and explanations for various GTK widgets, including text views. + +- [GNOME Human Interface Guidelines](https://developer.gnome.org/hig/): This link provides the GNOME Human Interface Guidelines, which offer design and usability recommendations for creating GNOME applications. It covers topics such as layout, navigation, and interaction patterns. + +## Error handling + +> Error dialogs should be avoided where possible, since they are disruptive. +> +> For simple non-critical errors, toasts can be a good alternative. \ No newline at end of file diff --git a/pkgs/clan-vm-manager/clan_vm_manager/app.py b/pkgs/clan-vm-manager/clan_vm_manager/app.py index 11d61e1f6..a1ce12302 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/app.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/app.py @@ -33,10 +33,8 @@ class MainApplication(Adw.Application): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__( - *args, application_id="org.clan.vm-manager", flags=Gio.ApplicationFlags.HANDLES_COMMAND_LINE, - **kwargs, ) self.add_main_option( @@ -48,7 +46,7 @@ class MainApplication(Adw.Application): None, ) - self.window: Adw.ApplicationWindow | None = None + self.window: "MainWindow" | None = None self.connect("activate", self.on_activate) self.connect("shutdown", self.on_shutdown) @@ -113,8 +111,10 @@ class MainApplication(Adw.Application): log.debug(f"Style css path: {resource_path}") css_provider = Gtk.CssProvider() css_provider.load_from_path(str(resource_path)) + display = Gdk.Display.get_default() + assert display is not None Gtk.StyleContext.add_provider_for_display( - Gdk.Display.get_default(), + display, css_provider, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION, ) diff --git a/pkgs/clan-vm-manager/clan_vm_manager/assets/style.css b/pkgs/clan-vm-manager/clan_vm_manager/assets/style.css index 1284730e2..c179744dd 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/assets/style.css +++ b/pkgs/clan-vm-manager/clan_vm_manager/assets/style.css @@ -17,6 +17,7 @@ avatar { } .join-list { + margin-top: 1px; margin-left: 2px; margin-right: 2px; @@ -56,3 +57,10 @@ avatar { searchbar { margin-bottom: 25px; } + + +.log-view { + margin-top: 12px; + font-family: monospace; + padding: 8px; +} diff --git a/pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py b/pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py index 247e8ca90..d4795f63c 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py @@ -134,8 +134,8 @@ class GKVStore(GObject.GObject, Gio.ListModel, Generic[K, V]): def do_get_item(self, position: int) -> V | None: return self.get_item(position) - def get_item_type(self) -> GObject.GType: - return self.gtype.__gtype__ + def get_item_type(self) -> Any: + return self.gtype.__gtype__ # type: ignore[attr-defined] def do_get_item_type(self) -> GObject.GType: return self.get_item_type() @@ -187,10 +187,10 @@ class GKVStore(GObject.GObject, Gio.ListModel, Generic[K, V]): return len(self._items) # O(1) operation - def __getitem__(self, key: K) -> V: + def __getitem__(self, key: K) -> V: # type: ignore[override] return self._items[key] - def __contains__(self, key: K) -> bool: + def __contains__(self, key: K) -> bool: # type: ignore[override] return key in self._items def __str__(self) -> str: diff --git a/pkgs/clan-vm-manager/clan_vm_manager/components/trayicon.py b/pkgs/clan-vm-manager/clan_vm_manager/components/trayicon.py index 88caf4242..89c900af7 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/components/trayicon.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/components/trayicon.py @@ -24,6 +24,9 @@ import sys from collections.abc import Callable from typing import Any, ClassVar +import gi + +gi.require_version("Gtk", "4.0") from gi.repository import GdkPixbuf, Gio, GLib, Gtk @@ -168,7 +171,7 @@ class ImplUnavailableError(Exception): class BaseImplementation: - def __init__(self, application: Gtk.Application) -> None: + def __init__(self, application: Any) -> None: self.application = application self.menu_items: dict[int, Any] = {} self.menu_item_id: int = 1 @@ -1090,8 +1093,8 @@ class Win32Implementation(BaseImplementation): class TrayIcon: - def __init__(self, application: Gtk.Application) -> None: - self.application: Gtk.Application = application + def __init__(self, application: Gio.Application) -> None: + self.application: Gio.Application = application self.available: bool = True self.implementation: Any = None diff --git a/pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py b/pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py index a713d6434..aa613e1e2 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py @@ -5,7 +5,7 @@ import tempfile import threading import time import weakref -from collections.abc import Generator +from collections.abc import Callable, Generator from contextlib import contextmanager from datetime import datetime from pathlib import Path @@ -13,7 +13,7 @@ from typing import IO, ClassVar import gi from clan_cli import vms -from clan_cli.clan_uri import ClanScheme, ClanURI +from clan_cli.clan_uri import ClanURI from clan_cli.history.add import HistoryEntry from clan_cli.machines.machines import Machine @@ -21,7 +21,7 @@ from clan_vm_manager.components.executor import MPProcess, spawn gi.require_version("GObject", "2.0") gi.require_version("Gtk", "4.0") -from gi.repository import GLib, GObject, Gtk +from gi.repository import Gio, GLib, GObject, Gtk log = logging.getLogger(__name__) @@ -29,34 +29,35 @@ log = logging.getLogger(__name__) class VMObject(GObject.Object): # Define a custom signal with the name "vm_stopped" and a string argument for the message __gsignals__: ClassVar = { - "vm_status_changed": (GObject.SignalFlags.RUN_FIRST, None, []) + "vm_status_changed": (GObject.SignalFlags.RUN_FIRST, None, []), + "vm_build_notify": (GObject.SignalFlags.RUN_FIRST, None, [bool, bool]), } - def _vm_status_changed_task(self) -> bool: - self.emit("vm_status_changed") - return GLib.SOURCE_REMOVE - - def update(self, data: HistoryEntry) -> None: - self.data = data - def __init__( self, icon: Path, data: HistoryEntry, + build_log_cb: Callable[[Gio.File], None], ) -> None: super().__init__() # Store the data from the history entry - self.data = data + self.data: HistoryEntry = data + + self.build_log_cb = build_log_cb # Create a process object to store the VM process - self.vm_process = MPProcess("vm_dummy", mp.Process(), Path("./dummy")) - self.build_process = MPProcess("build_dummy", mp.Process(), Path("./dummy")) + self.vm_process: MPProcess = MPProcess( + "vm_dummy", mp.Process(), Path("./dummy") + ) + self.build_process: MPProcess = MPProcess( + "build_dummy", mp.Process(), Path("./dummy") + ) self._start_thread: threading.Thread = threading.Thread() self.machine: Machine | None = None # Watcher to stop the VM - self.KILL_TIMEOUT = 20 # seconds + self.KILL_TIMEOUT: int = 20 # seconds self._stop_thread: threading.Thread = threading.Thread() # Build progress bar vars @@ -66,7 +67,7 @@ class VMObject(GObject.Object): self.prog_bar_id: int = 0 # Create a temporary directory to store the logs - self.log_dir = tempfile.TemporaryDirectory( + self.log_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory( prefix="clan_vm-", suffix=f"-{self.data.flake.flake_attr}" ) self._logs_id: int = 0 @@ -75,16 +76,26 @@ class VMObject(GObject.Object): # To be able to set the switch state programmatically # we need to store the handler id returned by the connect method # and block the signal while we change the state. This is cursed. - self.switch = Gtk.Switch() + self.switch: Gtk.Switch = Gtk.Switch() self.switch_handler_id: int = self.switch.connect( "notify::active", self._on_switch_toggle ) self.connect("vm_status_changed", self._on_vm_status_changed) # Make sure the VM is killed when the reference to this object is dropped - self._finalizer = weakref.finalize(self, self._kill_ref_drop) + self._finalizer: weakref.finalize = weakref.finalize(self, self._kill_ref_drop) + + def _vm_status_changed_task(self) -> bool: + self.emit("vm_status_changed") + return GLib.SOURCE_REMOVE + + def update(self, data: HistoryEntry) -> None: + self.data = data def _on_vm_status_changed(self, source: "VMObject") -> None: + # Signal may be emited multiple times + self.emit("vm_build_notify", self.is_building(), self.is_running()) + self.switch.set_state(self.is_running() and not self.is_building()) if self.switch.get_sensitive() is False and not self.is_building(): self.switch.set_sensitive(True) @@ -93,9 +104,8 @@ class VMObject(GObject.Object): exit_build = self.build_process.proc.exitcode exitc = exit_vm or exit_build if not self.is_running() and exitc != 0: - self.switch.handler_block(self.switch_handler_id) - self.switch.set_active(False) - self.switch.handler_unblock(self.switch_handler_id) + with self.switch.handler_block(self.switch_handler_id): + self.switch.set_active(False) log.error(f"VM exited with error. Exitcode: {exitc}") def _on_switch_toggle(self, switch: Gtk.Switch, user_state: bool) -> None: @@ -113,19 +123,19 @@ class VMObject(GObject.Object): @contextmanager def _create_machine(self) -> Generator[Machine, None, None]: uri = ClanURI.from_str( - url=self.data.flake.flake_url, flake_attr=self.data.flake.flake_attr + url=str(self.data.flake.flake_url), machine_name=self.data.flake.flake_attr ) - match uri.scheme: - case ClanScheme.LOCAL.value(path): - self.machine = Machine( - name=self.data.flake.flake_attr, - flake=path, # type: ignore - ) - case ClanScheme.REMOTE.value(url): - self.machine = Machine( - name=self.data.flake.flake_attr, - flake=url, # type: ignore - ) + if uri.flake_id.is_local(): + self.machine = Machine( + name=self.data.flake.flake_attr, + flake=uri.flake_id.path, + ) + if uri.flake_id.is_remote(): + self.machine = Machine( + name=self.data.flake.flake_attr, + flake=uri.flake_id.url, + ) + assert self.machine is not None yield self.machine self.machine = None @@ -151,6 +161,14 @@ class VMObject(GObject.Object): machine=machine, tmpdir=log_dir, ) + + gfile = Gio.File.new_for_path(str(log_dir / "build.log")) + # Gio documentation: + # Obtains a file monitor for the given file. + # If no file notification mechanism exists, then regular polling of the file is used. + g_monitor = gfile.monitor_file(Gio.FileMonitorFlags.NONE, None) + g_monitor.connect("changed", self.on_logs_changed) + GLib.idle_add(self._vm_status_changed_task) self.switch.set_sensitive(True) # Start the logs watcher @@ -203,6 +221,18 @@ class VMObject(GObject.Object): log.debug(f"VM {self.get_id()} has stopped") GLib.idle_add(self._vm_status_changed_task) + def on_logs_changed( + self, + monitor: Gio.FileMonitor, + file: Gio.File, + other_file: Gio.File, + event_type: Gio.FileMonitorEvent, + ) -> None: + if event_type == Gio.FileMonitorEvent.CHANGES_DONE_HINT: + # File was changed and the changes were written to disk + # wire up the callback for setting the logs + self.build_log_cb(file) + def start(self) -> None: if self.is_running(): log.warn("VM is already running. Ignoring start request") diff --git a/pkgs/clan-vm-manager/clan_vm_manager/singletons/toast.py b/pkgs/clan-vm-manager/clan_vm_manager/singletons/toast.py new file mode 100644 index 000000000..13d5843cc --- /dev/null +++ b/pkgs/clan-vm-manager/clan_vm_manager/singletons/toast.py @@ -0,0 +1,136 @@ +import logging +from collections.abc import Callable +from typing import Any + +import gi + +gi.require_version("Gtk", "4.0") +gi.require_version("Adw", "1") + +from gi.repository import Adw + +from clan_vm_manager.singletons.use_views import ViewStack +from clan_vm_manager.views.logs import Logs + +log = logging.getLogger(__name__) + + +class ToastOverlay: + """ + The ToastOverlay is a class that manages the display of toasts + It should be used as a singleton in your application to prevent duplicate toasts + Usage + """ + + # For some reason, the adw toast overlay cannot be subclassed + # Thats why it is added as a class property + overlay: Adw.ToastOverlay + active_toasts: set[str] + + _instance: "None | ToastOverlay" = None + + def __init__(self) -> None: + raise RuntimeError("Call use() instead") + + @classmethod + def use(cls: Any) -> "ToastOverlay": + if cls._instance is None: + cls._instance = cls.__new__(cls) + cls.overlay = Adw.ToastOverlay() + cls.active_toasts = set() + + return cls._instance + + def add_toast_unique(self, toast: Adw.Toast, key: str) -> None: + if key not in self.active_toasts: + self.active_toasts.add(key) + self.overlay.add_toast(toast) + toast.connect("dismissed", lambda toast: self.active_toasts.remove(key)) + + +class ErrorToast: + toast: Adw.Toast + + def __init__( + self, message: str, persistent: bool = False, details: str = "" + ) -> None: + super().__init__() + self.toast = Adw.Toast.new( + f"""❌ Error {message}""" + ) + self.toast.set_use_markup(True) + + self.toast.set_priority(Adw.ToastPriority.HIGH) + self.toast.set_button_label("Show more") + + if persistent: + self.toast.set_timeout(0) + + views = ViewStack.use().view + + # we cannot check this type, python is not smart enough + logs_view: Logs = views.get_child_by_name("logs") # type: ignore + logs_view.set_message(details) + + self.toast.connect( + "button-clicked", + lambda _: views.set_visible_child_name("logs"), + ) + + +class WarningToast: + toast: Adw.Toast + + def __init__(self, message: str, persistent: bool = False) -> None: + super().__init__() + self.toast = Adw.Toast.new( + f"⚠ Warning {message}" + ) + self.toast.set_use_markup(True) + + self.toast.set_priority(Adw.ToastPriority.NORMAL) + + if persistent: + self.toast.set_timeout(0) + + +class SuccessToast: + toast: Adw.Toast + + def __init__(self, message: str, persistent: bool = False) -> None: + super().__init__() + self.toast = Adw.Toast.new(f" {message}") + self.toast.set_use_markup(True) + + self.toast.set_priority(Adw.ToastPriority.NORMAL) + + if persistent: + self.toast.set_timeout(0) + + +class LogToast: + toast: Adw.Toast + + def __init__( + self, + message: str, + on_button_click: Callable[[], None], + button_label: str = "More", + persistent: bool = False, + ) -> None: + super().__init__() + self.toast = Adw.Toast.new( + f"""Logs are avilable {message}""" + ) + self.toast.set_use_markup(True) + + self.toast.set_priority(Adw.ToastPriority.NORMAL) + + if persistent: + self.toast.set_timeout(0) + + self.toast.set_button_label(button_label) + self.toast.connect( + "button-clicked", + lambda _: on_button_click(), + ) diff --git a/pkgs/clan-vm-manager/clan_vm_manager/singletons/use_join.py b/pkgs/clan-vm-manager/clan_vm_manager/singletons/use_join.py index b8d9963dc..6e71d1663 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/singletons/use_join.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/singletons/use_join.py @@ -1,7 +1,7 @@ import logging import threading from collections.abc import Callable -from typing import Any, ClassVar +from typing import Any, ClassVar, cast import gi from clan_cli.clan_uri import ClanURI @@ -31,8 +31,8 @@ class JoinValue(GObject.Object): def __init__(self, url: ClanURI) -> None: super().__init__() - self.url = url - self.entry = None + self.url: ClanURI = url + self.entry: HistoryEntry | None = None def __join(self) -> None: new_entry = add_history(self.url) @@ -62,8 +62,8 @@ class JoinList: cls._instance = cls.__new__(cls) cls.list_store = Gio.ListStore.new(JoinValue) - # Rerendering the join list every time an item changes in the clan_store ClanStore.use().register_on_deep_change(cls._instance._rerender_join_list) + return cls._instance def _rerender_join_list( @@ -83,7 +83,9 @@ class JoinList: """ value = JoinValue(uri) - if value.url.get_id() in [item.url.get_id() for item in self.list_store]: + if value.url.machine.get_id() in [ + cast(JoinValue, item).url.machine.get_id() for item in self.list_store + ]: log.info(f"Join request already exists: {value.url}. Ignoring.") return @@ -95,6 +97,7 @@ class JoinList: def _on_join_finished(self, source: JoinValue) -> None: log.info(f"Join finished: {source.url}") self.discard(source) + assert source.entry is not None ClanStore.use().push_history_entry(source.entry) def discard(self, value: JoinValue) -> None: diff --git a/pkgs/clan-vm-manager/clan_vm_manager/singletons/use_vms.py b/pkgs/clan-vm-manager/clan_vm_manager/singletons/use_vms.py index 8ac60d602..220cd96e7 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/singletons/use_vms.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/singletons/use_vms.py @@ -10,10 +10,12 @@ from clan_cli.history.add import HistoryEntry from clan_vm_manager import assets from clan_vm_manager.components.gkvstore import GKVStore from clan_vm_manager.components.vmobj import VMObject +from clan_vm_manager.singletons.use_views import ViewStack +from clan_vm_manager.views.logs import Logs gi.require_version("GObject", "2.0") gi.require_version("Gtk", "4.0") -from gi.repository import GLib +from gi.repository import Gio, GLib log = logging.getLogger(__name__) @@ -27,6 +29,10 @@ class ClanStore: _instance: "None | ClanStore" = None _clan_store: GKVStore[str, VMStore] + # set the vm that is outputting logs + # build logs are automatically streamed to the logs-view + _logging_vm: VMObject | None = None + # Make sure the VMS class is used as a singleton def __init__(self) -> None: raise RuntimeError("Call use() instead") @@ -41,6 +47,13 @@ class ClanStore: return cls._instance + def set_logging_vm(self, ident: str) -> VMObject | None: + vm = self.get_vm(ClanURI(f"clan://{ident}")) + if vm is not None: + self._logging_vm = vm + + return self._logging_vm + def register_on_deep_change( self, callback: Callable[[GKVStore, int, int, int], None] ) -> None: @@ -57,7 +70,7 @@ class ClanStore: store: "GKVStore", position: int, removed: int, added: int ) -> None: if added > 0: - store.register_on_change(on_vmstore_change) + store.values()[position].register_on_change(on_vmstore_change) callback(store, position, removed, added) self.clan_store.register_on_change(on_clanstore_change) @@ -73,18 +86,43 @@ class ClanStore: def push_history_entry(self, entry: HistoryEntry) -> None: # TODO: We shouldn't do this here but in the list view if entry.flake.icon is None: - icon = assets.loc / "placeholder.jpeg" + icon: Path = assets.loc / "placeholder.jpeg" else: - icon = entry.flake.icon + icon = Path(entry.flake.icon) - vm = VMObject( - icon=Path(icon), - data=entry, - ) + def log_details(gfile: Gio.File) -> None: + self.log_details(vm, gfile) + + vm = VMObject(icon=icon, data=entry, build_log_cb=log_details) self.push(vm) + def log_details(self, vm: VMObject, gfile: Gio.File) -> None: + views = ViewStack.use().view + logs_view: Logs = views.get_child_by_name("logs") # type: ignore + + def file_read_callback( + source_object: Gio.File, result: Gio.AsyncResult, _user_data: Any + ) -> None: + try: + # Finish the asynchronous read operation + res = source_object.load_contents_finish(result) + _success, contents, _etag_out = res + + # Convert the byte array to a string and print it + logs_view.set_message(contents.decode("utf-8")) + except Exception as e: + print(f"Error reading file: {e}") + + # only one vm can output logs at a time + if vm == self._logging_vm: + gfile.load_contents_async(None, file_read_callback, None) + else: + log.info("Log details of VM hidden, vm is not current logging VM.") + + # we cannot check this type, python is not smart enough + def push(self, vm: VMObject) -> None: - url = vm.data.flake.flake_url + url = str(vm.data.flake.flake_url) # Only write to the store if the Clan is not already in it # Every write to the KVStore rerenders bound widgets to the clan_store @@ -108,13 +146,14 @@ class ClanStore: vm_store.append(vm) def remove(self, vm: VMObject) -> None: - del self.clan_store[vm.data.flake.flake_url][vm.data.flake.flake_attr] + del self.clan_store[str(vm.data.flake.flake_url)][vm.data.flake.flake_attr] def get_vm(self, uri: ClanURI) -> None | VMObject: - clan = self.clan_store.get(uri.get_internal()) - if clan is None: + vm_store = self.clan_store.get(str(uri.flake_id)) + if vm_store is None: return None - return clan.get(uri.params.flake_attr, None) + machine = vm_store.get(uri.machine.name, None) + return machine def get_running_vms(self) -> list[VMObject]: return [ diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/details.py b/pkgs/clan-vm-manager/clan_vm_manager/views/details.py index 58f5b5950..c9ec2f93f 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/details.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/details.py @@ -1,16 +1,19 @@ import os from collections.abc import Callable from functools import partial -from typing import Any, Literal +from typing import Any, Literal, TypeVar import gi gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GObject, Gtk +# Define a TypeVar that is bound to GObject.Object +ListItem = TypeVar("ListItem", bound=GObject.Object) + def create_details_list( - model: Gio.ListStore, render_row: Callable[[Gtk.ListBox, GObject], Gtk.Widget] + model: Gio.ListStore, render_row: Callable[[Gtk.ListBox, ListItem], Gtk.Widget] ) -> Gtk.ListBox: boxed_list = Gtk.ListBox() boxed_list.set_selection_mode(Gtk.SelectionMode.NONE) @@ -49,7 +52,10 @@ class Details(Gtk.Box): def render_entry_row( self, boxed_list: Gtk.ListBox, item: PreferencesValue ) -> Gtk.Widget: - row = Adw.SpinRow.new_with_range(0, os.cpu_count(), 1) + cores: int | None = os.cpu_count() + fcores = float(cores) if cores else 1.0 + + row = Adw.SpinRow.new_with_range(0, fcores, 1) row.set_value(item.data) return row diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/list.py b/pkgs/clan-vm-manager/clan_vm_manager/views/list.py index f5c4e535e..59377c730 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/list.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/list.py @@ -1,25 +1,36 @@ +import base64 import logging from collections.abc import Callable from functools import partial -from typing import Any +from typing import Any, TypeVar import gi -from clan_cli import history from clan_cli.clan_uri import ClanURI from clan_vm_manager.components.interfaces import ClanConfig from clan_vm_manager.components.vmobj import VMObject +from clan_vm_manager.singletons.toast import ( + LogToast, + ToastOverlay, + WarningToast, +) from clan_vm_manager.singletons.use_join import JoinList, JoinValue +from clan_vm_manager.singletons.use_views import ViewStack from clan_vm_manager.singletons.use_vms import ClanStore, VMStore +from clan_vm_manager.views.logs import Logs gi.require_version("Adw", "1") from gi.repository import Adw, Gdk, Gio, GLib, GObject, Gtk log = logging.getLogger(__name__) +ListItem = TypeVar("ListItem", bound=GObject.Object) +CustomStore = TypeVar("CustomStore", bound=Gio.ListModel) + def create_boxed_list( - model: Gio.ListStore, render_row: Callable[[Gtk.ListBox, GObject], Gtk.Widget] + model: CustomStore, + render_row: Callable[[Gtk.ListBox, ListItem], Gtk.Widget], ) -> Gtk.ListBox: boxed_list = Gtk.ListBox() boxed_list.set_selection_mode(Gtk.SelectionMode.NONE) @@ -47,11 +58,11 @@ class ClanList(Gtk.Box): def __init__(self, config: ClanConfig) -> None: super().__init__(orientation=Gtk.Orientation.VERTICAL) - self.app = Gio.Application.get_default() - self.app.connect("join_request", self.on_join_request) + app = Gio.Application.get_default() + assert app is not None + app.connect("join_request", self.on_join_request) self.log_label: Gtk.Label = Gtk.Label() - self.__init_machines = history.add.list_history() # Add join list self.join_boxed_list = create_boxed_list( @@ -78,9 +89,10 @@ class ClanList(Gtk.Box): add_action = Gio.SimpleAction.new("add", GLib.VariantType.new("s")) add_action.connect("activate", self.on_add) app = Gio.Application.get_default() + assert app is not None app.add_action(add_action) - menu_model = Gio.Menu() + # menu_model = Gio.Menu() # TODO: Make this lazy, blocks UI startup for too long # for vm in machines.list.list_machines(flake_url=vm.data.flake.flake_url): # if vm not in vm_store: @@ -89,10 +101,16 @@ class ClanList(Gtk.Box): box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=5) box.set_valign(Gtk.Align.CENTER) - add_button = Gtk.MenuButton() - add_button.set_has_frame(False) - add_button.set_menu_model(menu_model) - add_button.set_label("Add machine") + add_button = Gtk.Button() + add_button_content = Adw.ButtonContent.new() + add_button_content.set_label("Add machine") + add_button_content.set_icon_name("list-add-symbolic") + add_button.add_css_class("flat") + add_button.set_child(add_button_content) + + # add_button.set_has_frame(False) + # add_button.set_menu_model(menu_model) + # add_button.set_label("Add machine") box.append(add_button) grp.set_header_suffix(box) @@ -157,13 +175,47 @@ class ClanList(Gtk.Box): ## Drop down menu open_action = Gio.SimpleAction.new("edit", GLib.VariantType.new("s")) open_action.connect("activate", self.on_edit) + + action_id = base64.b64encode(vm.get_id().encode("utf-8")).decode("utf-8") + + build_logs_action = Gio.SimpleAction.new( + f"logs.{action_id}", GLib.VariantType.new("s") + ) + + build_logs_action.connect("activate", self.on_show_build_logs) + build_logs_action.set_enabled(False) + app = Gio.Application.get_default() + assert app is not None + app.add_action(open_action) + app.add_action(build_logs_action) + + # set a callback function for conditionally enabling the build_logs action + def on_vm_build_notify( + vm: VMObject, is_building: bool, is_running: bool + ) -> None: + build_logs_action.set_enabled(is_building or is_running) + app.add_action(build_logs_action) + if is_building: + ToastOverlay.use().add_toast_unique( + LogToast( + """Build process running ...""", + on_button_click=lambda: self.show_vm_build_logs(vm.get_id()), + ).toast, + f"info.build.running.{vm}", + ) + + vm.connect("vm_build_notify", on_vm_build_notify) + menu_model = Gio.Menu() menu_model.append("Edit", f"app.edit::{vm.get_id()}") + menu_model.append("Show Logs", f"app.logs.{action_id}::{vm.get_id()}") + pref_button = Gtk.MenuButton() pref_button.set_icon_name("open-menu-symbolic") pref_button.set_menu_model(menu_model) + button_box.append(pref_button) ## VM switch button @@ -178,9 +230,33 @@ class ClanList(Gtk.Box): def on_edit(self, source: Any, parameter: Any) -> None: target = parameter.get_string() - print("Editing settings for machine", target) + def on_show_build_logs(self, _: Any, parameter: Any) -> None: + target = parameter.get_string() + self.show_vm_build_logs(target) + + def show_vm_build_logs(self, target: str) -> None: + vm = ClanStore.use().set_logging_vm(target) + if vm is None: + raise ValueError(f"VM {target} not found") + + views = ViewStack.use().view + # Reset the logs view + logs: Logs = views.get_child_by_name("logs") # type: ignore + + if logs is None: + raise ValueError("Logs view not found") + + name = vm.machine.name if vm.machine else "Unknown" + + logs.set_title(f"""📄 {name}""") + # initial message. Streaming happens automatically when the file is changed by the build process + with open(vm.build_process.out_file) as f: + logs.set_message(f.read()) + + views.set_visible_child_name("logs") + def render_join_row( self, boxed_list: Gtk.ListBox, join_val: JoinValue ) -> Gtk.Widget: @@ -190,8 +266,8 @@ class ClanList(Gtk.Box): log.debug("Rendering join row for %s", join_val.url) row = Adw.ActionRow() - row.set_title(join_val.url.params.flake_attr) - row.set_subtitle(join_val.url.get_internal()) + row.set_title(join_val.url.machine.name) + row.set_subtitle(str(join_val.url)) row.add_css_class("trust") vm = ClanStore.use().get_vm(join_val.url) @@ -199,12 +275,21 @@ class ClanList(Gtk.Box): # Can't do this here because clan store is empty at this point if vm is not None: sub = row.get_subtitle() + assert sub is not None + + ToastOverlay.use().add_toast_unique( + WarningToast( + f"""{join_val.url.machine.name!s} Already exists. Joining again will update it""" + ).toast, + "warning.duplicate.join", + ) + row.set_subtitle( sub + "\nClan already exists. Joining again will update it" ) avatar = Adw.Avatar() - avatar.set_text(str(join_val.url.params.flake_attr)) + avatar.set_text(str(join_val.url.machine.name)) avatar.set_show_initials(True) avatar.set_size(50) row.add_prefix(avatar) @@ -229,7 +314,7 @@ class ClanList(Gtk.Box): def on_join_request(self, source: Any, url: str) -> None: log.debug("Join request: %s", url) - clan_uri = ClanURI.from_str(url) + clan_uri = ClanURI(url) JoinList.use().push(clan_uri, self.on_after_join) def on_after_join(self, source: JoinValue) -> None: diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/logs.py b/pkgs/clan-vm-manager/clan_vm_manager/views/logs.py new file mode 100644 index 000000000..f7fb804f5 --- /dev/null +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/logs.py @@ -0,0 +1,65 @@ +import logging + +import gi + +gi.require_version("Adw", "1") +from gi.repository import Adw, Gio, Gtk + +from clan_vm_manager.singletons.use_views import ViewStack + +log = logging.getLogger(__name__) + + +class Logs(Gtk.Box): + """ + Simple log view + This includes a banner and a text view and a button to close the log and navigate back to the overview + """ + + def __init__(self) -> None: + super().__init__(orientation=Gtk.Orientation.VERTICAL) + + app = Gio.Application.get_default() + assert app is not None + + self.banner = Adw.Banner.new("") + self.banner.set_use_markup(True) + self.banner.set_revealed(True) + self.banner.set_button_label("Close") + + self.banner.connect( + "button-clicked", + lambda _: ViewStack.use().view.set_visible_child_name("list"), + ) + + self.text_view = Gtk.TextView() + self.text_view.set_editable(False) + self.text_view.set_wrap_mode(Gtk.WrapMode.WORD) + self.text_view.add_css_class("log-view") + + self.append(self.banner) + self.append(self.text_view) + + def set_title(self, title: str) -> None: + self.banner.set_title(title) + + def set_message(self, message: str) -> None: + """ + Set the log message. This will delete any previous message + """ + buffer = self.text_view.get_buffer() + buffer.set_text(message) + + mark = buffer.create_mark(None, buffer.get_end_iter(), False) # type: ignore + self.text_view.scroll_to_mark(mark, 0.05, True, 0.0, 1.0) + + def append_message(self, message: str) -> None: + """ + Append to the end of a potentially existent log message + """ + buffer = self.text_view.get_buffer() + end_iter = buffer.get_end_iter() + buffer.insert(end_iter, message) # type: ignore + + mark = buffer.create_mark(None, buffer.get_end_iter(), False) # type: ignore + self.text_view.scroll_to_mark(mark, 0.05, True, 0.0, 1.0) diff --git a/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py b/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py index d78fc81a8..887027325 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py @@ -5,10 +5,12 @@ import gi from clan_cli.history.list import list_history from clan_vm_manager.components.interfaces import ClanConfig +from clan_vm_manager.singletons.toast import ToastOverlay from clan_vm_manager.singletons.use_views import ViewStack from clan_vm_manager.singletons.use_vms import ClanStore from clan_vm_manager.views.details import Details from clan_vm_manager.views.list import ClanList +from clan_vm_manager.views.logs import Logs gi.require_version("Adw", "1") @@ -25,13 +27,17 @@ class MainWindow(Adw.ApplicationWindow): self.set_title("cLAN Manager") self.set_default_size(980, 650) + overlay = ToastOverlay.use().overlay view = Adw.ToolbarView() - self.set_content(view) + overlay.set_child(view) + + self.set_content(overlay) header = Adw.HeaderBar() view.add_top_bar(header) app = Gio.Application.get_default() + assert app is not None self.tray_icon: TrayIcon = TrayIcon(app) # Initialize all ClanStore @@ -40,21 +46,22 @@ class MainWindow(Adw.ApplicationWindow): # Initialize all views stack_view = ViewStack.use().view - scroll = Gtk.ScrolledWindow() - scroll.set_propagate_natural_height(True) - scroll.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC) - scroll.set_child(ClanList(config)) - - stack_view.add_named(scroll, "list") - stack_view.add_named(Details(), "details") - - stack_view.set_visible_child_name(config.initial_view) - clamp = Adw.Clamp() clamp.set_child(stack_view) clamp.set_maximum_size(1000) - view.set_content(clamp) + scroll = Gtk.ScrolledWindow() + scroll.set_propagate_natural_height(True) + scroll.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC) + scroll.set_child(clamp) + + stack_view.add_named(ClanList(config), "list") + stack_view.add_named(Details(), "details") + stack_view.add_named(Logs(), "logs") + + stack_view.set_visible_child_name(config.initial_view) + + view.set_content(scroll) self.connect("destroy", self.on_destroy) diff --git a/pkgs/clan-vm-manager/default.nix b/pkgs/clan-vm-manager/default.nix index 0395fec03..71ab3d60e 100644 --- a/pkgs/clan-vm-manager/default.nix +++ b/pkgs/clan-vm-manager/default.nix @@ -1,15 +1,17 @@ -{ python3 -, runCommand -, setuptools -, copyDesktopItems -, pygobject3 -, wrapGAppsHook -, gtk4 -, gnome -, gobject-introspection -, clan-cli -, makeDesktopItem -, libadwaita +{ + python3, + runCommand, + setuptools, + copyDesktopItems, + pygobject3, + wrapGAppsHook, + gtk4, + gnome, + pygobject-stubs, + gobject-introspection, + clan-cli, + makeDesktopItem, + libadwaita, }: let source = ./.; @@ -22,7 +24,7 @@ let mimeTypes = [ "x-scheme-handler/clan" ]; }; in -python3.pkgs.buildPythonApplication { +python3.pkgs.buildPythonApplication rec { name = "clan-vm-manager"; src = source; format = "pyproject"; @@ -40,12 +42,28 @@ python3.pkgs.buildPythonApplication { gobject-introspection ]; - buildInputs = [ gtk4 libadwaita gnome.adwaita-icon-theme ]; - propagatedBuildInputs = [ pygobject3 clan-cli ]; + buildInputs = [ + gtk4 + libadwaita + gnome.adwaita-icon-theme + ]; + + # We need to propagate the build inputs to nix fmt / treefmt + propagatedBuildInputs = [ + (python3.pkgs.toPythonModule clan-cli) + passthru.externalPythonDeps + ]; # also re-expose dependencies so we test them in CI passthru = { inherit desktop-file; + # Keep external dependencies in a separate lists to refer to thm elsewhere + # This helps avoiding issues like dev-shells accidentally depending on + # nix derivations of local packages. + externalPythonDeps = [ + pygobject3 + pygobject-stubs + ]; tests = { clan-vm-manager-no-breakpoints = runCommand "clan-vm-manager-no-breakpoints" { } '' if grep --include \*.py -Rq "breakpoint()" ${source}; then @@ -66,7 +84,5 @@ python3.pkgs.buildPythonApplication { checkPhase = '' PYTHONPATH= $out/bin/clan-vm-manager --help ''; - desktopItems = [ - desktop-file - ]; + desktopItems = [ desktop-file ]; } diff --git a/pkgs/clan-vm-manager/flake-module.nix b/pkgs/clan-vm-manager/flake-module.nix index 7264c1302..06de8e234 100644 --- a/pkgs/clan-vm-manager/flake-module.nix +++ b/pkgs/clan-vm-manager/flake-module.nix @@ -1,12 +1,15 @@ -{ ... }: { - perSystem = { config, pkgs, ... }: { - devShells.clan-vm-manager = pkgs.callPackage ./shell.nix { - inherit (config.packages) clan-cli clan-vm-manager; - }; - packages.clan-vm-manager = pkgs.python3.pkgs.callPackage ./default.nix { - inherit (config.packages) clan-cli; - }; +{ ... }: +{ + perSystem = + { config, pkgs, ... }: + { + devShells.clan-vm-manager = pkgs.callPackage ./shell.nix { + inherit (config.packages) clan-cli clan-vm-manager; + }; + packages.clan-vm-manager = pkgs.python3.pkgs.callPackage ./default.nix { + inherit (config.packages) clan-cli; + }; - checks = config.packages.clan-vm-manager.tests; - }; + checks = config.packages.clan-vm-manager.tests; + }; } diff --git a/pkgs/clan-vm-manager/pyproject.toml b/pkgs/clan-vm-manager/pyproject.toml index 6f8a2f6fe..8016c21eb 100644 --- a/pkgs/clan-vm-manager/pyproject.toml +++ b/pkgs/clan-vm-manager/pyproject.toml @@ -22,10 +22,6 @@ disallow_untyped_calls = true disallow_untyped_defs = true no_implicit_optional = true -[[tool.mypy.overrides]] -module = "gi.*" -ignore_missing_imports = true - [[tool.mypy.overrides]] module = "clan_cli.*" ignore_missing_imports = true diff --git a/pkgs/clan-vm-manager/screenshots/image.png b/pkgs/clan-vm-manager/screenshots/image.png new file mode 100644 index 000000000..a6f7f4a29 Binary files /dev/null and b/pkgs/clan-vm-manager/screenshots/image.png differ diff --git a/pkgs/clan-vm-manager/shell.nix b/pkgs/clan-vm-manager/shell.nix index 07d355fa1..360944228 100644 --- a/pkgs/clan-vm-manager/shell.nix +++ b/pkgs/clan-vm-manager/shell.nix @@ -1,11 +1,37 @@ -{ lib, runCommand, makeWrapper, stdenv, clan-vm-manager, gdb, gtk4, libadwaita, clan-cli, mkShell, ruff, desktop-file-utils, xdg-utils, mypy, python3, python3Packages }: +{ + lib, + runCommand, + makeWrapper, + stdenv, + clan-vm-manager, + gdb, + gtk4, + libadwaita, + clan-cli, + mkShell, + ruff, + desktop-file-utils, + xdg-utils, + mypy, + python3, + python3Packages, +}: mkShell ( let - pygdb = runCommand "pygdb" { buildInputs = [ gdb python3 makeWrapper ]; } '' - mkdir -p "$out/bin" - makeWrapper "${gdb}/bin/gdb" "$out/bin/pygdb" \ - --add-flags '-ex "source ${python3}/share/gdb/libpython.py"' - ''; + pygdb = + runCommand "pygdb" + { + buildInputs = [ + gdb + python3 + makeWrapper + ]; + } + '' + mkdir -p "$out/bin" + makeWrapper "${gdb}/bin/gdb" "$out/bin/pygdb" \ + --add-flags '-ex "source ${python3}/share/gdb/libpython.py"' + ''; in { inherit (clan-vm-manager) propagatedBuildInputs buildInputs; @@ -15,7 +41,6 @@ mkShell ( pygdb ]; - # To debug clan-vm-manger execute pygdb --args python ./bin/clan-vm-manager nativeBuildInputs = [ ruff @@ -24,7 +49,7 @@ mkShell ( python3Packages.ipdb gtk4.dev libadwaita.devdoc # has the demo called 'adwaita-1-demo' - ] ++ clan-vm-manager.nativeBuildInputs; + ] ++ clan-vm-manager.nativeBuildInputs ++ clan-vm-manager.propagatedBuildInputs; PYTHONBREAKPOINT = "ipdb.set_trace"; diff --git a/pkgs/flake-module.nix b/pkgs/flake-module.nix index 7d44417ce..d446ed955 100644 --- a/pkgs/flake-module.nix +++ b/pkgs/flake-module.nix @@ -1,39 +1,45 @@ -{ ... }: { +{ ... }: +{ imports = [ ./clan-cli/flake-module.nix ./clan-vm-manager/flake-module.nix ./installer/flake-module.nix ]; - perSystem = { pkgs, config, lib, ... }: { - packages = { - tea-create-pr = pkgs.callPackage ./tea-create-pr { }; - zerotier-members = pkgs.callPackage ./zerotier-members { }; - zt-tcp-relay = pkgs.callPackage ./zt-tcp-relay { }; - moonlight-sunshine-accept = pkgs.callPackage ./moonlight-sunshine-accept { }; - merge-after-ci = pkgs.callPackage ./merge-after-ci { - inherit (config.packages) tea-create-pr; - }; - pending-reviews = pkgs.callPackage ./pending-reviews { }; - } // lib.optionalAttrs pkgs.stdenv.isLinux { - wayland-proxy-virtwl = pkgs.callPackage ./wayland-proxy-virtwl { }; - waypipe = pkgs.waypipe.overrideAttrs - (_old: { - # https://gitlab.freedesktop.org/mstoeckl/waypipe - src = pkgs.fetchFromGitLab { - domain = "gitlab.freedesktop.org"; - owner = "mstoeckl"; - repo = "waypipe"; - rev = "4e4ff3bc1943cf7f6aeb56b06c060f40578d3570"; - hash = "sha256-dxz4AmeJAweffyPCayvykworQNntHtHeq6PXMXWsM5k="; - }; - }); - # halalify zerotierone - zerotierone = pkgs.zerotierone.overrideAttrs (_old: { - meta = _old.meta // { - license = lib.licenses.apsl20; + perSystem = + { + pkgs, + config, + lib, + ... + }: + { + packages = + { + tea-create-pr = pkgs.callPackage ./tea-create-pr { }; + zerotier-members = pkgs.callPackage ./zerotier-members { }; + zt-tcp-relay = pkgs.callPackage ./zt-tcp-relay { }; + merge-after-ci = pkgs.callPackage ./merge-after-ci { inherit (config.packages) tea-create-pr; }; + pending-reviews = pkgs.callPackage ./pending-reviews { }; + } + // lib.optionalAttrs pkgs.stdenv.isLinux { + wayland-proxy-virtwl = pkgs.callPackage ./wayland-proxy-virtwl { }; + waypipe = pkgs.waypipe.overrideAttrs (_old: { + # https://gitlab.freedesktop.org/mstoeckl/waypipe + src = pkgs.fetchFromGitLab { + domain = "gitlab.freedesktop.org"; + owner = "mstoeckl"; + repo = "waypipe"; + rev = "4e4ff3bc1943cf7f6aeb56b06c060f40578d3570"; + hash = "sha256-dxz4AmeJAweffyPCayvykworQNntHtHeq6PXMXWsM5k="; + }; + }); + # halalify zerotierone + zerotierone = pkgs.zerotierone.overrideAttrs (_old: { + meta = _old.meta // { + license = lib.licenses.apsl20; + }; + }); }; - }); }; - }; } diff --git a/pkgs/go-ssb/default.nix b/pkgs/go-ssb/default.nix index a650cbea8..a974d11e6 100644 --- a/pkgs/go-ssb/default.nix +++ b/pkgs/go-ssb/default.nix @@ -1,7 +1,7 @@ -{ lib -, buildGoModule -, fetchFromGitHub -, +{ + lib, + buildGoModule, + fetchFromGitHub, }: buildGoModule rec { pname = "go-ssb"; @@ -17,7 +17,10 @@ buildGoModule rec { vendorHash = "sha256-ZytuWFre7Cz6Qt01tLQoPEuNzDIyoC938OkdIrU8nZo="; - ldflags = [ "-s" "-w" ]; + ldflags = [ + "-s" + "-w" + ]; # take very long doCheck = false; diff --git a/pkgs/installer/flake-module.nix b/pkgs/installer/flake-module.nix index 0e9853786..fea1d77b5 100644 --- a/pkgs/installer/flake-module.nix +++ b/pkgs/installer/flake-module.nix @@ -1,19 +1,24 @@ { self, lib, ... }: let - installerModule = { config, pkgs, ... }: { - imports = [ - self.nixosModules.installer - self.nixosModules.hidden-ssh-announce - self.inputs.nixos-generators.nixosModules.all-formats - self.inputs.disko.nixosModules.disko + installerModule = + { config, pkgs, ... }: + { + imports = [ + self.nixosModules.installer + self.inputs.nixos-generators.nixosModules.all-formats + ]; + + system.stateVersion = config.system.nixos.version; + nixpkgs.pkgs = self.inputs.nixpkgs.legacyPackages.x86_64-linux; + }; + + installer = lib.nixosSystem { + modules = [ + installerModule + { disko.memSize = 4096; } # FIXME: otherwise the image builder goes OOM ]; - - system.stateVersion = config.system.nixos.version; - nixpkgs.pkgs = self.inputs.nixpkgs.legacyPackages.x86_64-linux; }; - installer = lib.nixosSystem { modules = [ installerModule ]; }; - clan = self.lib.buildClan { clanName = "clan-core"; directory = self; @@ -24,7 +29,9 @@ in flake.packages.x86_64-linux.install-iso = self.inputs.disko.lib.makeDiskImages { nixosConfig = installer; }; - flake.nixosConfigurations = clan.nixosConfigurations; + flake.nixosConfigurations = { + inherit (clan.nixosConfigurations) installer; + }; flake.clanInternals = clan.clanInternals; flake.apps.x86_64-linux.install-vm.program = installer.config.formats.vm.outPath; flake.apps.x86_64-linux.install-vm-nogui.program = installer.config.formats.vm-nogui.outPath; diff --git a/pkgs/merge-after-ci/default.nix b/pkgs/merge-after-ci/default.nix index 28195dd89..732c720d4 100644 --- a/pkgs/merge-after-ci/default.nix +++ b/pkgs/merge-after-ci/default.nix @@ -1,19 +1,19 @@ -{ bash -, callPackage -, coreutils -, git -, lib -, nix -, openssh -, tea -, tea-create-pr -, ... +{ + bash, + callPackage, + coreutils, + git, + lib, + nix, + openssh, + tea, + tea-create-pr, + ... }: let writers = callPackage ../builders/script-writers.nix { }; in -writers.writePython3Bin "merge-after-ci" -{ +writers.writePython3Bin "merge-after-ci" { makeWrapperArgs = [ "--prefix" "PATH" @@ -28,6 +28,4 @@ writers.writePython3Bin "merge-after-ci" tea-create-pr ]) ]; -} - ./merge-after-ci.py - +} ./merge-after-ci.py diff --git a/pkgs/merge-after-ci/merge-after-ci.py b/pkgs/merge-after-ci/merge-after-ci.py index 759cca5aa..02cf86874 100644 --- a/pkgs/merge-after-ci/merge-after-ci.py +++ b/pkgs/merge-after-ci/merge-after-ci.py @@ -2,7 +2,7 @@ import argparse import subprocess parser = argparse.ArgumentParser() -parser.add_argument("--reviewers", nargs="*") +parser.add_argument("--reviewers", nargs="*", default=[]) parser.add_argument("--no-review", action="store_true") parser.add_argument("args", nargs="*") args = parser.parse_args() @@ -17,8 +17,8 @@ subprocess.run( "origin", "main", "--assignees", - "clan-bot", - *([*args.reviewers] if args.reviewers else []), + ",".join(["clan-bot", *args.reviewers]), + *(["--labels", "needs-review"] if not args.no_review else []), *args.args, ] ) diff --git a/pkgs/pending-reviews/default.nix b/pkgs/pending-reviews/default.nix index dee603174..7aa4cafe2 100644 --- a/pkgs/pending-reviews/default.nix +++ b/pkgs/pending-reviews/default.nix @@ -1,6 +1,7 @@ -{ writeShellApplication -, bash -, curl +{ + writeShellApplication, + bash, + curl, }: writeShellApplication { name = "pending-reviews"; diff --git a/pkgs/tea-create-pr/default.nix b/pkgs/tea-create-pr/default.nix index 15ac802ae..a22b9a282 100644 --- a/pkgs/tea-create-pr/default.nix +++ b/pkgs/tea-create-pr/default.nix @@ -1,9 +1,10 @@ -{ writeShellApplication -, bash -, coreutils -, git -, tea -, openssh +{ + writeShellApplication, + bash, + coreutils, + git, + tea, + openssh, }: writeShellApplication { name = "tea-create-pr"; diff --git a/pkgs/tea-create-pr/script.sh b/pkgs/tea-create-pr/script.sh index a22ae5dd4..8216c027e 100644 --- a/pkgs/tea-create-pr/script.sh +++ b/pkgs/tea-create-pr/script.sh @@ -30,5 +30,4 @@ tea pr create \ --description "$rest" \ --head "$tempRemoteBranch" \ --base "$targetBranch" \ - --labels "needs-review" \ "$@" diff --git a/pkgs/wayland-proxy-virtwl/default.nix b/pkgs/wayland-proxy-virtwl/default.nix index b4303cd2d..0f2f98630 100644 --- a/pkgs/wayland-proxy-virtwl/default.nix +++ b/pkgs/wayland-proxy-virtwl/default.nix @@ -1,4 +1,9 @@ -{ wayland-proxy-virtwl, fetchFromGitHub, libdrm, ocaml-ng }: +{ + wayland-proxy-virtwl, + fetchFromGitHub, + libdrm, + ocaml-ng, +}: let ocaml-wayland = ocaml-ng.ocamlPackages_5_0.wayland.overrideAttrs (_old: { src = fetchFromGitHub { @@ -16,13 +21,15 @@ wayland-proxy-virtwl.overrideAttrs (_old: { rev = "652fca9d4e006a2bdeba920dfaf53190c5373a7d"; hash = "sha256-VgpqxjHgueK9eQSX987PF0KvscpzkScOzFkW3haYCOw="; }; - buildInputs = [ libdrm ] ++ (with ocaml-ng.ocamlPackages_5_0; [ - ocaml-wayland - dune-configurator - eio_main - ppx_cstruct - cmdliner - logs - ppx_cstruct - ]); + buildInputs = + [ libdrm ] + ++ (with ocaml-ng.ocamlPackages_5_0; [ + ocaml-wayland + dune-configurator + eio_main + ppx_cstruct + cmdliner + logs + ppx_cstruct + ]); }) diff --git a/pkgs/zerotier-members/default.nix b/pkgs/zerotier-members/default.nix index 450b0793d..550159147 100644 --- a/pkgs/zerotier-members/default.nix +++ b/pkgs/zerotier-members/default.nix @@ -1,4 +1,8 @@ -{ stdenv, python3, lib }: +{ + stdenv, + python3, + lib, +}: stdenv.mkDerivation { name = "zerotier-members"; diff --git a/pkgs/zt-tcp-relay/default.nix b/pkgs/zt-tcp-relay/default.nix index 3bdd313e5..1dd6aa361 100644 --- a/pkgs/zt-tcp-relay/default.nix +++ b/pkgs/zt-tcp-relay/default.nix @@ -1,6 +1,7 @@ -{ lib -, rustPlatform -, fetchFromGitHub +{ + lib, + rustPlatform, + fetchFromGitHub, }: rustPlatform.buildRustPackage { diff --git a/templates/flake-module.nix b/templates/flake-module.nix index d44882f36..b8fe8e9fa 100644 --- a/templates/flake-module.nix +++ b/templates/flake-module.nix @@ -1,4 +1,5 @@ -{ self, ... }: { +{ self, ... }: +{ flake.templates = { new-clan = { description = "Initialize a new clan flake"; diff --git a/templates/new-clan/flake.nix b/templates/new-clan/flake.nix index 38acce83a..2f34b17c1 100644 --- a/templates/new-clan/flake.nix +++ b/templates/new-clan/flake.nix @@ -3,7 +3,8 @@ inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core"; - outputs = { self, clan-core, ... }: + outputs = + { self, clan-core, ... }: let system = "x86_64-linux"; pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system}; @@ -17,9 +18,7 @@ inherit (clan) nixosConfigurations clanInternals; # add the cLAN cli tool to the dev shell devShells.${system}.default = pkgs.mkShell { - packages = [ - clan-core.packages.${system}.clan-cli - ]; + packages = [ clan-core.packages.${system}.clan-cli ]; }; }; }