diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index 839c74cb1..cc50bc0ad 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -1,6 +1,6 @@ name: deploy on: - push: + push: branches: - main jobs: @@ -10,4 +10,4 @@ jobs: - uses: actions/checkout@v3 - run: nix run .#deploy-docs env: - SSH_HOMEPAGE_KEY: ${{ secrets.SSH_HOMEPAGE_KEY }} \ No newline at end of file + SSH_HOMEPAGE_KEY: ${{ secrets.SSH_HOMEPAGE_KEY }} diff --git a/.gitignore b/.gitignore index 84945a95e..890024cdb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .direnv +**/.nixos-test-history ***/.hypothesis out.log .coverage.* diff --git a/checks/devshell/flake-module.nix b/checks/devshell/flake-module.nix new file mode 100644 index 000000000..2e9ea7641 --- /dev/null +++ b/checks/devshell/flake-module.nix @@ -0,0 +1,22 @@ +{ ... }: +{ + perSystem = + { self', pkgs, ... }: + { + checks.devshell = + pkgs.runCommand "check-devshell-not-depends-on-clan-cli" + { + exportReferencesGraph = [ + "graph" + self'.devShells.default + ]; + } + '' + if grep -q "${self'.packages.clan-cli}" ./graph; then + echo "devshell depends on clan-cli, which is not allowed"; + exit 1; + fi + mkdir $out + ''; + }; +} diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 6b591aa55..a6de83757 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -1,10 +1,11 @@ { self, ... }: { imports = [ - ./impure/flake-module.nix ./backups/flake-module.nix - ./installation/flake-module.nix + ./devshell/flake-module.nix ./flash/flake-module.nix + ./impure/flake-module.nix + ./installation/flake-module.nix ]; perSystem = { @@ -40,10 +41,11 @@ secrets = import ./secrets nixosTestArgs; container = import ./container nixosTestArgs; deltachat = import ./deltachat nixosTestArgs; - matrix-synapse = import ./matrix-synapse nixosTestArgs; - zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs; borgbackup = import ./borgbackup nixosTestArgs; + matrix-synapse = import ./matrix-synapse nixosTestArgs; + mumble = import ./mumble nixosTestArgs; syncthing = import ./syncthing nixosTestArgs; + zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs; postgresql = import ./postgresql nixosTestArgs; wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs; }; diff --git a/checks/mumble/default.nix b/checks/mumble/default.nix new file mode 100644 index 000000000..d29c8a7ae --- /dev/null +++ b/checks/mumble/default.nix @@ -0,0 +1,146 @@ +(import ../lib/test-base.nix) ( + { ... }: + let + common = + { self, pkgs, ... }: + { + imports = [ + self.clanModules.mumble + self.nixosModules.clanCore + (self.inputs.nixpkgs + "/nixos/tests/common/x11.nix") + { + clan.core.clanDir = ./.; + environment.systemPackages = [ pkgs.killall ]; + services.murmur.sslKey = "/etc/mumble-key"; + services.murmur.sslCert = "/etc/mumble-cert"; + clan.core.facts.services.mumble.secret."mumble-key".path = "/etc/mumble-key"; + clan.core.facts.services.mumble.public."mumble-cert".path = "/etc/mumble-cert"; + } + ]; + + }; + in + { + name = "mumble"; + + enableOCR = true; + + nodes.peer1 = + { ... }: + { + imports = [ + common + { + clan.core.machineName = "peer1"; + environment.etc = { + "mumble-key".source = ./peer_1/peer_1_test_key; + "mumble-cert".source = ./peer_1/peer_1_test_cert; + }; + systemd.tmpfiles.settings."vmsecrets" = { + "/etc/secrets/mumble-key" = { + C.argument = "${./peer_1/peer_1_test_key}"; + z = { + mode = "0400"; + user = "murmur"; + }; + }; + "/etc/secrets/mumble-cert" = { + C.argument = "${./peer_1/peer_1_test_cert}"; + z = { + mode = "0400"; + user = "murmur"; + }; + }; + }; + services.murmur.sslKey = "/etc/mumble-key"; + services.murmur.sslCert = "/etc/mumble-cert"; + clan.core.facts.services.mumble.secret."mumble-key".path = "/etc/mumble-key"; + clan.core.facts.services.mumble.public."mumble-cert".path = "/etc/mumble-cert"; + } + ]; + }; + nodes.peer2 = + { ... }: + { + imports = [ + common + { + clan.core.machineName = "peer2"; + environment.etc = { + "mumble-key".source = ./peer_2/peer_2_test_key; + "mumble-cert".source = ./peer_2/peer_2_test_cert; + }; + systemd.tmpfiles.settings."vmsecrets" = { + "/etc/secrets/mumble-key" = { + C.argument = "${./peer_2/peer_2_test_key}"; + z = { + mode = "0400"; + user = "murmur"; + }; + }; + "/etc/secrets/mumble-cert" = { + C.argument = "${./peer_2/peer_2_test_cert}"; + z = { + mode = "0400"; + user = "murmur"; + }; + }; + }; + } + ]; + }; + testScript = '' + start_all() + + with subtest("Waiting for x"): + peer1.wait_for_x() + peer2.wait_for_x() + + with subtest("Waiting for murmur"): + peer1.wait_for_unit("murmur.service") + peer2.wait_for_unit("murmur.service") + + with subtest("Starting Mumble"): + # starting mumble is blocking + peer1.execute("mumble >&2 &") + peer2.execute("mumble >&2 &") + + with subtest("Wait for Mumble"): + peer1.wait_for_window(r"^Mumble$") + peer2.wait_for_window(r"^Mumble$") + + with subtest("Wait for certificate creation"): + peer1.wait_for_window(r"^Mumble$") + peer1.sleep(3) # mumble is slow to register handlers + peer1.send_chars("\n") + peer1.send_chars("\n") + peer2.wait_for_window(r"^Mumble$") + peer2.sleep(3) # mumble is slow to register handlers + peer2.send_chars("\n") + peer2.send_chars("\n") + + with subtest("Wait for server connect"): + peer1.wait_for_window(r"^Mumble Server Connect$") + peer2.wait_for_window(r"^Mumble Server Connect$") + + with subtest("Check validity of server certificates"): + peer1.execute("killall .mumble-wrapped") + peer1.sleep(1) + peer1.execute("mumble mumble://peer2 >&2 &") + peer1.wait_for_window(r"^Mumble$") + peer1.sleep(3) # mumble is slow to register handlers + peer1.send_chars("\n") + peer1.send_chars("\n") + peer1.wait_for_text("Connected.") + + peer2.execute("killall .mumble-wrapped") + peer2.sleep(1) + peer2.execute("mumble mumble://peer1 >&2 &") + peer2.wait_for_window(r"^Mumble$") + peer2.sleep(3) # mumble is slow to register handlers + peer2.send_chars("\n") + peer2.send_chars("\n") + peer2.wait_for_text("Connected.") + ''; + } +) diff --git a/checks/mumble/machines/peer1/facts/mumble-cert b/checks/mumble/machines/peer1/facts/mumble-cert new file mode 100644 index 000000000..9d8d6c654 --- /dev/null +++ b/checks/mumble/machines/peer1/facts/mumble-cert @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUCUjfNkF0CDhTKbO3nNczcsCW4qEwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA2MjcwOTM2NDZaFw0yNDA3 +MjcwOTM2NDZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDCcdZEJvXJIeOKO5pF5XUFvUeJtCCiwfWvWS662bxc +R/5MZucRLqfTNYo9aBv4NITw5kxZsTaaubmS4zSGQoTEAVzqzVdi3a/gNvsdVLb+ +7CivpmweLllX/OGsTL0kHPEI+74AYiTBjXfdWV1Y5T1tuwc3G8ATrguQ33Uo5vvF +vcqsbTKcRZC0pB9O/nn4q03GsRdvlpaKakIhjMpRG/uZ3u7wtbyZ+WqjsjxZNfnY +aMyPoaipFqX1v+L7GKlOj2NpyEZFVVwa2ZqhVSYXyDfpAWQFznwKGzD5mjtcyKym +gnv/5LwrpH4Xj+JMt48hN+rPnu5vfXT8Y4KnID30OQW7AgMBAAGjUzBRMB0GA1Ud +DgQWBBQBBO8Wp975pAGioMjkaxANAVInfzAfBgNVHSMEGDAWgBQBBO8Wp975pAGi +oMjkaxANAVInfzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAg +F40MszTZXpR/A1z9B1CcXH47tNK67f8bCMR2dhvXODbpatwSihyxhQjtLb5R6kYH +5Yq/B4yrh303j0CXaobCQ4nQH7zI7fhViww+TzW7vDhgM7ueEyyXrqCXt6JY8avg +TuvIRtJSeWSQJ5aLNaYqmiwMf/tj9W3BMDpctGyLqu1WTSrbpYa9mA5Vudud70Yz +DgZ/aqHilB07cVNqzVYZzRZ56WJlTjGzVevRgnHZqPiZNVrU13H6gtWa3r8aV4Gj +i4F663eRAttj166cRgfl1QqpSG2IprNyV9UfuS2LlUaVNT3y0idawiJ4HhaA8pGB +ZqMUUkA4DSucb6xxEcTK +-----END CERTIFICATE----- + diff --git a/checks/mumble/machines/peer1/key.age b/checks/mumble/machines/peer1/key.age new file mode 100644 index 000000000..1c9755ab6 --- /dev/null +++ b/checks/mumble/machines/peer1/key.age @@ -0,0 +1 @@ +AGE-SECRET-KEY-1UCXEUJH6JXF8LFKWFHDM4N9AQE2CCGQZGXLUNV4TKR5KY0KC8FDQ2TY4NX diff --git a/checks/mumble/machines/peer1/peer_1_test_cert b/checks/mumble/machines/peer1/peer_1_test_cert new file mode 100644 index 000000000..effa81269 --- /dev/null +++ b/checks/mumble/machines/peer1/peer_1_test_cert @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHTCCAaKgAwIBAgIIT2gZuvqVFP0wCgYIKoZIzj0EAwIwSjESMBAGA1UEChMJ +U3luY3RoaW5nMSAwHgYDVQQLExdBdXRvbWF0aWNhbGx5IEdlbmVyYXRlZDESMBAG +A1UEAxMJc3luY3RoaW5nMB4XDTIzMTIwNjAwMDAwMFoXDTQzMTIwMTAwMDAwMFow +SjESMBAGA1UEChMJU3luY3RoaW5nMSAwHgYDVQQLExdBdXRvbWF0aWNhbGx5IEdl +bmVyYXRlZDESMBAGA1UEAxMJc3luY3RoaW5nMHYwEAYHKoZIzj0CAQYFK4EEACID +YgAEBAr1CsciwCa0vi7eC6xxuSGijY3txbjtsyFanec/fge4oJBD3rVpaLKFETb3 +TvHHsuvblzElcP483MEVq6FMUoxwuL9CzTtpJrRhtwSmAs8AHLFu8irVn8sZjgkL +sXMho1UwUzAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG +AQUFBwMCMAwGA1UdEwEB/wQCMAAwFAYDVR0RBA0wC4IJc3luY3RoaW5nMAoGCCqG +SM49BAMCA2kAMGYCMQDbrtLgfcyMMIkNQn+PJe9DHYAqj8C47LQcWuIY/nekhOu0 +aUfKctEAwyBtI60Y5zcCMQCEdgD/6CNBh7Qqq3z3CKPhlrpxHtCO5tNw17k0jfdH +haCwJInHZvZgclHk4EtFpTw= +-----END CERTIFICATE----- diff --git a/checks/mumble/machines/peer1/peer_1_test_key b/checks/mumble/machines/peer1/peer_1_test_key new file mode 100644 index 000000000..101f810c4 --- /dev/null +++ b/checks/mumble/machines/peer1/peer_1_test_key @@ -0,0 +1,6 @@ +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDA14Nqo17Xs/xRLGH2KLuyzjKp4eW9iWFobVNM93RZZbECT++W3XcQc +cEc5WVtiPmWgBwYFK4EEACKhZANiAAQECvUKxyLAJrS+Lt4LrHG5IaKNje3FuO2z +IVqd5z9+B7igkEPetWlosoURNvdO8cey69uXMSVw/jzcwRWroUxSjHC4v0LNO2km +tGG3BKYCzwAcsW7yKtWfyxmOCQuxcyE= +-----END EC PRIVATE KEY----- diff --git a/checks/mumble/machines/peer2/facts/mumble-cert b/checks/mumble/machines/peer2/facts/mumble-cert new file mode 100644 index 000000000..cbbae2413 --- /dev/null +++ b/checks/mumble/machines/peer2/facts/mumble-cert @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUfENbTtH5nr7giuawwQpDYqUpWJswDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA2MjcwOTQxNDNaFw0yNDA3 +MjcwOTQxNDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCfP6cZhCs9jOnWqyQP12vrOOxlBrWofYZFf9amUA24 +AfE7oGcSfkylanmkxzvGqQkhgLAvkHZj/GEvHujKyy8PgcEGP+pwmsfWNQMvU0Dz +j3syjWOTi3eIC/3DoUnHlWCT2qCil/bjqxgU1l7fO/OXUlq5kyvIjln7Za4sUHun +ixe/m96Er6l8a4Mh2pxh2C5pkLCvulkQhjjGG+R6MccH8wwQwmLg5oVBkFEZrnRE +pnRKBI0DvA+wk1aJFAPOI4d8Q5T7o/MyxH3f8TYGHqbeMQFCKwusnlWPRtrNdaIc +gaLvSpR0LVlroXGu8tYmRpvHPByoKGDbgVvO0Bwx8fmRAgMBAAGjUzBRMB0GA1Ud +DgQWBBR7r+mQWNUZ0TpQNwrwjgxgngvOjTAfBgNVHSMEGDAWgBR7r+mQWNUZ0TpQ +NwrwjgxgngvOjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCO +7B4s6uQEGE8jg3CQgy76oU/D8sazGcP8+/E4JLHSc0Nj49w4ztSpkOVk2HyEtzbm +uR3TreIw+SfqpbiOI/ivVNDbEBsb/vEeq7qPzDH1Bi72plHZNRVhNGGV5rd7ibga +TkfXHKPM9yt8ffffHHiu1ROvb8gg2B6JbQwboU4hvvmmorW7onyTFSYEzZVdNSpv +pUtKPldxYjTnLlbsJdXC4xyCC4PrJt2CC0n0jsWfICJ77LMxIxTODh8oZNjbPg6r +RdI7U/DsD+R072DjbIcrivvigotJM+jihzz5inZwbO8o0WQOHAbJLIG3C3BnRW3A +Ek4u3+HXZMl5a0LGJ76u +-----END CERTIFICATE----- + diff --git a/checks/mumble/machines/peer2/peer_2_test_cert b/checks/mumble/machines/peer2/peer_2_test_cert new file mode 100644 index 000000000..b0830f0ef --- /dev/null +++ b/checks/mumble/machines/peer2/peer_2_test_cert @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHjCCAaOgAwIBAgIJAKbMWefkf1rVMAoGCCqGSM49BAMCMEoxEjAQBgNVBAoT +CVN5bmN0aGluZzEgMB4GA1UECxMXQXV0b21hdGljYWxseSBHZW5lcmF0ZWQxEjAQ +BgNVBAMTCXN5bmN0aGluZzAeFw0yMzEyMDYwMDAwMDBaFw00MzEyMDEwMDAwMDBa +MEoxEjAQBgNVBAoTCVN5bmN0aGluZzEgMB4GA1UECxMXQXV0b21hdGljYWxseSBH +ZW5lcmF0ZWQxEjAQBgNVBAMTCXN5bmN0aGluZzB2MBAGByqGSM49AgEGBSuBBAAi +A2IABFZTMt4RfsfBue0va7QuNdjfXMI4HfZzJCEcG+b9MtV7FlDmwMKX5fgGykD9 +FBbC7yiza3+xCobdMb5bakz1qYJ7nUFCv1mwSDo2eNM+/XE+rJmlre8NwkwGmvzl +h1uhyqNVMFMwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr +BgEFBQcDAjAMBgNVHRMBAf8EAjAAMBQGA1UdEQQNMAuCCXN5bmN0aGluZzAKBggq +hkjOPQQDAgNpADBmAjEAwzhsroN6R4/quWeXj6dO5gt5CfSTLkLee6vrcuIP5i1U +rZvJ3OKQVmmGG6IWYe7iAjEAyuq3X2wznaqiw2YK3IDI4qVeYWpCUap0fwRNq7/x +4dC4k+BOzHcuJOwNBIY/bEuK +-----END CERTIFICATE----- diff --git a/checks/mumble/machines/peer2/peer_2_test_key b/checks/mumble/machines/peer2/peer_2_test_key new file mode 100644 index 000000000..7b9b28a04 --- /dev/null +++ b/checks/mumble/machines/peer2/peer_2_test_key @@ -0,0 +1,6 @@ +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDCXHGpvumKjjDRxB6SsjZOb7duw3w+rdlGQCJTIvRThLjD6zwjnyImi +7c3PD5nWtLqgBwYFK4EEACKhZANiAARWUzLeEX7HwbntL2u0LjXY31zCOB32cyQh +HBvm/TLVexZQ5sDCl+X4BspA/RQWwu8os2t/sQqG3TG+W2pM9amCe51BQr9ZsEg6 +NnjTPv1xPqyZpa3vDcJMBpr85Ydboco= +-----END EC PRIVATE KEY----- diff --git a/checks/mumble/peer_1/key.age b/checks/mumble/peer_1/key.age new file mode 100644 index 000000000..1c9755ab6 --- /dev/null +++ b/checks/mumble/peer_1/key.age @@ -0,0 +1 @@ +AGE-SECRET-KEY-1UCXEUJH6JXF8LFKWFHDM4N9AQE2CCGQZGXLUNV4TKR5KY0KC8FDQ2TY4NX diff --git a/checks/mumble/peer_1/peer_1_test_cert b/checks/mumble/peer_1/peer_1_test_cert new file mode 100644 index 000000000..9d8d6c654 --- /dev/null +++ b/checks/mumble/peer_1/peer_1_test_cert @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUCUjfNkF0CDhTKbO3nNczcsCW4qEwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA2MjcwOTM2NDZaFw0yNDA3 +MjcwOTM2NDZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDCcdZEJvXJIeOKO5pF5XUFvUeJtCCiwfWvWS662bxc +R/5MZucRLqfTNYo9aBv4NITw5kxZsTaaubmS4zSGQoTEAVzqzVdi3a/gNvsdVLb+ +7CivpmweLllX/OGsTL0kHPEI+74AYiTBjXfdWV1Y5T1tuwc3G8ATrguQ33Uo5vvF +vcqsbTKcRZC0pB9O/nn4q03GsRdvlpaKakIhjMpRG/uZ3u7wtbyZ+WqjsjxZNfnY +aMyPoaipFqX1v+L7GKlOj2NpyEZFVVwa2ZqhVSYXyDfpAWQFznwKGzD5mjtcyKym +gnv/5LwrpH4Xj+JMt48hN+rPnu5vfXT8Y4KnID30OQW7AgMBAAGjUzBRMB0GA1Ud +DgQWBBQBBO8Wp975pAGioMjkaxANAVInfzAfBgNVHSMEGDAWgBQBBO8Wp975pAGi +oMjkaxANAVInfzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAg +F40MszTZXpR/A1z9B1CcXH47tNK67f8bCMR2dhvXODbpatwSihyxhQjtLb5R6kYH +5Yq/B4yrh303j0CXaobCQ4nQH7zI7fhViww+TzW7vDhgM7ueEyyXrqCXt6JY8avg +TuvIRtJSeWSQJ5aLNaYqmiwMf/tj9W3BMDpctGyLqu1WTSrbpYa9mA5Vudud70Yz +DgZ/aqHilB07cVNqzVYZzRZ56WJlTjGzVevRgnHZqPiZNVrU13H6gtWa3r8aV4Gj +i4F663eRAttj166cRgfl1QqpSG2IprNyV9UfuS2LlUaVNT3y0idawiJ4HhaA8pGB +ZqMUUkA4DSucb6xxEcTK +-----END CERTIFICATE----- + diff --git a/checks/mumble/peer_1/peer_1_test_key b/checks/mumble/peer_1/peer_1_test_key new file mode 100644 index 000000000..c52c49f27 --- /dev/null +++ b/checks/mumble/peer_1/peer_1_test_key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDCcdZEJvXJIeOK +O5pF5XUFvUeJtCCiwfWvWS662bxcR/5MZucRLqfTNYo9aBv4NITw5kxZsTaaubmS +4zSGQoTEAVzqzVdi3a/gNvsdVLb+7CivpmweLllX/OGsTL0kHPEI+74AYiTBjXfd +WV1Y5T1tuwc3G8ATrguQ33Uo5vvFvcqsbTKcRZC0pB9O/nn4q03GsRdvlpaKakIh +jMpRG/uZ3u7wtbyZ+WqjsjxZNfnYaMyPoaipFqX1v+L7GKlOj2NpyEZFVVwa2Zqh +VSYXyDfpAWQFznwKGzD5mjtcyKymgnv/5LwrpH4Xj+JMt48hN+rPnu5vfXT8Y4Kn +ID30OQW7AgMBAAECggEAGVKn+/Iy+kG+l2cRvV6XseqnoWhjA69M5swviMgIfuAl +Xx/boeI4mwoS+dJQKi/0zEbB1MB+gwIDB/0s/vs0vS4MQswBQG/skr+2TmiU+Hgb +CF0dIYUZv5rAbScFTumx/mCCqxwc+1QIMzyLKqOYL203EFc92ZJGEVT4th321haZ +8Wd+dllcYAb7BbEeBhCrTqRe9T3zt5reZgtZTquTF5hGm8EAyBp6rLjZK7dyZ9dd +gyIsDbWgPC9vkRc6x/eANn70hgDbYOuoXwAP/qIFnWLL1Zzy8LKUyOsSgQ91S3S3 +Il4Lt6lEyU3+61MsCYss7jDoP/7REEjz5h6gfxlFSQKBgQD9u8nhHuwte4/d9VNU +rhSBW9h8IJzwPif/eS8vh9VaS2SjR2dDCcHg6rGYKnexeEzUcx56aQMA+p3nRJwy +Uwnx5BfEWs9FO6yPR8VEI0a2sBp+hoWKJX/Lvat+QCs6IFuGmlQpczD7/RYAkhG4 +mwyt/ymqzjukb9mFaeYIltOfPwKBgQDELnkH1ChTUH5u3HgDoelFbzR18okz6dxH +urMbfZMAl8W5h2zAvHsAX5qxyHHankOUsiH2y3BrAgqQtTuIA2a5W7j+yHBkYiEZ +EUNeI9YNA0KU+wwZpVVvRGUsRB5SUBo5LlcSYmX/V32f0oU5Np44i0vjl3Ju8esx +2MLfj1A2hQKBgQDCxtZZZ0h8Pb8Z7wpSFfQNvXi5CLwQvFYuClQLk6VXVErkAJsn +XiUjyGYeXnNVm/i2mcyKwXQZ20k90HBrPU2ED8mi5Ob5ya5Uqw6mmMHe2d7sw81d +WB37RBWSrCXC0DYSZQQ4cYHn3sd2Fqtd4EBijV7qDLjCKU582OdKLqYzNwKBgH31 +UKQkJZgIkIThbPT4GewI0GgCRvFb76DmUGUQJTg2Oi86siq1WUwOFiabie5RuxZX +oNLyH8W008/BbO2RMX1FVOvRCciJ8LJFkTl6TM6iDzfUUBqPOuFryoG3Yrh60btw +81rMbqyZIgFhi0QGu2OWnC0Oadyt2tJwV/5t55R5AoGBAPspZttDmOzVkAJDSn9Z +iByYt1KmwBQ6l7LpFg33a7ds9zWqW4+i6r0PzXvSewf/z69L0cAywSk5CaJJjDso +dTlNMqwux01wd6V+nQGR871xnsOg+qzgJ565TJZelWgRmNRUooi4DMp5POJA33xp +rqAISUfW0w2S+q7/5Lm0QiJE +-----END PRIVATE KEY----- diff --git a/checks/mumble/peer_2/peer_2_test_cert b/checks/mumble/peer_2/peer_2_test_cert new file mode 100644 index 000000000..cbbae2413 --- /dev/null +++ b/checks/mumble/peer_2/peer_2_test_cert @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUfENbTtH5nr7giuawwQpDYqUpWJswDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA2MjcwOTQxNDNaFw0yNDA3 +MjcwOTQxNDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCfP6cZhCs9jOnWqyQP12vrOOxlBrWofYZFf9amUA24 +AfE7oGcSfkylanmkxzvGqQkhgLAvkHZj/GEvHujKyy8PgcEGP+pwmsfWNQMvU0Dz +j3syjWOTi3eIC/3DoUnHlWCT2qCil/bjqxgU1l7fO/OXUlq5kyvIjln7Za4sUHun +ixe/m96Er6l8a4Mh2pxh2C5pkLCvulkQhjjGG+R6MccH8wwQwmLg5oVBkFEZrnRE +pnRKBI0DvA+wk1aJFAPOI4d8Q5T7o/MyxH3f8TYGHqbeMQFCKwusnlWPRtrNdaIc +gaLvSpR0LVlroXGu8tYmRpvHPByoKGDbgVvO0Bwx8fmRAgMBAAGjUzBRMB0GA1Ud +DgQWBBR7r+mQWNUZ0TpQNwrwjgxgngvOjTAfBgNVHSMEGDAWgBR7r+mQWNUZ0TpQ +NwrwjgxgngvOjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCO +7B4s6uQEGE8jg3CQgy76oU/D8sazGcP8+/E4JLHSc0Nj49w4ztSpkOVk2HyEtzbm +uR3TreIw+SfqpbiOI/ivVNDbEBsb/vEeq7qPzDH1Bi72plHZNRVhNGGV5rd7ibga +TkfXHKPM9yt8ffffHHiu1ROvb8gg2B6JbQwboU4hvvmmorW7onyTFSYEzZVdNSpv +pUtKPldxYjTnLlbsJdXC4xyCC4PrJt2CC0n0jsWfICJ77LMxIxTODh8oZNjbPg6r +RdI7U/DsD+R072DjbIcrivvigotJM+jihzz5inZwbO8o0WQOHAbJLIG3C3BnRW3A +Ek4u3+HXZMl5a0LGJ76u +-----END CERTIFICATE----- + diff --git a/checks/mumble/peer_2/peer_2_test_key b/checks/mumble/peer_2/peer_2_test_key new file mode 100644 index 000000000..fe022bdd6 --- /dev/null +++ b/checks/mumble/peer_2/peer_2_test_key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCfP6cZhCs9jOnW +qyQP12vrOOxlBrWofYZFf9amUA24AfE7oGcSfkylanmkxzvGqQkhgLAvkHZj/GEv +HujKyy8PgcEGP+pwmsfWNQMvU0Dzj3syjWOTi3eIC/3DoUnHlWCT2qCil/bjqxgU +1l7fO/OXUlq5kyvIjln7Za4sUHunixe/m96Er6l8a4Mh2pxh2C5pkLCvulkQhjjG +G+R6MccH8wwQwmLg5oVBkFEZrnREpnRKBI0DvA+wk1aJFAPOI4d8Q5T7o/MyxH3f +8TYGHqbeMQFCKwusnlWPRtrNdaIcgaLvSpR0LVlroXGu8tYmRpvHPByoKGDbgVvO +0Bwx8fmRAgMBAAECggEACAkjOnNj5zA0IIP0RuRc6rqtmw9ynTTwUJN51lyVxKI8 +dQDMEq/S2En+J2VyS7z92/XtbgkBIFx83u7VWl5UWpj2j4UsJFB7IwD7zyiJT4D+ ++3cM/kX8Wx4XyQZbfbm47N0MXAgFCkn45hxHH0acLReXwmN9wxoDyl7AIjZRdwvG +Qq0rnOnIc8kkkew7L6AiFwQS8b77eyzua3d6moKXN9hU/kfiJ6YUFG/WLe0pmQA1 +HbF27YghfeLnYUt50oDuX6jF6CzQhflchWVq/wn8/cxEpg/RMicWE8ulrTk7o27l +JwCrHrhYEBsPuZO4mxX/DHrAMmhTeFjLaV5bQlz0PQKBgQDgRPSOEixYnKz9iPs/ +EDTlji5LA3Rm6TytRCNsjYY6Trw60KcvYqwyDUCiEjruvOQ9mqgBiQm1VHSalrG3 +RcbVfpEMouyZbEwmTjS8KdOi5x4Z6AX+4yWDN31jX3b8sktgbxV/HRdg3sA3q7MJ +vExTUuoXg57W+FepIZ+XlhSoQwKBgQC1x6UMAlAeW45/yUUm/LFRcCgb/bdCQx+e +hSb8w3jdvVoNWgx1j7RsjjFKaZUnseK3qQvVfCm4Qjvlz6MpKDxslaUYuR162Ku0 +e153z/xc7XRoXyPyPLdGZFlWii30jirB7ZqPdyz6mwlWwqdImNerbUqdFt9R8bId +pYsyHB5zmwKBgBjYCq9iW/9E+/TqI8sMpI95fK9app5v4AThs3rnAqOa7Ucmrh6V +s7Wnui06D8U6r54Tb+EbqTOpM3Gcl/tRg4FLEA5yTfuA/76Ok1D04Tj+mVsNVPyz +dQhgMUe835WGusroA12df2V/x5NjNeYyMdJZMQ2ByyrNQAjAbMmCGq+5AoGBAIj8 +ERFysMOfxUvg9b7CkDFJrsAhOzew86P2vYGfIHchGTqUkG0LRTDFGrnzxNXsBGjY ++DUB40Kajx7IkTETxC0jvA1ceq23l/VjPrZVQt0YiC+a+rCyNn7SYkyHxsfTVr9b +ea0BZyDXMntyJrPbkjL6Ik8tDE9pLwuOU84ISJ5fAoGAZ2+Ams/VhdZj/wpRpMky +K4jtS4nzbCmJzzTa6vdVV7Kjer5kFxSFFqMrS/FtJ/RxHeHvxdze9dfGu9jIdTKK +vSzbyQdHFfZgRkmAKfcoN9u567z7Oc74AQ9UgFEGdEVFQUbfWOevmr8KIPt8nDQK +J9HuVfILi1kH0jzDd/64TvA= +-----END PRIVATE KEY----- diff --git a/checks/secrets/sops/machines/machine/key.json b/checks/secrets/sops/machines/machine/key.json index 75648379c..c05c0a321 100755 --- a/checks/secrets/sops/machines/machine/key.json +++ b/checks/secrets/sops/machines/machine/key.json @@ -1,4 +1,4 @@ { "publickey": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00", "type": "age" -} \ No newline at end of file +} diff --git a/checks/secrets/sops/users/admin/key.json b/checks/secrets/sops/users/admin/key.json index 75648379c..c05c0a321 100755 --- a/checks/secrets/sops/users/admin/key.json +++ b/checks/secrets/sops/users/admin/key.json @@ -1,4 +1,4 @@ { "publickey": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00", "type": "age" -} \ No newline at end of file +} diff --git a/clanModules/dyndns/README.md b/clanModules/dyndns/README.md new file mode 100644 index 000000000..b739c5efb --- /dev/null +++ b/clanModules/dyndns/README.md @@ -0,0 +1,6 @@ +--- +description = "A dynamic DNS service to update domain IPs" +--- + + + diff --git a/clanModules/dyndns/default.nix b/clanModules/dyndns/default.nix new file mode 100644 index 000000000..3363b251c --- /dev/null +++ b/clanModules/dyndns/default.nix @@ -0,0 +1,265 @@ +{ + config, + pkgs, + lib, + ... +}: + +let + name = "dyndns"; + cfg = config.clan.${name}; + + # We dedup secrets if they have the same provider + base domain + secret_id = opt: "${name}-${opt.provider}-${opt.domain}"; + secret_path = + opt: config.clan.core.facts.services."${secret_id opt}".secret."${secret_id opt}".path; + + # We check that a secret has not been set in extraSettings. + extraSettingsSafe = + opt: + if (builtins.hasAttr opt.secret_field_name opt.extraSettings) then + throw "Please do not set ${opt.secret_field_name} in extraSettings, it is automatically set by the dyndns module." + else + opt.extraSettings; + /* + We go from: + {home.example.com:{value:{domain:example.com,host:home, provider:namecheap}}} + To: + {settings: [{domain: example.com, host: home, provider: namecheap, password: dyndns-namecheap-example.com}]} + */ + service_config = { + settings = builtins.catAttrs "value" ( + builtins.attrValues ( + lib.mapAttrs (_: opt: { + value = + (extraSettingsSafe opt) + // { + domain = opt.domain; + provider = opt.provider; + } + // { + "${opt.secret_field_name}" = secret_id opt; + }; + }) cfg.settings + ) + ); + }; + + secret_generator = _: opt: { + name = secret_id opt; + value = { + secret.${secret_id opt} = { }; + generator.prompt = "Dyndns passphrase for ${secret_id opt}"; + generator.script = '' + echo "$prompt_value" > $secrets/${secret_id opt} + ''; + }; + }; +in +{ + options.clan.${name} = { + + user = lib.mkOption { + type = lib.types.str; + default = name; + description = "User to run the service as"; + }; + group = lib.mkOption { + type = lib.types.str; + default = name; + description = "Group to run the service as"; + }; + + server = { + enable = lib.mkEnableOption "dyndns webserver"; + domain = lib.mkOption { + type = lib.types.str; + description = "Domain to serve the webservice on"; + }; + port = lib.mkOption { + type = lib.types.int; + default = 54805; + description = "Port to listen on"; + }; + }; + + period = lib.mkOption { + type = lib.types.int; + default = 5; + description = "Domain update period in minutes"; + }; + + settings = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { ... }: + { + options = { + provider = lib.mkOption { + example = "namecheap"; + type = lib.types.str; + description = "The dyndns provider to use"; + }; + domain = lib.mkOption { + type = lib.types.str; + example = "example.com"; + description = "The top level domain to update."; + }; + secret_field_name = lib.mkOption { + example = [ + "password" + "api_key" + ]; + type = lib.types.enum [ + "password" + "token" + "api_key" + ]; + default = "password"; + description = "The field name for the secret"; + }; + # TODO: Ideally we would create a gigantic list of all possible settings / types + # optimally we would have a way to generate the options from the source code + extraSettings = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + description = '' + Extra settings for the provider. + Provider specific settings: https://github.com/qdm12/ddns-updater#configuration + ''; + }; + }; + } + ) + ); + default = [ ]; + description = "Configuration for which domains to update"; + }; + }; + + imports = [ + (lib.mkRemovedOptionModule [ + "clan" + "dyndns" + "enable" + ] "Just define clan.dyndns.settings to enable it") + ]; + + config = lib.mkMerge [ + (lib.mkIf (cfg.settings != { }) { + clan.core.facts.services = lib.mapAttrs' secret_generator cfg.settings; + + users.groups.${cfg.group} = { }; + users.users.${cfg.user} = { + group = cfg.group; + isSystemUser = true; + description = "User for ${name} service"; + home = "/var/lib/${name}"; + createHome = true; + }; + + networking.firewall.allowedTCPPorts = lib.mkIf cfg.server.enable [ + 80 + 443 + ]; + + services.nginx = lib.mkIf cfg.server.enable { + enable = true; + virtualHosts = { + "${cfg.server.domain}" = { + forceSSL = true; + enableACME = true; + locations."/" = { + proxyPass = "http://localhost:${toString cfg.server.port}"; + }; + }; + }; + }; + + systemd.services.${name} = { + path = [ ]; + description = "Dynamic DNS updater"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + environment = { + MYCONFIG = "${builtins.toJSON service_config}"; + SERVER_ENABLED = if cfg.server.enable then "yes" else "no"; + PERIOD = "${toString cfg.period}m"; + LISTENING_ADDRESS = ":${toString cfg.server.port}"; + }; + + serviceConfig = + let + pyscript = pkgs.writers.writePyPy3Bin "test.py" { libraries = [ ]; } '' + import json + from pathlib import Path + import os + + cred_dir = Path(os.getenv("CREDENTIALS_DIRECTORY")) + config_str = os.getenv("MYCONFIG") + + + def get_credential(name): + secret_p = cred_dir / name + with open(secret_p, 'r') as f: + return f.read().strip() + + + config = json.loads(config_str) + print(f"Config: {config}") + for attrset in config["settings"]: + if "password" in attrset: + attrset['password'] = get_credential(attrset['password']) + elif "token" in attrset: + attrset['token'] = get_credential(attrset['token']) + elif "api_key" in attrset: + attrset['api_key'] = get_credential(attrset['api_key']) + else: + raise ValueError(f"Missing secret field in {attrset}") + + # create directory data if it does not exist + data_dir = Path('data') + data_dir.mkdir(mode=0o770, exist_ok=True) + + # Write the config with secrets back + config_path = data_dir / 'config.json' + with open(config_path, 'w') as f: + f.write(json.dumps(config, indent=4)) + + # Set file permissions to read and write + # only by the user and group + config_path.chmod(0o660) + + # Set file permissions to read + # and write only by the user and group + for file in data_dir.iterdir(): + file.chmod(0o660) + ''; + in + { + ExecStartPre = lib.getExe pyscript; + ExecStart = lib.getExe pkgs.ddns-updater; + LoadCredential = lib.mapAttrsToList (_: opt: "${secret_id opt}:${secret_path opt}") cfg.settings; + User = cfg.user; + Group = cfg.group; + NoNewPrivileges = true; + PrivateTmp = true; + ProtectSystem = "strict"; + ReadOnlyPaths = "/"; + PrivateDevices = "yes"; + ProtectKernelModules = "yes"; + ProtectKernelTunables = "yes"; + + WorkingDirectory = "/var/lib/${name}"; + ReadWritePaths = [ + "/proc/self" + "/var/lib/${name}" + ]; + + Restart = "always"; + RestartSec = 60; + }; + }; + }) + ]; +} diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix index a5601ca83..60e1a32fc 100644 --- a/clanModules/flake-module.nix +++ b/clanModules/flake-module.nix @@ -4,18 +4,23 @@ borgbackup = ./borgbackup; borgbackup-static = ./borgbackup-static; deltachat = ./deltachat; + dyndns = ./dyndns; ergochat = ./ergochat; + garage = ./garage; + golem-provider = ./golem-provider; + iwd = ./iwd; localbackup = ./localbackup; localsend = ./localsend; - single-disk = ./single-disk; matrix-synapse = ./matrix-synapse; moonlight = ./moonlight; + mumble = ./mumble; packages = ./packages; postgresql = ./postgresql; root-password = ./root-password; + single-disk = ./single-disk; sshd = ./sshd; - sunshine = ./sunshine; static-hosts = ./static-hosts; + sunshine = ./sunshine; syncthing = ./syncthing; syncthing-static-peers = ./syncthing-static-peers; thelounge = ./thelounge; diff --git a/clanModules/garage/README.md b/clanModules/garage/README.md new file mode 100644 index 000000000..911ff0abc --- /dev/null +++ b/clanModules/garage/README.md @@ -0,0 +1,10 @@ +--- +description = "S3-compatible object store for small self-hosted geo-distributed deployments" +--- + +This module generates garage specific keys automatically. +When using garage in a distributed deployment the `rpc_key` between connected instances must be shared. +This is currently still a manual process. + +Options: [NixosModuleOptions](https://search.nixos.org/options?channel=unstable&size=50&sort=relevance&type=packages&query=garage) +Documentation: https://garagehq.deuxfleurs.fr/ diff --git a/clanModules/garage/default.nix b/clanModules/garage/default.nix new file mode 100644 index 000000000..637068e17 --- /dev/null +++ b/clanModules/garage/default.nix @@ -0,0 +1,33 @@ +{ config, pkgs, ... }: +{ + systemd.services.garage.serviceConfig = { + LoadCredential = [ + "rpc_secret_path:${config.clan.core.vars.generators.garage.files.rpc_secret.path}" + "admin_token_path:${config.clan.core.vars.generators.garage.files.admin_token.path}" + "metrics_token_path:${config.clan.core.vars.generators.garage.files.metrics_token.path}" + ]; + Environment = [ + "GARAGE_ALLOW_WORLD_READABLE_SECRETS=true" + "GARAGE_RPC_SECRET_FILE=%d/rpc_secret_path" + "GARAGE_ADMIN_TOKEN_FILE=%d/admin_token_path" + "GARAGE_METRICS_TOKEN_FILE=%d/metrics_token_path" + ]; + }; + + clan.core.vars.generators.garage = { + files.rpc_secret = { }; + files.admin_token = { }; + files.metrics_token = { }; + runtimeInputs = [ + pkgs.coreutils + pkgs.openssl + ]; + script = '' + openssl rand -hex -out $out/rpc_secret 32 + openssl rand -base64 -out $out/admin_token 32 + openssl rand -base64 -out $out/metrics_token 32 + ''; + }; + + clan.core.state.garage.folders = [ config.services.garage.settings.metadata_dir ]; +} diff --git a/clanModules/golem-provider/README.md b/clanModules/golem-provider/README.md new file mode 100644 index 000000000..f9b6bf789 --- /dev/null +++ b/clanModules/golem-provider/README.md @@ -0,0 +1,7 @@ +--- +description = "Golem Provider for the Golem Network, an open-source and decentralized platform where everyone can use and share each other's computing power without relying on centralized entities like cloud computing corporations" +--- + +By running a golem provider your machine's compute resources are offered via the golem network which will allow other members to execute compute tasks on your machine. If this happens, you will be compensated with GLM, an ERC20 token. + +More about golem providers: https://docs.golem.network/docs/golem/overview diff --git a/clanModules/golem-provider/default.nix b/clanModules/golem-provider/default.nix new file mode 100644 index 000000000..6a3325db9 --- /dev/null +++ b/clanModules/golem-provider/default.nix @@ -0,0 +1,34 @@ +{ config, pkgs, ... }: +let + cfg = config.clan.golem-provider; + yagna = pkgs.callPackage ../../pkgs/yagna { }; + accountFlag = if cfg.account != null then "--account ${cfg.account}" else ""; +in +{ + imports = [ ./interface.nix ]; + + users.users.golem = { + isSystemUser = true; + home = "/var/lib/golem"; + group = "golem"; + createHome = true; + }; + + users.groups.golem = { }; + + environment.systemPackages = [ yagna ]; + + systemd.services.golem-provider = { + description = "Golem Provider"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" ]; + serviceConfig = { + Type = "simple"; + ExecStart = "${yagna}/bin/golemsp run --no-interactive ${accountFlag}"; + Restart = "always"; + RestartSec = "5"; + User = "golem"; + Group = "golem"; + }; + }; +} diff --git a/clanModules/golem-provider/interface.nix b/clanModules/golem-provider/interface.nix new file mode 100644 index 000000000..98ae7ccf3 --- /dev/null +++ b/clanModules/golem-provider/interface.nix @@ -0,0 +1,20 @@ +{ lib, ... }: +let + inherit (lib) mkOption; + + inherit (lib.types) nullOr str; + +in +{ + options.clan.golem-provider = { + account = mkOption { + type = nullOr str; + description = '' + Ethereum address for payouts. + + Leave empty to automatically generate a new address upon first start. + ''; + default = null; + }; + }; +} diff --git a/clanModules/golem-provider/test/vm.nix b/clanModules/golem-provider/test/vm.nix new file mode 100644 index 000000000..006e5a042 --- /dev/null +++ b/clanModules/golem-provider/test/vm.nix @@ -0,0 +1,4 @@ +{ ... }: +{ + imports = [ ../. ]; +} diff --git a/clanModules/iwd/README.md b/clanModules/iwd/README.md new file mode 100644 index 000000000..c33051abc --- /dev/null +++ b/clanModules/iwd/README.md @@ -0,0 +1,6 @@ +--- +description = "Automatically provisions wifi credentials" +--- + + + diff --git a/clanModules/iwd/default.nix b/clanModules/iwd/default.nix new file mode 100644 index 000000000..afbcbc616 --- /dev/null +++ b/clanModules/iwd/default.nix @@ -0,0 +1,83 @@ +{ lib, config, ... }: + +let + cfg = config.clan.iwd; + secret_path = ssid: config.clan.core.facts.services."iwd.${ssid}".secret."iwd.${ssid}".path; + secret_generator = name: value: { + name = "iwd.${value.ssid}"; + value = + let + secret_name = "iwd.${value.ssid}"; + in + { + secret.${secret_name} = { }; + generator.prompt = "Wifi password for '${value.ssid}'"; + generator.script = '' + config=" + [Security] + Passphrase=$prompt_value + " + echo "$config" > $secrets/${secret_name} + ''; + }; + }; +in +{ + options.clan.iwd = { + networks = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + ssid = lib.mkOption { + type = lib.types.strMatching "^[a-zA-Z0-9._-]+$"; + default = name; + description = "The name of the wifi network"; + }; + }; + } + ) + ); + default = { }; + description = "Wifi networks to predefine"; + }; + }; + + imports = [ + (lib.mkRemovedOptionModule [ + "clan" + "iwd" + "enable" + ] "Just define clan.iwd.networks to enable it") + ]; + + config = lib.mkMerge [ + (lib.mkIf (cfg.networks != { }) { + # Systemd tmpfiles rule to create /var/lib/iwd/example.psk file + systemd.tmpfiles.rules = lib.mapAttrsToList ( + _: value: "C /var/lib/iwd/${value.ssid}.psk 0600 root root - ${secret_path value.ssid}" + ) cfg.networks; + + clan.core.facts.services = lib.mapAttrs' secret_generator cfg.networks; + + # TODO: restart the iwd.service if something changes + }) + { + # disable wpa supplicant + networking.wireless.enable = false; + + # Use iwd instead of wpa_supplicant. It has a user friendly CLI + networking.wireless.iwd = { + enable = true; + settings = { + Network = { + EnableIPv6 = true; + RoutePriorityOffset = 300; + }; + Settings.AutoConnect = true; + }; + }; + } + ]; +} diff --git a/clanModules/matrix-synapse/default.nix b/clanModules/matrix-synapse/default.nix index 5540f7bc6..cceb465c1 100644 --- a/clanModules/matrix-synapse/default.nix +++ b/clanModules/matrix-synapse/default.nix @@ -169,6 +169,11 @@ in ]; }; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; + services.nginx = { enable = true; virtualHosts = { diff --git a/clanModules/mumble/README.md b/clanModules/mumble/README.md new file mode 100644 index 000000000..38b62f239 --- /dev/null +++ b/clanModules/mumble/README.md @@ -0,0 +1,14 @@ +--- +description = "Open Source, Low Latency, High Quality Voice Chat." +categories = ["chat", "voice"] +--- +The mumble clan module gives you: + +- True low latency voice communication. +- Secure, authenticated encryption. +- Free software. +- Backed by a large and active open-source community. + +This all set up in a way that allows peer-to-peer hosting. +Every machine inside the clan can be a host for mumble, +and thus it doesn't matter who in the network is online - as long as two people are online they are able to chat with each other. diff --git a/clanModules/mumble/default.nix b/clanModules/mumble/default.nix new file mode 100644 index 000000000..53d6986d8 --- /dev/null +++ b/clanModules/mumble/default.nix @@ -0,0 +1,104 @@ +{ + lib, + config, + pkgs, + ... +}: +let + clanDir = config.clan.core.clanDir; + machineDir = clanDir + "/machines/"; + machinesFileSet = builtins.readDir machineDir; + machines = lib.mapAttrsToList (name: _: name) machinesFileSet; + machineJson = builtins.toJSON machines; + certificateMachinePath = machines: machineDir + "/${machines}" + "/facts/mumble-cert"; + certificatesUnchecked = builtins.map ( + machine: + let + fullPath = certificateMachinePath machine; + in + if builtins.pathExists fullPath then machine else null + ) machines; + certificate = lib.filter (machine: machine != null) certificatesUnchecked; + machineCert = builtins.map ( + machine: (lib.nameValuePair machine (builtins.readFile (certificateMachinePath machine))) + ) certificate; + machineCertJson = builtins.toJSON machineCert; + +in +{ + options.clan.services.mumble = { + user = lib.mkOption { + type = lib.types.str; + default = "alice"; + description = "The user mumble should be set up for."; + }; + }; + + config = { + services.murmur = { + enable = true; + logDays = -1; + registerName = config.clan.core.machineName; + openFirewall = true; + bonjour = true; + sslKey = config.clan.core.facts.services.mumble.secret.mumble-key.path; + sslCert = config.clan.core.facts.services.mumble.public.mumble-cert.path; + }; + + clan.core.state.mumble.folders = [ + "/var/lib/mumble" + "/var/lib/murmur" + ]; + + systemd.tmpfiles.rules = [ + "d '/var/lib/mumble' 0770 '${config.clan.services.mumble.user}' 'users' - -" + ]; + + environment.systemPackages = + let + mumbleCfgDir = "/var/lib/mumble"; + mumbleDatabasePath = "${mumbleCfgDir}/mumble.sqlite"; + mumbleCfgPath = "/var/lib/mumble/mumble_settings.json"; + populate-channels = pkgs.writers.writePython3 "mumble-populate-channels" { + libraries = [ + pkgs.python3Packages.cryptography + pkgs.python3Packages.pyopenssl + ]; + flakeIgnore = [ + # We don't live in the dark ages anymore. + # Languages like Python that are whitespace heavy will overrun + # 79 characters.. + "E501" + ]; + } (builtins.readFile ./mumble-populate-channels.py); + mumble = pkgs.writeShellScriptBin "mumble" '' + set -xeu + mkdir -p ${mumbleCfgDir} + pushd "${mumbleCfgDir}" + XDG_DATA_HOME=${mumbleCfgDir} + XDG_DATA_DIR=${mumbleCfgDir} + ${populate-channels} --ensure-config '${mumbleCfgPath}' --db-location ${mumbleDatabasePath} + echo ${machineCertJson} + ${populate-channels} --machines '${machineJson}' --username ${config.clan.core.machineName} --db-location ${mumbleDatabasePath} + ${populate-channels} --servers '${machineCertJson}' --username ${config.clan.core.machineName} --db-location ${mumbleDatabasePath} --cert True + ${pkgs.mumble}/bin/mumble --config ${mumbleCfgPath} "$@" + popd + ''; + in + [ mumble ]; + + clan.core.facts.services.mumble = { + secret.mumble-key = { }; + public.mumble-cert = { }; + generator.path = [ + pkgs.coreutils + pkgs.openssl + ]; + generator.script = '' + openssl genrsa -out $secrets/mumble-key 2048 + openssl req -new -x509 -key $secrets/mumble-key -out $facts/mumble-cert + ''; + }; + }; + +} diff --git a/clanModules/mumble/mumble-populate-channels.py b/clanModules/mumble/mumble-populate-channels.py new file mode 100644 index 000000000..56ee36668 --- /dev/null +++ b/clanModules/mumble/mumble-populate-channels.py @@ -0,0 +1,249 @@ +import argparse +import json +import os +import sqlite3 + + +def ensure_config(path: str, db_path: str) -> None: + # Default JSON structure if the file doesn't exist + default_json = { + "misc": { + "audio_wizard_has_been_shown": True, + "database_location": db_path, + "viewed_server_ping_consent_message": True, + }, + "settings_version": 1, + } + + # Check if the file exists + if os.path.exists(path): + with open(path) as file: + data = json.load(file) + else: + data = default_json + # Create the file with default JSON structure + with open(path, "w") as file: + json.dump(data, file, indent=4) + + # TODO: make sure to only update the diff + updated_data = {**default_json, **data} + + # Write the modified JSON object back to the file + with open(path, "w") as file: + json.dump(updated_data, file, indent=4) + + +def initialize_database(db_location: str) -> None: + """ + Initializes the database. If the database or the servers table does not exist, it creates them. + + :param db_location: The path to the SQLite database + """ + conn = sqlite3.connect(db_location) + try: + cursor = conn.cursor() + + # Create the servers table if it doesn't exist + cursor.execute(""" + CREATE TABLE IF NOT EXISTS servers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + hostname TEXT NOT NULL, + port INTEGER NOT NULL, + username TEXT NOT NULL, + password TEXT NOT NULL, + url TEXT + ) + """) + + # Commit the changes + conn.commit() + + except sqlite3.Error as e: + print(f"An error occurred while initializing the database: {e}") + finally: + conn.close() + + +def initialize_certificates( + db_location: str, hostname: str, port: str, digest: str +) -> None: + # Connect to the SQLite database + conn = sqlite3.connect(db_location) + + try: + # Create a cursor object + cursor = conn.cursor() + + # TODO: check if cert already there + # if server_check(cursor, name, hostname): + # print( + # f"Server with name '{name}' and hostname '{hostname}' already exists." + # ) + # return + + # SQL command to insert data into the servers table + insert_query = """ + INSERT INTO cert (hostname, port, digest) + VALUES (?, ?, ?) + """ + + # Data to be inserted + data = (hostname, port, digest) + + # Execute the insert command with the provided data + cursor.execute(insert_query, data) + + # Commit the changes + conn.commit() + + print("Data has been successfully inserted.") + except sqlite3.Error as e: + print(f"An error occurred: {e}") + finally: + # Close the connection + conn.close() + pass + + +def calculate_digest(cert: str) -> str: + from cryptography import x509 + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes + + cert = cert.strip() + cert = cert.encode("utf-8") + cert = x509.load_pem_x509_certificate(cert, default_backend()) + digest = cert.fingerprint(hashes.SHA1()).hex() + return digest + + +def server_check(cursor: str, name: str, hostname: str) -> bool: + """ + Check if a server with the given name and hostname already exists. + + :param cursor: The database cursor + :param name: The name of the server + :param hostname: The hostname of the server + :return: True if the server exists, False otherwise + """ + check_query = """ + SELECT 1 FROM servers WHERE name = ? AND hostname = ? + """ + cursor.execute(check_query, (name, hostname)) + return cursor.fetchone() is not None + + +def insert_server( + name: str, + hostname: str, + port: str, + username: str, + password: str, + url: str, + db_location: str, +) -> None: + """ + Inserts a new server record into the servers table. + + :param name: The name of the server + :param hostname: The hostname of the server + :param port: The port number + :param username: The username + :param password: The password + :param url: The URL + """ + # Connect to the SQLite database + conn = sqlite3.connect(db_location) + + try: + # Create a cursor object + cursor = conn.cursor() + + if server_check(cursor, name, hostname): + print( + f"Server with name '{name}' and hostname '{hostname}' already exists." + ) + return + + # SQL command to insert data into the servers table + insert_query = """ + INSERT INTO servers (name, hostname, port, username, password, url) + VALUES (?, ?, ?, ?, ?, ?) + """ + + # Data to be inserted + data = (name, hostname, port, username, password, url) + + # Execute the insert command with the provided data + cursor.execute(insert_query, data) + + # Commit the changes + conn.commit() + + print("Data has been successfully inserted.") + except sqlite3.Error as e: + print(f"An error occurred: {e}") + finally: + # Close the connection + conn.close() + + +if __name__ == "__main__": + port = 64738 + password = "" + url = None + + parser = argparse.ArgumentParser( + prog="initialize_mumble", + ) + + subparser = parser.add_subparsers(dest="certificates") + # cert_parser = subparser.add_parser("certificates") + + parser.add_argument("--cert") + parser.add_argument("--digest") + parser.add_argument("--machines") + parser.add_argument("--servers") + parser.add_argument("--username") + parser.add_argument("--db-location") + parser.add_argument("--ensure-config") + args = parser.parse_args() + + print(args) + + if args.ensure_config: + ensure_config(args.ensure_config, args.db_location) + print("Initialized config") + exit(0) + + if args.servers: + print(args.servers) + servers = json.loads(f"{args.servers}") + db_location = args.db_location + for server in servers: + digest = calculate_digest(server.get("value")) + name = server.get("name") + initialize_certificates(db_location, name, port, digest) + print("Initialized certificates") + exit(0) + + initialize_database(args.db_location) + + # Insert the server into the database + print(args.machines) + machines = json.loads(f"{args.machines}") + print(machines) + print(list(machines)) + + for machine in list(machines): + print(f"Inserting {machine}.") + insert_server( + machine, + machine, + port, + args.username, + password, + url, + args.db_location, + ) diff --git a/clanModules/mumble/test.nix b/clanModules/mumble/test.nix new file mode 100644 index 000000000..d2d115810 --- /dev/null +++ b/clanModules/mumble/test.nix @@ -0,0 +1,42 @@ +{ pkgs, self, ... }: +pkgs.nixosTest { + name = "mumble"; + nodes.peer1 = + { ... }: + { + imports = [ + self.nixosModules.mumble + self.inputs.clan-core.nixosModules.clanCore + { + config = { + clan.core.machineName = "peer1"; + clan.core.clanDir = ./.; + + documentation.enable = false; + }; + } + ]; + }; + nodes.peer2 = + { ... }: + { + imports = [ + self.nixosModules.mumble + self.inputs.clan-core.nixosModules.clanCore + { + config = { + + clan.core.machineName = "peer2"; + clan.core.clanDir = ./.; + + documentation.enable = false; + }; + } + ]; + }; + + testScript = '' + start_all() + ''; + +} diff --git a/devShell.nix b/devShell.nix index 206b29cff..d615ff71c 100644 --- a/devShell.nix +++ b/devShell.nix @@ -38,6 +38,7 @@ ]; shellHook = '' echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}" + export PROJECT_ROOT=$(git rev-parse --show-toplevel) ''; }; }; diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 8bd5d22ac..647e880e0 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -45,6 +45,7 @@ nav: - Configure: getting-started/configure.md - Secrets & Facts: getting-started/secrets.md - Deploy Machine: getting-started/deploy.md + - Disk Encryption: getting-started/disk-encryption.md - Mesh VPN: getting-started/mesh-vpn.md - Backup & Restore: getting-started/backups.md - Flake-parts: getting-started/flake-parts.md @@ -54,15 +55,20 @@ nav: - Reference: - reference/index.md - Clan Modules: - - reference/clanModules/index.md - reference/clanModules/borgbackup-static.md - reference/clanModules/borgbackup.md - reference/clanModules/deltachat.md + - reference/clanModules/dyndns.md - reference/clanModules/ergochat.md + - reference/clanModules/garage.md + - reference/clanModules/golem-provider.md + - reference/clanModules/index.md + - reference/clanModules/iwd.md - reference/clanModules/localbackup.md - reference/clanModules/localsend.md - reference/clanModules/matrix-synapse.md - reference/clanModules/moonlight.md + - reference/clanModules/mumble.md - reference/clanModules/packages.md - reference/clanModules/postgresql.md - reference/clanModules/root-password.md @@ -81,7 +87,6 @@ nav: - CLI: - reference/cli/index.md - reference/cli/backups.md - - reference/cli/config.md - reference/cli/facts.md - reference/cli/flakes.md - reference/cli/flash.md diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 184e71158..4cd88fc44 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -1,12 +1,19 @@ -{% extends "base.html" %} - -{% block extrahead %} - - - +{% extends "base.html" %} {% block extrahead %} + + + -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/docs/site/getting-started/deploy.md b/docs/site/getting-started/deploy.md index 3312a606f..8ff89b6ca 100644 --- a/docs/site/getting-started/deploy.md +++ b/docs/site/getting-started/deploy.md @@ -219,6 +219,7 @@ This is useful for machines that are not always online or are not part of the re ## What's next ? +- [**Disk Encryption**](./disk-encryption.md): Configure disk encryption with remote decryption - [**Mesh VPN**](./mesh-vpn.md): Configuring a secure mesh network. --- diff --git a/docs/site/getting-started/disk-encryption.md b/docs/site/getting-started/disk-encryption.md new file mode 100644 index 000000000..1af4ee47d --- /dev/null +++ b/docs/site/getting-started/disk-encryption.md @@ -0,0 +1,301 @@ +## Setting up Encryption with Remote Decryption in NixOS + +This guide provides an example setup for a single-disk ZFS system with native encryption, accessible for decryption remotely. This configuration only applies to `systemd-boot` enabled systems and requires UEFI booting. + +For a mirrored disk setup, add `mode = "mirror";` to `zroot`. Under the `disk` option, provide the additional disk identifier, e.g., `y = mirrorBoot /dev/disk/by-id/`. + +Replace the disk `nvme-eui.002538b931b59865` with your own. + +Below is the configuration for `disko.nix` +```nix +{ lib, ... }: +let + mirrorBoot = idx: { + type = "disk"; + device = "/dev/disk/by-id/${idx}"; + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; # for grub MBR + priority = 1; + }; + ESP = lib.mkIf (idx == "nvme-eui.002538b931b59865") { + size = "1G"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "nofail" ]; + }; + }; + zfs = { + size = "100%"; + content = { + type = "zfs"; + pool = "zroot"; + }; + }; + }; + }; + }; +in +{ + boot.loader.systemd-boot.enable = true; + + disko.devices = { + disk = { + x = mirrorBoot "nvme-eui.002538b931b59865"; + }; + zpool = { + zroot = { + type = "zpool"; + rootFsOptions = { + compression = "lz4"; + acltype = "posixacl"; + xattr = "sa"; + "com.sun:auto-snapshot" = "true"; + mountpoint = "none"; + }; + datasets = { + "root" = { + type = "zfs_fs"; + options = { + mountpoint = "none"; + encryption = "aes-256-gcm"; + keyformat = "passphrase"; + keylocation = "file:///tmp/secret.key"; + }; + }; + "root/nixos" = { + type = "zfs_fs"; + options.mountpoint = "/"; + mountpoint = "/"; + }; + "root/home" = { + type = "zfs_fs"; + options.mountpoint = "/home"; + mountpoint = "/home"; + }; + "root/tmp" = { + type = "zfs_fs"; + mountpoint = "/tmp"; + options = { + mountpoint = "/tmp"; + sync = "disabled"; + }; + }; + }; + }; + }; + }; +} +``` + +Add this to networking.nix and **replace** the `default` values as well as the name `gchq-local` and `networking.hostId` with your own. + +```nix +{ config, lib, ... }: +{ + options = { + networking.gchq-local.ipv4.address = lib.mkOption { + type = lib.types.str; + default = "192.168.178.177"; + }; + networking.gchq-local.ipv4.cidr = lib.mkOption { + type = lib.types.str; + default = "24"; + }; + + networking.gchq-local.ipv4.gateway = lib.mkOption { + type = lib.types.str; + default = "192.168.178.1"; + }; + + networking.gchq-local.ipv6.address = lib.mkOption { + type = lib.types.str; + default = "2003:100:6701:d500:fbbc:40fb:cff3:3b87"; + }; + + networking.gchq-local.ipv6.cidr = lib.mkOption { + type = lib.types.str; + default = "64"; + }; + networking.gchq-local.ipv6.gateway = lib.mkOption { + type = lib.types.str; + default = "fe80::3ea6:2fff:feef:3435"; + }; + }; + + config = { + networking.dhcpcd.enable = false; + networking.nameservers = [ "127.0.0.1" ]; + networking.hostId = "a76ebcca"; # Needs to be unique for each host + + # The '10' in the network name is the priority, so this will be the first network to be configured + systemd.network.networks."10-eth" = { + matchConfig.Type = "ether"; + addresses = [ + { + Address=config.networking.gchq-local.ipv4.address + "/" + config.networking.gchq-local.ipv4.cidr; + } + { + Address=config.networking.gchq-local.ipv6.address + "/" + config.networking.gchq-local.ipv6.cidr; + } + ]; + DHCP = "yes"; + }; + }; +} +``` + +Put this into initrd.nix and add your pubkey to `authorizedKeys`. +Replace `kernelModules` with the ethernet module loaded one on your system. +```nix +{config, pkgs, ...}: + +{ + + boot.initrd.systemd = { + enable = true; + network.networks."10-eth" = config.systemd.network.networks."10-eth"; + }; + boot.initrd.network = { + enable = true; + + ssh = { + enable = true; + port = 7172; + authorizedKeys = [ "" ]; + hostKeys = [ + "/var/lib/initrd-ssh-key" + ]; + }; + }; + boot.initrd.availableKernelModules = [ + "xhci_pci" + ]; + + # Check the network card by running `lspci -k` on the target machine + boot.initrd.kernelModules = [ "r8169" ]; +} +``` + + +### Step 1: Copying SSH Public Key + +Before starting the installation process, ensure that the SSH public key is copied to the NixOS installer. + +1. Copy your public SSH key to the installer, if it has not been copied already: + +```bash +ssh-copy-id -o PreferredAuthentications=password -o PubkeyAuthentication=no root@nixos-installer.local +``` + +### Step 1.5: Prepare Secret Key and Clear Disk Data + +1. Access the installer using SSH: + +```bash +ssh root@nixos-installer.local +``` + +2. Create a `secret.key` file in `/tmp` using `nano` or another text editor: + +```bash +nano /tmp/secret.key +``` + +3. Discard the old disk partition data: + +```bash +blkdiscard /dev/disk/by-id/nvme-eui.002538b931b59865 +``` + +4. Run the `clan` machine installation with the following command: + +```bash +clan machines install gchq-local root@nixos-installer --yes --no-reboot +``` + +### Step 2: ZFS Pool Import and System Installation + +1. SSH into the installer once again: + +```bash +ssh root@nixos-installer.local +``` + +2. Perform the following commands on the remote installation environment: + +```bash +zpool import zroot +zfs set keystate=prompt zroot/root +zfs load-key zroot/root +zfs set mountpoint=/mnt zroot/root/nixos +mount /dev/nvme0n1p2 /mnt/boot +``` + +3. Disconnect from the SSH session: + +```bash +CTRL+D +``` + +4. Securely copy your local `initrd_rsa_key` to the installer's `/mnt` directory: + +```bash +scp ~/.ssh/initrd_rsa_key root@nixos-installer.local:/mnt/var/lib/initrd-ssh-key +``` + +5. SSH back into the installer: + +```bash +ssh root@nixos-installer.local +``` + +6. Navigate to the `/mnt` directory, enter the `nixos-enter` environment, and then exit: + +```bash +cd /mnt +nixos-enter +realpath /run/current-system +exit +``` + +7. Run the `nixos-install` command with the appropriate system path ``: + +```bash +nixos-install --no-root-passwd --no-channel-copy --root /mnt --system +``` + +8. After the installation process, unmount `/mnt/boot`, change the ZFS mountpoint, and reboot the system: + +```bash +umount /mnt/boot +cd / +zfs set mountpoint=/ zroot/root/nixos +reboot +``` + +9. Perform a hard reboot of the machine and remove the USB stick. + +### Step 3: Accessing the Initial Ramdisk (initrd) Environment + +1. SSH into the initrd environment using the `initrd_rsa_key` and provided port: + +```bash +ssh -p 7172 root@192.168.178.141 +``` + +2. Run the `systemd-tty-ask-password-agent` utility to query a password: + +```bash +systemd-tty-ask-password-agent --query +``` + +After completing these steps, your NixOS should be successfully installed and ready for use. + +**Note:** Replace `root@nixos-installer.local` and `192.168.178.141` with the appropriate user and IP addresses for your setup. Also, adjust `` to reflect the correct system path for your environment. \ No newline at end of file diff --git a/docs/site/getting-started/installer.md b/docs/site/getting-started/installer.md index 591aec74f..797e6fec5 100644 --- a/docs/site/getting-started/installer.md +++ b/docs/site/getting-started/installer.md @@ -49,7 +49,7 @@ sudo umount /dev/sdb1 clan flash --flake git+https://git.clan.lol/clan/clan-core \ --ssh-pubkey $HOME/.ssh/id_ed25519.pub \ --keymap us \ - --language en_US.utf-8 \ + --language en_US.UTF-8 \ --disk main /dev/sd \ flash-installer ``` diff --git a/docs/site/static/extra.css b/docs/site/static/extra.css index 505b38d32..79325ade9 100644 --- a/docs/site/static/extra.css +++ b/docs/site/static/extra.css @@ -1,13 +1,13 @@ @font-face { - font-family: "Roboto"; - src: url(./Roboto-Regular.ttf) format('truetype'); + font-family: "Roboto"; + src: url(./Roboto-Regular.ttf) format("truetype"); } @font-face { - font-family: "Fira Code"; - src: url(./FiraCode-VF.ttf) format('truetype'); + font-family: "Fira Code"; + src: url(./FiraCode-VF.ttf) format("truetype"); } :root { - --md-text-font: "Roboto"; - --md-code-font: "Fira Code"; -} \ No newline at end of file + --md-text-font: "Roboto"; + --md-code-font: "Fira Code"; +} diff --git a/flake.lock b/flake.lock index e72901e57..007d5b16c 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1721417620, - "narHash": "sha256-6q9b1h8fI3hXg2DG6/vrKWCeG8c5Wj2Kvv22RCgedzg=", + "lastModified": 1723080788, + "narHash": "sha256-C5LbM5VMdcolt9zHeLQ0bYMRjUL+N+AL5pK7/tVTdes=", "owner": "nix-community", "repo": "disko", - "rev": "bec6e3cde912b8acb915fecdc509eda7c973fb42", + "rev": "ffc1f95f6c28e1c6d1e587b51a2147027a3e45ed", "type": "github" }, "original": { @@ -27,11 +27,11 @@ ] }, "locked": { - "lastModified": 1719994518, - "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", "type": "github" }, "original": { @@ -48,11 +48,11 @@ ] }, "locked": { - "lastModified": 1721571445, - "narHash": "sha256-2MnlPVcNJZ9Nbu90kFyo7+lng366gswErP4FExfrUbc=", + "lastModified": 1724028934, + "narHash": "sha256-2M5dqS7UbAKfrO+1U+P/t5S2QIGbuGIsTNMYJzwB17g=", "owner": "nix-community", "repo": "nixos-images", - "rev": "accee005735844d57b411d9969c5d0aabc6a55f6", + "rev": "b733f0680a42cc01d6ad53896fb5ca40a66d5e79", "type": "github" }, "original": { @@ -63,11 +63,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721571961, - "narHash": "sha256-jfF4gpRUpTBY2OxDB0FRySsgNGOiuDckEtu7YDQom3Y=", + "lastModified": 1724137240, + "narHash": "sha256-VjbV/91spoYpl+fD7cK1asDhQIjJduP0lT+SgeXtcIc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4cc8b29327bed3d52b40041f810f49734298af46", + "rev": "d2fa2514f041934a6aa261c66dc44829251cffd3", "type": "github" }, "original": { @@ -84,6 +84,7 @@ "nixos-images": "nixos-images", "nixpkgs": "nixpkgs", "sops-nix": "sops-nix", + "systems": "systems", "treefmt-nix": "treefmt-nix" } }, @@ -95,11 +96,11 @@ "nixpkgs-stable": [] }, "locked": { - "lastModified": 1721531171, - "narHash": "sha256-AsvPw7T0tBLb53xZGcUC3YPqlIpdxoSx56u8vPCr6gU=", + "lastModified": 1723501126, + "narHash": "sha256-N9IcHgj/p1+2Pvk8P4Zc1bfrMwld5PcosVA0nL6IGdE=", "owner": "Mic92", "repo": "sops-nix", - "rev": "909e8cfb60d83321d85c8d17209d733658a21c95", + "rev": "be0eec2d27563590194a9206f551a6f73d52fa34", "type": "github" }, "original": { @@ -108,6 +109,21 @@ "type": "github" } }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, "treefmt-nix": { "inputs": { "nixpkgs": [ @@ -115,11 +131,11 @@ ] }, "locked": { - "lastModified": 1721458737, - "narHash": "sha256-wNXLQ/ATs1S4Opg1PmuNoJ+Wamqj93rgZYV3Di7kxkg=", + "lastModified": 1723808491, + "narHash": "sha256-rhis3qNuGmJmYC/okT7Dkc4M8CeUuRCSvW6kC2f3hBc=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "888bfb10a9b091d9ed2f5f8064de8d488f7b7c97", + "rev": "1d07739554fdc4f8481068f1b11d6ab4c1a4167a", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 545eac1a8..c582ed3c2 100644 --- a/flake.nix +++ b/flake.nix @@ -14,12 +14,18 @@ nixos-images.inputs.nixos-stable.follows = ""; flake-parts.url = "github:hercules-ci/flake-parts"; flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + systems.url = "github:nix-systems/default"; treefmt-nix.url = "github:numtide/treefmt-nix"; treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = - inputs@{ flake-parts, self, ... }: + inputs@{ + flake-parts, + self, + systems, + ... + }: flake-parts.lib.mkFlake { inherit inputs; } ( { ... }: { @@ -27,11 +33,7 @@ meta.name = "clan-core"; directory = self; }; - systems = [ - "x86_64-linux" - "aarch64-linux" - "aarch64-darwin" - ]; + systems = import systems; imports = [ ./checks/flake-module.nix ./clanModules/flake-module.nix diff --git a/flakeModules/clan.nix b/flakeModules/clan.nix index f8a60d1fb..b837d92dd 100644 --- a/flakeModules/clan.nix +++ b/flakeModules/clan.nix @@ -3,126 +3,34 @@ clan-core: config, lib, flake-parts-lib, - inputs, self, + inputs, ... }: let - inherit (lib) mkOption types; - buildClan = import ../lib/build-clan { - inherit lib clan-core; - inherit (inputs) nixpkgs; - }; - cfg = config.clan; + inherit (lib) types; in { - imports = [ - # TODO: figure out how to print the deprecation warning - # "${inputs.nixpkgs}/nixos/modules/misc/assertions.nix" - (lib.mkRenamedOptionModule - [ - "clan" - "clanName" - ] - [ - "clan" - "meta" - "name" - ] - ) - (lib.mkRenamedOptionModule - [ - "clan" - "clanIcon" - ] - [ - "clan" - "meta" - "icon" - ] - ) - ]; - options.clan = { - directory = mkOption { - type = types.path; - description = "The directory containing the clan subdirectory"; - default = self; # default to the directory of the flake - }; - specialArgs = mkOption { - type = types.attrsOf types.raw; - default = { }; - description = "Extra arguments to pass to nixosSystem i.e. useful to make self available"; - }; - machines = mkOption { - type = types.attrsOf types.raw; - default = { }; - description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }"; - }; - inventory = mkOption { - #type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; }; - type = types.attrsOf types.raw; - default = { }; - description = '' - An abstract service layer for consistently configuring distributed services across machine boundaries. - See https://docs.clan.lol/concepts/inventory/ for more details. - ''; - }; - - # Checks are performed in 'buildClan' - # Not everyone uses flake-parts - meta = { - name = lib.mkOption { - type = types.nullOr types.str; - default = null; - description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to."; + options.clan = lib.mkOption { + type = types.submoduleWith { + specialArgs = { + inherit clan-core self; + inherit (inputs) nixpkgs; }; - icon = mkOption { - type = types.nullOr types.path; - default = null; - description = "A path to an icon to be used for the clan in the GUI"; - }; - description = mkOption { - type = types.nullOr types.str; - default = null; - description = "A short description of the clan"; - }; - }; - - pkgsForSystem = mkOption { - type = types.functionTo types.raw; - default = _system: null; - description = "A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system."; + modules = [ + ../lib/build-clan/interface.nix + ../lib/build-clan/module.nix + ]; }; }; - options.flake = flake-parts-lib.mkSubmoduleOptions { - clanInternals = lib.mkOption { - type = lib.types.submodule { - options = { - inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - inventoryFile = lib.mkOption { type = lib.types.unspecified; }; - clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; }; - source = lib.mkOption { type = lib.types.path; }; - meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; - machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; - machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; - }; - }; - }; + options.flake = flake-parts-lib.mkSubmoduleOptions { + clanInternals = lib.mkOption { type = types.raw; }; }; config = { - flake = buildClan { - inherit (cfg) - directory - specialArgs - machines - pkgsForSystem - meta - inventory - ; - }; + flake.clanInternals = config.clan.clanInternals; + flake.nixosConfigurations = config.clan.nixosConfigurations; }; _file = __curPos.file; } diff --git a/formatter.nix b/formatter.nix index 8f6cbb285..a0786c697 100644 --- a/formatter.nix +++ b/formatter.nix @@ -11,7 +11,42 @@ treefmt.programs.nixfmt.enable = true; treefmt.programs.nixfmt.package = pkgs.nixfmt-rfc-style; treefmt.programs.deadnix.enable = true; - + treefmt.settings.global.excludes = [ + "*.png" + "*.jpeg" + "*.gitignore" + ".vscode/*" + "*.toml" + "*.clan-flake" + "*.code-workspace" + "*.pub" + "*.typed" + "*.age" + "*.list" + "*.desktop" + ]; + treefmt.programs.prettier = { + enable = true; + includes = [ + "*.cjs" + "*.css" + "*.html" + "*.js" + "*.json5" + "*.jsx" + "*.mdx" + "*.mjs" + "*.scss" + "*.ts" + "*.tsx" + "*.vue" + "*.yaml" + "*.yml" + ]; + # plugins = [ + # "${self'.packages.prettier-plugin-tailwindcss}/lib/node_modules/prettier-plugin-tailwindcss/dist/index.mjs" + # ]; + }; treefmt.programs.mypy.directories = { "pkgs/clan-cli" = { @@ -20,8 +55,8 @@ }; "pkgs/clan-app" = { extraPythonPackages = - # clan-app currently only exists on linux (self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies; + extraPythonPaths = [ "../clan-cli" ]; modules = [ "clan_app" ]; }; } @@ -30,8 +65,8 @@ { "pkgs/clan-vm-manager" = { extraPythonPackages = - # # clan-app currently only exists on linux - self'.packages.clan-vm-manager.testDependencies; + self'.packages.clan-vm-manager.externalTestDeps ++ self'.packages.clan-cli.testDependencies; + extraPythonPaths = [ "../clan-cli" ]; modules = [ "clan_vm_manager" ]; }; } @@ -41,53 +76,5 @@ treefmt.programs.ruff.check = true; treefmt.programs.ruff.format = true; - # FIXME: currently broken in CI - #treefmt.settings.formatter.vale = - # let - # vocab = "cLAN"; - # style = "Docs"; - # config = pkgs.writeText "vale.ini" '' - # StylesPath = ${styles} - # Vocab = ${vocab} - - # [*.md] - # BasedOnStyles = Vale, ${style} - # Vale.Terms = No - # ''; - # styles = pkgs.symlinkJoin { - # name = "vale-style"; - # paths = [ - # accept - # headings - # ]; - # }; - # accept = pkgs.writeTextDir "config/vocabularies/${vocab}/accept.txt" '' - # Nix - # NixOS - # Nixpkgs - # clan.lol - # Clan - # monorepo - # ''; - # headings = pkgs.writeTextDir "${style}/headings.yml" '' - # extends: capitalization - # message: "'%s' should be in sentence case" - # level: error - # scope: heading - # # $title, $sentence, $lower, $upper, or a pattern. - # match: $sentence - # ''; - # in - # { - # command = "${pkgs.vale}/bin/vale"; - # options = [ "--config=${config}" ]; - # includes = [ "*.md" ]; - # # TODO: too much at once, fix piecemeal - # excludes = [ - # "docs/*" - # "clanModules/*" - # "pkgs/*" - # ]; - # }; }; } diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 01808d6db..ed7057e57 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -1,306 +1,39 @@ +## WARNING: Do not add core logic here. +## This is only a wrapper such that buildClan can be called as a function. +## Add any logic to ./module.nix { - clan-core, - nixpkgs, lib, + nixpkgs, + clan-core, }: { - directory, # The directory containing the machines subdirectory - specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available - machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... } - # DEPRECATED: use meta.name instead - clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to. - # DEPRECATED: use meta.icon instead - clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines - meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string + ## Inputs + directory, # The directory containing the machines subdirectory # allows to include machine-specific modules i.e. machines.${name} = { ... } # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. # This improves performance, but all nipxkgs.* options will be ignored. - pkgsForSystem ? (_system: null), - /* - Low level inventory configuration. - Overrides the services configuration. - */ + # deadnix: skip inventory ? { }, -}: + ## Sepcial inputs (not passed to the module system as config) + specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available # A set containing clan meta: name :: string, icon :: string, description :: string + ## + ... +}@attrs: let - # Internal inventory, this is the result of merging all potential inventory sources: - # - Default instances configured via 'services' - # - The inventory overrides - # - Machines that exist in inventory.machines - # - Machines explicitly configured via 'machines' argument - # - Machines that exist in the machines directory - # Checks on the module level: - # - Each service role must reference a valid machine after all machines are merged - - clanToInventory = - config: - { clanPath, inventoryPath }: - let - v = lib.attrByPath clanPath null config; - in - lib.optionalAttrs (v != null) (lib.setAttrByPath inventoryPath v); - - mergedInventory = - (lib.evalModules { - modules = [ - clan-core.lib.inventory.interface - { inherit meta; } - ( - if - builtins.pathExists "${directory}/inventory.json" - # Is recursively applied. Any explicit nix will override. - then - (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) - else - { } - ) - inventory - # Machines explicitly configured via 'machines' argument - { - # { ${name} :: meta // { name, tags } } - machines = lib.mapAttrs ( - name: machineConfig: - (lib.attrByPath [ - "clan" - "meta" - ] { } machineConfig) - // { - # meta.name default is the attribute name of the machine - name = lib.mkDefault ( - lib.attrByPath [ - "clan" - "meta" - "name" - ] name machineConfig - ); - } - # tags - // (clanToInventory machineConfig { - clanPath = [ - "clan" - "tags" - ]; - inventoryPath = [ "tags" ]; - }) - # system - // (clanToInventory machineConfig { - clanPath = [ - "nixpkgs" - "hostPlatform" - ]; - inventoryPath = [ "system" ]; - }) - # deploy.targetHost - // (clanToInventory machineConfig { - clanPath = [ - "clan" - "core" - "networking" - "targetHost" - ]; - inventoryPath = [ - "deploy" - "targetHost" - ]; - }) - ) machines; - } - - # Will be deprecated - { - machines = - lib.mapAttrs - ( - name: _: - # Use mkForce to make sure users migrate to the inventory system. - # When the settings.json exists the evaluation will print the deprecation warning. - lib.mkForce { - inherit name; - system = (machineSettings name).nixpkgs.hostSystem or null; - } - ) - ( - lib.filterAttrs ( - machineName: _: builtins.pathExists "${directory}/machines/${machineName}/settings.json" - ) machinesDirs - ); - } - - # Deprecated interface - (if clanName != null then { meta.name = clanName; } else { }) - (if clanIcon != null then { meta.icon = clanIcon; } else { }) - ]; - }).config; - - inherit (clan-core.lib.inventory) buildInventory; - - # map from machine name to service configuration - # { ${machineName} :: Config } - serviceConfigs = buildInventory { - inventory = mergedInventory; - inherit directory; + eval = import ./eval.nix { + inherit + lib + nixpkgs + specialArgs + clan-core + ; + self = directory; }; - - machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( - builtins.readDir (directory + /machines) - ); - - machineSettings = - machineName: - let - warn = lib.warn '' - The use of ./machines//settings.json is deprecated. - If your settings.json is empty, you can safely remove it. - !!! Consider using the inventory system. !!! - - File: ${directory + /machines/${machineName}/settings.json} - - If there are still features missing in the inventory system, please open an issue on the clan-core repository. - ''; - in - # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily - # This is useful for doing a dry-run before writing changes into the settings.json - # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval - if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then - warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) - else - lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( - warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) - ); - - machineImports = - machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); - - deprecationWarnings = [ - (lib.warnIf ( - clanName != null - ) "clanName in buildClan is deprecated, please use meta.name instead." null) - (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null) - ]; - - # TODO: remove default system once we have a hardware-config mechanism - nixosConfiguration = - { - system ? "x86_64-linux", - name, - pkgs ? null, - extraConfig ? { }, - }: - nixpkgs.lib.nixosSystem { - modules = - let - settings = machineSettings name; - in - (machineImports settings) - ++ [ - { - # Autoinclude configuration.nix and hardware-configuration.nix - imports = builtins.filter (p: builtins.pathExists p) [ - "${directory}/machines/${name}/configuration.nix" - "${directory}/machines/${name}/hardware-configuration.nix" - ]; - } - settings - clan-core.nixosModules.clanCore - extraConfig - (machines.${name} or { }) - # Inherit the inventory assertions ? - { inherit (mergedInventory) assertions; } - { imports = serviceConfigs.${name} or { }; } - ( - { - # Settings - clan.core.clanDir = directory; - # Inherited from clan wide settings - clan.core.clanName = meta.name or clanName; - clan.core.clanIcon = meta.icon or clanIcon; - - # Machine specific settings - clan.core.machineName = name; - networking.hostName = lib.mkDefault name; - nixpkgs.hostPlatform = lib.mkDefault system; - - # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) - nix.registry.nixpkgs.to = { - type = "path"; - path = lib.mkDefault nixpkgs; - }; - } - // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } - ) - ]; - specialArgs = { - inherit clan-core; - } // specialArgs; - }; - - allMachines = mergedInventory.machines or { }; - - supportedSystems = [ - "x86_64-linux" - "aarch64-linux" - "riscv64-linux" - "x86_64-darwin" - "aarch64-darwin" - ]; - - nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; - - # This instantiates nixos for each system that we support: - # configPerSystem = ..nixosConfiguration - # We need this to build nixos secret generators for each system - configsPerSystem = builtins.listToAttrs ( - builtins.map ( - system: - lib.nameValuePair system ( - lib.mapAttrs ( - name: _: - nixosConfiguration { - inherit name system; - pkgs = pkgsForSystem system; - } - ) allMachines - ) - ) supportedSystems - ); - - configsFuncPerSystem = builtins.listToAttrs ( - builtins.map ( - system: - lib.nameValuePair system ( - lib.mapAttrs ( - name: _: args: - nixosConfiguration ( - args - // { - inherit name system; - pkgs = pkgsForSystem system; - } - ) - ) allMachines - ) - ) supportedSystems - ); + rest = builtins.removeAttrs attrs [ "specialArgs" ]; in -builtins.deepSeq deprecationWarnings { - inherit nixosConfigurations; - - clanInternals = { - inherit (clan-core) clanModules; - source = "${clan-core}"; - - meta = mergedInventory.meta; - inventory = mergedInventory; - - inventoryFile = "${directory}/inventory.json"; - - # machine specifics - machines = configsPerSystem; - machinesFunc = configsFuncPerSystem; - all-machines-json = lib.mapAttrs ( - system: configs: - nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( - lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs - ) - ) configsPerSystem; - }; +eval { + imports = [ + rest + # implementation + ./module.nix + ]; } diff --git a/lib/build-clan/eval.nix b/lib/build-clan/eval.nix new file mode 100644 index 000000000..123d980f0 --- /dev/null +++ b/lib/build-clan/eval.nix @@ -0,0 +1,19 @@ +{ + lib, + nixpkgs, + clan-core, + specialArgs ? { }, + self, +}: +# Returns a function that takes self, which should point to the directory of the flake +module: +(lib.evalModules { + specialArgs = { + inherit self clan-core nixpkgs; + }; + modules = [ + ./interface.nix + module + { inherit specialArgs; } + ]; +}).config diff --git a/lib/build-clan/flake-module.nix b/lib/build-clan/flake-module.nix new file mode 100644 index 000000000..13310663a --- /dev/null +++ b/lib/build-clan/flake-module.nix @@ -0,0 +1,41 @@ +{ self, inputs, ... }: +let + inputOverrides = builtins.concatStringsSep " " ( + builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs) + ); +in +{ + + perSystem = + { + pkgs, + lib, + system, + ... + }: + # let + + # in + { + + # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests + legacyPackages.evalTests-build-clan = import ./tests.nix { + inherit lib; + inherit (inputs) nixpkgs; + clan-core = self; + buildClan = self.lib.buildClan; + }; + checks = { + lib-build-clan-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' + export HOME="$(realpath .)" + + nix-unit --eval-store "$HOME" \ + --extra-experimental-features flakes \ + ${inputOverrides} \ + --flake ${self}#legacyPackages.${system}.evalTests-build-clan + + touch $out + ''; + }; + }; +} diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix new file mode 100644 index 000000000..2e11b531e --- /dev/null +++ b/lib/build-clan/interface.nix @@ -0,0 +1,74 @@ +{ lib, self, ... }: +let + types = lib.types; +in +{ + options = { + # Required options + directory = lib.mkOption { + type = types.path; + default = self; + description = "The directory containing the clan subdirectory"; + }; + + specialArgs = lib.mkOption { + type = types.attrsOf types.raw; + default = { }; + description = "Extra arguments to pass to nixosSystem i.e. useful to make self available"; + }; + + # Optional + machines = lib.mkOption { + type = types.attrsOf types.deferredModule; + default = { }; + }; + inventory = lib.mkOption { + type = types.submodule { imports = [ ../inventory/build-inventory/interface.nix ]; }; + }; + + # Meta + meta = lib.mkOption { + type = types.nullOr ( + types.submodule { + options = { + name = lib.mkOption { + type = types.nullOr types.str; + description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to."; + }; + }; + } + ); + default = null; + }; + + pkgsForSystem = lib.mkOption { + type = types.functionTo (types.nullOr types.attrs); + default = _: null; + }; + + # Outputs + nixosConfigurations = lib.mkOption { + type = types.lazyAttrsOf types.raw; + default = { }; + }; + # flake.clanInternals + clanInternals = lib.mkOption { + # type = types.raw; + # ClanInternals + type = types.submodule { + options = { + # Those options are interfaced by the CLI + # We don't speficy the type here, for better performance. + inventory = lib.mkOption { type = lib.types.raw; }; + inventoryFile = lib.mkOption { type = lib.types.raw; }; + clanModules = lib.mkOption { type = lib.types.raw; }; + source = lib.mkOption { type = lib.types.raw; }; + meta = lib.mkOption { type = lib.types.raw; }; + all-machines-json = lib.mkOption { type = lib.types.raw; }; + machines = lib.mkOption { type = lib.types.raw; }; + machinesFunc = lib.mkOption { type = lib.types.raw; }; + }; + }; + }; + }; +} diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix new file mode 100644 index 000000000..390625248 --- /dev/null +++ b/lib/build-clan/module.nix @@ -0,0 +1,228 @@ +{ + config, + clan-core, + nixpkgs, + lib, + ... +}: +let + inherit (config) + directory + machines + pkgsForSystem + specialArgs + ; + + inherit (config.clanInternals) inventory; + + inherit (clan-core.lib.inventory) buildInventory; + + # map from machine name to service configuration + # { ${machineName} :: Config } + serviceConfigs = ( + buildInventory { + inherit inventory; + inherit directory; + } + ); + + machineSettings = + machineName: + let + warn = lib.warn '' + The use of ./machines//settings.json is deprecated. + If your settings.json is empty, you can safely remove it. + !!! Consider using the inventory system. !!! + + File: ${directory + /machines/${machineName}/settings.json} + + If there are still features missing in the inventory system, please open an issue on the clan-core repository. + ''; + in + # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily + # This is useful for doing a dry-run before writing changes into the settings.json + # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then + warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) + else + lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( + warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) + ); + + machineImports = + machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); + + # TODO: remove default system once we have a hardware-config mechanism + nixosConfiguration = + { + system ? "x86_64-linux", + name, + pkgs ? null, + extraConfig ? { }, + }: + nixpkgs.lib.nixosSystem { + modules = + let + settings = machineSettings name; + in + (machineImports settings) + ++ [ + { + # Autoinclude configuration.nix and hardware-configuration.nix + imports = builtins.filter builtins.pathExists [ + "${directory}/machines/${name}/configuration.nix" + "${directory}/machines/${name}/hardware-configuration.nix" + ]; + } + settings + clan-core.nixosModules.clanCore + extraConfig + (machines.${name} or { }) + # Inherit the inventory assertions ? + # { inherit (mergedInventory) assertions; } + { imports = serviceConfigs.${name} or [ ]; } + ( + { + # Settings + clan.core.clanDir = directory; + # Inherited from clan wide settings + # TODO: remove these + clan.core.name = config.inventory.meta.name; + clan.core.icon = config.inventory.meta.icon; + + # Machine specific settings + clan.core.machineName = name; + networking.hostName = lib.mkDefault name; + nixpkgs.hostPlatform = lib.mkDefault system; + + # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) + nix.registry.nixpkgs.to = { + type = "path"; + path = lib.mkDefault nixpkgs; + }; + } + // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } + ) + ]; + + specialArgs = { + inherit clan-core; + } // specialArgs; + }; + + # TODO: Will be deprecated + # We must migrate the tests, that create a settings.json to add a machine. + ################################################## + testMachines = + lib.mapAttrs + (name: _: { + inherit name; + system = (machineSettings name).nixpkgs.hostSystem or null; + }) + ( + lib.filterAttrs ( + machineName: _: + if builtins.pathExists "${directory}/machines/${machineName}/settings.json" then + lib.warn '' + The use of ./machines//settings.json is deprecated. + If your settings.json is empty, you can safely remove it. + !!! Consider using the inventory system. !!! + + File: ${directory + /machines/${machineName}/settings.json} + + If there are still features missing in the inventory system, please open an issue on the clan-core repository. + '' true + else + false + ) machinesDirs + ); + machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( + builtins.readDir (directory + /machines) + ); + ################################################## + + allMachines = inventory.machines or { } // config.machines or { } // testMachines; + + supportedSystems = [ + "x86_64-linux" + "aarch64-linux" + "riscv64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + nixosConfigurations = lib.mapAttrs (name: _: nixosConfiguration { inherit name; }) allMachines; + + # This instantiates nixos for each system that we support: + # configPerSystem = ..nixosConfiguration + # We need this to build nixos secret generators for each system + configsPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: + nixosConfiguration { + inherit name system; + pkgs = pkgsForSystem system; + } + ) allMachines + ) + ) supportedSystems + ); + + configsFuncPerSystem = builtins.listToAttrs ( + builtins.map ( + system: + lib.nameValuePair system ( + lib.mapAttrs ( + name: _: args: + nixosConfiguration ( + args + // { + inherit name system; + pkgs = pkgsForSystem system; + } + ) + ) allMachines + ) + ) supportedSystems + ); + + inventoryFile = "${directory}/inventory.json"; + + inventoryLoaded = + if builtins.pathExists inventoryFile then + (builtins.fromJSON (builtins.readFile inventoryFile)) + else + { }; +in +{ + imports = [ + # Merge the inventory file + { inventory = inventoryLoaded; } + # Merge the meta attributes from the buildClan function + { inventory.meta = if config.meta != null then config.meta else { }; } + ]; + + inherit nixosConfigurations; + + clanInternals = { + inherit (clan-core) clanModules; + inherit inventoryFile; + inventory = config.inventory; + meta = config.inventory.meta; + + source = "${clan-core}"; + + # machine specifics + machines = configsPerSystem; + machinesFunc = configsFuncPerSystem; + all-machines-json = lib.mapAttrs ( + system: configs: + nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" ( + lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs + ) + ) configsPerSystem; + }; +} diff --git a/lib/build-clan/tests.nix b/lib/build-clan/tests.nix new file mode 100644 index 000000000..eb0250051 --- /dev/null +++ b/lib/build-clan/tests.nix @@ -0,0 +1,153 @@ +{ + lib, + nixpkgs, + clan-core, + buildClan, + ... +}: +let + evalClan = import ./eval.nix { + inherit lib nixpkgs clan-core; + self = ./.; + }; +in +####### +{ + test_only_required = + let + config = evalClan { + meta.name = "test"; + imports = [ ./module.nix ]; + }; + in + { + expr = config.inventory ? meta; + expected = true; + }; + + test_all_simple = + let + config = evalClan { + directory = ./.; + machines = { }; + inventory = { + meta.name = "test"; + }; + pkgsForSystem = _system: { }; + }; + in + { + expr = config ? inventory; + expected = true; + }; + + test_outputs_clanInternals = + let + config = evalClan { + imports = [ + # What the user needs to specif + { + directory = ./.; + inventory.meta.name = "test"; + } + + ./module.nix + # Explicit output, usually defined by flake-parts + { options.nixosConfigurations = lib.mkOption { type = lib.types.raw; }; } + ]; + }; + in + { + expr = config.clanInternals.meta; + expected = { + description = null; + icon = null; + name = "test"; + }; + }; + + test_fn_simple = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + }; + in + { + expr = result.clanInternals.meta; + expected = { + description = null; + icon = null; + name = "test"; + }; + }; + + test_fn_extensiv_meta = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + meta.description = "test"; + meta.icon = "test"; + inventory.meta.name = "superclan"; + inventory.meta.description = "description"; + inventory.meta.icon = "icon"; + }; + in + { + expr = result.clanInternals.meta; + expectedError = { + type = "ThrownError"; + msg = ""; + }; + }; + + test_fn_clan_core = + let + result = buildClan { + directory = ../../.; + meta.name = "test-clan-core"; + }; + in + { + expr = builtins.attrNames result.nixosConfigurations; + expected = [ "test-inventory-machine" ]; + }; + + test_buildClan_all_machines = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + inventory.machines.machine1.meta.name = "machine1"; + + machines.machine2 = { }; + + }; + in + { + expr = builtins.attrNames result.nixosConfigurations; + expected = [ + "machine1" + "machine2" + ]; + }; + + test_buildClan_specialArgs = + let + result = buildClan { + directory = ./.; + meta.name = "test"; + specialArgs.foo = "dream2nix"; + machines.machine2 = + { foo, ... }: + { + networking.hostName = foo; + }; + }; + in + { + expr = result.nixosConfigurations.machine2.config.networking.hostName; + expected = "dream2nix"; + }; +} diff --git a/lib/default.nix b/lib/default.nix index 91a23d6df..23556ee91 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -6,7 +6,7 @@ }: { evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; }; - buildClan = import ./build-clan { inherit clan-core lib nixpkgs; }; + buildClan = import ./build-clan { inherit lib nixpkgs clan-core; }; facts = import ./facts.nix { inherit lib; }; inventory = import ./inventory { inherit lib clan-core; }; jsonschema = import ./jsonschema { inherit lib; }; diff --git a/lib/eval-clan-modules/default.nix b/lib/eval-clan-modules/default.nix index 814388d7e..ccef35868 100644 --- a/lib/eval-clan-modules/default.nix +++ b/lib/eval-clan-modules/default.nix @@ -12,7 +12,7 @@ let imports = (import (pkgs.path + "/nixos/modules/module-list.nix")) ++ [ { nixpkgs.hostPlatform = "x86_64-linux"; - clan.core.clanName = "dummy"; + clan.core.name = "dummy"; } ]; }; diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 0785872a5..e9ab69a54 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -8,6 +8,7 @@ imports = [ ./jsonschema/flake-module.nix ./inventory/flake-module.nix + ./build-clan/flake-module.nix ]; flake.lib = import ./default.nix { inherit lib inputs; diff --git a/lib/inventory/build-inventory/assertions.nix b/lib/inventory/build-inventory/assertions.nix new file mode 100644 index 000000000..102e0c851 --- /dev/null +++ b/lib/inventory/build-inventory/assertions.nix @@ -0,0 +1,77 @@ +# Integrity validation of the inventory +{ config, lib, ... }: +{ + # Assertion must be of type + # { assertion :: bool, message :: string, severity :: "error" | "warning" } + imports = [ + # Check that each machine used in a service is defined in the top-level machines + { + assertions = lib.foldlAttrs ( + ass1: serviceName: c: + ass1 + ++ lib.foldlAttrs ( + ass2: instanceName: instanceConfig: + let + topLevelMachines = lib.attrNames config.machines; + # All machines must be defined in the top-level machines + assertions = lib.foldlAttrs ( + assertions: roleName: role: + assertions + ++ builtins.filter (a: !a.assertion) ( + builtins.map (m: { + assertion = builtins.elem m topLevelMachines; + message = '' + Machine '${m}' is not defined in the inventory. This might still work, if the machine is defined via nix. + + Defined in service: '${serviceName}' instance: '${instanceName}' role: '${roleName}'. + + Inventory machines: + ${builtins.concatStringsSep "\n" (map (n: "'${n}'") topLevelMachines)} + ''; + severity = "warning"; + }) role.machines + ) + ) [ ] instanceConfig.roles; + in + ass2 ++ assertions + ) [ ] c + ) [ ] config.services; + } + # Check that each tag used in a role is defined in at least one machines tags + { + assertions = lib.foldlAttrs ( + ass1: serviceName: c: + ass1 + ++ lib.foldlAttrs ( + ass2: instanceName: instanceConfig: + let + allTags = lib.foldlAttrs ( + tags: _machineName: machine: + tags ++ machine.tags + ) [ ] config.machines; + # All machines must be defined in the top-level machines + assertions = lib.foldlAttrs ( + assertions: roleName: role: + assertions + ++ builtins.filter (a: !a.assertion) ( + builtins.map (m: { + assertion = builtins.elem m allTags; + message = '' + Tag '${m}' is not defined in the inventory. + + Defined in service: '${serviceName}' instance: '${instanceName}' role: '${roleName}'. + + Available tags: + ${builtins.concatStringsSep "\n" (map (n: "'${n}'") allTags)} + ''; + severity = "error"; + }) role.tags + ) + ) [ ] instanceConfig.roles; + in + ass2 ++ assertions + ) [ ] c + ) [ ] config.services; + } + ]; +} diff --git a/lib/inventory/build-inventory/default.nix b/lib/inventory/build-inventory/default.nix index 1617d8f8c..1ae86a151 100644 --- a/lib/inventory/build-inventory/default.nix +++ b/lib/inventory/build-inventory/default.nix @@ -1,10 +1,7 @@ # Generate partial NixOS configurations for every machine in the inventory # This function is responsible for generating the module configuration for every machine in the inventory. { lib, clan-core }: -{ inventory, directory }: let - machines = machinesFromInventory inventory; - resolveTags = # Inventory, { machines :: [string], tags :: [string] } { @@ -45,8 +42,41 @@ let machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration } */ - machinesFromInventory = + + # { client_1_machine = { tags = [ "backup" ]; }; client_2_machine = { tags = [ "backup" ]; }; not_used_machine = { }; } + getAllMachines = inventory: + lib.foldlAttrs ( + res: serviceName: serviceConfigs: + (lib.foldlAttrs ( + res: instanceName: serviceConfig: + lib.foldlAttrs ( + res: roleName: members: + let + resolved = resolveTags { + inherit + serviceName + instanceName + roleName + inventory + members + ; + }; + in + res + // builtins.listToAttrs ( + builtins.map (m: { + name = m; + value = { }; + }) resolved.machines + ) + ) res serviceConfig.roles + ) res serviceConfigs) + ) { } (inventory.services or { }) + // inventory.machines or { }; + + buildInventory = + { inventory, directory }: # For every machine in the inventory, build a NixOS configuration # For each machine generate config, forEach service, if the machine is used. builtins.mapAttrs ( @@ -152,6 +182,8 @@ let config.clan.core.networking.targetHost = machineConfig.deploy.targetHost; }) ] - ) inventory.machines or { }; + ) (getAllMachines inventory); in -machines +{ + inherit buildInventory getAllMachines; +} diff --git a/lib/inventory/build-inventory/interface.nix b/lib/inventory/build-inventory/interface.nix index 0f4ea2bf3..133e65c58 100644 --- a/lib/inventory/build-inventory/interface.nix +++ b/lib/inventory/build-inventory/interface.nix @@ -51,6 +51,7 @@ let }; in { + imports = [ ./assertions.nix ]; options = { assertions = lib.mkOption { type = types.listOf types.unspecified; @@ -126,39 +127,4 @@ in ); }; }; - - # Smoke validation of the inventory - config.assertions = - let - # Inventory assertions - # - All referenced machines must exist in the top-level machines - serviceAssertions = lib.foldlAttrs ( - ass1: serviceName: c: - ass1 - ++ lib.foldlAttrs ( - ass2: instanceName: instanceConfig: - let - serviceMachineNames = lib.attrNames instanceConfig.machines; - topLevelMachines = lib.attrNames config.machines; - # All machines must be defined in the top-level machines - assertions = builtins.map (m: { - assertion = builtins.elem m topLevelMachines; - message = "${serviceName}.${instanceName}.machines.${m}. Should be one of [ ${builtins.concatStringsSep " | " topLevelMachines} ]"; - }) serviceMachineNames; - in - ass2 ++ assertions - ) [ ] c - ) [ ] config.services; - - # Machine assertions - # - A machine must define their host system - machineAssertions = map ( - { name, ... }: - { - assertion = true; - message = "Machine ${name} should define its host system in the inventory. ()"; - } - ) (lib.attrsToList (lib.filterAttrs (_n: v: v.system or null == null) config.machines)); - in - machineAssertions ++ serviceAssertions; } diff --git a/lib/inventory/default.nix b/lib/inventory/default.nix index 78a4c1ac4..4550a4299 100644 --- a/lib/inventory/default.nix +++ b/lib/inventory/default.nix @@ -1,5 +1,5 @@ { lib, clan-core }: { - buildInventory = import ./build-inventory { inherit lib clan-core; }; + inherit (import ./build-inventory { inherit lib clan-core; }) buildInventory; interface = ./build-inventory/interface.nix; } diff --git a/lib/inventory/flake-module.nix b/lib/inventory/flake-module.nix index 14394022a..c4962c99e 100644 --- a/lib/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -17,10 +17,12 @@ in ... }: let - buildInventory = import ./build-inventory { - clan-core = self; - inherit lib; - }; + inventory = ( + import ./build-inventory { + clan-core = self; + inherit lib; + } + ); getSchema = import ./interface-to-schema.nix { inherit lib self; }; @@ -98,7 +100,7 @@ in # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests legacyPackages.evalTests-inventory = import ./tests { - inherit buildInventory; + inherit inventory; clan-core = self; }; diff --git a/lib/inventory/tests/default.nix b/lib/inventory/tests/default.nix index 35e993b1f..3ab5a5ee2 100644 --- a/lib/inventory/tests/default.nix +++ b/lib/inventory/tests/default.nix @@ -1,5 +1,37 @@ -{ buildInventory, clan-core, ... }: +{ inventory, clan-core, ... }: +let + inherit (inventory) buildInventory getAllMachines; +in { + test_get_all_used_machines = { + # Test that all machines are returned + expr = getAllMachines { + machines = { + machine_3 = { + tags = [ "tag_3" ]; + }; + }; + services = { + borgbackup.instance_1 = { + roles.server.machines = [ "backup_server" ]; + roles.client.machines = [ + "client_1_machine" + "client_2_machine" + ]; + roles.client.tags = [ "tag_3" ]; + }; + }; + }; + expected = { + backup_server = { }; + client_1_machine = { }; + client_2_machine = { }; + machine_3 = { + tags = [ "tag_3" ]; + }; + }; + }; + test_inventory_empty = { # Empty inventory should return an empty module expr = buildInventory { diff --git a/nixosModules/bcachefs.nix b/nixosModules/bcachefs.nix index 24968b878..43dc7b43c 100644 --- a/nixosModules/bcachefs.nix +++ b/nixosModules/bcachefs.nix @@ -1,17 +1,16 @@ -{ lib, pkgs, ... }: - { - # use latest kernel we can support to get more hardware support - boot.kernelPackages = - lib.mkForce - (pkgs.zfs.override { removeLinuxDRM = pkgs.hostPlatform.isAarch64; }).latestCompatibleLinuxPackages; - boot.zfs.removeLinuxDRM = lib.mkDefault pkgs.hostPlatform.isAarch64; + lib, + pkgs, + config, + ... +}: +{ + # If we also need zfs, we can use the unstable version as we otherwise don't have a new enough kernel version + boot.zfs.package = pkgs.zfsUnstable; + boot.kernelPackages = lib.mkIf config.boot.zfs.enabled ( + lib.mkForce config.boot.zfs.package.latestCompatibleLinuxPackages + ); # Enable bcachefs support boot.supportedFilesystems.bcachefs = lib.mkDefault true; - - environment.systemPackages = with pkgs; [ - bcachefs-tools - keyutils - ]; } diff --git a/nixosModules/clanCore/facts/default.nix b/nixosModules/clanCore/facts/default.nix index f1121e2de..bb0a505ef 100644 --- a/nixosModules/clanCore/facts/default.nix +++ b/nixosModules/clanCore/facts/default.nix @@ -78,7 +78,7 @@ Each service can have a generator script which generates the secrets and facts. The generator script is expected to generate all secrets and facts defined for this service. - A `service` does not need to ba analogous to a systemd service, it can be any group of facts and secrets that need to be generated together. + A `service` does not need to be analogous to a systemd service, it can be any group of facts and secrets that need to be generated together. ''; default = { }; type = lib.types.attrsOf ( diff --git a/nixosModules/clanCore/meta/interface.nix b/nixosModules/clanCore/meta/interface.nix index 3b44046e7..aef373d57 100644 --- a/nixosModules/clanCore/meta/interface.nix +++ b/nixosModules/clanCore/meta/interface.nix @@ -6,5 +6,4 @@ in options.clan.meta.name = lib.mkOption { type = lib.types.str; }; options.clan.meta.description = lib.mkOption { type = optStr; }; options.clan.meta.icon = lib.mkOption { type = optStr; }; - options.clan.tags = lib.mkOption { type = lib.types.listOf lib.types.str; }; } diff --git a/nixosModules/clanCore/metadata.nix b/nixosModules/clanCore/metadata.nix index d9947296b..6f037a8a0 100644 --- a/nixosModules/clanCore/metadata.nix +++ b/nixosModules/clanCore/metadata.nix @@ -1,11 +1,33 @@ { lib, pkgs, ... }: { + imports = [ + (lib.mkRemovedOptionModule [ + "clan" + "core" + "clanName" + ] "clanName has been removed. Use clan.core.name instead.") + (lib.mkRemovedOptionModule [ + "clan" + "core" + "clanIcon" + ] "clanIcon has been removed. Use clan.core.icon instead.") + ]; options.clan.core = { - clanName = lib.mkOption { + name = lib.mkOption { type = lib.types.str; description = '' the name of the clan ''; + # Set by the flake, so it's read-only in the maschine + readOnly = true; + }; + icon = lib.mkOption { + type = lib.types.nullOr lib.types.path; + description = '' + the location of the clan icon + ''; + # Set by the flake, so it's read-only in the maschine + readOnly = true; }; machineIcon = lib.mkOption { type = lib.types.nullOr lib.types.path; @@ -28,12 +50,6 @@ the location of the flake repo, used to calculate the location of facts and secrets ''; }; - clanIcon = lib.mkOption { - type = lib.types.nullOr lib.types.path; - description = '' - the location of the clan icon - ''; - }; machineName = lib.mkOption { type = lib.types.str; default = "nixos"; diff --git a/nixosModules/clanCore/state.nix b/nixosModules/clanCore/state.nix index 17757fed7..5e0425092 100644 --- a/nixosModules/clanCore/state.nix +++ b/nixosModules/clanCore/state.nix @@ -106,8 +106,6 @@ ); }; - # defaults - config.clan.core.state.HOME.folders = [ "/home" ]; config.environment.systemPackages = lib.optional (config.clan.core.state != { }) ( pkgs.runCommand "state-commands" { } '' ${builtins.concatStringsSep "\n" ( diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index 396f4e531..d52acee1f 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -39,7 +39,12 @@ in vars = { generators = lib.flip lib.mapAttrs config.clan.core.vars.generators ( _name: generator: { - inherit (generator) dependencies finalScript prompts; + inherit (generator) + dependencies + finalScript + prompts + share + ; files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; }); } ); diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index c86f6925d..b54f0686b 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -168,6 +168,15 @@ in internal = true; visible = false; }; + share = { + description = '' + Whether the generated vars should be shared between machines. + Shared vars are only generated once, when the first machine using it is deployed. + Subsequent machines will re-use the already generated values. + ''; + type = bool; + default = false; + }; }; }) ); diff --git a/nixosModules/clanCore/vars/secret/sops/default.nix b/nixosModules/clanCore/vars/secret/sops/default.nix index b1614f06e..43c0983d1 100644 --- a/nixosModules/clanCore/vars/secret/sops/default.nix +++ b/nixosModules/clanCore/vars/secret/sops/default.nix @@ -10,9 +10,17 @@ let inherit (import ./funcs.nix { inherit lib; }) listVars; - varsDir = config.clan.core.clanDir + "/sops/vars"; + varsDirMachines = config.clan.core.clanDir + "/sops/vars/per-machine"; + varsDirShared = config.clan.core.clanDir + "/sops/vars/shared"; - vars = listVars varsDir; + varsUnfiltered = (listVars varsDirMachines) ++ (listVars varsDirShared); + filterVars = + vars: + builtins.elem vars.machine [ + config.clan.core.machineName + "shared" + ]; + vars = lib.filter filterVars varsUnfiltered; in { @@ -33,7 +41,7 @@ in flip map vars (secret: { name = secret.id; value = { - sopsFile = config.clan.core.clanDir + "/sops/vars/${secret.id}/secret"; + sopsFile = secret.sopsFile; format = "binary"; }; }) diff --git a/nixosModules/clanCore/vars/secret/sops/funcs.nix b/nixosModules/clanCore/vars/secret/sops/funcs.nix index b5700794f..4b6dded74 100644 --- a/nixosModules/clanCore/vars/secret/sops/funcs.nix +++ b/nixosModules/clanCore/vars/secret/sops/funcs.nix @@ -23,6 +23,7 @@ rec { generator = generator_name; name = secret_name; id = "${machine_name}/${generator_name}/${secret_name}"; + sopsFile = "${varsDir}/${machine_name}/${generator_name}/${secret_name}/secret"; }) ) ); diff --git a/nixosModules/clanCore/vm.nix b/nixosModules/clanCore/vm.nix index ee9d88a54..88cea4524 100644 --- a/nixosModules/clanCore/vm.nix +++ b/nixosModules/clanCore/vm.nix @@ -20,6 +20,8 @@ let ./waypipe.nix ]; + clan.core.state.HOME.folders = [ "/home" ]; + clan.services.waypipe = { inherit (config.clan.core.vm.inspect.waypipe) enable command; }; @@ -39,18 +41,14 @@ let boot.initrd.systemd.enable = true; - # currently needed for system.etc.overlay.enable - boot.kernelPackages = pkgs.linuxPackages_latest; - boot.initrd.systemd.storePaths = [ pkgs.util-linux pkgs.e2fsprogs ]; boot.initrd.systemd.emergencyAccess = true; - # sysusers is faster than nixos's perl scripts - # and doesn't require state. - systemd.sysusers.enable = true; + # sysusers would be faster because it doesn't need perl, but it cannot create normal users + systemd.sysusers.enable = false; users.mutableUsers = false; users.allowNoPasswordLogin = true; @@ -252,8 +250,8 @@ in config = { # for clan vm inspect clan.core.vm.inspect = { - clan_name = config.clan.core.clanName; - machine_icon = config.clan.core.machineIcon or config.clan.core.clanIcon; + clan_name = config.clan.core.name; + machine_icon = config.clan.core.machineIcon or config.clan.core.icon; machine_name = config.clan.core.machineName; machine_description = config.clan.core.machineDescription; memory_size = config.clan.virtualisation.memorySize; diff --git a/nixosModules/clanCore/zerotier/default.nix b/nixosModules/clanCore/zerotier/default.nix index a26974b0b..59804d911 100644 --- a/nixosModules/clanCore/zerotier/default.nix +++ b/nixosModules/clanCore/zerotier/default.nix @@ -23,8 +23,8 @@ in }; name = lib.mkOption { type = lib.types.str; - default = config.clan.core.clanName; - defaultText = "config.clan.core.clanName"; + default = config.clan.core.name; + defaultText = "config.clan.core.name"; description = '' zerotier network name ''; @@ -89,11 +89,7 @@ in ({ # Override license so that we can build zerotierone without # having to re-import nixpkgs. - services.zerotierone.package = lib.mkDefault ( - pkgs.zerotierone.overrideAttrs (_old: { - meta = { }; - }) - ); + services.zerotierone.package = lib.mkDefault (pkgs.callPackage ../../../pkgs/zerotierone { }); }) (lib.mkIf ((facts.zerotier-ip.value or null) != null) { environment.etc."zerotier/ip".text = facts.zerotier-ip.value; diff --git a/nixosModules/flake-module.nix b/nixosModules/flake-module.nix index 22797f8d4..32ff61eae 100644 --- a/nixosModules/flake-module.nix +++ b/nixosModules/flake-module.nix @@ -3,10 +3,12 @@ flake.nixosModules = { hidden-ssh-announce.imports = [ ./hidden-ssh-announce.nix ]; bcachefs.imports = [ ./bcachefs.nix ]; + zfs.imports = [ ./zfs.nix ]; installer.imports = [ ./installer self.nixosModules.hidden-ssh-announce self.nixosModules.bcachefs + self.nixosModules.zfs ]; clanCore.imports = [ inputs.sops-nix.nixosModules.sops diff --git a/nixosModules/zfs.nix b/nixosModules/zfs.nix new file mode 100644 index 000000000..01b69613b --- /dev/null +++ b/nixosModules/zfs.nix @@ -0,0 +1,16 @@ +{ lib, config, ... }: +{ + # Use the same default hostID as the NixOS install ISO and nixos-anywhere. + # This allows us to import zfs pool without using a force import. + # ZFS has this as a safety mechanism for networked block storage (ISCSI), but + # in practice we found it causes more breakages like unbootable machines, + # while people using ZFS on ISCSI is quite rare. + networking.hostId = lib.mkDefault "8425e349"; + + services.zfs = lib.mkIf (config.boot.zfs.enabled) { + autoSnapshot.enable = true; + # defaults to 12, which is a bit much given how much data is written + autoSnapshot.monthly = lib.mkDefault 1; + autoScrub.enable = true; + }; +} diff --git a/pkgs/clan-app/clan_app/api/__init__.py b/pkgs/clan-app/clan_app/api/__init__.py index c577dc467..73d424ac7 100644 --- a/pkgs/clan-app/clan_app/api/__init__.py +++ b/pkgs/clan-app/clan_app/api/__init__.py @@ -60,6 +60,10 @@ class ImplFunc(GObject.Object, Generic[P, B]): return result +# TODO: Reimplement this such that it uses a multiprocessing.Array of type ctypes.c_char +# all fn arguments are serialized to json and passed to the new process over the Array +# the new process deserializes the json and calls the function +# the result is serialized to json and passed back to the main process over another Array class MethodExecutor(threading.Thread): def __init__( self, function: Callable[..., Any], *args: Any, **kwargs: dict[str, Any] diff --git a/pkgs/clan-app/clan_app/api/file.py b/pkgs/clan-app/clan_app/api/file.py index a86f23874..c3235f926 100644 --- a/pkgs/clan-app/clan_app/api/file.py +++ b/pkgs/clan-app/clan_app/api/file.py @@ -4,6 +4,8 @@ import gi gi.require_version("Gtk", "4.0") import logging +from pathlib import Path +from typing import Any from clan_cli.api import ErrorDataClass, SuccessDataClass from clan_cli.api.directory import FileRequest @@ -14,10 +16,14 @@ from clan_app.api import ImplFunc log = logging.getLogger(__name__) +def remove_none(_list: list) -> list: + return [i for i in _list if i is not None] + + # This implements the abstract function open_file with one argument, file_request, # which is a FileRequest object and returns a string or None. class open_file( - ImplFunc[[FileRequest, str], SuccessDataClass[str | None] | ErrorDataClass] + ImplFunc[[FileRequest, str], SuccessDataClass[list[str] | None] | ErrorDataClass] ): def __init__(self) -> None: super().__init__() @@ -27,7 +33,7 @@ class open_file( try: gfile = file_dialog.open_finish(task) if gfile: - selected_path = gfile.get_path() + selected_path = remove_none([gfile.get_path()]) self.returns( SuccessDataClass( op_key=op_key, data=selected_path, status="success" @@ -36,16 +42,39 @@ class open_file( except Exception as e: print(f"Error getting selected file or directory: {e}") + def on_file_select_multiple( + file_dialog: Gtk.FileDialog, task: Gio.Task + ) -> None: + try: + gfiles: Any = file_dialog.open_multiple_finish(task) + if gfiles: + selected_paths = remove_none([gfile.get_path() for gfile in gfiles]) + self.returns( + SuccessDataClass( + op_key=op_key, data=selected_paths, status="success" + ) + ) + else: + self.returns( + SuccessDataClass(op_key=op_key, data=None, status="success") + ) + except Exception as e: + print(f"Error getting selected files: {e}") + def on_folder_select(file_dialog: Gtk.FileDialog, task: Gio.Task) -> None: try: gfile = file_dialog.select_folder_finish(task) if gfile: - selected_path = gfile.get_path() + selected_path = remove_none([gfile.get_path()]) self.returns( SuccessDataClass( op_key=op_key, data=selected_path, status="success" ) ) + else: + self.returns( + SuccessDataClass(op_key=op_key, data=None, status="success") + ) except Exception as e: print(f"Error getting selected directory: {e}") @@ -53,12 +82,16 @@ class open_file( try: gfile = file_dialog.save_finish(task) if gfile: - selected_path = gfile.get_path() + selected_path = remove_none([gfile.get_path()]) self.returns( SuccessDataClass( op_key=op_key, data=selected_path, status="success" ) ) + else: + self.returns( + SuccessDataClass(op_key=op_key, data=None, status="success") + ) except Exception as e: print(f"Error getting selected file: {e}") @@ -90,9 +123,21 @@ class open_file( filters.append(file_filters) dialog.set_filters(filters) + if file_request.initial_file: + p = Path(file_request.initial_file).expanduser() + f = Gio.File.new_for_path(str(p)) + dialog.set_initial_file(f) + + if file_request.initial_folder: + p = Path(file_request.initial_folder).expanduser() + f = Gio.File.new_for_path(str(p)) + dialog.set_initial_folder(f) + # if select_folder if file_request.mode == "select_folder": dialog.select_folder(callback=on_folder_select) + if file_request.mode == "open_multiple_files": + dialog.open_multiple(callback=on_file_select_multiple) elif file_request.mode == "open_file": dialog.open(callback=on_file_select) elif file_request.mode == "save": diff --git a/pkgs/clan-app/clan_app/assets/style.css b/pkgs/clan-app/clan_app/assets/style.css index c179744dd..772b9ad7c 100644 --- a/pkgs/clan-app/clan_app/assets/style.css +++ b/pkgs/clan-app/clan_app/assets/style.css @@ -1,66 +1,63 @@ /* Insert custom styles here */ navigation-view { - padding: 5px; - /* padding-left: 5px; + padding: 5px; + /* padding-left: 5px; padding-right: 5px; padding-bottom: 5px; */ } avatar { - margin: 2px; + margin: 2px; } .trust { - padding-top: 25px; - padding-bottom: 25px; + padding-top: 25px; + padding-bottom: 25px; } .join-list { - margin-top: 1px; - margin-left: 2px; - margin-right: 2px; - + margin-top: 1px; + margin-left: 2px; + margin-right: 2px; } .progress-bar { - margin-right: 25px; - min-width: 200px; + margin-right: 25px; + min-width: 200px; } .group-list { - background-color: inherit; + background-color: inherit; } .group-list > .activatable:hover { - background-color: unset; + background-color: unset; } .group-list > row { - margin-top: 12px; - border-bottom: unset; + margin-top: 12px; + border-bottom: unset; } - .vm-list { - margin-top: 25px; - margin-bottom: 25px; + margin-top: 25px; + margin-bottom: 25px; } .no-shadow { - box-shadow: none; + box-shadow: none; } .search-entry { - margin-bottom: 12px; + margin-bottom: 12px; } searchbar { - margin-bottom: 25px; + margin-bottom: 25px; } - .log-view { - margin-top: 12px; - font-family: monospace; - padding: 8px; + margin-top: 12px; + font-family: monospace; + padding: 8px; } diff --git a/pkgs/clan-app/clan_app/components/executor.py b/pkgs/clan-app/clan_app/components/executor.py new file mode 100644 index 000000000..b97013c50 --- /dev/null +++ b/pkgs/clan-app/clan_app/components/executor.py @@ -0,0 +1,127 @@ +import dataclasses +import logging +import multiprocessing as mp +import os +import signal +import sys +import traceback +from collections.abc import Callable +from pathlib import Path +from typing import Any + +log = logging.getLogger(__name__) + + +# Kill the new process and all its children by sending a SIGTERM signal to the process group +def _kill_group(proc: mp.Process) -> None: + pid = proc.pid + if proc.is_alive() and pid: + os.killpg(pid, signal.SIGTERM) + else: + log.warning(f"Process '{proc.name}' with pid '{pid}' is already dead") + + +@dataclasses.dataclass(frozen=True) +class MPProcess: + name: str + proc: mp.Process + out_file: Path + + # Kill the new process and all its children by sending a SIGTERM signal to the process group + def kill_group(self) -> None: + _kill_group(proc=self.proc) + + +def _set_proc_name(name: str) -> None: + if sys.platform != "linux": + return + import ctypes + + # Define the prctl function with the appropriate arguments and return type + libc = ctypes.CDLL("libc.so.6") + prctl = libc.prctl + prctl.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_ulong, + ctypes.c_ulong, + ctypes.c_ulong, + ] + prctl.restype = ctypes.c_int + + # Set the process name to "my_process" + prctl(15, name.encode(), 0, 0, 0) + + +def _init_proc( + func: Callable, + out_file: Path, + proc_name: str, + on_except: Callable[[Exception, mp.process.BaseProcess], None] | None, + **kwargs: Any, +) -> None: + # Create a new process group + os.setsid() + + # Open stdout and stderr + with open(out_file, "w") as out_fd: + os.dup2(out_fd.fileno(), sys.stdout.fileno()) + os.dup2(out_fd.fileno(), sys.stderr.fileno()) + + # Print some information + pid = os.getpid() + gpid = os.getpgid(pid=pid) + + # Set the process name + _set_proc_name(proc_name) + + # Close stdin + sys.stdin.close() + + linebreak = "=" * 5 + # Execute the main function + print(linebreak + f" {func.__name__}:{pid} " + linebreak, file=sys.stderr) + try: + func(**kwargs) + except Exception as ex: + traceback.print_exc() + if on_except is not None: + on_except(ex, mp.current_process()) + + # Kill the new process and all its children by sending a SIGTERM signal to the process group + pid = os.getpid() + gpid = os.getpgid(pid=pid) + print(f"Killing process group pid={pid} gpid={gpid}", file=sys.stderr) + os.killpg(gpid, signal.SIGTERM) + sys.exit(1) + # Don't use a finally block here, because we want the exitcode to be set to + # 0 if the function returns normally + + +def spawn( + *, + out_file: Path, + on_except: Callable[[Exception, mp.process.BaseProcess], None] | None, + func: Callable, + **kwargs: Any, +) -> MPProcess: + # Decouple the process from the parent + if mp.get_start_method(allow_none=True) is None: + mp.set_start_method(method="forkserver") + + # Set names + proc_name = f"MPExec:{func.__name__}" + + # Start the process + proc = mp.Process( + target=_init_proc, + args=(func, out_file, proc_name, on_except), + name=proc_name, + kwargs=kwargs, + ) + proc.start() + + # Return the process + mp_proc = MPProcess(name=proc_name, proc=proc, out_file=out_file) + + return mp_proc diff --git a/pkgs/clan-app/clan_app/views/webview.py b/pkgs/clan-app/clan_app/views/webview.py index 467883038..40835723b 100644 --- a/pkgs/clan-app/clan_app/views/webview.py +++ b/pkgs/clan-app/clan_app/views/webview.py @@ -1,4 +1,3 @@ -import dataclasses import json import logging from typing import Any @@ -116,10 +115,10 @@ class WebExecutor(GObject.Object): # Introspect the function and create the expected dataclass from dict dynamically # Depending on the introspected argument_type arg_class = self.jschema_api.get_method_argtype(method_name, k) - if dataclasses.is_dataclass(arg_class): - reconciled_arguments[k] = from_dict(arg_class, v) - else: - reconciled_arguments[k] = v + + # TODO: rename from_dict into something like construct_checked_value + # from_dict really takes Anything and returns an instance of the type/class + reconciled_arguments[k] = from_dict(arg_class, v) GLib.idle_add(fn_instance._async_run, reconciled_arguments) diff --git a/pkgs/clan-app/clan_app/windows/main_window.py b/pkgs/clan-app/clan_app/windows/main_window.py index d540dbc1e..cd599f800 100644 --- a/pkgs/clan-app/clan_app/windows/main_window.py +++ b/pkgs/clan-app/clan_app/windows/main_window.py @@ -1,4 +1,5 @@ import logging +import os import gi from clan_cli.api import API @@ -18,7 +19,7 @@ log = logging.getLogger(__name__) class MainWindow(Adw.ApplicationWindow): def __init__(self, config: ClanConfig) -> None: super().__init__() - self.set_title("Clan Manager") + self.set_title("Clan App") self.set_default_size(980, 850) # Overlay for GTK side exclusive toasts @@ -47,3 +48,4 @@ class MainWindow(Adw.ApplicationWindow): def on_destroy(self, source: "Adw.ApplicationWindow") -> None: log.debug("Destroying Adw.ApplicationWindow") + os._exit(0) diff --git a/pkgs/clan-app/pyproject.toml b/pkgs/clan-app/pyproject.toml index 1502af61f..96602631d 100644 --- a/pkgs/clan-app/pyproject.toml +++ b/pkgs/clan-app/pyproject.toml @@ -36,10 +36,6 @@ disallow_untyped_calls = true disallow_untyped_defs = true no_implicit_optional = true -[[tool.mypy.overrides]] -module = "argcomplete.*" -ignore_missing_imports = true - [[tool.mypy.overrides]] module = "clan_cli.*" ignore_missing_imports = true diff --git a/pkgs/clan-app/shell.nix b/pkgs/clan-app/shell.nix index 36d0da3bc..74878aac6 100644 --- a/pkgs/clan-app/shell.nix +++ b/pkgs/clan-app/shell.nix @@ -65,6 +65,5 @@ mkShell { export XDG_DATA_DIRS=${gtk4}/share/gsettings-schemas/gtk4-4.14.4:$XDG_DATA_DIRS export XDG_DATA_DIRS=${gsettings-desktop-schemas}/share/gsettings-schemas/gsettings-desktop-schemas-46.0:$XDG_DATA_DIRS - ''; } diff --git a/pkgs/clan-app/tests/test_cli.py b/pkgs/clan-app/tests/test_cli.py index 9a3b5cdb8..49b8e3805 100644 --- a/pkgs/clan-app/tests/test_cli.py +++ b/pkgs/clan-app/tests/test_cli.py @@ -2,6 +2,6 @@ import pytest from helpers import cli -def test_help(capfd: pytest.CaptureFixture) -> None: +def test_help() -> None: with pytest.raises(SystemExit): cli.run(["clan-app", "--help"]) diff --git a/pkgs/clan-cli/.vscode/launch.json b/pkgs/clan-cli/.vscode/launch.json index 4e2c20a75..163754efe 100644 --- a/pkgs/clan-cli/.vscode/launch.json +++ b/pkgs/clan-cli/.vscode/launch.json @@ -1,26 +1,24 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Clan Webui", - "type": "python", - "request": "launch", - "module": "clan_cli.webui", - "justMyCode": false, - "args": [ "--reload", "--no-open", "--log-level", "debug" ], - - }, - { - "name": "Clan Cli VMs", - "type": "python", - "request": "launch", - "module": "clan_cli", - "justMyCode": false, - "args": [ "vms" ], - - } - ] -} \ No newline at end of file + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Clan Webui", + "type": "python", + "request": "launch", + "module": "clan_cli.webui", + "justMyCode": false, + "args": ["--reload", "--no-open", "--log-level", "debug"] + }, + { + "name": "Clan Cli VMs", + "type": "python", + "request": "launch", + "module": "clan_cli", + "justMyCode": false, + "args": ["vms"] + } + ] +} diff --git a/pkgs/clan-cli/.vscode/settings.json b/pkgs/clan-cli/.vscode/settings.json index e5c263238..c40fd15aa 100644 --- a/pkgs/clan-cli/.vscode/settings.json +++ b/pkgs/clan-cli/.vscode/settings.json @@ -1,22 +1,22 @@ { - "python.testing.pytestArgs": [ - // Coverage is not supported by vscode: - // https://github.com/Microsoft/vscode-python/issues/693 - // Note that this will make pytest fail if pytest-cov is not installed, - // if that's the case, then this option needs to be be removed (overrides - // can be set at a workspace level, it's up to you to decide what's the - // best approach). You might also prefer to only set this option - // per-workspace (wherever coverage is used). - "--no-cov", - "tests" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "search.exclude": { - "**/.direnv": true - }, - "python.linting.mypyPath": "mypy", - "python.linting.mypyEnabled": true, - "python.linting.enabled": true, - "python.defaultInterpreterPath": "python" -} \ No newline at end of file + "python.testing.pytestArgs": [ + // Coverage is not supported by vscode: + // https://github.com/Microsoft/vscode-python/issues/693 + // Note that this will make pytest fail if pytest-cov is not installed, + // if that's the case, then this option needs to be be removed (overrides + // can be set at a workspace level, it's up to you to decide what's the + // best approach). You might also prefer to only set this option + // per-workspace (wherever coverage is used). + "--no-cov", + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "search.exclude": { + "**/.direnv": true + }, + "python.linting.mypyPath": "mypy", + "python.linting.mypyEnabled": true, + "python.linting.enabled": true, + "python.defaultInterpreterPath": "python" +} diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index e155a04d9..7fac1fd93 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -5,20 +5,18 @@ from pathlib import Path from types import ModuleType # These imports are unused, but necessary for @API.register to run once. -from clan_cli.api import directory, mdns_discovery, modules +from clan_cli.api import directory, disk, mdns_discovery, modules from clan_cli.arg_actions import AppendOptionAction from clan_cli.clan import show, update # API endpoints that are not used in the cli. -__all__ = ["directory", "mdns_discovery", "modules", "update"] +__all__ = ["directory", "mdns_discovery", "modules", "update", "disk"] from . import ( backups, clan, - config, facts, flash, - flatpak, history, machines, secrets, @@ -178,18 +176,6 @@ For more detailed information, visit: {help_hyperlink("getting-started", "https: clan.register_parser(parser_flake) - parser_config = subparsers.add_parser( - "config", - help="read a nixos configuration option", - description="read a nixos configuration option", - epilog=( - """ - """ - ), - formatter_class=argparse.RawTextHelpFormatter, - ) - config.register_parser(parser_config) - parser_ssh = subparsers.add_parser( "ssh", help="ssh to a remote machine", @@ -408,8 +394,6 @@ def main() -> None: if getattr(args, "debug", False): setup_logging(logging.DEBUG, root_log_name=__name__.split(".")[0]) log.debug("Debug log activated") - if flatpak.is_flatpak(): - log.debug("Running inside a flatpak sandbox") else: setup_logging(logging.INFO, root_log_name=__name__.split(".")[0]) diff --git a/pkgs/clan-cli/clan_cli/api/cli.py b/pkgs/clan-cli/clan_cli/api/cli.py new file mode 100755 index 000000000..763d5ec24 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/api/cli.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +import argparse +import json + +from clan_cli.api import API + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Debug the API.") + args = parser.parse_args() + + schema = API.to_json_schema() + print(json.dumps(schema, indent=4)) diff --git a/pkgs/clan-cli/clan_cli/api/directory.py b/pkgs/clan-cli/clan_cli/api/directory.py index c7f95b87f..895749709 100644 --- a/pkgs/clan-cli/clan_cli/api/directory.py +++ b/pkgs/clan-cli/clan_cli/api/directory.py @@ -12,24 +12,26 @@ from . import API @dataclass class FileFilter: - title: str | None - mime_types: list[str] | None - patterns: list[str] | None - suffixes: list[str] | None + title: str | None = field(default=None) + mime_types: list[str] | None = field(default=None) + patterns: list[str] | None = field(default=None) + suffixes: list[str] | None = field(default=None) @dataclass class FileRequest: # Mode of the os dialog window - mode: Literal["open_file", "select_folder", "save"] + mode: Literal["open_file", "select_folder", "save", "open_multiple_files"] # Title of the os dialog window - title: str | None = None + title: str | None = field(default=None) # Pre-applied filters for the file dialog - filters: FileFilter | None = None + filters: FileFilter | None = field(default=None) + initial_file: str | None = field(default=None) + initial_folder: str | None = field(default=None) @API.register_abstract -def open_file(file_request: FileRequest) -> str | None: +def open_file(file_request: FileRequest) -> list[str] | None: """ Abstract api method to open a file dialog window. It must return the name of the selected file or None if no file was selected. @@ -88,6 +90,8 @@ def get_directory(current_path: str) -> Directory: @dataclass class BlkInfo: name: str + id_link: str + path: str rm: str size: str ro: bool @@ -103,21 +107,53 @@ class Blockdevices: def blk_from_dict(data: dict) -> BlkInfo: return BlkInfo( name=data["name"], + path=data["path"], rm=data["rm"], size=data["size"], ro=data["ro"], mountpoints=data["mountpoints"], - type_=data["type"], # renamed here + type_=data["type"], # renamed + id_link=data["id-link"], # renamed ) +@dataclass +class BlockDeviceOptions: + hostname: str | None = None + keyfile: str | None = None + + @API.register -def show_block_devices() -> Blockdevices: +def show_block_devices(options: BlockDeviceOptions) -> Blockdevices: """ Abstract api method to show block devices. It must return a list of block devices. """ - cmd = nix_shell(["nixpkgs#util-linux"], ["lsblk", "--json"]) + keyfile = options.keyfile + remote = ( + [ + "ssh", + *(["-i", f"{keyfile}"] if keyfile else []), + # Disable strict host key checking + "-o StrictHostKeyChecking=no", + # Disable known hosts file + "-o UserKnownHostsFile=/dev/null", + f"{options.hostname}", + ] + if options.hostname + else [] + ) + + cmd = nix_shell( + ["nixpkgs#util-linux", *(["nixpkgs#openssh"] if options.hostname else [])], + [ + *remote, + "lsblk", + "--json", + "--output", + "PATH,NAME,RM,SIZE,RO,MOUNTPOINTS,TYPE,ID-LINK", + ], + ) proc = run_no_stdout(cmd) res = proc.stdout.strip() diff --git a/pkgs/clan-cli/clan_cli/api/disk.py b/pkgs/clan-cli/clan_cli/api/disk.py new file mode 100644 index 000000000..63b50f846 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/api/disk.py @@ -0,0 +1,65 @@ +from clan_cli.api import API +from clan_cli.inventory import ( + ServiceMeta, + ServiceSingleDisk, + ServiceSingleDiskRole, + ServiceSingleDiskRoleDefault, + SingleDiskConfig, + load_inventory_eval, + load_inventory_json, + save_inventory, +) + + +def get_instance_name(machine_name: str) -> str: + return f"{machine_name}-single-disk" + + +@API.register +def set_single_disk_uuid( + base_path: str, + machine_name: str, + disk_uuid: str, +) -> None: + """ + Set the disk UUID of single disk machine + """ + inventory = load_inventory_json(base_path) + + instance_name = get_instance_name(machine_name) + + single_disk_config: ServiceSingleDisk = ServiceSingleDisk( + meta=ServiceMeta(name=instance_name), + roles=ServiceSingleDiskRole( + default=ServiceSingleDiskRoleDefault( + config=SingleDiskConfig(device=disk_uuid) + ) + ), + ) + + inventory.services.single_disk[instance_name] = single_disk_config + + save_inventory( + inventory, + base_path, + f"Set disk UUID: '{disk_uuid}' on machine: '{machine_name}'", + ) + + +@API.register +def get_single_disk_uuid( + base_path: str, + machine_name: str, +) -> str | None: + """ + Get the disk UUID of single disk machine + """ + inventory = load_inventory_eval(base_path) + + instance_name = get_instance_name(machine_name) + + single_disk_config: ServiceSingleDisk = inventory.services.single_disk[ + instance_name + ] + + return single_disk_config.roles.default.config.device diff --git a/pkgs/clan-cli/clan_cli/api/modules.py b/pkgs/clan-cli/clan_cli/api/modules.py index 290689bf4..78b6beb63 100644 --- a/pkgs/clan-cli/clan_cli/api/modules.py +++ b/pkgs/clan-cli/clan_cli/api/modules.py @@ -3,13 +3,16 @@ import re import tomllib from dataclasses import dataclass from pathlib import Path +from typing import Any, get_args, get_type_hints from clan_cli.cmd import run_no_stdout from clan_cli.errors import ClanCmdError, ClanError -from clan_cli.inventory import Inventory, load_inventory_json +from clan_cli.inventory import Inventory, load_inventory_json, save_inventory +from clan_cli.inventory.classes import Service from clan_cli.nix import nix_eval from . import API +from .serde import from_dict @dataclass @@ -153,3 +156,35 @@ def get_module_info( @API.register def get_inventory(base_path: str) -> Inventory: return load_inventory_json(base_path) + + +@API.register +def set_service_instance( + base_path: str, module_name: str, instance_name: str, config: dict[str, Any] +) -> None: + """ + A function that allows to set any service instance in the inventory. + Takes any untyped dict. The dict is then checked and converted into the correct type using the type hints of the service. + If any conversion error occurs, the function will raise an error. + """ + service_keys = get_type_hints(Service).keys() + + if module_name not in service_keys: + raise ValueError( + f"{module_name} is not a valid Service attribute. Expected one of {', '.join(service_keys)}." + ) + + inventory = load_inventory_json(base_path) + target_type = get_args(get_type_hints(Service)[module_name])[1] + + module_instance_map: dict[str, Any] = getattr(inventory.services, module_name, {}) + + module_instance_map[instance_name] = from_dict(target_type, config) + + setattr(inventory.services, module_name, module_instance_map) + + save_inventory( + inventory, base_path, f"Update {module_name} instance {instance_name}" + ) + + # TODO: Add a check that rolls back the inventory if the service config is not valid or causes conflicts. diff --git a/pkgs/clan-cli/clan_cli/api/serde.py b/pkgs/clan-cli/clan_cli/api/serde.py index 57345c3db..a3ff265d5 100644 --- a/pkgs/clan-cli/clan_cli/api/serde.py +++ b/pkgs/clan-cli/clan_cli/api/serde.py @@ -29,17 +29,21 @@ Dependencies: Note: This module assumes the presence of other modules and classes such as `ClanError` and `ErrorDetails` from the `clan_cli.errors` module. """ +import dataclasses import json from dataclasses import dataclass, fields, is_dataclass from pathlib import Path +from types import UnionType from typing import ( + Annotated, Any, + Literal, TypeVar, + Union, + get_args, + get_origin, ) -from pydantic import TypeAdapter, ValidationError -from pydantic_core import ErrorDetails - from clan_cli.errors import ClanError @@ -64,7 +68,8 @@ def dataclass_to_dict(obj: Any, *, use_alias: bool = True) -> Any: field.metadata.get("alias", field.name) if use_alias else field.name ): _to_dict(getattr(obj, field.name)) for field in fields(obj) - if not field.name.startswith("_") # type: ignore + if not field.name.startswith("_") + and getattr(obj, field.name) is not None # type: ignore } elif isinstance(obj, list | tuple): return [_to_dict(item) for item in obj] @@ -81,26 +86,169 @@ def dataclass_to_dict(obj: Any, *, use_alias: bool = True) -> Any: T = TypeVar("T", bound=dataclass) # type: ignore +G = TypeVar("G") # type: ignore -def from_dict(t: type[T], data: Any) -> T: +def is_union_type(type_hint: type | UnionType) -> bool: + return ( + type(type_hint) is UnionType + or isinstance(type_hint, UnionType) + or get_origin(type_hint) is Union + ) + + +def is_type_in_union(union_type: type | UnionType, target_type: type) -> bool: + if get_origin(union_type) is UnionType: + return any(issubclass(arg, target_type) for arg in get_args(union_type)) + return union_type == target_type + + +def unwrap_none_type(type_hint: type | UnionType) -> type: """ + Takes a type union and returns the first non-None type. + None | str + => + str + """ + + if is_union_type(type_hint): + # Return the first non-None type + return next(t for t in get_args(type_hint) if t is not type(None)) + + return type_hint # type: ignore + + +JsonValue = str | float | dict[str, Any] | list[Any] | None + + +def construct_value(t: type, field_value: JsonValue, loc: list[str] = []) -> Any: + """ + Construct a field value from a type hint and a field value. + """ + if t is None and field_value: + raise ClanError(f"Expected None but got: {field_value}", location=f"{loc}") + # If the field is another dataclass + # Field_value must be a dictionary + if is_dataclass(t) and isinstance(field_value, dict): + return construct_dataclass(t, field_value) + + # If the field expects a path + # Field_value must be a string + elif is_type_in_union(t, Path): + if not isinstance(field_value, str): + raise ClanError( + f"Expected string, cannot construct pathlib.Path() from: {field_value} ", + location=f"{loc}", + ) + + return Path(field_value) + + # Trivial values + elif t is str: + if not isinstance(field_value, str): + raise ClanError(f"Expected string, got {field_value}", location=f"{loc}") + + return field_value + + elif t is int and not isinstance(field_value, str): + return int(field_value) # type: ignore + elif t is float and not isinstance(field_value, str): + return float(field_value) # type: ignore + elif t is bool and isinstance(field_value, bool): + return field_value # type: ignore + + # Union types construct the first non-None type + elif is_union_type(t): + # Unwrap the union type + t = unwrap_none_type(t) + # Construct the field value + return construct_value(t, field_value) + + # Nested types + # list + # dict + elif get_origin(t) is list: + if not isinstance(field_value, list): + raise ClanError(f"Expected list, got {field_value}", location=f"{loc}") + + return [construct_value(get_args(t)[0], item) for item in field_value] + elif get_origin(t) is dict and isinstance(field_value, dict): + return { + key: construct_value(get_args(t)[1], value) + for key, value in field_value.items() + } + elif get_origin(t) is Literal: + valid_values = get_args(t) + if field_value not in valid_values: + raise ClanError( + f"Expected one of {valid_values}, got {field_value}", location=f"{loc}" + ) + return field_value + + elif get_origin(t) is Annotated: + (base_type,) = get_args(t) + return construct_value(base_type, field_value) + + # elif get_origin(t) is Union: + + # Unhandled + else: + raise ClanError(f"Unhandled field type {t} with value {field_value}") + + +def construct_dataclass(t: type[T], data: dict[str, Any], path: list[str] = []) -> T: + """ + type t MUST be a dataclass Dynamically instantiate a data class from a dictionary, handling nested data classes. - We use dataclasses. But the deserialization logic of pydantic takes a lot of complexity. """ - adapter = TypeAdapter(t) - try: - return adapter.validate_python( - data, - ) - except ValidationError as e: - fst_error: ErrorDetails = e.errors()[0] - if not fst_error: - raise ClanError(msg=str(e)) + if not is_dataclass(t): + raise ClanError(f"{t.__name__} is not a dataclass") - msg = fst_error.get("msg") - loc = fst_error.get("loc") - field_path = "Unknown" - if loc: - field_path = str(loc) - raise ClanError(msg=msg, location=f"{t!s}: {field_path}", description=str(e)) + # Attempt to create an instance of the data_class# + field_values: dict[str, Any] = {} + required: list[str] = [] + + for field in fields(t): + if field.name.startswith("_"): + continue + # The first type in a Union + # str <- None | str | Path + field_type: type[Any] = unwrap_none_type(field.type) # type: ignore + data_field_name = field.metadata.get("alias", field.name) + + if ( + field.default is dataclasses.MISSING + and field.default_factory is dataclasses.MISSING + ): + required.append(field.name) + + # Populate the field_values dictionary with the field value + # if present in the data + if data_field_name in data: + field_value = data.get(data_field_name) + + if field_value is None and ( + field.type is None or is_type_in_union(field.type, type(None)) + ): + field_values[field.name] = None + else: + field_values[field.name] = construct_value(field_type, field_value) + + # Check that all required field are present. + for field_name in required: + if field_name not in field_values: + formatted_path = " ".join(path) + raise ClanError( + f"Default value missing for: '{field_name}' in {t} {formatted_path}, got Value: {data}" + ) + + return t(**field_values) # type: ignore + + +def from_dict(t: type[G], data: dict[str, Any] | Any, path: list[str] = []) -> G: + if is_dataclass(t): + if not isinstance(data, dict): + raise ClanError(f"{data} is not a dict. Expected {t}") + return construct_dataclass(t, data, path) # type: ignore + else: + return construct_value(t, data, path) diff --git a/pkgs/clan-cli/clan_cli/clan/inspect.py b/pkgs/clan-cli/clan_cli/clan/inspect.py index 0c5b28636..b82ded161 100644 --- a/pkgs/clan-cli/clan_cli/clan/inspect.py +++ b/pkgs/clan-cli/clan_cli/clan/inspect.py @@ -6,7 +6,7 @@ from ..clan_uri import FlakeId from ..cmd import run from ..dirs import machine_gcroot from ..errors import ClanError -from ..machines.list import list_machines +from ..machines.list import list_nixos_machines from ..machines.machines import Machine from ..nix import nix_add_to_gcroots, nix_build, nix_config, nix_eval, nix_metadata from ..vms.inspect import VmConfig, inspect_vm @@ -40,7 +40,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig: system = config["system"] # Check if the machine exists - machines = list_machines(flake_url, False) + machines: list[str] = list_nixos_machines(flake_url) if machine_name not in machines: raise ClanError( f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}" @@ -57,7 +57,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig: # Get the Clan name cmd = nix_eval( [ - f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clan.core.clanName' + f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clan.core.name' ] ) res = run_cmd(cmd) @@ -66,7 +66,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig: # Get the clan icon path cmd = nix_eval( [ - f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clan.core.clanIcon' + f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clan.core.icon' ] ) res = run_cmd(cmd) @@ -79,9 +79,9 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig: cmd = nix_build( [ - f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clan.core.clanIcon' + f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clan.core.icon' ], - machine_gcroot(flake_url=str(flake_url)) / "clanIcon", + machine_gcroot(flake_url=str(flake_url)) / "icon", ) run_cmd(cmd) diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index 1085bc314..06cd10d57 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -4,12 +4,10 @@ import json import logging import os import re -import sys from pathlib import Path from typing import Any, get_origin from clan_cli.cmd import run -from clan_cli.completions import add_dynamic_completer, complete_machines from clan_cli.dirs import machine_settings_file from clan_cli.errors import ClanError from clan_cli.git import commit_file @@ -305,65 +303,3 @@ def set_option( repo_dir=flake_dir, commit_message=f"Set option {option_description}", ) - - -# takes a (sub)parser and configures it -def register_parser( - parser: argparse.ArgumentParser | None, -) -> None: - if parser is None: - parser = argparse.ArgumentParser( - description="Set or show NixOS options", - ) - - # inject callback function to process the input later - parser.set_defaults(func=get_option) - set_machine_action = parser.add_argument( - "--machine", - "-m", - help="Machine to configure", - type=str, - default="default", - ) - add_dynamic_completer(set_machine_action, complete_machines) - - parser.add_argument( - "--show-trace", - help="Show nix trace on evaluation error", - action="store_true", - ) - - parser.add_argument( - "--options-file", - help="JSON file with options", - type=Path, - ) - - parser.add_argument( - "--settings-file", - help="JSON file with settings", - type=Path, - ) - parser.add_argument( - "--quiet", - help="Do not print the value", - action="store_true", - ) - - parser.add_argument( - "option", - help="Option to read or set (e.g. foo.bar)", - type=str, - ) - - -def main(argv: list[str] | None = None) -> None: - if argv is None: - argv = sys.argv - parser = argparse.ArgumentParser() - register_parser(parser) - parser.parse_args(argv[1:]) - - -if __name__ == "__main__": - main() diff --git a/pkgs/clan-cli/clan_cli/config/machine.py b/pkgs/clan-cli/clan_cli/config/machine.py index 9155918c2..e52845118 100644 --- a/pkgs/clan-cli/clan_cli/config/machine.py +++ b/pkgs/clan-cli/clan_cli/config/machine.py @@ -18,7 +18,7 @@ def verify_machine_config( ) -> str | None: """ Verify that the machine evaluates successfully - Returns a tuple of (success, error_message) + Returns None, in case of success, or a String containing the error_message """ if config is None: config = config_for_machine(flake_dir, machine_name) diff --git a/pkgs/clan-cli/clan_cli/config/schema.py b/pkgs/clan-cli/clan_cli/config/schema.py index bebce02c7..351130908 100644 --- a/pkgs/clan-cli/clan_cli/config/schema.py +++ b/pkgs/clan-cli/clan_cli/config/schema.py @@ -11,6 +11,8 @@ from clan_cli.errors import ClanError, ClanHttpError from clan_cli.nix import nix_eval +# TODO: When moving the api to `clan-app`, the whole config module should be +# ported to the `clan-app`, because it is not used by the cli at all. @API.register def machine_schema( flake_dir: Path, @@ -86,9 +88,9 @@ def machine_schema( [ clan-core.nixosModules.clanCore # potentially the config might affect submodule options, - # therefore we need to import it + # therefore we need to import it config - {{ clan.core.clanName = "fakeClan"; }} + {{ clan.core.name = "fakeClan"; }} ] # add all clan modules specified via clanImports ++ (map (name: clan-core.clanModules.${{name}}) config.clanImports or []); diff --git a/pkgs/clan-cli/clan_cli/facts/generate.py b/pkgs/clan-cli/clan_cli/facts/generate.py index 21b80ee31..ed6a941d4 100644 --- a/pkgs/clan-cli/clan_cli/facts/generate.py +++ b/pkgs/clan-cli/clan_cli/facts/generate.py @@ -105,14 +105,8 @@ def generate_service_facts( ) files_to_commit = [] # store secrets - for secret in machine.facts_data[service]["secret"]: - if isinstance(secret, str): - # TODO: This is the old NixOS module, can be dropped everyone has updated. - secret_name = secret - groups = [] - else: - secret_name = secret["name"] - groups = secret.get("groups", []) + for secret_name, secret in machine.facts_data[service]["secret"].items(): + groups = secret.get("groups", []) secret_file = secrets_dir / secret_name if not secret_file.is_file(): diff --git a/pkgs/clan-cli/clan_cli/facts/upload.py b/pkgs/clan-cli/clan_cli/facts/upload.py index 6cb6bf8be..1df5fb194 100644 --- a/pkgs/clan-cli/clan_cli/facts/upload.py +++ b/pkgs/clan-cli/clan_cli/facts/upload.py @@ -31,9 +31,11 @@ def upload_secrets(machine: Machine) -> None: "rsync", "-e", " ".join(["ssh"] + ssh_cmd[2:]), - "-az", + "--recursive", + "--links", + "--times", + "--compress", "--delete", - "--chown=root:root", "--chmod=D700,F600", f"{tempdir!s}/", f"{host.user}@{host.host}:{machine.secrets_upload_directory}/", diff --git a/pkgs/clan-cli/clan_cli/flash.py b/pkgs/clan-cli/clan_cli/flash.py index f846068a5..4850b36cc 100644 --- a/pkgs/clan-cli/clan_cli/flash.py +++ b/pkgs/clan-cli/clan_cli/flash.py @@ -6,7 +6,7 @@ import os import shutil import textwrap from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -19,23 +19,118 @@ from .completions import add_dynamic_completer, complete_machines from .errors import ClanError from .facts.secret_modules import SecretStoreBase from .machines.machines import Machine -from .nix import nix_shell +from .nix import nix_build, nix_shell log = logging.getLogger(__name__) +@dataclass +class WifiConfig: + ssid: str + password: str + + +@dataclass +class SystemConfig: + language: str | None = field(default=None) + keymap: str | None = field(default=None) + ssh_keys_path: list[str] | None = field(default=None) + wifi_settings: list[WifiConfig] | None = field(default=None) + + +@API.register +def list_possible_keymaps() -> list[str]: + cmd = nix_build(["nixpkgs#kbd"]) + result = run(cmd, log=Log.STDERR, error_msg="Failed to find kbdinfo") + keymaps_dir = Path(result.stdout.strip()) / "share" / "keymaps" + + if not keymaps_dir.exists(): + raise FileNotFoundError(f"Keymaps directory '{keymaps_dir}' does not exist.") + + keymap_files = [] + + for root, _, files in os.walk(keymaps_dir): + for file in files: + if file.endswith(".map.gz"): + # Remove '.map.gz' ending + name_without_ext = file[:-7] + keymap_files.append(name_without_ext) + + return keymap_files + + +@API.register +def list_possible_languages() -> list[str]: + cmd = nix_build(["nixpkgs#glibcLocales"]) + result = run(cmd, log=Log.STDERR, error_msg="Failed to find glibc locales") + locale_file = Path(result.stdout.strip()) / "share" / "i18n" / "SUPPORTED" + + if not locale_file.exists(): + raise FileNotFoundError(f"Locale file '{locale_file}' does not exist.") + + with locale_file.open() as f: + lines = f.readlines() + + languages = [] + for line in lines: + if line.startswith("#"): + continue + if "SUPPORTED-LOCALES" in line: + continue + # Split by '/' and take the first part + language = line.split("/")[0].strip() + languages.append(language) + + return languages + + @API.register def flash_machine( machine: Machine, *, mode: str, disks: dict[str, str], - system_config: dict[str, Any], + system_config: SystemConfig, dry_run: bool, write_efi_boot_entries: bool, debug: bool, extra_args: list[str] = [], ) -> None: + system_config_nix: dict[str, Any] = {} + + if system_config.wifi_settings: + wifi_settings = {} + for wifi in system_config.wifi_settings: + wifi_settings[wifi.ssid] = {"password": wifi.password} + system_config_nix["clan"] = {"iwd": {"networks": wifi_settings}} + + if system_config.language: + if system_config.language not in list_possible_languages(): + raise ClanError( + f"Language '{system_config.language}' is not a valid language. " + f"Run 'clan flash --list-languages' to see a list of possible languages." + ) + system_config_nix["i18n"] = {"defaultLocale": system_config.language} + + if system_config.keymap: + if system_config.keymap not in list_possible_keymaps(): + raise ClanError( + f"Keymap '{system_config.keymap}' is not a valid keymap. " + f"Run 'clan flash --list-keymaps' to see a list of possible keymaps." + ) + system_config_nix["console"] = {"keyMap": system_config.keymap} + + if system_config.ssh_keys_path: + root_keys = [] + for key_path in map(lambda x: Path(x), system_config.ssh_keys_path): + try: + root_keys.append(key_path.read_text()) + except OSError as e: + raise ClanError(f"Cannot read SSH public key file: {key_path}: {e}") + system_config_nix["users"] = { + "users": {"root": {"openssh": {"authorizedKeys": {"keys": root_keys}}}} + } + secret_facts_module = importlib.import_module(machine.secret_facts_module) secret_facts_store: SecretStoreBase = secret_facts_module.SecretStore( machine=machine @@ -58,7 +153,8 @@ def flash_machine( raise ClanError( "sudo is required to run disko-install as a non-root user" ) - disko_install.append("sudo") + wrapper = 'set -x; disko_install=$(command -v disko-install); exec sudo "$disko_install" "$@"' + disko_install.extend(["bash", "-c", wrapper]) disko_install.append("disko-install") if write_efi_boot_entries: @@ -76,7 +172,7 @@ def flash_machine( disko_install.extend( [ "--system-config", - json.dumps(system_config), + json.dumps(system_config_nix), ] ) disko_install.extend(["--option", "dry-run", "true"]) @@ -94,15 +190,13 @@ class FlashOptions: flake: FlakeId machine: str disks: dict[str, str] - ssh_keys_path: list[Path] dry_run: bool confirm: bool debug: bool mode: str - language: str - keymap: str write_efi_boot_entries: bool nix_options: list[str] + system_config: SystemConfig class AppendDiskAction(argparse.Action): @@ -126,17 +220,36 @@ def flash_command(args: argparse.Namespace) -> None: flake=args.flake, machine=args.machine, disks=args.disk, - ssh_keys_path=args.ssh_pubkey, dry_run=args.dry_run, confirm=not args.yes, debug=args.debug, mode=args.mode, - language=args.language, - keymap=args.keymap, + system_config=SystemConfig( + language=args.language, + keymap=args.keymap, + ssh_keys_path=args.ssh_pubkey, + wifi_settings=None, + ), write_efi_boot_entries=args.write_efi_boot_entries, nix_options=args.option, ) + if args.list_languages: + for language in list_possible_languages(): + print(language) + return + + if args.list_keymaps: + for keymap in list_possible_keymaps(): + print(keymap) + return + + if args.wifi: + opts.system_config.wifi_settings = [ + WifiConfig(ssid=ssid, password=password) + for ssid, password in args.wifi.items() + ] + machine = Machine(opts.machine, flake=opts.flake) if opts.confirm and not opts.dry_run: disk_str = ", ".join(f"{name}={device}" for name, device in opts.disks.items()) @@ -148,28 +261,11 @@ def flash_command(args: argparse.Namespace) -> None: if ask != "y": return - extra_config: dict[str, Any] = {} - if opts.ssh_keys_path: - root_keys = [] - for key_path in opts.ssh_keys_path: - try: - root_keys.append(key_path.read_text()) - except OSError as e: - raise ClanError(f"Cannot read SSH public key file: {key_path}: {e}") - extra_config["users"] = { - "users": {"root": {"openssh": {"authorizedKeys": {"keys": root_keys}}}} - } - if opts.keymap: - extra_config["console"] = {"keyMap": opts.keymap} - - if opts.language: - extra_config["i18n"] = {"defaultLocale": opts.language} - flash_machine( machine, mode=opts.mode, disks=opts.disks, - system_config=extra_config, + system_config=opts.system_config, dry_run=opts.dry_run, debug=opts.debug, write_efi_boot_entries=opts.write_efi_boot_entries, @@ -202,6 +298,15 @@ def register_parser(parser: argparse.ArgumentParser) -> None: Mount is useful for updating an existing system without losing data. """ ) + parser.add_argument( + "--wifi", + type=str, + nargs=2, + metavar=("ssid", "password"), + action=AppendDiskAction, + help="wifi network to connect to", + default={}, + ) parser.add_argument( "--mode", type=str, @@ -221,6 +326,18 @@ def register_parser(parser: argparse.ArgumentParser) -> None: type=str, help="system language", ) + parser.add_argument( + "--list-languages", + help="List possible languages", + default=False, + action="store_true", + ) + parser.add_argument( + "--list-keymaps", + help="List possible keymaps", + default=False, + action="store_true", + ) parser.add_argument( "--keymap", type=str, diff --git a/pkgs/clan-cli/clan_cli/flatpak.py b/pkgs/clan-cli/clan_cli/flatpak.py deleted file mode 100644 index d9a563fa8..000000000 --- a/pkgs/clan-cli/clan_cli/flatpak.py +++ /dev/null @@ -1,16 +0,0 @@ -import os - - -def is_flatpak() -> bool: - """Check if the current process is running inside a flatpak sandbox.""" - # FLATPAK_ID environment variable check - flatpak_env = "FLATPAK_ID" in os.environ - - flatpak_file = False - try: - with open("/.flatpak-info"): - flatpak_file = True - except FileNotFoundError: - pass - - return flatpak_env and flatpak_file diff --git a/pkgs/clan-cli/clan_cli/history/add.py b/pkgs/clan-cli/clan_cli/history/add.py index e13938e86..93646be30 100644 --- a/pkgs/clan-cli/clan_cli/history/add.py +++ b/pkgs/clan-cli/clan_cli/history/add.py @@ -7,7 +7,7 @@ import logging from typing import Any from clan_cli.clan.inspect import FlakeConfig, inspect_flake -from clan_cli.machines.list import list_machines +from clan_cli.machines.list import list_nixos_machines from ..clan_uri import ClanURI from ..dirs import user_history_file @@ -72,7 +72,7 @@ def new_history_entry(url: str, machine: str) -> HistoryEntry: def add_all_to_history(uri: ClanURI) -> list[HistoryEntry]: history = list_history() new_entries: list[HistoryEntry] = [] - for machine in list_machines(uri.get_url()): + for machine in list_nixos_machines(uri.get_url()): new_entry = _add_maschine_to_history_list(uri.get_url(), machine, history) new_entries.append(new_entry) write_history_file(history) diff --git a/pkgs/clan-cli/clan_cli/inventory/__init__.py b/pkgs/clan-cli/clan_cli/inventory/__init__.py index acf680b75..3aabe6686 100644 --- a/pkgs/clan-cli/clan_cli/inventory/__init__.py +++ b/pkgs/clan-cli/clan_cli/inventory/__init__.py @@ -32,6 +32,10 @@ from .classes import ( ServiceBorgbackupRoleClient, ServiceBorgbackupRoleServer, ServiceMeta, + ServiceSingleDisk, + ServiceSingleDiskRole, + ServiceSingleDiskRoleDefault, + SingleDiskConfig, ) # Re export classes here @@ -49,6 +53,11 @@ __all__ = [ "ServiceBorgbackupRole", "ServiceBorgbackupRoleClient", "ServiceBorgbackupRoleServer", + # Single Disk service + "ServiceSingleDisk", + "ServiceSingleDiskRole", + "ServiceSingleDiskRoleDefault", + "SingleDiskConfig", ] @@ -82,6 +91,7 @@ def load_inventory_eval(flake_dir: str | Path) -> Inventory: "--json", ] ) + proc = run_no_stdout(cmd) try: diff --git a/pkgs/clan-cli/clan_cli/machines/__init__.py b/pkgs/clan-cli/clan_cli/machines/__init__.py index bad8706fb..c6b0631e1 100644 --- a/pkgs/clan-cli/clan_cli/machines/__init__.py +++ b/pkgs/clan-cli/clan_cli/machines/__init__.py @@ -6,7 +6,6 @@ from .delete import register_delete_parser from .hardware import register_hw_generate from .install import register_install_parser from .list import register_list_parser -from .show import register_show_parser from .update import register_update_parser @@ -86,17 +85,6 @@ For more detailed information, visit: https://docs.clan.lol/getting-started/conf ) register_hw_generate(generate_hw_parser) - show_parser = subparser.add_parser( - "show", - help="Show a machine", - epilog=( - """ -This subcommand shows the details of a machine managed by this clan like icon, description, etc -""" - ), - ) - register_show_parser(show_parser) - install_parser = subparser.add_parser( "install", help="Install a machine", diff --git a/pkgs/clan-cli/clan_cli/machines/create.py b/pkgs/clan-cli/clan_cli/machines/create.py index cea2412a7..f5b858602 100644 --- a/pkgs/clan-cli/clan_cli/machines/create.py +++ b/pkgs/clan-cli/clan_cli/machines/create.py @@ -31,6 +31,8 @@ def create_machine(flake: FlakeId, machine: Machine) -> None: if machine.name in full_inventory.machines.keys(): raise ClanError(f"Machine with the name {machine.name} already exists") + print(f"Define machine {machine.name}", machine) + inventory.machines.update({machine.name: machine}) save_inventory(inventory, flake.path, f"Create machine {machine.name}") diff --git a/pkgs/clan-cli/clan_cli/machines/hardware.py b/pkgs/clan-cli/clan_cli/machines/hardware.py index 4d9fe3393..c9f47272b 100644 --- a/pkgs/clan-cli/clan_cli/machines/hardware.py +++ b/pkgs/clan-cli/clan_cli/machines/hardware.py @@ -94,6 +94,7 @@ def generate_machine_hardware_info( machine_name: str, hostname: str | None = None, password: str | None = None, + keyfile: str | None = None, force: bool | None = False, ) -> HardwareInfo: """ @@ -117,15 +118,15 @@ def generate_machine_hardware_info( [ *(["sshpass", "-p", f"{password}"] if password else []), "ssh", - # Disable strict host key checking - "-o", - "StrictHostKeyChecking=no", + *(["-i", f"{keyfile}"] if keyfile else []), # Disable known hosts file "-o", "UserKnownHostsFile=/dev/null", "-p", str(machine.target_host.port), target_host, + "-o UserKnownHostsFile=/dev/null", + f"{hostname}", "nixos-generate-config", # Filesystems are managed by disko "--no-filesystems", diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 5d7a3cae3..17ea04152 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -3,10 +3,12 @@ import importlib import json import logging import os -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from tempfile import TemporaryDirectory +from clan_cli.api import API + from ..clan_uri import FlakeId from ..cmd import Log, run from ..completions import add_dynamic_completer, complete_machines @@ -91,15 +93,30 @@ def install_nixos( @dataclass class InstallOptions: + # flake to install flake: FlakeId machine: str target_host: str - kexec: str | None - confirm: bool - debug: bool - no_reboot: bool - json_ssh_deploy: dict[str, str] | None - nix_options: list[str] + kexec: str | None = None + debug: bool = False + no_reboot: bool = False + json_ssh_deploy: dict[str, str] | None = None + nix_options: list[str] = field(default_factory=list) + + +@API.register +def install_machine(opts: InstallOptions, password: str | None) -> None: + machine = Machine(opts.machine, flake=opts.flake) + machine.target_host_address = opts.target_host + + install_nixos( + machine, + kexec=opts.kexec, + debug=opts.debug, + password=password, + no_reboot=opts.no_reboot, + extra_args=opts.nix_options, + ) def install_command(args: argparse.Namespace) -> None: @@ -123,32 +140,23 @@ def install_command(args: argparse.Namespace) -> None: target_host = args.target_host password = None - opts = InstallOptions( - flake=args.flake, - machine=args.machine, - target_host=target_host, - kexec=args.kexec, - confirm=not args.yes, - debug=args.debug, - no_reboot=args.no_reboot, - json_ssh_deploy=json_ssh_deploy, - nix_options=args.option, - ) - machine = Machine(opts.machine, flake=opts.flake) - machine.target_host_address = opts.target_host - - if opts.confirm: - ask = input(f"Install {machine.name} to {opts.target_host}? [y/N] ") + if not args.yes: + ask = input(f"Install {args.machine} to {target_host}? [y/N] ") if ask != "y": return - install_nixos( - machine, - kexec=opts.kexec, - debug=opts.debug, - password=password, - no_reboot=opts.no_reboot, - extra_args=opts.nix_options, + return install_machine( + InstallOptions( + flake=args.flake, + machine=args.machine, + target_host=target_host, + kexec=args.kexec, + debug=args.debug, + no_reboot=args.no_reboot, + json_ssh_deploy=json_ssh_deploy, + nix_options=args.option, + ), + password, ) diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index f8df311d5..bc754d661 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -1,22 +1,121 @@ import argparse +import json import logging +from dataclasses import dataclass from pathlib import Path +from typing import Literal from clan_cli.api import API +from clan_cli.cmd import run_no_stdout +from clan_cli.errors import ClanCmdError, ClanError from clan_cli.inventory import Machine, load_inventory_eval +from clan_cli.nix import nix_eval, nix_shell log = logging.getLogger(__name__) @API.register -def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]: +def list_inventory_machines(flake_url: str | Path) -> dict[str, Machine]: inventory = load_inventory_eval(flake_url) return inventory.machines +@dataclass +class MachineDetails: + machine: Machine + has_hw_specs: bool = False + # TODO: + # has_disk_specs: bool = False + + +@API.register +def get_inventory_machine_details( + flake_url: str | Path, machine_name: str +) -> MachineDetails: + inventory = load_inventory_eval(flake_url) + machine = inventory.machines.get(machine_name) + if machine is None: + raise ClanError(f"Machine {machine_name} not found in inventory") + + hw_config_path = ( + Path(flake_url) / "machines" / Path(machine_name) / "hardware-configuration.nix" + ) + + return MachineDetails( + machine=machine, + has_hw_specs=hw_config_path.exists(), + ) + + +@API.register +def list_nixos_machines(flake_url: str | Path) -> list[str]: + cmd = nix_eval( + [ + f"{flake_url}#nixosConfigurations", + "--apply", + "builtins.attrNames", + "--json", + ] + ) + proc = run_no_stdout(cmd) + + try: + res = proc.stdout.strip() + data = json.loads(res) + return data + except json.JSONDecodeError as e: + raise ClanError(f"Error decoding machines from flake: {e}") + + +@dataclass +class ConnectionOptions: + keyfile: str | None = None + timeout: int = 2 + + +@API.register +def check_machine_online( + flake_url: str | Path, machine_name: str, opts: ConnectionOptions | None +) -> Literal["Online", "Offline"]: + machine = load_inventory_eval(flake_url).machines.get(machine_name) + if not machine: + raise ClanError(f"Machine {machine_name} not found in inventory") + + hostname = machine.deploy.targetHost + + if not hostname: + raise ClanError(f"Machine {machine_name} does not specify a targetHost") + + timeout = opts.timeout if opts and opts.timeout else 2 + + cmd = nix_shell( + ["nixpkgs#util-linux", *(["nixpkgs#openssh"] if hostname else [])], + [ + "ssh", + *(["-i", f"{opts.keyfile}"] if opts and opts.keyfile else []), + # Disable strict host key checking + "-o StrictHostKeyChecking=no", + # Disable known hosts file + "-o UserKnownHostsFile=/dev/null", + f"-o ConnectTimeout={timeout}", + f"{hostname}", + "true", + "&> /dev/null", + ], + ) + try: + proc = run_no_stdout(cmd) + if proc.returncode != 0: + return "Offline" + + return "Online" + except ClanCmdError: + return "Offline" + + def list_command(args: argparse.Namespace) -> None: flake_path = args.flake.path - for name in list_machines(flake_path, args.debug).keys(): + for name in list_nixos_machines(flake_path): print(name) diff --git a/pkgs/clan-cli/clan_cli/machines/show.py b/pkgs/clan-cli/clan_cli/machines/show.py deleted file mode 100644 index 29284b034..000000000 --- a/pkgs/clan-cli/clan_cli/machines/show.py +++ /dev/null @@ -1,59 +0,0 @@ -import argparse -import dataclasses -import json -import logging -from pathlib import Path - -from clan_cli.api import API - -from ..cmd import run_no_stdout -from ..completions import add_dynamic_completer, complete_machines -from ..nix import nix_config, nix_eval -from .types import machine_name_type - -log = logging.getLogger(__name__) - - -@dataclasses.dataclass -class MachineInfo: - machine_name: str - machine_description: str | None - machine_icon: str | None - - -@API.register -def show_machine(flake_url: str | Path, machine_name: str) -> MachineInfo: - config = nix_config() - system = config["system"] - cmd = nix_eval( - [ - f"{flake_url}#clanInternals.machines.{system}.{machine_name}", - "--apply", - "machine: { inherit (machine.config.clan.core) machineDescription machineIcon machineName; }", - "--json", - ] - ) - proc = run_no_stdout(cmd) - res = proc.stdout.strip() - machine = json.loads(res) - - return MachineInfo( - machine_name=machine.get("machineName"), - machine_description=machine.get("machineDescription", None), - machine_icon=machine.get("machineIcon", None), - ) - - -def show_command(args: argparse.Namespace) -> None: - machine = show_machine(args.flake.path, args.machine) - print(f"Name: {machine.machine_name}") - print(f"Description: {machine.machine_description or ''}") - print(f"Icon: {machine.machine_icon or ''}") - - -def register_show_parser(parser: argparse.ArgumentParser) -> None: - parser.set_defaults(func=show_command) - machine_parser = parser.add_argument( - "machine", help="the name of the machine", type=machine_name_type - ) - add_dynamic_completer(machine_parser, complete_machines) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index c49377070..c0beae8b3 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -5,11 +5,15 @@ import os import shlex import sys +from clan_cli.api import API +from clan_cli.clan_uri import FlakeId + from ..cmd import run from ..completions import add_dynamic_completer, complete_machines from ..errors import ClanError from ..facts.generate import generate_facts from ..facts.upload import upload_secrets +from ..inventory import Machine as InventoryMachine from ..machines.machines import Machine from ..nix import nix_command, nix_metadata from ..ssh import HostKeyCheck @@ -81,6 +85,25 @@ def upload_sources( ) +@API.register +def update_machines(base_path: str, machines: list[InventoryMachine]) -> None: + group_machines: list[Machine] = [] + + # Convert InventoryMachine to Machine + for machine in machines: + m = Machine( + name=machine.name, + flake=FlakeId(base_path), + ) + if not machine.deploy.targetHost: + raise ClanError(f"'TargetHost' is not set for machine '{machine.name}'") + # Copy targetHost to machine + m.target_host_address = machine.deploy.targetHost + group_machines.append(m) + + deploy_machine(MachineGroup(group_machines)) + + def deploy_machine(machines: MachineGroup) -> None: """ Deploy to all hosts in parallel @@ -97,8 +120,10 @@ def deploy_machine(machines: MachineGroup) -> None: generate_vars([machine], None, False) upload_secrets(machine) - path = upload_sources(".", target) - + path = upload_sources( + str(machine.flake.path) if machine.flake.is_local() else machine.flake.url, + target, + ) if host.host_key_check != HostKeyCheck.STRICT: ssh_arg += " -o StrictHostKeyChecking=no" if host.host_key_check == HostKeyCheck.NONE: @@ -109,6 +134,7 @@ def deploy_machine(machines: MachineGroup) -> None: cmd = [ "nixos-rebuild", "switch", + "--show-trace", "--fast", "--option", "keep-going", diff --git a/pkgs/clan-cli/clan_cli/nix/__init__.py b/pkgs/clan-cli/clan_cli/nix/__init__.py index e343263a5..66e39bc37 100644 --- a/pkgs/clan-cli/clan_cli/nix/__init__.py +++ b/pkgs/clan-cli/clan_cli/nix/__init__.py @@ -35,6 +35,7 @@ def nix_build(flags: list[str], gcroot: Path | None = None) -> list[str]: str(gcroot), "--print-out-paths", "--no-write-lock-file", + "--show-trace", ] ) + flags @@ -47,6 +48,7 @@ def nix_build(flags: list[str], gcroot: Path | None = None) -> list[str]: "--no-link", "--print-out-paths", "--no-write-lock-file", + "--show-trace", ] ) + flags diff --git a/pkgs/clan-cli/clan_cli/secrets/machines.py b/pkgs/clan-cli/clan_cli/secrets/machines.py index af7543ff3..5ca91fd65 100644 --- a/pkgs/clan-cli/clan_cli/secrets/machines.py +++ b/pkgs/clan-cli/clan_cli/secrets/machines.py @@ -47,10 +47,16 @@ def get_machine(flake_dir: Path, name: str) -> str: def has_machine(flake_dir: Path, name: str) -> bool: + """ + Checks if a machine exists in the sops machines folder + """ return (sops_machines_folder(flake_dir) / name / "key.json").exists() -def list_machines(flake_dir: Path) -> list[str]: +def list_sops_machines(flake_dir: Path) -> list[str]: + """ + Lists all machines in the sops machines folder + """ path = sops_machines_folder(flake_dir) def validate(name: str) -> bool: @@ -86,7 +92,7 @@ def remove_secret(flake_dir: Path, machine: str, secret: str) -> None: def list_command(args: argparse.Namespace) -> None: if args.flake is None: raise ClanError("Could not find clan flake toplevel directory") - lst = list_machines(args.flake.path) + lst = list_sops_machines(args.flake.path) if len(lst) > 0: print("\n".join(lst)) diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 1cebdc4fe..de5128d54 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -163,7 +163,10 @@ def remove_command(args: argparse.Namespace) -> None: def add_secret_argument(parser: argparse.ArgumentParser, autocomplete: bool) -> None: secrets_parser = parser.add_argument( - "secret", help="the name of the secret", type=secret_name_type + "secret", + metavar="secret-name", + help="the name of the secret", + type=secret_name_type, ) if autocomplete: add_dynamic_completer(secrets_parser, complete_secrets) diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index d6f80e01a..f711da5d6 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -57,6 +57,7 @@ def decrypt_dependencies( generator_name: str, secret_vars_store: SecretStoreBase, public_vars_store: FactStoreBase, + shared: bool, ) -> dict[str, dict[str, bytes]]: generator = machine.vars_generators[generator_name] dependencies = set(generator["dependencies"]) @@ -67,11 +68,11 @@ def decrypt_dependencies( for file_name, file in dep_files.items(): if file["secret"]: decrypted_dependencies[dep_generator][file_name] = ( - secret_vars_store.get(dep_generator, file_name) + secret_vars_store.get(dep_generator, file_name, shared=shared) ) else: decrypted_dependencies[dep_generator][file_name] = ( - public_vars_store.get(dep_generator, file_name) + public_vars_store.get(dep_generator, file_name, shared=shared) ) return decrypted_dependencies @@ -109,10 +110,11 @@ def execute_generator( msg += "fact/secret generation is only supported for local flakes" generator = machine.vars_generators[generator_name]["finalScript"] + is_shared = machine.vars_generators[generator_name]["share"] # build temporary file tree of dependencies decrypted_dependencies = decrypt_dependencies( - machine, generator_name, secret_vars_store, public_vars_store + machine, generator_name, secret_vars_store, public_vars_store, shared=is_shared ) env = os.environ.copy() with TemporaryDirectory() as tmp: @@ -159,11 +161,18 @@ def execute_generator( raise ClanError(msg) if file["secret"]: file_path = secret_vars_store.set( - generator_name, file_name, secret_file.read_bytes(), groups + generator_name, + file_name, + secret_file.read_bytes(), + groups, + shared=is_shared, ) else: file_path = public_vars_store.set( - generator_name, file_name, secret_file.read_bytes() + generator_name, + file_name, + secret_file.read_bytes(), + shared=is_shared, ) if file_path: files_to_commit.append(file_path) @@ -260,18 +269,18 @@ def generate_vars( ) -> bool: was_regenerated = False for machine in machines: - errors = 0 + errors = [] try: was_regenerated |= _generate_vars_for_machine( machine, generator_name, regenerate ) except Exception as exc: log.error(f"Failed to generate facts for {machine.name}: {exc}") - errors += 1 - if errors > 0: + errors += [exc] + if len(errors) > 0: raise ClanError( - f"Failed to generate facts for {errors} hosts. Check the logs above" - ) + f"Failed to generate facts for {len(errors)} hosts. Check the logs above" + ) from errors[0] if not was_regenerated: print("All secrets and facts are already up to date") diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py b/pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py index a53ba10c0..41e859cdc 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py @@ -10,16 +10,18 @@ class FactStoreBase(ABC): pass @abstractmethod - def exists(self, service: str, name: str) -> bool: + def exists(self, service: str, name: str, shared: bool = False) -> bool: pass @abstractmethod - def set(self, service: str, name: str, value: bytes) -> Path | None: + def set( + self, service: str, name: str, value: bytes, shared: bool = False + ) -> Path | None: pass # get a single fact @abstractmethod - def get(self, service: str, name: str) -> bytes: + def get(self, service: str, name: str, shared: bool = False) -> bytes: pass # get all facts diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py index c8c2c892c..6e4f2ccc8 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py @@ -10,17 +10,22 @@ class FactStore(FactStoreBase): def __init__(self, machine: Machine) -> None: self.machine = machine self.works_remotely = False + self.per_machine_folder = ( + self.machine.flake_dir / "vars" / "per-machine" / self.machine.name + ) + self.shared_folder = self.machine.flake_dir / "vars" / "shared" - def set(self, generator_name: str, name: str, value: bytes) -> Path | None: + def _var_path(self, generator_name: str, name: str, shared: bool) -> Path: + if shared: + return self.shared_folder / generator_name / name + else: + return self.per_machine_folder / generator_name / name + + def set( + self, generator_name: str, name: str, value: bytes, shared: bool = False + ) -> Path | None: if self.machine.flake.is_local(): - fact_path = ( - self.machine.flake.path - / "machines" - / self.machine.name - / "vars" - / generator_name - / name - ) + fact_path = self._var_path(generator_name, name, shared) fact_path.parent.mkdir(parents=True, exist_ok=True) fact_path.touch() fact_path.write_bytes(value) @@ -30,35 +35,21 @@ class FactStore(FactStoreBase): f"in_flake fact storage is only supported for local flakes: {self.machine.flake}" ) - def exists(self, generator_name: str, name: str) -> bool: - fact_path = ( - self.machine.flake_dir - / "machines" - / self.machine.name - / "vars" - / generator_name - / name - ) - return fact_path.exists() + def exists(self, generator_name: str, name: str, shared: bool = False) -> bool: + return self._var_path(generator_name, name, shared).exists() # get a single fact - def get(self, generator_name: str, name: str) -> bytes: - fact_path = ( - self.machine.flake_dir - / "machines" - / self.machine.name - / "vars" - / generator_name - / name - ) - return fact_path.read_bytes() + def get(self, generator_name: str, name: str, shared: bool = False) -> bytes: + return self._var_path(generator_name, name, shared).read_bytes() # get all public vars def get_all(self) -> dict[str, dict[str, bytes]]: - facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "vars" facts: dict[str, dict[str, bytes]] = {} facts["TODO"] = {} - if facts_folder.exists(): - for fact_path in facts_folder.iterdir(): + if self.per_machine_folder.exists(): + for fact_path in self.per_machine_folder.iterdir(): + facts["TODO"][fact_path.name] = fact_path.read_bytes() + if self.shared_folder.exists(): + for fact_path in self.shared_folder.iterdir(): facts["TODO"][fact_path.name] = fact_path.read_bytes() return facts diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py index 10e0c6b7c..1c9171953 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py @@ -17,18 +17,20 @@ class FactStore(FactStoreBase): self.dir = vm_state_dir(str(machine.flake), machine.name) / "facts" log.debug(f"FactStore initialized with dir {self.dir}") - def exists(self, service: str, name: str) -> bool: + def exists(self, service: str, name: str, shared: bool = False) -> bool: fact_path = self.dir / service / name return fact_path.exists() - def set(self, service: str, name: str, value: bytes) -> Path | None: + def set( + self, service: str, name: str, value: bytes, shared: bool = False + ) -> Path | None: fact_path = self.dir / service / name fact_path.parent.mkdir(parents=True, exist_ok=True) fact_path.write_bytes(value) return None # get a single fact - def get(self, service: str, name: str) -> bytes: + def get(self, service: str, name: str, shared: bool = False) -> bytes: fact_path = self.dir / service / name if fact_path.exists(): return fact_path.read_bytes() diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py index 5e26009c7..0952e62ad 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py @@ -11,16 +11,21 @@ class SecretStoreBase(ABC): @abstractmethod def set( - self, service: str, name: str, value: bytes, groups: list[str] + self, + service: str, + name: str, + value: bytes, + groups: list[str], + shared: bool = False, ) -> Path | None: pass @abstractmethod - def get(self, service: str, name: str) -> bytes: + def get(self, service: str, name: str, shared: bool = False) -> bytes: pass @abstractmethod - def exists(self, service: str, name: str) -> bool: + def exists(self, service: str, name: str, shared: bool = False) -> bool: pass def update_check(self) -> bool: diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index de4c81168..8a2d50109 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -12,8 +12,25 @@ class SecretStore(SecretStoreBase): def __init__(self, machine: Machine) -> None: self.machine = machine + @property + def _password_store_dir(self) -> str: + return os.environ.get( + "PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store" + ) + + def _var_path(self, generator_name: str, name: str, shared: bool) -> Path: + if shared: + return Path(f"shared/{generator_name}/{name}") + else: + return Path(f"machines/{self.machine.name}/{generator_name}/{name}") + def set( - self, generator_name: str, name: str, value: bytes, groups: list[str] + self, + generator_name: str, + name: str, + value: bytes, + groups: list[str], + shared: bool = False, ) -> Path | None: subprocess.run( nix_shell( @@ -22,7 +39,7 @@ class SecretStore(SecretStoreBase): "pass", "insert", "-m", - f"machines/{self.machine.name}/{generator_name}/{name}", + str(self._var_path(generator_name, name, shared)), ], ), input=value, @@ -30,34 +47,28 @@ class SecretStore(SecretStoreBase): ) return None # we manage the files outside of the git repo - def get(self, generator_name: str, name: str) -> bytes: + def get(self, generator_name: str, name: str, shared: bool = False) -> bytes: return subprocess.run( nix_shell( ["nixpkgs#pass"], [ "pass", "show", - f"machines/{self.machine.name}/{generator_name}/{name}", + str(self._var_path(generator_name, name, shared)), ], ), check=True, stdout=subprocess.PIPE, ).stdout - def exists(self, generator_name: str, name: str) -> bool: - password_store = os.environ.get( - "PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store" - ) - secret_path = ( - Path(password_store) - / f"machines/{self.machine.name}/{generator_name}/{name}.gpg" - ) - return secret_path.exists() + def exists(self, generator_name: str, name: str, shared: bool = False) -> bool: + return ( + Path(self._password_store_dir) + / f"{self._var_path(generator_name, name, shared)}.gpg" + ).exists() def generate_hash(self) -> bytes: - password_store = os.environ.get( - "PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store" - ) + password_store = self._password_store_dir hashes = [] hashes.append( subprocess.run( @@ -117,15 +128,17 @@ class SecretStore(SecretStoreBase): return local_hash.decode() == remote_hash + # TODO: fixme def upload(self, output_dir: Path) -> None: - for service in self.machine.facts_data: - for secret in self.machine.facts_data[service]["secret"]: - if isinstance(secret, dict): - secret_name = secret["name"] - else: - # TODO: drop old format soon - secret_name = secret - with (output_dir / secret_name).open("wb") as f: - f.chmod(0o600) - f.write(self.get(service, secret_name)) - (output_dir / ".pass_info").write_bytes(self.generate_hash()) + pass + # for service in self.machine.facts_data: + # for secret in self.machine.facts_data[service]["secret"]: + # if isinstance(secret, dict): + # secret_name = secret["name"] + # else: + # # TODO: drop old format soon + # secret_name = secret + # with (output_dir / secret_name).open("wb") as f: + # f.chmod(0o600) + # f.write(self.get(service, secret_name)) + # (output_dir / ".pass_info").write_bytes(self.generate_hash()) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index 5784ebfd6..2194c2996 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -36,20 +36,30 @@ class SecretStore(SecretStoreBase): ) add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) - def secret_path(self, generator_name: str, secret_name: str) -> Path: - return ( - self.machine.flake_dir - / "sops" - / "vars" - / self.machine.name - / generator_name - / secret_name - ) + def secret_path( + self, generator_name: str, secret_name: str, shared: bool = False + ) -> Path: + if shared: + base_path = self.machine.flake_dir / "sops" / "vars" / "shared" + else: + base_path = ( + self.machine.flake_dir + / "sops" + / "vars" + / "per-machine" + / self.machine.name + ) + return base_path / generator_name / secret_name def set( - self, generator_name: str, name: str, value: bytes, groups: list[str] + self, + generator_name: str, + name: str, + value: bytes, + groups: list[str], + shared: bool = False, ) -> Path | None: - path = self.secret_path(generator_name, name) + path = self.secret_path(generator_name, name, shared) encrypt_secret( self.machine.flake_dir, path, @@ -59,14 +69,14 @@ class SecretStore(SecretStoreBase): ) return path - def get(self, generator_name: str, name: str) -> bytes: + def get(self, generator_name: str, name: str, shared: bool = False) -> bytes: return decrypt_secret( - self.machine.flake_dir, self.secret_path(generator_name, name) + self.machine.flake_dir, self.secret_path(generator_name, name, shared) ).encode("utf-8") - def exists(self, generator_name: str, name: str) -> bool: + def exists(self, generator_name: str, name: str, shared: bool = False) -> bool: return has_secret( - self.secret_path(generator_name, name), + self.secret_path(generator_name, name, shared), ) def upload(self, output_dir: Path) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py index fc3ea3cdd..2efd4daf1 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py @@ -15,18 +15,23 @@ class SecretStore(SecretStoreBase): self.dir.mkdir(parents=True, exist_ok=True) def set( - self, service: str, name: str, value: bytes, groups: list[str] + self, + service: str, + name: str, + value: bytes, + groups: list[str], + shared: bool = False, ) -> Path | None: secret_file = self.dir / service / name secret_file.parent.mkdir(parents=True, exist_ok=True) secret_file.write_bytes(value) return None # we manage the files outside of the git repo - def get(self, service: str, name: str) -> bytes: + def get(self, service: str, name: str, shared: bool = False) -> bytes: secret_file = self.dir / service / name return secret_file.read_bytes() - def exists(self, service: str, name: str) -> bool: + def exists(self, service: str, name: str, shared: bool = False) -> bool: return (self.dir / service / name).exists() def upload(self, output_dir: Path) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/upload.py b/pkgs/clan-cli/clan_cli/vars/upload.py index 6cb6bf8be..d80af092f 100644 --- a/pkgs/clan-cli/clan_cli/vars/upload.py +++ b/pkgs/clan-cli/clan_cli/vars/upload.py @@ -13,14 +13,14 @@ log = logging.getLogger(__name__) def upload_secrets(machine: Machine) -> None: - secret_facts_module = importlib.import_module(machine.secret_facts_module) - secret_facts_store = secret_facts_module.SecretStore(machine=machine) + secret_store_module = importlib.import_module(machine.secret_facts_module) + secret_store = secret_store_module.SecretStore(machine=machine) - if secret_facts_store.update_check(): + if secret_store.update_check(): log.info("Secrets already up to date") return with TemporaryDirectory() as tempdir: - secret_facts_store.upload(Path(tempdir)) + secret_store.upload(Path(tempdir)) host = machine.target_host ssh_cmd = host.ssh_cmd() @@ -31,9 +31,11 @@ def upload_secrets(machine: Machine) -> None: "rsync", "-e", " ".join(["ssh"] + ssh_cmd[2:]), - "-az", + "--recursive", + "--links", + "--times", + "--compress", "--delete", - "--chown=root:root", "--chmod=D700,F600", f"{tempdir!s}/", f"{host.user}@{host.host}:{machine.secrets_upload_directory}/", diff --git a/pkgs/clan-cli/clan_cli/vms/qemu.py b/pkgs/clan-cli/clan_cli/vms/qemu.py index 9f5969fff..f8c90f95d 100644 --- a/pkgs/clan-cli/clan_cli/vms/qemu.py +++ b/pkgs/clan-cli/clan_cli/vms/qemu.py @@ -94,6 +94,7 @@ def qemu_command( virtiofsd_socket: Path, qmp_socket_file: Path, qga_socket_file: Path, + portmap: list[tuple[int, int]] = [], ) -> QemuCommand: kernel_cmdline = [ (Path(nixos_config["toplevel"]) / "kernel-params").read_text(), @@ -103,6 +104,7 @@ def qemu_command( ] if not vm.waypipe: kernel_cmdline.append("console=tty0") + hostfwd = ",".join(f"hostfwd=tcp::{h}-:{g}" for h, g in portmap) # fmt: off command = [ "qemu-kvm", @@ -116,7 +118,7 @@ def qemu_command( # speed-up boot by not waiting for the boot menu "-boot", "menu=off,strict=on", "-device", "virtio-rng-pci", - "-netdev", "user,id=user.0", + "-netdev", f"user,id=user.0,{hostfwd}", "-device", "virtio-net-pci,netdev=user.0,romfile=", "-chardev", f"socket,id=char1,path={virtiofsd_socket}", "-device", "vhost-user-fs-pci,chardev=char1,tag=nix-store", diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 6f2082d70..57cd3c618 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -108,6 +108,7 @@ def run_vm( cachedir: Path | None = None, socketdir: Path | None = None, nix_options: list[str] = [], + portmap: list[tuple[int, int]] = [], ) -> None: with ExitStack() as stack: machine = Machine(name=vm.machine_name, flake=vm.flake_url) @@ -168,6 +169,7 @@ def run_vm( virtiofsd_socket=virtiofsd_socket, qmp_socket_file=qmp_socket_file, qga_socket_file=qga_socket_file, + portmap=portmap, ) packages = ["nixpkgs#qemu"] @@ -199,7 +201,9 @@ def run_command( vm: VmConfig = inspect_vm(machine=machine_obj) - run_vm(vm, nix_options=args.option) + portmap = [(h, g) for h, g in (p.split(":") for p in args.publish)] + + run_vm(vm, nix_options=args.option, portmap=portmap) def register_run_parser(parser: argparse.ArgumentParser) -> None: @@ -207,4 +211,13 @@ def register_run_parser(parser: argparse.ArgumentParser) -> None: "machine", type=str, help="machine in the flake to run" ) add_dynamic_completer(machine_action, complete_machines) + # option: --publish 2222:22 + parser.add_argument( + "--publish", + "-p", + action="append", + type=str, + default=[], + help="Forward ports from host to guest", + ) parser.set_defaults(func=lambda args: run_command(args)) diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 516bd4cb4..8db403407 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -17,8 +17,6 @@ setuptools, stdenv, - pydantic, - # custom args clan-core-path, nixpkgs, @@ -30,7 +28,6 @@ let pythonDependencies = [ argcomplete # Enables shell completions - pydantic # Dataclass deserialisation / validation / schemas ]; # load nixpkgs runtime dependencies from a json file @@ -63,9 +60,7 @@ let source = runCommand "clan-cli-source" { } '' cp -r ${./.} $out chmod -R +w $out - rm $out/clan_cli/config/jsonschema ln -sf ${nixpkgs'} $out/clan_cli/nixpkgs - cp -r ${../../lib/jsonschema} $out/clan_cli/config/jsonschema cp -r ${../../templates} $out/clan_cli/templates ${classgen}/bin/classgen ${inventory-schema}/schema.json $out/clan_cli/inventory/classes.py diff --git a/pkgs/clan-cli/tests/age_keys.py b/pkgs/clan-cli/tests/age_keys.py index 02f15c02b..cf90170f4 100644 --- a/pkgs/clan-cli/tests/age_keys.py +++ b/pkgs/clan-cli/tests/age_keys.py @@ -1,4 +1,8 @@ +import os +from pathlib import Path + import pytest +from helpers import cli class KeyPair: @@ -11,6 +15,22 @@ class SopsSetup: def __init__(self, keys: list[KeyPair]) -> None: self.keys = keys + def init(self, flake_path: Path | None = None) -> None: + if flake_path is None: + flake_path = Path.cwd() + self.user = os.environ.get("USER", "user") + cli.run( + [ + "secrets", + "users", + "add", + "--flake", + str(flake_path), + self.user, + self.keys[0].pubkey, + ] + ) + KEYS = [ KeyPair( diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index a0374aeea..80220daee 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -15,6 +15,7 @@ pytest_plugins = [ "ports", "host_group", "fixtures_flakes", + "stdout", ] diff --git a/pkgs/clan-cli/tests/data/secrets.yaml b/pkgs/clan-cli/tests/data/secrets.yaml index 3bc636597..c59abca52 100644 --- a/pkgs/clan-cli/tests/data/secrets.yaml +++ b/pkgs/clan-cli/tests/data/secrets.yaml @@ -1,23 +1,23 @@ secret-key: ENC[AES256_GCM,data:gjX4OmCUdd3TlA4p,iv:3yZVpyd6FqkITQY0nU2M1iubmzvkR6PfkK2m/s6nQh8=,tag:Abgp9xkiFFylZIyAlap6Ew==,type:str] nested: - secret-key: ENC[AES256_GCM,data:iUMgDhhIjwvd7wL4,iv:jiJIrh12dSu/sXX+z9ITVoEMNDMjwIlFBnyv40oN4LE=,tag:G9VmAa66Km1sc7JEhW5AvA==,type:str] + secret-key: ENC[AES256_GCM,data:iUMgDhhIjwvd7wL4,iv:jiJIrh12dSu/sXX+z9ITVoEMNDMjwIlFBnyv40oN4LE=,tag:G9VmAa66Km1sc7JEhW5AvA==,type:str] sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: - - recipient: age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0eWdRVjlydXlXOVZFQ3lO - bzU1eG9Iam5Ka29Sdlo0cHJ4b1R6bjdNSzBjCkgwRndCbWZQWHlDU0x1cWRmaGVt - N29lbjR6UjN0L2RhaXEzSG9zQmRsZGsKLS0tIEdsdWgxSmZwU3BWUDVxVWRSSC9M - eVZ6bjgwZnR2TTM5MkRYZWNFSFplQWsKmSzv12/dftL9jx2y35UZUGVK6xWdatE8 - BGJiCvMlp0BQNrh2s/+YaEaBa48w8LL79U/XJnEZ+ZUwxmlbSTn6Hg== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2023-08-08T14:27:20Z" - mac: ENC[AES256_GCM,data:iRWWX+L5Q5nKn3fBCLaWoz/mvqGnNnRd93gJmYXDZbRjFoHa9IFJZst5QDIDa1ZRYUe6G0/+lV5SBi+vwRm1pHysJ3c0ZWYjBP+e1jw3jLXxLV5gACsDC8by+6rFUCho0Xgu+Nqu2ehhNenjQQnCvDH5ivWbW70KFT5ynNgR9Tw=,iv:RYnnbLMC/hNfMwWPreMq9uvY0khajwQTZENO/P34ckY=,tag:Xi1PS5vM1c+sRkroHkPn1Q==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0eWdRVjlydXlXOVZFQ3lO + bzU1eG9Iam5Ka29Sdlo0cHJ4b1R6bjdNSzBjCkgwRndCbWZQWHlDU0x1cWRmaGVt + N29lbjR6UjN0L2RhaXEzSG9zQmRsZGsKLS0tIEdsdWgxSmZwU3BWUDVxVWRSSC9M + eVZ6bjgwZnR2TTM5MkRYZWNFSFplQWsKmSzv12/dftL9jx2y35UZUGVK6xWdatE8 + BGJiCvMlp0BQNrh2s/+YaEaBa48w8LL79U/XJnEZ+ZUwxmlbSTn6Hg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2023-08-08T14:27:20Z" + mac: ENC[AES256_GCM,data:iRWWX+L5Q5nKn3fBCLaWoz/mvqGnNnRd93gJmYXDZbRjFoHa9IFJZst5QDIDa1ZRYUe6G0/+lV5SBi+vwRm1pHysJ3c0ZWYjBP+e1jw3jLXxLV5gACsDC8by+6rFUCho0Xgu+Nqu2ehhNenjQQnCvDH5ivWbW70KFT5ynNgR9Tw=,iv:RYnnbLMC/hNfMwWPreMq9uvY0khajwQTZENO/P34ckY=,tag:Xi1PS5vM1c+sRkroHkPn1Q==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.3 diff --git a/pkgs/clan-cli/tests/fixtures_flakes.py b/pkgs/clan-cli/tests/fixtures_flakes.py index 49760f3b8..1ddfb444f 100644 --- a/pkgs/clan-cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/tests/fixtures_flakes.py @@ -1,4 +1,3 @@ -import fileinput import json import logging import os @@ -25,20 +24,26 @@ def substitute( flake: Path = Path(__file__).parent, ) -> None: sops_key = str(flake.joinpath("sops.key")) - for line in fileinput.input(file, inplace=True): - line = line.replace("__NIXPKGS__", str(nixpkgs_source())) - if clan_core_flake: - line = line.replace("__CLAN_CORE__", str(clan_core_flake)) - line = line.replace( - "git+https://git.clan.lol/clan/clan-core", str(clan_core_flake) - ) - line = line.replace( - "https://git.clan.lol/clan/clan-core/archive/main.tar.gz", - str(clan_core_flake), - ) - line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key) - line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake)) - print(line, end="") + buf = "" + with file.open() as f: + for line in f: + line = line.replace("__NIXPKGS__", str(nixpkgs_source())) + if clan_core_flake: + line = line.replace("__CLAN_CORE__", str(clan_core_flake)) + line = line.replace( + "git+https://git.clan.lol/clan/clan-core", str(clan_core_flake) + ) + line = line.replace( + "https://git.clan.lol/clan/clan-core/archive/main.tar.gz", + str(clan_core_flake), + ) + line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key) + line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake)) + buf += line + print(f"file: {file}") + print(f"clan_core: {clan_core_flake}") + print(f"flake: {flake}") + file.write_text(buf) class FlakeForTest(NamedTuple): @@ -91,10 +96,13 @@ def generate_flake( for file in flake.rglob("*"): if file.is_file(): print(f"Final Content of {file}:") - for line in fileinput.input(file, inplace=True): - for key, value in substitutions.items(): - line = line.replace(key, value) - print(line, end="") + buf = "" + with file.open() as f: + for line in f: + for key, value in substitutions.items(): + line = line.replace(key, value) + buf += line + file.write_text(buf) # generate machines from machineConfigs for machine_name, machine_config in machine_configs.items(): diff --git a/pkgs/clan-cli/tests/helpers/nixos_config.py b/pkgs/clan-cli/tests/helpers/nixos_config.py new file mode 100644 index 000000000..b922c6bf9 --- /dev/null +++ b/pkgs/clan-cli/tests/helpers/nixos_config.py @@ -0,0 +1,11 @@ +from collections import defaultdict +from collections.abc import Callable +from typing import Any + + +def def_value() -> defaultdict: + return defaultdict(def_value) + + +# allows defining nested dictionary in a single line +nested_dict: Callable[[], dict[str, Any]] = lambda: defaultdict(def_value) diff --git a/pkgs/clan-cli/tests/helpers/vms.py b/pkgs/clan-cli/tests/helpers/vms.py new file mode 100644 index 000000000..417b85034 --- /dev/null +++ b/pkgs/clan-cli/tests/helpers/vms.py @@ -0,0 +1,95 @@ +import contextlib +import os +import socket +import sys +import threading +import traceback +from pathlib import Path +from time import sleep + +from clan_cli.dirs import vm_state_dir +from clan_cli.qemu.qga import QgaSession +from clan_cli.qemu.qmp import QEMUMonitorProtocol + +from . import cli + + +def find_free_port() -> int: + """Find an unused localhost port from 1024-65535 and return it.""" + with contextlib.closing(socket.socket(type=socket.SOCK_STREAM)) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def run_vm_in_thread(machine_name: str, ssh_port: int | None = None) -> int: + # runs machine and prints exceptions + if ssh_port is None: + ssh_port = find_free_port() + + def run() -> None: + try: + cli.run(["vms", "run", machine_name, "--publish", f"{ssh_port}:22"]) + except Exception: + # print exception details + print(traceback.format_exc(), file=sys.stderr) + print(sys.exc_info()[2], file=sys.stderr) + + # run the machine in a separate thread + t = threading.Thread(target=run, name="run") + t.daemon = True + t.start() + return ssh_port + + +# wait for qmp socket to exist +def wait_vm_up(machine_name: str, flake_url: str | None = None) -> None: + if flake_url is None: + flake_url = str(Path.cwd()) + socket_file = vm_state_dir(flake_url, machine_name) / "qmp.sock" + timeout: float = 600 + while True: + if timeout <= 0: + raise TimeoutError( + f"qmp socket {socket_file} not found. Is the VM running?" + ) + if socket_file.exists(): + break + sleep(0.1) + timeout -= 0.1 + + +# wait for vm to be down by checking if qmp socket is down +def wait_vm_down(machine_name: str, flake_url: str | None = None) -> None: + if flake_url is None: + flake_url = str(Path.cwd()) + socket_file = vm_state_dir(flake_url, machine_name) / "qmp.sock" + timeout: float = 300 + while socket_file.exists(): + if timeout <= 0: + raise TimeoutError( + f"qmp socket {socket_file} still exists. Is the VM down?" + ) + sleep(0.1) + timeout -= 0.1 + + +# wait for vm to be up then connect and return qmp instance +def qmp_connect(machine_name: str, flake_url: str | None = None) -> QEMUMonitorProtocol: + if flake_url is None: + flake_url = str(Path.cwd()) + state_dir = vm_state_dir(flake_url, machine_name) + wait_vm_up(machine_name, flake_url) + qmp = QEMUMonitorProtocol( + address=str(os.path.realpath(state_dir / "qmp.sock")), + ) + qmp.connect() + return qmp + + +# wait for vm to be up then connect and return qga instance +def qga_connect(machine_name: str, flake_url: str | None = None) -> QgaSession: + if flake_url is None: + flake_url = str(Path.cwd()) + state_dir = vm_state_dir(flake_url, machine_name) + wait_vm_up(machine_name, flake_url) + return QgaSession(os.path.realpath(state_dir / "qga.sock")) diff --git a/pkgs/clan-cli/tests/stdout.py b/pkgs/clan-cli/tests/stdout.py new file mode 100644 index 000000000..5106498db --- /dev/null +++ b/pkgs/clan-cli/tests/stdout.py @@ -0,0 +1,23 @@ +from typing import Any + +import pytest +from pytest import CaptureFixture + + +class CaptureOutput: + def __init__(self, capsys: CaptureFixture) -> None: + self.capsys = capsys + + def __enter__(self) -> "CaptureOutput": + self.capsys.readouterr() + return self + + def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> bool: + res = self.capsys.readouterr() + self.out = res.out + self.err = res.err + + +@pytest.fixture +def capture_output(capsys: CaptureFixture) -> CaptureOutput: + return CaptureOutput(capsys) diff --git a/pkgs/clan-cli/tests/test_cli.py b/pkgs/clan-cli/tests/test_cli.py index 5f4762bcb..03b0c5ae1 100644 --- a/pkgs/clan-cli/tests/test_cli.py +++ b/pkgs/clan-cli/tests/test_cli.py @@ -1,9 +1,9 @@ import pytest from helpers import cli +from stdout import CaptureOutput -def test_help(capsys: pytest.CaptureFixture) -> None: - with pytest.raises(SystemExit): +def test_help(capture_output: CaptureOutput) -> None: + with capture_output as output, pytest.raises(SystemExit): cli.run(["--help"]) - captured = capsys.readouterr() - assert captured.out.startswith("usage:") + assert output.out.startswith("usage:") diff --git a/pkgs/clan-cli/tests/test_config.py b/pkgs/clan-cli/tests/test_config.py index 4cc91d5ac..e0ffd4262 100644 --- a/pkgs/clan-cli/tests/test_config.py +++ b/pkgs/clan-cli/tests/test_config.py @@ -1,8 +1,6 @@ from pathlib import Path import pytest -from fixtures_flakes import FlakeForTest -from helpers import cli from clan_cli import config from clan_cli.config import parsing @@ -11,28 +9,6 @@ from clan_cli.errors import ClanError example_options = f"{Path(config.__file__).parent}/jsonschema/options.json" -def test_configure_machine( - test_flake: FlakeForTest, - capsys: pytest.CaptureFixture, -) -> None: - # clear the output buffer - capsys.readouterr() - # read a option value - cli.run( - [ - "config", - "--flake", - str(test_flake.path), - "-m", - "machine1", - "clan.jitsi.enable", - ] - ) - - # read the output - assert capsys.readouterr().out == "false\n" - - def test_walk_jsonschema_all_types() -> None: schema = dict( type="object", diff --git a/pkgs/clan-cli/tests/test_create_flake.py b/pkgs/clan-cli/tests/test_create_flake.py index d0e920c80..75f7bdd80 100644 --- a/pkgs/clan-cli/tests/test_create_flake.py +++ b/pkgs/clan-cli/tests/test_create_flake.py @@ -5,14 +5,15 @@ from pathlib import Path import pytest from fixtures_flakes import substitute from helpers import cli +from stdout import CaptureOutput @pytest.mark.impure def test_create_flake( monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, temporary_home: Path, clan_core: Path, + capture_output: CaptureOutput, ) -> None: flake_dir = temporary_home / "test-flake" @@ -29,7 +30,6 @@ def test_create_flake( monkeypatch.chdir(flake_dir) cli.run(["machines", "create", "machine1"]) - capsys.readouterr() # flush cache # create a hardware-configuration.nix that doesn't throw an eval error @@ -39,8 +39,9 @@ def test_create_flake( ) as hw_config_nix: hw_config_nix.write("{}") - cli.run(["machines", "list"]) - assert "machine1" in capsys.readouterr().out + with capture_output as output: + cli.run(["machines", "list"]) + assert "machine1" in output.out flake_show = subprocess.run( ["nix", "flake", "show", "--json"], check=True, @@ -57,9 +58,9 @@ def test_create_flake( @pytest.mark.impure def test_ui_template( monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture, temporary_home: Path, clan_core: Path, + capture_output: CaptureOutput, ) -> None: flake_dir = temporary_home / "test-flake" url = f"{clan_core}#minimal" @@ -73,10 +74,10 @@ def test_ui_template( monkeypatch.chdir(flake_dir) cli.run(["machines", "create", "machine1"]) - capsys.readouterr() # flush cache - cli.run(["machines", "list"]) - assert "machine1" in capsys.readouterr().out + with capture_output as output: + cli.run(["machines", "list"]) + assert "machine1" in output.out flake_show = subprocess.run( ["nix", "flake", "show", "--json"], check=True, diff --git a/pkgs/clan-cli/tests/test_deserializers.py b/pkgs/clan-cli/tests/test_deserializers.py index f87b65adc..b475c00b9 100644 --- a/pkgs/clan-cli/tests/test_deserializers.py +++ b/pkgs/clan-cli/tests/test_deserializers.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field from pathlib import Path +from typing import Literal import pytest @@ -18,6 +19,7 @@ from clan_cli.inventory import ( ServiceBorgbackupRoleServer, ServiceMeta, ) +from clan_cli.machines import machines def test_simple() -> None: @@ -45,11 +47,11 @@ def test_nested() -> None: class Person: name: str # deeply nested dataclasses + home: Path | str | None age: Age age_list: list[Age] age_dict: dict[str, Age] # Optional field - home: Path | None person_dict = { "name": "John", @@ -72,6 +74,55 @@ def test_nested() -> None: assert from_dict(Person, person_dict) == expected_person +def test_nested_nullable() -> None: + @dataclass + class SystemConfig: + language: str | None = field(default=None) + keymap: str | None = field(default=None) + ssh_keys_path: list[str] | None = field(default=None) + + @dataclass + class FlashOptions: + machine: machines.Machine + mode: str + disks: dict[str, str] + system_config: SystemConfig + dry_run: bool + write_efi_boot_entries: bool + debug: bool + + data = { + "machine": { + "name": "flash-installer", + "flake": {"loc": "git+https://git.clan.lol/clan/clan-core"}, + }, + "mode": "format", + "disks": {"main": "/dev/sda"}, + "system_config": {"language": "en_US.UTF-8", "keymap": "en"}, + "dry_run": False, + "write_efi_boot_entries": False, + "debug": False, + "op_key": "jWnTSHwYhSgr7Qz3u4ppD", + } + + expected = FlashOptions( + machine=machines.Machine( + name="flash-installer", + flake=machines.FlakeId("git+https://git.clan.lol/clan/clan-core"), + ), + mode="format", + disks={"main": "/dev/sda"}, + system_config=SystemConfig( + language="en_US.UTF-8", keymap="en", ssh_keys_path=None + ), + dry_run=False, + write_efi_boot_entries=False, + debug=False, + ) + + assert from_dict(FlashOptions, data) == expected + + def test_simple_field_missing() -> None: @dataclass class Person: @@ -83,6 +134,44 @@ def test_simple_field_missing() -> None: from_dict(Person, person_dict) +def test_nullable() -> None: + @dataclass + class Person: + name: None + + person_dict = { + "name": None, + } + + from_dict(Person, person_dict) + + +def test_nullable_non_exist() -> None: + @dataclass + class Person: + name: None + + person_dict = {} + + with pytest.raises(ClanError): + from_dict(Person, person_dict) + + +def test_list() -> None: + data = [ + {"name": "John"}, + {"name": "Sarah"}, + ] + + @dataclass + class Name: + name: str + + result = from_dict(list[Name], data) + + assert result == [Name("John"), Name("Sarah")] + + def test_deserialize_extensive_inventory() -> None: # TODO: Make this an abstract test, so it doesn't break the test if the inventory changes data = { @@ -177,3 +266,19 @@ def test_private_public_fields() -> None: assert from_dict(Person, data) == expected assert dataclass_to_dict(expected) == data + + +def test_literal_field() -> None: + @dataclass + class Person: + name: Literal["open_file", "select_folder", "save"] + + data = {"name": "open_file"} + expected = Person(name="open_file") + assert from_dict(Person, data) == expected + + assert dataclass_to_dict(expected) == data + + with pytest.raises(ClanError): + # Not a valid value + from_dict(Person, {"name": "open"}) diff --git a/pkgs/clan-cli/tests/test_flakes_cli.py b/pkgs/clan-cli/tests/test_flakes_cli.py index 6bb57fc86..5ff74a2ef 100644 --- a/pkgs/clan-cli/tests/test_flakes_cli.py +++ b/pkgs/clan-cli/tests/test_flakes_cli.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING import pytest from fixtures_flakes import FlakeForTest from helpers import cli +from stdout import CaptureOutput if TYPE_CHECKING: pass @@ -10,18 +11,17 @@ if TYPE_CHECKING: @pytest.mark.impure def test_flakes_inspect( - test_flake_with_core: FlakeForTest, capsys: pytest.CaptureFixture + test_flake_with_core: FlakeForTest, capture_output: CaptureOutput ) -> None: - cli.run( - [ - "flakes", - "inspect", - "--flake", - str(test_flake_with_core.path), - "--machine", - "vm1", - ] - ) - out = capsys.readouterr() # empty the buffer - - assert "Icon" in out.out + with capture_output as output: + cli.run( + [ + "flakes", + "inspect", + "--flake", + str(test_flake_with_core.path), + "--machine", + "vm1", + ] + ) + assert "Icon" in output.out diff --git a/pkgs/clan-cli/tests/test_history_cli.py b/pkgs/clan-cli/tests/test_history_cli.py index e65c7b2af..88b7e4ed1 100644 --- a/pkgs/clan-cli/tests/test_history_cli.py +++ b/pkgs/clan-cli/tests/test_history_cli.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING import pytest from fixtures_flakes import FlakeForTest from helpers import cli -from pytest import CaptureFixture +from stdout import CaptureOutput from clan_cli.dirs import user_history_file from clan_cli.history.add import HistoryEntry @@ -32,17 +32,15 @@ def test_history_add( @pytest.mark.impure def test_history_list( - capsys: CaptureFixture, + capture_output: CaptureOutput, test_flake_with_core: FlakeForTest, ) -> None: - cmd = [ - "history", - "list", - ] - - cli.run(cmd) - assert str(test_flake_with_core.path) not in capsys.readouterr().out + with capture_output as output: + cli.run(["history", "list"]) + assert str(test_flake_with_core.path) not in output.out cli.run(["history", "add", f"clan://{test_flake_with_core.path}#vm1"]) - cli.run(cmd) - assert str(test_flake_with_core.path) in capsys.readouterr().out + + with capture_output as output: + cli.run(["history", "list"]) + assert str(test_flake_with_core.path) in output.out diff --git a/pkgs/clan-cli/tests/test_import_sops_cli.py b/pkgs/clan-cli/tests/test_import_sops_cli.py index 1147a5e71..ec0230877 100644 --- a/pkgs/clan-cli/tests/test_import_sops_cli.py +++ b/pkgs/clan-cli/tests/test_import_sops_cli.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING import pytest from fixtures_flakes import FlakeForTest from helpers import cli +from stdout import CaptureOutput if TYPE_CHECKING: from age_keys import KeyPair @@ -12,7 +13,7 @@ if TYPE_CHECKING: def test_import_sops( test_root: Path, test_flake: FlakeForTest, - capsys: pytest.CaptureFixture, + capture_output: CaptureOutput, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], ) -> None: @@ -88,11 +89,11 @@ def test_import_sops( ] cli.run(cmd) - capsys.readouterr() - cli.run(["secrets", "users", "list", "--flake", str(test_flake.path)]) - users = sorted(capsys.readouterr().out.rstrip().split()) + with capture_output as output: + cli.run(["secrets", "users", "list", "--flake", str(test_flake.path)]) + users = sorted(output.out.rstrip().split()) assert users == ["user1", "user2"] - capsys.readouterr() - cli.run(["secrets", "get", "--flake", str(test_flake.path), "secret-key"]) - assert capsys.readouterr().out == "secret-value" + with capture_output as output: + cli.run(["secrets", "get", "--flake", str(test_flake.path), "secret-key"]) + assert output.out == "secret-value" diff --git a/pkgs/clan-cli/tests/test_machines_cli.py b/pkgs/clan-cli/tests/test_machines_cli.py index 10c8f22a6..f8994471b 100644 --- a/pkgs/clan-cli/tests/test_machines_cli.py +++ b/pkgs/clan-cli/tests/test_machines_cli.py @@ -1,40 +1,32 @@ import pytest from fixtures_flakes import FlakeForTest from helpers import cli +from stdout import CaptureOutput @pytest.mark.impure def test_machine_subcommands( - test_flake_with_core: FlakeForTest, capsys: pytest.CaptureFixture + test_flake_with_core: FlakeForTest, + capture_output: CaptureOutput, ) -> None: cli.run( ["machines", "create", "--flake", str(test_flake_with_core.path), "machine1"] ) - capsys.readouterr() - cli.run(["machines", "list", "--flake", str(test_flake_with_core.path)]) + with capture_output as output: + cli.run(["machines", "list", "--flake", str(test_flake_with_core.path)]) - out = capsys.readouterr() - - assert "machine1" in out.out - assert "vm1" in out.out - assert "vm2" in out.out - - capsys.readouterr() - cli.run(["machines", "show", "--flake", str(test_flake_with_core.path), "machine1"]) - out = capsys.readouterr() - assert "machine1" in out.out - assert "Description" in out.out - print(out) + print(output.out) + assert "machine1" in output.out + assert "vm1" in output.out + assert "vm2" in output.out cli.run( ["machines", "delete", "--flake", str(test_flake_with_core.path), "machine1"] ) - capsys.readouterr() - cli.run(["machines", "list", "--flake", str(test_flake_with_core.path)]) - out = capsys.readouterr() - - assert "machine1" not in out.out - assert "vm1" in out.out - assert "vm2" in out.out + with capture_output as output: + cli.run(["machines", "list", "--flake", str(test_flake_with_core.path)]) + assert "machine1" not in output.out + assert "vm1" in output.out + assert "vm2" in output.out diff --git a/pkgs/clan-cli/tests/test_machines_config.py b/pkgs/clan-cli/tests/test_machines_config.py deleted file mode 100644 index 0bc6c50d3..000000000 --- a/pkgs/clan-cli/tests/test_machines_config.py +++ /dev/null @@ -1,48 +0,0 @@ -import pytest -from fixtures_flakes import FlakeForTest - -from clan_cli.clan_uri import FlakeId -from clan_cli.config.machine import ( - config_for_machine, - set_config_for_machine, - verify_machine_config, -) -from clan_cli.config.schema import machine_schema -from clan_cli.inventory import Machine, MachineDeploy -from clan_cli.machines.create import create_machine -from clan_cli.machines.list import list_machines - - -@pytest.mark.with_core -def test_schema_for_machine(test_flake_with_core: FlakeForTest) -> None: - schema = machine_schema(test_flake_with_core.path, config={}) - assert "properties" in schema - - -@pytest.mark.with_core -def test_create_machine_on_minimal_clan(test_flake_minimal: FlakeForTest) -> None: - assert list_machines(test_flake_minimal.path) == {} - - create_machine( - FlakeId(test_flake_minimal.path), - Machine( - name="foo", - system="x86_64-linux", - description="A test machine", - tags=["test"], - icon=None, - deploy=MachineDeploy(), - ), - ) - - result = list_machines(test_flake_minimal.path) - assert list(result.keys()) == ["foo"] - - # Writes into settings.json - set_config_for_machine( - test_flake_minimal.path, "foo", dict(services=dict(openssh=dict(enable=True))) - ) - - config = config_for_machine(test_flake_minimal.path, "foo") - assert config["services"]["openssh"]["enable"] - assert verify_machine_config(test_flake_minimal.path, "foo") is None diff --git a/pkgs/clan-cli/tests/test_secrets_cli.py b/pkgs/clan-cli/tests/test_secrets_cli.py index f47fe8eb9..0deeb622c 100644 --- a/pkgs/clan-cli/tests/test_secrets_cli.py +++ b/pkgs/clan-cli/tests/test_secrets_cli.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING import pytest from fixtures_flakes import FlakeForTest from helpers import cli +from stdout import CaptureOutput from clan_cli.errors import ClanError @@ -19,7 +20,7 @@ log = logging.getLogger(__name__) def _test_identities( what: str, test_flake: FlakeForTest, - capsys: pytest.CaptureFixture, + capture_output: CaptureOutput, age_keys: list["KeyPair"], ) -> None: sops_folder = test_flake.path / "sops" @@ -64,24 +65,22 @@ def _test_identities( ] ) - capsys.readouterr() # empty the buffer - cli.run( - [ - "secrets", - what, - "get", - "--flake", - str(test_flake.path), - "foo", - ] - ) - out = capsys.readouterr() # empty the buffer - assert age_keys[1].pubkey in out.out + with capture_output as output: + cli.run( + [ + "secrets", + what, + "get", + "--flake", + str(test_flake.path), + "foo", + ] + ) + assert age_keys[1].pubkey in output.out - capsys.readouterr() # empty the buffer - cli.run(["secrets", what, "list", "--flake", str(test_flake.path)]) - out = capsys.readouterr() # empty the buffer - assert "foo" in out.out + with capture_output as output: + cli.run(["secrets", what, "list", "--flake", str(test_flake.path)]) + assert "foo" in output.out cli.run(["secrets", what, "remove", "--flake", str(test_flake.path), "foo"]) assert not (sops_folder / what / "foo" / "key.json").exists() @@ -89,30 +88,29 @@ def _test_identities( with pytest.raises(ClanError): # already removed cli.run(["secrets", what, "remove", "--flake", str(test_flake.path), "foo"]) - capsys.readouterr() - cli.run(["secrets", what, "list", "--flake", str(test_flake.path)]) - out = capsys.readouterr() - assert "foo" not in out.out + with capture_output as output: + cli.run(["secrets", what, "list", "--flake", str(test_flake.path)]) + assert "foo" not in output.out def test_users( - test_flake: FlakeForTest, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: FlakeForTest, capture_output: CaptureOutput, age_keys: list["KeyPair"] ) -> None: - _test_identities("users", test_flake, capsys, age_keys) + _test_identities("users", test_flake, capture_output, age_keys) def test_machines( - test_flake: FlakeForTest, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: FlakeForTest, capture_output: CaptureOutput, age_keys: list["KeyPair"] ) -> None: - _test_identities("machines", test_flake, capsys, age_keys) + _test_identities("machines", test_flake, capture_output, age_keys) def test_groups( - test_flake: FlakeForTest, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: FlakeForTest, capture_output: CaptureOutput, age_keys: list["KeyPair"] ) -> None: - capsys.readouterr() # empty the buffer - cli.run(["secrets", "groups", "list", "--flake", str(test_flake.path)]) - assert capsys.readouterr().out == "" + with capture_output as output: + cli.run(["secrets", "groups", "list", "--flake", str(test_flake.path)]) + assert output.out == "" with pytest.raises(ClanError): # machine does not exist yet cli.run( @@ -197,9 +195,9 @@ def test_groups( ] ) - capsys.readouterr() # empty the buffer - cli.run(["secrets", "groups", "list", "--flake", str(test_flake.path)]) - out = capsys.readouterr().out + with capture_output as output: + cli.run(["secrets", "groups", "list", "--flake", str(test_flake.path)]) + out = output.out assert "user1" in out assert "machine1" in out @@ -243,20 +241,20 @@ def use_key(key: str, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: def test_secrets( test_flake: FlakeForTest, - capsys: pytest.CaptureFixture, + capture_output: CaptureOutput, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], ) -> None: - capsys.readouterr() # empty the buffer - cli.run(["secrets", "list", "--flake", str(test_flake.path)]) - assert capsys.readouterr().out == "" + with capture_output as output: + cli.run(["secrets", "list", "--flake", str(test_flake.path)]) + assert output.out == "" monkeypatch.setenv("SOPS_NIX_SECRET", "foo") monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(test_flake.path / ".." / "age.key")) cli.run(["secrets", "key", "generate", "--flake", str(test_flake.path)]) - capsys.readouterr() # empty the buffer - cli.run(["secrets", "key", "show", "--flake", str(test_flake.path)]) - key = capsys.readouterr().out + with capture_output as output: + cli.run(["secrets", "key", "show", "--flake", str(test_flake.path)]) + key = output.out assert key.startswith("age1") cli.run( ["secrets", "users", "add", "--flake", str(test_flake.path), "testuser", key] @@ -265,12 +263,12 @@ def test_secrets( with pytest.raises(ClanError): # does not exist yet cli.run(["secrets", "get", "--flake", str(test_flake.path), "nonexisting"]) cli.run(["secrets", "set", "--flake", str(test_flake.path), "initialkey"]) - capsys.readouterr() - cli.run(["secrets", "get", "--flake", str(test_flake.path), "initialkey"]) - assert capsys.readouterr().out == "foo" - capsys.readouterr() - cli.run(["secrets", "users", "list", "--flake", str(test_flake.path)]) - users = capsys.readouterr().out.rstrip().split("\n") + with capture_output as output: + cli.run(["secrets", "get", "--flake", str(test_flake.path), "initialkey"]) + assert output.out == "foo" + with capture_output as output: + cli.run(["secrets", "users", "list", "--flake", str(test_flake.path)]) + users = output.out.rstrip().split("\n") assert len(users) == 1, f"users: {users}" owner = users[0] @@ -280,17 +278,17 @@ def test_secrets( cli.run(["secrets", "rename", "--flake", str(test_flake.path), "initialkey", "key"]) - capsys.readouterr() # empty the buffer - cli.run(["secrets", "list", "--flake", str(test_flake.path)]) - assert capsys.readouterr().out == "key\n" + with capture_output as output: + cli.run(["secrets", "list", "--flake", str(test_flake.path)]) + assert output.out == "key\n" - capsys.readouterr() # empty the buffer - cli.run(["secrets", "list", "--flake", str(test_flake.path), "nonexisting"]) - assert capsys.readouterr().out == "" + with capture_output as output: + cli.run(["secrets", "list", "--flake", str(test_flake.path), "nonexisting"]) + assert output.out == "" - capsys.readouterr() # empty the buffer - cli.run(["secrets", "list", "--flake", str(test_flake.path), "key"]) - assert capsys.readouterr().out == "key\n" + with capture_output as output: + cli.run(["secrets", "list", "--flake", str(test_flake.path), "key"]) + assert output.out == "key\n" cli.run( [ @@ -314,15 +312,14 @@ def test_secrets( "key", ] ) - capsys.readouterr() - cli.run(["secrets", "machines", "list", "--flake", str(test_flake.path)]) - assert capsys.readouterr().out == "machine1\n" + with capture_output as output: + cli.run(["secrets", "machines", "list", "--flake", str(test_flake.path)]) + assert output.out == "machine1\n" with use_key(age_keys[1].privkey, monkeypatch): - capsys.readouterr() - cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) - - assert capsys.readouterr().out == "foo" + with capture_output as output: + cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) + assert output.out == "foo" # rotate machines key cli.run( @@ -340,10 +337,9 @@ def test_secrets( # should also rotate the encrypted secret with use_key(age_keys[0].privkey, monkeypatch): - capsys.readouterr() - cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) - - assert capsys.readouterr().out == "foo" + with capture_output as output: + cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) + assert output.out == "foo" cli.run( [ @@ -379,10 +375,9 @@ def test_secrets( "key", ] ) - capsys.readouterr() - with use_key(age_keys[1].privkey, monkeypatch): + with capture_output as output, use_key(age_keys[1].privkey, monkeypatch): cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) - assert capsys.readouterr().out == "foo" + assert output.out == "foo" cli.run( [ "secrets", @@ -441,7 +436,6 @@ def test_secrets( ] ) - capsys.readouterr() # empty the buffer cli.run( [ "secrets", @@ -455,9 +449,9 @@ def test_secrets( ) with use_key(age_keys[1].privkey, monkeypatch): - capsys.readouterr() - cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) - assert capsys.readouterr().out == "foo" + with capture_output as output: + cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) + assert output.out == "foo" # extend group will update secrets cli.run( @@ -484,9 +478,9 @@ def test_secrets( ) with use_key(age_keys[2].privkey, monkeypatch): # user2 - capsys.readouterr() - cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) - assert capsys.readouterr().out == "foo" + with capture_output as output: + cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) + assert output.out == "foo" cli.run( [ @@ -501,9 +495,9 @@ def test_secrets( ) with pytest.raises(ClanError), use_key(age_keys[2].privkey, monkeypatch): # user2 is not in the group anymore - capsys.readouterr() - cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) - print(capsys.readouterr().out) + with capture_output as output: + cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"]) + print(output.out) cli.run( [ @@ -520,6 +514,6 @@ def test_secrets( cli.run(["secrets", "remove", "--flake", str(test_flake.path), "key"]) cli.run(["secrets", "remove", "--flake", str(test_flake.path), "key2"]) - capsys.readouterr() # empty the buffer - cli.run(["secrets", "list", "--flake", str(test_flake.path)]) - assert capsys.readouterr().out == "" + with capture_output as output: + cli.run(["secrets", "list", "--flake", str(test_flake.path)]) + assert output.out == "" diff --git a/pkgs/clan-cli/tests/test_serializers.py b/pkgs/clan-cli/tests/test_serializers.py index fa6557d90..c0ce5b3a9 100644 --- a/pkgs/clan-cli/tests/test_serializers.py +++ b/pkgs/clan-cli/tests/test_serializers.py @@ -104,3 +104,23 @@ def test_dataclass_to_dict_defaults() -> None: "foo": {"home": {"a": "b"}, "work": ["a", "b"]}, } assert dataclass_to_dict(real_person) == expected + + +def test_filters_null_fields() -> None: + @dataclass + class Foo: + home: str | None = None + work: str | None = None + + # None fields are filtered out + instance = Foo() + + assert instance.home is None + assert dataclass_to_dict(instance) == {} + + # fields that are set are not filtered + instance = Foo(home="home") + + assert instance.home == "home" + assert instance.work is None + assert dataclass_to_dict(instance) == {"home": "home"} diff --git a/pkgs/clan-cli/tests/test_ssh_cli.py b/pkgs/clan-cli/tests/test_ssh_cli.py index 4b1aca3cb..604530d1a 100644 --- a/pkgs/clan-cli/tests/test_ssh_cli.py +++ b/pkgs/clan-cli/tests/test_ssh_cli.py @@ -4,19 +4,20 @@ import sys import pytest import pytest_subprocess.fake_process from pytest_subprocess import utils +from stdout import CaptureOutput import clan_cli from clan_cli.ssh import cli def test_no_args( - capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch + monkeypatch: pytest.MonkeyPatch, + capture_output: CaptureOutput, ) -> None: monkeypatch.setattr(sys, "argv", ["", "ssh"]) - with pytest.raises(SystemExit): + with capture_output as output, pytest.raises(SystemExit): clan_cli.main() - captured = capsys.readouterr() - assert captured.err.startswith("usage:") + assert output.err.startswith("usage:") # using fp fixture from pytest-subprocess diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py index fe1a03e0d..570d89f46 100644 --- a/pkgs/clan-cli/tests/test_vars.py +++ b/pkgs/clan-cli/tests/test_vars.py @@ -1,30 +1,20 @@ -import os import subprocess -from collections import defaultdict -from collections.abc import Callable from io import StringIO from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any import pytest -from age_keys import SopsSetup -from fixtures_flakes import generate_flake -from helpers import cli -from root import CLAN_CORE from clan_cli.clan_uri import FlakeId from clan_cli.machines.machines import Machine from clan_cli.nix import nix_shell +from clan_cli.vars.public_modules import in_repo from clan_cli.vars.secret_modules import password_store, sops - - -def def_value() -> defaultdict: - return defaultdict(def_value) - - -# allows defining nested dictionary in a single line -nested_dict: Callable[[], dict[str, Any]] = lambda: defaultdict(def_value) +from tests.age_keys import SopsSetup +from tests.fixtures_flakes import generate_flake +from tests.helpers import cli +from tests.helpers.nixos_config import nested_dict +from tests.root import CLAN_CORE def test_get_subgraph() -> None: @@ -89,11 +79,9 @@ def test_generate_public_var( ) monkeypatch.chdir(flake.path) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - var_file_path = ( - flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_value" - ) - assert var_file_path.is_file() - assert var_file_path.read_text() == "hello\n" + store = in_repo.FactStore(Machine(name="my_machine", flake=FlakeId(flake.path))) + assert store.exists("my_generator", "my_value") + assert store.get("my_generator", "my_value").decode() == "hello\n" @pytest.mark.impure @@ -102,7 +90,6 @@ def test_generate_secret_var_sops( temporary_home: Path, sops_setup: SopsSetup, ) -> None: - user = os.environ.get("USER", "user") config = nested_dict() my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] my_generator["files"]["my_secret"]["secret"] = True @@ -113,22 +100,12 @@ def test_generate_secret_var_sops( machine_configs=dict(my_machine=config), ) monkeypatch.chdir(flake.path) - cli.run( - [ - "secrets", - "users", - "add", - "--flake", - str(flake.path), - user, - sops_setup.keys[0].pubkey, - ] - ) + sops_setup.init() cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - var_file_path = ( - flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret" + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(flake.path)) ) - assert not var_file_path.is_file() + assert not in_repo_store.exists("my_generator", "my_secret") sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path))) assert sops_store.exists("my_generator", "my_secret") assert sops_store.get("my_generator", "my_secret").decode() == "hello\n" @@ -140,7 +117,6 @@ def test_generate_secret_var_sops_with_default_group( temporary_home: Path, sops_setup: SopsSetup, ) -> None: - user = os.environ.get("USER", "user") config = nested_dict() config["clan"]["core"]["sops"]["defaultGroups"] = ["my_group"] my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] @@ -152,34 +128,15 @@ def test_generate_secret_var_sops_with_default_group( machine_configs=dict(my_machine=config), ) monkeypatch.chdir(flake.path) - cli.run( - [ - "secrets", - "users", - "add", - "--flake", - str(flake.path), - user, - sops_setup.keys[0].pubkey, - ] - ) - cli.run(["secrets", "groups", "add-user", "my_group", user]) + sops_setup.init() + cli.run(["secrets", "groups", "add-user", "my_group", sops_setup.user]) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - assert not ( - flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret" - ).is_file() + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(flake.path)) + ) + assert not in_repo_store.exists("my_generator", "my_secret") sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path))) assert sops_store.exists("my_generator", "my_secret") - assert ( - flake.path - / "sops" - / "vars" - / "my_machine" - / "my_generator" - / "my_secret" - / "groups" - / "my_group" - ).exists() assert sops_store.get("my_generator", "my_secret").decode() == "hello\n" @@ -226,10 +183,6 @@ def test_generate_secret_var_password_store( nix_shell(["nixpkgs#pass"], ["pass", "init", "test@local"]), check=True ) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - var_file_path = ( - flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret" - ) - assert not var_file_path.is_file() store = password_store.SecretStore( Machine(name="my_machine", flake=FlakeId(flake.path)) ) @@ -243,7 +196,6 @@ def test_generate_secret_for_multiple_machines( temporary_home: Path, sops_setup: SopsSetup, ) -> None: - user = os.environ.get("USER", "user") machine1_config = nested_dict() machine1_generator = machine1_config["clan"]["core"]["vars"]["generators"][ "my_generator" @@ -268,29 +220,19 @@ def test_generate_secret_for_multiple_machines( machine_configs=dict(machine1=machine1_config, machine2=machine2_config), ) monkeypatch.chdir(flake.path) - cli.run( - [ - "secrets", - "users", - "add", - "--flake", - str(flake.path), - user, - sops_setup.keys[0].pubkey, - ] - ) + sops_setup.init() cli.run(["vars", "generate", "--flake", str(flake.path)]) # check if public vars have been created correctly - machine1_var_file_path = ( - flake.path / "machines" / "machine1" / "vars" / "my_generator" / "my_value" + in_repo_store1 = in_repo.FactStore( + Machine(name="machine1", flake=FlakeId(flake.path)) ) - machine2_var_file_path = ( - flake.path / "machines" / "machine2" / "vars" / "my_generator" / "my_value" + in_repo_store2 = in_repo.FactStore( + Machine(name="machine2", flake=FlakeId(flake.path)) ) - assert machine1_var_file_path.is_file() - assert machine1_var_file_path.read_text() == "machine1\n" - assert machine2_var_file_path.is_file() - assert machine2_var_file_path.read_text() == "machine2\n" + assert in_repo_store1.exists("my_generator", "my_value") + assert in_repo_store2.exists("my_generator", "my_value") + assert in_repo_store1.get("my_generator", "my_value").decode() == "machine1\n" + assert in_repo_store2.get("my_generator", "my_value").decode() == "machine2\n" # check if secret vars have been created correctly sops_store1 = sops.SecretStore(Machine(name="machine1", flake=FlakeId(flake.path))) sops_store2 = sops.SecretStore(Machine(name="machine2", flake=FlakeId(flake.path))) @@ -320,16 +262,13 @@ def test_dependant_generators( ) monkeypatch.chdir(flake.path) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - parent_file_path = ( - flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value" + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(flake.path)) ) - assert parent_file_path.is_file() - assert parent_file_path.read_text() == "hello\n" - child_file_path = ( - flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value" - ) - assert child_file_path.is_file() - assert child_file_path.read_text() == "hello\n" + assert in_repo_store.exists("parent_generator", "my_value") + assert in_repo_store.get("parent_generator", "my_value").decode() == "hello\n" + assert in_repo_store.exists("child_generator", "my_value") + assert in_repo_store.get("child_generator", "my_value").decode() == "hello\n" @pytest.mark.impure @@ -362,8 +301,55 @@ def test_prompt( monkeypatch.chdir(flake.path) monkeypatch.setattr("sys.stdin", StringIO(input_value)) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - var_file_path = ( - flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_value" + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(flake.path)) ) - assert var_file_path.is_file() - assert var_file_path.read_text() == input_value + assert in_repo_store.exists("my_generator", "my_value") + assert in_repo_store.get("my_generator", "my_value").decode() == input_value + + +@pytest.mark.impure +def test_share_flag( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, + sops_setup: SopsSetup, +) -> None: + config = nested_dict() + shared_generator = config["clan"]["core"]["vars"]["generators"]["shared_generator"] + shared_generator["files"]["my_secret"]["secret"] = True + shared_generator["files"]["my_value"]["secret"] = False + shared_generator["script"] = ( + "echo hello > $out/my_secret && echo hello > $out/my_value" + ) + shared_generator["share"] = True + unshared_generator = config["clan"]["core"]["vars"]["generators"][ + "unshared_generator" + ] + unshared_generator["files"]["my_secret"]["secret"] = True + unshared_generator["files"]["my_value"]["secret"] = False + unshared_generator["script"] = ( + "echo hello > $out/my_secret && echo hello > $out/my_value" + ) + unshared_generator["share"] = False + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs=dict(my_machine=config), + ) + monkeypatch.chdir(flake.path) + sops_setup.init() + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path))) + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(flake.path)) + ) + # check secrets stored correctly + assert sops_store.exists("shared_generator", "my_secret", shared=True) + assert not sops_store.exists("shared_generator", "my_secret", shared=False) + assert sops_store.exists("unshared_generator", "my_secret", shared=False) + assert not sops_store.exists("unshared_generator", "my_secret", shared=True) + # check values stored correctly + assert in_repo_store.exists("shared_generator", "my_value", shared=True) + assert not in_repo_store.exists("shared_generator", "my_value", shared=False) + assert in_repo_store.exists("unshared_generator", "my_value", shared=False) + assert not in_repo_store.exists("unshared_generator", "my_value", shared=True) diff --git a/pkgs/clan-cli/tests/test_vars_deployment.py b/pkgs/clan-cli/tests/test_vars_deployment.py new file mode 100644 index 000000000..2b2c361d7 --- /dev/null +++ b/pkgs/clan-cli/tests/test_vars_deployment.py @@ -0,0 +1,42 @@ +from pathlib import Path + +import pytest + +from tests.age_keys import SopsSetup +from tests.fixtures_flakes import generate_flake +from tests.helpers import cli +from tests.helpers.nixos_config import nested_dict +from tests.helpers.vms import qga_connect, run_vm_in_thread, wait_vm_down +from tests.root import CLAN_CORE + + +@pytest.mark.impure +def test_vm_deployment( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, + sops_setup: SopsSetup, +) -> None: + config = nested_dict() + config["clan"]["virtualisation"]["graphics"] = False + config["services"]["getty"]["autologinUser"] = "root" + config["services"]["openssh"]["enable"] = True + config["networking"]["firewall"]["enable"] = False + my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] + my_generator["files"]["my_secret"]["secret"] = True + my_generator["files"]["my_value"]["secret"] = False + my_generator["script"] = "echo hello > $out/my_secret && echo hello > $out/my_value" + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs=dict(my_machine=config), + ) + monkeypatch.chdir(flake.path) + sops_setup.init() + cli.run(["vars", "generate", "my_machine"]) + run_vm_in_thread("my_machine") + qga = qga_connect("my_machine") + qga.run("ls /run/secrets/my_machine/my_generator/my_secret", check=True) + _, out, _ = qga.run("cat /run/secrets/my_machine/my_generator/my_secret") + assert out == "hello\n" + qga.exec_cmd("poweroff") + wait_vm_down("my_machine") diff --git a/pkgs/clan-cli/tests/test_vms_cli.py b/pkgs/clan-cli/tests/test_vms_cli.py index 818da7496..3aeb7f78b 100644 --- a/pkgs/clan-cli/tests/test_vms_cli.py +++ b/pkgs/clan-cli/tests/test_vms_cli.py @@ -1,93 +1,29 @@ import os -import sys -import threading -import traceback from pathlib import Path -from time import sleep from typing import TYPE_CHECKING import pytest -from fixtures_flakes import FlakeForTest, generate_flake -from helpers import cli -from root import CLAN_CORE +from stdout import CaptureOutput -from clan_cli.dirs import vm_state_dir -from clan_cli.qemu.qga import QgaSession -from clan_cli.qemu.qmp import QEMUMonitorProtocol +from tests.fixtures_flakes import FlakeForTest, generate_flake +from tests.helpers import cli +from tests.helpers.nixos_config import nested_dict +from tests.helpers.vms import qga_connect, qmp_connect, run_vm_in_thread, wait_vm_down +from tests.root import CLAN_CORE if TYPE_CHECKING: - from age_keys import KeyPair + from tests.age_keys import KeyPair no_kvm = not os.path.exists("/dev/kvm") -def run_vm_in_thread(machine_name: str) -> None: - # runs machine and prints exceptions - def run() -> None: - try: - cli.run(["vms", "run", machine_name]) - except Exception: - # print exception details - print(traceback.format_exc(), file=sys.stderr) - print(sys.exc_info()[2], file=sys.stderr) - - # run the machine in a separate thread - t = threading.Thread(target=run, name="run") - t.daemon = True - t.start() - - -# wait for qmp socket to exist -def wait_vm_up(state_dir: Path) -> None: - socket_file = state_dir / "qga.sock" - timeout: float = 100 - while True: - if timeout <= 0: - raise TimeoutError( - f"qga socket {socket_file} not found. Is the VM running?" - ) - if socket_file.exists(): - break - sleep(0.1) - timeout -= 0.1 - - -# wait for vm to be down by checking if qga socket is down -def wait_vm_down(state_dir: Path) -> None: - socket_file = state_dir / "qga.sock" - timeout: float = 300 - while socket_file.exists(): - if timeout <= 0: - raise TimeoutError( - f"qga socket {socket_file} still exists. Is the VM down?" - ) - sleep(0.1) - timeout -= 0.1 - - -# wait for vm to be up then connect and return qmp instance -def qmp_connect(state_dir: Path) -> QEMUMonitorProtocol: - wait_vm_up(state_dir) - qmp = QEMUMonitorProtocol( - address=str(os.path.realpath(state_dir / "qmp.sock")), - ) - qmp.connect() - return qmp - - -# wait for vm to be up then connect and return qga instance -def qga_connect(state_dir: Path) -> QgaSession: - wait_vm_up(state_dir) - return QgaSession(os.path.realpath(state_dir / "qga.sock")) - - @pytest.mark.impure def test_inspect( - test_flake_with_core: FlakeForTest, capsys: pytest.CaptureFixture + test_flake_with_core: FlakeForTest, capture_output: CaptureOutput ) -> None: - cli.run(["vms", "inspect", "--flake", str(test_flake_with_core.path), "vm1"]) - out = capsys.readouterr() # empty the buffer - assert "Cores" in out.out + with capture_output as output: + cli.run(["vms", "inspect", "--flake", str(test_flake_with_core.path), "vm1"]) + assert "Cores" in output.out @pytest.mark.skipif(no_kvm, reason="Requires KVM") @@ -129,7 +65,7 @@ def test_vm_qmp( # set up a simple clan flake flake = generate_flake( temporary_home, - flake_template=CLAN_CORE / "templates" / "new-clan", + flake_template=CLAN_CORE / "templates" / "minimal", machine_configs=dict( my_machine=dict( clan=dict( @@ -144,14 +80,11 @@ def test_vm_qmp( # 'clan vms run' must be executed from within the flake monkeypatch.chdir(flake.path) - # the state dir is a point of reference for qemu interactions as it links to the qga/qmp sockets - state_dir = vm_state_dir(str(flake.path), "my_machine") - # start the VM run_vm_in_thread("my_machine") # connect with qmp - qmp = qmp_connect(state_dir) + qmp = qmp_connect("my_machine") # verify that issuing a command works # result = qmp.cmd_obj({"execute": "query-status"}) @@ -169,121 +102,60 @@ def test_vm_persistence( temporary_home: Path, ) -> None: # set up a clan flake with some systemd services to test persistence + config = nested_dict() + # logrotate-checkconf doesn't work in VM because /nix/store is owned by nobody + config["my_machine"]["systemd"]["services"]["logrotate-checkconf"]["enable"] = False + config["my_machine"]["services"]["getty"]["autologinUser"] = "root" + config["my_machine"]["clan"]["virtualisation"] = {"graphics": False} + config["my_machine"]["clan"]["networking"] = {"targetHost": "client"} + config["my_machine"]["clan"]["core"]["state"]["my_state"]["folders"] = [ + # to be owned by root + "/var/my-state", + # to be owned by user 'test' + "/var/user-state", + ] + config["my_machine"]["users"]["users"] = { + "test": {"password": "test", "isNormalUser": True}, + "root": {"password": "root"}, + } + flake = generate_flake( temporary_home, - flake_template=CLAN_CORE / "templates" / "new-clan", - machine_configs=dict( - my_machine=dict( - services=dict(getty=dict(autologinUser="root")), - clanCore=dict( - state=dict( - my_state=dict( - folders=[ - # to be owned by root - "/var/my-state", - # to be owned by user 'test' - "/var/user-state", - ] - ) - ) - ), - # create test user to test if state can be owned by user - users=dict( - users=dict( - test=dict( - password="test", - isNormalUser=True, - ), - root=dict(password="root"), - ) - ), - # create a systemd service to create a file in the state folder - # and another to read it after reboot - systemd=dict( - services=dict( - create_state=dict( - description="Create a file in the state folder", - wantedBy=["multi-user.target"], - script=""" - if [ ! -f /var/my-state/root ]; then - echo "Creating a file in the state folder" - echo "dream2nix" > /var/my-state/root - # create /var/my-state/test owned by user test - echo "dream2nix" > /var/my-state/test - chown test /var/my-state/test - # make sure /var/user-state is owned by test - chown test /var/user-state - fi - """, - serviceConfig=dict( - Type="oneshot", - ), - ), - reboot=dict( - description="Reboot the machine", - wantedBy=["multi-user.target"], - after=["my-state.service"], - script=""" - if [ ! -f /var/my-state/rebooting ]; then - echo "Rebooting the machine" - touch /var/my-state/rebooting - poweroff - else - touch /var/my-state/rebooted - fi - """, - ), - read_after_reboot=dict( - description="Read a file in the state folder", - wantedBy=["multi-user.target"], - after=["reboot.service"], - # TODO: currently state folders itself cannot be owned by users - script=""" - if ! cat /var/my-state/test; then - echo "cannot read from state file" > /var/my-state/error - # ensure root file is owned by root - elif [ "$(stat -c '%U' /var/my-state/root)" != "root" ]; then - echo "state file /var/my-state/root is not owned by user root" > /var/my-state/error - # ensure test file is owned by test - elif [ "$(stat -c '%U' /var/my-state/test)" != "test" ]; then - echo "state file /var/my-state/test is not owned by user test" > /var/my-state/error - # ensure /var/user-state is owned by test - elif [ "$(stat -c '%U' /var/user-state)" != "test" ]; then - echo "state folder /var/user-state is not owned by user test" > /var/my-state/error - fi - - """, - serviceConfig=dict( - Type="oneshot", - ), - ), - ) - ), - clan=dict( - virtualisation=dict(graphics=False), - networking=dict(targetHost="client"), - ), - ) - ), + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs=config, ) - monkeypatch.chdir(flake.path) - # the state dir is a point of reference for qemu interactions as it links to the qga/qmp sockets - state_dir = vm_state_dir(str(flake.path), "my_machine") + monkeypatch.chdir(flake.path) run_vm_in_thread("my_machine") - # wait for the VM to start - wait_vm_up(state_dir) + # wait for the VM to start and connect qga + qga = qga_connect("my_machine") + + # create state via qmp command instead of systemd service + qga.run("echo 'dream2nix' > /var/my-state/root", check=True) + qga.run("echo 'dream2nix' > /var/my-state/test", check=True) + qga.run("chown test /var/my-state/test", check=True) + qga.run("chown test /var/user-state", check=True) + qga.run("touch /var/my-state/rebooting", check=True) + qga.exec_cmd("poweroff") # wait for socket to be down (systemd service 'poweroff' rebooting machine) - wait_vm_down(state_dir) + wait_vm_down("my_machine") # start vm again run_vm_in_thread("my_machine") # connect second time - qga = qga_connect(state_dir) + qga = qga_connect("my_machine") + # check state exists + qga.run("cat /var/my-state/test", check=True) + # ensure root file is owned by root + qga.run("stat -c '%U' /var/my-state/root", check=True) + # ensure test file is owned by test + qga.run("stat -c '%U' /var/my-state/test", check=True) + # ensure /var/user-state is owned by test + qga.run("stat -c '%U' /var/user-state", check=True) # ensure that the file created by the service is still there and has the expected content exitcode, out, err = qga.run("cat /var/my-state/test") @@ -301,5 +173,5 @@ def test_vm_persistence( assert exitcode == 0, out # use qmp to shutdown the machine (prevent zombie qemu processes) - qmp = qmp_connect(state_dir) + qmp = qmp_connect("my_machine") qmp.command("system_powerdown") diff --git a/pkgs/clan-vm-manager/clan_vm_manager/assets/style.css b/pkgs/clan-vm-manager/clan_vm_manager/assets/style.css index c179744dd..772b9ad7c 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/assets/style.css +++ b/pkgs/clan-vm-manager/clan_vm_manager/assets/style.css @@ -1,66 +1,63 @@ /* Insert custom styles here */ navigation-view { - padding: 5px; - /* padding-left: 5px; + padding: 5px; + /* padding-left: 5px; padding-right: 5px; padding-bottom: 5px; */ } avatar { - margin: 2px; + margin: 2px; } .trust { - padding-top: 25px; - padding-bottom: 25px; + padding-top: 25px; + padding-bottom: 25px; } .join-list { - margin-top: 1px; - margin-left: 2px; - margin-right: 2px; - + margin-top: 1px; + margin-left: 2px; + margin-right: 2px; } .progress-bar { - margin-right: 25px; - min-width: 200px; + margin-right: 25px; + min-width: 200px; } .group-list { - background-color: inherit; + background-color: inherit; } .group-list > .activatable:hover { - background-color: unset; + background-color: unset; } .group-list > row { - margin-top: 12px; - border-bottom: unset; + margin-top: 12px; + border-bottom: unset; } - .vm-list { - margin-top: 25px; - margin-bottom: 25px; + margin-top: 25px; + margin-bottom: 25px; } .no-shadow { - box-shadow: none; + box-shadow: none; } .search-entry { - margin-bottom: 12px; + margin-bottom: 12px; } searchbar { - margin-bottom: 25px; + margin-bottom: 25px; } - .log-view { - margin-top: 12px; - font-family: monospace; - padding: 8px; + margin-top: 12px; + font-family: monospace; + padding: 8px; } diff --git a/pkgs/clan-vm-manager/clan_vm_manager/components/executor.py b/pkgs/clan-vm-manager/clan_vm_manager/components/executor.py index 7ca59d52f..b97013c50 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/components/executor.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/components/executor.py @@ -1,19 +1,14 @@ +import dataclasses import logging +import multiprocessing as mp import os import signal import sys import traceback +from collections.abc import Callable from pathlib import Path from typing import Any -import gi - -gi.require_version("GdkPixbuf", "2.0") - -import dataclasses -import multiprocessing as mp -from collections.abc import Callable - log = logging.getLogger(__name__) diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/list.py b/pkgs/clan-vm-manager/clan_vm_manager/views/list.py index 65444fb4c..95c37eb05 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/list.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/list.py @@ -113,7 +113,7 @@ class ClanList(Gtk.Box): # menu_model = Gio.Menu() # TODO: Make this lazy, blocks UI startup for too long - # for vm in machines.list.list_machines(flake_url=vm.data.flake.flake_url): + # for vm in machines.list.list_nixos_machines(flake_url=vm.data.flake.flake_url): # if vm not in vm_store: # menu_model.append(vm, f"app.add::{vm}") diff --git a/pkgs/classgen/main.py b/pkgs/classgen/main.py index 8cafe0b6c..c698c480a 100644 --- a/pkgs/classgen/main.py +++ b/pkgs/classgen/main.py @@ -165,14 +165,18 @@ def get_field_def( if not default and not default_factory and not field_meta: return f"{field_name}: {serialised_types}" field_init = "field(" - if default: - field_init += f"default = {default}" - if default_factory: - field_init += f"default_factory = {default_factory}" - if field_meta: - field_init += f", metadata = {field_meta}" - return f"{field_name}: {serialised_types} = {field_init})" + init_args = [] + if default: + init_args.append(f"default = {default}") + if default_factory: + init_args.append(f"default_factory = {default_factory}") + if field_meta: + init_args.append(f"metadata = {field_meta}") + + field_init += ", ".join(init_args) + ")" + + return f"{field_name}: {serialised_types} = {field_init}" # Recursive function to generate dataclasses from JSON schema @@ -223,8 +227,11 @@ def generate_dataclass(schema: dict[str, Any], class_name: str = root_class) -> known_classes.add(nested_class_name) elif inner_type and inner_type.get("type") != "object": - # Trivial type - field_types = map_json_type(inner_type) + # Trivial type: + # dict[str, inner_type] + field_types = { + f"""dict[str, {" | ".join(map_json_type(inner_type))}]""" + } elif not inner_type: # The type is a class diff --git a/pkgs/flake-module.nix b/pkgs/flake-module.nix index 1909c9c3e..67c8efb6a 100644 --- a/pkgs/flake-module.nix +++ b/pkgs/flake-module.nix @@ -1,4 +1,4 @@ -{ ... }: +{ inputs, ... }: { imports = [ @@ -11,32 +11,27 @@ ./distro-packages/flake-module.nix ]; + flake.packages.x86_64-linux = + let + pkgs = inputs.nixpkgs.legacyPackages.x86_64-linux; + in + { + yagna = pkgs.callPackage ./yagna { }; + }; + perSystem = + { pkgs, config, ... }: { - pkgs, - config, - lib, - ... - }: - { - packages = - { - tea-create-pr = pkgs.callPackage ./tea-create-pr { }; - zerotier-members = pkgs.callPackage ./zerotier-members { }; - zt-tcp-relay = pkgs.callPackage ./zt-tcp-relay { }; - moonlight-sunshine-accept = pkgs.callPackage ./moonlight-sunshine-accept { }; - merge-after-ci = pkgs.callPackage ./merge-after-ci { inherit (config.packages) tea-create-pr; }; - pending-reviews = pkgs.callPackage ./pending-reviews { }; - editor = pkgs.callPackage ./editor/clan-edit-codium.nix { }; - classgen = pkgs.callPackage ./classgen { }; - } - // lib.optionalAttrs pkgs.stdenv.isLinux { - # halalify zerotierone - zerotierone = pkgs.zerotierone.overrideAttrs (_old: { - meta = _old.meta // { - license = lib.licenses.apsl20; - }; - }); - }; + packages = { + tea-create-pr = pkgs.callPackage ./tea-create-pr { }; + zerotier-members = pkgs.callPackage ./zerotier-members { }; + zt-tcp-relay = pkgs.callPackage ./zt-tcp-relay { }; + moonlight-sunshine-accept = pkgs.callPackage ./moonlight-sunshine-accept { }; + merge-after-ci = pkgs.callPackage ./merge-after-ci { inherit (config.packages) tea-create-pr; }; + pending-reviews = pkgs.callPackage ./pending-reviews { }; + editor = pkgs.callPackage ./editor/clan-edit-codium.nix { }; + classgen = pkgs.callPackage ./classgen { }; + zerotierone = pkgs.callPackage ./zerotierone { }; + }; }; } diff --git a/pkgs/installer/base64.nix b/pkgs/installer/base64.nix new file mode 100644 index 000000000..588d1dfd1 --- /dev/null +++ b/pkgs/installer/base64.nix @@ -0,0 +1,60 @@ +{ lib, ... }: +{ + toBase64 = + text: + let + inherit (lib) + sublist + mod + stringToCharacters + concatMapStrings + ; + inherit (lib.strings) charToInt; + inherit (builtins) + substring + foldl' + genList + elemAt + length + concatStringsSep + stringLength + ; + lookup = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + sliceN = + size: list: n: + sublist (n * size) size list; + pows = [ + (64 * 64 * 64) + (64 * 64) + 64 + 1 + ]; + intSextets = i: map (j: mod (i / j) 64) pows; + compose = + f: g: x: + f (g x); + intToChar = elemAt lookup; + convertTripletInt = sliceInt: concatMapStrings intToChar (intSextets sliceInt); + sliceToInt = foldl' (acc: val: acc * 256 + val) 0; + convertTriplet = compose convertTripletInt sliceToInt; + join = concatStringsSep ""; + convertLastSlice = + slice: + let + len = length slice; + in + if len == 1 then + (substring 0 2 (convertTripletInt ((sliceToInt slice) * 256 * 256))) + "==" + else if len == 2 then + (substring 0 3 (convertTripletInt ((sliceToInt slice) * 256))) + "=" + else + ""; + len = stringLength text; + nFullSlices = len / 3; + bytes = map charToInt (stringToCharacters text); + tripletAt = sliceN 3 bytes; + head = genList (compose convertTriplet tripletAt) nFullSlices; + tail = convertLastSlice (tripletAt nFullSlices); + in + join (head ++ [ tail ]); +} diff --git a/pkgs/installer/flake-module.nix b/pkgs/installer/flake-module.nix index b00c22a79..29c25e370 100644 --- a/pkgs/installer/flake-module.nix +++ b/pkgs/installer/flake-module.nix @@ -1,32 +1,15 @@ { self, lib, ... }: let - wifiModule = - { ... }: - { - # use iwd instead of wpa_supplicant - networking.wireless.enable = false; - - # Use iwd instead of wpa_supplicant. It has a user friendly CLI - networking.wireless.iwd = { - enable = true; - settings = { - Network = { - EnableIPv6 = true; - RoutePriorityOffset = 300; - }; - Settings.AutoConnect = true; - }; - }; - }; flashInstallerModule = { config, ... }: { imports = [ - wifiModule + ./iwd.nix self.nixosModules.installer ]; + system.stateVersion = config.system.nixos.version; nixpkgs.pkgs = self.inputs.nixpkgs.legacyPackages.x86_64-linux; } diff --git a/pkgs/installer/iwd.nix b/pkgs/installer/iwd.nix new file mode 100644 index 000000000..8717a1c38 --- /dev/null +++ b/pkgs/installer/iwd.nix @@ -0,0 +1,67 @@ +{ + lib, + pkgs, + config, + ... +}: + +let + cfg = config.clan.iwd; + toBase64 = (pkgs.callPackage ./base64.nix { inherit lib; }).toBase64; + wifi_config = password: '' + [Security] + Passphrase=${password} + ''; +in +{ + options.clan.iwd = { + networks = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + ssid = lib.mkOption { + type = lib.types.strMatching "^[a-zA-Z0-9._-]+$"; + default = name; + description = "The name of the wifi network"; + }; + password = lib.mkOption { + type = lib.types.str; + description = "The password of the wifi network"; + }; + }; + } + ) + ); + default = { }; + description = "Wifi networks to predefine"; + }; + }; + config = lib.mkMerge [ + (lib.mkIf (cfg.networks != { }) { + # Systemd tmpfiles rule to create /var/lib/iwd/example.psk file + systemd.tmpfiles.rules = lib.mapAttrsToList ( + _: value: + "f+~ /var/lib/iwd/${value.ssid}.psk 0600 root root - ${toBase64 (wifi_config value.password)}" + ) cfg.networks; + + }) + { + # disable wpa supplicant + networking.wireless.enable = false; + + # Use iwd instead of wpa_supplicant. It has a user friendly CLI + networking.wireless.iwd = { + enable = true; + settings = { + Network = { + EnableIPv6 = true; + RoutePriorityOffset = 300; + }; + Settings.AutoConnect = true; + }; + }; + } + ]; +} diff --git a/pkgs/tea-create-pr/script.sh b/pkgs/tea-create-pr/script.sh index 3b4fbcf8e..87c7c219a 100755 --- a/pkgs/tea-create-pr/script.sh +++ b/pkgs/tea-create-pr/script.sh @@ -10,7 +10,6 @@ currentBranch="$(git rev-parse --abbrev-ref HEAD)" user_unparsed="$(tea whoami)" user="$(echo "$user_unparsed" | tr -d '\n' | cut -f4 -d' ')" tempRemoteBranch="$user-$currentBranch" -root_dir=$(git rev-parse --show-toplevel) # Function to check if a remote exists @@ -30,7 +29,7 @@ if ! check_remote "$remoteUpstream"; then exit 1 fi -treefmt --no-cache --fail-on-change -C "$root_dir" +nix fmt -- --fail-on-change --no-cache upstream_url=$(git remote get-url "$remoteUpstream") set -x @@ -42,7 +41,7 @@ repo=$(echo "$upstream_url" | sed -E 's#.*:([^/]+/[^.]+)\.git#\1#') git log --reverse --pretty="format:%s%n%n%b%n%n" "$remoteUpstream/$targetBranch..HEAD" > "$TMPDIR"/commit-msg -$EDITOR "$TMPDIR"/commit-msg +"$EDITOR" "$TMPDIR"/commit-msg COMMIT_MSG=$(cat "$TMPDIR"/commit-msg) diff --git a/pkgs/webview-ui/.vscode/settings.json b/pkgs/webview-ui/.vscode/settings.json index 335c251a2..f9c1a6bc2 100644 --- a/pkgs/webview-ui/.vscode/settings.json +++ b/pkgs/webview-ui/.vscode/settings.json @@ -1,5 +1,7 @@ { - "typescript.tsdk": "node_modules/typescript/lib", - "tailwindCSS.experimental.classRegex": [["cx\\(([^)]*)\\)", "[\"'`]([^\"'`]*)[\"'`]"]], - "editor.wordWrap": "on" + "typescript.tsdk": "node_modules/typescript/lib", + "tailwindCSS.experimental.classRegex": [ + ["cx\\(([^)]*)\\)", "[\"'`]([^\"'`]*)[\"'`]"] + ], + "editor.wordWrap": "on" } diff --git a/pkgs/webview-ui/app/README.md b/pkgs/webview-ui/app/README.md index 6a1764536..f171a99cd 100644 --- a/pkgs/webview-ui/app/README.md +++ b/pkgs/webview-ui/app/README.md @@ -1,8 +1,10 @@ ## Usage -Those templates dependencies are maintained via [pnpm](https://pnpm.io) via `pnpm up -Lri`. +Those templates dependencies are maintained via [pnpm](https://pnpm.io) via +`pnpm up -Lri`. -This is the reason you see a `pnpm-lock.yaml`. That being said, any package manager will work. This file can be safely be removed once you clone a template. +This is the reason you see a `pnpm-lock.yaml`. That being said, any package +manager will work. This file can be safely be removed once you clone a template. ```bash $ npm install # or pnpm install or yarn install @@ -16,19 +18,20 @@ In the project directory, you can run: ### `npm run dev` or `npm start` -Runs the app in the development mode.
-Open [http://localhost:3000](http://localhost:3000) to view it in the browser. +Runs the app in the development mode.
Open +[http://localhost:3000](http://localhost:3000) to view it in the browser. The page will reload if you make edits.
### `npm run build` -Builds the app for production to the `dist` folder.
-It correctly bundles Solid in production mode and optimizes the build for the best performance. +Builds the app for production to the `dist` folder.
It correctly bundles +Solid in production mode and optimizes the build for the best performance. -The build is minified and the filenames include the hashes.
-Your app is ready to be deployed! +The build is minified and the filenames include the hashes.
Your app is +ready to be deployed! ## Deployment -You can deploy the `dist` folder to any static host provider (netlify, surge, now, etc.) +You can deploy the `dist` folder to any static host provider (netlify, surge, +now, etc.) diff --git a/pkgs/webview-ui/app/eslint.config.mjs b/pkgs/webview-ui/app/eslint.config.mjs index 619db917b..858cc8fec 100644 --- a/pkgs/webview-ui/app/eslint.config.mjs +++ b/pkgs/webview-ui/app/eslint.config.mjs @@ -28,5 +28,5 @@ export default tseslint.config( "no-unused-vars": "off", "@typescript-eslint/no-unused-vars": "off", }, - } + }, ); diff --git a/pkgs/webview-ui/app/gtk.webview.js b/pkgs/webview-ui/app/gtk.webview.js index 522d93cc6..a768e1b5e 100644 --- a/pkgs/webview-ui/app/gtk.webview.js +++ b/pkgs/webview-ui/app/gtk.webview.js @@ -43,7 +43,7 @@ fs.readFile(manifestPath, { encoding: "utf8" }, (err, data) => { console.log(`Rewriting CSS url(): ${asset.url} to ${res}`); return res; }, - }) + }), ) .process(css, { from: `dist/${cssEntry}`, diff --git a/pkgs/webview-ui/app/index.html b/pkgs/webview-ui/app/index.html index 68f32847f..0c649c3fb 100644 --- a/pkgs/webview-ui/app/index.html +++ b/pkgs/webview-ui/app/index.html @@ -1,4 +1,4 @@ - + Solid App diff --git a/pkgs/webview-ui/app/package-lock.json b/pkgs/webview-ui/app/package-lock.json index 5f435d99c..317efb017 100644 --- a/pkgs/webview-ui/app/package-lock.json +++ b/pkgs/webview-ui/app/package-lock.json @@ -12,6 +12,7 @@ "@floating-ui/dom": "^1.6.8", "@modular-forms/solid": "^0.21.0", "@solid-primitives/storage": "^3.7.1", + "@solidjs/router": "^0.14.2", "@tanstack/eslint-plugin-query": "^5.51.12", "@tanstack/solid-query": "^5.51.2", "material-icons": "^1.13.12", @@ -1551,6 +1552,14 @@ "solid-js": "^1.6.12" } }, + "node_modules/@solidjs/router": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/@solidjs/router/-/router-0.14.2.tgz", + "integrity": "sha512-JaJe7XJcZTyOfMOIVHmLO+3wP3akm5QQesrDU4XLn/JRMxozBzCaNXBsK7F8pBuDgxzRRxTV8RvXeS09HGXv6Q==", + "peerDependencies": { + "solid-js": "^1.8.6" + } + }, "node_modules/@tailwindcss/typography": { "version": "0.5.13", "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.13.tgz", diff --git a/pkgs/webview-ui/app/package.json b/pkgs/webview-ui/app/package.json index 356215a3e..8681e2830 100644 --- a/pkgs/webview-ui/app/package.json +++ b/pkgs/webview-ui/app/package.json @@ -41,6 +41,7 @@ "@floating-ui/dom": "^1.6.8", "@modular-forms/solid": "^0.21.0", "@solid-primitives/storage": "^3.7.1", + "@solidjs/router": "^0.14.2", "@tanstack/eslint-plugin-query": "^5.51.12", "@tanstack/solid-query": "^5.51.2", "material-icons": "^1.13.12", diff --git a/pkgs/webview-ui/app/src/App.tsx b/pkgs/webview-ui/app/src/App.tsx index 016cff09f..d5ad8b40e 100644 --- a/pkgs/webview-ui/app/src/App.tsx +++ b/pkgs/webview-ui/app/src/App.tsx @@ -1,24 +1,12 @@ -import { createEffect, createSignal, type Component } from "solid-js"; -import { Layout } from "./layout/layout"; -import { Route, Router } from "./Routes"; -import { Toaster } from "solid-toast"; -import { effect } from "solid-js/web"; +import { createSignal } from "solid-js"; import { makePersisted } from "@solid-primitives/storage"; -// Some global state -const [route, setRoute] = createSignal("machines"); -createEffect(() => { - console.log(route()); -}); - -export { route, setRoute }; - const [activeURI, setActiveURI] = makePersisted( createSignal(null), { name: "activeURI", storage: localStorage, - } + }, ); export { activeURI, setActiveURI }; @@ -29,19 +17,3 @@ const [clanList, setClanList] = makePersisted(createSignal([]), { }); export { clanList, setClanList }; - -const App: Component = () => { - effect(() => { - if (clanList().length === 0) { - setRoute("welcome"); - } - }); - return [ - , - - - , - ]; -}; - -export default App; diff --git a/pkgs/webview-ui/app/src/Routes.tsx b/pkgs/webview-ui/app/src/Routes.tsx deleted file mode 100644 index a9a618fff..000000000 --- a/pkgs/webview-ui/app/src/Routes.tsx +++ /dev/null @@ -1,86 +0,0 @@ -import { Accessor, For, Match, Switch } from "solid-js"; -import { MachineListView } from "./routes/machines/view"; -import { colors } from "./routes/colors/view"; -import { CreateClan } from "./routes/clan/view"; -import { HostList } from "./routes/hosts/view"; -import { BlockDevicesView } from "./routes/blockdevices/view"; -import { Flash } from "./routes/flash/view"; -import { Settings } from "./routes/settings"; -import { Welcome } from "./routes/welcome"; -import { Deploy } from "./routes/deploy"; -import { CreateMachine } from "./routes/machines/create"; -import { DiskView } from "./routes/disk/view"; - -export type Route = keyof typeof routes; - -export const routes = { - createClan: { - child: CreateClan, - label: "Create Clan", - icon: "groups", - }, - machines: { - child: MachineListView, - label: "Machines", - icon: "devices_other", - }, - "machines/add": { - child: CreateMachine, - label: "create Machine", - icon: "add", - }, - hosts: { - child: HostList, - label: "hosts", - icon: "devices_other", - }, - flash: { - child: Flash, - label: "create_flash_installer", - icon: "devices_other", - }, - blockdevices: { - child: BlockDevicesView, - label: "blockdevices", - icon: "devices_other", - }, - colors: { - child: colors, - label: "Colors", - icon: "color_lens", - }, - settings: { - child: Settings, - label: "Settings", - icon: "settings", - }, - welcome: { - child: Welcome, - label: "welcome", - icon: "settings", - }, - deploy: { - child: Deploy, - label: "deploy", - icon: "content_copy", - }, - diskConfig: { - child: DiskView, - label: "diskConfig", - icon: "disk", - }, -}; - -interface RouterProps { - route: Accessor; -} -export const Router = (props: RouterProps) => { - const { route } = props; - return ( - route {route()} not found

}> - - {([key, { child }]) => {child}} - - - ); -}; diff --git a/pkgs/webview-ui/app/src/Sidebar.tsx b/pkgs/webview-ui/app/src/Sidebar.tsx index 28deabea1..cb9bc3f55 100644 --- a/pkgs/webview-ui/app/src/Sidebar.tsx +++ b/pkgs/webview-ui/app/src/Sidebar.tsx @@ -1,33 +1,73 @@ -import { Accessor, For, Setter } from "solid-js"; -import { Route, routes } from "./Routes"; +import { For, Show } from "solid-js"; +import { activeURI } from "./App"; +import { createQuery } from "@tanstack/solid-query"; +import { callApi } from "./api"; +import { A, RouteSectionProps } from "@solidjs/router"; +import { AppRoute, routes } from "./index"; + +export const Sidebar = (props: RouteSectionProps) => { + const query = createQuery(() => ({ + queryKey: [activeURI(), "meta"], + queryFn: async () => { + const curr = activeURI(); + if (curr) { + const result = await callApi("show_clan_meta", { uri: curr }); + if (result.status === "error") throw new Error("Failed to fetch data"); + return result.data; + } + }, + })); -interface SidebarProps { - route: Accessor; - setRoute: Setter; -} -export const Sidebar = (props: SidebarProps) => { - const { route, setRoute } = props; return ( -