Merge pull request 'clan machines generations' (#4848) from Qubasa/clan-core:add_generate_cli into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4848

clanServices/monitoring/default.nix

@@ -10,15 +10,15 @@
       { lib, ... }:
       {
         options.allowAllInterfaces = lib.mkOption {
-          type = lib.types.bool;
-          default = false;
-          description = "If true, Telegraf will listen on all interfaces. Otherwise, it will only listen on the interfaces specified in `interfaces`";
+          type = lib.types.nullOr lib.types.bool;
+          default = null;
+          description = "Deprecated. Has no effect.";
         };
 
         options.interfaces = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
-          default = [ "zt+" ];
-          description = "List of interfaces to expose the metrics to";
+          type = lib.types.nullOr (lib.types.listOf lib.types.str);
+          default = null;
+          description = "Deprecated. Has no effect.";
         };
       };
   };

clanServices/monitoring/telegraf.nix

@@ -14,30 +14,51 @@
           auth_user = "prometheus";
         in
         {
+          warnings =
+            lib.optionals (settings.allowAllInterfaces != null) [
+              "monitoring.settings.allowAllInterfaces is deprecated and and has no effect. Please remove it from your inventory."
+              "The monitoring service will now always listen on all interfaces over https."
+            ]
+            ++ (lib.optionals (settings.interfaces != null) [
+              "monitoring.settings.interfaces is deprecated and and has no effect. Please remove it from your inventory."
+              "The monitoring service will now always listen on all interfaces over https."
+            ]);
 
-          networking.firewall.interfaces = lib.mkIf (settings.allowAllInterfaces == false) (
-            builtins.listToAttrs (
-              map (name: {
-                inherit name;
-                value.allowedTCPPorts = [
-                  9273
-                  9990
-                ];
-              }) settings.interfaces
-            )
-          );
-
-          networking.firewall.allowedTCPPorts = lib.mkIf (settings.allowAllInterfaces == true) [
+          networking.firewall.allowedTCPPorts = [
             9273
             9990
           ];
 
-          clan.core.vars.generators."telegraf" = {
+          clan.core.vars.generators."telegraf-certs" = {
+            files.crt = {
+              restartUnits = [ "telegraf.service" ];
+              deploy = true;
+              secret = false;
+            };
+            files.key = {
+              mode = "0600";
+              restartUnits = [ "telegraf.service" ];
+            };
 
+            runtimeInputs = [
+              pkgs.openssl
+            ];
+
+            script = ''
+              openssl req -x509 -nodes -newkey rsa:4096 \
+                -keyout "$out"/key \
+                -out "$out"/crt \
+                -subj "/C=US/ST=CA/L=San Francisco/O=Example Corp/OU=IT/CN=example.com"
+            '';
+          };
+
+          clan.core.vars.generators."telegraf" = {
             files.password.restartUnits = [ "telegraf.service" ];
             files.password-env.restartUnits = [ "telegraf.service" ];
             files.miniserve-auth.restartUnits = [ "telegraf.service" ];
 
+            dependencies = [ "telegraf-certs" ];
+
             runtimeInputs = [
               pkgs.coreutils
               pkgs.xkcdpass
@@ -60,16 +81,33 @@
             serviceConfig = {
               LoadCredential = [
                 "auth_file_path:${config.clan.core.vars.generators.telegraf.files.miniserve-auth.path}"
+                "telegraf_crt_path:${config.clan.core.vars.generators.telegraf-certs.files.crt.path}"
+                "telegraf_key_path:${config.clan.core.vars.generators.telegraf-certs.files.key.path}"
               ];
               Environment = [
                 "AUTH_FILE_PATH=%d/auth_file_path"
+                "CRT_PATH=%d/telegraf_crt_path"
+                "KEY_PATH=%d/telegraf_key_path"
               ];
               Restart = "on-failure";
               User = "telegraf";
               Group = "telegraf";
               RuntimeDirectory = "telegraf-www";
             };
-            script = "${pkgs.miniserve}/bin/miniserve -p 9990 /run/telegraf-www --auth-file \"$AUTH_FILE_PATH\"";
+            script = "${pkgs.miniserve}/bin/miniserve -p 9990 /run/telegraf-www --auth-file \"$AUTH_FILE_PATH\" --tls-cert \"$CRT_PATH\" --tls-key \"$KEY_PATH\"";
+          };
+
+          systemd.services.telegraf = {
+            serviceConfig = {
+              LoadCredential = [
+                "telegraf_crt_path:${config.clan.core.vars.generators.telegraf-certs.files.crt.path}"
+                "telegraf_key_path:${config.clan.core.vars.generators.telegraf-certs.files.key.path}"
+              ];
+              Environment = [
+                "CRT_PATH=%d/telegraf_crt_path"
+                "KEY_PATH=%d/telegraf_key_path"
+              ];
+            };
           };
 
           services.telegraf = {
@@ -77,6 +115,7 @@
             environmentFiles = [
               (builtins.toString config.clan.core.vars.generators.telegraf.files.password-env.path)
             ];
+
             extraConfig = {
               agent.interval = "60s";
               inputs = {
@@ -112,6 +151,8 @@
                 metric_version = 2;
                 basic_username = "${auth_user}";
                 basic_password = "$${BASIC_AUTH_PWD}";
+                tls_cert = "$${CRT_PATH}";
+                tls_key = "$${KEY_PATH}";
               };
 
               outputs.file = {

clanServices/monitoring/tests/vm/default.nix

@@ -1,4 +1,4 @@
-{ packages, pkgs, ... }:
+{ ... }:
 {
   name = "monitoring";
 
@@ -14,9 +14,6 @@
         module.input = "self";
 
         roles.telegraf.machines.peer1 = { };
-        roles.telegraf.settings = {
-          allowAllInterfaces = true;
-        };
       };
     };
   };
@@ -28,6 +25,8 @@
         services.telegraf.extraConfig = {
           agent.interval = lib.mkForce "1s";
           outputs.prometheus_client = {
+            # BUG: We have to disable basic auth here because the prometheus_client
+            # output plugin will otherwise deadlock Telegraf on startup.
             basic_password = lib.mkForce "";
             basic_username = lib.mkForce "";
           };
@@ -35,17 +34,16 @@
       };
   };
 
-  extraPythonPackages = _p: [
-    (pkgs.python3.pkgs.toPythonModule packages.${pkgs.system}.clan-cli)
-  ];
-
+  # !!! ANY CHANGES HERE MUST BE REFLECTED IN:
+  # clan_lib/metrics/telegraf.py::get_metrics
   testScript =
-    { ... }:
+    { nodes, ... }:
     ''
       import time
       import os
       import sys
       import subprocess 
+      import ssl
       import json
       import shlex
       import urllib.request
@@ -54,45 +52,44 @@
 
       peer1.wait_for_unit("network-online.target")
       peer1.wait_for_unit("telegraf.service")
-
       peer1.wait_for_unit("telegraf-json.service")
-      peer1.succeed("curl http://localhost:9990/telegraf.json")
-      peer1.succeed("curl http://localhost:9273/metrics")
 
       # Fetch the basic auth password from the secret file
-      password = peer1.succeed("cat /var/run/secrets/vars/telegraf/password")
-      url = f"http://192.168.1.1:9990/telegraf.json"
+      password = peer1.succeed("cat ${nodes.peer1.clan.core.vars.generators.telegraf.files.password.path}").strip()
       credentials = f"prometheus:{password}"
-      print("Using credentials:", credentials)
-      time.sleep(10)  # wait a bit for telegraf to collect some data
 
-      # Fetch the json output from miniserve
+      print("Using credentials:", credentials)
+      peer1.succeed(f"curl -k -u {credentials}  https://localhost:9990/telegraf.json")
+      peer1.succeed(f"curl -k -u {credentials}  https://localhost:9273/metrics")
+
+      cert_path = "${nodes.peer1.clan.core.vars.generators.telegraf-certs.files.crt.path}"
+      url = "https://192.168.1.1:9990/telegraf.json"  # HTTPS required
+
+      print("Waiting for /var/run/telegraf-www/telegraf.json to be bigger then 200 bytes")
+      peer1.wait_until_succeeds(f"test \"$(stat -c%s /var/run/telegraf-www/telegraf.json)\" -ge 200", timeout=30)
+
       encoded_credentials = b64encode(credentials.encode("utf-8")).decode("utf-8")
       headers = {"Authorization": f"Basic {encoded_credentials}"}
       req = urllib.request.Request(url, headers=headers)  # noqa: S310
-      response = urllib.request.urlopen(req)
 
-      # Look for the nixos_systems metric in the json output
+      # Trust the provided CA/server certificate
+      context = ssl.create_default_context(cafile=cert_path)
+      context.check_hostname = False
+      context.verify_mode = ssl.CERT_REQUIRED
+
       found_system = False
-      for line in response:
-          line_str = line.decode("utf-8").strip()
-          line = json.loads(line_str)
-          if line["name"] == "nixos_systems":
-              found_system = True
-              print("Found nixos_systems metric in json output")
-              break
-      assert found_system, "nixos_systems metric not found in json output"
+      with urllib.request.urlopen(req, context=context, timeout=5) as response:
+          for raw_line in response:
+              line_str = raw_line.decode("utf-8").strip()
+              if not line_str:
+                  continue
+              obj = json.loads(line_str)
+              if obj.get("name") == "nixos_systems":
+                  found_system = True
+                  print("Found nixos_systems metric in json output")
+                  break
 
-      # TODO: I would like to test the python code here but it's not working yet
-      # Missing: I need a way to get the encrypted var from the clan
-      #from clan_lib.metrics.version import get_nixos_systems
-      #from clan_lib.machines.machines import Machine as ClanMachine
-      #from clan_lib.flake import Flake
-      #from clan_lib.ssh.remote import Remote
-      #target_host = Remote("peer1", "192.168.1.1")
-      #machine = ClanMachine("peer1", flake=Flake("${./.}"))
-      # data = get_nixos_systems(mymachine, target_host)
-      # assert data["current_system"] is not None
+      assert found_system, "nixos_systems metric not found in json output"
 
     '';
 }

clanServices/monitoring/tests/vm/sops/machines/peer1/key.json

@@ -1,6 +1,6 @@
 [
   {
-    "publickey": "age18vspwr3de7jp0awyu66kk9psd5x4sy9suv0zt7ux2kqw0s6h2ueqwkgjxm",
+    "publickey": "age1ntpf7lqqw4zrk8swjvwtyfak7f2wg04uf7ggu6vk2yyt9qt74qkswn25ck",
     "type": "age"
   }
 ]

clanServices/monitoring/tests/vm/sops/secrets/peer1-age.key/secret

@@ -1,14 +1,14 @@
 {
-	"data": "ENC[AES256_GCM,data:raon/RshBCUAHgav5phuv5ZrXQDed03F/ZdjW6iUyj3UJJ/mYdkSdBmmYwhFh32Yluqyy9bUajHoEgDQqV27IE567o9M5YqmnVw=,iv:rpyv2/fAAg77JLyOUeWGkTmd8TYIrrb8pS89/AjSc+4=,tag:WA+UaqrhEhcbg6K21JgAsA==,type:str]",
+	"data": "ENC[AES256_GCM,data:ACFpRJRDIgVPurZwHYW0J1MnvyuiRGnXMeQj1nb9rDAIqHbZzZk8+E0Nu1+EdXwk78ziP6tHR1GQP2ILTtpLME4lXXRVjouW5Eo=,iv:ctR1HENO3XGIq1/gzYi47nateYzsSK317EKn92ptqDI=,tag:q1yuk/ZMx3nuORkiT/XXqg==,type:str]",
 	"sops": {
 		"age": [
 			{
 				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4SzVMVHhUMzAzcFdTSjEx\nRWZabEZEZzQzUzJuZkVYSzZBcFFYM3JaVmpBCm9xY1pJdUdNWGRaMFprdXhkdysw\nZXh4Q1dva1lMbmtVT2MzNGhQWEd1cUUKLS0tIFZhM1NjYnA1SldmcTFwQnZYZkpF\nVmtzbGIyL3JQbklPVk5ESFhHcWtPWUEKf04KPkSts5hiF4uImepznlfzbkb48YD4\nbtQ3toBSzW0wUnbxEHfA9nmuZFb6DF6majNCd1pVVr02c0MMinxVPw==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvMUtabnp3V0dzNFFYRzk0\nd0ZJbUtDMXRPRGxpRjhYR1MyQzdJYWdJTUFrCjBNV0pPTTlIOHBBbzlEQkFzVy92\ndENxcDdIZlNDSm1oZTNveUtIeVc3MXcKLS0tIGtocENjMFNYT0s1LzhYNy92QU5G\nREVEdjErb0xPSE1yb0g5bGlackh6bEUKwxBoDteD7+JfnlFF71CHx4oEdV/TFYcF\n3JPYUbTWAIyMtUu/CLbX+Pn9Mv+McrEIqhwT7TWL/YbELKVadX/k5Q==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
-		"lastmodified": "2025-08-20T19:56:05Z",
-		"mac": "ENC[AES256_GCM,data:/P15LdZuAUOrKagmmq2+EZ+70GZ5sGT/QbkPrOw/t1P281kFQXTL/OjXEyQcLUMSYElPjK+qEdDuIN4hsh24lOEe2ChJ/xEo/Gm0NvPRwb3CeIsfYs7mdOAVitUgc0q5kjpEBvljaGwN3SUvVD15I7jZkgY2BkVHXHTRxueKcvw=,iv:dmb7qnNR6hKBDHANwS2LHXdJrVBicdsOoJ42uhYd9Fo=,tag:OiHWr4nCK+9BvKnn/2LlBQ==,type:str]",
+		"lastmodified": "2025-09-18T14:33:37Z",
+		"mac": "ENC[AES256_GCM,data:4631iJmioJ2vZ2PTFbdEJu7UqtyQbp43XBlgEbFAviGZdugb3weVI24rJ8m1Rdnxq8uciEeiX6YHBhURdWQY4JNm2wTGnjz7e2PwQ8FCwOmxCcIQPpdKKsziq/M4HArgD66eUxIWfTt1yJfHgBcUuuANbrbH8MirllT+hJTBhqE=,iv:rM8a/MpKbK7DlqjuR4BG77XDHLK11Q+E2rzZLDJalhk=,tag:bbGMn4anXrLHg4eLA0/CXA==,type:str]",
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.10.2"
 	}

clanServices/monitoring/tests/vm/vars/per-machine/peer1/state-version/version/value

@@ -0,0 +1 @@
+25.11

clanServices/monitoring/tests/vm/vars/per-machine/peer1/telegraf-certs/crt/value

@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFuTCCA6GgAwIBAgIUMXnA00bMrYvYSq0PjU5/HhXTpmcwDQYJKoZIhvcNAQEL
+BQAwbDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJh
+bmNpc2NvMRUwEwYDVQQKDAxFeGFtcGxlIENvcnAxCzAJBgNVBAsMAklUMRQwEgYD
+VQQDDAtleGFtcGxlLmNvbTAeFw0yNTA5MTgxNDMzMzZaFw0yNTEwMTgxNDMzMzZa
+MGwxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5j
+aXNjbzEVMBMGA1UECgwMRXhhbXBsZSBDb3JwMQswCQYDVQQLDAJJVDEUMBIGA1UE
+AwwLZXhhbXBsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC7
+sdy27E/XMAyKrgeFcXY70R/vX0gx6EcZlWGp2vZSUVAfW1ni/Vq/LVC02sxGEGwv
+10+42yP2yghi89doKo8oCoLsbVu+Pi+TmRsgAijy4jN8pHqbn9/Vk8M8utLa1u4z
+VonSIx9pzCYd2+IIdwVuWoyPAAnK/JIKS3n0A8KWkZ/1lq6YDl2whj8iY4YF2Ekg
+M0SWhquLZiaApAs7STTYvcP7iLfL4U6cH65dRAbwWMpMErPuLf/CedkXiSUp8Zqx
+YIXXE5lf7wqt7tM6k6BHic9FEzAo1HnBWBXV5eB5fs1lX9M1VPmx43XINCfzKwxE
+xODtIBrmvj+qOp6/ihBsu3LlOoOikxmL+T9Wgvf7fOuFC4BgmX85mGUV+EMZCDoJ
+44jlwFF8wgrfG/ZawkP+opNsQLsdOm9DbAdWpx5+JYdgWBahjxuH4z2eIiBmMKgj
+puqDgXdZzcERiYtOEEn0p0tvIkVLO3Tm2GjtHbmg1yF2nwsZjupGfcOGTVX4Zi5x
+ZCs7vYgBtZy96kNAuyZcFl8eBUr/oVg//i3Zc9Vnw/UJryB7I6dvj228hlrSz0Ve
+pGoeZXbcCzRv8NX2V0V1VTtrblSA3w5WRxVzK7UAVetPZ4dlJX+eyx3x2wiC3TiW
+ZYH8haFubQqr1h9oXFHgDE5xYZKr51T3SRGfpn6KvQIDAQABo1MwUTAdBgNVHQ4E
+FgQUJHOErJYWaGdla1XhxWha4XBKFYgwHwYDVR0jBBgwFoAUJHOErJYWaGdla1Xh
+xWha4XBKFYgwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAXqcg
+DW6qzFccR+JTqNR5HBOneB07LxaUqfBTAzU5GTRljY3mVpnTa6vVvXlStChqdmwU
+JJdRhWzTpzE4K92l4UKiYKy486PT1ff34aPLPX5BB9OzL4dgvC3gO0MYDJ84AFZl
+6BN/MRTinioG+s14SsxmgcUTl+HXsxt75r3WKjXvqECqhONLPXEXDJ6TVmfb2yd5
+X9cE6HLS2IXqfvs0EdXmQhSQVS7AlUQWZPDeoBTDUA1tT6ZKCcG0BuHEFnHxg4Yg
+W9xp/wMJCEly+9eNJYZYzyK1AHRGnTMRCSifTJEybwI4A35v68FyRLfAC0lM2qVL
+yQIGjj55+r4yGCK7bySSKjs59LLLxi6Px3S61OxAYq9KMT65nBLK9JAPFyTnikw9
+q/xW208lL+kcRtG+ARo5ycx5QUjWdsHn7TCnqxnDhHznwSV4KGbJFaGQZTtgfcz0
+g5a1GwxqHjEZ9IWiN38f2l4kpLLybKhwVQMYeG000s7rDa5hgjbh13qtQN6vUvI6
+VozzZPnFcR1Rsa8RR9njDugxbVwlJQfGkoMiMZwNGgXnZRC2XaI6SCyPwqTPBuVP
+ZR1eWv4qwsIGKJzJYcdChb5dimlTuVSfZmONpnrOP/4mhQLyaWr3XLqxxP3mIXsz
+k1PNWTkgLsXO8DNkCudxcvPElXfmaw6zwaLrZys=
+-----END CERTIFICATE-----

clanServices/monitoring/tests/vm/vars/per-machine/peer1/telegraf-certs/key/machines/peer1

@@ -0,0 +1 @@
+../../../../../../sops/machines/peer1

clanServices/monitoring/tests/vm/vars/per-machine/peer1/telegraf-certs/key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:eWZyDgPQppMI/wNGSGsXowQ35I1KW1KH9p3GfxMFKNfoG2rnNwiBG11ARd9CDVMnY5OUt6RxL2sRKBlvqqjouCICDEEj3CWNnEpA55JGnmp3jj+kCRiA/te67F5vDXWus/mLGgI00apHwqUkwRkdck0URgniEIektncP9mQhcKDT7Lksm1S8oTHGDRcdiG4MxhrOq0qumVWdwS3qkAuwOvFMlYeCec6nfKBV5QTGeDxe8m8tijr7RTfM8cEaXrwaJDct1IIiHsl1U+V7+rz0KEvJ8ofeyOLP2zNSq4JfwM9rg/EwVuPsKf6LNmm6G/JdePlaCrwTaLchwb20/Tnf9nvrZu3P5w86IuniIyjFByvLR3bc6wKjxkWDU/+9UoTXfms5qKYNsgylFdg1xfqPjK0SgWiUL4IlxTBYPoPouNp/NZO+vzB+nkAcljCNGnYrfCz53F3gsTwBXIGmye2gvmNMvP+rs2/ySEt3XIzMEiWlBjDlurpAaYgqHhxVuc2jiqX56W8nu/QStopKP6sziPQbRqKDERSACxJ/WWumXTVO56dVJzqTpYnkqpq28tFoRd2yG7cJjlAbgqyxRuNkcLwnTEjGeGSSdVvmBeCqr4LuIh5qd2B4lrHQ6fR9xE/EHuJ2bcAH/x8ukOE7CZrACIEr6HfcpsnNhnpFYdA6gf4Gle21UJpK7hpY3+nCMNEPdfTjYkCvi/guzjG+X+UQPY466qbiVhUnNK4sg35axAJyNH1Jk6lK6+L/o4EVHBvnEUagLN2xFD5w0kXYMpzvQWEMaexyciDs6Natn7MzYVhmea8OfKXVE6dQz3Y5YFJ3uEQGGjuNO4fPyfnVgUULeaAs/IWkoPl2HV0x0KdxMEKGw2CAl7XuHYfV1rFTur+Wvf72rECUiiDmOgDU1g4plcBxQ6ocp34kize3lt1PdEL0R9lWg5c6l8LsqFhLqK8lpPV6neRdXX4UDzPjxnf3Ra/p1Hn283QSAv55pIwJQAo+kjWGckzr9CleUnLfPxQUKJQ7Jpjb/HtuhTQGA0mTsCbEHR6VWM/EYS4WzUd6opmfBstzSplD+kSBFIBoee+0dkUjfZcdFIWJRcabtjnn2TEsHHCK+dAguYY77OGeAh+tw7r66gONgtNlwjCN+KrzWH8cTu8BEaUoZH35lExs/wn+Ucj8IXDUXYLTTzGgokBybEeis+BDWFpDrhsZKFSwRE8tsrxfpgr7R1Ue9zMLoHnKeDZ6ndkm6fMinZ81OOchfE8bElRecCEzs9N/zU9nCtXKSAiYc86VntdbDFcPAm+bZ4hVkQpiRvQVGFYhgLuol7i9xhKD86TuIkqwMybEnT0ruqMNEVljxMWK7Cy+CAWg68w+hY2Pd54vXyC9ORndrYG7zbtVEe2dR7peeWTDTjU+5gVqIlC9lIhnIjgDprzvjszukHzc6TE98W9bnEKieSNGbQntm+YPohprg3CdVoPc1GfVueRqyXfXG0WVkLgfrhgfuLaJGKgwo438cUcRV8qH2wgCa7CGPMgvxzXJrK2dSRmZA/vPgZDpX9r78YlFGo+g/ghGhiNVonMYtMhohlSrzrQARA2AYuMgM91aXPnoKtqDy8+UL4g344bu7Jh3SKyGoqBo3TFLJyQgutzIx6EHG/eIDnTfc/I/3RgBtwo7RR/g+g899nhsiBLKVQId0/EZ+rKSndRTguCnFkjwCvXNW1z5uoiom/J5Q+J0xC1lqcjWF0zn9UwStQmvXDOABJUsGu+AZnj5l27MdRWvTfP2p3r12TXbyPEwOGuJa2LKSL/k4XmuaO8HkxSsfC1ImPOuPGbjgVkh62Y2oMqI90dtVrZ2HyosHwxv4tKzGAZbvH5vkK7TZXgoXCgAq+XwCPG9gtW2sIA2qoxw+SLOG5CEnHt6VlSgelLce9lU6kETdJ13fSqjMwZTQD07vXVnrtCHhsC6s+aY/7/2lJ2x8VmRBXVW7yREF56AdjYYVYgiAoHQqaQ0/OHpr6hacckqBTP0VzlNHLAzwm5zlgsZLDt3NxjTUZdgJEvFxF+rjzZHgyXwMA8hfzPbfVjftDW8hCMD1p8wJSY+CqaH+6/Ui9Q0X4F3YcZbhn/i9ZmMrB+CzBcjVzGrZIA0FLFoJWD2bFVPmMbcmDsT5ei0HafGBb2NBQ1gYvceGlN3WVQbTYCG54QavABNAyGFH+eQHvnk5jCg2DYspoCOPjEvIHjKM+gluIrozrnzMO2+hzp4Z+AscJCOm91LmL4PIFviyWzqy6AV1BLYPMLybdqrbEqUCFIzkXdFW3AZxV69hwhnBaZbLAaLeOG9YUz48o7oOITsDKVtuzUxkYDj+vBxI6zf7SvqjmopNXuZ2+4J+oa/p7xCpNUJTi0V4Ac38BZMiUcpXidu1V0pkGWbca4Dfqf2vBOzOcpLxrorizsyROv1SJAA7mR8KQut28HnkXgshIhB4cY99tnmKN/E1oiLGU0NkUHR6fCBtV2Ak8k7PNCVzhU0y6/NCJoSKqKQpuPEMVT+0QaKNfjtGvWgvZrvcchoMNAAGQa1OMSkmcZ4KdnAUaMROrS5LH3IBwpmSwtTBFkx9Shl3xMm2SpF6SdWnpweUbRAQqKNmRvSQLsXiEwOwxIO018mo8CgyiDyyIf4k0gFlNTapYyacwRO4vTMc3vfXjTcwK1LzUZVeG+e61WVDmmu2e6zls0JhXe7V58OkbnYWnzNzBSxWJluicno/P9h5vefBOHfysKe6SlGye/H0BO7piVG96cjqC0hTul8k1ysQoXtFgf4fbrlqs/D1kR9xVHcr3hAeWd9c4LwXEcSCeVuBd0bsoo2sYIeNSWNdJo9bSF0vb49snroh/RgbzntW3+geL94DEZaXMmf+RLujLEIgoNLlZ6r2jTMvlV6DWbSRE3cii6LFOXdQq53fmG/cI73R3hGNdQaLhZDaOi7hLnxbAMAjtEVQQOQg93a43d/BDGFzgNhKjYqyjZ9mM/Tk37DLlZ+xeIEJpALLIAaOguSG5cg3ALBrdGRec+SPf0r6M6DVkS1VHFz54kPx1eGkJQyQTotcykafNIt1Ahbqif0Z7U2bF0LxUbrZxcoldFteBNzihlXxa4zrY5Uj3BWEOrd6E8zHUIW97KwUAdttMTlNoOrMOgLY4790cVX+K7sa9ZPWz8Lts7o99sdcF7+dHoVxvfM0O3vXdzA/2O1opKqD6ZfPmU1UyWL/N2d4d9JerDhD6RFuBJP7nsv8osf2NHyWdHV9Luj0gOiBZvoOuSI4nvE05rPIXR/UEjXBw+1XaGHqcj8x/6rE6oTAma/1DH+E+N0j6mUd97vHFa48rbABCLWK4n9MrjXpQAVYNlXsSRgmEaVcq3S4RdRHKIp6yhhsUfNI8B8i8obQ3lBj7ktx1BNynnSJKTbQVOritYsQEY3t/+PvCdr4RKflftx0KzwcFTscVSrX22+aZZD+VrPZ3o8OUH8yxBWUsK5hdhuVOfNEjL6TpgDUZgbFUdlTDHmzPm5RxDxK6qGLxr0JwfLNm/+nYliKoyiTFKVKWFDE5Z+Rt0yKj+pDrWXBpKPySTfWX80VbioPW0curpiLt4tjVFfzhZ6V60vPfjcCjHlGz/pA5atUTGlZBP6DynDFJVV4QO0uhRYRfDvk+D6YOjZSHAX0e82IFg5l4d3fcF9WveqIfKRhJEVt3s4PLhCul/ESTWp45h1IA9ZfI4wvmuP0hCUvLgTOKx75QnwfVQRKJ5xa+R0e2Igywnobz63LaX9+yC8KJ23U8ZHS0Wc3E2NqTVEiP93ds98pMRMepoln20bsLUypcW2/py0WYb/YEGzlww9MxywAEQX+Pce8XhI7iylSfUzUmk863Y8cE1RMAiDeMFIQ8vZBT+LKwJ5zdik8jqJFED5XVGtYai7vEjj1tZKrfL+fR6CtDdQqyP1fWS+Xi5CZ7rdr2HiD943Vre1ZA8B7byozkMuahiYVzfTKIGI6lUMvXmmVNkdWXmj26YRy4l4X1KYM9L7f4NX8jRe61sUXanWJgcScxQTNKfGDOiKWRFQjo5UgCXOvjGtFCpRQyksY19TatFHRGrNdV2CmZhFTaaGbCbqD5QlfdoY1StT0Ko3x/YJR4/4Yoa2oCr2cVzNZ0/xPW0bC5NszLnKMjVI8Nj1nNFvMm4yZBpaz6YKk2REf9nndbkbhcppdrZN4Vt7wdt2gV2+5OpXRZ8OaxnegFpNiYuJb61gzXFYmYjWCkU6V9ncGV/71fXWMlxSlu4kLVhIQqD2+RI/VWAcS+cFEvb0Ntjft/gkyQcrLCeeFzdxXSNnlX1h5DigeRwyNtW4Mrk8vFQ6o2Oi3HiBKmvAD7sPkJg+lOJngQ/hI0477c0=,iv:q3j8EAokyyxiszf+wyRqxEr2igaD1bX7YnFx/NbsGg8=,tag:HKKYWRJEUwW2/TxL+5dSng==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1ntpf7lqqw4zrk8swjvwtyfak7f2wg04uf7ggu6vk2yyt9qt74qkswn25ck",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaeXRjU214aWk5ajl1aW9E\naGJlb1ViaVRmMTBHdkFDQUNDZS94WFZiNUNvCllmWTJBck9hR3U3V09VWDZwQ2xI\nd3ZEQnBIUG5ZSTVIdS8rQ2FMYVhyNk0KLS0tIEE1UG8rSzFyU01sVXhGVHpoaE9i\nSis4Qi9tMGFqbTNMTDZUVk1ZdXkrM28Km4VkfaOsZ69ckjvrg+os43H/O1IoWHzC\nt4LqZRz1Tk7/d1aLWavSPPjVYrCOMZeNBqGbQpGfjjuXrafClRNQdQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3R1RHTGViTnRLVVkyM3J0\nbm96cGVPTlo4NXBNL0g1eEVSNG9DUkgwVFRBCmRKVTlMRmV3Tmg2RTZIclBlWlcr\ndzI5MUxhcllzbE1IMDNxa08zVkpITmsKLS0tIG01Y2dyQkY3UmRudFk2d0p6bThn\nemlaWnZoS3p4VHhMTFFwTm9VN0ttYzQKVbLFgtK6NIRIiryWHeeOPD45iwUds4QD\n7b8xYYoxlo+DETggxK6Vz3IdT/BSK5bFtgAxl864b5gW+Aw4c6AO5w==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-09-18T14:33:37Z",
+		"mac": "ENC[AES256_GCM,data:XKCnd0QrAlOCECSeSvbLYHMLbmUh4fMRnLaTb5ARoP4Zc9joWGsCaRZxokc2/sG4BXA/6pkbQXHyIOudKbcBpVjjvs9E+6Mnzt53nfRoH/iOkYPbN2EO49okVZJXW0M1rlBxrxvGuiDlz2p2p6L7neKLy4EB482pYea5+dUr2Yw=,iv:oj/MkZCfkvCmAb79uzEvKwEAm1bKtWhS4rPRAWSgRgw=,tag:h5TPPILXkhJplnDT2Gqtfw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/monitoring/tests/vm/vars/per-machine/peer1/telegraf-certs/key/users/admin

@@ -0,0 +1 @@
+../../../../../../sops/users/admin

clanServices/monitoring/tests/vm/vars/per-machine/peer1/telegraf/miniserve-auth/secret

@@ -1,18 +1,18 @@
 {
-	"data": "ENC[AES256_GCM,data:ePdnRA2Rkwl3C1Ugp0GjZ3gncdgu+vxTMZe87tI23FKRX7KxJoobKeivWH4=,iv:h3Mjf+zfWMC98KarOYKdAr1/I0HSDd7iSxnlgxIFL7Y=,tag:GVave5z1hT2MG1WW0p6H4g==,type:str]",
+	"data": "ENC[AES256_GCM,data:Q0Vn7J0nERccBYT8HZxHF0Zi5qxmMu40n0H1c+L2SCRF6vRLdURxXKDwvh8xtTU=,iv:ucExjoYDFYy19GsBbNNldJRPBSpT+L+x4PrwTG+m2K8=,tag:/Quupyy/nnUNZsDudEMmNA==,type:str]",
 	"sops": {
 		"age": [
 			{
-				"recipient": "age18vspwr3de7jp0awyu66kk9psd5x4sy9suv0zt7ux2kqw0s6h2ueqwkgjxm",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUOXJ3UVJkdHl1T0REODBB\nUU9pamtPQ092L3FWT3pHeWpoNitLNlllSzFrClNLSlRIYlMxRUZIQ1JZaXpiclpm\nZlV4SUo0USs3NmluV2ZHVkdtZjZXUGMKLS0tIFJscmkxVG03dUNiUkJLZ1F6UEkw\nQmVZQndkNm9TczcwaXRoN2hTVGVPSVkKJHjTSZrR0VTPh3IiIfvoRAsWBA4lvXp5\n3+9x3zN802z4+62SmI1y7497GEUe4iVcMIvxup1az+sbFpN31eC9+w==\n-----END AGE ENCRYPTED FILE-----\n"
+				"recipient": "age1ntpf7lqqw4zrk8swjvwtyfak7f2wg04uf7ggu6vk2yyt9qt74qkswn25ck",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQWWo5OEJ5N1RTR0xMaDhL\nQnlUV2RrRXIzM01OemhQWjVkd3FNZjRhR2dzCi9IeE56b3VZTkNkdW9DMzVia3Zx\nbklxWmFpenRjdEIrc0ZDTGdmSTAxRTQKLS0tIHZJdjdYUzhhY0YzQjRqS0psZmpI\nVHJpUjNZNHRpc2ZWSml1TVNNejhiT28K8TTP/J+XspXZ7TVYj9YaBhEodPIXjojB\nRLqAIgJXRaK4NCLukC6l0IMii6w5J/512RnO2ZBTGhKfbdLfyLOFqg==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
 				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJZEhBSW0rWFRDb2tCMzln\nVzZEZFIxdXBlQkVxaEl1N1p3UUI4elpTSEVFCnYvUC9iV0RZVURDODRiNGFjZnY1\nSUVNeUo3UUg3YVV4OXdsbGtaMGlaY1UKLS0tIHFhdXpOdzd5TVBSQm5NdGtxZzdr\nL01odEF4QnRReTNRVkZkMk44cHk3SzAKHUs4hgsOMK9ZIIyUDbTqbWmk1GHGBa+B\nENSaH5UL8AYnOvGd3vV4VQcznVmhYh+VPkJUbu7gXkrYyVNBjsWx0Q==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrZVc5b0FhbzNXcG1zUDlD\neEVWcWpSRkRCMkxBTHdBM3dCbjVpR3FBa0VjCitlTmx4eUJOMHlaU0dFZEhpK3ZD\nZzlMQXVuZWpnaUNmQW9kOGtOaGVDMU0KLS0tIFNlUi9LSzF0UEJCSVBiRlRSNFQz\nNHhMbmNlRXd4ZEJQWVcvTWdCRWEzMUkKls7RbmNOdPDx8z15F+7qay9qIWx6jNsN\nTahT+GgbG29t1aGQCb0yEzKuUyAp39maxxSWToPsfCgJSYJ8RYiUng==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
-		"lastmodified": "2025-08-20T19:56:05Z",
-		"mac": "ENC[AES256_GCM,data:0g+PNdDOZ+pQnt3d5cfYVOtToIbuBnyMnqfk86HjK4YDMObFRkyLe9k0aTDAH2NvERBX/BPb+lTffBlwM2Fl6p/EDXh+x0Q3IELjzcOfhX6jbR45rMRJF+CcU8pbNSGApp153QWai2ku60SUUOdpe5tIbmTah/QfdMO4x4/fM+M=,iv:vLhzyZpfcuShncd0K0+GzFNNmhBOOeNxnboe+3VkJGY=,tag:VgbJA1xfISWK9JJPIgxHew==,type:str]",
+		"lastmodified": "2025-09-18T14:33:39Z",
+		"mac": "ENC[AES256_GCM,data:g+9/fRiqom2+W28ZpiF+oBj9V6ieq5Xz3sRz3GyzvHoLr6yw51JvpG2QuYNYANW0WCiUjFDkU0qPj/9gLHcuX52nc+gNaTzznb1QGPg7WCGSQI7xaMzyYsPxHpg/BOdj5CL8GyLiOWstD1ch0kc3bJmyu68sJUs04uGtHAADzsE=,iv:oASrYaZarEPDu0R3hd/jMazLgwG5r//hIdMyU/tN15o=,tag:o1fgf5oy+rlWXg88FN5Nfw==,type:str]",
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.10.2"
 	}

clanServices/monitoring/tests/vm/vars/per-machine/peer1/telegraf/password-env/secret

@@ -1,18 +1,18 @@
 {
-	"data": "ENC[AES256_GCM,data:u9lTWO6Z4wgw35zBhhfbPDv3bc1MIWttbWUzkS3Hjzgwq06Zp4z61e4Wgt1QBkCC,iv:1uD5R0hQ/6Su1bg4nqL0MjJ22HvHjLGmgrL02DxegpY=,tag:mypBmsq27JHUujCdNYpR+Q==,type:str]",
+	"data": "ENC[AES256_GCM,data:4NIUEK05kEQAKjR8F9mU3M/XvtZXw+X6CejVI0usMcb4WzagNz7XTVDhLWXZ9St5Ev0Y,iv:bD2+rDLMoWSqUAIZRJof0wRrJVya1xwZUTIJBdCs98I=,tag:g2s4byFHTzwU3ikcBGMElA==,type:str]",
 	"sops": {
 		"age": [
 			{
-				"recipient": "age18vspwr3de7jp0awyu66kk9psd5x4sy9suv0zt7ux2kqw0s6h2ueqwkgjxm",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKMElLR3dhWmZJd1NmUytD\nVitaeW1JbkpnMGt0VFk4M0VRWnpCazRWeTFnCis4eWxnZVdVajVPWVUwMGFKQkRB\nbDVkTk95RzZZVW9BQll0M09VekZCNkkKLS0tIHFpZm5JRDlueTRsMGUxekxxWjVz\nRW5KVEtZRE85KzNFM1BINUtJcExtME0K5aOLpzy9Y35McN19UEm1Wy6bU2oeXGxZ\nCjw5tLHHzxUOzfoE1RfIZinRmBXRZpCpQVH6iK3IaIq8aouK36Pa1Q==\n-----END AGE ENCRYPTED FILE-----\n"
+				"recipient": "age1ntpf7lqqw4zrk8swjvwtyfak7f2wg04uf7ggu6vk2yyt9qt74qkswn25ck",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQeVh2M2tqSGlOVkpzNlhU\nd0pMd1R0c0tQWnZzdXViWmtxcjl1Wk1Ka0FNCnBUUWJVbjlyR1hSNGpXNWlPRHJB\nNnMzN3BMQ2NDamFBMlhHbVdJUEZ6cjQKLS0tIEJjWmI0ZDl1NXgrSW9uc0R0LzAr\neEwwOC9DdDg2RTJHQ0M3QTFlcVBaSE0K2Du4NguefdEyY1gS6OuVdO3gHga4omcR\n8B+K1wUfIQbArxZLawPxrj7WNDoW5d4mF9fA3MeV1DFyc4KwtYZmUw==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
 				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2dEJkTnZzekhaSjRsaFJw\na2xPK1BoMkVkc3N0UnNqZTFZZWpWQ2lIYUJFClBaemplMjhPYkJsRmIvWmsyQm56\nYUswOVkzTUd1cTRtNVFoV1RZQlBtNzAKLS0tIC9ERjl1Q2lBaXFHcHMyRGt3VTRs\nV25tTEZsak5KQ2lNOEFLUVR5c1lnNjgKaGDYoK6UJSbkBs8+eiqEFEx/tbzlNGPz\nw96ttHtR3j/jkCbwOpAb/D5yChfJf9mQlpjbtKvEJ0SJSEPT5fniiw==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvWkdBakVrMVR4RU8xdDlF\nRDkvL0Mrb3ltazhIMjRLZDVlSTVlaFY2ODBBCnlQM2s0SGEvZjFDN3dGWDhIN0dK\nenhQbjZ1ZS9QZzg5SE5XazZXS3dFSkkKLS0tIHJhKzhadGpjTXd4L3hOQkhpR0Fy\nYzhTN2dxVSt3OE5uZFpuWmVlYW4vd1kKwHOxP0C5mLcm4oIT/sGQtUsdsmu3LSN0\nSola5+N+IrAZ+HKnuZlDLZ5JmJSc5j/YhGNn7KR1xhkhfGSS1e3UZw==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
-		"lastmodified": "2025-08-20T19:56:05Z",
-		"mac": "ENC[AES256_GCM,data:GH6B/83FRG4KDdD/ZHmAVjeOlzVFMbuoaq+yyc/XRVIdTi5t5uLMGZePd1AiHzAz4lLubxJWVp2bqw56m9G9ICPCCtmCpE5SEoXllZ8cRZ9H0yb8ywAT/66pRnWT60cNCLYrX0LJyPw5HLbA09xETXhNt3HYhom6tt3VA3/ghW4=,iv:UcZoY2P8g8KS/NvsOd3B983vu5D6fC/ClF6hDXkjvq0=,tag:J1KxVXW/EpNiOo4mhwqokw==,type:str]",
+		"lastmodified": "2025-09-18T14:33:39Z",
+		"mac": "ENC[AES256_GCM,data:ehbrYqTJcsBKGHUB25JHFnKXrJ6z3LkcElZ89xVr4XxLet+odbhsjIoP2FCcxex7PlXcegMduhHBpXwNGUbX+IUNAXTxlWA9CLDmYhWuS2WLiEVXrS11NE03/zUyHdVx/C38dbIPrWD9iaYSrAiuOyfqDTh9k/Bn7vehLTtadoE=,iv:Nk2WVuJydi5tfsb1Mib4A6NocBCDp9QoIbSadq3bIDI=,tag:IaoyfCv3SkmtemXMR9XnkA==,type:str]",
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.10.2"
 	}

clanServices/monitoring/tests/vm/vars/per-machine/peer1/telegraf/password/secret

@@ -1,18 +1,18 @@
 {
-	"data": "ENC[AES256_GCM,data:rXxZC/HU9O/3CR6lDR+twbgtq3SjPHR7mz7iUnHF4MA=,iv:SpRivg1us5RutN7h/YR5dh3QG8/wBYM4GgE1t7u0YVM=,tag:miFjFfqYWy0yFdPPUB+T6Q==,type:str]",
+	"data": "ENC[AES256_GCM,data:0BmP+NwG/NGe6R5yU55/MdPEQ8E5u+VXWtvstHc4GpDtmBY=,iv:vo8XBcN7KcYjiyKvvp+XDOdP9yR9B7wJi0XlaiCdVbk=,tag:brK9ntAPSuOvw/C+oDo51g==,type:str]",
 	"sops": {
 		"age": [
 			{
-				"recipient": "age18vspwr3de7jp0awyu66kk9psd5x4sy9suv0zt7ux2kqw0s6h2ueqwkgjxm",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMMllDWGZvSXRMTFM4SFBQ\ncEo4dFZ4SHdVL3hYQisrQTlFOTBWQ2JPTmowCktIak55S29YVnZ2OGtJa3hVcURx\nWDdEa1hmZGt1UkZxdmpnang4NThMbHMKLS0tIFU4UTVaSnpQdzZCNFNJWnRSaUNU\nSDRuYnpvbDJsL2d0OEcxQXVxVDExTHcKsijOA0GChkmjNGuPiD4/5ohXuBcTmxxD\nTOC6jdf3TEo0b9ZRmNk/TpJpUhe7PQiv48oqFfbyj7VTicNMKbtw1A==\n-----END AGE ENCRYPTED FILE-----\n"
+				"recipient": "age1ntpf7lqqw4zrk8swjvwtyfak7f2wg04uf7ggu6vk2yyt9qt74qkswn25ck",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4Tk1INGtybUVlejlNNlZE\nVms3TkdRVVF1T0E4TmV3NmxvYWVEL2U3WVhNCjJIaHhBcWVlMEYxRjg5bzJpTWdJ\neUhaRTNRTmtlTW0zUXQxTVZEMkQ2MFEKLS0tIFNGWDI4b2FXTE8xQ2xqb0cyK3FI\ncktHWnE5c1ZSVFpmQU1HZmU2VVB1QmcK/s1fVmwpMMg4BYkkAJzSY7hVQWae1F7g\nmfH8EGlr74mifWUNEbd49/K13nl8atQx6bcau83JIEQR+yyihuY4Jw==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
 				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVZzNsNGJBbEU2eXhpenBj\nTkRlRThJbHpMd1NlMWVXNU9EcGRvemYzQXhjCi9Xc2gzQW82T3A2WGs0KzE5M1JV\nb1ZZT0ptVHhubjlTTTM3MkUydUN2d28KLS0tIFJScGZudjBVTFpnenllaW96NEVi\nZmNyaU1PUDY0QWZFNnNuaVBYdmlXdzQKfmGQ5EUjUxGzZddENlu4ZSaHYuT33Kfj\nBMJsbYovtgJA4UsBufcMY+ohN86C2Xo23JxYpgA2Qzu1KA46qXlrHw==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsL2FXVytUUVZnVU90bG5L\nYURiYjgwN3RuTldWMGl4clpUWmxkeUsrVzM0CkhKZFgwWHl4dWhNSWRQRXVPNDR6\na3hHNmp2RG9YNDhNM2MyV2FuOGY2UlUKLS0tIFpNU2tNOHdhRDRTdHhYWVh2NGZa\nU3J3S0hpclZzWGIwTlFyczdNZkZSZTAKXCZrLaIOVq90ejoKMaRiK0xNw8WOPcnm\nz2uxProEYvQhY8k29mhCFX5HCN0tGn1XTtHeDL7uHuKuFsnSG/fgYQ==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
-		"lastmodified": "2025-08-20T19:56:05Z",
-		"mac": "ENC[AES256_GCM,data:vtO+KbN36c38DkFyqlQAnz22CodxE8S6yfadRLtjbKPaBD4cn60R6Md90jHB25uqiXyr48J+3mMbpqBVwl2Sjxt/PtcsS4UBz/xt9pGtdEssX5veQnIdEA4dlBpAeYWba/aAZn/rxYYsm+yswLAd1DCwQf/moetF6asQCZSXTBc=,iv:DI5m3qopcdPSvpVSaNDIhUoNDcVpumkMI/nz1MV/fF4=,tag:HNG0pupG1tzQh7bQJobVDA==,type:str]",
+		"lastmodified": "2025-09-18T14:33:39Z",
+		"mac": "ENC[AES256_GCM,data:QkGJKj/H+MI9Mr9Up5NDUToSddY5eTz47egc2+IatfxR8RebKJ2/mYaeLV26vPdmY60bIac4N/nZkoa6IVBhkHHMvsEHsx3nD6Lro9Wf/pWP8Zddzr90LF5p2+wusq25JutKQiPKOb2gmrcagmSsH/7V/UqI/my3PMeKmw6irhw=,iv:hOtHF/cDFdNfvqCKRhJsOwAHEiQmCPjENzsg23sKG+Q=,tag:K7qG9b4fQD0VbAV8OYp3vw==,type:str]",
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.10.2"
 	}

pkgs/clan-cli/clan_cli/cli.py

@@ -549,11 +549,11 @@ def main() -> None:
 
     try:
         args.func(args)
-    except ClanError:
+    except ClanError as e:
         if debug:
             log.exception("Exited with error")
         else:
-            log.exception("Exited with error")
+            log.error(e)  # noqa: TRY400
         sys.exit(1)
     except KeyboardInterrupt as ex:
         log.warning("Interrupted by user", exc_info=ex)

pkgs/clan-cli/clan_cli/machines/cli.py

@@ -3,6 +3,7 @@ import argparse
 
 from .create import register_create_parser
 from .delete import register_delete_parser
+from .generations import register_generations_parser
 from .hardware import register_update_hardware_config
 from .install import register_install_parser
 from .list import register_list_parser
@@ -145,3 +146,19 @@ For more detailed information, visit: https://docs.clan.lol/guides/getting-start
         formatter_class=argparse.RawTextHelpFormatter,
     )
     register_install_parser(install_parser)
+
+    generations_parser = subparser.add_parser(
+        "generations",
+        help="list generations of machines",
+        description="list generations of machines",
+        epilog=(
+            """
+  List NixOS generations of the machine.
+  The generations are the different versions of the machine that are installed on the target host.
+  Examples:
+  $ clan generations [MACHINE]
+            """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    register_generations_parser(generations_parser)

pkgs/clan-cli/clan_cli/machines/generations.py

@@ -0,0 +1,285 @@
+import argparse
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Literal, TypeVar, get_args
+
+from clan_lib.async_run import AsyncContext, AsyncFuture, AsyncOpts, AsyncRuntime
+from clan_lib.errors import ClanError, text_heading
+from clan_lib.flake import require_flake
+from clan_lib.machines.generations import MachineGeneration, get_machine_generations
+from clan_lib.machines.machines import Machine
+from clan_lib.metrics.telegraf import MonitoringNotEnabledError
+from clan_lib.metrics.version import check_machine_up_to_date
+from clan_lib.network.network import get_best_remote
+from clan_lib.ssh.host_key import HostKeyCheck
+from clan_lib.ssh.localhost import LocalHost
+from clan_lib.ssh.remote import Remote
+
+from clan_cli.completions import (
+    add_dynamic_completer,
+    complete_machines,
+    complete_tags,
+)
+from clan_cli.machines.update import get_machines_for_update
+
+if TYPE_CHECKING:
+    from clan_lib.ssh.host import Host
+
+log = logging.getLogger(__name__)
+
+UpToDateType = Literal["up-to-date", "out-of-date", "unknown"]
+
+
+def print_generations(
+    generations: list[MachineGeneration],
+    needs_update: UpToDateType = "unknown",
+) -> None:
+    headers = [
+        "Generation (Up-To-Date)",
+        "Date",
+        "NixOS Version",
+        "Kernel Version",
+    ]
+    rows = []
+    for gen in generations:
+        gen_marker = f" ← ({needs_update})" if gen.current else ""
+        gen_str = f"{gen.generation}{gen_marker}"
+        row = [
+            gen_str,
+            gen.date,
+            gen.nixos_version,
+            gen.kernel_version,
+        ]
+        rows.append(row)
+
+    elided_rows = rows
+
+    col_widths = [
+        max(len(str(item)) for item in [header] + [row[i] for row in elided_rows])
+        for i, header in enumerate(headers)
+    ]
+
+    # Print header
+    header_row = " | ".join(
+        header.ljust(col_widths[i]) for i, header in enumerate(headers)
+    )
+    print(header_row)
+    print("-+-".join("-" * w for w in col_widths))
+
+    # Print rows
+    for row in elided_rows:
+        print(" | ".join(row[i].ljust(col_widths[i]) for i in range(len(headers))))
+
+    print()
+
+
+def print_summary_table(
+    machine_data: dict[Machine, tuple[list[MachineGeneration], UpToDateType]],
+) -> None:
+    print(text_heading("Current Generations Summary"))
+    headers = ["Machine", "Current Generation", "Date", "NixOS Version", "Up-To-Date"]
+    rows = []
+
+    for machine, (generations, needs_update) in machine_data.items():
+        current_gen = None
+        for gen in generations:
+            if gen.current:
+                current_gen = gen
+                break
+
+        if current_gen is None:
+            continue
+
+        status = needs_update
+        row = [
+            machine.name,
+            str(current_gen.generation),
+            current_gen.date,
+            current_gen.nixos_version,
+            status,
+        ]
+        rows.append(row)
+
+    if not rows:
+        print("Couldn't retrieve data from any machine.")
+        return
+
+    col_widths = [
+        max(len(str(item)) for item in [header] + [row[i] for row in rows])
+        for i, header in enumerate(headers)
+    ]
+
+    # Print header
+    header_row = " | ".join(
+        header.ljust(col_widths[i]) for i, header in enumerate(headers)
+    )
+    print(header_row)
+    print("-+-".join("-" * w for w in col_widths))
+
+    # Print rows
+    for row in rows:
+        print(" | ".join(row[i].ljust(col_widths[i]) for i in range(len(headers))))
+
+    print()
+
+
+@dataclass(frozen=True)
+class MachineVersionData:
+    generations: AsyncFuture[list[MachineGeneration]]
+    machine_update: AsyncFuture[bool] | None
+
+
+def generations_command(args: argparse.Namespace) -> None:
+    flake = require_flake(args.flake)
+
+    machines_to_update = get_machines_for_update(flake, args.machines, args.tags)
+
+    if args.target_host is not None and len(machines_to_update) > 1:
+        msg = "Target Host can only be set for one machines"
+        raise ClanError(msg)
+
+    host_key_check = args.host_key_check
+    machine_generations: dict[Machine, MachineVersionData] = {}
+    with AsyncRuntime() as runtime:
+        for machine in machines_to_update:
+            if args.target_host:
+                target_host: Host | None = None
+                if args.target_host == "localhost":
+                    target_host = LocalHost()
+                else:
+                    target_host = Remote.from_ssh_uri(
+                        machine_name=machine.name,
+                        address=args.target_host,
+                    ).override(host_key_check=host_key_check)
+            else:
+                try:
+                    with get_best_remote(machine) as _remote:
+                        target_host = machine.target_host().override(
+                            host_key_check=host_key_check
+                        )
+                except ClanError:
+                    log.warning(
+                        f"Skipping {machine.name} as it has no target host configured."
+                    )
+                    continue
+            generations = runtime.async_run(
+                AsyncOpts(
+                    tid=machine.name,
+                    async_ctx=AsyncContext(prefix=machine.name),
+                ),
+                get_machine_generations,
+                target_host=target_host,
+            )
+            if args.skip_outdated_check:
+                machine_update = None
+            else:
+                machine_update = runtime.async_run(
+                    AsyncOpts(
+                        tid=machine.name + "-needs-update",
+                        async_ctx=AsyncContext(prefix=machine.name),
+                    ),
+                    check_machine_up_to_date,
+                    machine=machine,
+                    target_host=target_host,
+                )
+            machine_generations[machine] = MachineVersionData(
+                generations, machine_update
+            )
+        runtime.join_all()
+
+    R = TypeVar("R")
+
+    errors: dict[Machine, Exception] = {}
+    successful_machines: dict[
+        Machine, tuple[list[MachineGeneration], UpToDateType]
+    ] = {}
+
+    for machine, async_version_data in machine_generations.items():
+
+        def get_result(async_future: AsyncFuture[R]) -> R | Exception:
+            aresult = async_future.get_result()
+            if aresult is None:
+                msg = "Generations result should never be None"
+                raise ClanError(msg)
+            if aresult.error is not None:
+                return aresult.error
+            return aresult.result
+
+        mgenerations = get_result(async_version_data.generations)
+        if isinstance(mgenerations, Exception):
+            errors[machine] = mgenerations
+            continue
+
+        if async_version_data.machine_update is None:
+            needs_update: UpToDateType = "unknown"
+        else:
+            eneeds_update = get_result(async_version_data.machine_update)
+            if isinstance(eneeds_update, MonitoringNotEnabledError):
+                log.warning(
+                    f"Skipping up-to-date check for {machine.name} as monitoring is not enabled."
+                )
+                needs_update = "unknown"
+            elif isinstance(eneeds_update, Exception):
+                errors[machine] = eneeds_update
+                continue
+            else:
+                needs_update = "out-of-date" if eneeds_update else "up-to-date"
+
+        successful_machines[machine] = (mgenerations, needs_update)
+
+    # Check if specific machines were requested
+    specific_machines_requested = bool(args.machines or args.tags)
+
+    if specific_machines_requested:
+        # Print detailed generations for each machine
+        for mgenerations, needs_update in successful_machines.values():
+            print_generations(
+                generations=mgenerations,
+                needs_update=needs_update,
+            )
+    else:
+        # Print summary table
+        print_summary_table(successful_machines)
+
+    for machine, error in errors.items():
+        msg = f"Failed for machine {machine.name}: {error}"
+        raise ClanError(msg) from error
+
+
+def register_generations_parser(parser: argparse.ArgumentParser) -> None:
+    machines_parser = parser.add_argument(
+        "machines",
+        type=str,
+        nargs="*",
+        default=[],
+        metavar="MACHINE",
+        help="Machine to update. If no machines are specified, all machines that don't require explicit updates will be updated.",
+    )
+    add_dynamic_completer(machines_parser, complete_machines)
+
+    tag_parser = parser.add_argument(
+        "--tags",
+        nargs="+",
+        default=[],
+        help="Tags that machines should be queried for. Multiple tags will intersect.",
+    )
+    add_dynamic_completer(tag_parser, complete_tags)
+
+    parser.add_argument(
+        "--host-key-check",
+        choices=list(get_args(HostKeyCheck)),
+        default="ask",
+        help="Host key (.ssh/known_hosts) check mode.",
+    )
+    parser.add_argument(
+        "--target-host",
+        type=str,
+        help="Address of the machine to update, in the format of user@host:1234.",
+    )
+
+    parser.add_argument(
+        "--skip-outdated-check",
+        action="store_true",
+        help="Skip checking if the current generation is outdated (faster).",
+    )
+    parser.set_defaults(func=generations_command)

pkgs/clan-cli/clan_cli/vars/get.py

@@ -17,6 +17,10 @@ from .list import get_machine_vars
 log = logging.getLogger(__name__)
 
 
+class VarNotFoundError(ClanError):
+    pass
+
+
 def get_machine_var(machine: Machine, var_id: str) -> Var:
     log.debug(f"getting var: {var_id} from machine: {machine.name}")
     vars_ = get_machine_vars(machine)
@@ -29,11 +33,14 @@ def get_machine_var(machine: Machine, var_id: str) -> Var:
         if var.id.startswith(var_id):
             results.append(var)
     if len(results) == 0:
-        msg = f"Couldn't find var: {var_id} for machine: {machine}"
-        raise ClanError(msg)
+        msg = f"Couldn't find var: {var_id} for machine: {machine.name}"
+        raise VarNotFoundError(msg)
     if len(results) > 1:
-        error = f"Found multiple vars in {machine} for {var_id}:\n  - " + "\n  - ".join(
-            [str(var) for var in results],
+        error = (
+            f"Found multiple vars in {machine.name} for {var_id}:\n  - "
+            + "\n  - ".join(
+                [str(var) for var in results],
+            )
         )
         raise ClanError(error)
     # we have exactly one result at this point

pkgs/clan-cli/clan_lib/machines/generations.py

@@ -0,0 +1,44 @@
+import json
+from dataclasses import dataclass, field
+
+from clan_lib.api import API
+from clan_lib.ssh.localhost import LocalHost
+from clan_lib.ssh.remote import Remote
+
+
+@dataclass(order=True, frozen=True)
+class MachineGeneration:
+    generation: int
+    date: str
+    nixos_version: str
+    kernel_version: str
+    configuration_revision: str
+    specialisations: list[str] = field(default_factory=list)
+    current: bool = False
+
+
+@API.register
+def get_machine_generations(target_host: Remote | LocalHost) -> list[MachineGeneration]:
+    """Get the nix generations installed on the target host and compare them with the machine."""
+    with target_host.host_connection() as target_host_conn:
+        cmd = [
+            "nixos-rebuild",
+            "list-generations",
+            "--json",
+        ]
+        res = target_host_conn.run(cmd)
+
+        data = json.loads(res.stdout.strip())
+        sorted_data = sorted(data, key=lambda gen: gen.get("generation", 0))
+        return [
+            MachineGeneration(
+                generation=gen.get("generation"),
+                date=gen.get("date"),
+                nixos_version=gen.get("nixosVersion", ""),
+                kernel_version=gen.get("kernelVersion", ""),
+                configuration_revision=gen.get("configurationRevision", ""),
+                specialisations=gen.get("specialisations", []),
+                current=gen.get("current", False),
+            )
+            for gen in sorted_data
+        ]

pkgs/clan-cli/clan_lib/metrics/telegraf.py

@@ -1,11 +1,12 @@
 import json
 import logging
+import ssl
 import urllib.request
 from base64 import b64encode
 from collections.abc import Iterator
 from typing import Any, TypedDict, cast
 
-from clan_cli.vars.get import get_machine_var
+from clan_cli.vars.get import VarNotFoundError, get_machine_var
 
 from clan_lib.errors import ClanError
 from clan_lib.machines.machines import Machine
@@ -21,6 +22,11 @@ class MetricSample(TypedDict):
     timestamp: int
 
 
+class MonitoringNotEnabledError(ClanError):
+    pass
+
+
+# Tests for this function are in the 'monitoring' clanService tests
 def get_metrics(
     machine: Machine,
     target_host: Host,
@@ -36,14 +42,20 @@ def get_metrics(
 
     """
     # Example: fetch Prometheus metrics with basic auth
-    url = f"http://{target_host.address}:9990/telegraf.json"
+    url = f"https://{target_host.address}:9990/telegraf.json"
     username = "prometheus"
-    var_name = "telegraf/password"
-    password_var = get_machine_var(machine, var_name)
-    if not password_var.exists:
+
+    try:
+        password_var = get_machine_var(machine, "telegraf/password")
+        cert_var = get_machine_var(machine, "telegraf-certs/crt")
+    except VarNotFoundError as e:
+        msg = "Module 'monitoring' is required to fetch metrics from machine."
+        raise MonitoringNotEnabledError(msg) from e
+
+    if not password_var.exists or not cert_var.exists:
         msg = (
-            f"Missing required var '{var_name}' for machine '{machine.name}'.\n"
-            "Ensure the 'monitoring' clanService is enabled and run `clan machines update {machine.name}`."
+            f"Missing required var.\n"
+            f"Ensure the 'monitoring' clanService is enabled and run `clan machines update {machine.name}`."
             "For more information, see: https://docs.clan.lol/reference/clanServices/monitoring/"
         )
         raise ClanError(msg)
@@ -53,21 +65,30 @@ def get_metrics(
 
     encoded_credentials = b64encode(credentials.encode("utf-8")).decode("utf-8")
     headers = {"Authorization": f"Basic {encoded_credentials}"}
+
+    cert_path = machine.select(
+        "config.clan.core.vars.generators.telegraf-certs.files.crt.path"
+    )
+    context = ssl.create_default_context(cafile=cert_path)
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_REQUIRED
+
     req = urllib.request.Request(url, headers=headers)  # noqa: S310
 
     try:
-        response = urllib.request.urlopen(req)  # noqa: S310
-        for line in response:
-            line_str = line.decode("utf-8").strip()
-            if line_str:
-                try:
-                    yield cast("MetricSample", json.loads(line_str))
-                except json.JSONDecodeError:
-                    log.warning(f"Skipping invalid JSON line: {line_str}")
-                    continue
+        machine.info(f"Fetching Prometheus metrics from {url}")
+        with urllib.request.urlopen(req, context=context, timeout=10) as response:  # noqa: S310
+            for line in response:
+                line_str = line.decode("utf-8").strip()
+                if line_str:
+                    try:
+                        yield cast("MetricSample", json.loads(line_str))
+                    except json.JSONDecodeError:
+                        machine.warn(f"Skipping invalid JSON line: {line_str}")
+                        continue
     except Exception as e:
         msg = (
-            f"Failed to fetch Prometheus metrics from {url} for machine '{machine.name}': {e}\n"
+            f"Failed to fetch Prometheus metrics from {url}: {e}\n"
             "Ensure the telegraf.service is running and accessible."
         )
         raise ClanError(msg) from e

pkgs/clan-cli/clan_lib/metrics/version.py

@@ -67,8 +67,8 @@ def check_machine_up_to_date(
         ],
     )
 
-    log.debug(
-        f"Checking if {machine.name} needs an update:\n"
+    machine.debug(
+        f"Checking up-to-date:\n"
         f"Machine outPath: {nixos_systems.current_system}\n"
         f"Git outPath    : {git_out_path}\n",
     )