Merge pull request 'Add library function to read public vars' (#5628) from lib-vars-helper into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/5628
Reviewed-by: Kenji Berthold <aks.kenji@protonmail.com>

clanServices/data-mesher/default.nix

@@ -1,4 +1,7 @@
-{ ... }:
+{
+  clanLib,
+  ...
+}:
 let
   sharedInterface =
     { lib, ... }:
@@ -51,15 +54,15 @@ let
       builtins.foldl' (
         urls: name:
         let
-          ipPath = "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value";
+          ip = clanLib.vars.getPublicValue {
+            flake = config.clan.core.settings.directory;
+            machine = name;
+            generator = "zerotier";
+            file = "zerotier-ip";
+            default = null;
+          };
         in
-        if builtins.pathExists ipPath then
+        if ip != null then urls ++ [ "[${ip}]:${builtins.toString settings.network.port}" ] else urls
-          let
-            ip = builtins.readFile ipPath;
-          in
-          urls ++ [ "[${ip}]:${builtins.toString settings.network.port}" ]
-        else
-          urls
       ) [ ] (builtins.attrNames ((roles.admin.machines or { }) // (roles.signer.machines or { })))
     );
 
@@ -156,9 +159,14 @@ in
                   readHostKey =
                     machine:
                     let
-                      path = "${config.clan.core.settings.directory}/vars/per-machine/${machine}/data-mesher-host-key/public_key/value";
+                      publicKey = clanLib.vars.getPublicValue {
+                        flake = config.clan.core.settings.directory;
+                        inherit machine;
+                        generator = "data-mesher-host-key";
+                        file = "public_key";
+                      };
                     in
-                    builtins.elemAt (lib.splitString "\n" (builtins.readFile path)) 1;
+                    builtins.elemAt (lib.splitString "\n" publicKey) 1;
                 in
                 {
                   enable = true;

clanServices/wireguard/default.nix

@@ -54,7 +54,10 @@
       - For other controllers: The controller's /56 subnet
 */
 
-{ ... }:
+{
+  clanLib,
+  ...
+}:
 let
   # Shared module for extraHosts configuration
   extraHostsModule =
@@ -74,10 +77,12 @@ let
           controllerHosts = lib.mapAttrsToList (
             name: _value:
             let
-              prefix = builtins.readFile (
+              prefix = clanLib.vars.getPublicValue {
-                config.clan.core.settings.directory
+                flake = config.clan.core.settings.directory;
-                + "/vars/per-machine/${name}/wireguard-network-${instanceName}/prefix/value"
+                machine = name;
-              );
+                generator = "wireguard-network-${instanceName}";
+                file = "prefix";
+              };
               # Controller IP is always ::1 in their subnet
               ip = prefix + "::1";
             in
@@ -88,20 +93,24 @@ let
           peerHosts = lib.mapAttrsToList (
             peerName: peerValue:
             let
-              peerSuffix = builtins.readFile (
+              peerSuffix = clanLib.vars.getPublicValue {
-                config.clan.core.settings.directory
+                flake = config.clan.core.settings.directory;
-                + "/vars/per-machine/${peerName}/wireguard-network-${instanceName}/suffix/value"
+                machine = peerName;
-              );
+                generator = "wireguard-network-${instanceName}";
+                file = "suffix";
+              };
               # Determine designated controller
               designatedController =
                 if (builtins.length (builtins.attrNames roles.controller.machines) == 1) then
                   (builtins.head (builtins.attrNames roles.controller.machines))
                 else
                   peerValue.settings.controller;
-              controllerPrefix = builtins.readFile (
+              controllerPrefix = clanLib.vars.getPublicValue {
-                config.clan.core.settings.directory
+                flake = config.clan.core.settings.directory;
-                + "/vars/per-machine/${designatedController}/wireguard-network-${instanceName}/prefix/value"
+                machine = designatedController;
-              );
+                generator = "wireguard-network-${instanceName}";
+                file = "prefix";
+              };
               peerIP = controllerPrefix + ":" + peerSuffix;
             in
             "${peerIP} ${peerName}.${domain}"
@@ -220,10 +229,12 @@ in
                 lib.mapAttrsToList (
                   ctrlName: _:
                   let
-                    controllerPrefix = builtins.readFile (
+                    controllerPrefix = clanLib.vars.getPublicValue {
-                      config.clan.core.settings.directory
+                      flake = config.clan.core.settings.directory;
-                      + "/vars/per-machine/${ctrlName}/wireguard-network-${instanceName}/prefix/value"
+                      machine = ctrlName;
-                    );
+                      generator = "wireguard-network-${instanceName}";
+                      file = "prefix";
+                    };
                     peerIP = controllerPrefix + ":" + peerSuffix;
                   in
                   "${peerIP}/56"
@@ -234,20 +245,22 @@ in
 
               # Connect to all controllers
               peers = lib.mapAttrsToList (name: value: {
-                publicKey = (
+                publicKey = clanLib.vars.getPublicValue {
-                  builtins.readFile (
+                  flake = config.clan.core.settings.directory;
-                    config.clan.core.settings.directory
+                  machine = name;
-                    + "/vars/per-machine/${name}/wireguard-keys-${instanceName}/publickey/value"
+                  generator = "wireguard-keys-${instanceName}";
-                  )
+                  file = "publickey";
-                );
+                };
 
                 # Allow each controller's /56 subnet
                 allowedIPs = [
                   "${
-                    builtins.readFile (
+                    clanLib.vars.getPublicValue {
-                      config.clan.core.settings.directory
+                      flake = config.clan.core.settings.directory;
-                      + "/vars/per-machine/${name}/wireguard-network-${instanceName}/prefix/value"
+                      machine = name;
-                    )
+                      generator = "wireguard-network-${instanceName}";
+                      file = "prefix";
+                    }
                   }::/56"
                 ];
 
@@ -349,25 +362,29 @@ in
                 if allPeers ? ${name} then
                   # For peers: they now have our entire /56 subnet
                   {
-                    publicKey = (
+                    publicKey = clanLib.vars.getPublicValue {
-                      builtins.readFile (
+                      flake = config.clan.core.settings.directory;
-                        config.clan.core.settings.directory
+                      machine = name;
-                        + "/vars/per-machine/${name}/wireguard-keys-${instanceName}/publickey/value"
+                      generator = "wireguard-keys-${instanceName}";
-                      )
+                      file = "publickey";
-                    );
+                    };
 
                     # Allow the peer's /96 range in ALL controller subnets
                     allowedIPs = lib.mapAttrsToList (
                       ctrlName: _:
                       let
-                        controllerPrefix = builtins.readFile (
+                        controllerPrefix = clanLib.vars.getPublicValue {
-                          config.clan.core.settings.directory
+                          flake = config.clan.core.settings.directory;
-                          + "/vars/per-machine/${ctrlName}/wireguard-network-${instanceName}/prefix/value"
+                          machine = ctrlName;
-                        );
+                          generator = "wireguard-network-${instanceName}";
-                        peerSuffix = builtins.readFile (
+                          file = "prefix";
-                          config.clan.core.settings.directory
+                        };
-                          + "/vars/per-machine/${name}/wireguard-network-${instanceName}/suffix/value"
+                        peerSuffix = clanLib.vars.getPublicValue {
-                        );
+                          flake = config.clan.core.settings.directory;
+                          machine = name;
+                          generator = "wireguard-network-${instanceName}";
+                          file = "suffix";
+                        };
                       in
                       "${controllerPrefix}:${peerSuffix}/96"
                     ) roles.controller.machines;
@@ -377,19 +394,21 @@ in
                 else
                   # For other controllers: use their /56 subnet
                   {
-                    publicKey = (
+                    publicKey = clanLib.vars.getPublicValue {
-                      builtins.readFile (
+                      flake = config.clan.core.settings.directory;
-                        config.clan.core.settings.directory
+                      machine = name;
-                        + "/vars/per-machine/${name}/wireguard-keys-${instanceName}/publickey/value"
+                      generator = "wireguard-keys-${instanceName}";
-                      )
+                      file = "publickey";
-                    );
+                    };
 
                     allowedIPs = [
                       "${
-                        builtins.readFile (
+                        clanLib.vars.getPublicValue {
-                          config.clan.core.settings.directory
+                          flake = config.clan.core.settings.directory;
-                          + "/vars/per-machine/${name}/wireguard-network-${instanceName}/prefix/value"
+                          machine = name;
-                        )
+                          generator = "wireguard-network-${instanceName}";
+                          file = "prefix";
+                        }
                       }::/56"
                     ];
 

clanServices/zerotier/default.nix

@@ -1,4 +1,7 @@
-{ ... }:
+{
+  clanLib,
+  ...
+}:
 {
   _class = "clan.service";
   manifest.name = "clan-core/zerotier";
@@ -39,6 +42,7 @@
             imports = [
               (import ./shared.nix {
                 inherit
+                  clanLib
                   instanceName
                   roles
                   config
@@ -90,6 +94,7 @@
             imports = [
               (import ./shared.nix {
                 inherit
+                  clanLib
                   instanceName
                   roles
                   config
@@ -142,6 +147,7 @@
             imports = [
               (import ./shared.nix {
                 inherit
+                  clanLib
                   instanceName
                   roles
                   config
@@ -160,15 +166,16 @@
                   );
                   networkIps = builtins.foldl' (
                     ips: name:
-                    if
+                    let
-                      builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
+                      ztIp = clanLib.vars.getPublicValue {
-                    then
+                        flake = config.clan.core.settings.directory;
-                      ips
+                        machine = name;
-                      ++ [
+                        generator = "zerotier";
-                        (builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value")
+                        file = "zerotier-ip";
-                      ]
+                        default = null;
-                    else
+                      };
-                      ips
+                    in
+                    if ztIp != null then ips ++ [ ztIp ] else ips
                   ) [ ] machines;
                   allHostIPs = settings.allowedIps ++ networkIps;
                 in

clanServices/zerotier/shared.nix

@@ -1,4 +1,5 @@
 {
+  clanLib,
   lib,
   config,
   pkgs,
@@ -8,20 +9,26 @@
 }:
 let
   controllerMachine = builtins.head (lib.attrNames roles.controller.machines or { });
-  networkIdPath = "${config.clan.core.settings.directory}/vars/per-machine/${controllerMachine}/zerotier/zerotier-network-id/value";
+  networkId = clanLib.vars.getPublicValue {
-  networkId = if builtins.pathExists networkIdPath then builtins.readFile networkIdPath else null;
+    flake = config.clan.core.settings.directory;
+    machine = controllerMachine;
+    generator = "zerotier";
+    file = "zerotier-network-id";
+    default = null;
+  };
   moons = lib.attrNames (roles.moon.machines or { });
   moonIps = builtins.foldl' (
     ips: name:
-    if
+    let
-      builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
+      moonIp = clanLib.vars.getPublicValue {
-    then
+        flake = config.clan.core.settings.directory;
-      ips
+        machine = name;
-      ++ [
+        generator = "zerotier";
-        (builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value")
+        file = "zerotier-ip";
-      ]
+        default = null;
-    else
+      };
-      ips
+    in
+    if moonIp != null then ips ++ [ moonIp ] else ips
   ) [ ] moons;
 in
 {

lib/default.nix

@@ -30,6 +30,8 @@ lib.fix (
       jsonschema = import ./jsonschema { inherit lib; };
       docs = import ./docs.nix { inherit lib; };
 
+      vars = import ./vars.nix { inherit lib; };
+
       # flakes
       flakes = clanLib.callLib ./flakes.nix { };
 

lib/inventory/distributed-service/all-services-wrapper.nix

@@ -37,6 +37,10 @@ in
               { name, ... }:
               {
                 _module.args._ctx = [ name ];
+                _module.args.clanLib = specialArgs.clanLib;
+                _module.args.exports = config.exports;
+                _module.args.directory = directory;
+
               }
             )
             ./service-module.nix

lib/tests.nix

@@ -212,6 +212,57 @@ in
       };
     };
 
+  test_get_var_machine =
+    let
+      varsLib = import ./vars.nix { };
+    in
+    {
+      expr = varsLib.getPublicValue {
+        backend = "in_repo";
+        default = "test";
+        shared = false;
+        generator = "test-generator";
+        machine = "test-machine";
+        file = "test-file";
+        flake = ./vars-test-flake;
+      };
+      expected = "foo-machine";
+    };
+
+  test_get_var_shared =
+    let
+      varsLib = import ./vars.nix { };
+    in
+    {
+      expr = varsLib.getPublicValue {
+        backend = "in_repo";
+        default = "test";
+        shared = true;
+        generator = "test-generator";
+        machine = "test-machine";
+        file = "test-file";
+        flake = ./vars-test-flake;
+      };
+      expected = "foo-shared";
+    };
+
+  test_get_var_default =
+    let
+      varsLib = import ./vars.nix { };
+    in
+    {
+      expr = varsLib.getPublicValue {
+        backend = "in_repo";
+        default = "test-default";
+        shared = true;
+        generator = "test-generator-wrong";
+        machine = "test-machine";
+        file = "test-file";
+        flake = ./vars-test-flake;
+      };
+      expected = "test-default";
+    };
+
   test_clan_all_machines_laziness =
     let
       eval = clan {

lib/vars-test-flake/vars/per-machine/test-machine/test-generator/test-file/value

@@ -0,0 +1 @@
+foo-machine

lib/vars-test-flake/vars/shared/test-generator/test-file/value

@@ -0,0 +1 @@
+foo-shared

lib/vars.nix

@@ -0,0 +1,25 @@
+_: {
+  getPublicValue =
+    {
+
+      backend ? "in_repo",
+      default ? throw "getPublicValue: Public value ${machine}/${generator}/${file} not found!",
+      shared ? false,
+      generator,
+      machine,
+      file,
+      flake,
+    }:
+
+    if backend == "in_repo" then
+      let
+        path =
+          if shared then
+            "${flake}/vars/shared/${generator}/${file}/value"
+          else
+            "${flake}/vars/per-machine/${machine}/${generator}/${file}/value";
+      in
+      if builtins.pathExists path then builtins.readFile path else default
+    else
+      throw "backend ${backend} does not implement getPublicValue";
+}

lib/vars_test.nix

@@ -0,0 +1,3 @@
+{
+
+}

modules/inventoryClass/inventory.nix

@@ -167,7 +167,7 @@ in
       '';
       type = types.submoduleWith {
         specialArgs = {
-          inherit (config) machines;
+          inherit (config) machines clanLib;
         };
         modules = [
           {