yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'zerotier: migrate from facts to vars' (#3383) from zerotier-vars into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3383
This commit is contained in:
Mic92
2025-04-22 16:48:32 +00:00
parent b5cc250237 043077269a
commit 38cbe58af4
28 changed files with 173 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
clanModules/data-mesher/shared.nix
View File

@@ -13,10 +13,10 @@ let
defaultBootstrapNodes = builtins.foldl' (
urls: name:
if
builtins.pathExists "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip"
builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
then
let
ip = builtins.readFile "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip";
ip = builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value";
in
urls ++ "${ip}:${cfg.network.port}"
else

4
clanModules/static-hosts/default.nix
View File

@@ -26,8 +26,8 @@
config.networking.hosts =
let
dir = config.clan.core.settings.directory;
machineDir = dir + "/machines/";
zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
machineDir = "${dir}/vars/per-machine";
zerotierIpMachinePath = machine: "${machineDir}/${machine}/zerotier/zerotier-ip/value";
machinesFileSet = builtins.readDir machineDir;
machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
networkIpsUnchecked = builtins.map (

9
clanModules/syncthing-static-peers/default.nix
View File

@@ -6,10 +6,9 @@
}:
let
dir = config.clan.core.settings.directory;
machineDir = dir + "/machines/";
machineVarDir = dir + "/vars/per-machine/";
syncthingPublicKeyPath = machines: machineVarDir + machines + "/syncthing/id/value";
machinesFileSet = builtins.readDir machineDir;
machineVarDir = "${dir}/vars/per-machine/";
syncthingPublicKeyPath = machine: "${machineVarDir}/${machine}/syncthing/id/value";
machinesFileSet = builtins.readDir machineVarDir;
machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
syncthingPublicKeysUnchecked = builtins.map (
machine:
@@ -19,7 +18,7 @@ let
if builtins.pathExists fullPath then machine else null
) machines;
syncthingPublicKeyMachines = lib.filter (machine: machine != null) syncthingPublicKeysUnchecked;
zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
zerotierIpMachinePath = machine: "${machineVarDir}/${machine}/zerotier/zerotier-ip/value";
networkIpsUnchecked = builtins.map (
machine:
let

21
clanModules/zerotier-static-peers/default.nix
View File

@@ -6,15 +6,16 @@
}:
let
dir = config.clan.core.settings.directory;
machineDir = dir + "/machines/";
machineDir = "${dir}/vars/per-machine";
# TODO: This should use the inventory
# However we are probably going to replace this with the network module.
machinesFileSet = builtins.readDir machineDir;
machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
zerotierNetworkIdPath = machines: machineDir + machines + "/facts/zerotier-network-id";
networkIdsUnchecked = builtins.map (
machine:
let
fullPath = zerotierNetworkIdPath machine;
fullPath = "${machineDir}/vars/per-machine/${machine}/zerotier/zerotier-network-id/value";
in
if builtins.pathExists fullPath then builtins.readFile fullPath else null
) machines;
@@ -45,13 +46,9 @@ in
config.systemd.services.zerotier-static-peers-autoaccept =
let
zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
zerotierIpFor = machine: "${machineDir}/vars/per-machine/${machine}/zerotier/zerotier-ip/value";
networkIpsUnchecked = builtins.map (
machine:
let
fullPath = zerotierIpMachinePath machine;
in
if builtins.pathExists fullPath then machine else null
machine: if builtins.pathExists (zerotierIpFor machine) then machine else null
) machines;
networkIps = lib.filter (machine: machine != null) networkIpsUnchecked;
machinesWithIp = lib.filterAttrs (name: _: (lib.elem name networkIps)) machinesFileSet;
@@ -60,11 +57,7 @@ in
) machinesWithIp;
hosts = lib.mapAttrsToList (host: _: host) (
lib.mapAttrs' (
machine: _:
let
fullPath = zerotierIpMachinePath machine;
in
lib.nameValuePair (builtins.readFile fullPath) [ machine ]
machine: _: lib.nameValuePair (builtins.readFile (zerotierIpFor machine)) [ machine ]
) filteredMachines
);
allHostIPs = config.clan.zerotier-static-peers.networkIps ++ hosts;

4
clanModules/zerotier/roles/controller.nix
View File

@@ -23,11 +23,11 @@ in
networkIps = builtins.foldl' (
ips: name:
if
builtins.pathExists "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip"
builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
then
ips
++ [
(builtins.readFile "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip")
(builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value")
]
else
ips

17
clanModules/zerotier/shared.nix
View File

@@ -10,17 +10,24 @@ let
zeroTierInstance = config.clan.inventory.services.zerotier.${instanceName};
roles = zeroTierInstance.roles;
controllerMachine = builtins.head roles.controller.machines;
networkIdPath = "${config.clan.core.settings.directory}/machines/${controllerMachine}/facts/zerotier-network-id";
networkId = if builtins.pathExists networkIdPath then builtins.readFile networkIdPath else null;
networkIdPath = "${config.clan.core.settings.directory}/vars/per-machine/${controllerMachine}/zerotier/zerotier-network-id/value";
networkId =
if builtins.pathExists networkIdPath then
builtins.readFile networkIdPath
else
builtins.throw ''
No zerotier network id found for ${controllerMachine}.
Please run `clan vars generate ${controllerMachine}` first.
'';
moons = roles.moon.machines;
moonIps = builtins.foldl' (
ips: name:
if
builtins.pathExists "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip"
builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
then
ips
++ [
(builtins.readFile "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip")
(builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value")
]
else
ips
@@ -62,7 +69,7 @@ in
clan.core.networking.zerotier.networkId = networkId;
clan.core.networking.zerotier.name = instanceName;
# TODO: in future we want to have the node id of our moons in our facts
# TODO: in future we want to have the node id of our moons in our vars
systemd.services.zerotierone.serviceConfig.ExecStartPost = lib.mkIf (moonIps != [ ]) (
lib.mkAfter [
"+${pkgs.writeScript "orbit-moons-by-ip" ''

3
devShell.nix
View File

@@ -29,8 +29,7 @@
select-shell
pkgs.nix-unit
pkgs.tea
# Better error messages than nix 2.18
pkgs.nixVersions.latest
pkgs.nix
self'.packages.tea-create-pr
self'.packages.merge-after-ci
self'.packages.pending-reviews

2
docs/site/getting-started/mesh-vpn.md
View File

@@ -86,7 +86,7 @@ This guide shows you how to configure `zerotier` either through `NixOS Options`
configuration, substituting `<CONTROLLER>` with the controller machine name:
```nix
{ config, ... }: {
clan.core.networking.zerotier.networkId = builtins.readFile (config.clan.core.settings.directory + "/machines/<CONTROLLER>/facts/zerotier-network-id");
clan.core.networking.zerotier.networkId = builtins.readFile ../../vars/per-machine/<CONTROLLER>/zerotier/zerotier-network-id/value;
}
```
1. **Update the New Machine**: Execute:

71
nixosModules/clanCore/zerotier/default.nix
View File

@@ -2,15 +2,11 @@
config,
lib,
pkgs,
options,
...
}:
let
cfg = config.clan.core.networking.zerotier;
facts = config.clan.core.facts.services.zerotier.public or { };
genMoonScript = pkgs.runCommand "genmoon" { nativeBuildInputs = [ pkgs.python3 ]; } ''
install -Dm755 ${./genmoon.py} $out/bin/genmoon
patchShebangs $out/bin/genmoon
'';
in
{
options.clan.core.networking.zerotier = {
@@ -89,15 +85,14 @@ in
};
};
config = lib.mkMerge [
({
{
# Override license so that we can build zerotierone without
# having to re-import nixpkgs.
services.zerotierone.package = lib.mkDefault (pkgs.callPackage ../../../pkgs/zerotierone { });
})
(lib.mkIf ((facts.zerotier-ip.value or null) != null) {
environment.etc."zerotier/ip".text = facts.zerotier-ip.value;
})
}
(lib.mkIf (cfg.networkId != null) {
environment.etc."zerotier/ip".text =
config.clan.core.vars.generators.zerotier.files.zerotier-ip.value;
systemd.network.networks."09-zerotier" = {
matchConfig.Name = "zt*";
@@ -112,12 +107,12 @@ in
"+${pkgs.writeShellScript "init-zerotier" ''
# compare hashes of the current identity secret and the one in the config
hash1=$(sha256sum /var/lib/zerotier-one/identity.secret | cut -d ' ' -f 1)
hash2=$(sha256sum ${config.clan.core.facts.services.zerotier.secret.zerotier-identity-secret.path} | cut -d ' ' -f 1)
hash2=$(sha256sum ${config.clan.core.vars.generators.zerotier.files.zerotier-identity-secret.path} | cut -d ' ' -f 1)
if [[ "$hash1" != "$hash2" ]]; then
echo "Identity secret has changed, backing up old identity to /var/lib/zerotier-one/identity.secret.bac"
cp /var/lib/zerotier-one/identity.secret /var/lib/zerotier-one/identity.secret.bac
cp /var/lib/zerotier-one/identity.public /var/lib/zerotier-one/identity.public.bac
cp ${config.clan.core.facts.services.zerotier.secret.zerotier-identity-secret.path} /var/lib/zerotier-one/identity.secret
cp ${config.clan.core.vars.generators.zerotier.files.zerotier-identity-secret.path} /var/lib/zerotier-one/identity.secret
zerotier-idtool getpublic /var/lib/zerotier-one/identity.secret > /var/lib/zerotier-one/identity.public
fi
@@ -129,7 +124,12 @@ in
if [[ ! -f /var/lib/zerotier-one/moon.json ]]; then
zerotier-idtool initmoon /var/lib/zerotier-one/identity.public > /var/lib/zerotier-one/moon.json
fi
${genMoonScript}/bin/genmoon /var/lib/zerotier-one/moon.json ${builtins.toFile "moon.json" (builtins.toJSON cfg.moon.stableEndpoints)} /var/lib/zerotier-one/moons.d
${
pkgs.runCommand "genmoon" { nativeBuildInputs = [ pkgs.python3 ]; } ''
install -Dm755 ${./genmoon.py} $out/bin/genmoon
patchShebangs $out/bin/genmoon
''
}/bin/genmoon /var/lib/zerotier-one/moon.json ${builtins.toFile "moon.json" (builtins.toJSON cfg.moon.stableEndpoints)} /var/lib/zerotier-one/moons.d
''}
# cleanup old networks
@@ -181,22 +181,26 @@ in
services.zerotierone.localConf.settings.tcpFallbackRelay = "65.21.12.51/4443";
})
(lib.mkIf cfg.controller.enable {
environment.etc."zerotier/ip".text =
config.clan.core.vars.generators.zerotier.files.zerotier-ip.value;
# only the controller needs to have the key in the repo, the other clients can be dynamic
# we generate the zerotier code manually for the controller, since it's part of the bootstrap command
clan.core.facts.services.zerotier = {
public.zerotier-ip = { };
public.zerotier-network-id = { };
secret.zerotier-identity-secret = { };
generator.path = [
clan.core.vars.generators.zerotier = {
migrateFact = "zerotier";
files.zerotier-ip.secret = false;
files.zerotier-network-id.secret = false;
files.zerotier-identity-secret = { };
runtimeInputs = [
config.services.zerotierone.package
pkgs.python3
];
generator.script = ''
script = ''
source ${(pkgs.callPackage ../../../pkgs/minifakeroot { })}/share/minifakeroot/rc
python3 ${./generate.py} --mode network \
--ip "$facts/zerotier-ip" \
--identity-secret "$secrets/zerotier-identity-secret" \
--network-id "$facts/zerotier-network-id"
--ip "$out/zerotier-ip" \
--identity-secret "$out/zerotier-identity-secret" \
--network-id "$out/zerotier-network-id"
'';
};
clan.core.state.zerotier.folders = [ "/var/lib/zerotier-one" ];
@@ -204,23 +208,25 @@ in
environment.systemPackages = [ config.clan.core.clanPkgs.zerotier-members ];
})
(lib.mkIf (!cfg.controller.enable && cfg.networkId != null) {
clan.core.facts.services.zerotier = {
public.zerotier-ip = { };
secret.zerotier-identity-secret = { };
generator.path = [
clan.core.vars.generators.zerotier = {
migrateFact = "zerotier";
files.zerotier-ip.secret = false;
files.zerotier-identity-secret = { };
runtimeInputs = [
config.services.zerotierone.package
pkgs.python3
];
generator.script = ''
script = ''
python3 ${./generate.py} --mode identity \
--ip "$facts/zerotier-ip" \
--identity-secret "$secrets/zerotier-identity-secret" \
--ip "$out/zerotier-ip" \
--identity-secret "$out/zerotier-identity-secret" \
--network-id ${cfg.networkId}
'';
};
})
(lib.mkIf (cfg.controller.enable && (facts.zerotier-network-id.value or null) != null) {
clan.core.networking.zerotier.networkId = facts.zerotier-network-id.value;
(lib.mkIf (cfg.controller.enable && config.clan.core.vars.generators ? zerotier) {
clan.core.networking.zerotier.networkId =
config.clan.core.vars.generators.zerotier.files.zerotier-network-id.value;
clan.core.networking.zerotier.settings = {
authTokens = [ null ];
authorizationEndpoint = "";
@@ -259,7 +265,8 @@ in
zt = false;
};
};
environment.etc."zerotier/network-id".text = facts.zerotier-network-id.value;
environment.etc."zerotier/network-id".text =
config.clan.core.vars.generators.zerotier.files.zerotier-network-id.value;
systemd.services.zerotierone.serviceConfig.ExecStartPost = [
"+${pkgs.writeShellScript "whitelist-controller" ''
${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow ${

10
pkgs/clan-cli/clan_cli/facts/check.py
View File

@@ -1,5 +1,4 @@
import argparse
import importlib
import logging
from clan_cli.completions import add_dynamic_completer, complete_machines
@@ -9,11 +8,6 @@ log = logging.getLogger(__name__)
def check_secrets(machine: Machine, service: None | str = None) -> bool:
secret_facts_module = importlib.import_module(machine.secret_facts_module)
secret_facts_store = secret_facts_module.SecretStore(machine=machine)
public_facts_module = importlib.import_module(machine.public_facts_module)
public_facts_store = public_facts_module.FactStore(machine=machine)
missing_secret_facts = []
missing_public_facts = []
services = [service] if service else list(machine.facts_data.keys())
@@ -23,14 +17,14 @@ def check_secrets(machine: Machine, service: None | str = None) -> bool:
secret_name = secret_fact
else:
secret_name = secret_fact["name"]
if not secret_facts_store.exists(service, secret_name):
if not machine.secret_facts_store.exists(service, secret_name):
machine.info(
f"Secret fact '{secret_fact}' for service '{service}' is missing."
)
missing_secret_facts.append((service, secret_name))
for public_fact in machine.facts_data[service]["public"]:
if not public_facts_store.exists(service, public_fact):
if not machine.public_facts_store.exists(service, public_fact):
machine.info(
f"Public fact '{public_fact}' for service '{service}' is missing."
)

10
pkgs/clan-cli/clan_cli/facts/generate.py
View File

@@ -1,5 +1,4 @@
import argparse
import importlib
import logging
import os
import sys
@@ -161,11 +160,6 @@ def _generate_facts_for_machine(
) -> bool:
local_temp = tmpdir / machine.name
local_temp.mkdir()
secret_facts_module = importlib.import_module(machine.secret_facts_module)
secret_facts_store = secret_facts_module.SecretStore(machine=machine)
public_facts_module = importlib.import_module(machine.public_facts_module)
public_facts_store = public_facts_module.FactStore(machine=machine)
machine_updated = False
@@ -184,8 +178,8 @@ def _generate_facts_for_machine(
machine=machine,
service=service,
regenerate=regenerate,
secret_facts_store=secret_facts_store,
public_facts_store=public_facts_store,
secret_facts_store=machine.secret_facts_store,
public_facts_store=machine.public_facts_store,
tmpdir=local_temp,
prompt=prompt,
)

17
pkgs/clan-cli/clan_cli/facts/list.py
View File

@@ -1,8 +1,6 @@
import argparse
import importlib
import json
import logging
from typing import Any
from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.machines.machines import Machine
@@ -10,24 +8,11 @@ from clan_cli.machines.machines import Machine
log = logging.getLogger(__name__)
# TODO get also secret facts
def get_all_facts(machine: Machine) -> dict:
public_facts_store = get_public_facts_store(machine)
return public_facts_store.get_all()
def get_public_facts_store(machine: Machine) -> Any:
public_facts_module = importlib.import_module(machine.public_facts_module)
public_facts_store = public_facts_module.FactStore(machine=machine)
return public_facts_store
def get_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake=args.flake)
# the raw_facts are bytestrings making them not json serializable
raw_facts = get_all_facts(machine)
raw_facts = machine.public_facts_store.get_all()
facts = {}
for key in raw_facts["TODO"]:
facts[key] = raw_facts["TODO"][key].decode("utf8")

8
pkgs/clan-cli/clan_cli/facts/upload.py
View File

@@ -1,5 +1,4 @@
import argparse
import importlib
import logging
from pathlib import Path
from tempfile import TemporaryDirectory
@@ -12,16 +11,13 @@ log = logging.getLogger(__name__)
def upload_secrets(machine: Machine) -> None:
secret_facts_module = importlib.import_module(machine.secret_facts_module)
secret_facts_store = secret_facts_module.SecretStore(machine=machine)
if not secret_facts_store.needs_upload():
if not machine.secret_facts_store.needs_upload():
machine.info("Secrets already uploaded")
return
with TemporaryDirectory(prefix="facts-upload-") as _tempdir:
local_secret_dir = Path(_tempdir).resolve()
secret_facts_store.upload(local_secret_dir)
machine.secret_facts_store.upload(local_secret_dir)
remote_secret_dir = Path(machine.secrets_upload_directory)
upload(machine.target_host, local_secret_dir, remote_secret_dir)

10
pkgs/clan-cli/clan_cli/flash/flash.py
View File

@@ -1,4 +1,3 @@
import importlib
import json
import logging
import os
@@ -11,10 +10,10 @@ from clan_cli.api import API
from clan_cli.cmd import Log, RunOpts, cmd_with_root, run
from clan_cli.errors import ClanError
from clan_cli.facts.generate import generate_facts
from clan_cli.facts.secret_modules import SecretStoreBase
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_shell
from clan_cli.vars.generate import generate_vars
from clan_cli.vars.upload import upload_secret_vars
from .automount import pause_automounting
from .list import list_possible_keymaps, list_possible_languages
@@ -96,10 +95,6 @@ def flash_machine(
msg = f"Partitioning time secrets are not supported with `clan flash write`: clan.core.vars.generators.{generator.name}.files.{file.name}"
raise ClanError(msg)
secret_facts_module = importlib.import_module(machine.secret_facts_module)
secret_facts_store: SecretStoreBase = secret_facts_module.SecretStore(
machine=machine
)
with TemporaryDirectory(prefix="disko-install-") as _tmpdir:
tmpdir = Path(_tmpdir)
upload_dir = machine.secrets_upload_directory
@@ -110,7 +105,8 @@ def flash_machine(
local_dir = tmpdir / upload_dir
local_dir.mkdir(parents=True)
secret_facts_store.upload(local_dir)
machine.secret_facts_store.upload(local_dir)
upload_secret_vars(machine, local_dir)
disko_install = []
if os.geteuid() != 0:

10
pkgs/clan-cli/clan_cli/machines/delete.py
View File

@@ -14,12 +14,6 @@ from clan_cli.secrets.machines import remove_machine as secrets_machine_remove
from clan_cli.secrets.secrets import (
list_secrets,
)
from clan_cli.vars.list import (
public_store as vars_public_store,
)
from clan_cli.vars.list import (
secret_store as vars_secret_store,
)
from .machines import Machine
@@ -55,8 +49,8 @@ def delete_machine(flake: Flake, name: str) -> None:
shutil.rmtree(secret_path)
machine = Machine(name, flake)
changed_paths.extend(vars_public_store(machine).delete_store())
changed_paths.extend(vars_secret_store(machine).delete_store())
changed_paths.extend(machine.public_vars_store.delete_store())
changed_paths.extend(machine.secret_vars_store.delete_store())
# Remove the machine's key, and update secrets & vars that referenced it:
if secrets_has_machine(flake.path, name):
secrets_machine_remove(flake.path, name)

11
pkgs/clan-cli/clan_cli/machines/facts.py
View File

@@ -1,11 +0,0 @@
from pathlib import Path
from clan_cli.dirs import specific_machine_dir
def machine_has_fact(flake_dir: Path, machine: str, fact: str) -> bool:
return (specific_machine_dir(flake_dir, machine) / "facts" / fact).exists()
def machine_get_fact(flake_dir: Path, machine: str, fact: str) -> str:
return (specific_machine_dir(flake_dir, machine) / "facts" / fact).read_text()

36
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -5,7 +5,7 @@ import re
from dataclasses import dataclass, field
from functools import cached_property
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal
from typing import TYPE_CHECKING, Any
from clan_cli.cmd import Log, RunOpts, run_no_stdout
from clan_cli.errors import ClanCmdError, ClanError
@@ -94,50 +94,24 @@ class Machine:
)
return val
@property
def secret_facts_module(
self,
) -> Literal[
"clan_cli.facts.secret_modules.sops",
"clan_cli.facts.secret_modules.vm",
"clan_cli.facts.secret_modules.password_store",
]:
return self.deployment["facts"]["secretModule"]
@property
def public_facts_module(
self,
) -> Literal[
"clan_cli.facts.public_modules.in_repo", "clan_cli.facts.public_modules.vm"
]:
return self.deployment["facts"]["publicModule"]
@cached_property
def secret_facts_store(self) -> facts_secret_modules.SecretStoreBase:
module = importlib.import_module(self.secret_facts_module)
module = importlib.import_module(self.deployment["facts"]["secretModule"])
return module.SecretStore(machine=self)
@cached_property
def public_facts_store(self) -> facts_public_modules.FactStoreBase:
module = importlib.import_module(self.public_facts_module)
module = importlib.import_module(self.deployment["facts"]["publicModule"])
return module.FactStore(machine=self)
@property
def secret_vars_module(self) -> str:
return self.deployment["vars"]["secretModule"]
@property
def public_vars_module(self) -> str:
return self.deployment["vars"]["publicModule"]
@cached_property
def secret_vars_store(self) -> StoreBase:
module = importlib.import_module(self.secret_vars_module)
module = importlib.import_module(self.deployment["vars"]["secretModule"])
return module.SecretStore(machine=self)
@cached_property
def public_vars_store(self) -> StoreBase:
module = importlib.import_module(self.public_vars_module)
module = importlib.import_module(self.deployment["vars"]["publicModule"])
return module.FactStore(machine=self)
@property

1
pkgs/clan-cli/clan_cli/tests/test_modules.py
View File

@@ -18,7 +18,6 @@ from clan_cli.tests.fixtures_flakes import FlakeForTest
if TYPE_CHECKING:
from .age_keys import KeyPair
# from clan_cli.vars.var import machine_get_fact
from clan_cli.machines.machines import Machine as MachineMachine
from clan_cli.tests.helpers import cli

89
pkgs/clan-cli/clan_cli/tests/test_secrets_generate.py
View File

@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING
import pytest
from clan_cli.facts.secret_modules.sops import SecretStore
from clan_cli.flake import Flake
from clan_cli.machines.facts import machine_get_fact
from clan_cli.machines.machines import Machine
from clan_cli.secrets.folders import sops_secrets_folder
from clan_cli.tests.fixtures_flakes import FlakeForTest
@@ -45,66 +44,90 @@ def test_generate_secret(
"user1",
]
)
cmd = ["facts", "generate", "--flake", str(test_flake_with_core.path), "vm1"]
cmd = [
"vars",
"generate",
"--flake",
str(test_flake_with_core.path),
"vm1",
"--generator",
"zerotier",
]
cli.run(cmd)
store1 = SecretStore(
Machine(name="vm1", flake=Flake(str(test_flake_with_core.path)))
)
assert store1.exists("", "age.key")
assert store1.exists("", "zerotier-identity-secret")
network_id = machine_get_fact(
test_flake_with_core.path, "vm1", "zerotier-network-id"
)
network_id = (
test_flake_with_core.path
/ "vars"
/ "per-machine"
/ "vm1"
/ "zerotier"
/ "zerotier-network-id"
/ "value"
).read_text()
assert len(network_id) == 16
secrets_folder = sops_secrets_folder(test_flake_with_core.path)
age_key = secrets_folder / "vm1-age.key" / "secret"
identity_secret = secrets_folder / "vm1-zerotier-identity-secret" / "secret"
identity_secret = (
test_flake_with_core.path
/ "vars"
/ "per-machine"
/ "vm1"
/ "zerotier"
/ "zerotier-identity-secret"
/ "secret"
)
age_key_mtime = age_key.lstat().st_mtime_ns
secret1_mtime = identity_secret.lstat().st_mtime_ns
# Assert that the age key is valid
age_secret = store1.get("", "age.key").decode()
assert age_secret.isprintable()
assert is_valid_age_key(age_secret)
# test idempotency for vm1 and also generate for vm2
cli.run(["facts", "generate", "--flake", str(test_flake_with_core.path)])
cli.run(
[
"vars",
"generate",
"--flake",
str(test_flake_with_core.path),
"--generator",
"zerotier",
]
)
assert age_key.lstat().st_mtime_ns == age_key_mtime
assert identity_secret.lstat().st_mtime_ns == secret1_mtime
assert (
secrets_folder / "vm1-zerotier-identity-secret" / "machines" / "vm1"
).exists()
store2 = SecretStore(
Machine(name="vm2", flake=Flake(str(test_flake_with_core.path)))
)
# clan vars generate
# TODO: Test vars
# varsStore = VarsSecretStore(
# machine=Machine(name="vm2", flake=FlakeId(str(test_flake_with_core.path)))
# )
# generators = get_generators(str(test_flake_with_core.path), "vm2")
# generator = next((gen for gen in generators if gen.name == "root-password"), None)
# if not generator:
# raise Exception("Generator not found")
# password_update = GeneratorUpdate(
# generator=generator.name, prompt_values={"password": "1234"}
# )
# set_prompts(str(test_flake_with_core.path), "vm2", [password_update])
# assert varsStore.exists(generator, "root-password")
assert store2.exists("", "age.key")
assert store2.exists("", "zerotier-identity-secret")
(
test_flake_with_core.path
/ "vars"
/ "per-machine"
/ "vm2"
/ "zerotier"
/ "zerotier-identity-secret"
/ "secret"
).exists()
ip = machine_get_fact(test_flake_with_core.path, "vm1", "zerotier-ip")
ip = (
test_flake_with_core.path
/ "vars"
/ "per-machine"
/ "vm2"
/ "zerotier"
/ "zerotier-ip"
/ "value"
).read_text()
assert ipaddress.IPv6Address(ip).is_private
# Assert that the age key is valid
age_secret = store2.get("", "age.key").decode()
assert age_secret.isprintable()
assert is_valid_age_key(age_secret)

84
pkgs/clan-cli/clan_cli/tests/test_secrets_password_store.py
View File

@@ -1,84 +0,0 @@
import subprocess
from pathlib import Path
import pytest
from clan_cli.facts.secret_modules.password_store import SecretStore
from clan_cli.flake import Flake
from clan_cli.machines.facts import machine_get_fact
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_shell
from clan_cli.ssh.host import Host
from clan_cli.tests.fixtures_flakes import ClanFlake
from clan_cli.tests.helpers import cli
@pytest.mark.impure
def test_upload_secret(
monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake,
temporary_home: Path,
hosts: list[Host],
) -> None:
flake.clan_modules = [
"root-password",
"user-password",
"sshd",
]
config = flake.machines["vm1"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
config["clan"]["core"]["networking"]["zerotier"]["controller"]["enable"] = True
host = hosts[0]
addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
config["clan"]["core"]["networking"]["targetHost"] = addr
config["clan"]["user-password"]["user"] = "alice"
config["clan"]["user-password"]["prompt"] = False
facts = config["clan"]["core"]["facts"]
facts["secretStore"] = "password-store"
facts["secretUploadDirectory"]["_type"] = "override"
facts["secretUploadDirectory"]["content"] = str(
temporary_home / "flake" / "secrets"
)
facts["secretUploadDirectory"]["priority"] = 50
flake.refresh()
monkeypatch.chdir(flake.path)
gnupghome = temporary_home / "gpg"
gnupghome.mkdir(mode=0o700)
monkeypatch.setenv("GNUPGHOME", str(gnupghome))
monkeypatch.setenv("PASSWORD_STORE_DIR", str(temporary_home / "pass"))
gpg_key_spec = temporary_home / "gpg_key_spec"
gpg_key_spec.write_text(
"""
Key-Type: 1
Key-Length: 1024
Name-Real: Root Superuser
Name-Email: test@local
Expire-Date: 0
%no-protection
"""
)
subprocess.run(
nix_shell(["gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)]),
check=True,
)
subprocess.run(nix_shell(["pass"], ["pass", "init", "test@local"]), check=True)
cli.run(["facts", "generate", "vm1", "--flake", str(flake.path)])
store = SecretStore(Machine(name="vm1", flake=Flake(str(flake.path))))
network_id = machine_get_fact(flake.path, "vm1", "zerotier-network-id")
assert len(network_id) == 16
identity_secret = (
temporary_home / "pass" / "machines" / "vm1" / "zerotier-identity-secret.gpg"
)
secret1_mtime = identity_secret.lstat().st_mtime_ns
# test idempotency
cli.run(["facts", "generate", "vm1"])
assert identity_secret.lstat().st_mtime_ns == secret1_mtime
cli.run(["facts", "upload", "vm1"])
zerotier_identity_secret = flake.path / "secrets" / "zerotier-identity-secret"
assert zerotier_identity_secret.exists()
assert store.exists("", "zerotier-identity-secret")
assert store.exists("", "zerotier-identity-secret")

21
pkgs/clan-cli/clan_cli/vars/check.py
View File

@@ -1,11 +1,9 @@
import argparse
import importlib
import logging
from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from clan_cli.vars._types import StoreBase
log = logging.getLogger(__name__)
@@ -30,11 +28,6 @@ class VarStatus:
def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatus:
secret_vars_module = importlib.import_module(machine.secret_vars_module)
secret_vars_store: StoreBase = secret_vars_module.SecretStore(machine=machine)
public_vars_module = importlib.import_module(machine.public_vars_module)
public_vars_store: StoreBase = public_vars_module.FactStore(machine=machine)
missing_secret_vars = []
missing_public_vars = []
# signals if a var needs to be updated (eg. needs re-encryption due to new users added)
@@ -55,17 +48,19 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu
for generator in generators:
generator.machine(machine)
for file in generator.files:
file.store(secret_vars_store if file.secret else public_vars_store)
file.store(
machine.secret_vars_store if file.secret else machine.public_vars_store
)
file.generator(generator)
if file.secret:
if not secret_vars_store.exists(generator, file.name):
if not machine.secret_vars_store.exists(generator, file.name):
machine.info(
f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing."
)
missing_secret_vars.append(file)
else:
msg = secret_vars_store.health_check(
msg = machine.secret_vars_store.health_check(
generator=generator,
file_name=file.name,
)
@@ -75,15 +70,15 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu
)
unfixed_secret_vars.append(file)
elif not public_vars_store.exists(generator, file.name):
elif not machine.public_vars_store.exists(generator, file.name):
machine.info(
f"Public var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing."
)
missing_public_vars.append(file)
# check if invalidation hash is up to date
if not (
secret_vars_store.hash_is_valid(generator)
and public_vars_store.hash_is_valid(generator)
machine.secret_vars_store.hash_is_valid(generator)
and machine.public_vars_store.hash_is_valid(generator)
):
invalid_generators.append(generator.name)
machine.info(

11
pkgs/clan-cli/clan_cli/vars/fix.py
View File

@@ -1,21 +1,14 @@
import argparse
import importlib
import logging
from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from clan_cli.vars._types import StoreBase
log = logging.getLogger(__name__)
def fix_vars(machine: Machine, generator_name: None | str = None) -> None:
secret_vars_module = importlib.import_module(machine.secret_vars_module)
secret_vars_store: StoreBase = secret_vars_module.SecretStore(machine=machine)
public_vars_module = importlib.import_module(machine.public_vars_module)
public_vars_store: StoreBase = public_vars_module.FactStore(machine=machine)
generators = machine.vars_generators
if generator_name:
for generator in generators:
@@ -29,8 +22,8 @@ def fix_vars(machine: Machine, generator_name: None | str = None) -> None:
raise ClanError(err_msg)
for generator in generators:
public_vars_store.fix(generator=generator)
secret_vars_store.fix(generator=generator)
machine.public_vars_store.fix(generator=generator)
machine.secret_vars_store.fix(generator=generator)
def fix_command(args: argparse.Namespace) -> None:

24
pkgs/clan-cli/clan_cli/vars/list.py
View File

@@ -1,5 +1,4 @@
import argparse
import importlib
import logging
from clan_cli.api import API
@@ -7,7 +6,6 @@ from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.errors import ClanError
from clan_cli.flake import Flake
from clan_cli.machines.machines import Machine
from clan_cli.vars._types import StoreBase
from ._types import GeneratorUpdate
from .generate import Generator, Prompt, Var, execute_generator
@@ -15,21 +13,11 @@ from .generate import Generator, Prompt, Var, execute_generator
log = logging.getLogger(__name__)
def public_store(machine: Machine) -> StoreBase:
public_vars_module = importlib.import_module(machine.public_vars_module)
return public_vars_module.FactStore(machine=machine)
def secret_store(machine: Machine) -> StoreBase:
secret_vars_module = importlib.import_module(machine.secret_vars_module)
return secret_vars_module.SecretStore(machine=machine)
@API.register
def get_vars(base_dir: str, machine_name: str) -> list[Var]:
machine = Machine(name=machine_name, flake=Flake(base_dir))
pub_store = public_store(machine)
sec_store = secret_store(machine)
pub_store = machine.public_vars_store
sec_store = machine.secret_vars_store
all_vars = []
for generator in machine.vars_generators:
for var in generator.files:
@@ -50,10 +38,10 @@ def _get_previous_value(
if not prompt.persist:
return None
pub_store = public_store(machine)
pub_store = machine.public_vars_store
if pub_store.exists(generator, prompt.name):
return pub_store.get(generator, prompt.name).decode()
sec_store = secret_store(machine)
sec_store = machine.secret_vars_store
if sec_store.exists(generator, prompt.name):
return sec_store.get(generator, prompt.name).decode()
return None
@@ -87,8 +75,8 @@ def set_prompts(
execute_generator(
machine,
generator,
secret_vars_store=secret_store(machine),
public_vars_store=public_store(machine),
secret_vars_store=machine.secret_vars_store,
public_vars_store=machine.public_vars_store,
prompt_values=update.prompt_values,
)

9
pkgs/clan-cli/clan_cli/vars/upload.py
View File

@@ -1,5 +1,4 @@
import argparse
import importlib
import logging
from pathlib import Path
@@ -10,12 +9,12 @@ log = logging.getLogger(__name__)
def upload_secret_vars(machine: Machine, directory: Path | None = None) -> None:
secret_store_module = importlib.import_module(machine.secret_vars_module)
secret_store = secret_store_module.SecretStore(machine=machine)
if directory:
secret_store.populate_dir(directory, phases=["activation", "users", "services"])
machine.secret_vars_store.populate_dir(
directory, phases=["activation", "users", "services"]
)
else:
secret_store.upload(phases=["activation", "users", "services"])
machine.secret_vars_store.upload(phases=["activation", "users", "services"])
def upload_command(args: argparse.Namespace) -> None:

14
pkgs/clan-cli/clan_cli/vms/run.py
View File

@@ -1,5 +1,4 @@
import argparse
import importlib
import json
import logging
import os
@@ -22,6 +21,8 @@ from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_shell
from clan_cli.qemu.qga import QgaSession
from clan_cli.qemu.qmp import QEMUMonitorProtocol
from clan_cli.vars.generate import generate_vars
from clan_cli.vars.upload import upload_secret_vars
from .inspect import VmConfig, inspect_vm
from .qemu import qemu_command
@@ -55,9 +56,7 @@ def build_vm(
nix_options = []
secrets_dir = get_secrets(machine, tmpdir)
public_facts_module = importlib.import_module(machine.public_facts_module)
public_facts_store = public_facts_module.FactStore(machine=machine)
public_facts = public_facts_store.get_all()
public_facts = machine.public_facts_store.get_all()
nixos_config_file = machine.build_nix(
"config.system.clan.vm.create",
@@ -81,12 +80,11 @@ def get_secrets(
secrets_dir = tmpdir / "secrets"
secrets_dir.mkdir(parents=True, exist_ok=True)
secret_facts_module = importlib.import_module(machine.secret_facts_module)
secret_facts_store = secret_facts_module.SecretStore(machine=machine)
generate_facts([machine])
generate_vars([machine])
secret_facts_store.upload(secrets_dir)
machine.secret_facts_store.upload(secrets_dir)
upload_secret_vars(machine, secrets_dir)
return secrets_dir

6
pkgs/clan-cli/default.nix
View File

@@ -63,10 +63,8 @@ let
nixpkgs' =
runCommand "nixpkgs"
{
nativeBuildInputs = [
# old nix version doesn't support --flake flag
(if lib.versionAtLeast nix.version "2.24" then nix else nixVersions.latest)
];
# Not all versions have `nix flake update --flake` option
nativeBuildInputs = [ nixVersions.stable ];
}
''
mkdir $out

2
templates/clan/flake-parts/flake.nix
View File

@@ -100,7 +100,7 @@
This will allow sara to share the VPN overlay network with jon
The networkId is generated by the first deployment of jon
*/
# clan.core.networking.zerotier.networkId = builtins.readFile ../jon/facts/zerotier-network-id;
# clan.core.networking.zerotier.networkId = builtins.readFile ../../vars/per-machine/jon/zerotier/zerotier-network-id/value;
};
};
};

2
templates/clan/new-clan/machines/sara/configuration.nix
View File

@@ -30,5 +30,5 @@
This will allow sara to share the VPN overlay network with jon
The networkId is generated by the first deployment of jon
*/
# clan.core.networking.zerotier.networkId = builtins.readFile ../jon/facts/zerotier-network-id;
# clan.core.networking.zerotier.networkId = builtins.readFile ../../vars/per-machine/jon/zerotier/zerotier-network-id/value;
}