yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'refactor: decouple vars stores from machine instances' (#4269) from davhau/vars-new into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4269
This commit is contained in:
kenji
2025-07-08 18:11:03 +00:00
parent 311f11698b 0aa6288edb
commit 07f30928e6
16 changed files with 350 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

288
pkgs/clan-cli/clan_cli/tests/test_vars.py
View File

@@ -161,7 +161,7 @@ def test_generate_public_and_secret_vars(
assert check_vars(machine.name, machine.flake) assert check_vars(machine.name, machine.flake)
# get last commit message # get last commit message
commit_message = run( commit_message = run(
["git", "log", "-3", "--pretty=%B"], ["git", "log", "-5", "--pretty=%B"],
).stdout.strip() ).stdout.strip()
assert ( assert (
"Update vars via generator my_generator for machine my_machine" "Update vars via generator my_generator for machine my_machine"
@@ -184,18 +184,18 @@ def test_generate_public_and_secret_vars(
== "shared" == "shared"
) )
vars_text = stringify_all_vars(machine) vars_text = stringify_all_vars(machine)
in_repo_store = in_repo.FactStore( flake_obj = Flake(str(flake.path))
machine="my_machine", flake=Flake(str(flake.path)) my_generator = Generator("my_generator", machine="my_machine", _flake=flake_obj)
) dependent_generator = Generator(
assert not in_repo_store.exists(Generator("my_generator"), "my_secret") "dependent_generator", machine="my_machine", _flake=flake_obj
sops_store = sops.SecretStore(machine="my_machine", flake=Flake(str(flake.path)))
assert sops_store.exists(Generator("my_generator"), "my_secret")
assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "secret"
assert sops_store.exists(Generator("dependent_generator"), "my_secret")
assert (
sops_store.get(Generator("dependent_generator"), "my_secret").decode()
== "shared"
) )
in_repo_store = in_repo.FactStore(flake=flake_obj)
assert not in_repo_store.exists(my_generator, "my_secret")
sops_store = sops.SecretStore(flake=flake_obj)
assert sops_store.exists(my_generator, "my_secret")
assert sops_store.get(my_generator, "my_secret").decode() == "secret"
assert sops_store.exists(dependent_generator, "my_secret")
assert sops_store.get(dependent_generator, "my_secret").decode() == "shared"
assert "my_generator/my_value: public" in vars_text assert "my_generator/my_value: public" in vars_text
assert "my_generator/my_secret" in vars_text assert "my_generator/my_secret" in vars_text
@@ -262,19 +262,20 @@ def test_generate_secret_var_sops_with_default_group(
monkeypatch.chdir(flake.path) monkeypatch.chdir(flake.path)
cli.run(["secrets", "groups", "add-user", "my_group", sops_setup.user]) cli.run(["secrets", "groups", "add-user", "my_group", sops_setup.user])
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
in_repo_store = in_repo.FactStore( flake_obj = Flake(str(flake.path))
machine="my_machine", flake=Flake(str(flake.path)) first_generator = Generator(
"first_generator", machine="my_machine", _flake=flake_obj
) )
assert not in_repo_store.exists(Generator("first_generator"), "my_secret") second_generator = Generator(
sops_store = sops.SecretStore(machine="my_machine", flake=Flake(str(flake.path))) "second_generator", machine="my_machine", _flake=flake_obj
assert sops_store.exists(Generator("first_generator"), "my_secret")
assert (
sops_store.get(Generator("first_generator"), "my_secret").decode() == "hello\n"
)
assert sops_store.exists(Generator("second_generator"), "my_secret")
assert (
sops_store.get(Generator("second_generator"), "my_secret").decode() == "hello\n"
) )
in_repo_store = in_repo.FactStore(flake=flake_obj)
assert not in_repo_store.exists(first_generator, "my_secret")
sops_store = sops.SecretStore(flake=flake_obj)
assert sops_store.exists(first_generator, "my_secret")
assert sops_store.get(first_generator, "my_secret").decode() == "hello\n"
assert sops_store.exists(second_generator, "my_secret")
assert sops_store.get(second_generator, "my_secret").decode() == "hello\n"
# add another user to the group and check if secret gets re-encrypted # add another user to the group and check if secret gets re-encrypted
pubkey_user2 = sops_setup.keys[1] pubkey_user2 = sops_setup.keys[1]
@@ -292,12 +293,14 @@ def test_generate_secret_var_sops_with_default_group(
cli.run(["secrets", "groups", "add-user", "my_group", "user2"]) cli.run(["secrets", "groups", "add-user", "my_group", "user2"])
# check if new user can access the secret # check if new user can access the secret
monkeypatch.setenv("USER", "user2") monkeypatch.setenv("USER", "user2")
assert sops_store.user_has_access( first_generator_with_share = Generator(
"user2", Generator("first_generator", share=False), "my_secret" "first_generator", share=False, machine="my_machine", _flake=flake_obj
) )
assert sops_store.user_has_access( second_generator_with_share = Generator(
"user2", Generator("second_generator", share=False), "my_secret" "second_generator", share=False, machine="my_machine", _flake=flake_obj
) )
assert sops_store.user_has_access("user2", first_generator_with_share, "my_secret")
assert sops_store.user_has_access("user2", second_generator_with_share, "my_secret")
# Rotate key of a user # Rotate key of a user
pubkey_user3 = sops_setup.keys[2] pubkey_user3 = sops_setup.keys[2]
@@ -314,12 +317,8 @@ def test_generate_secret_var_sops_with_default_group(
] ]
) )
monkeypatch.setenv("USER", "user2") monkeypatch.setenv("USER", "user2")
assert sops_store.user_has_access( assert sops_store.user_has_access("user2", first_generator_with_share, "my_secret")
"user2", Generator("first_generator", share=False), "my_secret" assert sops_store.user_has_access("user2", second_generator_with_share, "my_secret")
)
assert sops_store.user_has_access(
"user2", Generator("second_generator", share=False), "my_secret"
)
@pytest.mark.with_core @pytest.mark.with_core
@@ -351,21 +350,21 @@ def test_generated_shared_secret_sops(
cli.run(["vars", "generate", "--flake", str(flake.path), "machine2"]) cli.run(["vars", "generate", "--flake", str(flake.path), "machine2"])
assert check_vars(machine2.name, machine2.flake) assert check_vars(machine2.name, machine2.flake)
assert check_vars(machine2.name, machine2.flake) assert check_vars(machine2.name, machine2.flake)
m1_sops_store = sops.SecretStore(machine1.name, machine1.flake) m1_sops_store = sops.SecretStore(machine1.flake)
m2_sops_store = sops.SecretStore(machine2.name, machine2.flake) m2_sops_store = sops.SecretStore(machine2.flake)
assert m1_sops_store.exists( # Create generators with machine context for testing
Generator("my_shared_generator", share=True), "my_shared_secret" generator_m1 = Generator(
"my_shared_generator", share=True, machine="machine1", _flake=machine1.flake
) )
assert m2_sops_store.exists( generator_m2 = Generator(
Generator("my_shared_generator", share=True), "my_shared_secret" "my_shared_generator", share=True, machine="machine2", _flake=machine2.flake
)
assert m1_sops_store.machine_has_access(
Generator("my_shared_generator", share=True), "my_shared_secret"
)
assert m2_sops_store.machine_has_access(
Generator("my_shared_generator", share=True), "my_shared_secret"
) )
assert m1_sops_store.exists(generator_m1, "my_shared_secret")
assert m2_sops_store.exists(generator_m2, "my_shared_secret")
assert m1_sops_store.machine_has_access(generator_m1, "my_shared_secret")
assert m2_sops_store.machine_has_access(generator_m2, "my_shared_secret")
@pytest.mark.with_core @pytest.mark.with_core
def test_generate_secret_var_password_store( def test_generate_secret_var_password_store(
@@ -412,43 +411,72 @@ def test_generate_secret_var_password_store(
["git", "config", "user.name", "Test User"], cwd=password_store_dir, check=True ["git", "config", "user.name", "Test User"], cwd=password_store_dir, check=True
) )
machine = Machine(name="my_machine", flake=Flake(str(flake.path))) flake_obj = Flake(str(flake.path))
machine = Machine(name="my_machine", flake=flake_obj)
assert not check_vars(machine.name, machine.flake) assert not check_vars(machine.name, machine.flake)
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
assert check_vars(machine.name, machine.flake) assert check_vars(machine.name, machine.flake)
store = password_store.SecretStore( store = password_store.SecretStore(flake=flake_obj)
machine="my_machine", flake=Flake(str(flake.path)) my_generator = Generator(
"my_generator", share=False, files=[], machine="my_machine", _flake=flake_obj
) )
assert store.exists(Generator("my_generator", share=False, files=[]), "my_secret") my_generator_shared = Generator(
assert not store.exists( "my_generator", share=True, files=[], machine="my_machine", _flake=flake_obj
Generator("my_generator", share=True, files=[]), "my_secret"
) )
assert store.exists( my_shared_generator = Generator(
Generator("my_shared_generator", share=True, files=[]), "my_shared_secret" "my_shared_generator",
share=True,
files=[],
machine="my_machine",
_flake=flake_obj,
) )
assert not store.exists( my_shared_generator_not_shared = Generator(
Generator("my_shared_generator", share=False, files=[]), "my_shared_secret" "my_shared_generator",
share=False,
files=[],
machine="my_machine",
_flake=flake_obj,
) )
assert store.exists(my_generator, "my_secret")
assert not store.exists(my_generator_shared, "my_secret")
assert store.exists(my_shared_generator, "my_shared_secret")
assert not store.exists(my_shared_generator_not_shared, "my_shared_secret")
generator = Generator(name="my_generator", share=False, files=[]) generator = Generator(
name="my_generator",
share=False,
files=[],
machine="my_machine",
_flake=flake_obj,
)
assert store.get(generator, "my_secret").decode() == "hello\n" assert store.get(generator, "my_secret").decode() == "hello\n"
vars_text = stringify_all_vars(machine) vars_text = stringify_all_vars(machine)
assert "my_generator/my_secret" in vars_text assert "my_generator/my_secret" in vars_text
my_generator = Generator("my_generator", share=False, files=[]) my_generator = Generator(
"my_generator", share=False, files=[], machine="my_machine", _flake=flake_obj
)
var_name = "my_secret" var_name = "my_secret"
store.delete(my_generator, var_name) store.delete(my_generator, var_name)
assert not store.exists(my_generator, var_name) assert not store.exists(my_generator, var_name)
store.delete_store() store.delete_store("my_machine")
store.delete_store() # check idempotency store.delete_store("my_machine") # check idempotency
my_generator2 = Generator("my_generator2", share=False, files=[]) my_generator2 = Generator(
"my_generator2", share=False, files=[], machine="my_machine", _flake=flake_obj
)
var_name = "my_secret2" var_name = "my_secret2"
assert not store.exists(my_generator2, var_name) assert not store.exists(my_generator2, var_name)
# The shared secret should still be there, # The shared secret should still be there,
# not sure if we can delete those automatically: # not sure if we can delete those automatically:
my_shared_generator = Generator("my_shared_generator", share=True, files=[]) my_shared_generator = Generator(
"my_shared_generator",
share=True,
files=[],
machine="my_machine",
_flake=flake_obj,
)
var_name = "my_shared_secret" var_name = "my_shared_secret"
assert store.exists(my_shared_generator, var_name) assert store.exists(my_shared_generator, var_name)
@@ -492,29 +520,25 @@ def test_generate_secret_for_multiple_machines(
monkeypatch.chdir(flake.path) monkeypatch.chdir(flake.path)
cli.run(["vars", "generate", "--flake", str(flake.path)]) cli.run(["vars", "generate", "--flake", str(flake.path)])
# check if public vars have been created correctly # check if public vars have been created correctly
in_repo_store1 = in_repo.FactStore(machine="machine1", flake=Flake(str(flake.path))) flake_obj = Flake(str(flake.path))
in_repo_store2 = in_repo.FactStore(machine="machine2", flake=Flake(str(flake.path))) in_repo_store1 = in_repo.FactStore(flake=flake_obj)
assert in_repo_store1.exists(Generator("my_generator"), "my_value") in_repo_store2 = in_repo.FactStore(flake=flake_obj)
assert in_repo_store2.exists(Generator("my_generator"), "my_value")
assert ( # Create generators for each machine
in_repo_store1.get(Generator("my_generator"), "my_value").decode() gen1 = Generator("my_generator", machine="machine1", _flake=flake_obj)
== "machine1\n" gen2 = Generator("my_generator", machine="machine2", _flake=flake_obj)
)
assert ( assert in_repo_store1.exists(gen1, "my_value")
in_repo_store2.get(Generator("my_generator"), "my_value").decode() assert in_repo_store2.exists(gen2, "my_value")
== "machine2\n" assert in_repo_store1.get(gen1, "my_value").decode() == "machine1\n"
) assert in_repo_store2.get(gen2, "my_value").decode() == "machine2\n"
# check if secret vars have been created correctly # check if secret vars have been created correctly
sops_store1 = sops.SecretStore(machine="machine1", flake=Flake(str(flake.path))) sops_store1 = sops.SecretStore(flake=flake_obj)
sops_store2 = sops.SecretStore(machine="machine2", flake=Flake(str(flake.path))) sops_store2 = sops.SecretStore(flake=flake_obj)
assert sops_store1.exists(Generator("my_generator"), "my_secret") assert sops_store1.exists(gen1, "my_secret")
assert sops_store2.exists(Generator("my_generator"), "my_secret") assert sops_store2.exists(gen2, "my_secret")
assert ( assert sops_store1.get(gen1, "my_secret").decode() == "machine1\n"
sops_store1.get(Generator("my_generator"), "my_secret").decode() == "machine1\n" assert sops_store2.get(gen2, "my_secret").decode() == "machine2\n"
)
assert (
sops_store2.get(Generator("my_generator"), "my_secret").decode() == "machine2\n"
)
@pytest.mark.with_core @pytest.mark.with_core
@@ -550,28 +574,27 @@ def test_prompt(
iter(["line input", "my\nmultiline\ninput\n", "prompt_persist"]), iter(["line input", "my\nmultiline\ninput\n", "prompt_persist"]),
) )
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
in_repo_store = in_repo.FactStore( flake_obj = Flake(str(flake.path))
machine="my_machine", flake=Flake(str(flake.path)) my_generator = Generator("my_generator", machine="my_machine", _flake=flake_obj)
) my_generator_with_details = Generator(
assert in_repo_store.exists(Generator("my_generator"), "line_value") name="my_generator",
assert ( share=False,
in_repo_store.get(Generator("my_generator"), "line_value").decode() files=[],
== "line input" machine="my_machine",
_flake=flake_obj,
) )
in_repo_store = in_repo.FactStore(flake=flake_obj)
assert in_repo_store.exists(my_generator, "line_value")
assert in_repo_store.get(my_generator, "line_value").decode() == "line input"
assert in_repo_store.exists(Generator("my_generator"), "multiline_value") assert in_repo_store.exists(my_generator, "multiline_value")
assert ( assert (
in_repo_store.get(Generator("my_generator"), "multiline_value").decode() in_repo_store.get(my_generator, "multiline_value").decode()
== "my\nmultiline\ninput\n" == "my\nmultiline\ninput\n"
) )
sops_store = sops.SecretStore(machine="my_machine", flake=Flake(str(flake.path))) sops_store = sops.SecretStore(flake=flake_obj)
assert sops_store.exists( assert sops_store.exists(my_generator_with_details, "prompt_persist")
Generator(name="my_generator", share=False, files=[]), "prompt_persist" assert sops_store.get(my_generator, "prompt_persist").decode() == "prompt_persist"
)
assert (
sops_store.get(Generator(name="my_generator"), "prompt_persist").decode()
== "prompt_persist"
)
@pytest.mark.with_core @pytest.mark.with_core
@@ -606,21 +629,27 @@ def test_multi_machine_shared_vars(
monkeypatch.chdir(flake.path) monkeypatch.chdir(flake.path)
machine1 = Machine(name="machine1", flake=Flake(str(flake.path))) machine1 = Machine(name="machine1", flake=Flake(str(flake.path)))
machine2 = Machine(name="machine2", flake=Flake(str(flake.path))) machine2 = Machine(name="machine2", flake=Flake(str(flake.path)))
sops_store_1 = sops.SecretStore(machine1.name, machine1.flake) sops_store_1 = sops.SecretStore(machine1.flake)
sops_store_2 = sops.SecretStore(machine2.name, machine2.flake) sops_store_2 = sops.SecretStore(machine2.flake)
in_repo_store_1 = in_repo.FactStore(machine1.name, machine1.flake) in_repo_store_1 = in_repo.FactStore(machine1.flake)
in_repo_store_2 = in_repo.FactStore(machine2.name, machine2.flake) in_repo_store_2 = in_repo.FactStore(machine2.flake)
generator = Generator("shared_generator", share=True) # Create generators with machine context for testing
generator_m1 = Generator(
"shared_generator", share=True, machine="machine1", _flake=machine1.flake
)
generator_m2 = Generator(
"shared_generator", share=True, machine="machine2", _flake=machine2.flake
)
# generate for machine 1 # generate for machine 1
cli.run(["vars", "generate", "--flake", str(flake.path), "machine1"]) cli.run(["vars", "generate", "--flake", str(flake.path), "machine1"])
# read out values for machine 1 # read out values for machine 1
m1_secret = sops_store_1.get(generator, "my_secret") m1_secret = sops_store_1.get(generator_m1, "my_secret")
m1_value = in_repo_store_1.get(generator, "my_value") m1_value = in_repo_store_1.get(generator_m1, "my_value")
# generate for machine 2 # generate for machine 2
cli.run(["vars", "generate", "--flake", str(flake.path), "machine2"]) cli.run(["vars", "generate", "--flake", str(flake.path), "machine2"])
# ensure values are the same for both machines # ensure values are the same for both machines
assert sops_store_2.get(generator, "my_secret") == m1_secret assert sops_store_2.get(generator_m2, "my_secret") == m1_secret
assert in_repo_store_2.get(generator, "my_value") == m1_value assert in_repo_store_2.get(generator_m2, "my_value") == m1_value
# ensure shared secret stays available for all machines after regeneration # ensure shared secret stays available for all machines after regeneration
# regenerate for machine 1 # regenerate for machine 1
@@ -628,15 +657,15 @@ def test_multi_machine_shared_vars(
["vars", "generate", "--flake", str(flake.path), "machine1", "--regenerate"] ["vars", "generate", "--flake", str(flake.path), "machine1", "--regenerate"]
) )
# ensure values changed # ensure values changed
new_secret_1 = sops_store_1.get(generator, "my_secret") new_secret_1 = sops_store_1.get(generator_m1, "my_secret")
new_value_1 = in_repo_store_1.get(generator, "my_value") new_value_1 = in_repo_store_1.get(generator_m1, "my_value")
new_secret_2 = sops_store_2.get(generator, "my_secret") new_secret_2 = sops_store_2.get(generator_m2, "my_secret")
assert new_secret_1 != m1_secret assert new_secret_1 != m1_secret
assert new_value_1 != m1_value assert new_value_1 != m1_value
# ensure that both machines still have access to the same secret # ensure that both machines still have access to the same secret
assert new_secret_1 == new_secret_2 assert new_secret_1 == new_secret_2
assert sops_store_1.machine_has_access(generator, "my_secret") assert sops_store_1.machine_has_access(generator_m1, "my_secret")
assert sops_store_2.machine_has_access(generator, "my_secret") assert sops_store_2.machine_has_access(generator_m2, "my_secret")
@pytest.mark.with_core @pytest.mark.with_core
@@ -665,9 +694,10 @@ def test_api_set_prompts(
}, },
) )
machine = Machine(name="my_machine", flake=Flake(str(flake.path))) machine = Machine(name="my_machine", flake=Flake(str(flake.path)))
store = in_repo.FactStore(machine.name, machine.flake) store = in_repo.FactStore(machine.flake)
assert store.exists(Generator("my_generator"), "prompt1") my_generator = Generator("my_generator", machine="my_machine", _flake=machine.flake)
assert store.get(Generator("my_generator"), "prompt1").decode() == "input1" assert store.exists(my_generator, "prompt1")
assert store.get(my_generator, "prompt1").decode() == "input1"
run_generators( run_generators(
machine_name="my_machine", machine_name="my_machine",
base_dir=flake.path, base_dir=flake.path,
@@ -678,7 +708,7 @@ def test_api_set_prompts(
} }
}, },
) )
assert store.get(Generator("my_generator"), "prompt1").decode() == "input2" assert store.get(my_generator, "prompt1").decode() == "input2"
generators = get_generators( generators = get_generators(
machine_name="my_machine", machine_name="my_machine",
@@ -815,19 +845,21 @@ def test_migration(
assert "Migrated var my_generator/my_value" in caplog.text assert "Migrated var my_generator/my_value" in caplog.text
assert "Migrated secret var my_generator/my_secret" in caplog.text assert "Migrated secret var my_generator/my_secret" in caplog.text
in_repo_store = in_repo.FactStore( flake_obj = Flake(str(flake.path))
machine="my_machine", flake=Flake(str(flake.path)) my_generator = Generator("my_generator", machine="my_machine", _flake=flake_obj)
other_generator = Generator(
"other_generator", machine="my_machine", _flake=flake_obj
) )
sops_store = sops.SecretStore(machine="my_machine", flake=Flake(str(flake.path))) in_repo_store = in_repo.FactStore(flake=flake_obj)
assert in_repo_store.exists(Generator("my_generator"), "my_value") sops_store = sops.SecretStore(flake=flake_obj)
assert in_repo_store.get(Generator("my_generator"), "my_value").decode() == "hello" assert in_repo_store.exists(my_generator, "my_value")
assert sops_store.exists(Generator("my_generator"), "my_secret") assert in_repo_store.get(my_generator, "my_value").decode() == "hello"
assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello" assert sops_store.exists(my_generator, "my_secret")
assert sops_store.get(my_generator, "my_secret").decode() == "hello"
assert in_repo_store.exists(Generator("other_generator"), "other_value") assert in_repo_store.exists(other_generator, "other_value")
assert ( assert (
in_repo_store.get(Generator("other_generator"), "other_value").decode() in_repo_store.get(other_generator, "other_value").decode() == "value-from-vars"
== "value-from-vars"
) )

27
pkgs/clan-cli/clan_cli/vars/_types.py
View File

@@ -29,8 +29,7 @@ class GeneratorUpdate:
class StoreBase(ABC): class StoreBase(ABC):
def __init__(self, machine: str, flake: Flake) -> None: def __init__(self, flake: Flake) -> None:
self.machine = machine
self.flake = flake self.flake = flake
@property @property
@@ -38,6 +37,19 @@ class StoreBase(ABC):
def store_name(self) -> str: def store_name(self) -> str:
pass pass
def get_machine(self, generator: "Generator") -> str:
"""Get machine name from generator, asserting it's not None for now."""
if generator.machine is None:
if generator.share:
# Shared generators don't need a machine for most operations
# but some operations (like SOPS key management) might still need one
# This is a temporary workaround - we should handle this better
msg = f"Shared generator '{generator.name}' requires a machine context for this operation"
raise ClanError(msg)
msg = f"Generator '{generator.name}' has no machine associated"
raise ClanError(msg)
return generator.machine
# get a single fact # get a single fact
@abstractmethod @abstractmethod
def get(self, generator: "Generator", name: str) -> bytes: def get(self, generator: "Generator", name: str) -> bytes:
@@ -65,6 +77,7 @@ class StoreBase(ABC):
def health_check( def health_check(
self, self,
machine: str,
generator: "Generator | None" = None, generator: "Generator | None" = None,
file_name: str | None = None, file_name: str | None = None,
) -> str | None: ) -> str | None:
@@ -72,6 +85,7 @@ class StoreBase(ABC):
def fix( def fix(
self, self,
machine: str,
generator: "Generator | None" = None, generator: "Generator | None" = None,
file_name: str | None = None, file_name: str | None = None,
) -> None: ) -> None:
@@ -87,7 +101,8 @@ class StoreBase(ABC):
def rel_dir(self, generator: "Generator", var_name: str) -> Path: def rel_dir(self, generator: "Generator", var_name: str) -> Path:
if generator.share: if generator.share:
return Path("shared") / generator.name / var_name return Path("shared") / generator.name / var_name
return Path("per-machine") / self.machine / generator.name / var_name machine = self.get_machine(generator)
return Path("per-machine") / machine / generator.name / var_name
def directory(self, generator: "Generator", var_name: str) -> Path: def directory(self, generator: "Generator", var_name: str) -> Path:
return self.flake.path / "vars" / self.rel_dir(generator, var_name) return self.flake.path / "vars" / self.rel_dir(generator, var_name)
@@ -134,7 +149,7 @@ class StoreBase(ABC):
""" """
@abstractmethod @abstractmethod
def delete_store(self) -> Iterable[Path]: def delete_store(self, machine: str) -> Iterable[Path]:
"""Delete the store (all vars) for this machine. """Delete the store (all vars) for this machine.
.. note:: .. note::
@@ -181,9 +196,9 @@ class StoreBase(ABC):
return stored_hash == target_hash return stored_hash == target_hash
@abstractmethod @abstractmethod
def populate_dir(self, output_dir: Path, phases: list[str]) -> None: def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
pass pass
@abstractmethod @abstractmethod
def upload(self, host: Remote, phases: list[str]) -> None: def upload(self, machine: str, host: Remote, phases: list[str]) -> None:
pass pass

1
pkgs/clan-cli/clan_cli/vars/check.py
View File

@@ -65,6 +65,7 @@ def vars_status(
missing_secret_vars.append(file) missing_secret_vars.append(file)
else: else:
msg = machine.secret_vars_store.health_check( msg = machine.secret_vars_store.health_check(
machine=machine.name,
generator=generator, generator=generator,
file_name=file.name, file_name=file.name,
) )

4
pkgs/clan-cli/clan_cli/vars/fix.py
View File

@@ -24,8 +24,8 @@ def fix_vars(machine: Machine, generator_name: None | str = None) -> None:
raise ClanError(err_msg) raise ClanError(err_msg)
for generator in generators: for generator in generators:
machine.public_vars_store.fix(generator=generator) machine.public_vars_store.fix(machine.name, generator=generator)
machine.secret_vars_store.fix(generator=generator) machine.secret_vars_store.fix(machine.name, generator=generator)
def fix_command(args: argparse.Namespace) -> None: def fix_command(args: argparse.Namespace) -> None:

8
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -533,8 +533,12 @@ def create_machine_vars_interactive(
_generator = generator _generator = generator
break break
pub_healtcheck_msg = machine.public_vars_store.health_check(_generator) pub_healtcheck_msg = machine.public_vars_store.health_check(
sec_healtcheck_msg = machine.secret_vars_store.health_check(_generator) machine.name, _generator
)
sec_healtcheck_msg = machine.secret_vars_store.health_check(
machine.name, _generator
)
if pub_healtcheck_msg or sec_healtcheck_msg: if pub_healtcheck_msg or sec_healtcheck_msg:
msg = f"Health check failed for machine {machine.name}:\n" msg = f"Health check failed for machine {machine.name}:\n"

12
pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py
View File

@@ -14,8 +14,8 @@ class FactStore(StoreBase):
def is_secret_store(self) -> bool: def is_secret_store(self) -> bool:
return False return False
def __init__(self, machine: str, flake: Flake) -> None: def __init__(self, flake: Flake) -> None:
super().__init__(machine, flake) super().__init__(flake)
self.works_remotely = False self.works_remotely = False
@property @property
@@ -61,18 +61,18 @@ class FactStore(StoreBase):
fact_folder.rmdir() fact_folder.rmdir()
return [fact_folder] return [fact_folder]
def delete_store(self) -> Iterable[Path]: def delete_store(self, machine: str) -> Iterable[Path]:
flake_root = self.flake.path flake_root = self.flake.path
store_folder = flake_root / "vars/per-machine" / self.machine store_folder = flake_root / "vars/per-machine" / machine
if not store_folder.exists(): if not store_folder.exists():
return [] return []
shutil.rmtree(store_folder) shutil.rmtree(store_folder)
return [store_folder] return [store_folder]
def populate_dir(self, output_dir: Path, phases: list[str]) -> None: def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
msg = "populate_dir is not implemented for public vars stores" msg = "populate_dir is not implemented for public vars stores"
raise NotImplementedError(msg) raise NotImplementedError(msg)
def upload(self, host: Remote, phases: list[str]) -> None: def upload(self, machine: str, host: Remote, phases: list[str]) -> None:
msg = "upload is not implemented for public vars stores" msg = "upload is not implemented for public vars stores"
raise NotImplementedError(msg) raise NotImplementedError(msg)

43
pkgs/clan-cli/clan_cli/vars/public_modules/vm.py
View File

@@ -18,21 +18,26 @@ class FactStore(StoreBase):
def is_secret_store(self) -> bool: def is_secret_store(self) -> bool:
return False return False
def __init__(self, machine: str, flake: Flake) -> None: def __init__(self, flake: Flake) -> None:
super().__init__(machine, flake) super().__init__(flake)
self.works_remotely = False self.works_remotely = False
self.dir = vm_state_dir(flake.identifier, machine) / "facts"
log.debug(
f"FactStore initialized with dir {self.dir}",
extra={"command_prefix": machine},
)
@property @property
def store_name(self) -> str: def store_name(self) -> str:
return "vm" return "vm"
def get_dir(self, machine: str) -> Path:
"""Get the directory for a given machine."""
vars_dir = vm_state_dir(self.flake.identifier, machine) / "facts"
log.debug(
f"FactStore using dir {vars_dir}",
extra={"command_prefix": machine},
)
return vars_dir
def exists(self, generator: Generator, name: str) -> bool: def exists(self, generator: Generator, name: str) -> bool:
fact_path = self.dir / generator.name / name machine = self.get_machine(generator)
fact_path = self.get_dir(machine) / generator.name / name
return fact_path.exists() return fact_path.exists()
def _set( def _set(
@@ -41,21 +46,24 @@ class FactStore(StoreBase):
var: Var, var: Var,
value: bytes, value: bytes,
) -> Path | None: ) -> Path | None:
fact_path = self.dir / generator.name / var.name machine = self.get_machine(generator)
fact_path = self.get_dir(machine) / generator.name / var.name
fact_path.parent.mkdir(parents=True, exist_ok=True) fact_path.parent.mkdir(parents=True, exist_ok=True)
fact_path.write_bytes(value) fact_path.write_bytes(value)
return None return None
# get a single fact # get a single fact
def get(self, generator: Generator, name: str) -> bytes: def get(self, generator: Generator, name: str) -> bytes:
fact_path = self.dir / generator.name / name machine = self.get_machine(generator)
fact_path = self.get_dir(machine) / generator.name / name
if fact_path.exists(): if fact_path.exists():
return fact_path.read_bytes() return fact_path.read_bytes()
msg = f"Fact {name} for service {generator.name} not found" msg = f"Fact {name} for service {generator.name} not found"
raise ClanError(msg) raise ClanError(msg)
def delete(self, generator: Generator, name: str) -> Iterable[Path]: def delete(self, generator: Generator, name: str) -> Iterable[Path]:
fact_dir = self.dir / generator.name machine = self.get_machine(generator)
fact_dir = self.get_dir(machine) / generator.name
fact_file = fact_dir / name fact_file = fact_dir / name
fact_file.unlink() fact_file.unlink()
empty = None empty = None
@@ -63,16 +71,17 @@ class FactStore(StoreBase):
fact_dir.rmdir() fact_dir.rmdir()
return [fact_file] return [fact_file]
def delete_store(self) -> Iterable[Path]: def delete_store(self, machine: str) -> Iterable[Path]:
if not self.dir.exists(): vars_dir = self.get_dir(machine)
if not vars_dir.exists():
return [] return []
shutil.rmtree(self.dir) shutil.rmtree(vars_dir)
return [self.dir] return [vars_dir]
def populate_dir(self, output_dir: Path, phases: list[str]) -> None: def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
msg = "populate_dir is not implemented for public vars stores" msg = "populate_dir is not implemented for public vars stores"
raise NotImplementedError(msg) raise NotImplementedError(msg)
def upload(self, host: Remote, phases: list[str]) -> None: def upload(self, machine: str, host: Remote, phases: list[str]) -> None:
msg = "upload is not implemented for public vars stores" msg = "upload is not implemented for public vars stores"
raise NotImplementedError(msg) raise NotImplementedError(msg)

10
pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py
View File

@@ -13,8 +13,8 @@ class SecretStore(StoreBase):
def is_secret_store(self) -> bool: def is_secret_store(self) -> bool:
return True return True
def __init__(self, machine: str, flake: Flake) -> None: def __init__(self, flake: Flake) -> None:
super().__init__(machine, flake) super().__init__(flake)
self.dir = Path(tempfile.gettempdir()) / "clan_secrets" self.dir = Path(tempfile.gettempdir()) / "clan_secrets"
self.dir.mkdir(parents=True, exist_ok=True) self.dir.mkdir(parents=True, exist_ok=True)
@@ -40,7 +40,7 @@ class SecretStore(StoreBase):
secret_file = self.dir / generator.name / name secret_file = self.dir / generator.name / name
return secret_file.read_bytes() return secret_file.read_bytes()
def populate_dir(self, output_dir: Path, phases: list[str]) -> None: def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
if output_dir.exists(): if output_dir.exists():
shutil.rmtree(output_dir) shutil.rmtree(output_dir)
shutil.copytree(self.dir, output_dir) shutil.copytree(self.dir, output_dir)
@@ -52,11 +52,11 @@ class SecretStore(StoreBase):
secret_file.unlink() secret_file.unlink()
return [] return []
def delete_store(self) -> list[Path]: def delete_store(self, machine: str) -> list[Path]:
if self.dir.exists(): if self.dir.exists():
shutil.rmtree(self.dir) shutil.rmtree(self.dir)
return [] return []
def upload(self, host: Remote, phases: list[str]) -> None: def upload(self, machine: str, host: Remote, phases: list[str]) -> None:
msg = "Cannot upload secrets with FS backend" msg = "Cannot upload secrets with FS backend"
raise NotImplementedError(msg) raise NotImplementedError(msg)

76
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
View File

@@ -20,8 +20,8 @@ class SecretStore(StoreBase):
def is_secret_store(self) -> bool: def is_secret_store(self) -> bool:
return True return True
def __init__(self, machine: str, flake: Flake) -> None: def __init__(self, flake: Flake) -> None:
super().__init__(machine, flake) super().__init__(flake)
self.entry_prefix = "clan-vars" self.entry_prefix = "clan-vars"
self._store_dir: Path | None = None self._store_dir: Path | None = None
@@ -29,25 +29,25 @@ class SecretStore(StoreBase):
def store_name(self) -> str: def store_name(self) -> str:
return "password_store" return "password_store"
@property def store_dir(self, machine: str) -> Path:
def store_dir(self) -> Path: """Get the password store directory, cached per machine."""
"""Get the password store directory, cached after first access.""" if not self._store_dir:
if self._store_dir is None: result = self._run_pass(
result = self._run_pass("git", "rev-parse", "--show-toplevel", check=False) machine, "git", "rev-parse", "--show-toplevel", check=False
)
if result.returncode != 0: if result.returncode != 0:
msg = "Password store must be a git repository" msg = "Password store must be a git repository"
raise ValueError(msg) raise ValueError(msg)
self._store_dir = Path(result.stdout.strip().decode()) self._store_dir = Path(result.stdout.strip().decode())
return self._store_dir return self._store_dir
@property def _pass_command(self, machine: str) -> str:
def _pass_command(self) -> str:
out_path = self.flake.select_machine( out_path = self.flake.select_machine(
self.machine, "config.clan.core.vars.password-store.passPackage.outPath" machine, "config.clan.core.vars.password-store.passPackage.outPath"
) )
main_program = ( main_program = (
self.flake.select_machine( self.flake.select_machine(
self.machine, machine,
"config.clan.core.vars.password-store.passPackage.?meta.?mainProgram", "config.clan.core.vars.password-store.passPackage.?meta.?mainProgram",
) )
.get("meta", {}) .get("meta", {})
@@ -80,11 +80,12 @@ class SecretStore(StoreBase):
def _run_pass( def _run_pass(
self, self,
machine: str,
*args: str, *args: str,
input: bytes | None = None, # noqa: A002 input: bytes | None = None, # noqa: A002
check: bool = True, check: bool = True,
) -> subprocess.CompletedProcess[bytes]: ) -> subprocess.CompletedProcess[bytes]:
cmd = [self._pass_command, *args] cmd = [self._pass_command(machine), *args]
# We need bytes support here, so we can not use clan cmd. # We need bytes support here, so we can not use clan cmd.
# If you change this to run( add bytes support to it first! # If you change this to run( add bytes support to it first!
# otherwise we mangle binary secrets (which is annoying to debug) # otherwise we mangle binary secrets (which is annoying to debug)
@@ -101,37 +102,44 @@ class SecretStore(StoreBase):
var: Var, var: Var,
value: bytes, value: bytes,
) -> Path | None: ) -> Path | None:
machine = self.get_machine(generator)
pass_call = ["insert", "-m", str(self.entry_dir(generator, var.name))] pass_call = ["insert", "-m", str(self.entry_dir(generator, var.name))]
self._run_pass(*pass_call, input=value, check=True) self._run_pass(machine, *pass_call, input=value, check=True)
return None # we manage the files outside of the git repo return None # we manage the files outside of the git repo
def get(self, generator: Generator, name: str) -> bytes: def get(self, generator: Generator, name: str) -> bytes:
machine = self.get_machine(generator)
pass_name = str(self.entry_dir(generator, name)) pass_name = str(self.entry_dir(generator, name))
return self._run_pass("show", pass_name).stdout return self._run_pass(machine, "show", pass_name).stdout
def exists(self, generator: Generator, name: str) -> bool: def exists(self, generator: Generator, name: str) -> bool:
machine = self.get_machine(generator)
pass_name = str(self.entry_dir(generator, name)) pass_name = str(self.entry_dir(generator, name))
# Check if the file exists with either .age or .gpg extension # Check if the file exists with either .age or .gpg extension
age_file = self.store_dir / f"{pass_name}.age" store_dir = self.store_dir(machine)
gpg_file = self.store_dir / f"{pass_name}.gpg" age_file = store_dir / f"{pass_name}.age"
gpg_file = store_dir / f"{pass_name}.gpg"
return age_file.exists() or gpg_file.exists() return age_file.exists() or gpg_file.exists()
def delete(self, generator: Generator, name: str) -> Iterable[Path]: def delete(self, generator: Generator, name: str) -> Iterable[Path]:
machine = self.get_machine(generator)
pass_name = str(self.entry_dir(generator, name)) pass_name = str(self.entry_dir(generator, name))
self._run_pass("rm", "--force", pass_name, check=True) self._run_pass(machine, "rm", "--force", pass_name, check=True)
return [] return []
def delete_store(self) -> Iterable[Path]: def delete_store(self, machine: str) -> Iterable[Path]:
machine_dir = Path(self.entry_prefix) / "per-machine" / self.machine machine_dir = Path(self.entry_prefix) / "per-machine" / machine
# Check if the directory exists in the password store before trying to delete # Check if the directory exists in the password store before trying to delete
result = self._run_pass("ls", str(machine_dir), check=False) result = self._run_pass(machine, "ls", str(machine_dir), check=False)
if result.returncode == 0: if result.returncode == 0:
self._run_pass("rm", "--force", "--recursive", str(machine_dir), check=True) self._run_pass(
machine, "rm", "--force", "--recursive", str(machine_dir), check=True
)
return [] return []
def generate_hash(self) -> bytes: def generate_hash(self, machine: str) -> bytes:
result = self._run_pass( result = self._run_pass(
"git", "log", "-1", "--format=%H", self.entry_prefix, check=False machine, "git", "log", "-1", "--format=%H", self.entry_prefix, check=False
) )
git_hash = result.stdout.strip() git_hash = result.stdout.strip()
@@ -141,7 +149,7 @@ class SecretStore(StoreBase):
from clan_cli.vars.generate import Generator from clan_cli.vars.generate import Generator
manifest = [] manifest = []
generators = Generator.generators_from_flake(self.machine, self.flake) generators = Generator.generators_from_flake(machine, self.flake)
for generator in generators: for generator in generators:
for file in generator.files: for file in generator.files:
manifest.append(f"{generator.name}/{file.name}".encode()) manifest.append(f"{generator.name}/{file.name}".encode())
@@ -149,8 +157,8 @@ class SecretStore(StoreBase):
manifest.append(git_hash) manifest.append(git_hash)
return b"\n".join(manifest) return b"\n".join(manifest)
def needs_upload(self, host: Remote) -> bool: def needs_upload(self, machine: str, host: Remote) -> bool:
local_hash = self.generate_hash() local_hash = self.generate_hash(machine)
if not local_hash: if not local_hash:
return True return True
@@ -159,7 +167,7 @@ class SecretStore(StoreBase):
remote_hash = host.run( remote_hash = host.run(
[ [
"cat", "cat",
f"{self.flake.select_machine(self.machine, 'config.clan.core.vars.password-store.secretLocation')}/.pass_info", f"{self.flake.select_machine(machine, 'config.clan.core.vars.password-store.secretLocation')}/.pass_info",
], ],
RunOpts(log=Log.STDERR, check=False), RunOpts(log=Log.STDERR, check=False),
).stdout.strip() ).stdout.strip()
@@ -169,10 +177,10 @@ class SecretStore(StoreBase):
return local_hash != remote_hash.encode() return local_hash != remote_hash.encode()
def populate_dir(self, output_dir: Path, phases: list[str]) -> None: def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
from clan_cli.vars.generate import Generator from clan_cli.vars.generate import Generator
vars_generators = Generator.generators_from_flake(self.machine, self.flake) vars_generators = Generator.generators_from_flake(machine, self.flake)
if "users" in phases: if "users" in phases:
with tarfile.open( with tarfile.open(
output_dir / "secrets_for_users.tar.gz", "w:gz" output_dir / "secrets_for_users.tar.gz", "w:gz"
@@ -231,23 +239,23 @@ class SecretStore(StoreBase):
out_file.parent.mkdir(parents=True, exist_ok=True) out_file.parent.mkdir(parents=True, exist_ok=True)
out_file.write_bytes(self.get(generator, file.name)) out_file.write_bytes(self.get(generator, file.name))
hash_data = self.generate_hash() hash_data = self.generate_hash(machine)
if hash_data: if hash_data:
(output_dir / ".pass_info").write_bytes(hash_data) (output_dir / ".pass_info").write_bytes(hash_data)
def upload(self, host: Remote, phases: list[str]) -> None: def upload(self, machine: str, host: Remote, phases: list[str]) -> None:
if "partitioning" in phases: if "partitioning" in phases:
msg = "Cannot upload partitioning secrets" msg = "Cannot upload partitioning secrets"
raise NotImplementedError(msg) raise NotImplementedError(msg)
if not self.needs_upload(host): if not self.needs_upload(machine, host):
log.info("Secrets already uploaded") log.info("Secrets already uploaded")
return return
with TemporaryDirectory(prefix="vars-upload-") as _tempdir: with TemporaryDirectory(prefix="vars-upload-") as _tempdir:
pass_dir = Path(_tempdir).resolve() pass_dir = Path(_tempdir).resolve()
self.populate_dir(pass_dir, phases) self.populate_dir(machine, pass_dir, phases)
upload_dir = Path( upload_dir = Path(
self.flake.select_machine( self.flake.select_machine(
self.machine, "config.clan.core.vars.password-store.secretLocation" machine, "config.clan.core.vars.password-store.secretLocation"
) )
) )
upload(host, pass_dir, upload_dir) upload(host, pass_dir, upload_dir)

71
pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py
View File

@@ -48,13 +48,15 @@ class SecretStore(StoreBase):
def is_secret_store(self) -> bool: def is_secret_store(self) -> bool:
return True return True
def __init__(self, machine: str, flake: Flake) -> None: def __init__(self, flake: Flake) -> None:
super().__init__(machine, flake) super().__init__(flake)
def ensure_machine_key(self, machine: str) -> None:
"""Ensure machine has sops keys initialized."""
# no need to generate keys if we don't manage secrets # no need to generate keys if we don't manage secrets
from clan_cli.vars.generate import Generator from clan_cli.vars.generate import Generator
vars_generators = Generator.generators_from_flake(self.machine, self.flake) vars_generators = Generator.generators_from_flake(machine, self.flake)
if not vars_generators: if not vars_generators:
return return
has_secrets = False has_secrets = False
@@ -65,19 +67,19 @@ class SecretStore(StoreBase):
if not has_secrets: if not has_secrets:
return return
if has_machine(self.flake.path, self.machine): if has_machine(self.flake.path, machine):
return return
priv_key, pub_key = sops.generate_private_key() priv_key, pub_key = sops.generate_private_key()
encrypt_secret( encrypt_secret(
self.flake.path, self.flake.path,
sops_secrets_folder(self.flake.path) / f"{self.machine}-age.key", sops_secrets_folder(self.flake.path) / f"{machine}-age.key",
priv_key, priv_key,
add_groups=self.flake.select_machine( add_groups=self.flake.select_machine(
self.machine, "config.clan.core.sops.defaultGroups" machine, "config.clan.core.sops.defaultGroups"
), ),
age_plugins=load_age_plugins(self.flake), age_plugins=load_age_plugins(self.flake),
) )
add_machine(self.flake.path, self.machine, pub_key, False) add_machine(self.flake.path, machine, pub_key, False)
@property @property
def store_name(self) -> str: def store_name(self) -> str:
@@ -90,7 +92,9 @@ class SecretStore(StoreBase):
return self.key_has_access(key_dir, generator, secret_name) return self.key_has_access(key_dir, generator, secret_name)
def machine_has_access(self, generator: Generator, secret_name: str) -> bool: def machine_has_access(self, generator: Generator, secret_name: str) -> bool:
key_dir = sops_machines_folder(self.flake.path) / self.machine machine = self.get_machine(generator)
self.ensure_machine_key(machine)
key_dir = sops_machines_folder(self.flake.path) / machine
return self.key_has_access(key_dir, generator, secret_name) return self.key_has_access(key_dir, generator, secret_name)
def key_has_access( def key_has_access(
@@ -106,7 +110,10 @@ class SecretStore(StoreBase):
@override @override
def health_check( def health_check(
self, generator: Generator | None = None, file_name: str | None = None self,
machine: str,
generator: Generator | None = None,
file_name: str | None = None,
) -> str | None: ) -> str | None:
""" """
Apply local updates to secrets like re-encrypting with missing keys Apply local updates to secrets like re-encrypting with missing keys
@@ -116,7 +123,7 @@ class SecretStore(StoreBase):
if generator is None: if generator is None:
from clan_cli.vars.generate import Generator from clan_cli.vars.generate import Generator
generators = Generator.generators_from_flake(self.machine, self.flake) generators = Generator.generators_from_flake(machine, self.flake)
else: else:
generators = [generator] generators = [generator]
file_found = False file_found = False
@@ -141,7 +148,7 @@ class SecretStore(StoreBase):
if outdated: if outdated:
msg = ( msg = (
"The local state of some secret vars is inconsistent and needs to be updated.\n" "The local state of some secret vars is inconsistent and needs to be updated.\n"
f"Run 'clan vars fix {self.machine}' to apply the necessary changes." f"Run 'clan vars fix {machine}' to apply the necessary changes."
"Problems to fix:\n" "Problems to fix:\n"
"\n".join(o[2] for o in outdated if o[2]) "\n".join(o[2] for o in outdated if o[2])
) )
@@ -154,6 +161,8 @@ class SecretStore(StoreBase):
var: Var, var: Var,
value: bytes, value: bytes,
) -> Path | None: ) -> Path | None:
machine = self.get_machine(generator)
self.ensure_machine_key(machine)
secret_folder = self.secret_path(generator, var.name) secret_folder = self.secret_path(generator, var.name)
# create directory if it doesn't exist # create directory if it doesn't exist
secret_folder.mkdir(parents=True, exist_ok=True) secret_folder.mkdir(parents=True, exist_ok=True)
@@ -162,9 +171,9 @@ class SecretStore(StoreBase):
self.flake.path, self.flake.path,
secret_folder, secret_folder,
value, value,
add_machines=[self.machine] if var.deploy else [], add_machines=[machine] if var.deploy else [],
add_groups=self.flake.select_machine( add_groups=self.flake.select_machine(
self.machine, "config.clan.core.sops.defaultGroups" machine, "config.clan.core.sops.defaultGroups"
), ),
git_commit=False, git_commit=False,
age_plugins=load_age_plugins(self.flake), age_plugins=load_age_plugins(self.flake),
@@ -182,20 +191,20 @@ class SecretStore(StoreBase):
shutil.rmtree(secret_dir) shutil.rmtree(secret_dir)
return [secret_dir] return [secret_dir]
def delete_store(self) -> Iterable[Path]: def delete_store(self, machine: str) -> Iterable[Path]:
flake_root = self.flake.path flake_root = self.flake.path
store_folder = flake_root / "vars/per-machine" / self.machine store_folder = flake_root / "vars/per-machine" / machine
if not store_folder.exists(): if not store_folder.exists():
return [] return []
shutil.rmtree(store_folder) shutil.rmtree(store_folder)
return [store_folder] return [store_folder]
def populate_dir(self, output_dir: Path, phases: list[str]) -> None: def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
from clan_cli.vars.generate import Generator from clan_cli.vars.generate import Generator
vars_generators = Generator.generators_from_flake(self.machine, self.flake) vars_generators = Generator.generators_from_flake(machine, self.flake)
if "users" in phases or "services" in phases: if "users" in phases or "services" in phases:
key_name = f"{self.machine}-age.key" key_name = f"{machine}-age.key"
if not has_secret(sops_secrets_folder(self.flake.path) / key_name): if not has_secret(sops_secrets_folder(self.flake.path) / key_name):
# skip uploading the secret, not managed by us # skip uploading the secret, not managed by us
return return
@@ -237,13 +246,13 @@ class SecretStore(StoreBase):
target_path.chmod(file.mode) target_path.chmod(file.mode)
@override @override
def upload(self, host: Remote, phases: list[str]) -> None: def upload(self, machine: str, host: Remote, phases: list[str]) -> None:
if "partitioning" in phases: if "partitioning" in phases:
msg = "Cannot upload partitioning secrets" msg = "Cannot upload partitioning secrets"
raise NotImplementedError(msg) raise NotImplementedError(msg)
with TemporaryDirectory(prefix="sops-upload-") as _tempdir: with TemporaryDirectory(prefix="sops-upload-") as _tempdir:
sops_upload_dir = Path(_tempdir).resolve() sops_upload_dir = Path(_tempdir).resolve()
self.populate_dir(sops_upload_dir, phases) self.populate_dir(machine, sops_upload_dir, phases)
upload(host, sops_upload_dir, Path("/var/lib/sops-nix")) upload(host, sops_upload_dir, Path("/var/lib/sops-nix"))
def exists(self, generator: Generator, name: str) -> bool: def exists(self, generator: Generator, name: str) -> bool:
@@ -251,17 +260,18 @@ class SecretStore(StoreBase):
return (secret_folder / "secret").exists() return (secret_folder / "secret").exists()
def ensure_machine_has_access(self, generator: Generator, name: str) -> None: def ensure_machine_has_access(self, generator: Generator, name: str) -> None:
machine = self.get_machine(generator)
if self.machine_has_access(generator, name): if self.machine_has_access(generator, name):
return return
secret_folder = self.secret_path(generator, name) secret_folder = self.secret_path(generator, name)
add_secret( add_secret(
self.flake.path, self.flake.path,
self.machine, machine,
secret_folder, secret_folder,
age_plugins=load_age_plugins(self.flake), age_plugins=load_age_plugins(self.flake),
) )
def collect_keys_for_secret(self, path: Path) -> set[sops.SopsKey]: def collect_keys_for_secret(self, machine: str, path: Path) -> set[sops.SopsKey]:
from clan_cli.secrets.secrets import ( from clan_cli.secrets.secrets import (
collect_keys_for_path, collect_keys_for_path,
collect_keys_for_type, collect_keys_for_type,
@@ -269,7 +279,7 @@ class SecretStore(StoreBase):
keys = collect_keys_for_path(path) keys = collect_keys_for_path(path)
for group in self.flake.select_machine( for group in self.flake.select_machine(
self.machine, "config.clan.core.sops.defaultGroups" machine, "config.clan.core.sops.defaultGroups"
): ):
keys.update( keys.update(
collect_keys_for_type( collect_keys_for_type(
@@ -285,9 +295,10 @@ class SecretStore(StoreBase):
return keys return keys
def needs_fix(self, generator: Generator, name: str) -> tuple[bool, str | None]: def needs_fix(self, generator: Generator, name: str) -> tuple[bool, str | None]:
machine = self.get_machine(generator)
secret_path = self.secret_path(generator, name) secret_path = self.secret_path(generator, name)
current_recipients = sops.get_recipients(secret_path) current_recipients = sops.get_recipients(secret_path)
wanted_recipients = self.collect_keys_for_secret(secret_path) wanted_recipients = self.collect_keys_for_secret(machine, secret_path)
needs_update = current_recipients != wanted_recipients needs_update = current_recipients != wanted_recipients
recipients_to_add = wanted_recipients - current_recipients recipients_to_add = wanted_recipients - current_recipients
var_id = f"{generator.name}/{name}" var_id = f"{generator.name}/{name}"
@@ -295,20 +306,23 @@ class SecretStore(StoreBase):
f"One or more recipient keys were added to secret{' shared' if generator.share else ''} var '{var_id}', but it was never re-encrypted.\n" f"One or more recipient keys were added to secret{' shared' if generator.share else ''} var '{var_id}', but it was never re-encrypted.\n"
f"This could have been a malicious actor trying to add their keys, please investigate.\n" f"This could have been a malicious actor trying to add their keys, please investigate.\n"
f"Added keys: {', '.join(f'{r.key_type.name}:{r.pubkey}' for r in recipients_to_add)}\n" f"Added keys: {', '.join(f'{r.key_type.name}:{r.pubkey}' for r in recipients_to_add)}\n"
f"If this is intended, run 'clan vars fix {self.machine}' to re-encrypt the secret." f"If this is intended, run 'clan vars fix {machine}' to re-encrypt the secret."
) )
return needs_update, msg return needs_update, msg
@override @override
def fix( def fix(
self, generator: Generator | None = None, file_name: str | None = None self,
machine: str,
generator: Generator | None = None,
file_name: str | None = None,
) -> None: ) -> None:
from clan_cli.secrets.secrets import update_keys from clan_cli.secrets.secrets import update_keys
if generator is None: if generator is None:
from clan_cli.vars.generate import Generator from clan_cli.vars.generate import Generator
generators = Generator.generators_from_flake(self.machine, self.flake) generators = Generator.generators_from_flake(machine, self.flake)
else: else:
generators = [generator] generators = [generator]
file_found = False file_found = False
@@ -327,8 +341,9 @@ class SecretStore(StoreBase):
age_plugins = load_age_plugins(self.flake) age_plugins = load_age_plugins(self.flake)
gen_machine = self.get_machine(generator)
for group in self.flake.select_machine( for group in self.flake.select_machine(
self.machine, "config.clan.core.sops.defaultGroups" gen_machine, "config.clan.core.sops.defaultGroups"
): ):
allow_member( allow_member(
groups_folder(secret_path), groups_folder(secret_path),

40
pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py
View File

@@ -14,35 +14,43 @@ class SecretStore(StoreBase):
def is_secret_store(self) -> bool: def is_secret_store(self) -> bool:
return True return True
def __init__(self, machine: str, flake: Flake) -> None: def __init__(self, flake: Flake) -> None:
super().__init__(machine, flake) super().__init__(flake)
self.dir = vm_state_dir(flake.identifier, machine) / "secrets"
self.dir.mkdir(parents=True, exist_ok=True)
@property @property
def store_name(self) -> str: def store_name(self) -> str:
return "vm" return "vm"
def get_dir(self, machine: str) -> Path:
"""Get the directory for a given machine, creating it if needed."""
vars_dir = vm_state_dir(self.flake.identifier, machine) / "secrets"
vars_dir.mkdir(parents=True, exist_ok=True)
return vars_dir
def _set( def _set(
self, self,
generator: Generator, generator: Generator,
var: Var, var: Var,
value: bytes, value: bytes,
) -> Path | None: ) -> Path | None:
secret_file = self.dir / generator.name / var.name machine = self.get_machine(generator)
secret_file = self.get_dir(machine) / generator.name / var.name
secret_file.parent.mkdir(parents=True, exist_ok=True) secret_file.parent.mkdir(parents=True, exist_ok=True)
secret_file.write_bytes(value) secret_file.write_bytes(value)
return None # we manage the files outside of the git repo return None # we manage the files outside of the git repo
def exists(self, generator: "Generator", name: str) -> bool: def exists(self, generator: "Generator", name: str) -> bool:
return (self.dir / generator.name / name).exists() machine = self.get_machine(generator)
return (self.get_dir(machine) / generator.name / name).exists()
def get(self, generator: Generator, name: str) -> bytes: def get(self, generator: Generator, name: str) -> bytes:
secret_file = self.dir / generator.name / name machine = self.get_machine(generator)
secret_file = self.get_dir(machine) / generator.name / name
return secret_file.read_bytes() return secret_file.read_bytes()
def delete(self, generator: Generator, name: str) -> Iterable[Path]: def delete(self, generator: Generator, name: str) -> Iterable[Path]:
secret_dir = self.dir / generator.name machine = self.get_machine(generator)
secret_dir = self.get_dir(machine) / generator.name
secret_file = secret_dir / name secret_file = secret_dir / name
secret_file.unlink() secret_file.unlink()
empty = None empty = None
@@ -50,17 +58,19 @@ class SecretStore(StoreBase):
secret_dir.rmdir() secret_dir.rmdir()
return [secret_file] return [secret_file]
def delete_store(self) -> Iterable[Path]: def delete_store(self, machine: str) -> Iterable[Path]:
if not self.dir.exists(): vars_dir = self.get_dir(machine)
if not vars_dir.exists():
return [] return []
shutil.rmtree(self.dir) shutil.rmtree(vars_dir)
return [self.dir] return [vars_dir]
def populate_dir(self, output_dir: Path, phases: list[str]) -> None: def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
vars_dir = self.get_dir(machine)
if output_dir.exists(): if output_dir.exists():
shutil.rmtree(output_dir) shutil.rmtree(output_dir)
shutil.copytree(self.dir, output_dir) shutil.copytree(vars_dir, output_dir)
def upload(self, host: Remote, phases: list[str]) -> None: def upload(self, machine: str, host: Remote, phases: list[str]) -> None:
msg = "Cannot upload secrets to VMs" msg = "Cannot upload secrets to VMs"
raise NotImplementedError(msg) raise NotImplementedError(msg)

6
pkgs/clan-cli/clan_cli/vars/upload.py
View File

@@ -10,12 +10,14 @@ log = logging.getLogger(__name__)
def upload_secret_vars(machine: Machine, host: Remote) -> None: def upload_secret_vars(machine: Machine, host: Remote) -> None:
machine.secret_vars_store.upload(host, phases=["activation", "users", "services"]) machine.secret_vars_store.upload(
machine.name, host, phases=["activation", "users", "services"]
)
def populate_secret_vars(machine: Machine, directory: Path) -> None: def populate_secret_vars(machine: Machine, directory: Path) -> None:
machine.secret_vars_store.populate_dir( machine.secret_vars_store.populate_dir(
directory, phases=["activation", "users", "services"] machine.name, directory, phases=["activation", "users", "services"]
) )

4
pkgs/clan-cli/clan_lib/machines/delete.py
View File

@@ -58,8 +58,8 @@ def delete_machine(machine: Machine) -> None:
changed_paths.append(secret_path) changed_paths.append(secret_path)
shutil.rmtree(secret_path) shutil.rmtree(secret_path)
changed_paths.extend(machine.public_vars_store.delete_store()) changed_paths.extend(machine.public_vars_store.delete_store(machine.name))
changed_paths.extend(machine.secret_vars_store.delete_store()) changed_paths.extend(machine.secret_vars_store.delete_store(machine.name))
# Remove the machine's key, and update secrets & vars that referenced it: # Remove the machine's key, and update secrets & vars that referenced it:
if secrets_has_machine(machine.flake.path, machine.name): if secrets_has_machine(machine.flake.path, machine.name):
secrets_machine_remove(machine.flake.path, machine.name) secrets_machine_remove(machine.flake.path, machine.name)

4
pkgs/clan-cli/clan_lib/machines/install.py
View File

@@ -66,13 +66,13 @@ def run_machine_install(opts: InstallOptions, target_host: Remote) -> None:
upload_dir.mkdir(parents=True) upload_dir.mkdir(parents=True)
machine.secret_facts_store.upload(upload_dir) machine.secret_facts_store.upload(upload_dir)
machine.secret_vars_store.populate_dir( machine.secret_vars_store.populate_dir(
upload_dir, phases=["activation", "users", "services"] machine.name, upload_dir, phases=["activation", "users", "services"]
) )
partitioning_secrets = base_directory / "partitioning_secrets" partitioning_secrets = base_directory / "partitioning_secrets"
partitioning_secrets.mkdir(parents=True) partitioning_secrets.mkdir(parents=True)
machine.secret_vars_store.populate_dir( machine.secret_vars_store.populate_dir(
partitioning_secrets, phases=["partitioning"] machine.name, partitioning_secrets, phases=["partitioning"]
) )
if opts.password: if opts.password:

4
pkgs/clan-cli/clan_lib/machines/machines.py
View File

@@ -104,13 +104,13 @@ class Machine:
def secret_vars_store(self) -> StoreBase: def secret_vars_store(self) -> StoreBase:
secret_module = self.select("config.clan.core.vars.settings.secretModule") secret_module = self.select("config.clan.core.vars.settings.secretModule")
module = importlib.import_module(secret_module) module = importlib.import_module(secret_module)
return module.SecretStore(machine=self.name, flake=self.flake) return module.SecretStore(flake=self.flake)
@cached_property @cached_property
def public_vars_store(self) -> StoreBase: def public_vars_store(self) -> StoreBase:
public_module = self.select("config.clan.core.vars.settings.publicModule") public_module = self.select("config.clan.core.vars.settings.publicModule")
module = importlib.import_module(public_module) module = importlib.import_module(public_module)
return module.FactStore(machine=self.name, flake=self.flake) return module.FactStore(flake=self.flake)
@property @property
def facts_data(self) -> dict[str, dict[str, Any]]: def facts_data(self) -> dict[str, dict[str, Any]]:

4
pkgs/clan-cli/clan_lib/machines/morph.py
View File

@@ -81,7 +81,9 @@ def morph_machine(
generate_vars([machine], generator_name=None, regenerate=False) generate_vars([machine], generator_name=None, regenerate=False)
machine.secret_vars_store.populate_dir( machine.secret_vars_store.populate_dir(
output_dir=Path("/run/secrets"), phases=["activation", "users", "services"] machine.name,
output_dir=Path("/run/secrets"),
phases=["activation", "users", "services"],
) )
# run(["nixos-facter", "-o", f"{flakedir}/machines/{name}/facter.json"]).stdout # run(["nixos-facter", "-o", f"{flakedir}/machines/{name}/facter.json"]).stdout