yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

vars: introduce share flag

Browse Source
This commit is contained in:
DavHau
2024-08-03 12:34:46 +07:00
parent 7788767058
commit d1c2f0b622
12 changed files with 237 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
nixosModules/clanCore/vars/default.nix
View File

@@ -39,7 +39,12 @@ in
vars = {
generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
_name: generator: {
inherit (generator) dependencies finalScript prompts;
inherit (generator)
dependencies
finalScript
prompts
share
;
files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
}
);

9
nixosModules/clanCore/vars/interface.nix
View File

@@ -168,6 +168,15 @@ in
internal = true;
visible = false;
};
share = {
description = ''
Whether the generated vars should be shared between machines.
Shared vars are only generated once, when the first machine using it is deployed.
Subsequent machines will re-use the already generated values.
'';
type = bool;
default = false;
};
};
})
);

29
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -57,6 +57,7 @@ def decrypt_dependencies(
generator_name: str,
secret_vars_store: SecretStoreBase,
public_vars_store: FactStoreBase,
shared: bool,
) -> dict[str, dict[str, bytes]]:
generator = machine.vars_generators[generator_name]
dependencies = set(generator["dependencies"])
@@ -67,11 +68,11 @@ def decrypt_dependencies(
for file_name, file in dep_files.items():
if file["secret"]:
decrypted_dependencies[dep_generator][file_name] = (
secret_vars_store.get(dep_generator, file_name)
secret_vars_store.get(dep_generator, file_name, shared=shared)
)
else:
decrypted_dependencies[dep_generator][file_name] = (
public_vars_store.get(dep_generator, file_name)
public_vars_store.get(dep_generator, file_name, shared=shared)
)
return decrypted_dependencies
@@ -109,10 +110,11 @@ def execute_generator(
msg += "fact/secret generation is only supported for local flakes"
generator = machine.vars_generators[generator_name]["finalScript"]
is_shared = machine.vars_generators[generator_name]["share"]
# build temporary file tree of dependencies
decrypted_dependencies = decrypt_dependencies(
machine, generator_name, secret_vars_store, public_vars_store
machine, generator_name, secret_vars_store, public_vars_store, shared=is_shared
)
env = os.environ.copy()
with TemporaryDirectory() as tmp:
@@ -159,11 +161,18 @@ def execute_generator(
raise ClanError(msg)
if file["secret"]:
file_path = secret_vars_store.set(
generator_name, file_name, secret_file.read_bytes(), groups
generator_name,
file_name,
secret_file.read_bytes(),
groups,
shared=is_shared,
)
else:
file_path = public_vars_store.set(
generator_name, file_name, secret_file.read_bytes()
generator_name,
file_name,
secret_file.read_bytes(),
shared=is_shared,
)
if file_path:
files_to_commit.append(file_path)
@@ -260,18 +269,18 @@ def generate_vars(
) -> bool:
was_regenerated = False
for machine in machines:
errors = 0
errors = []
try:
was_regenerated |= _generate_vars_for_machine(
machine, generator_name, regenerate
)
except Exception as exc:
log.error(f"Failed to generate facts for {machine.name}: {exc}")
errors += 1
if errors > 0:
errors += [exc]
if len(errors) > 0:
raise ClanError(
f"Failed to generate facts for {errors} hosts. Check the logs above"
)
f"Failed to generate facts for {len(errors)} hosts. Check the logs above"
) from errors[0]
if not was_regenerated:
print("All secrets and facts are already up to date")

8
pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py
View File

@@ -10,16 +10,18 @@ class FactStoreBase(ABC):
pass
@abstractmethod
def exists(self, service: str, name: str) -> bool:
def exists(self, service: str, name: str, shared: bool = False) -> bool:
pass
@abstractmethod
def set(self, service: str, name: str, value: bytes) -> Path | None:
def set(
self, service: str, name: str, value: bytes, shared: bool = False
) -> Path | None:
pass
# get a single fact
@abstractmethod
def get(self, service: str, name: str) -> bytes:
def get(self, service: str, name: str, shared: bool = False) -> bytes:
pass
# get all facts

55
pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py
View File

@@ -10,17 +10,22 @@ class FactStore(FactStoreBase):
def __init__(self, machine: Machine) -> None:
self.machine = machine
self.works_remotely = False
self.per_machine_folder = (
self.machine.flake_dir / "vars" / "per-machine" / self.machine.name
)
self.shared_folder = self.machine.flake_dir / "vars" / "shared"
def set(self, generator_name: str, name: str, value: bytes) -> Path | None:
def _var_path(self, generator_name: str, name: str, shared: bool) -> Path:
if shared:
return self.shared_folder / generator_name / name
else:
return self.per_machine_folder / generator_name / name
def set(
self, generator_name: str, name: str, value: bytes, shared: bool = False
) -> Path | None:
if self.machine.flake.is_local():
fact_path = (
self.machine.flake.path
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
)
fact_path = self._var_path(generator_name, name, shared)
fact_path.parent.mkdir(parents=True, exist_ok=True)
fact_path.touch()
fact_path.write_bytes(value)
@@ -30,35 +35,21 @@ class FactStore(FactStoreBase):
f"in_flake fact storage is only supported for local flakes: {self.machine.flake}"
)
def exists(self, generator_name: str, name: str) -> bool:
fact_path = (
self.machine.flake_dir
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
)
return fact_path.exists()
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
return self._var_path(generator_name, name, shared).exists()
# get a single fact
def get(self, generator_name: str, name: str) -> bytes:
fact_path = (
self.machine.flake_dir
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
)
return fact_path.read_bytes()
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
return self._var_path(generator_name, name, shared).read_bytes()
# get all public vars
def get_all(self) -> dict[str, dict[str, bytes]]:
facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "vars"
facts: dict[str, dict[str, bytes]] = {}
facts["TODO"] = {}
if facts_folder.exists():
for fact_path in facts_folder.iterdir():
if self.per_machine_folder.exists():
for fact_path in self.per_machine_folder.iterdir():
facts["TODO"][fact_path.name] = fact_path.read_bytes()
if self.shared_folder.exists():
for fact_path in self.shared_folder.iterdir():
facts["TODO"][fact_path.name] = fact_path.read_bytes()
return facts

8
pkgs/clan-cli/clan_cli/vars/public_modules/vm.py
View File

@@ -17,18 +17,20 @@ class FactStore(FactStoreBase):
self.dir = vm_state_dir(str(machine.flake), machine.name) / "facts"
log.debug(f"FactStore initialized with dir {self.dir}")
def exists(self, service: str, name: str) -> bool:
def exists(self, service: str, name: str, shared: bool = False) -> bool:
fact_path = self.dir / service / name
return fact_path.exists()
def set(self, service: str, name: str, value: bytes) -> Path | None:
def set(
self, service: str, name: str, value: bytes, shared: bool = False
) -> Path | None:
fact_path = self.dir / service / name
fact_path.parent.mkdir(parents=True, exist_ok=True)
fact_path.write_bytes(value)
return None
# get a single fact
def get(self, service: str, name: str) -> bytes:
def get(self, service: str, name: str, shared: bool = False) -> bytes:
fact_path = self.dir / service / name
if fact_path.exists():
return fact_path.read_bytes()

11
pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py
View File

@@ -11,16 +11,21 @@ class SecretStoreBase(ABC):
@abstractmethod
def set(
self, service: str, name: str, value: bytes, groups: list[str]
self,
service: str,
name: str,
value: bytes,
groups: list[str],
shared: bool = False,
) -> Path | None:
pass
@abstractmethod
def get(self, service: str, name: str) -> bytes:
def get(self, service: str, name: str, shared: bool = False) -> bytes:
pass
@abstractmethod
def exists(self, service: str, name: str) -> bool:
def exists(self, service: str, name: str, shared: bool = False) -> bool:
pass
def update_check(self) -> bool:

63
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
View File

@@ -12,8 +12,25 @@ class SecretStore(SecretStoreBase):
def __init__(self, machine: Machine) -> None:
self.machine = machine
@property
def _password_store_dir(self) -> str:
return os.environ.get(
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
)
def _var_path(self, generator_name: str, name: str, shared: bool) -> Path:
if shared:
return Path(f"shared/{generator_name}/{name}")
else:
return Path(f"machines/{self.machine.name}/{generator_name}/{name}")
def set(
self, generator_name: str, name: str, value: bytes, groups: list[str]
self,
generator_name: str,
name: str,
value: bytes,
groups: list[str],
shared: bool = False,
) -> Path | None:
subprocess.run(
nix_shell(
@@ -22,7 +39,7 @@ class SecretStore(SecretStoreBase):
"pass",
"insert",
"-m",
f"machines/{self.machine.name}/{generator_name}/{name}",
str(self._var_path(generator_name, name, shared)),
],
),
input=value,
@@ -30,34 +47,28 @@ class SecretStore(SecretStoreBase):
)
return None # we manage the files outside of the git repo
def get(self, generator_name: str, name: str) -> bytes:
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
return subprocess.run(
nix_shell(
["nixpkgs#pass"],
[
"pass",
"show",
f"machines/{self.machine.name}/{generator_name}/{name}",
str(self._var_path(generator_name, name, shared)),
],
),
check=True,
stdout=subprocess.PIPE,
).stdout
def exists(self, generator_name: str, name: str) -> bool:
password_store = os.environ.get(
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
)
secret_path = (
Path(password_store)
/ f"machines/{self.machine.name}/{generator_name}/{name}.gpg"
)
return secret_path.exists()
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
return (
Path(self._password_store_dir)
/ f"{self._var_path(generator_name, name, shared)}.gpg"
).exists()
def generate_hash(self) -> bytes:
password_store = os.environ.get(
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
)
password_store = self._password_store_dir
hashes = []
hashes.append(
subprocess.run(
@@ -117,13 +128,15 @@ class SecretStore(SecretStoreBase):
return local_hash.decode() == remote_hash
# TODO: fixme
def upload(self, output_dir: Path) -> None:
for service in self.machine.facts_data:
for secret in self.machine.facts_data[service]["secret"]:
if isinstance(secret, dict):
secret_name = secret["name"]
else:
# TODO: drop old format soon
secret_name = secret
(output_dir / secret_name).write_bytes(self.get(service, secret_name))
(output_dir / ".pass_info").write_bytes(self.generate_hash())
pass
# for service in self.machine.facts_data:
# for secret in self.machine.facts_data[service]["secret"]:
# if isinstance(secret, dict):
# secret_name = secret["name"]
# else:
# # TODO: drop old format soon
# secret_name = secret
# (output_dir / secret_name).write_bytes(self.get(service, secret_name))
# (output_dir / ".pass_info").write_bytes(self.generate_hash())

40
pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py
View File

@@ -36,20 +36,30 @@ class SecretStore(SecretStoreBase):
)
add_machine(self.machine.flake_dir, self.machine.name, pub_key, False)
def secret_path(self, generator_name: str, secret_name: str) -> Path:
return (
self.machine.flake_dir
/ "sops"
/ "vars"
/ self.machine.name
/ generator_name
/ secret_name
)
def secret_path(
self, generator_name: str, secret_name: str, shared: bool = False
) -> Path:
if shared:
base_path = self.machine.flake_dir / "sops" / "vars" / "shared"
else:
base_path = (
self.machine.flake_dir
/ "sops"
/ "vars"
/ "per-machine"
/ self.machine.name
)
return base_path / generator_name / secret_name
def set(
self, generator_name: str, name: str, value: bytes, groups: list[str]
self,
generator_name: str,
name: str,
value: bytes,
groups: list[str],
shared: bool = False,
) -> Path | None:
path = self.secret_path(generator_name, name)
path = self.secret_path(generator_name, name, shared)
encrypt_secret(
self.machine.flake_dir,
path,
@@ -59,14 +69,14 @@ class SecretStore(SecretStoreBase):
)
return path
def get(self, generator_name: str, name: str) -> bytes:
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
return decrypt_secret(
self.machine.flake_dir, self.secret_path(generator_name, name)
self.machine.flake_dir, self.secret_path(generator_name, name, shared)
).encode("utf-8")
def exists(self, generator_name: str, name: str) -> bool:
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
return has_secret(
self.secret_path(generator_name, name),
self.secret_path(generator_name, name, shared),
)
def upload(self, output_dir: Path) -> None:

11
pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py
View File

@@ -15,18 +15,23 @@ class SecretStore(SecretStoreBase):
self.dir.mkdir(parents=True, exist_ok=True)
def set(
self, service: str, name: str, value: bytes, groups: list[str]
self,
service: str,
name: str,
value: bytes,
groups: list[str],
shared: bool = False,
) -> Path | None:
secret_file = self.dir / service / name
secret_file.parent.mkdir(parents=True, exist_ok=True)
secret_file.write_bytes(value)
return None # we manage the files outside of the git repo
def get(self, service: str, name: str) -> bytes:
def get(self, service: str, name: str, shared: bool = False) -> bytes:
secret_file = self.dir / service / name
return secret_file.read_bytes()
def exists(self, service: str, name: str) -> bool:
def exists(self, service: str, name: str, shared: bool = False) -> bool:
return (self.dir / service / name).exists()
def upload(self, output_dir: Path) -> None:

8
pkgs/clan-cli/clan_cli/vars/upload.py
View File

@@ -13,14 +13,14 @@ log = logging.getLogger(__name__)
def upload_secrets(machine: Machine) -> None:
secret_facts_module = importlib.import_module(machine.secret_facts_module)
secret_facts_store = secret_facts_module.SecretStore(machine=machine)
secret_store_module = importlib.import_module(machine.secret_facts_module)
secret_store = secret_store_module.SecretStore(machine=machine)
if secret_facts_store.update_check():
if secret_store.update_check():
log.info("Secrets already up to date")
return
with TemporaryDirectory() as tempdir:
secret_facts_store.upload(Path(tempdir))
secret_store.upload(Path(tempdir))
host = machine.target_host
ssh_cmd = host.ssh_cmd()

133
pkgs/clan-cli/tests/test_vars.py
View File

@@ -16,6 +16,7 @@ from root import CLAN_CORE
from clan_cli.clan_uri import FlakeId
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_shell
from clan_cli.vars.public_modules import in_repo
from clan_cli.vars.secret_modules import password_store, sops
@@ -89,11 +90,9 @@ def test_generate_public_var(
)
monkeypatch.chdir(flake.path)
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
var_file_path = (
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_value"
)
assert var_file_path.is_file()
assert var_file_path.read_text() == "hello\n"
store = in_repo.FactStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
assert store.exists("my_generator", "my_value")
assert store.get("my_generator", "my_value").decode() == "hello\n"
@pytest.mark.impure
@@ -125,10 +124,10 @@ def test_generate_secret_var_sops(
]
)
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
var_file_path = (
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(flake.path))
)
assert not var_file_path.is_file()
assert not in_repo_store.exists("my_generator", "my_secret")
sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
assert sops_store.exists("my_generator", "my_secret")
assert sops_store.get("my_generator", "my_secret").decode() == "hello\n"
@@ -165,21 +164,12 @@ def test_generate_secret_var_sops_with_default_group(
)
cli.run(["secrets", "groups", "add-user", "my_group", user])
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
assert not (
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
).is_file()
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(flake.path))
)
assert not in_repo_store.exists("my_generator", "my_secret")
sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
assert sops_store.exists("my_generator", "my_secret")
assert (
flake.path
/ "sops"
/ "vars"
/ "my_machine"
/ "my_generator"
/ "my_secret"
/ "groups"
/ "my_group"
).exists()
assert sops_store.get("my_generator", "my_secret").decode() == "hello\n"
@@ -226,10 +216,6 @@ def test_generate_secret_var_password_store(
nix_shell(["nixpkgs#pass"], ["pass", "init", "test@local"]), check=True
)
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
var_file_path = (
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
)
assert not var_file_path.is_file()
store = password_store.SecretStore(
Machine(name="my_machine", flake=FlakeId(flake.path))
)
@@ -281,16 +267,16 @@ def test_generate_secret_for_multiple_machines(
)
cli.run(["vars", "generate", "--flake", str(flake.path)])
# check if public vars have been created correctly
machine1_var_file_path = (
flake.path / "machines" / "machine1" / "vars" / "my_generator" / "my_value"
in_repo_store1 = in_repo.FactStore(
Machine(name="machine1", flake=FlakeId(flake.path))
)
machine2_var_file_path = (
flake.path / "machines" / "machine2" / "vars" / "my_generator" / "my_value"
in_repo_store2 = in_repo.FactStore(
Machine(name="machine2", flake=FlakeId(flake.path))
)
assert machine1_var_file_path.is_file()
assert machine1_var_file_path.read_text() == "machine1\n"
assert machine2_var_file_path.is_file()
assert machine2_var_file_path.read_text() == "machine2\n"
assert in_repo_store1.exists("my_generator", "my_value")
assert in_repo_store2.exists("my_generator", "my_value")
assert in_repo_store1.get("my_generator", "my_value").decode() == "machine1\n"
assert in_repo_store2.get("my_generator", "my_value").decode() == "machine2\n"
# check if secret vars have been created correctly
sops_store1 = sops.SecretStore(Machine(name="machine1", flake=FlakeId(flake.path)))
sops_store2 = sops.SecretStore(Machine(name="machine2", flake=FlakeId(flake.path)))
@@ -320,16 +306,13 @@ def test_dependant_generators(
)
monkeypatch.chdir(flake.path)
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
parent_file_path = (
flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value"
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(flake.path))
)
assert parent_file_path.is_file()
assert parent_file_path.read_text() == "hello\n"
child_file_path = (
flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value"
)
assert child_file_path.is_file()
assert child_file_path.read_text() == "hello\n"
assert in_repo_store.exists("parent_generator", "my_value")
assert in_repo_store.get("parent_generator", "my_value").decode() == "hello\n"
assert in_repo_store.exists("child_generator", "my_value")
assert in_repo_store.get("child_generator", "my_value").decode() == "hello\n"
@pytest.mark.impure
@@ -362,8 +345,66 @@ def test_prompt(
monkeypatch.chdir(flake.path)
monkeypatch.setattr("sys.stdin", StringIO(input_value))
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
var_file_path = (
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_value"
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(flake.path))
)
assert var_file_path.is_file()
assert var_file_path.read_text() == input_value
assert in_repo_store.exists("my_generator", "my_value")
assert in_repo_store.get("my_generator", "my_value").decode() == input_value
@pytest.mark.impure
def test_share_flag(
monkeypatch: pytest.MonkeyPatch,
temporary_home: Path,
sops_setup: SopsSetup,
) -> None:
user = os.environ.get("USER", "user")
config = nested_dict()
shared_generator = config["clan"]["core"]["vars"]["generators"]["shared_generator"]
shared_generator["files"]["my_secret"]["secret"] = True
shared_generator["files"]["my_value"]["secret"] = False
shared_generator["script"] = (
"echo hello > $out/my_secret && echo hello > $out/my_value"
)
shared_generator["share"] = True
unshared_generator = config["clan"]["core"]["vars"]["generators"][
"unshared_generator"
]
unshared_generator["files"]["my_secret"]["secret"] = True
unshared_generator["files"]["my_value"]["secret"] = False
unshared_generator["script"] = (
"echo hello > $out/my_secret && echo hello > $out/my_value"
)
unshared_generator["share"] = False
flake = generate_flake(
temporary_home,
flake_template=CLAN_CORE / "templates" / "minimal",
machine_configs=dict(my_machine=config),
)
monkeypatch.chdir(flake.path)
cli.run(
[
"secrets",
"users",
"add",
"--flake",
str(flake.path),
user,
sops_setup.keys[0].pubkey,
]
)
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(flake.path))
)
# check secrets stored correctly
assert sops_store.exists("shared_generator", "my_secret", shared=True)
assert not sops_store.exists("shared_generator", "my_secret", shared=False)
assert sops_store.exists("unshared_generator", "my_secret", shared=False)
assert not sops_store.exists("unshared_generator", "my_secret", shared=True)
# check values stored correctly
assert in_repo_store.exists("shared_generator", "my_value", shared=True)
assert not in_repo_store.exists("shared_generator", "my_value", shared=False)
assert in_repo_store.exists("unshared_generator", "my_value", shared=False)
assert not in_repo_store.exists("unshared_generator", "my_value", shared=True)