yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

vars: introduce share flag

Browse Source
This commit is contained in:
DavHau
2024-08-03 12:34:46 +07:00
parent a40ddd2b24
commit cc9c828598
12 changed files with 237 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -57,6 +57,7 @@ def decrypt_dependencies(
generator_name: str,
secret_vars_store: SecretStoreBase,
public_vars_store: FactStoreBase,
shared: bool,
) -> dict[str, dict[str, bytes]]:
generator = machine.vars_generators[generator_name]
dependencies = set(generator["dependencies"])
@@ -67,11 +68,11 @@ def decrypt_dependencies(
for file_name, file in dep_files.items():
if file["secret"]:
decrypted_dependencies[dep_generator][file_name] = (
secret_vars_store.get(dep_generator, file_name)
secret_vars_store.get(dep_generator, file_name, shared=shared)
)
else:
decrypted_dependencies[dep_generator][file_name] = (
public_vars_store.get(dep_generator, file_name)
public_vars_store.get(dep_generator, file_name, shared=shared)
)
return decrypted_dependencies
@@ -109,10 +110,11 @@ def execute_generator(
msg += "fact/secret generation is only supported for local flakes"
generator = machine.vars_generators[generator_name]["finalScript"]
is_shared = machine.vars_generators[generator_name]["share"]
# build temporary file tree of dependencies
decrypted_dependencies = decrypt_dependencies(
machine, generator_name, secret_vars_store, public_vars_store
machine, generator_name, secret_vars_store, public_vars_store, shared=is_shared
)
env = os.environ.copy()
with TemporaryDirectory() as tmp:
@@ -159,11 +161,18 @@ def execute_generator(
raise ClanError(msg)
if file["secret"]:
file_path = secret_vars_store.set(
generator_name, file_name, secret_file.read_bytes(), groups
generator_name,
file_name,
secret_file.read_bytes(),
groups,
shared=is_shared,
)
else:
file_path = public_vars_store.set(
generator_name, file_name, secret_file.read_bytes()
generator_name,
file_name,
secret_file.read_bytes(),
shared=is_shared,
)
if file_path:
files_to_commit.append(file_path)
@@ -260,18 +269,18 @@ def generate_vars(
) -> bool:
was_regenerated = False
for machine in machines:
errors = 0
errors = []
try:
was_regenerated |= _generate_vars_for_machine(
machine, generator_name, regenerate
)
except Exception as exc:
log.error(f"Failed to generate facts for {machine.name}: {exc}")
errors += 1
if errors > 0:
errors += [exc]
if len(errors) > 0:
raise ClanError(
f"Failed to generate facts for {errors} hosts. Check the logs above"
)
f"Failed to generate facts for {len(errors)} hosts. Check the logs above"
) from errors[0]
if not was_regenerated:
print("All secrets and facts are already up to date")

8
pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py
View File

@@ -10,16 +10,18 @@ class FactStoreBase(ABC):
pass
@abstractmethod
def exists(self, service: str, name: str) -> bool:
def exists(self, service: str, name: str, shared: bool = False) -> bool:
pass
@abstractmethod
def set(self, service: str, name: str, value: bytes) -> Path | None:
def set(
self, service: str, name: str, value: bytes, shared: bool = False
) -> Path | None:
pass
# get a single fact
@abstractmethod
def get(self, service: str, name: str) -> bytes:
def get(self, service: str, name: str, shared: bool = False) -> bytes:
pass
# get all facts

55
pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py
View File

@@ -10,17 +10,22 @@ class FactStore(FactStoreBase):
def __init__(self, machine: Machine) -> None:
self.machine = machine
self.works_remotely = False
self.per_machine_folder = (
self.machine.flake_dir / "vars" / "per-machine" / self.machine.name
)
self.shared_folder = self.machine.flake_dir / "vars" / "shared"
def set(self, generator_name: str, name: str, value: bytes) -> Path | None:
def _var_path(self, generator_name: str, name: str, shared: bool) -> Path:
if shared:
return self.shared_folder / generator_name / name
else:
return self.per_machine_folder / generator_name / name
def set(
self, generator_name: str, name: str, value: bytes, shared: bool = False
) -> Path | None:
if self.machine.flake.is_local():
fact_path = (
self.machine.flake.path
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
)
fact_path = self._var_path(generator_name, name, shared)
fact_path.parent.mkdir(parents=True, exist_ok=True)
fact_path.touch()
fact_path.write_bytes(value)
@@ -30,35 +35,21 @@ class FactStore(FactStoreBase):
f"in_flake fact storage is only supported for local flakes: {self.machine.flake}"
)
def exists(self, generator_name: str, name: str) -> bool:
fact_path = (
self.machine.flake_dir
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
)
return fact_path.exists()
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
return self._var_path(generator_name, name, shared).exists()
# get a single fact
def get(self, generator_name: str, name: str) -> bytes:
fact_path = (
self.machine.flake_dir
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
)
return fact_path.read_bytes()
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
return self._var_path(generator_name, name, shared).read_bytes()
# get all public vars
def get_all(self) -> dict[str, dict[str, bytes]]:
facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "vars"
facts: dict[str, dict[str, bytes]] = {}
facts["TODO"] = {}
if facts_folder.exists():
for fact_path in facts_folder.iterdir():
if self.per_machine_folder.exists():
for fact_path in self.per_machine_folder.iterdir():
facts["TODO"][fact_path.name] = fact_path.read_bytes()
if self.shared_folder.exists():
for fact_path in self.shared_folder.iterdir():
facts["TODO"][fact_path.name] = fact_path.read_bytes()
return facts

8
pkgs/clan-cli/clan_cli/vars/public_modules/vm.py
View File

@@ -17,18 +17,20 @@ class FactStore(FactStoreBase):
self.dir = vm_state_dir(str(machine.flake), machine.name) / "facts"
log.debug(f"FactStore initialized with dir {self.dir}")
def exists(self, service: str, name: str) -> bool:
def exists(self, service: str, name: str, shared: bool = False) -> bool:
fact_path = self.dir / service / name
return fact_path.exists()
def set(self, service: str, name: str, value: bytes) -> Path | None:
def set(
self, service: str, name: str, value: bytes, shared: bool = False
) -> Path | None:
fact_path = self.dir / service / name
fact_path.parent.mkdir(parents=True, exist_ok=True)
fact_path.write_bytes(value)
return None
# get a single fact
def get(self, service: str, name: str) -> bytes:
def get(self, service: str, name: str, shared: bool = False) -> bytes:
fact_path = self.dir / service / name
if fact_path.exists():
return fact_path.read_bytes()

11
pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py
View File

@@ -11,16 +11,21 @@ class SecretStoreBase(ABC):
@abstractmethod
def set(
self, service: str, name: str, value: bytes, groups: list[str]
self,
service: str,
name: str,
value: bytes,
groups: list[str],
shared: bool = False,
) -> Path | None:
pass
@abstractmethod
def get(self, service: str, name: str) -> bytes:
def get(self, service: str, name: str, shared: bool = False) -> bytes:
pass
@abstractmethod
def exists(self, service: str, name: str) -> bool:
def exists(self, service: str, name: str, shared: bool = False) -> bool:
pass
def update_check(self) -> bool:

63
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
View File

@@ -12,8 +12,25 @@ class SecretStore(SecretStoreBase):
def __init__(self, machine: Machine) -> None:
self.machine = machine
@property
def _password_store_dir(self) -> str:
return os.environ.get(
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
)
def _var_path(self, generator_name: str, name: str, shared: bool) -> Path:
if shared:
return Path(f"shared/{generator_name}/{name}")
else:
return Path(f"machines/{self.machine.name}/{generator_name}/{name}")
def set(
self, generator_name: str, name: str, value: bytes, groups: list[str]
self,
generator_name: str,
name: str,
value: bytes,
groups: list[str],
shared: bool = False,
) -> Path | None:
subprocess.run(
nix_shell(
@@ -22,7 +39,7 @@ class SecretStore(SecretStoreBase):
"pass",
"insert",
"-m",
f"machines/{self.machine.name}/{generator_name}/{name}",
str(self._var_path(generator_name, name, shared)),
],
),
input=value,
@@ -30,34 +47,28 @@ class SecretStore(SecretStoreBase):
)
return None # we manage the files outside of the git repo
def get(self, generator_name: str, name: str) -> bytes:
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
return subprocess.run(
nix_shell(
["nixpkgs#pass"],
[
"pass",
"show",
f"machines/{self.machine.name}/{generator_name}/{name}",
str(self._var_path(generator_name, name, shared)),
],
),
check=True,
stdout=subprocess.PIPE,
).stdout
def exists(self, generator_name: str, name: str) -> bool:
password_store = os.environ.get(
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
)
secret_path = (
Path(password_store)
/ f"machines/{self.machine.name}/{generator_name}/{name}.gpg"
)
return secret_path.exists()
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
return (
Path(self._password_store_dir)
/ f"{self._var_path(generator_name, name, shared)}.gpg"
).exists()
def generate_hash(self) -> bytes:
password_store = os.environ.get(
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
)
password_store = self._password_store_dir
hashes = []
hashes.append(
subprocess.run(
@@ -117,13 +128,15 @@ class SecretStore(SecretStoreBase):
return local_hash.decode() == remote_hash
# TODO: fixme
def upload(self, output_dir: Path) -> None:
for service in self.machine.facts_data:
for secret in self.machine.facts_data[service]["secret"]:
if isinstance(secret, dict):
secret_name = secret["name"]
else:
# TODO: drop old format soon
secret_name = secret
(output_dir / secret_name).write_bytes(self.get(service, secret_name))
(output_dir / ".pass_info").write_bytes(self.generate_hash())
pass
# for service in self.machine.facts_data:
# for secret in self.machine.facts_data[service]["secret"]:
# if isinstance(secret, dict):
# secret_name = secret["name"]
# else:
# # TODO: drop old format soon
# secret_name = secret
# (output_dir / secret_name).write_bytes(self.get(service, secret_name))
# (output_dir / ".pass_info").write_bytes(self.generate_hash())

40
pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py
View File

@@ -36,20 +36,30 @@ class SecretStore(SecretStoreBase):
)
add_machine(self.machine.flake_dir, self.machine.name, pub_key, False)
def secret_path(self, generator_name: str, secret_name: str) -> Path:
return (
self.machine.flake_dir
/ "sops"
/ "vars"
/ self.machine.name
/ generator_name
/ secret_name
)
def secret_path(
self, generator_name: str, secret_name: str, shared: bool = False
) -> Path:
if shared:
base_path = self.machine.flake_dir / "sops" / "vars" / "shared"
else:
base_path = (
self.machine.flake_dir
/ "sops"
/ "vars"
/ "per-machine"
/ self.machine.name
)
return base_path / generator_name / secret_name
def set(
self, generator_name: str, name: str, value: bytes, groups: list[str]
self,
generator_name: str,
name: str,
value: bytes,
groups: list[str],
shared: bool = False,
) -> Path | None:
path = self.secret_path(generator_name, name)
path = self.secret_path(generator_name, name, shared)
encrypt_secret(
self.machine.flake_dir,
path,
@@ -59,14 +69,14 @@ class SecretStore(SecretStoreBase):
)
return path
def get(self, generator_name: str, name: str) -> bytes:
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
return decrypt_secret(
self.machine.flake_dir, self.secret_path(generator_name, name)
self.machine.flake_dir, self.secret_path(generator_name, name, shared)
).encode("utf-8")
def exists(self, generator_name: str, name: str) -> bool:
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
return has_secret(
self.secret_path(generator_name, name),
self.secret_path(generator_name, name, shared),
)
def upload(self, output_dir: Path) -> None:

11
pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py
View File

@@ -15,18 +15,23 @@ class SecretStore(SecretStoreBase):
self.dir.mkdir(parents=True, exist_ok=True)
def set(
self, service: str, name: str, value: bytes, groups: list[str]
self,
service: str,
name: str,
value: bytes,
groups: list[str],
shared: bool = False,
) -> Path | None:
secret_file = self.dir / service / name
secret_file.parent.mkdir(parents=True, exist_ok=True)
secret_file.write_bytes(value)
return None # we manage the files outside of the git repo
def get(self, service: str, name: str) -> bytes:
def get(self, service: str, name: str, shared: bool = False) -> bytes:
secret_file = self.dir / service / name
return secret_file.read_bytes()
def exists(self, service: str, name: str) -> bool:
def exists(self, service: str, name: str, shared: bool = False) -> bool:
return (self.dir / service / name).exists()
def upload(self, output_dir: Path) -> None:

8
pkgs/clan-cli/clan_cli/vars/upload.py
View File

@@ -13,14 +13,14 @@ log = logging.getLogger(__name__)
def upload_secrets(machine: Machine) -> None:
secret_facts_module = importlib.import_module(machine.secret_facts_module)
secret_facts_store = secret_facts_module.SecretStore(machine=machine)
secret_store_module = importlib.import_module(machine.secret_facts_module)
secret_store = secret_store_module.SecretStore(machine=machine)
if secret_facts_store.update_check():
if secret_store.update_check():
log.info("Secrets already up to date")
return
with TemporaryDirectory() as tempdir:
secret_facts_store.upload(Path(tempdir))
secret_store.upload(Path(tempdir))
host = machine.target_host
ssh_cmd = host.ssh_cmd()