diff --git a/pkgs/clan-cli/clan_cli/machines/inventory.py b/pkgs/clan-cli/clan_cli/machines/inventory.py index 0eeac102c..a2b6397bf 100644 --- a/pkgs/clan-cli/clan_cli/machines/inventory.py +++ b/pkgs/clan-cli/clan_cli/machines/inventory.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path from clan_cli.clan_uri import FlakeId @@ -16,6 +17,10 @@ def get_all_machines(flake: FlakeId, nix_options: list[str]) -> list[Machine]: nix_build([f'{flake}#clanInternals.all-machines-json."{system}"']) ).stdout + tmp_store = os.environ.get("TMP_STORE", None) + if tmp_store: + json_path = f"{tmp_store}/{json_path}" + machines_json = json.loads(Path(json_path.rstrip()).read_text()) machines = [] diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index be7547263..ff06ebf1f 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -1,6 +1,7 @@ import importlib import json import logging +import os from dataclasses import dataclass, field from functools import cached_property from pathlib import Path @@ -329,6 +330,10 @@ class Machine: return self._build_cache[attr] output = self.nix("build", attr, extra_config, nix_options) + assert isinstance(output, Path), "Nix build did not result in a single path" + tmp_store = os.environ.get("TMP_STORE", None) + if tmp_store is not None: + output = Path(f"{tmp_store}/{output!s}") if isinstance(output, Path): self._build_cache[attr] = output return output diff --git a/pkgs/clan-cli/clan_cli/nix/__init__.py b/pkgs/clan-cli/clan_cli/nix/__init__.py index a7cef7f63..d1dc6a083 100644 --- a/pkgs/clan-cli/clan_cli/nix/__init__.py +++ b/pkgs/clan-cli/clan_cli/nix/__init__.py @@ -10,7 +10,11 @@ from clan_cli.errors import ClanError def nix_command(flags: list[str]) -> list[str]: - return ["nix", "--extra-experimental-features", "nix-command flakes", *flags] + args = ["nix", "--extra-experimental-features", "nix-command flakes", *flags] + store = os.environ.get("TMP_STORE", None) + if store: + args += ["--store", store] + return args def nix_flake_show(flake_url: str | Path) -> list[str]: diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 2ff012557..dcfc96454 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -150,14 +150,38 @@ python3.pkgs.buildPythonApplication { ''; clan-pytest-with-core = runCommand "clan-pytest-with-core" - { nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies; } + { + nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies; + buildInputs = [ + pkgs.bash + pkgs.coreutils + pkgs.nix + ]; + closureInfo = pkgs.closureInfo { + rootPaths = [ + pkgs.bash + pkgs.coreutils + pkgs.jq.dev + pkgs.stdenv + pkgs.stdenvNoCC + ]; + }; + } '' cp -r ${source} ./src chmod +w -R ./src cd ./src export CLAN_CORE=${clan-core-path} - export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error + export NIX_STATE_DIR=$TMPDIR/nix + export IN_NIX_SANDBOX=1 + export PYTHONWARNINGS=error + export TMP_STORE=$TMPDIR/store + # required to prevent concurrent 'nix flake lock' operations + export LOCK_NIX=$TMPDIR/nix_lock + mkdir -p $TMP_STORE/nix/store + xargs cp --recursive --target "$TMP_STORE/nix/store" < "$closureInfo/store-paths" + nix-store --load-db --store $TMP_STORE < "$closureInfo/registration" ${pythonWithTestDeps}/bin/python -m pytest -m "not impure and with_core" ./tests touch $out ''; diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 297c4ee8d..5aadcaca2 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -14,30 +14,88 @@ let flakeLock = lib.importJSON (self + /flake.lock); flakeInputs = builtins.removeAttrs inputs [ "self" ]; - flakeLockVendoredDeps = flakeLock // { - nodes = - flakeLock.nodes - // (lib.flip lib.mapAttrs flakeInputs ( - name: _: - flakeLock.nodes.${name} - // { - locked = { - inherit (flakeLock.nodes.${name}.locked) narHash; - lastModified = - # lol, nixpkgs has a different timestamp on the fs??? - if name == "nixpkgs" then 0 else 1; - path = "${inputs.${name}}"; - type = "path"; - }; - } - )); + flakeLockVendoredDeps = + flakeLock: + flakeLock + // { + nodes = + flakeLock.nodes + // (lib.flip lib.mapAttrs flakeInputs ( + name: _: + # remove follows and let 'nix flake lock' re-compute it later + # (lib.removeAttrs flakeLock.nodes.${name} ["inputs"]) + flakeLock.nodes.${name} + // { + locked = { + inherit (flakeLock.nodes.${name}.locked) narHash; + lastModified = + # lol, nixpkgs has a different timestamp on the fs??? + if name == "nixpkgs" then 0 else 1; + path = "${inputs.${name}}"; + type = "path"; + }; + } + )); + }; + clanCoreLock = flakeLockVendoredDeps flakeLock; + clanCoreLockFile = builtins.toFile "clan-core-flake.lock" (builtins.toJSON clanCoreLock); + clanCoreNode = { + inputs = lib.mapAttrs (name: _input: name) flakeInputs; + locked = { + lastModified = 1; + path = "${self}"; + type = "path"; + }; + original = { + type = "tarball"; + url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; + }; }; - flakeLockFile = builtins.toFile "clan-core-flake.lock" (builtins.toJSON flakeLockVendoredDeps); - clanCoreWithVendoredDeps = pkgs.runCommand "clan-core-with-vendored-deps" { } '' - cp -r ${self} $out - chmod +w -R $out - cp ${flakeLockFile} $out/flake.lock - ''; + # generate a lock file that nix will accept for our flake templates, + # in order to not require internet access during tests. + templateLock = clanCoreLock // { + nodes = clanCoreLock.nodes // { + clan-core = clanCoreNode; + nixpkgs-lib = clanCoreLock.nodes.nixpkgs; # required by flake-parts + root = clanCoreLock.nodes.root // { + inputs = clanCoreLock.nodes.root.inputs // { + clan-core = "clan-core"; + nixpkgs = "nixpkgs"; + nixpkgs-lib = "nixpkgs-lib"; + }; + }; + }; + }; + templateLockFile = builtins.toFile "template-flake.lock" (builtins.toJSON templateLock); + clanCoreWithVendoredDeps = + pkgs.runCommand "clan-core-with-vendored-deps" + { + buildInputs = [ + pkgs.findutils + pkgs.git + pkgs.jq + pkgs.nix + ]; + } + '' + set -e + export HOME=$(realpath .) + export NIX_STATE_DIR=$HOME + cp -r ${self} $out + chmod +w -R $out + cp ${clanCoreLockFile} $out/flake.lock + nix flake lock $out --extra-experimental-features 'nix-command flakes' + clanCoreHash=$(nix hash path ${self} --extra-experimental-features 'nix-command') + for templateDir in $(find $out/templates -mindepth 1 -maxdepth 1 -type d); do + if ! [ -e "$templateDir/flake.nix" ]; then + continue + fi + cp ${templateLockFile} $templateDir/flake.lock + cat $templateDir/flake.lock | jq ".nodes.\"clan-core\".locked.narHash = \"$clanCoreHash\"" > $templateDir/flake.lock.final + mv $templateDir/flake.lock.final $templateDir/flake.lock + nix flake lock $templateDir --extra-experimental-features 'nix-command flakes' + done + ''; in { devShells.clan-cli = pkgs.callPackage ./shell.nix { diff --git a/pkgs/clan-cli/tests/fixtures_flakes.py b/pkgs/clan-cli/tests/fixtures_flakes.py index c9b7f2e15..2e92f1006 100644 --- a/pkgs/clan-cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/tests/fixtures_flakes.py @@ -11,11 +11,16 @@ from typing import Any, NamedTuple import pytest from clan_cli.dirs import nixpkgs_source +from clan_cli.locked_open import locked_open from fixture_error import FixtureError from root import CLAN_CORE log = logging.getLogger(__name__) +lock_nix = os.environ.get("LOCK_NIX", "") +if not lock_nix: + lock_nix = tempfile.NamedTemporaryFile().name # NOQA: SIM115 + # allows defining nested dictionary in a single line def def_value() -> defaultdict: @@ -151,13 +156,20 @@ class ClanFlake: sp.run(["chmod", "+w", "-R", str(self.path)], check=True) self.substitute() if not (self.path / ".git").exists(): - sp.run( - ["nix", "flake", "lock"], - cwd=self.path, - check=True, - ) - with pytest.MonkeyPatch.context() as mp: - init_git(mp, self.path) + with locked_open(Path(lock_nix), "w"): + sp.run( + [ + "nix", + "flake", + "lock", + "--extra-experimental-features", + "flakes nix-command", + ], + cwd=self.path, + check=True, + ) + with pytest.MonkeyPatch.context() as mp: + init_git(mp, self.path) def refresh(self) -> None: if not self.path.exists(): diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py index 16a8bbee3..6335fedf1 100644 --- a/pkgs/clan-cli/tests/test_vars.py +++ b/pkgs/clan-cli/tests/test_vars.py @@ -93,7 +93,7 @@ def test_required_generators() -> None: ] -@pytest.mark.impure +@pytest.mark.with_core def test_generate_public_var( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -128,7 +128,7 @@ def test_generate_public_var( assert json.loads(vars_eval) == "hello\n" -@pytest.mark.impure +@pytest.mark.with_core def test_generate_secret_var_sops( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -166,7 +166,7 @@ def test_generate_secret_var_sops( # TODO: it doesn't actually test if the group has access -@pytest.mark.impure +@pytest.mark.with_core def test_generate_secret_var_sops_with_default_group( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -219,7 +219,7 @@ def test_generate_secret_var_sops_with_default_group( ) -@pytest.mark.impure +@pytest.mark.with_core def test_generated_shared_secret_sops( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -264,7 +264,7 @@ def test_generated_shared_secret_sops( ) -@pytest.mark.impure +@pytest.mark.with_core def test_generate_secret_var_password_store( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -316,7 +316,7 @@ def test_generate_secret_var_password_store( assert "my_generator/my_secret" in vars_text -@pytest.mark.impure +@pytest.mark.with_core def test_generate_secret_for_multiple_machines( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -378,7 +378,7 @@ def test_generate_secret_for_multiple_machines( ) -@pytest.mark.impure +@pytest.mark.with_core def test_dependant_generators( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -409,7 +409,7 @@ def test_dependant_generators( ) -@pytest.mark.impure +@pytest.mark.with_core @pytest.mark.parametrize( ("prompt_type", "input_value"), [ @@ -447,7 +447,7 @@ def test_prompt( ) -@pytest.mark.impure +@pytest.mark.with_core def test_share_flag( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -514,7 +514,7 @@ def test_share_flag( assert json.loads(vars_eval) == "hello\n" -@pytest.mark.impure +@pytest.mark.with_core def test_depending_on_shared_secret_succeeds( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -540,7 +540,7 @@ def test_depending_on_shared_secret_succeeds( cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) -@pytest.mark.impure +@pytest.mark.with_core def test_prompt_create_file( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -572,7 +572,7 @@ def test_prompt_create_file( ) -@pytest.mark.impure +@pytest.mark.with_core def test_api_get_prompts( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -595,7 +595,7 @@ def test_api_get_prompts( assert api_prompts[0].prompts[0].previous_value == "input1" -@pytest.mark.impure +@pytest.mark.with_core def test_api_set_prompts( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -634,7 +634,7 @@ def test_api_set_prompts( assert store.get(Generator("my_generator"), "prompt1").decode() == "input2" -@pytest.mark.impure +@pytest.mark.with_core def test_commit_message( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -691,7 +691,7 @@ def test_commit_message( ) -@pytest.mark.impure +@pytest.mark.with_core def test_default_value( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -728,7 +728,7 @@ def test_default_value( assert json.loads(value_eval) == "hello" -@pytest.mark.impure +@pytest.mark.with_core def test_stdout_of_generate( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -809,7 +809,7 @@ def test_stdout_of_generate( assert "hello" not in output.out -@pytest.mark.impure +@pytest.mark.with_core def test_migration_skip( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -837,7 +837,7 @@ def test_migration_skip( assert in_repo_store.get(Generator("my_generator"), "my_value").decode() == "world" -@pytest.mark.impure +@pytest.mark.with_core def test_migration( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -873,7 +873,7 @@ def test_migration( assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello" -@pytest.mark.impure +@pytest.mark.with_core def test_fails_when_files_are_left_from_other_backend( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -915,7 +915,7 @@ def test_fails_when_files_are_left_from_other_backend( ) -@pytest.mark.impure +@pytest.mark.with_core def test_keygen( monkeypatch: pytest.MonkeyPatch, temporary_home: Path, @@ -935,7 +935,7 @@ def test_keygen( assert (temporary_home / "sops" / "users" / "user").is_dir() -@pytest.mark.impure +@pytest.mark.with_core def test_vars_get( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, @@ -962,7 +962,7 @@ def test_vars_get( ) -@pytest.mark.impure +@pytest.mark.with_core def test_invalidation( monkeypatch: pytest.MonkeyPatch, flake: ClanFlake, diff --git a/templates/minimal-flake-parts/flake.nix b/templates/minimal-flake-parts/flake.nix index bf3f961e4..be0c31e8b 100644 --- a/templates/minimal-flake-parts/flake.nix +++ b/templates/minimal-flake-parts/flake.nix @@ -1,10 +1,10 @@ { inputs = { - nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - clan.url = "git+https://git.clan.lol/clan/clan-core"; - clan.inputs.nixpkgs.follows = "nixpkgs"; - clan.inputs.flake-parts.follows = "flake-parts"; + clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; + clan-core.inputs.nixpkgs.follows = "nixpkgs"; + clan-core.inputs.flake-parts.follows = "flake-parts"; flake-parts.url = "github:hercules-ci/flake-parts"; flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; diff --git a/templates/minimal/flake.nix b/templates/minimal/flake.nix index 78301b9c9..3618b47df 100644 --- a/templates/minimal/flake.nix +++ b/templates/minimal/flake.nix @@ -1,5 +1,6 @@ { - inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core"; + inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; + inputs.nixpkgs.follows = "clan-core/nixpkgs"; outputs = { self, clan-core, ... }: diff --git a/templates/new-clan/flake.nix b/templates/new-clan/flake.nix index 892b0bbd4..10594dcb5 100644 --- a/templates/new-clan/flake.nix +++ b/templates/new-clan/flake.nix @@ -2,6 +2,7 @@ description = ""; inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; + inputs.nixpkgs.follows = "clan-core/nixpkgs"; outputs = { self, clan-core, ... }: