yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'vars: make all python tests work in nix sandbox' (#2502) from DavHau/clan-core:DavHau-dave into main

Browse Source
This commit is contained in:
clan-bot
2024-11-27 07:38:23 +00:00
parent dcd2581b41 61576649ff
commit 5db3783976
10 changed files with 170 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkgs/clan-cli/clan_cli/machines/inventory.py
View File

@@ -1,4 +1,5 @@
import json import json
import os
from pathlib import Path from pathlib import Path
from clan_cli.clan_uri import FlakeId from clan_cli.clan_uri import FlakeId
@@ -16,6 +17,10 @@ def get_all_machines(flake: FlakeId, nix_options: list[str]) -> list[Machine]:
nix_build([f'{flake}#clanInternals.all-machines-json."{system}"']) nix_build([f'{flake}#clanInternals.all-machines-json."{system}"'])
).stdout ).stdout
tmp_store = os.environ.get("TMP_STORE", None)
if tmp_store:
json_path = f"{tmp_store}/{json_path}"
machines_json = json.loads(Path(json_path.rstrip()).read_text()) machines_json = json.loads(Path(json_path.rstrip()).read_text())
machines = [] machines = []

5
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -1,6 +1,7 @@
import importlib import importlib
import json import json
import logging import logging
import os
from dataclasses import dataclass, field from dataclasses import dataclass, field
from functools import cached_property from functools import cached_property
from pathlib import Path from pathlib import Path
@@ -329,6 +330,10 @@ class Machine:
return self._build_cache[attr] return self._build_cache[attr]
output = self.nix("build", attr, extra_config, nix_options) output = self.nix("build", attr, extra_config, nix_options)
assert isinstance(output, Path), "Nix build did not result in a single path"
tmp_store = os.environ.get("TMP_STORE", None)
if tmp_store is not None:
output = Path(f"{tmp_store}/{output!s}")
if isinstance(output, Path): if isinstance(output, Path):
self._build_cache[attr] = output self._build_cache[attr] = output
return output return output

6
pkgs/clan-cli/clan_cli/nix/__init__.py
View File

@@ -10,7 +10,11 @@ from clan_cli.errors import ClanError
def nix_command(flags: list[str]) -> list[str]: def nix_command(flags: list[str]) -> list[str]:
return ["nix", "--extra-experimental-features", "nix-command flakes", *flags] args = ["nix", "--extra-experimental-features", "nix-command flakes", *flags]
store = os.environ.get("TMP_STORE", None)
if store:
args += ["--store", store]
return args
def nix_flake_show(flake_url: str | Path) -> list[str]: def nix_flake_show(flake_url: str | Path) -> list[str]:

28
pkgs/clan-cli/default.nix
View File

@@ -150,14 +150,38 @@ python3.pkgs.buildPythonApplication {
''; '';
clan-pytest-with-core = clan-pytest-with-core =
runCommand "clan-pytest-with-core" runCommand "clan-pytest-with-core"
{ nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies; } {
nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies;
buildInputs = [
pkgs.bash
pkgs.coreutils
pkgs.nix
];
closureInfo = pkgs.closureInfo {
rootPaths = [
pkgs.bash
pkgs.coreutils
pkgs.jq.dev
pkgs.stdenv
pkgs.stdenvNoCC
];
};
}
'' ''
cp -r ${source} ./src cp -r ${source} ./src
chmod +w -R ./src chmod +w -R ./src
cd ./src cd ./src
export CLAN_CORE=${clan-core-path} export CLAN_CORE=${clan-core-path}
export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error export NIX_STATE_DIR=$TMPDIR/nix
export IN_NIX_SANDBOX=1
export PYTHONWARNINGS=error
export TMP_STORE=$TMPDIR/store
# required to prevent concurrent 'nix flake lock' operations
export LOCK_NIX=$TMPDIR/nix_lock
mkdir -p $TMP_STORE/nix/store
xargs cp --recursive --target "$TMP_STORE/nix/store" < "$closureInfo/store-paths"
nix-store --load-db --store $TMP_STORE < "$closureInfo/registration"
${pythonWithTestDeps}/bin/python -m pytest -m "not impure and with_core" ./tests ${pythonWithTestDeps}/bin/python -m pytest -m "not impure and with_core" ./tests
touch $out touch $out
''; '';

104
pkgs/clan-cli/flake-module.nix
View File

@@ -14,30 +14,88 @@
let let
flakeLock = lib.importJSON (self + /flake.lock); flakeLock = lib.importJSON (self + /flake.lock);
flakeInputs = builtins.removeAttrs inputs [ "self" ]; flakeInputs = builtins.removeAttrs inputs [ "self" ];
flakeLockVendoredDeps = flakeLock // { flakeLockVendoredDeps =
nodes = flakeLock:
flakeLock.nodes flakeLock
// (lib.flip lib.mapAttrs flakeInputs ( // {
name: _: nodes =
flakeLock.nodes.${name} flakeLock.nodes
// { // (lib.flip lib.mapAttrs flakeInputs (
locked = { name: _:
inherit (flakeLock.nodes.${name}.locked) narHash; # remove follows and let 'nix flake lock' re-compute it later
lastModified = # (lib.removeAttrs flakeLock.nodes.${name} ["inputs"])
# lol, nixpkgs has a different timestamp on the fs??? flakeLock.nodes.${name}
if name == "nixpkgs" then 0 else 1; // {
path = "${inputs.${name}}"; locked = {
type = "path"; inherit (flakeLock.nodes.${name}.locked) narHash;
}; lastModified =
} # lol, nixpkgs has a different timestamp on the fs???
)); if name == "nixpkgs" then 0 else 1;
path = "${inputs.${name}}";
type = "path";
};
}
));
};
clanCoreLock = flakeLockVendoredDeps flakeLock;
clanCoreLockFile = builtins.toFile "clan-core-flake.lock" (builtins.toJSON clanCoreLock);
clanCoreNode = {
inputs = lib.mapAttrs (name: _input: name) flakeInputs;
locked = {
lastModified = 1;
path = "${self}";
type = "path";
};
original = {
type = "tarball";
url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
};
}; };
flakeLockFile = builtins.toFile "clan-core-flake.lock" (builtins.toJSON flakeLockVendoredDeps); # generate a lock file that nix will accept for our flake templates,
clanCoreWithVendoredDeps = pkgs.runCommand "clan-core-with-vendored-deps" { } '' # in order to not require internet access during tests.
cp -r ${self} $out templateLock = clanCoreLock // {
chmod +w -R $out nodes = clanCoreLock.nodes // {
cp ${flakeLockFile} $out/flake.lock clan-core = clanCoreNode;
''; nixpkgs-lib = clanCoreLock.nodes.nixpkgs; # required by flake-parts
root = clanCoreLock.nodes.root // {
inputs = clanCoreLock.nodes.root.inputs // {
clan-core = "clan-core";
nixpkgs = "nixpkgs";
nixpkgs-lib = "nixpkgs-lib";
};
};
};
};
templateLockFile = builtins.toFile "template-flake.lock" (builtins.toJSON templateLock);
clanCoreWithVendoredDeps =
pkgs.runCommand "clan-core-with-vendored-deps"
{
buildInputs = [
pkgs.findutils
pkgs.git
pkgs.jq
pkgs.nix
];
}
''
set -e
export HOME=$(realpath .)
export NIX_STATE_DIR=$HOME
cp -r ${self} $out
chmod +w -R $out
cp ${clanCoreLockFile} $out/flake.lock
nix flake lock $out --extra-experimental-features 'nix-command flakes'
clanCoreHash=$(nix hash path ${self} --extra-experimental-features 'nix-command')
for templateDir in $(find $out/templates -mindepth 1 -maxdepth 1 -type d); do
if ! [ -e "$templateDir/flake.nix" ]; then
continue
fi
cp ${templateLockFile} $templateDir/flake.lock
cat $templateDir/flake.lock | jq ".nodes.\"clan-core\".locked.narHash = \"$clanCoreHash\"" > $templateDir/flake.lock.final
mv $templateDir/flake.lock.final $templateDir/flake.lock
nix flake lock $templateDir --extra-experimental-features 'nix-command flakes'
done
'';
in in
{ {
devShells.clan-cli = pkgs.callPackage ./shell.nix { devShells.clan-cli = pkgs.callPackage ./shell.nix {

26
pkgs/clan-cli/tests/fixtures_flakes.py
View File

@@ -11,11 +11,16 @@ from typing import Any, NamedTuple
import pytest import pytest
from clan_cli.dirs import nixpkgs_source from clan_cli.dirs import nixpkgs_source
from clan_cli.locked_open import locked_open
from fixture_error import FixtureError from fixture_error import FixtureError
from root import CLAN_CORE from root import CLAN_CORE
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
lock_nix = os.environ.get("LOCK_NIX", "")
if not lock_nix:
lock_nix = tempfile.NamedTemporaryFile().name # NOQA: SIM115
# allows defining nested dictionary in a single line # allows defining nested dictionary in a single line
def def_value() -> defaultdict: def def_value() -> defaultdict:
@@ -151,13 +156,20 @@ class ClanFlake:
sp.run(["chmod", "+w", "-R", str(self.path)], check=True) sp.run(["chmod", "+w", "-R", str(self.path)], check=True)
self.substitute() self.substitute()
if not (self.path / ".git").exists(): if not (self.path / ".git").exists():
sp.run( with locked_open(Path(lock_nix), "w"):
["nix", "flake", "lock"], sp.run(
cwd=self.path, [
check=True, "nix",
) "flake",
with pytest.MonkeyPatch.context() as mp: "lock",
init_git(mp, self.path) "--extra-experimental-features",
"flakes nix-command",
],
cwd=self.path,
check=True,
)
with pytest.MonkeyPatch.context() as mp:
init_git(mp, self.path)
def refresh(self) -> None: def refresh(self) -> None:
if not self.path.exists(): if not self.path.exists():

44
pkgs/clan-cli/tests/test_vars.py
View File

@@ -93,7 +93,7 @@ def test_required_generators() -> None:
] ]
@pytest.mark.impure @pytest.mark.with_core
def test_generate_public_var( def test_generate_public_var(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -128,7 +128,7 @@ def test_generate_public_var(
assert json.loads(vars_eval) == "hello\n" assert json.loads(vars_eval) == "hello\n"
@pytest.mark.impure @pytest.mark.with_core
def test_generate_secret_var_sops( def test_generate_secret_var_sops(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -166,7 +166,7 @@ def test_generate_secret_var_sops(
# TODO: it doesn't actually test if the group has access # TODO: it doesn't actually test if the group has access
@pytest.mark.impure @pytest.mark.with_core
def test_generate_secret_var_sops_with_default_group( def test_generate_secret_var_sops_with_default_group(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -219,7 +219,7 @@ def test_generate_secret_var_sops_with_default_group(
) )
@pytest.mark.impure @pytest.mark.with_core
def test_generated_shared_secret_sops( def test_generated_shared_secret_sops(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -264,7 +264,7 @@ def test_generated_shared_secret_sops(
) )
@pytest.mark.impure @pytest.mark.with_core
def test_generate_secret_var_password_store( def test_generate_secret_var_password_store(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -316,7 +316,7 @@ def test_generate_secret_var_password_store(
assert "my_generator/my_secret" in vars_text assert "my_generator/my_secret" in vars_text
@pytest.mark.impure @pytest.mark.with_core
def test_generate_secret_for_multiple_machines( def test_generate_secret_for_multiple_machines(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -378,7 +378,7 @@ def test_generate_secret_for_multiple_machines(
) )
@pytest.mark.impure @pytest.mark.with_core
def test_dependant_generators( def test_dependant_generators(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -409,7 +409,7 @@ def test_dependant_generators(
) )
@pytest.mark.impure @pytest.mark.with_core
@pytest.mark.parametrize( @pytest.mark.parametrize(
("prompt_type", "input_value"), ("prompt_type", "input_value"),
[ [
@@ -447,7 +447,7 @@ def test_prompt(
) )
@pytest.mark.impure @pytest.mark.with_core
def test_share_flag( def test_share_flag(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -514,7 +514,7 @@ def test_share_flag(
assert json.loads(vars_eval) == "hello\n" assert json.loads(vars_eval) == "hello\n"
@pytest.mark.impure @pytest.mark.with_core
def test_depending_on_shared_secret_succeeds( def test_depending_on_shared_secret_succeeds(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -540,7 +540,7 @@ def test_depending_on_shared_secret_succeeds(
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
@pytest.mark.impure @pytest.mark.with_core
def test_prompt_create_file( def test_prompt_create_file(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -572,7 +572,7 @@ def test_prompt_create_file(
) )
@pytest.mark.impure @pytest.mark.with_core
def test_api_get_prompts( def test_api_get_prompts(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -595,7 +595,7 @@ def test_api_get_prompts(
assert api_prompts[0].prompts[0].previous_value == "input1" assert api_prompts[0].prompts[0].previous_value == "input1"
@pytest.mark.impure @pytest.mark.with_core
def test_api_set_prompts( def test_api_set_prompts(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -634,7 +634,7 @@ def test_api_set_prompts(
assert store.get(Generator("my_generator"), "prompt1").decode() == "input2" assert store.get(Generator("my_generator"), "prompt1").decode() == "input2"
@pytest.mark.impure @pytest.mark.with_core
def test_commit_message( def test_commit_message(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -691,7 +691,7 @@ def test_commit_message(
) )
@pytest.mark.impure @pytest.mark.with_core
def test_default_value( def test_default_value(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -728,7 +728,7 @@ def test_default_value(
assert json.loads(value_eval) == "hello" assert json.loads(value_eval) == "hello"
@pytest.mark.impure @pytest.mark.with_core
def test_stdout_of_generate( def test_stdout_of_generate(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -809,7 +809,7 @@ def test_stdout_of_generate(
assert "hello" not in output.out assert "hello" not in output.out
@pytest.mark.impure @pytest.mark.with_core
def test_migration_skip( def test_migration_skip(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -837,7 +837,7 @@ def test_migration_skip(
assert in_repo_store.get(Generator("my_generator"), "my_value").decode() == "world" assert in_repo_store.get(Generator("my_generator"), "my_value").decode() == "world"
@pytest.mark.impure @pytest.mark.with_core
def test_migration( def test_migration(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -873,7 +873,7 @@ def test_migration(
assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello" assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello"
@pytest.mark.impure @pytest.mark.with_core
def test_fails_when_files_are_left_from_other_backend( def test_fails_when_files_are_left_from_other_backend(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -915,7 +915,7 @@ def test_fails_when_files_are_left_from_other_backend(
) )
@pytest.mark.impure @pytest.mark.with_core
def test_keygen( def test_keygen(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
temporary_home: Path, temporary_home: Path,
@@ -935,7 +935,7 @@ def test_keygen(
assert (temporary_home / "sops" / "users" / "user").is_dir() assert (temporary_home / "sops" / "users" / "user").is_dir()
@pytest.mark.impure @pytest.mark.with_core
def test_vars_get( def test_vars_get(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,
@@ -962,7 +962,7 @@ def test_vars_get(
) )
@pytest.mark.impure @pytest.mark.with_core
def test_invalidation( def test_invalidation(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake, flake: ClanFlake,

8
templates/minimal-flake-parts/flake.nix
View File

@@ -1,10 +1,10 @@
{ {
inputs = { inputs = {
nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable"; nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
clan.url = "git+https://git.clan.lol/clan/clan-core"; clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
clan.inputs.nixpkgs.follows = "nixpkgs"; clan-core.inputs.nixpkgs.follows = "nixpkgs";
clan.inputs.flake-parts.follows = "flake-parts"; clan-core.inputs.flake-parts.follows = "flake-parts";
flake-parts.url = "github:hercules-ci/flake-parts"; flake-parts.url = "github:hercules-ci/flake-parts";
flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";

3
templates/minimal/flake.nix
View File

@@ -1,5 +1,6 @@
{ {
inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core"; inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
inputs.nixpkgs.follows = "clan-core/nixpkgs";
outputs = outputs =
{ self, clan-core, ... }: { self, clan-core, ... }:

1
templates/new-clan/flake.nix
View File

@@ -2,6 +2,7 @@
description = "<Put your description here>"; description = "<Put your description here>";
inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
inputs.nixpkgs.follows = "clan-core/nixpkgs";
outputs = outputs =
{ self, clan-core, ... }: { self, clan-core, ... }: