yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'Better install instructions for macos' (#2550) from arm64 into main

Browse Source
This commit is contained in:
clan-bot
2024-12-04 16:32:02 +00:00
parent 742b0c2c45 d5eb60d904
commit 8da4fceb24
20 changed files with 131 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
docs/site/getting-started/configure.md
View File

@@ -37,13 +37,6 @@ In the `flake.nix` file:
meta.name = "Lobsters";
# Should usually point to the directory of flake.nix
directory = ./.;
machines = {
jon = {
# ...
};
# ...
}
}
```
@@ -55,13 +48,6 @@ In the `flake.nix` file:
clan = {
# Set a unique name
meta.name = "Lobsters";
machines = {
jon = {
# ...
};
# ...
}
};
```
@@ -129,6 +115,32 @@ Adding or configuring a new machine requires two simple steps:
}
```
You can also create additional machines using the `clan machines create` command:
```
$ clan machines create --help
usage: clan [-h] [SUBCOMMAND] machines create [-h] [--tags TAGS [TAGS ...]] [--template-name TEMPLATE_NAME]
[--target-host TARGET_HOST] [--debug] [--option name value] [--flake PATH]
machine_name
positional arguments:
machine_name The name of the machine to create
options:
-h, --help show this help message and exit
--tags TAGS [TAGS ...]
Tags to associate with the machine. Can be used to assign multiple machines to services.
--template-name TEMPLATE_NAME
The name of the template machine to import
--target-host TARGET_HOST
Address of the machine to install and update, in the format of user@host:1234
--debug Enable debug logging
--option name value Nix option to set
--flake PATH path to the flake where the clan resides in, can be a remote flake or local, can be set through
the [CLAN_DIR] environment variable
```
!!! Info "Replace `__YOUR_USERNAME__` with the ip of your machine, if you use avahi you can also use your hostname"
!!! Info "Replace `__IP__` with the ip of your machine, if you use avahi you can also use your hostname"
!!! Info "Replace `__CHANGE_ME__` with the appropriate identifier, such as `nvme-eui.e8238fa6bf530001001b448b4aec2929`"

28
docs/site/getting-started/installer.md
View File

@@ -68,12 +68,6 @@ sudo umount /dev/sdb1
```
If you do not have an ssh key yet, you can generate one with `ssh-keygen -t ed25519` command.
- **Wifi Option**:
To add wifi credentials into the installer image append the option:
```
--wifi <ssid> <password>
```
- **List Keymaps**:
You can get a list of all keymaps with the following command:
```
@@ -95,7 +89,7 @@ sudo umount /dev/sdb1
For x86_64:
```shellSession
https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-installer-x86_64-linux.iso
wget https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-installer-x86_64-linux.iso
```
For generic arm64 / aarch64 (probably does not work on raspberry pi...)
@@ -104,20 +98,34 @@ sudo umount /dev/sdb1
wget https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-installer-aarch64-linux.iso
```
!!! Note
If you don't have `wget` installed, you can use `curl --progress-bar -OL <url>` instead.
### Step 2.5 Flash the Installer to the USB Drive
!!! Danger "Specifying the wrong device can lead to unrecoverable data loss."
The `dd` utility will erase the disk. Make sure to specify the correct device (`of=...`)
For example if the USB device is `sdb` use `of=/dev/sdb`.
For example if the USB device is `sdb` use `of=/dev/sdb` (on macOS it will look more like /dev/disk1)
On Linux, you can use the `lsblk` utility to identify the correct disko
```
lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
```
Use the `dd` utility to write the NixOS installer image to your USB drive:
On macos use `diskutil`:
```
diskutil list
```
Use the `dd` utility to write the NixOS installer image to your USB drive.
Replace `/dev/sd<X>` with your external drive from above.
```shellSession
sudo dd bs=4M conv=fsync oflag=direct status=progress if=./nixos-installer-x86_64-linux.iso of=/dev/sd<X>
sudo dd bs=4M conv=fsync status=progress if=./nixos-installer-x86_64-linux.iso of=/dev/sd<X>
```

10
lib/build-clan/module.nix
View File

@@ -67,13 +67,21 @@ let
# Machine specific settings
clan.core.machineName = name;
networking.hostName = lib.mkDefault name;
nixpkgs.hostPlatform = lib.mkIf (system != null) (lib.mkDefault system);
# speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs)
nix.registry.nixpkgs.to = lib.mkDefault {
type = "path";
path = lib.mkDefault nixpkgs;
};
# For vars we need to override the system so we run vars
# generators on the machine that runs `clan vars generate`. If a
# users is using the `pkgsForSystem`, we don't set
# nixpkgs.hostPlatform it would conflict with the `nixpkgs.pkgs`
# option.
nixpkgs.hostPlatform = lib.mkIf (system != null && (pkgsForSystem system) != null) (
lib.mkForce system
);
}
// lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; }
)

9
pkgs/clan-cli/clan_cli/cmd.py
View File

@@ -83,8 +83,10 @@ def handle_io(
return b""
# Extra information passed to the logger
stdout_extra = {"command_prefix": prefix}
stderr_extra = {"command_prefix": prefix}
stdout_extra = {}
stderr_extra = {}
if prefix:
stdout_extra["command_prefix"] = stderr_extra["command_prefix"] = prefix
if msg_color and msg_color.stderr:
stdout_extra["color"] = msg_color.stderr.value
if msg_color and msg_color.stdout:
@@ -267,9 +269,6 @@ def run(
if options.cwd is None:
options.cwd = Path.cwd()
if options.prefix is None:
options.prefix = "$"
if options.input:
if any(not ch.isprintable() for ch in options.input.decode("ascii", "replace")):
filtered_input = "<<binary_blob>>"

10
pkgs/clan-cli/clan_cli/custom_logger.py
View File

@@ -25,12 +25,9 @@ class PrefixFormatter(logging.Formatter):
print errors in red and warnings in yellow
"""
def __init__(
self, trace_prints: bool = False, default_prefix: str | None = None
) -> None:
def __init__(self, trace_prints: bool = False) -> None:
super().__init__()
self.default_prefix = default_prefix
self.trace_prints = trace_prints
self.hostnames: list[str] = []
self.hostname_color_offset = 0
@@ -51,7 +48,7 @@ class PrefixFormatter(logging.Formatter):
msg_color = AnsiColor.DEFAULT.value
# If extra["command_prefix"] is set, use that as the logging prefix.
command_prefix = getattr(record, "command_prefix", self.default_prefix)
command_prefix = getattr(record, "command_prefix", None)
# If color is disabled, don't use color.
if DISABLE_COLOR:
@@ -154,7 +151,6 @@ def print_trace(msg: str, logger: logging.Logger, prefix: str | None) -> None:
def setup_logging(
level: Any,
root_log_name: str = __name__.split(".")[0],
default_prefix: str = "clan",
) -> None:
# Get the root logger and set its level
main_logger = logging.getLogger(root_log_name)
@@ -166,5 +162,5 @@ def setup_logging(
# Create and add your custom handler
default_handler.setLevel(level)
trace_prints = bool(int(os.environ.get("TRACE_PRINT", "0")))
default_handler.setFormatter(PrefixFormatter(trace_prints, default_prefix))
default_handler.setFormatter(PrefixFormatter(trace_prints))
main_logger.addHandler(default_handler)

12
pkgs/clan-cli/clan_cli/flash/flash.py
View File

@@ -23,17 +23,11 @@ from .list import list_possible_keymaps, list_possible_languages
log = logging.getLogger(__name__)
@dataclass
class WifiConfig:
ssid: str
@dataclass
class SystemConfig:
language: str | None = field(default=None)
keymap: str | None = field(default=None)
ssh_keys_path: list[str] | None = field(default=None)
wifi_settings: list[WifiConfig] | None = field(default=None)
@dataclass
@@ -65,12 +59,6 @@ def flash_machine(
)
generate_facts([machine])
if system_config.wifi_settings:
wifi_settings: dict[str, dict[str, str]] = {}
for wifi in system_config.wifi_settings:
wifi_settings[wifi.ssid] = {}
system_config_nix["clan"] = {"iwd": {"networks": wifi_settings}}
if system_config.language:
if system_config.language not in list_possible_languages():
msg = (

13
pkgs/clan-cli/clan_cli/flash/flash_cmd.py
View File

@@ -10,7 +10,7 @@ from clan_cli.clan_uri import FlakeId
from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.machines.machines import Machine
from .flash import Disk, SystemConfig, WifiConfig, flash_machine
from .flash import Disk, SystemConfig, flash_machine
log = logging.getLogger(__name__)
@@ -69,15 +69,11 @@ def flash_command(args: argparse.Namespace) -> None:
language=args.language,
keymap=args.keymap,
ssh_keys_path=args.ssh_pubkey,
wifi_settings=None,
),
write_efi_boot_entries=args.write_efi_boot_entries,
nix_options=args.option,
)
if args.wifi:
opts.system_config.wifi_settings = [WifiConfig(ssid=ssid) for ssid in args.wifi]
machine = Machine(opts.machine, flake=opts.flake)
if opts.confirm and not opts.dry_run:
disk_str = ", ".join(f"{disk.name}={disk.device}" for disk in opts.disks)
@@ -126,13 +122,6 @@ def register_flash_write_parser(parser: argparse.ArgumentParser) -> None:
Mount is useful for updating an existing system without losing data.
"""
)
parser.add_argument(
"--wifi",
type=str,
action="append",
help="wifi ssid to connect to",
default=[],
)
parser.add_argument(
"--mode",
type=str,

3
pkgs/clan-cli/clan_cli/inventory/classes.py
View File

@@ -1,4 +1,5 @@
# DON NOT EDIT THIS FILE MANUALLY. IT IS GENERATED.
# DO NOT EDIT THIS FILE MANUALLY. IT IS GENERATED.
# This file was generated by running `pkgs/clan-cli/clan_cli/inventory/update.sh`
#
# ruff: noqa: N815
# ruff: noqa: N806

6
pkgs/clan-cli/clan_cli/inventory/update.sh
View File

@@ -1,4 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
jsonSchema=$(nix build .#schemas.inventory-schema-abstract --print-out-paths)/schema.json
nix run .#classgen "$jsonSchema" "$PKG_ROOT/clan_cli/inventory/classes.py" -- --stop-at "Service"
SCRIPT_DIR=$(dirname "$0")
cd "$SCRIPT_DIR"
nix run .#classgen -- "$jsonSchema" "../../../clan-cli/clan_cli/inventory/classes.py" --stop-at "Service"

23
pkgs/clan-cli/clan_cli/machines/create.py
View File

@@ -7,12 +7,12 @@ from pathlib import Path
from tempfile import TemporaryDirectory
from clan_cli.api import API
from clan_cli.clan.create import git_command
from clan_cli.clan_uri import FlakeId
from clan_cli.cmd import Log, RunOpts, run
from clan_cli.completions import add_dynamic_completer, complete_tags
from clan_cli.dirs import TemplateType, clan_templates, get_clan_flake_toplevel_or_env
from clan_cli.errors import ClanError
from clan_cli.git import commit_file
from clan_cli.inventory import Machine as InventoryMachine
from clan_cli.inventory import (
MachineDeploy,
@@ -109,10 +109,8 @@ def create_machine(opts: CreateOptions) -> None:
src = tmpdirp / "machines" / opts.template_name
if (
not (src / "configuration.nix").exists()
and not (src / "inventory.json").exists()
):
has_inventory = (dst / "inventory.json").exists()
if not (src / "configuration.nix").exists() and not has_inventory:
msg = f"Template machine '{opts.template_name}' does not contain a configuration.nix or inventory.json"
description = (
"Template machine must contain a configuration.nix or inventory.json"
@@ -126,12 +124,16 @@ def create_machine(opts: CreateOptions) -> None:
shutil.copytree(src, dst, ignore_dangling_symlinks=True, copy_function=log_copy)
run(git_command(clan_dir, "add", f"machines/{machine_name}"), RunOpts(cwd=clan_dir))
commit_file(
clan_dir / "machines" / machine_name,
repo_dir=clan_dir,
commit_message=f"Add machine {machine_name}",
)
inventory = load_inventory_json(clan_dir)
# Merge the inventory from the template
if (dst / "inventory.json").exists():
if has_inventory:
template_inventory = load_inventory_json(dst)
merge_template_inventory(inventory, template_inventory, machine_name)
@@ -141,6 +143,13 @@ def create_machine(opts: CreateOptions) -> None:
new_machine = InventoryMachine(
name=machine_name, deploy=deploy, tags=opts.machine.tags
)
if (
not has_inventory
and len(opts.machine.tags) == 0
and new_machine.deploy.targetHost is None
):
# no need to update inventory if there are no tags or target host
return
inventory.machines.update({new_machine.name: dataclass_to_dict(new_machine)})
set_inventory(inventory, clan_dir, "Imported machine from template")

43
pkgs/clan-cli/clan_cli/nix/__init__.py
View File

@@ -1,4 +1,5 @@
import json
import logging
import os
import tempfile
from pathlib import Path
@@ -8,6 +9,8 @@ from clan_cli.cmd import run, run_no_stdout
from clan_cli.dirs import nixpkgs_flake, nixpkgs_source
from clan_cli.errors import ClanError
log = logging.getLogger(__name__)
def nix_command(flags: list[str]) -> list[str]:
args = ["nix", "--extra-experimental-features", "nix-command flakes", *flags]
@@ -23,38 +26,22 @@ def nix_flake_show(flake_url: str | Path) -> list[str]:
"flake",
"show",
"--json",
"--show-trace",
f"{flake_url}",
*(["--show-trace"] if log.isEnabledFor(logging.DEBUG) else []),
str(flake_url),
]
)
def nix_build(flags: list[str], gcroot: Path | None = None) -> list[str]:
if gcroot is not None:
return (
nix_command(
[
"build",
"--out-link",
str(gcroot),
"--print-out-paths",
"--show-trace",
"--print-build-logs",
]
)
+ flags
)
return (
nix_command(
[
"build",
"--no-link",
"--print-out-paths",
"--show-trace",
"--print-build-logs",
]
)
+ flags
return nix_command(
[
"build",
"--print-out-paths",
"--print-build-logs",
*(["--show-trace"] if log.isEnabledFor(logging.DEBUG) else []),
*(["--out-root", str(gcroot)] if gcroot is not None else []),
*flags,
]
)
@@ -77,7 +64,7 @@ def nix_eval(flags: list[str]) -> list[str]:
default_flags = nix_command(
[
"eval",
"--show-trace",
*(["--show-trace"] if log.isEnabledFor(logging.DEBUG) else []),
"--json",
"--print-build-logs",
]

2
pkgs/clan-cli/pyproject.toml
View File

@@ -31,7 +31,7 @@ clan_cli = [
testpaths = "tests"
faulthandler_timeout = 60
log_level = "DEBUG"
log_format = "%(levelname)s: %(message)s\n %(pathname)s:%(lineno)d::%(funcName)s"
log_format = "%(message)s"
addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --durations 5 --color=yes --new-first -W error -n auto" # Add --pdb for debugging
norecursedirs = "tests/helpers"
markers = ["impure", "with_core"]

1
pkgs/clan-cli/tests/getpwnam-preload.c
View File

@@ -20,7 +20,6 @@ struct passwd *getpwnam(const char *name) {
fprintf(stderr, "no LOGIN_SHELL set\n");
exit(1);
}
fprintf(stderr, "SHELL:%s\n", shell);
pw->pw_shell = strdup(shell);
}
return pw;

2
pkgs/clan-cli/tests/test_flake_with_core/flake.nix
View File

@@ -15,6 +15,7 @@
vm1 =
{ lib, ... }:
{
nixpkgs.hostPlatform = "x86_64-linux";
clan.core.networking.targetHost = "__CLAN_TARGET_ADDRESS__";
system.stateVersion = lib.version;
sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
@@ -28,6 +29,7 @@
vm2 =
{ lib, ... }:
{
nixpkgs.hostPlatform = "x86_64-linux";
imports = [
clan-core.clanModules.sshd
clan-core.clanModules.root-password

10
pkgs/clan-cli/tests/test_machines_cli.py
View File

@@ -10,7 +10,15 @@ def test_machine_subcommands(
capture_output: CaptureOutput,
) -> None:
cli.run(
["machines", "create", "--flake", str(test_flake_with_core.path), "machine1"]
[
"machines",
"create",
"--flake",
str(test_flake_with_core.path),
"machine1",
"--tags",
"vm",
]
)
with capture_output as output:

11
pkgs/clan-cli/tests/test_vars.py
View File

@@ -323,6 +323,7 @@ def test_generate_secret_for_multiple_machines(
sops_setup: SopsSetup,
) -> None:
machine1_config = flake.machines["machine1"]
machine1_config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
machine1_generator = machine1_config["clan"]["core"]["vars"]["generators"][
"my_generator"
]
@@ -332,6 +333,7 @@ def test_generate_secret_for_multiple_machines(
"echo machine1 > $out/my_secret && echo machine1 > $out/my_value"
)
machine2_config = flake.machines["machine2"]
machine2_config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
machine2_generator = machine2_config["clan"]["core"]["vars"]["generators"][
"my_generator"
]
@@ -384,6 +386,7 @@ def test_dependant_generators(
flake: ClanFlake,
) -> None:
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
parent_gen = config["clan"]["core"]["vars"]["generators"]["parent_generator"]
parent_gen["files"]["my_value"]["secret"] = False
parent_gen["script"] = "echo hello > $out/my_value"
@@ -426,6 +429,7 @@ def test_prompt(
input_value: str,
) -> None:
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["files"]["my_value"]["secret"] = False
my_generator["prompts"]["prompt1"]["description"] = "dream2nix"
@@ -521,6 +525,7 @@ def test_depending_on_shared_secret_succeeds(
sops_setup: SopsSetup,
) -> None:
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
shared_generator = config["clan"]["core"]["vars"]["generators"]["shared_generator"]
shared_generator["share"] = True
shared_generator["files"]["my_secret"]["secret"] = True
@@ -550,6 +555,7 @@ def test_prompt_create_file(
Test that the createFile flag in the prompt configuration works as expected
"""
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["prompts"]["prompt1"]["createFile"] = True
my_generator["prompts"]["prompt2"]["createFile"] = False
@@ -580,6 +586,7 @@ def test_api_get_prompts(
from clan_cli.vars.list import get_prompts
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["prompts"]["prompt1"]["type"] = "line"
my_generator["files"]["prompt1"]["secret"] = False
@@ -604,6 +611,7 @@ def test_api_set_prompts(
from clan_cli.vars.list import set_prompts
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["prompts"]["prompt1"]["type"] = "line"
my_generator["files"]["prompt1"]["secret"] = False
@@ -641,6 +649,7 @@ def test_commit_message(
sops_setup: SopsSetup,
) -> None:
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["files"]["my_value"]["secret"] = False
my_generator["script"] = "echo hello > $out/my_value"
@@ -952,6 +961,7 @@ def test_vars_get(
flake: ClanFlake,
) -> None:
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["files"]["my_value"]["secret"] = False
my_generator["script"] = "echo -n hello > $out/my_value"
@@ -979,6 +989,7 @@ def test_invalidation(
flake: ClanFlake,
) -> None:
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["files"]["my_value"]["secret"] = False
my_generator["script"] = "echo -n $RANDOM > $out/my_value"

1
pkgs/clan-cli/tests/test_vms_cli.py
View File

@@ -61,6 +61,7 @@ def test_vm_persistence(
) -> None:
# set up a clan flake with some systemd services to test persistence
config = flake.machines["my_machine"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
# logrotate-checkconf doesn't work in VM because /nix/store is owned by nobody
config["systemd"]["services"]["logrotate-checkconf"]["enable"] = False
config["services"]["getty"]["autologinUser"] = "root"

3
pkgs/classgen/main.py
View File

@@ -349,7 +349,8 @@ def run_gen(args: argparse.Namespace) -> None:
with args.output.open("w") as f:
f.write(
"""# DON NOT EDIT THIS FILE MANUALLY. IT IS GENERATED.
"""# DO NOT EDIT THIS FILE MANUALLY. IT IS GENERATED.
# This file was generated by running `pkgs/clan-cli/clan_cli/inventory/update.sh`
#
# ruff: noqa: N815
# ruff: noqa: N806

1
pkgs/webview-ui/app/src/routes/flash/view.tsx
View File

@@ -157,7 +157,6 @@ export const Flash = () => {
language: values.language,
keymap: values.keymap,
ssh_keys_path: values.sshKeys.map((file) => file.name),
wifi_settings: values.wifi,
},
dry_run: false,
write_efi_boot_entries: false,

9
templates/clan/new-clan/flake.nix
View File

@@ -13,14 +13,17 @@
# Ensure this is unique among all clans you want to use.
meta.name = "__CHANGE_ME__";
# All machines in the ./machines will be imported.
# Prerequisite: boot into the installer.
# See: https://docs.clan.lol/getting-started/installer
# local> mkdir -p ./machines/machine1
# local> Edit ./machines/<machine>/configuration.nix to your liking.
machines = {
# The name will be used as hostname by default.
jon = { };
sara = { };
# You can also specify additional machines here.
# somemachine = {
# imports = [ ./some-machine/configuration.nix ];
# }
};
};
in