yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'Inventory: docs improvements' (#2132) from hsjobeki/clan-core:hsjobeki-inventory-docs into main

Browse Source
This commit is contained in:
clan-bot
2024-09-16 20:58:28 +00:00
parent 8ccc695b92 3721fc4340
commit ba8e46fe7a
29 changed files with 240 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
clanModules/admin/README.md
View File

@@ -1,4 +1,5 @@
---
description = "Convenient Administration for the Clan App"
categories = ["administration"]
categories = ["Utility"]
features = [ "inventory" ]
---

1
clanModules/admin/roles/default.nix
View File

@@ -1 +0,0 @@
{ }

5
clanModules/borgbackup-static/README.md
View File

@@ -1,6 +1,11 @@
---
description = "Statically configure borgbackup with sane defaults."
---
!!! Danger "Deprecated"
Use [borgbackup](borgbackup.md) instead.
Don't use borgbackup-static through [inventory](../../guides/inventory.md).
This module implements the `borgbackup` backend and implements sane defaults
for backup management through `borgbackup` for members of the clan.

3
clanModules/borgbackup/README.md
View File

@@ -1,6 +1,7 @@
---
description = "Efficient, deduplicating backup program with optional compression and secure encryption."
categories = ["backup"]
categories = ["System"]
features = [ "inventory" ]
---
BorgBackup (short: Borg) gives you:

1
clanModules/disk-id/README.md
View File

@@ -1,3 +1,4 @@
---
description = "Generates a uuid for use in disk device naming"
features = [ "inventory" ]
---

1
clanModules/disk-id/roles/default.nix
View File

@@ -1 +0,0 @@
{ }

1
clanModules/iwd/README.md
View File

@@ -1,5 +1,6 @@
---
description = "Automatically provisions wifi credentials"
features = [ "inventory" ]
---
!!! Warning

1
clanModules/iwd/roles/default.nix
View File

@@ -1 +0,0 @@
{ }

1
clanModules/machine-id/README.md
View File

@@ -1,3 +1,4 @@
---
description = "Sets the /etc/machine-id and exposes it as a nix option"
features = [ "inventory" ]
---

1
clanModules/machine-id/roles/default.nix
View File

@@ -1 +0,0 @@
{ }

2
clanModules/mumble/README.md
View File

@@ -1,6 +1,6 @@
---
description = "Open Source, Low Latency, High Quality Voice Chat."
categories = ["chat", "voice"]
categories = ["Audio", "Social"]
---
The mumble clan module gives you:

3
clanModules/packages/README.md
View File

@@ -1,4 +1,5 @@
---
description = "Define package sets from nixpkgs and install them on one or more machines"
categories = ["packages"]
categories = ["System"]
features = [ "inventory" ]
---

1
clanModules/packages/roles/default.nix
View File

@@ -1 +0,0 @@
{ }

3
clanModules/single-disk/README.md
View File

@@ -1,6 +1,7 @@
---
description = "Configures partitioning of the main disk"
categories = ["disk-layout"]
categories = ["System"]
features = [ "inventory" ]
---
# Primary Disk Layout

1
clanModules/single-disk/roles/default.nix
View File

@@ -1 +0,0 @@
{ }

1
clanModules/state-version/README.md
View File

@@ -1,5 +1,6 @@
---
description = "Automatically generate the state version of the nixos installation."
features = [ "inventory" ]
---
This module generates the `system.stateVersion` of the nixos installation automatically.

1
clanModules/state-version/roles/default.nix
View File

@@ -1 +0,0 @@
{ }

3
docs/mkdocs.yml
View File

@@ -51,10 +51,12 @@ nav:
- Flake-parts: getting-started/flake-parts.md
- Guides:
- guides/index.md
- Adding Machines: guides/add-machines.md
- Inventory: guides/inventory.md
- Reference:
- reference/index.md
- Clan Modules:
- reference/clanModules/index.md
- reference/clanModules/admin.md
- reference/clanModules/borgbackup-static.md
- reference/clanModules/borgbackup.md
@@ -65,7 +67,6 @@ nav:
- reference/clanModules/garage.md
- reference/clanModules/golem-provider.md
- reference/clanModules/heisenbridge.md
- reference/clanModules/index.md
- reference/clanModules/iwd.md
- reference/clanModules/localbackup.md
- reference/clanModules/localsend.md

55
docs/nix/render_options/__init__.py
View File

@@ -28,7 +28,7 @@ import os
from pathlib import Path
from typing import Any
from clan_cli.api.modules import Frontmatter, extract_frontmatter
from clan_cli.api.modules import Frontmatter, extract_frontmatter, get_roles
from clan_cli.errors import ClanError
# Get environment variables
@@ -125,14 +125,15 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
return res
def module_header(module_name: str) -> str:
return f"# {module_name}\n\n"
def module_header(module_name: str, has_inventory_feature: bool = False) -> str:
indicator = " 🔹" if has_inventory_feature else ""
return f"# {module_name}{indicator}\n\n"
def module_usage(module_name: str) -> str:
return f"""## Usage
To use this module, import it like this:
To use this module, import it like th:
```nix
{{config, lib, inputs, ...}}: {{
@@ -198,25 +199,13 @@ def render_roles(roles: list[str] | None, module_name: str) -> str:
if roles:
roles_list = "\n".join([f" - `{r}`" for r in roles])
return f"""
???+ tip "Inventory usage"
## Inventory Roles
Predefined roles:
Predefined roles
{roles_list}
Usage:
```{{.nix hl_lines="4"}}
buildClan {{
inventory.services = {{
{module_name}.instance_1 = {{
roles.{roles[0]}.machines = [ "sara_machine" ];
# ...
}};
}};
}}
```
For more information, see the [inventory guide](../../guides/inventory.md).
"""
return ""
@@ -226,6 +215,21 @@ clan_modules_descr = """Clan modules are [NixOS modules](https://wiki.nixos.org/
"""
def render_categories(categories: list[str], frontmatter: Frontmatter) -> str:
cat_info = frontmatter.categories_info
res = """<div style="display: grid; grid-template-columns: repeat(3, 1fr); gap: 10px;">"""
for cat in categories:
color = cat_info[cat]["color"]
# description = cat_info[cat]["description"]
res += f"""
<div style="background-color: {color}; color: white; padding: 10px; border-radius: 20px; text-align: center;">
{cat}
</div>
"""
res += "</div>"
return res
def produce_clan_modules_docs() -> None:
if not CLAN_MODULES:
msg = f"Environment variables are not set correctly: $out={CLAN_MODULES}"
@@ -270,16 +274,21 @@ def produce_clan_modules_docs() -> None:
with (Path(options_file) / "share/doc/nixos/options.json").open() as f:
options: dict[str, dict[str, Any]] = json.load(f)
print(f"Rendering options for {module_name}...")
output = module_header(module_name)
output = module_header(module_name, "inventory" in frontmatter.features)
if frontmatter.description:
output += f"**{frontmatter.description}**\n\n"
output += "## Categories\n\n"
output += render_categories(frontmatter.categories, frontmatter)
output += "\n---\n\n"
output += f"{readme_content}\n"
# get_roles(str) -> list[str] | None
# roles = get_roles(CLAN_CORE_PATH / "clanModules" / module_name)
# if roles:
# output += render_roles(roles, module_name)
roles = get_roles(CLAN_CORE_PATH / "clanModules" / module_name)
if roles:
output += render_roles(roles, module_name)
output += module_usage(module_name)

50
docs/site/guides/add-machines.md Normal file
View File

@@ -0,0 +1,50 @@
# How to add machines
Clan has two general methods of adding machines
- **Automatic**: Detects every folder in the `machines` folder.
- **Declarative**: Explicit declarations in nix.
## Automatic register
Every machine of the form `machines/{machineName}` will be registered automatically.
Automatically imported:
- [x] ``machines/{machineName}/configuration.nix`
- [x] ``machines/{machineName}/hardware-configuration.nix`
- [x] ``machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](../blog/posts/nixos-facter.md)
## Manual declaration
Machines can also be added manually under `buildClan`, `clan.*` in flake-parts or via [`inventory`](../guides/inventory.md).
!!! Note
It is possible to use `inventory` and `buildClan` together at the same time.
=== "**Individual Machine Configuration**"
```{.nix}
buildClan {
machines = {
"jon" = {
# Any valid nixos config
};
};
}
```
=== "**Inventory Configuration**"
```{.nix}
buildClan {
inventory = {
machines = {
"jon" = {
# Inventory machines can set tags
tags = [ "zone1" ];
};
};
};
}
```

1
docs/site/guides/index.md
View File

@@ -2,4 +2,5 @@
Detailed guides into the following subtopics:
- [Adding Machines](./add-machines.md): How to add machines.
- [Inventory](./inventory.md): Configuring Services across machine boundaries

103
docs/site/guides/inventory.md
View File

@@ -4,73 +4,36 @@
See [Inventory API Documentation](../reference/nix-api/inventory.md)
This guide will walk you through setting up a backup-service, where the inventory becomes useful.
This guide will walk you through setting up a backup service, where the inventory becomes useful.
## Prerequisites Meta (optional)
!!! example "Experimental status"
The inventory implementation is not considered stable yet.
We are actively soliciting feedback from users.
Metadata about the clan, will be displayed upfront in the upcomming clan-app, make sure to choose a unique name.
Stabilizing the API is a priority.
Make sure to set `name` either via `inventory.meta` OR via `clan.meta`.
## Categories
```{.nix hl_lines="3-8"}
buildClan {
inventory = {
meta = {
name = "Superclan"
description = "Awesome backups and family stuff"
};
};
}
```
<div style="display: grid; grid-template-columns: repeat(3, 1fr); gap: 10px;">
<!-- Chip 1 -->
<div style="background-color: #F5B7B1; color: white; padding: 10px; border-radius: 20px; text-align: center;">
Light Coral
</div>
<!-- Chip 2 -->
<div style="background-color: #85C1E9; color: white; padding: 10px; border-radius: 20px; text-align: center;">
Baby Blue
</div>
</div>
## How to add machines
## Prerequisites
Every machine of the form `machines/{machineName}/configuration.nix` will be registered automatically.
Machines can also be manually added under `inventory.machines` OR via `buildClan` directly.
!!! Note
It doesn't matter where the machine gets introduced to buildClan - All delarations are valid, duplications are merged.
However the clan-app (UI) will create machines in the inventory, because it cannot create arbitrary nix code or nixos configs.
In the following example `backup_server` is one machine - it may specify parts of its configuration in different places.
```{.nix hl_lines="3-5 12-20"}
buildClan {
machines = {
"backup_server" = {
# Any valid nixos config
};
"jon" = {
# Any valid nixos config
};
};
inventory = {
machines = {
"backup_server" = {
# Don't include any nixos config here.
# See the Inventory API Docs for the available attributes.
};
"jon" = {
# Same as above
};
};
};
}
```
- [x] [Add machines](./add-machines.md) to your clan.
## Services
### Available clanModules
The inventory defines `services`. Membership of `machines` is defined via roles exclusively.
Currently the inventory interface is implemented by the following clanModules
- [borgbackup](../reference/clanModules/borgbackup.md)
- [packages](../reference/clanModules/packages.md)
- [single-disk](../reference/clanModules/single-disk.md)
See the respective module documentation for available roles.
See the each [module documentation](../reference/clanModules/index.md) for available roles.
!!! Note
It is possible to use any [clanModule](../reference/clanModules/index.md) in the inventory and add machines via
@@ -95,21 +58,12 @@ Each service can still be customized and configured according to the modules opt
See also: [Multiple Service Instances](#multiple-service-instances)
```{.nix hl_lines="14-17"}
```{.nix hl_lines="6-7"}
buildClan {
inventory = {
machines = {
"backup_server" = {
# Don't include any nixos config here
# See inventory.Machines for available options
};
"jon" = {
# Don't include any nixos config here
# See inventory.Machines for available options
};
};
services = {
borgbackup.instance_1 = {
# Machines can be added here.
roles.client.machines = [ "jon" ];
roles.server.machines = [ "backup_server" ];
};
@@ -120,24 +74,23 @@ Each service can still be customized and configured according to the modules opt
### Scaling the Backup
It is possible to add services to multiple machines via tags. The service instance gets added in the specified role. In this case `role = "client"`
The inventory allows machines to set Tags
It is possible to add services to multiple machines via tags as shown
!!! Example "Tags Example"
```{.nix hl_lines="9 12 17"}
```{.nix hl_lines="5 8 14"}
buildClan {
inventory = {
machines = {
"backup_server" = {
# Don't include any nixos config here
# See inventory.Machines for available options
};
"jon" = {
tags = [ "backup" ];
};
"sara" = {
tags = [ "backup" ];
};
# ...
};
services = {
borgbackup.instance_1 = {
@@ -172,7 +125,7 @@ It is possible to add services to multiple machines via tags. The service instan
roles.client.machines = [ "jon" ];
roles.server.machines = [ "backup_server" ];
};
borgbackup.instance_1 = {
borgbackup.instance_2 = {
roles.client.machines = [ "backup_server" ];
roles.server.machines = [ "backup_backup_server" ];
};

5
lib/build-clan/module.nix
View File

@@ -68,7 +68,7 @@ let
hwConfig = "${directory}/machines/${name}/hardware-configuration.nix";
facterModules = lib.optionals (builtins.pathExists facterJson) [
"${clan-core.inputs.nixos-facter-modules}/modules/nixos/facter.nix"
clan-core.inputs.nixos-facter-modules.nixosModules.facter
{ config.facter.reportPath = facterJson; }
];
in
@@ -231,6 +231,9 @@ in
)
);
}
{
inventory.machines = lib.mapAttrs (_n: _: { }) config.machines;
}
# Merge the meta attributes from the buildClan function
{ inventory.meta = if config.meta != null then config.meta else { }; }
];

21
lib/description.nix
View File

@@ -12,7 +12,7 @@ rec {
in
readmeContents;
getShortDescription =
getFrontmatter =
modulename:
let
content = getReadme modulename;
@@ -21,14 +21,21 @@ rec {
parsed = builtins.partition ({ index, ... }: if index >= 2 then false else true) (
lib.filter ({ index, ... }: index != 0) (lib.imap0 (index: part: { inherit index part; }) parts)
);
# Use this if the content is needed
# readmeContent = lib.concatMapStrings (v: "---" + v.part) parsed.wrong;
meta = builtins.fromTOML (builtins.head parsed.right).part;
in
if (builtins.length parts >= 3) then
meta.description
meta
else
throw "Short description delimiter `---` not found in README.md for module ${modulename}";
throw ''
TOML Frontmatter not found in README.md for module ${modulename}
Please add the following to the top of your README.md:
---
description = "Your description here"
categories = [ "Your categories here" ]
features = [ "inventory" ]
---
...rest of your README.md...
'';
}

1
lib/inventory/build-inventory/assertions.nix
View File

@@ -74,4 +74,5 @@
) [ ] config.services;
}
];
}

72
lib/inventory/build-inventory/default.nix
View File

@@ -37,44 +37,18 @@ let
) [ ] members.tags or [ ]);
};
checkService =
serviceName:
let
frontmatter = clan-core.lib.modules.getFrontmatter serviceName;
in
if builtins.elem "inventory" frontmatter.features or [ ] then true else false;
/*
Returns a NixOS configuration for every machine in the inventory.
machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
*/
# { client_1_machine = { tags = [ "backup" ]; }; client_2_machine = { tags = [ "backup" ]; }; not_used_machine = { }; }
getAllMachines =
inventory:
lib.foldlAttrs (
res: serviceName: serviceConfigs:
(lib.foldlAttrs (
res: instanceName: serviceConfig:
lib.foldlAttrs (
res: roleName: members:
let
resolved = resolveTags {
inherit
serviceName
instanceName
roleName
inventory
members
;
};
in
res
// builtins.listToAttrs (
builtins.map (m: {
name = m;
value = { };
}) resolved.machines
)
) res serviceConfig.roles
) res serviceConfigs)
) { } (inventory.services or { })
// inventory.machines or { };
buildInventory =
{ inventory, directory }:
# For every machine in the inventory, build a NixOS configuration
@@ -178,7 +152,7 @@ let
]
else
acc2
) [ ] serviceConfigs)
) [ ] (serviceConfigs))
) [ ] inventory.services
# Append each machine config
++ [
@@ -188,9 +162,35 @@ let
(lib.optionalAttrs (machineConfig.deploy.targetHost or null != null) {
config.clan.core.networking.targetHost = machineConfig.deploy.targetHost;
})
{
assertions = lib.foldlAttrs (
acc: serviceName: _:
acc
++ [
{
assertion = checkService serviceName;
message = ''
Service ${serviceName} cannot be used in inventory. It does not declare the 'inventory' feature.
To allow it add the following to the beginning of the README.md of the module:
---
...
features = [ "inventory" ]
---
Also make sure to test the module with the 'inventory' feature enabled.
'';
}
]
) (getAllMachines inventory);
) [ ] inventory.services;
}
]
) (inventory.machines or { });
in
{
inherit buildInventory getAllMachines;
inherit buildInventory;
}

37
lib/inventory/tests/default.nix
View File

@@ -1,37 +1,8 @@
{ inventory, clan-core, ... }:
let
inherit (inventory) buildInventory getAllMachines;
inherit (inventory) buildInventory;
in
{
test_get_all_used_machines = {
# Test that all machines are returned
expr = getAllMachines {
machines = {
machine_3 = {
tags = [ "tag_3" ];
};
};
services = {
borgbackup.instance_1 = {
roles.server.machines = [ "backup_server" ];
roles.client.machines = [
"client_1_machine"
"client_2_machine"
];
roles.client.tags = [ "tag_3" ];
};
};
};
expected = {
backup_server = { };
client_1_machine = { };
client_2_machine = { };
machine_3 = {
tags = [ "tag_3" ];
};
};
};
test_inventory_empty = {
# Empty inventory should return an empty module
expr = buildInventory {
@@ -117,9 +88,9 @@ in
not_used_machine = builtins.length configs.not_used_machine;
};
expected = {
client_1_machine = 5;
client_2_machine = 5;
not_used_machine = 2;
client_1_machine = 6;
client_2_machine = 6;
not_used_machine = 3;
};
};

43
pkgs/clan-cli/clan_cli/api/modules.py
View File

@@ -1,9 +1,9 @@
import json
import re
import tomllib
from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, get_args, get_type_hints
from typing import Any, TypedDict, get_args, get_type_hints
from clan_cli.cmd import run_no_stdout
from clan_cli.errors import ClanCmdError, ClanError
@@ -15,10 +15,47 @@ from . import API
from .serde import from_dict
class CategoryInfo(TypedDict):
color: str
description: str
@dataclass
class Frontmatter:
description: str
categories: list[str] | None = None
categories: list[str] = field(default_factory=lambda: ["Uncategorized"])
features: list[str] = field(default_factory=list)
@property
def categories_info(self) -> dict[str, CategoryInfo]:
category_map: dict[str, CategoryInfo] = {
"AudioVideo": {
"color": "#AEC6CF",
"description": "Applications for presenting, creating, or processing multimedia (audio/video)",
},
"Audio": {"color": "#CFCFC4", "description": "Audio"},
"Video": {"color": "#FFD1DC", "description": "Video"},
"Development": {"color": "#F49AC2", "description": "Development"},
"Education": {"color": "#B39EB5", "description": "Education"},
"Game": {"color": "#FFB347", "description": "Game"},
"Graphics": {"color": "#FF6961", "description": "Graphics"},
"Social": {"color": "#76D7C4", "description": "Social"},
"Network": {"color": "#77DD77", "description": "Network"},
"Office": {"color": "#85C1E9", "description": "Office"},
"Science": {"color": "#779ECB", "description": "Science"},
"System": {"color": "#F5C3C0", "description": "System"},
"Settings": {"color": "#03C03C", "description": "Settings"},
"Utility": {"color": "#B19CD9", "description": "Utility"},
"Uncategorized": {"color": "#C23B22", "description": "Uncategorized"},
}
return category_map
def __post_init__(self) -> None:
for category in self.categories:
if category not in self.categories_info:
msg = f"Invalid category: {category}"
raise ValueError(msg)
def extract_frontmatter(readme_content: str, err_scope: str) -> tuple[Frontmatter, str]:

9
pkgs/schemas/flake-module.nix
View File

@@ -23,11 +23,14 @@
modulename: _: jsonLib.parseOptions (optionsFromModule modulename) { }
) clanModules;
clanModuleFunctionSchemas = lib.mapAttrsFlatten (modulename: _: {
clanModuleFunctionSchemas = lib.mapAttrsFlatten (
modulename: _:
(self.lib.modules.getFrontmatter modulename)
// {
name = modulename;
description = self.lib.modules.getShortDescription modulename;
parameters = jsonLib.parseOptions (optionsFromModule modulename) { };
}) clanModules;
}
) clanModules;
in
rec {
checks = {