diff --git a/docs/.gitignore b/docs/.gitignore index 932a47563..1658c2ad8 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1,6 @@ -/site/reference +/site/reference/clan-core +/site/reference/clanModules +/site/reference/nix-api/inventory.md +/site/reference/cli /site/static/Roboto-Regular.ttf /site/static/FiraCode-VF.ttf \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index e704599ec..8bd5d22ac 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -48,11 +48,13 @@ nav: - Mesh VPN: getting-started/mesh-vpn.md - Backup & Restore: getting-started/backups.md - Flake-parts: getting-started/flake-parts.md - - Concepts: - - Configuration: concepts/configuration.md - - Inventory: concepts/inventory.md + - Guides: + - guides/index.md + - Inventory: guides/inventory.md - Reference: + - reference/index.md - Clan Modules: + - reference/clanModules/index.md - reference/clanModules/borgbackup-static.md - reference/clanModules/borgbackup.md - reference/clanModules/deltachat.md @@ -77,13 +79,13 @@ nav: - reference/clanModules/zerotier-static-peers.md - reference/clanModules/zt-tcp-relay.md - CLI: + - reference/cli/index.md - reference/cli/backups.md - reference/cli/config.md - reference/cli/facts.md - reference/cli/flakes.md - reference/cli/flash.md - reference/cli/history.md - - reference/cli/index.md - reference/cli/machines.md - reference/cli/secrets.md - reference/cli/show.md @@ -98,6 +100,10 @@ nav: - reference/clan-core/state.md - reference/clan-core/deployment.md - reference/clan-core/networking.md + - Nix API: + - reference/nix-api/index.md + - buildClan: reference/nix-api/buildclan.md + - Inventory: reference/nix-api/inventory.md - Contributing: contributing/contributing.md - Blog: - blog/index.md diff --git a/docs/nix/default.nix b/docs/nix/default.nix index 01dae322f..914024538 100644 --- a/docs/nix/default.nix +++ b/docs/nix/default.nix @@ -2,6 +2,7 @@ pkgs, module-docs, clan-cli-docs, + inventory-api-docs, asciinema-player-js, asciinema-player-css, roboto, @@ -32,6 +33,7 @@ pkgs.stdenv.mkDerivation { mkdir -p ./site/reference/cli cp -af ${module-docs}/* ./site/reference/ cp -af ${clan-cli-docs}/* ./site/reference/cli/ + cp -af ${inventory-api-docs} ./site/reference/nix-api/inventory.md mkdir -p ./site/static/asciinema-player ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js diff --git a/docs/nix/flake-module.nix b/docs/nix/flake-module.nix index bd44e94b7..89052d53f 100644 --- a/docs/nix/flake-module.nix +++ b/docs/nix/flake-module.nix @@ -73,7 +73,7 @@ in { devShells.docs = pkgs.callPackage ./shell.nix { - inherit (self'.packages) docs clan-cli-docs; + inherit (self'.packages) docs clan-cli-docs inventory-api-docs; inherit asciinema-player-js asciinema-player-css @@ -83,7 +83,7 @@ }; packages = { docs = pkgs.python3.pkgs.callPackage ./default.nix { - inherit (self'.packages) clan-cli-docs; + inherit (self'.packages) clan-cli-docs inventory-api-docs; inherit (inputs) nixpkgs; inherit module-docs; inherit asciinema-player-js; diff --git a/docs/nix/scripts/renderOptions.py b/docs/nix/scripts/renderOptions.py index 807cb12b7..323dbc528 100644 --- a/docs/nix/scripts/renderOptions.py +++ b/docs/nix/scripts/renderOptions.py @@ -220,6 +220,11 @@ def render_roles(roles: list[str] | None, module_name: str) -> str: return "" +clan_modules_descr = """Clan modules are [NixOS modules](https://wiki.nixos.org/wiki/NixOS_modules) which have been enhanced with additional features provided by Clan, with certain option types restricted to enable configuration through a graphical interface. + +""" + + def produce_clan_modules_docs() -> None: if not CLAN_MODULES: raise ValueError( @@ -245,6 +250,12 @@ def produce_clan_modules_docs() -> None: # print(meta_map) # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... } + + modules_index = "# Modules Overview\n\n" + modules_index += clan_modules_descr + modules_index += "## Overview\n\n" + modules_index += '
\n\n' + for module_name, options_file in links.items(): readme_file = Path(CLAN_CORE_PATH) / "clanModules" / module_name / "README.md" print(module_name, readme_file) @@ -254,6 +265,8 @@ def produce_clan_modules_docs() -> None: frontmatter, readme_content = extract_frontmatter(readme, str(readme_file)) print(frontmatter, readme_content) + modules_index += build_option_card(module_name, frontmatter) + with open(Path(options_file) / "share/doc/nixos/options.json") as f: options: dict[str, dict[str, Any]] = json.load(f) print(f"Rendering options for {module_name}...") @@ -282,6 +295,39 @@ def produce_clan_modules_docs() -> None: with open(outfile, "w") as of: of.write(output) + modules_index += "
" + modules_index += "\n" + modules_outfile = Path(OUT) / "clanModules/index.md" + + with open(modules_outfile, "w") as of: + of.write(modules_index) + + +def build_option_card(module_name: str, frontmatter: Frontmatter) -> str: + """ + Build the overview index card for each reference target option. + """ + + def indent_all(text: str, indent_size: int = 4) -> str: + """ + Indent all lines in a string. + """ + indent = " " * indent_size + lines = text.split("\n") + indented_text = indent + ("\n" + indent).join(lines) + return indented_text + + def to_md_li(module_name: str, frontmatter: Frontmatter) -> str: + md_li = ( + f"""- **[{module_name}](./{"-".join(module_name.split(" "))}.md)**\n\n""" + ) + md_li += f"""{indent_all("---", 4)}\n\n""" + fmd = f"\n{frontmatter.description.strip()}" if frontmatter.description else "" + md_li += f"""{indent_all(fmd, 4)}""" + return md_li + + return f"{to_md_li(module_name, frontmatter)}\n\n" + if __name__ == "__main__": produce_clan_core_docs() diff --git a/docs/nix/shell.nix b/docs/nix/shell.nix index 29432d906..fd9246265 100644 --- a/docs/nix/shell.nix +++ b/docs/nix/shell.nix @@ -3,6 +3,7 @@ pkgs, module-docs, clan-cli-docs, + inventory-api-docs, asciinema-player-js, asciinema-player-css, roboto, @@ -19,6 +20,8 @@ pkgs.mkShell { mkdir -p ./site/reference/cli cp -af ${module-docs}/* ./site/reference/ cp -af ${clan-cli-docs}/* ./site/reference/cli/ + cp -af ${inventory-api-docs} ./site/reference/nix-api/inventory.md + chmod +w ./site/reference/* echo "Generated API documentation in './site/reference/' " diff --git a/docs/site/concepts/configuration.md b/docs/site/concepts/configuration.md deleted file mode 100644 index 1ba0df378..000000000 --- a/docs/site/concepts/configuration.md +++ /dev/null @@ -1,56 +0,0 @@ -# Configuration - -## Introduction - -When managing machine configuration this can be done through many possible ways. -Ranging from writing `nix` expression in a `flake.nix` file; placing `autoincluded` files into your machine directory; or configuring everything in a simple UI (upcomming). - -clan currently offers the following methods to configure machines: - -!!! Success "Recommended for nix people" - - - flake.nix (i.e. via `buildClan`) - - `machine` argument - - `inventory` argument - - - machines/`machine_name`/configuration.nix (`autoincluded` if it exists) - -???+ Note "Used by CLI & UI" - - - inventory.json - - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists) - - -!!! Warning "Deprecated" - - machines/`machine_name`/settings.json - -## BuildClan - -The core function that produces a clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services. - -### Inputs - -`directory` -: The directory containing the machines subdirectory - -`machines` -: Allows to include machine-specific modules i.e. machines.${name} = { ... } - -`meta` -: An optional set - -: `{ name :: string, icon :: string, description :: string }` - -`inventory` -: Service set for easily configuring distributed services, such as backups - -: For more details see [Inventory](./inventory.md) - -`specialArgs` -: Extra arguments to pass to nixosSystem i.e. useful to make self available - -`pkgsForSystem` -: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system. - This improves performance, but all nipxkgs.* options will be ignored. - `(string -> pkgs )` diff --git a/docs/site/getting-started/configure.md b/docs/site/getting-started/configure.md index 8f191d532..c06b25cf9 100644 --- a/docs/site/getting-started/configure.md +++ b/docs/site/getting-started/configure.md @@ -1,5 +1,31 @@ # Configuration - How to configure clan with your own machines +Managing machine configurations can be done in the following ways: + +- writing `nix` expressions in a `flake.nix` file, +- placing `autoincluded` files into your machine directory, +- configuring everything in a simple UI (upcoming). + +Clan currently offers the following methods to configure machines: + +!!! Success "Recommended for nix people" + + - flake.nix (i.e. via `buildClan`) + - `machine` argument + - `inventory` argument + + - machines/`machine_name`/configuration.nix (`autoincluded` if it exists) + +???+ Note "Used by CLI & UI" + + - inventory.json + - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists) + + +!!! Warning "Deprecated" + + machines/`machine_name`/settings.json + ## Global configuration In the `flake.nix` file: diff --git a/docs/site/guides/index.md b/docs/site/guides/index.md new file mode 100644 index 000000000..b4a625c49 --- /dev/null +++ b/docs/site/guides/index.md @@ -0,0 +1,5 @@ +# Guides + +Detailed guides into the following subtopics: + +- [Inventory](./inventory.md): Configuring Services across machine boundaries diff --git a/docs/site/concepts/inventory.md b/docs/site/guides/inventory.md similarity index 75% rename from docs/site/concepts/inventory.md rename to docs/site/guides/inventory.md index 6d966ab0e..7e9bd93dc 100644 --- a/docs/site/concepts/inventory.md +++ b/docs/site/guides/inventory.md @@ -2,31 +2,35 @@ `Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries. -## Meta +See [Inventory API Documentation](../reference/nix-api/inventory.md) + +This guide will walk you through setting up a backup-service, where the inventory becomes useful. + +## Prerequisites Meta (optional) Metadata about the clan, will be displayed upfront in the upcomming clan-app, make sure to choose a unique name. +Make sure to set `name` either via `inventory.meta` OR via `clan.meta`. + ```{.nix hl_lines="3-8"} buildClan { inventory = { meta = { - # The following options are available - # name: string # Required, name of the clan. - # description: null | string - # icon: null | string + name = "Superclan" + description = "Awesome backups and family stuff" }; }; } ``` -## Machines +## How to add machines -Machines and a small pieve of their configuration can be added via `inventory.machines`. +Machines can be added via `inventory.machines` OR via `buildClan` directly. !!! Note It doesn't matter where the machine gets introduced to buildClan - All delarations are valid, duplications are merged. - However the clan-app (UI) will create machines in the inventory, because it cannot create arbitrary nixos configs. + However the clan-app (UI) will create machines in the inventory, because it cannot create arbitrary nix code or nixos configs. In the following example `backup_server` is one machine - it may specify parts of its configuration in different places. @@ -78,9 +82,10 @@ A module can be added to one or multiple machines via `Roles`. clan's `Role` int Each service can still be customized and configured according to the modules options. - Per instance configuration via `services...config` +- Per role configuration via `services...roles..config` - Per machine configuration via `services...machines..config` -### Configuration Examples +### Setting up the Backup Service !!! Example "Borgbackup Example" @@ -112,49 +117,33 @@ Each service can still be customized and configured according to the modules opt } ``` -!!! Example "Packages Example" +### Scalabling the Backup - This example shows how to add `pkgs.firefox` via the inventory interface. +The inventory allows machines to set **Tags** - ```{.nix hl_lines="8-11"} - buildClan { - inventory = { - machines = { - "sara" = {}; - "jon" = {}; - }; - services = { - packages.set_1 = { - roles.default.machines = [ "jon" "sara" ]; - # Packages is a configuration option of the "packages" clanModule - config.packages = ["firefox"]; - }; - }; - }; - } - ``` - -### Tags - -It is possible to add services to multiple machines via tags. The service instance gets added in the specified role. In this case `role = "default"` +It is possible to add services to multiple machines via tags. The service instance gets added in the specified role. In this case `role = "client"` !!! Example "Tags Example" - ```{.nix hl_lines="5 8 13"} + ```{.nix hl_lines="9 12 17"} buildClan { inventory = { machines = { - "sara" = { - tags = ["browsing"]; + "backup_server" = { + # Don't include any nixos config here + # See inventory.Machines for available options }; "jon" = { - tags = ["browsing"]; + tags = [ "backup" ]; + }; + "sara" = { + tags = [ "backup" ]; }; }; services = { - packages.set_1 = { - roles.default.tags = [ "browsing" ]; - config.packages = ["firefox"]; + borgbackup.instance_1 = { + roles.client.tags = [ "backup" ]; + roles.server.machines = [ "backup_server" ]; }; }; }; @@ -164,9 +153,8 @@ It is possible to add services to multiple machines via tags. The service instan ### Multiple Service Instances !!! danger "Important" - Not all modules support multiple instances yet. - -Some modules have support for adding multiple instances of the same service in different roles or configurations. + Not all modules implement support for multiple instances yet. + Multiple instance usage could create complexity, refer to each modules documentation, for intended usage. !!! Example @@ -194,9 +182,11 @@ Some modules have support for adding multiple instances of the same service in d } ``` -### Schema specification +### API specification -The complete schema specification can be retrieved via: +**The complete schema specification is available [here](../reference/nix-api/inventory.md)** + +Or it can build anytime via: ```sh nix build git+https://git.clan.lol/clan/clan-core#inventory-schema diff --git a/docs/site/reference/index.md b/docs/site/reference/index.md new file mode 100644 index 000000000..d9e1b00fd --- /dev/null +++ b/docs/site/reference/index.md @@ -0,0 +1,8 @@ +# Documentation + +This section of the site contains information about the following topics: + +- How to use the [Clan CLI](./cli/index.md) +- Available services and application [modules](./clanModules/index.md) +- [Configuration options](./clan-core/index.md) controlling the essential features +- Descriptions of the [Nix interfaces](./nix-api/index.md) for defining a Clan diff --git a/docs/site/reference/nix-api/buildclan.md b/docs/site/reference/nix-api/buildclan.md new file mode 100644 index 000000000..089e03f64 --- /dev/null +++ b/docs/site/reference/nix-api/buildclan.md @@ -0,0 +1,29 @@ +# buildClan + +The core [function](https://git.clan.lol/clan/clan-core/src/branch/main/lib/build-clan/default.nix) that produces a Clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services. + +## Inputs + +`directory` +: The directory containing the machines subdirectory + +`machines` +: Allows to include machine-specific modules i.e. machines.${name} = { ... } + +`meta` +: An optional set + +: `{ name :: string, icon :: string, description :: string }` + +`inventory` +: Service set for easily configuring distributed services, such as backups + +: For more details see [Inventory](./inventory.md) + +`specialArgs` +: Extra arguments to pass to nixosSystem i.e. useful to make self available + +`pkgsForSystem` +: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system. + This improves performance, but all nipxkgs.* options will be ignored. + `(string -> pkgs )` diff --git a/docs/site/reference/nix-api/index.md b/docs/site/reference/nix-api/index.md new file mode 100644 index 000000000..00e148b35 --- /dev/null +++ b/docs/site/reference/nix-api/index.md @@ -0,0 +1,6 @@ +# Nix API Overview + +There are two top-level components of the Nix API, which together allow for the declarative definition of a Clan: + +- the [Inventory](./inventory.md), a structure representing the machines, services, custom configurations, and other data that constitute a Clan, and +- the [`buildClan`](./buildclan.md) function, which constructs a Clan from an Inventory definition. diff --git a/docs/site/reference/nix-api/inventory.md b/docs/site/reference/nix-api/inventory.md new file mode 100644 index 000000000..75fa5e970 --- /dev/null +++ b/docs/site/reference/nix-api/inventory.md @@ -0,0 +1,185 @@ +# Inventory API + +*Inventory* is an abstract service layer for consistently configuring distributed services across machine boundaries. + +The following is a specification of the inventory in [cuelang](https://cuelang.org/) format. + +```cue +package compose + +#Root: { + @jsonschema(schema="http://json-schema.org/draft-07/schema#") + machines?: [string]: { + deploy?: { + // Configuration for the deployment of the machine + targetHost: null | string + } + description: null | string + icon: null | string + name: string + system: null | string + tags: [...string] + } + meta?: { + description: null | string + icon: null | string + name: string + } + services?: { + borgbackup?: [string]: { + // borgbackup-config + config?: { + // destinations where the machine should be backuped to + destinations?: { + [string]: { + // the name of the backup job + name: string + + // the borgbackup repository to backup to + repo: string + } + } + } | *{ + ... + } + machines?: [string]: { + // borgbackup-config + config?: { + // destinations where the machine should be backuped to + destinations?: { + [string]: { + // the name of the backup job + name: string + + // the borgbackup repository to backup to + repo: string + } + } + } | *{ + ... + } + imports: [...string] + } + meta?: { + description: null | string + icon: null | string + name: string + } + roles?: { + client?: { + // borgbackup-config + config?: { + // destinations where the machine should be backuped to + destinations?: { + [string]: { + // the name of the backup job + name: string + + // the borgbackup repository to backup to + repo: string + } + } + } | *{ + ... + } + imports: [...string] + machines: [...string] + tags: [...string] + } + server?: { + // borgbackup-config + config?: { + // destinations where the machine should be backuped to + destinations?: { + [string]: { + // the name of the backup job + name: string + + // the borgbackup repository to backup to + repo: string + } + } + } | *{ + ... + } + imports: [...string] + machines: [...string] + tags: [...string] + } + } + } + packages?: [string]: { + // packages-config + config?: { + // The packages to install on the machine + packages: [...string] + } | *{ + ... + } + machines?: [string]: { + // packages-config + config?: { + // The packages to install on the machine + packages: [...string] + } | *{ + ... + } + imports: [...string] + } + meta?: { + description: null | string + icon: null | string + name: string + } + roles?: default?: { + // packages-config + config?: { + // The packages to install on the machine + packages: [...string] + } | *{ + ... + } + imports: [...string] + machines: [...string] + tags: [...string] + } + } + "single-disk"?: [string]: { + // single-disk-config + config?: { + // The primary disk device to install the system on + device: null | string + } | *{ + ... + } + machines?: [string]: { + // single-disk-config + config?: { + // The primary disk device to install the system on + device: null | string + } | *{ + ... + } + imports: [...string] + } + meta?: { + description: null | string + icon: null | string + name: string + } + roles?: default?: { + // single-disk-config + config?: { + // The primary disk device to install the system on + device: null | string + } | *{ + ... + } + imports: [...string] + machines: [...string] + tags: [...string] + } + } + } +} +``` diff --git a/lib/inventory/flake-module.nix b/lib/inventory/flake-module.nix index 07ad18f5c..14394022a 100644 --- a/lib/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -46,6 +46,29 @@ in }; # Inventory schema with concrete module implementations + packages.inventory-api-docs = pkgs.stdenv.mkDerivation { + name = "inventory-schema"; + buildInputs = [ ]; + src = ./.; + buildPhase = '' + cat < "$out" + # Inventory API + + *Inventory* is an abstract service layer for consistently configuring distributed services across machine boundaries. + + The following is a specification of the inventory in [cuelang](https://cuelang.org/) format. + + \`\`\`cue + EOF + + cat ${self'.packages.inventory-schema-pretty}/schema.cue >> $out + + cat <> $out + \`\`\` + EOF + ''; + }; + packages.inventory-schema = pkgs.stdenv.mkDerivation { name = "inventory-schema"; buildInputs = [ pkgs.cue ];