From 35d8deb3939c5dc2d9d35f9114c22797623be95e Mon Sep 17 00:00:00 2001 From: clan-bot Date: Thu, 9 Oct 2025 10:01:53 +0000 Subject: [PATCH 01/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index b90758790..18261257f 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1759968599, - "narHash": "sha256-OdJ4OPAdvaIXZvwomVzjHWNTDdAX6++v4Ynjm2sXxBw=", + "lastModified": 1760000589, + "narHash": "sha256-9xBwxeb8x5XOo3alaJvv2ZwL7UhW3/oYUUBK+odWGrk=", "ref": "main", - "rev": "28d8a91a309985aa2b8586ff120365de6b0241b3", + "rev": "e2f20b5ffcd4ff59e2528d29649056e3eb8d22bb", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From 684aa27068d00d96706e8991b3f898297607a17e Mon Sep 17 00:00:00 2001 From: clan-bot Date: Thu, 9 Oct 2025 10:02:12 +0000 Subject: [PATCH 02/39] Update nixpkgs-dev in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index b90758790..30b12f3c4 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -105,11 +105,11 @@ }, "nixpkgs-dev": { "locked": { - "lastModified": 1759860509, - "narHash": "sha256-c7eJvqAlWLhwNc9raHkQ7mvoFbHLUO/cLMrww1ds4Zg=", + "lastModified": 1759989671, + "narHash": "sha256-3Wk0I5TYsd7cyIO8vYGxjOuQ8zraZEUFZqEhSSIhQLs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b574dcadf3fb578dee8d104b565bd745a5a9edc0", + "rev": "837076de579c67aa0c2ce2ab49948b24d907d449", "type": "github" }, "original": { From 6fa0062573c5876264ce562be2ce9bb3905f0286 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 9 Oct 2025 15:24:30 +0200 Subject: [PATCH 03/39] app: fix ClanSettings story --- .../ClanSettingsModal.stories.tsx | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/pkgs/clan-app/ui/src/modals/ClanSettingsModal/ClanSettingsModal.stories.tsx b/pkgs/clan-app/ui/src/modals/ClanSettingsModal/ClanSettingsModal.stories.tsx index f99007ac6..41c071c43 100644 --- a/pkgs/clan-app/ui/src/modals/ClanSettingsModal/ClanSettingsModal.stories.tsx +++ b/pkgs/clan-app/ui/src/modals/ClanSettingsModal/ClanSettingsModal.stories.tsx @@ -11,28 +11,35 @@ export default meta; type Story = StoryObj; -export const Default: Story = { - args: { - onClose: fn(), - model: { - uri: "/home/foo/my-clan", +const props: ClanSettingsModalProps = { + onClose: fn(), + model: { + uri: "/home/foo/my-clan", + details: { name: "Sol", description: null, icon: null, - fieldsSchema: { - name: { - readonly: true, - reason: null, - }, - description: { - readonly: false, - reason: null, - }, - icon: { - readonly: false, - reason: null, - }, + }, + fieldsSchema: { + name: { + readonly: true, + reason: null, + readonly_members: [], + }, + description: { + readonly: false, + reason: null, + readonly_members: [], + }, + icon: { + readonly: false, + reason: null, + readonly_members: [], }, }, }, }; + +export const Default: Story = { + args: props, +}; From 9cc85b36c6e943778d95f8f0520b8db99bf7bedf Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Mon, 29 Sep 2025 15:53:03 +0100 Subject: [PATCH 04/39] feat(ui): switch to webkit for storybook tests --- pkgs/clan-app/shell.nix | 18 +++++++++++++++--- pkgs/clan-app/ui/package-lock.json | 16 ++++++++-------- pkgs/clan-app/ui/package.json | 2 +- pkgs/clan-app/ui/tsconfig.json | 6 +++++- pkgs/clan-app/ui/vitest.config.ts | 9 ++++++++- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/pkgs/clan-app/shell.nix b/pkgs/clan-app/shell.nix index 5778d9e55..215d6cb97 100644 --- a/pkgs/clan-app/shell.nix +++ b/pkgs/clan-app/shell.nix @@ -113,15 +113,27 @@ mkShell { # todo darwin support needs some work (lib.optionalString stdenv.hostPlatform.isLinux '' # configure playwright for storybook snapshot testing - export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 + # we only want webkit as that matches what the app is rendered with + export PLAYWRIGHT_BROWSERS_PATH=${ playwright-driver.browsers.override { withFfmpeg = false; withFirefox = false; + withWebkit = true; withChromium = false; - withChromiumHeadlessShell = true; + withChromiumHeadlessShell = false; } } - export PLAYWRIGHT_HOST_PLATFORM_OVERRIDE="ubuntu-24.04" + + # stop playwright from trying to validate it has downloaded the necessary browsers + # we are providing them manually via nix + + export PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS=true + + # playwright browser drivers are versioned e.g. webkit-2191 + # this helps us avoid having to update the playwright js dependency everytime we update nixpkgs and vice versa + # see vitest.config.js for corresponding launch configuration + + export PLAYWRIGHT_WEBKIT_EXECUTABLE=$(find -L "$PLAYWRIGHT_BROWSERS_PATH" -type f -name "pw_run.sh") ''); } diff --git a/pkgs/clan-app/ui/package-lock.json b/pkgs/clan-app/ui/package-lock.json index e7076b902..0803bd39b 100644 --- a/pkgs/clan-app/ui/package-lock.json +++ b/pkgs/clan-app/ui/package-lock.json @@ -53,7 +53,7 @@ "jsdom": "^26.1.0", "knip": "^5.61.2", "markdown-to-jsx": "^7.7.10", - "playwright": "~1.53.2", + "playwright": "~1.55.1", "postcss": "^8.4.38", "postcss-url": "^10.1.3", "prettier": "^3.2.5", @@ -6956,13 +6956,13 @@ } }, "node_modules/playwright": { - "version": "1.53.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.53.2.tgz", - "integrity": "sha512-6K/qQxVFuVQhRQhFsVZ9fGeatxirtrpPgxzBYWyZLEXJzqYwuL4fuNmfOfD5et1tJE4GScKyPNeLhZeRwuTU3A==", + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.1.tgz", + "integrity": "sha512-cJW4Xd/G3v5ovXtJJ52MAOclqeac9S/aGGgRzLabuF8TnIb6xHvMzKIa6JmrRzUkeXJgfL1MhukP0NK6l39h3A==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.53.2" + "playwright-core": "1.55.1" }, "bin": { "playwright": "cli.js" @@ -6975,9 +6975,9 @@ } }, "node_modules/playwright-core": { - "version": "1.53.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.53.2.tgz", - "integrity": "sha512-ox/OytMy+2w1jcYEYlOo1Hhp8hZkLCximMTUTMBXjGUA1KoFfiSZ+DU+3a739jsPY0yoKH2TFy9S2fsJas8yAw==", + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.1.tgz", + "integrity": "sha512-Z6Mh9mkwX+zxSlHqdr5AOcJnfp+xUWLCt9uKV18fhzA8eyxUd8NUWzAjxUh55RZKSYwDGX0cfaySdhZJGMoJ+w==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/pkgs/clan-app/ui/package.json b/pkgs/clan-app/ui/package.json index a1e37647d..36e5a7851 100644 --- a/pkgs/clan-app/ui/package.json +++ b/pkgs/clan-app/ui/package.json @@ -48,7 +48,7 @@ "jsdom": "^26.1.0", "knip": "^5.61.2", "markdown-to-jsx": "^7.7.10", - "playwright": "~1.53.2", + "playwright": "~1.55.1", "postcss": "^8.4.38", "postcss-url": "^10.1.3", "prettier": "^3.2.5", diff --git a/pkgs/clan-app/ui/tsconfig.json b/pkgs/clan-app/ui/tsconfig.json index 3544efda6..5e4b93ca3 100644 --- a/pkgs/clan-app/ui/tsconfig.json +++ b/pkgs/clan-app/ui/tsconfig.json @@ -9,7 +9,11 @@ "esModuleInterop": true, "jsx": "preserve", "jsxImportSource": "solid-js", - "types": ["vite/client", "vite-plugin-solid-svg/types-component-solid"], + "types": [ + "vite/client", + "vite-plugin-solid-svg/types-component-solid", + "@vitest/browser/providers/playwright" + ], "noEmit": true, "resolveJsonModule": true, "allowJs": true, diff --git a/pkgs/clan-app/ui/vitest.config.ts b/pkgs/clan-app/ui/vitest.config.ts index 30ccb0860..ea912c4a6 100644 --- a/pkgs/clan-app/ui/vitest.config.ts +++ b/pkgs/clan-app/ui/vitest.config.ts @@ -40,7 +40,14 @@ export default mergeConfig( enabled: true, headless: true, provider: "playwright", - instances: [{ browser: "chromium" }], + instances: [ + { + browser: "webkit", + launch: { + executablePath: process.env.PLAYWRIGHT_WEBKIT_EXECUTABLE, + }, + }, + ], }, // This setup file applies Storybook project annotations for Vitest // More info at: https://storybook.js.org/docs/api/portable-stories/portable-stories-vitest#setprojectannotations From feef0a513eb93c7b8ef393c74bd2ecce7aa7c8fd Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Tue, 30 Sep 2025 11:48:12 +0100 Subject: [PATCH 05/39] fix(storybook): remove cubes storybook It wasn't adding much value and requires a mock Clan context which is a lot of effort at the min. --- pkgs/clan-app/ui/src/scene/cubes.stories.tsx | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 pkgs/clan-app/ui/src/scene/cubes.stories.tsx diff --git a/pkgs/clan-app/ui/src/scene/cubes.stories.tsx b/pkgs/clan-app/ui/src/scene/cubes.stories.tsx deleted file mode 100644 index 79c53375e..000000000 --- a/pkgs/clan-app/ui/src/scene/cubes.stories.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import { Meta, StoryObj } from "@kachurun/storybook-solid"; -import { CubeScene } from "./cubes"; - -const meta: Meta = { - title: "scene/cubes", - component: CubeScene, -}; - -export default meta; - -type Story = StoryObj; - -export const Default: Story = { - args: {}, -}; From 2814c46e68c62b1d0fd4411b121ea16382080fd9 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Tue, 30 Sep 2025 11:49:59 +0100 Subject: [PATCH 06/39] fix(storybook): button stories - role="button" was removed at some point during refactoring which broke how the story was finding buttons - button no longer has automatic loading state, instead it is now controlled. --- .../src/components/Button/Button.stories.tsx | 51 +++---------------- .../ui/src/components/Button/Button.tsx | 1 + 2 files changed, 7 insertions(+), 45 deletions(-) diff --git a/pkgs/clan-app/ui/src/components/Button/Button.stories.tsx b/pkgs/clan-app/ui/src/components/Button/Button.stories.tsx index eba98b1c1..a16f03467 100644 --- a/pkgs/clan-app/ui/src/components/Button/Button.stories.tsx +++ b/pkgs/clan-app/ui/src/components/Button/Button.stories.tsx @@ -1,7 +1,7 @@ import type { Meta, StoryObj } from "@kachurun/storybook-solid"; import { Button, ButtonProps } from "./Button"; import { Component } from "solid-js"; -import { expect, fn, waitFor } from "storybook/test"; +import { expect, fn, waitFor, within } from "storybook/test"; import { StoryContext } from "@kachurun/storybook-solid-vite"; const getCursorStyle = (el: Element) => window.getComputedStyle(el).cursor; @@ -216,17 +216,11 @@ const timeout = process.env.NODE_ENV === "test" ? 500 : 2000; export const Primary: Story = { args: { hierarchy: "primary", - onAction: fn(async () => { - // wait 500 ms to simulate an action - await new Promise((resolve) => setTimeout(resolve, timeout)); - // randomly fail to check that the loading state still returns to normal - if (Math.random() > 0.5) { - throw new Error("Action failure"); - } - }), + onClick: fn(), }, - play: async ({ canvas, step, userEvent, args }: StoryContext) => { + play: async ({ canvasElement, step, userEvent, args }: StoryContext) => { + const canvas = within(canvasElement); const buttons = await canvas.findAllByRole("button"); for (const button of buttons) { @@ -238,14 +232,6 @@ export const Primary: Story = { } await step(`Click on ${testID}`, async () => { - // check for the loader - const loaders = button.getElementsByClassName("loader"); - await expect(loaders.length).toEqual(1); - - // assert its width is 0 before we click - const [loader] = loaders; - await expect(loader.clientWidth).toEqual(0); - // move the mouse over the button await userEvent.hover(button); @@ -255,33 +241,8 @@ export const Primary: Story = { // click the button await userEvent.click(button); - // check the button has changed - await waitFor( - async () => { - // the action handler should have been called - await expect(args.onAction).toHaveBeenCalled(); - // the button should have a loading class - await expect(button).toHaveClass("loading"); - // the loader should be visible - await expect(loader.clientWidth).toBeGreaterThan(0); - // the pointer should have changed to wait - await expect(getCursorStyle(button)).toEqual("wait"); - }, - { timeout: timeout + 500 }, - ); - - // wait for the action handler to finish - await waitFor( - async () => { - // the loading class should be removed - await expect(button).not.toHaveClass("loading"); - // the loader should be hidden - await expect(loader.clientWidth).toEqual(0); - // the pointer should be normal - await expect(getCursorStyle(button)).toEqual("pointer"); - }, - { timeout: timeout + 500 }, - ); + // the click handler should have been called + await expect(args.onClick).toHaveBeenCalled(); }); } }, diff --git a/pkgs/clan-app/ui/src/components/Button/Button.tsx b/pkgs/clan-app/ui/src/components/Button/Button.tsx index b9149b3b4..ea8f7b4c1 100644 --- a/pkgs/clan-app/ui/src/components/Button/Button.tsx +++ b/pkgs/clan-app/ui/src/components/Button/Button.tsx @@ -57,6 +57,7 @@ export const Button = (props: ButtonProps) => { return ( Date: Tue, 30 Sep 2025 12:01:37 +0100 Subject: [PATCH 07/39] fix(storybook): clan settings mock data shape changed --- .../ui/src/modals/ClanSettingsModal/ClanSettingsModal.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-app/ui/src/modals/ClanSettingsModal/ClanSettingsModal.tsx b/pkgs/clan-app/ui/src/modals/ClanSettingsModal/ClanSettingsModal.tsx index 6bf98b03f..e8bab2cfc 100644 --- a/pkgs/clan-app/ui/src/modals/ClanSettingsModal/ClanSettingsModal.tsx +++ b/pkgs/clan-app/ui/src/modals/ClanSettingsModal/ClanSettingsModal.tsx @@ -22,9 +22,9 @@ import { Alert } from "@/src/components/Alert/Alert"; import { removeClanURI } from "@/src/stores/clan"; const schema = v.object({ - name: v.pipe(v.optional(v.string())), - description: v.nullish(v.string()), - icon: v.pipe(v.nullish(v.string())), + name: v.string(), + description: v.optional(v.string()), + icon: v.optional(v.string()), }); export interface ClanSettingsModalProps { From 1df329fe0df6e9e3ebb5908c402352418fa432e5 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Tue, 30 Sep 2025 12:13:35 +0100 Subject: [PATCH 08/39] fix(storybook): disable service workflow stories Temporary until we can decide how best to mock state. --- .../src/workflows/Service/Service.stories.tsx | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pkgs/clan-app/ui/src/workflows/Service/Service.stories.tsx b/pkgs/clan-app/ui/src/workflows/Service/Service.stories.tsx index c38a72681..f69ab9576 100644 --- a/pkgs/clan-app/ui/src/workflows/Service/Service.stories.tsx +++ b/pkgs/clan-app/ui/src/workflows/Service/Service.stories.tsx @@ -165,23 +165,23 @@ export default meta; type Story = StoryObj; -export const Default: Story = { - args: {}, -}; - -export const SelectRoleMembers: Story = { - render: () => ( - { - console.log("Submitted instance:", instance); - }} - onClose={() => { - console.log("Closed"); - }} - initialStep="select:members" - initialStore={{ - currentRole: "peer", - }} - /> - ), -}; +// export const Default: Story = { +// args: {}, +// }; +// +// export const SelectRoleMembers: Story = { +// render: () => ( +// { +// console.log("Submitted instance:", instance); +// }} +// onClose={() => { +// console.log("Closed"); +// }} +// initialStep="select:members" +// initialStore={{ +// currentRole: "peer", +// }} +// /> +// ), +// }; From 9626e22db7917fb47b1e1c9179bae819b36a3a16 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Tue, 30 Sep 2025 12:36:43 +0100 Subject: [PATCH 09/39] fix(storybook): adjust flash installer on mount It needs to handle possible missing state in the store on mount. --- .../src/workflows/InstallMachine/steps/createInstaller.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkgs/clan-app/ui/src/workflows/InstallMachine/steps/createInstaller.tsx b/pkgs/clan-app/ui/src/workflows/InstallMachine/steps/createInstaller.tsx index bbfc8c102..88971563e 100644 --- a/pkgs/clan-app/ui/src/workflows/InstallMachine/steps/createInstaller.tsx +++ b/pkgs/clan-app/ui/src/workflows/InstallMachine/steps/createInstaller.tsx @@ -304,11 +304,10 @@ const FlashProgress = () => { const [store, set] = getStepStore(stepSignal); onMount(async () => { - const result = await store.flash.progress.result; - if (result.status == "success") { - console.log("Flashing Success"); + const result = await store.flash?.progress?.result; + if (result?.status == "success") { + stepSignal.next(); } - stepSignal.next(); }); const handleCancel = async () => { From 27d43ee21dc1d70c64b448373a600a2fdee27a01 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Tue, 30 Sep 2025 12:39:49 +0100 Subject: [PATCH 10/39] fix(storybook): disable Sidebar story until we have a better mock data approach --- .../components/Sidebar/Sidebar.stories.tsx | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/pkgs/clan-app/ui/src/components/Sidebar/Sidebar.stories.tsx b/pkgs/clan-app/ui/src/components/Sidebar/Sidebar.stories.tsx index 039f27df0..568ef3594 100644 --- a/pkgs/clan-app/ui/src/components/Sidebar/Sidebar.stories.tsx +++ b/pkgs/clan-app/ui/src/components/Sidebar/Sidebar.stories.tsx @@ -160,47 +160,47 @@ const mockFetcher = ( }, }) satisfies ApiCall; -export const Default: Story = { - args: {}, - decorators: [ - (Story: StoryObj) => { - const queryClient = new QueryClient({ - defaultOptions: { - queries: { - retry: false, - staleTime: Infinity, - }, - }, - }); - - Object.entries(queryData).forEach(([clanURI, clan]) => { - queryClient.setQueryData( - ["clans", encodeBase64(clanURI), "details"], - clan.details, - ); - - const machines = clan.machines || {}; - - queryClient.setQueryData( - ["clans", encodeBase64(clanURI), "machines"], - machines, - ); - - Object.entries(machines).forEach(([name, machine]) => { - queryClient.setQueryData( - ["clans", encodeBase64(clanURI), "machine", name, "state"], - machine.state, - ); - }); - }); - - return ( - - - - - - ); - }, - ], -}; +// export const Default: Story = { +// args: {}, +// decorators: [ +// (Story: StoryObj) => { +// const queryClient = new QueryClient({ +// defaultOptions: { +// queries: { +// retry: false, +// staleTime: Infinity, +// }, +// }, +// }); +// +// Object.entries(queryData).forEach(([clanURI, clan]) => { +// queryClient.setQueryData( +// ["clans", encodeBase64(clanURI), "details"], +// clan.details, +// ); +// +// const machines = clan.machines || {}; +// +// queryClient.setQueryData( +// ["clans", encodeBase64(clanURI), "machines"], +// machines, +// ); +// +// Object.entries(machines).forEach(([name, machine]) => { +// queryClient.setQueryData( +// ["clans", encodeBase64(clanURI), "machine", name, "state"], +// machine.state, +// ); +// }); +// }); +// +// return ( +// +// +// +// +// +// ); +// }, +// ], +// }; From 50a26ece3213af4d2017a9821a5724fb05254bff Mon Sep 17 00:00:00 2001 From: clan-bot Date: Thu, 9 Oct 2025 15:01:53 +0000 Subject: [PATCH 11/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 534251681..6c1a98aa9 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760000589, - "narHash": "sha256-9xBwxeb8x5XOo3alaJvv2ZwL7UhW3/oYUUBK+odWGrk=", + "lastModified": 1760019726, + "narHash": "sha256-tn1QNTOKCmz0p78X87EBNONF7EmNjZdIVvB8OpSXYjM=", "ref": "main", - "rev": "e2f20b5ffcd4ff59e2528d29649056e3eb8d22bb", + "rev": "8f224b00a63ad6a318070e2c0685e0ed3a6fefea", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From d829aa583896d032f36f3e34ae6de7510dae9dfa Mon Sep 17 00:00:00 2001 From: clan-bot Date: Thu, 9 Oct 2025 20:01:50 +0000 Subject: [PATCH 12/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 6c1a98aa9..3c41b31c4 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760019726, - "narHash": "sha256-tn1QNTOKCmz0p78X87EBNONF7EmNjZdIVvB8OpSXYjM=", + "lastModified": 1760022577, + "narHash": "sha256-gIqmVekNsfc7pPF4A/0lOSvo2Pjl0A/WU3NjsHKDFik=", "ref": "main", - "rev": "8f224b00a63ad6a318070e2c0685e0ed3a6fefea", + "rev": "fd6619668bd358abdb32695f1a2a7351041467f7", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From dd1cab5daa30efde94e4f881c262a4c712251883 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Fri, 10 Oct 2025 00:01:51 +0000 Subject: [PATCH 13/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 3c41b31c4..9c512234f 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760022577, - "narHash": "sha256-gIqmVekNsfc7pPF4A/0lOSvo2Pjl0A/WU3NjsHKDFik=", + "lastModified": 1760040583, + "narHash": "sha256-i63BhjXoAz8HKxGGKVe/IyTPEzw5dSSnTothNUK+Pfo=", "ref": "main", - "rev": "fd6619668bd358abdb32695f1a2a7351041467f7", + "rev": "32edae4ebd20f9460d6ac587c56217cdaaab3951", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From f224d4b20c18f9e781179c2e244911e9fc9067dc Mon Sep 17 00:00:00 2001 From: clan-bot Date: Fri, 10 Oct 2025 05:01:54 +0000 Subject: [PATCH 14/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 9c512234f..6a18311a1 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760040583, - "narHash": "sha256-i63BhjXoAz8HKxGGKVe/IyTPEzw5dSSnTothNUK+Pfo=", + "lastModified": 1760055069, + "narHash": "sha256-sPC5gV955CzdFnpu4j33iOVeIdSMfZXHHjUY6a04XRs=", "ref": "main", - "rev": "32edae4ebd20f9460d6ac587c56217cdaaab3951", + "rev": "47aa0a3b8e528ae08fd289edb50e6871c469f0b8", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From 749bac63f457fd27a3f8cfcc00fa434ddb99f2e9 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Fri, 10 Oct 2025 10:01:53 +0000 Subject: [PATCH 15/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 6a18311a1..85a373202 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760055069, - "narHash": "sha256-sPC5gV955CzdFnpu4j33iOVeIdSMfZXHHjUY6a04XRs=", + "lastModified": 1760072968, + "narHash": "sha256-cIyA2H2SzNMfYV85Ph1rouV7WU4p4qU8i61Akj8U/0w=", "ref": "main", - "rev": "47aa0a3b8e528ae08fd289edb50e6871c469f0b8", + "rev": "2bac2ec7ee23d530577fdc5ebec1d59dfd792e17", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From f93ae13448b5ed2b66ad53530bd3bc09d10bd7a6 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Fri, 10 Oct 2025 10:02:12 +0000 Subject: [PATCH 16/39] Update nixpkgs-dev in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 6a18311a1..2692239f8 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -105,11 +105,11 @@ }, "nixpkgs-dev": { "locked": { - "lastModified": 1759989671, - "narHash": "sha256-3Wk0I5TYsd7cyIO8vYGxjOuQ8zraZEUFZqEhSSIhQLs=", + "lastModified": 1760075832, + "narHash": "sha256-yFPaAoVMivTE2cpNkVl6jrkNd+mIq1cae/4D0pN14XQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "837076de579c67aa0c2ce2ab49948b24d907d449", + "rev": "daebeba791763abfe3cce5e0f16376ddf1b724d4", "type": "github" }, "original": { From 9a664c323c00a8b753be33803f1f01073fded2e3 Mon Sep 17 00:00:00 2001 From: a-kenji Date: Fri, 10 Oct 2025 13:35:29 +0200 Subject: [PATCH 17/39] docs: Remove surprising statement on the front of documentation --- docs/site/index.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/site/index.md b/docs/site/index.md index aaeccea7b..4a941cdc4 100644 --- a/docs/site/index.md +++ b/docs/site/index.md @@ -70,8 +70,6 @@ hide: .clamp-toggle:checked ~ .clamp-more::after { content: "Read less"; } -trivial change -
From 6014ddcd9a288c9d38a2edc8e286f1de885aff8a Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Mon, 6 Oct 2025 17:04:31 +0200 Subject: [PATCH 18/39] checks: fix SSH debugging over `vsock` not working --- checks/installation/test-helpers.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/checks/installation/test-helpers.nix b/checks/installation/test-helpers.nix index 202a79328..fdd53c79a 100644 --- a/checks/installation/test-helpers.nix +++ b/checks/installation/test-helpers.nix @@ -15,7 +15,6 @@ let networking.useNetworkd = true; services.openssh.enable = true; services.openssh.settings.UseDns = false; - services.openssh.settings.PasswordAuthentication = false; system.nixos.variant_id = "installer"; environment.systemPackages = [ pkgs.nixos-facter From bc09d5c886754bba6f2213bc9d7a0055b9b515cc Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Fri, 10 Oct 2025 14:07:03 +0200 Subject: [PATCH 19/39] cli: ensure `init-hardware-config` passes Nix options to nixos-anywhere --- pkgs/clan-cli/clan_lib/machines/hardware.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/clan-cli/clan_lib/machines/hardware.py b/pkgs/clan-cli/clan_lib/machines/hardware.py index 5e574b5f9..089627438 100644 --- a/pkgs/clan-cli/clan_lib/machines/hardware.py +++ b/pkgs/clan-cli/clan_lib/machines/hardware.py @@ -119,6 +119,9 @@ def run_machine_hardware_info_init( if opts.debug: cmd += ["--debug"] + # Add nix options to nixos-anywhere + cmd.extend(opts.machine.flake.nix_options or []) + cmd += [target_host.target] cmd = nix_shell( ["nixos-anywhere"], From 0a3e564ec0495da18ceaa942b548a19834148a9a Mon Sep 17 00:00:00 2001 From: clan-bot Date: Fri, 10 Oct 2025 15:01:52 +0000 Subject: [PATCH 20/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 3bfe928e3..44dd9b5de 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760072968, - "narHash": "sha256-cIyA2H2SzNMfYV85Ph1rouV7WU4p4qU8i61Akj8U/0w=", + "lastModified": 1760107210, + "narHash": "sha256-kypCMk6s4AGDkq9l8dg9a8NDdW7kPtyEOSAOnFx7mBY=", "ref": "main", - "rev": "2bac2ec7ee23d530577fdc5ebec1d59dfd792e17", + "rev": "f6b8d660d8b8ef08f11b232ddcfb7943fe8ff5a5", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From 1fbb4f5014e86608bd76d7f74e897f428c9d10a4 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Fri, 10 Oct 2025 20:01:49 +0000 Subject: [PATCH 21/39] Update treefmt-nix --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 2914f1f6c..d07756c15 100644 --- a/flake.lock +++ b/flake.lock @@ -181,11 +181,11 @@ ] }, "locked": { - "lastModified": 1758728421, - "narHash": "sha256-ySNJ008muQAds2JemiyrWYbwbG+V7S5wg3ZVKGHSFu8=", + "lastModified": 1760120816, + "narHash": "sha256-gq9rdocpmRZCwLS5vsHozwB6b5nrOBDNc2kkEaTXHfg=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "5eda4ee8121f97b218f7cc73f5172098d458f1d1", + "rev": "761ae7aff00907b607125b2f57338b74177697ed", "type": "github" }, "original": { From f15a113f52919deeeab2ab4a3c5b8e6b0698968e Mon Sep 17 00:00:00 2001 From: clan-bot Date: Fri, 10 Oct 2025 20:01:50 +0000 Subject: [PATCH 22/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 44dd9b5de..e1c7f5881 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760107210, - "narHash": "sha256-kypCMk6s4AGDkq9l8dg9a8NDdW7kPtyEOSAOnFx7mBY=", + "lastModified": 1760110834, + "narHash": "sha256-HCD5JTa1htWZ1L1eqI3pyFeY+0w4iczdEKmvtfjIxBE=", "ref": "main", - "rev": "f6b8d660d8b8ef08f11b232ddcfb7943fe8ff5a5", + "rev": "980a3c90b53c0210f1b3a392f20bfcd31c196fa3", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From 0e0f8e73ec6752bf74d647f2f254c88cc4ddfca1 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Fri, 10 Oct 2025 20:02:13 +0000 Subject: [PATCH 23/39] Update treefmt-nix in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 44dd9b5de..019076d42 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -208,11 +208,11 @@ "nixpkgs": [] }, "locked": { - "lastModified": 1758728421, - "narHash": "sha256-ySNJ008muQAds2JemiyrWYbwbG+V7S5wg3ZVKGHSFu8=", + "lastModified": 1760120816, + "narHash": "sha256-gq9rdocpmRZCwLS5vsHozwB6b5nrOBDNc2kkEaTXHfg=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "5eda4ee8121f97b218f7cc73f5172098d458f1d1", + "rev": "761ae7aff00907b607125b2f57338b74177697ed", "type": "github" }, "original": { From 7d3aa5936d3406712f90ddc11167655daeda7e81 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Sat, 11 Oct 2025 00:01:51 +0000 Subject: [PATCH 24/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index d0d2752ce..99d516126 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760110834, - "narHash": "sha256-HCD5JTa1htWZ1L1eqI3pyFeY+0w4iczdEKmvtfjIxBE=", + "lastModified": 1760127134, + "narHash": "sha256-9E4Ukwl7HdoWVfIk5CS3JLBdgMwn/3una6byrrWhYnY=", "ref": "main", - "rev": "980a3c90b53c0210f1b3a392f20bfcd31c196fa3", + "rev": "f8f8efbb88de690effaee6fcb2e5d4bbdbb82d3e", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From 652677d06ffa2602049756e80d87baa166a06fd9 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Sat, 11 Oct 2025 05:01:53 +0000 Subject: [PATCH 25/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 99d516126..31b009db8 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760127134, - "narHash": "sha256-9E4Ukwl7HdoWVfIk5CS3JLBdgMwn/3una6byrrWhYnY=", + "lastModified": 1760141373, + "narHash": "sha256-7++RIlzB4FWu0+DZpzJzD8a3qxVeVPk8tXoKQluAwpA=", "ref": "main", - "rev": "f8f8efbb88de690effaee6fcb2e5d4bbdbb82d3e", + "rev": "ec163657cd307e50a71f8c7f2250aa254850fbc8", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From 869a04e5af068a848c930fac03d8be53765b66d5 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Sat, 11 Oct 2025 10:01:50 +0000 Subject: [PATCH 26/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 31b009db8..2a4d81e72 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760141373, - "narHash": "sha256-7++RIlzB4FWu0+DZpzJzD8a3qxVeVPk8tXoKQluAwpA=", + "lastModified": 1760159356, + "narHash": "sha256-HRJ+xTXdiXdDMNpWTiYbbjSJJqnsUB5Sz11C4Il+nnY=", "ref": "main", - "rev": "ec163657cd307e50a71f8c7f2250aa254850fbc8", + "rev": "d09fdc35287a37aca7893ca4799b10ff1f9ae22d", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From f33c8e98fe34908d823eba4d0836e59e6835202b Mon Sep 17 00:00:00 2001 From: clan-bot Date: Sat, 11 Oct 2025 10:02:05 +0000 Subject: [PATCH 27/39] Update nixpkgs-dev in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index 31b009db8..08cd7b8d8 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -105,11 +105,11 @@ }, "nixpkgs-dev": { "locked": { - "lastModified": 1760075832, - "narHash": "sha256-yFPaAoVMivTE2cpNkVl6jrkNd+mIq1cae/4D0pN14XQ=", + "lastModified": 1760161054, + "narHash": "sha256-PO3cKHFIQEPI0dr/SzcZwG50cHXfjoIqP2uS5W78OXg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "daebeba791763abfe3cce5e0f16376ddf1b724d4", + "rev": "e18d8ec6fafaed55561b7a1b54eb1c1ce3ffa2c5", "type": "github" }, "original": { From 84d4660a8d9955c94298393eba54a892bd4d3715 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 11 Oct 2025 15:57:42 +0200 Subject: [PATCH 28/39] inventory: wrap autoloaded machines with correct file --- lib/modules/clan/module.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/modules/clan/module.nix b/lib/modules/clan/module.nix index 866233b3b..ed738b116 100644 --- a/lib/modules/clan/module.nix +++ b/lib/modules/clan/module.nix @@ -134,11 +134,12 @@ in ) { # TODO: Figure out why this causes infinite recursion - inventory.machines = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( - builtins.mapAttrs (_n: _v: { }) ( - lib.filterAttrs (_: t: t == "directory") (builtins.readDir "${directory}/machines") - ) - ); + inventory = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ({ + imports = lib.mapAttrsToList (name: _t: { + _file = "${directory}/machines/${name}"; + machines.${name} = { }; + }) ((lib.filterAttrs (_: t: t == "directory") (builtins.readDir "${directory}/machines"))); + }); } { inventory.machines = lib.mapAttrs (_n: _: { }) config.machines; From 18e3c72ef0f3a237c1248fede1617338f7542a5b Mon Sep 17 00:00:00 2001 From: clan-bot Date: Sat, 11 Oct 2025 15:01:51 +0000 Subject: [PATCH 29/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index e8c3cc8d0..c28ce45d2 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760159356, - "narHash": "sha256-HRJ+xTXdiXdDMNpWTiYbbjSJJqnsUB5Sz11C4Il+nnY=", + "lastModified": 1760177669, + "narHash": "sha256-ZoFH4wxQp5daL7DAbifgsx+FyFRR5jeX3ra1L+oA+CQ=", "ref": "main", - "rev": "d09fdc35287a37aca7893ca4799b10ff1f9ae22d", + "rev": "13c3e1411a4717fb753af5a5d4c03a5b53f31a52", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From 7f49449f94d6642141ee5e26269b13f00b578912 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 11 Oct 2025 18:02:32 +0200 Subject: [PATCH 30/39] clan-core/nixos: remove autoloading magic in favour of simple code --- checks/flake-module.nix | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/checks/flake-module.nix b/checks/flake-module.nix index c4af4f3d1..9edb8dc72 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -20,19 +20,7 @@ let in { imports = - let - clanCoreModulesDir = ../nixosModules/clanCore; - getClanCoreTestModules = - let - moduleNames = attrNames (builtins.readDir clanCoreModulesDir); - testPaths = map ( - moduleName: clanCoreModulesDir + "/${moduleName}/tests/flake-module.nix" - ) moduleNames; - in - filter pathExists testPaths; - in - getClanCoreTestModules - ++ filter pathExists [ + filter pathExists [ ./devshell/flake-module.nix ./flash/flake-module.nix ./installation/flake-module.nix @@ -40,6 +28,10 @@ in ./morph/flake-module.nix ./nixos-documentation/flake-module.nix ./dont-depend-on-repo-root.nix + # clan core submodule tests + ../nixosModules/clanCore/machine-id/tests/flake-module.nix + ../nixosModules/clanCore/postgresql/tests/flake-module.nix + ../nixosModules/clanCore/state-version/tests/flake-module.nix ]; flake.check = genAttrs [ "x86_64-linux" "aarch64-darwin" ] ( system: From 2be6cedec4e0214d593a0bc9c21bd861cda6e720 Mon Sep 17 00:00:00 2001 From: clan-bot Date: Sat, 11 Oct 2025 20:01:49 +0000 Subject: [PATCH 31/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index c28ce45d2..ead303ad5 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760177669, - "narHash": "sha256-ZoFH4wxQp5daL7DAbifgsx+FyFRR5jeX3ra1L+oA+CQ=", + "lastModified": 1760198437, + "narHash": "sha256-c+c2QHcoYMav9P1aNC62QpaseSdDyiZkCrLoXUxGaCA=", "ref": "main", - "rev": "13c3e1411a4717fb753af5a5d4c03a5b53f31a52", + "rev": "1f7bfa4e34e4355844ae4a0ca91dd5ef3cf3cdeb", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From b74b6ff4496d00d0bb93c02ef51d928c6b26c35f Mon Sep 17 00:00:00 2001 From: clan-bot Date: Sun, 12 Oct 2025 00:01:53 +0000 Subject: [PATCH 32/39] Update clan-core-for-checks in devFlake --- devFlake/flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devFlake/flake.lock b/devFlake/flake.lock index ead303ad5..02ef74978 100644 --- a/devFlake/flake.lock +++ b/devFlake/flake.lock @@ -3,10 +3,10 @@ "clan-core-for-checks": { "flake": false, "locked": { - "lastModified": 1760198437, - "narHash": "sha256-c+c2QHcoYMav9P1aNC62QpaseSdDyiZkCrLoXUxGaCA=", + "lastModified": 1760213549, + "narHash": "sha256-XosVRUEcdsoEdRtXyz9HrRc4Dt9Ke+viM5OVF7tLK50=", "ref": "main", - "rev": "1f7bfa4e34e4355844ae4a0ca91dd5ef3cf3cdeb", + "rev": "9c8797e77031d8d472d057894f18a53bdc9bbe1e", "shallow": true, "type": "git", "url": "https://git.clan.lol/clan/clan-core" From 1d38ffa9c2cfe8d4c9baf59cc518e1faf6dd2038 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 12 Oct 2025 16:07:08 +0200 Subject: [PATCH 33/39] inventory: unit test autoloading with a virtual fs --- checks/flake-module.nix | 27 +++++---- lib/clanTest/virtual-fs.nix | 51 +++++++++++++++++ lib/default.nix | 4 ++ lib/modules/clan/module.nix | 6 +- lib/modules/dir_test.nix | 108 ++++++++++++++++++++++++++++++++++++ lib/modules/tests.nix | 1 + 6 files changed, 180 insertions(+), 17 deletions(-) create mode 100644 lib/clanTest/virtual-fs.nix create mode 100644 lib/modules/dir_test.nix diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 9edb8dc72..5c895c856 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -19,20 +19,19 @@ let nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { }; in { - imports = - filter pathExists [ - ./devshell/flake-module.nix - ./flash/flake-module.nix - ./installation/flake-module.nix - ./update/flake-module.nix - ./morph/flake-module.nix - ./nixos-documentation/flake-module.nix - ./dont-depend-on-repo-root.nix - # clan core submodule tests - ../nixosModules/clanCore/machine-id/tests/flake-module.nix - ../nixosModules/clanCore/postgresql/tests/flake-module.nix - ../nixosModules/clanCore/state-version/tests/flake-module.nix - ]; + imports = filter pathExists [ + ./devshell/flake-module.nix + ./flash/flake-module.nix + ./installation/flake-module.nix + ./update/flake-module.nix + ./morph/flake-module.nix + ./nixos-documentation/flake-module.nix + ./dont-depend-on-repo-root.nix + # clan core submodule tests + ../nixosModules/clanCore/machine-id/tests/flake-module.nix + ../nixosModules/clanCore/postgresql/tests/flake-module.nix + ../nixosModules/clanCore/state-version/tests/flake-module.nix + ]; flake.check = genAttrs [ "x86_64-linux" "aarch64-darwin" ] ( system: let diff --git a/lib/clanTest/virtual-fs.nix b/lib/clanTest/virtual-fs.nix new file mode 100644 index 000000000..385463002 --- /dev/null +++ b/lib/clanTest/virtual-fs.nix @@ -0,0 +1,51 @@ +{ lib }: +let + sanitizePath = + rootPath: path: + let + storePrefix = builtins.unsafeDiscardStringContext ("${rootPath}"); + pathStr = lib.removePrefix "/" ( + lib.removePrefix storePrefix (builtins.unsafeDiscardStringContext (toString path)) + ); + in + pathStr; + + mkFunctions = rootPath: passthru: virtual_fs: { + # Some functions to override lib functions + pathExists = + path: + let + pathStr = sanitizePath rootPath path; + isPassthru = builtins.any (exclude: (builtins.match exclude pathStr) != null) passthru; + in + if isPassthru then + builtins.pathExists path + else + let + res = virtual_fs ? ${pathStr}; + in + lib.trace "pathExists: '${pathStr}' -> '${lib.generators.toPretty { } res}'" res; + readDir = + path: + let + pathStr = sanitizePath rootPath path; + base = (pathStr + "/"); + res = lib.mapAttrs' (name: fileInfo: { + name = lib.removePrefix base name; + value = fileInfo.type; + }) (lib.filterAttrs (n: _: lib.hasPrefix base n) virtual_fs); + isPassthru = builtins.any (exclude: (builtins.match exclude pathStr) != null) passthru; + in + if isPassthru then + builtins.readDir path + else + lib.trace "readDir: '${pathStr}' -> '${lib.generators.toPretty { } res}'" res; + }; +in +{ + virtual_fs, + rootPath, + # Patterns + passthru ? [ ], +}: +mkFunctions rootPath passthru virtual_fs diff --git a/lib/default.nix b/lib/default.nix index e77897acb..f9734ed0f 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -36,6 +36,10 @@ lib.fix ( # TODO: Flatten our lib functions like this: resolveModule = clanLib.callLib ./resolve-module { }; + + fs = { + inherit (builtins) pathExists readDir; + }; }; in f diff --git a/lib/modules/clan/module.nix b/lib/modules/clan/module.nix index ed738b116..66d762e87 100644 --- a/lib/modules/clan/module.nix +++ b/lib/modules/clan/module.nix @@ -133,12 +133,12 @@ in } ) { - # TODO: Figure out why this causes infinite recursion - inventory = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ({ + # Note: we use clanLib.fs here, so that we can override it in tests + inventory = lib.optionalAttrs (clanLib.fs.pathExists "${directory}/machines") ({ imports = lib.mapAttrsToList (name: _t: { _file = "${directory}/machines/${name}"; machines.${name} = { }; - }) ((lib.filterAttrs (_: t: t == "directory") (builtins.readDir "${directory}/machines"))); + }) ((lib.filterAttrs (_: t: t == "directory") (clanLib.fs.readDir "${directory}/machines"))); }); } { diff --git a/lib/modules/dir_test.nix b/lib/modules/dir_test.nix new file mode 100644 index 000000000..26ac302f1 --- /dev/null +++ b/lib/modules/dir_test.nix @@ -0,0 +1,108 @@ +{ + lib ? import , +}: +let + clanLibOrig = (import ./.. { inherit lib; }).__unfix__; + clanLibWithFs = + { virtual_fs }: + lib.fix ( + lib.extends ( + final: _: + let + clan-core = { + clanLib = final; + modules.clan.default = lib.modules.importApply ./clan { inherit clan-core; }; + + # Note: Can add other things to "clan-core" + # ... Not needed for this test + }; + in + { + clan = import ../clan { + inherit lib clan-core; + }; + + # Override clanLib.fs for unit-testing against a virtual filesystem + fs = import ../clanTest/virtual-fs.nix { inherit lib; } { + inherit rootPath virtual_fs; + # Example of a passthru + # passthru = [ + # ".*inventory\.json$" + # ]; + }; + } + ) clanLibOrig + ); + + rootPath = ./.; +in +{ + test_autoload_directories = + let + vclan = + (clanLibWithFs { + virtual_fs = { + "machines" = { + type = "directory"; + }; + "machines/foo-machine" = { + type = "directory"; + }; + "machines/bar-machine" = { + type = "directory"; + }; + }; + }).clan + { config.directory = rootPath; }; + in + { + inherit vclan; + expr = { + machines = lib.attrNames vclan.config.inventory.machines; + definedInMachinesDir = map ( + p: lib.hasInfix "/machines/" p + ) vclan.options.inventory.valueMeta.configuration.options.machines.files; + }; + expected = { + machines = [ + "bar-machine" + "foo-machine" + ]; + definedInMachinesDir = [ + true # /machines/foo-machine + true # /machines/bar-machine + false # /module.nix defines "machines" without members + ]; + }; + }; + + # Could probably be unified with the previous test + # This is here for the sake to show that 'virtual_fs' is a test parameter + test_files_are_not_machines = + let + vclan = + (clanLibWithFs { + virtual_fs = { + "machines" = { + type = "directory"; + }; + "machines/foo.nix" = { + type = "file"; + }; + "machines/bar.nix" = { + type = "file"; + }; + }; + }).clan + { config.directory = rootPath; }; + in + { + inherit vclan; + expr = { + machines = lib.attrNames vclan.config.inventory.machines; + }; + expected = { + machines = [ ]; + }; + }; +} diff --git a/lib/modules/tests.nix b/lib/modules/tests.nix index 5ec8707d8..56ed2edbf 100644 --- a/lib/modules/tests.nix +++ b/lib/modules/tests.nix @@ -12,6 +12,7 @@ let in ####### { + autoloading = import ./dir_test.nix { inherit lib; }; test_missing_self = let eval = clan { From 347668a57f64a3cd28c0f0775ca7ec7e7dbc4d10 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 12 Oct 2025 17:49:05 +0200 Subject: [PATCH 34/39] lib: remove unused facts utils --- lib/default.nix | 1 - lib/facts.nix | 71 ------------------------------------------------- 2 files changed, 72 deletions(-) delete mode 100644 lib/facts.nix diff --git a/lib/default.nix b/lib/default.nix index f9734ed0f..48f3aaebd 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -28,7 +28,6 @@ lib.fix ( # Plain imports. introspection = import ./introspection { inherit lib; }; jsonschema = import ./jsonschema { inherit lib; }; - facts = import ./facts.nix { inherit lib; }; docs = import ./docs.nix { inherit lib; }; # flakes diff --git a/lib/facts.nix b/lib/facts.nix deleted file mode 100644 index 967b44828..000000000 --- a/lib/facts.nix +++ /dev/null @@ -1,71 +0,0 @@ -{ lib, ... }: -clanDir: -let - - allMachineNames = lib.mapAttrsToList (name: _: name) (builtins.readDir clanDir); - - getFactPath = machine: fact: "${clanDir}/machines/${machine}/facts/${fact}"; - - readFact = - machine: fact: - let - path = getFactPath machine fact; - in - if builtins.pathExists path then builtins.readFile path else null; - - # Example: - # - # readFactFromAllMachines zerotier-ip - # => { - # machineA = "1.2.3.4"; - # machineB = "5.6.7.8"; - # }; - readFactFromAllMachines = - fact: - let - machines = allMachineNames; - facts = lib.genAttrs machines (machine: readFact machine fact); - filteredFacts = lib.filterAttrs (_machine: fact: fact != null) facts; - in - filteredFacts; - - # all given facts are are set and factvalues are never null. - # - # Example: - # - # readFactsFromAllMachines [ "zerotier-ip" "syncthing.pub" ] - # => { - # machineA = - # { - # "zerotier-ip" = "1.2.3.4"; - # "synching.pub" = "1234"; - # }; - # machineB = - # { - # "zerotier-ip" = "5.6.7.8"; - # "synching.pub" = "23456719"; - # }; - # }; - readFactsFromAllMachines = - facts: - let - # machine -> fact -> factvalue - machinesFactsAttrs = lib.genAttrs allMachineNames ( - machine: lib.genAttrs facts (fact: readFact machine fact) - ); - # remove all machines which don't have all facts set - filteredMachineFactAttrs = lib.filterAttrs ( - _machine: values: builtins.all (fact: values.${fact} != null) facts - ) machinesFactsAttrs; - in - filteredMachineFactAttrs; -in -{ - inherit - allMachineNames - getFactPath - readFact - readFactFromAllMachines - readFactsFromAllMachines - ; -} From 7b95fa039f6d0b549850a8d7c0770138cfa8e555 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 12 Oct 2025 18:00:52 +0200 Subject: [PATCH 35/39] clan-cli: remove unused test fixture --- .../.clan-flake | 0 .../flake.nix | 24 ------------------- 2 files changed, 24 deletions(-) delete mode 100644 pkgs/clan-cli/clan_cli/tests/test_flake_with_core_dynamic_machines/.clan-flake delete mode 100644 pkgs/clan-cli/clan_cli/tests/test_flake_with_core_dynamic_machines/flake.nix diff --git a/pkgs/clan-cli/clan_cli/tests/test_flake_with_core_dynamic_machines/.clan-flake b/pkgs/clan-cli/clan_cli/tests/test_flake_with_core_dynamic_machines/.clan-flake deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkgs/clan-cli/clan_cli/tests/test_flake_with_core_dynamic_machines/flake.nix b/pkgs/clan-cli/clan_cli/tests/test_flake_with_core_dynamic_machines/flake.nix deleted file mode 100644 index 730b3f81b..000000000 --- a/pkgs/clan-cli/clan_cli/tests/test_flake_with_core_dynamic_machines/flake.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - # Use this path to our repo root e.g. for UI test - # inputs.clan-core.url = "../../../../."; - - # this placeholder is replaced by the path to nixpkgs - inputs.clan-core.url = "__CLAN_CORE__"; - - outputs = - { self, clan-core }: - let - clan = clan-core.lib.clan { - inherit self; - meta.name = "test_flake_with_core_dynamic_machines"; - machines = - let - machineModules = builtins.readDir (self + "/machines"); - in - builtins.mapAttrs (name: _type: import (self + "/machines/${name}")) machineModules; - }; - in - { - inherit (clan.config) nixosConfigurations nixosModules clanInternals; - }; -} From 3c4b3e180ea0ca1245c6db368e4b7f819a1b8706 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 13 Oct 2025 09:44:58 +0200 Subject: [PATCH 36/39] facts: add bigger migration warnings --- nixosModules/clanCore/facts/compat.nix | 26 ++++++++++++++++++------- nixosModules/clanCore/facts/default.nix | 12 +++++++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/nixosModules/clanCore/facts/compat.nix b/nixosModules/clanCore/facts/compat.nix index 02f1bad83..58a6b6ab1 100644 --- a/nixosModules/clanCore/facts/compat.nix +++ b/nixosModules/clanCore/facts/compat.nix @@ -164,13 +164,25 @@ config = lib.mkIf (config.clan.core.secrets != { }) { clan.core.facts.services = lib.mapAttrs' ( name: service: - lib.warn "clan.core.secrets.${name} is deprecated, use clan.core.facts.services.${name} instead" ( - lib.nameValuePair name ({ - secret = service.secrets; - public = service.facts; - generator = service.generator; - }) - ) + lib.warn + '' + ############################################################################### + # # + # clan.core.secrets.${name} clan.core.facts.services.${name} is deprecated # + # in favor of "vars" # + # # + # Refer to https://docs.clan.lol/guides/migrations/migration-facts-vars/ # + # for migration instructions. # + # # + ############################################################################### + '' + ( + lib.nameValuePair name ({ + secret = service.secrets; + public = service.facts; + generator = service.generator; + }) + ) ) config.clan.core.secrets; }; } diff --git a/nixosModules/clanCore/facts/default.nix b/nixosModules/clanCore/facts/default.nix index 0a1798c31..2b8800510 100644 --- a/nixosModules/clanCore/facts/default.nix +++ b/nixosModules/clanCore/facts/default.nix @@ -6,7 +6,17 @@ }: { config.warnings = lib.optionals (config.clan.core.facts.services != { }) [ - "Facts are deprecated, please migrate them to vars instead, see: https://docs.clan.lol/guides/migrations/migration-facts-vars/" + '' + ############################################################################### + # # + # Facts are deprecated please migrate any usages to vars instead # + # # + # # + # Refer to https://docs.clan.lol/guides/migrations/migration-facts-vars/ # + # for migration instructions. # + # # + ############################################################################### + '' ]; options.clan.core.facts = { From 1c3282bb63b181e5b1aa37473fbd45794bb80f32 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 13 Oct 2025 10:05:15 +0200 Subject: [PATCH 37/39] vars: simplify collectFiles --- .../vars/secret/sops/collectFiles.nix | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/nixosModules/clanCore/vars/secret/sops/collectFiles.nix b/nixosModules/clanCore/vars/secret/sops/collectFiles.nix index c6cbe0550..697c59861 100644 --- a/nixosModules/clanCore/vars/secret/sops/collectFiles.nix +++ b/nixosModules/clanCore/vars/secret/sops/collectFiles.nix @@ -5,33 +5,31 @@ let inherit (lib) filterAttrs - flatten mapAttrsToList ; -in -generators: -let - relevantFiles = - generator: - filterAttrs ( - _name: f: f.secret && f.deploy && (f.neededFor == "users" || f.neededFor == "services") - ) generator.files; - allFiles = flatten ( - mapAttrsToList ( - gen_name: generator: - mapAttrsToList (fname: file: { - name = fname; - generator = gen_name; - neededForUsers = file.neededFor == "users"; - inherit (generator) share; - inherit (file) - owner - group - mode - restartUnits - ; - }) (relevantFiles generator) - ) generators + + relevantFiles = filterAttrs ( + _name: f: f.secret && f.deploy && (f.neededFor == "users" || f.neededFor == "services") ); + + collectFiles = + generators: + builtins.concatLists ( + mapAttrsToList ( + gen_name: generator: + mapAttrsToList (fname: file: { + name = fname; + generator = gen_name; + neededForUsers = file.neededFor == "users"; + inherit (generator) share; + inherit (file) + owner + group + mode + restartUnits + ; + }) (relevantFiles generator.files) + ) generators + ); in -allFiles +collectFiles From fe8f7e919ec89e0c11e9e01c653630e38e56d5ee Mon Sep 17 00:00:00 2001 From: pinpox Date: Mon, 13 Oct 2025 12:51:42 +0200 Subject: [PATCH 38/39] Fix ssh docs --- clanServices/sshd/README.md | 90 ++++++++----------------------------- 1 file changed, 19 insertions(+), 71 deletions(-) diff --git a/clanServices/sshd/README.md b/clanServices/sshd/README.md index 88df4ab25..5989c6e65 100644 --- a/clanServices/sshd/README.md +++ b/clanServices/sshd/README.md @@ -1,91 +1,39 @@ -# Clan service: sshd -What it does -- Generates and persists SSH host keys via `vars`. -- Optionally issues CA‑signed host certificates for servers. -- Installs the `server` CA public key into `clients` `known_hosts` for TOFU‑less verification. +The `sshd` Clan service manages SSH to make it easy to securely access your +machines over the internet. The service uses `vars` to store the SSH host keys +for each machine to ensure they remain stable across deployments. +`sshd` also generates SSH certificates for both servers and clients allowing for +certificate-based authentication for SSH. -When to use it -- Zero‑TOFU SSH for dynamic fleets: admins/CI can connect to frequently rebuilt hosts (e.g., server-1.example.com) without prompts or per‑host `known_hosts` churn. +The service also disables password-based authentication over SSH, to access your +machines you'll need to use public key authentication or certificate-based +authentication. -Roles -- Server: runs sshd, presents a CA‑signed host certificate for `.`. -- Client: trusts the CA for the given domains to verify servers’ certificates. - Tip: assign both roles to a machine if it should both present a cert and verify others. - -Quick start (with host certificates) -Useful if you never want to get a prompt about trusting the ssh fingerprint. -```nix -{ - inventory.instances = { - sshd-with-certs = { - module = { name = "sshd"; input = "clan-core"; }; - # Servers present certificates for .example.com - roles.server.tags.all = { }; - roles.server.settings = { - certificate.searchDomains = [ "example.com" ]; - # Optional: also add RSA host keys - # hostKeys.rsa.enable = true; - }; - # Clients trust the CA for *.example.com - roles.client.tags.all = { }; - roles.client.settings = { - certificate.searchDomains = [ "example.com" ]; - }; - }; - }; -} -``` - -Basic: only add persistent host keys (ed25519), no certificates -Useful if you want to get an ssh "trust this server" prompt once and then never again. +## Usage ```nix { inventory.instances = { + # By default this service only generates ed25519 host keys sshd-basic = { module = { name = "sshd"; input = "clan-core"; }; roles.server.tags.all = { }; + roles.client.tags.all = { }; }; - }; -} -``` - -Example: selective trust per environment -Admins should trust only production; CI should trust prod and staging. Servers are reachable under both domains. -```nix -{ - inventory.instances = { - sshd-env-scoped = { - module = { name = "sshd"; input = "clan-core"; }; - - # Servers present certs for both prod and staging FQDNs + # Also generate RSA host keys for all servers + sshd-with-rsa = { + module = { + name = "sshd"; + input = "clan-core"; + }; roles.server.tags.all = { }; roles.server.settings = { - certificate.searchDomains = [ "prod.example.com" "staging.example.com" ]; - }; - - # Admin laptop: trust prod only - roles.client.machines."admin-laptop".settings = { - certificate.searchDomains = [ "prod.example.com" ]; - }; - - # CI runner: trust prod and staging - roles.client.machines."ci-runner-1".settings = { - certificate.searchDomains = [ "prod.example.com" "staging.example.com" ]; + hostKeys.rsa.enable = true; }; + roles.client.tags.all = { }; }; }; } ``` -- Admin -> server1.prod.example.com: zero‑TOFU (verified via cert). -- Admin -> server1.staging.example.com: falls back to TOFU (or is blocked by policy). -- CI -> either prod or staging: zero‑TOFU for both. -Note: server and client searchDomains don’t have to be identical; they only need to overlap for the hostnames you actually use. - -Notes -- Connect using a name that matches a cert principal (e.g., `server1.example.com`); wildcards are not allowed inside the certificate. -- CA private key stays in `vars` (not deployed); only the CA public key is distributed. -- Logins still require your user SSH keys on the server (passwords are disabled). \ No newline at end of file From bf41a9ef00b82f7fabcf25f9d41358d41f8991ab Mon Sep 17 00:00:00 2001 From: DavHau Date: Mon, 13 Oct 2025 18:43:40 +0700 Subject: [PATCH 39/39] vars/sops: stop writing on `clan vars check` This fixes an issue where check_vars() would add machine keys or authorize machines for shared vars. These write operations should only ever be done on a `clan vars generate`, which `clan vars check` should be a read-only operation --- pkgs/clan-cli/clan_cli/tests/test_vars.py | 39 +++++++++++++++- pkgs/clan-cli/clan_cli/vars/check.py | 45 +++++++++++++++++-- pkgs/clan-cli/clan_cli/vars/generator.py | 4 ++ .../clan_cli/vars/secret_modules/sops.py | 6 +-- pkgs/clan-cli/clan_lib/vars/generate.py | 30 ++++++++++--- 5 files changed, 110 insertions(+), 14 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 50c20cd6d..1943fb936 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -1,5 +1,6 @@ import json import logging +import os import shutil import subprocess import time @@ -429,9 +430,43 @@ def test_generated_shared_secret_sops( machine1 = Machine(name="machine1", flake=Flake(str(flake.path))) machine2 = Machine(name="machine2", flake=Flake(str(flake.path))) cli.run(["vars", "generate", "--flake", str(flake.path), "machine1"]) - assert check_vars(machine1.name, machine1.flake) + + # Get the initial state of the flake directory after generation + def get_file_mtimes(path: str) -> dict[str, float]: + """Get modification times of all files in a directory tree.""" + mtimes = {} + for root, _dirs, files in os.walk(path): + # Skip .git directory + if ".git" in root: + continue + for file in files: + filepath = Path(root) / file + mtimes[str(filepath)] = filepath.stat().st_mtime + return mtimes + + initial_mtimes = get_file_mtimes(str(flake.path)) + + # First check_vars should not write anything + assert check_vars(machine1.name, machine1.flake), ( + "machine1 has already generated vars, so check_vars should return True\n" + f"Check result:\n{check_vars(machine1.name, machine1.flake)}" + ) + # Verify no files were modified + after_check_mtimes = get_file_mtimes(str(flake.path)) + assert initial_mtimes == after_check_mtimes, ( + "check_vars should not modify any files when vars are already valid" + ) + + assert not check_vars(machine2.name, machine2.flake), ( + "machine2 has not generated vars yet, so check_vars should return False" + ) + # Verify no files were modified + after_check_mtimes_2 = get_file_mtimes(str(flake.path)) + assert initial_mtimes == after_check_mtimes_2, ( + "check_vars should not modify any files when vars are not valid" + ) + cli.run(["vars", "generate", "--flake", str(flake.path), "machine2"]) - assert check_vars(machine2.name, machine2.flake) m1_sops_store = sops.SecretStore(machine1.flake) m2_sops_store = sops.SecretStore(machine2.flake) # Create generators with machine context for testing diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index 362d21b0d..70de575a6 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -3,6 +3,7 @@ import logging from typing import TYPE_CHECKING from clan_cli.completions import add_dynamic_completer, complete_machines +from clan_cli.vars.secret_modules import sops from clan_lib.errors import ClanError from clan_lib.flake import Flake, require_flake from clan_lib.machines.machines import Machine @@ -26,6 +27,26 @@ class VarStatus: self.unfixed_secret_vars = unfixed_secret_vars self.invalid_generators = invalid_generators + def text(self) -> str: + log = "" + if self.missing_secret_vars: + log += "Missing secret vars:\n" + for var in self.missing_secret_vars: + log += f" - {var.id}\n" + if self.missing_public_vars: + log += "Missing public vars:\n" + for var in self.missing_public_vars: + log += f" - {var.id}\n" + if self.unfixed_secret_vars: + log += "Unfixed secret vars:\n" + for var in self.unfixed_secret_vars: + log += f" - {var.id}\n" + if self.invalid_generators: + log += "Invalid generators (outdated invalidation hash):\n" + for gen in self.invalid_generators: + log += f" - {gen}\n" + return log if log else "All vars are present and valid." + def vars_status( machine_name: str, @@ -66,15 +87,32 @@ def vars_status( f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing.", ) missing_secret_vars.append(file) + if ( + isinstance(machine.secret_vars_store, sops.SecretStore) + and generator.share + and file.exists + and not machine.secret_vars_store.machine_has_access( + generator=generator, + secret_name=file.name, + machine=machine.name, + ) + ): + msg = ( + f"Secret var '{generator.name}/{file.name}' is marked for deployment to machine '{machine.name}', but the machine does not have access to it.\n" + f"Run 'clan vars generate {machine.name}' to fix this.\n" + ) + machine.info(msg) + missing_secret_vars.append(file) + else: - msg = machine.secret_vars_store.health_check( + health_msg = machine.secret_vars_store.health_check( machine=machine.name, generators=[generator], file_name=file.name, ) - if msg: + if health_msg is not None: machine.info( - f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} needs update: {msg}", + f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} needs update: {health_msg}", ) unfixed_secret_vars.append(file) @@ -106,6 +144,7 @@ def check_vars( generator_name: None | str = None, ) -> bool: status = vars_status(machine_name, flake, generator_name=generator_name) + log.info(f"Check results for machine '{machine_name}': \n{status.text()}") return not ( status.missing_secret_vars or status.missing_public_vars diff --git a/pkgs/clan-cli/clan_cli/vars/generator.py b/pkgs/clan-cli/clan_cli/vars/generator.py index 3daf33392..acb4e687c 100644 --- a/pkgs/clan-cli/clan_cli/vars/generator.py +++ b/pkgs/clan-cli/clan_cli/vars/generator.py @@ -259,6 +259,10 @@ class Generator: _secret_store=sec_store, ) + # link generator to its files + for file in files: + file.generator(generator) + if share: # For shared generators, check if we already created it existing = next( diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index 1df98dc0e..c96a2e1d8 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -98,7 +98,8 @@ class SecretStore(StoreBase): def machine_has_access( self, generator: Generator, secret_name: str, machine: str ) -> bool: - self.ensure_machine_key(machine) + if not has_machine(self.flake.path, machine): + return False key_dir = sops_machines_folder(self.flake.path) / machine return self.key_has_access(key_dir, generator, secret_name) @@ -156,8 +157,6 @@ class SecretStore(StoreBase): else: continue if file.secret and self.exists(generator, file.name): - if file.deploy: - self.ensure_machine_has_access(generator, file.name, machine) needs_update, msg = self.needs_fix(generator, file.name, machine) if needs_update: outdated.append((generator.name, file.name, msg)) @@ -283,6 +282,7 @@ class SecretStore(StoreBase): ) -> None: if self.machine_has_access(generator, name, machine): return + self.ensure_machine_key(machine) secret_folder = self.secret_path(generator, name) add_secret( self.flake.path, diff --git a/pkgs/clan-cli/clan_lib/vars/generate.py b/pkgs/clan-cli/clan_lib/vars/generate.py index e4ccf281e..41ad4951e 100644 --- a/pkgs/clan-cli/clan_lib/vars/generate.py +++ b/pkgs/clan-cli/clan_lib/vars/generate.py @@ -5,6 +5,7 @@ from clan_cli.vars import graph from clan_cli.vars.generator import Generator from clan_cli.vars.graph import requested_closure from clan_cli.vars.migration import check_can_migrate, migrate_files +from clan_cli.vars.secret_modules import sops from clan_lib.api import API from clan_lib.errors import ClanError @@ -152,15 +153,15 @@ def run_generators( if not machines: msg = "At least one machine must be provided" raise ClanError(msg) + all_generators = get_generators(machines, full_closure=True) if isinstance(generators, list): # List of generator names - use them exactly as provided if len(generators) == 0: return - all_generators = get_generators(machines, full_closure=True) - generator_objects = [g for g in all_generators if g.key.name in generators] + generators_to_run = [g for g in all_generators if g.key.name in generators] else: # None or single string - use get_generators with closure parameter - generator_objects = get_generators( + generators_to_run = get_generators( machines, full_closure=full_closure, generator_name=generators, @@ -170,13 +171,30 @@ def run_generators( # TODO: make this more lazy and ask for every generator on execution if callable(prompt_values): prompt_values = { - generator.name: prompt_values(generator) for generator in generator_objects + generator.name: prompt_values(generator) for generator in generators_to_run } # execute health check for machine in machines: _ensure_healthy(machine=machine) + # ensure all selected machines have access to all selected shared generators + for machine in machines: + # This is only relevant for the sops store + # TODO: improve store abstraction to use Protocols and introduce a proper SecretStore interface + if not isinstance(machine.secret_vars_store, sops.SecretStore): + continue + for generator in all_generators: + if generator.share: + for file in generator.files: + if not file.secret or not file.exists: + continue + machine.secret_vars_store.ensure_machine_has_access( + generator, + file.name, + machine.name, + ) + # get the flake via any machine (they are all the same) flake = machines[0].flake @@ -188,13 +206,13 @@ def run_generators( # preheat the select cache, to reduce repeated calls during execution selectors = [] - for generator in generator_objects: + for generator in generators_to_run: machine = get_generator_machine(generator) selectors.append(generator.final_script_selector(machine.name)) flake.precache(selectors) # execute generators - for generator in generator_objects: + for generator in generators_to_run: machine = get_generator_machine(generator) if check_can_migrate(machine, generator): migrate_files(machine, generator)