From 52c73265acbb336336313e96d3f23e228978d484 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Sat, 5 Aug 2023 12:19:14 +0000 Subject: [PATCH 01/36] Fixed wrong flake template name (#89) Co-authored-by: Luis-Hebendanz Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/89 --- pkgs/clan-cli/clan_cli/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/admin.py b/pkgs/clan-cli/clan_cli/admin.py index 2197b5235..91edc563a 100644 --- a/pkgs/clan-cli/clan_cli/admin.py +++ b/pkgs/clan-cli/clan_cli/admin.py @@ -13,7 +13,7 @@ def create(args: argparse.Namespace) -> None: "flake", "init", "-t", - "git+https://git.clan.lol/clan/clan-core#clan-template", + "git+https://git.clan.lol/clan/clan-core#new-clan", ] ) From c3b49fdce04c8df5d03af7d6e691a518a73b19bc Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Aug 2023 17:26:24 +0200 Subject: [PATCH 02/36] init: layout with simple side navigation --- pkgs/ui/next.config.js | 1 + pkgs/ui/nix/pdefs.nix | 151 +++++++++++++++++++++++ pkgs/ui/package-lock.json | 26 ++++ pkgs/ui/package.json | 1 + pkgs/ui/prettier.config.js | 4 - pkgs/ui/public/logo.svg | 1 + pkgs/ui/src/app/dashboard/page.tsx | 25 ---- pkgs/ui/src/app/globals.css | 24 ---- pkgs/ui/src/app/layout.tsx | 68 +++++----- pkgs/ui/src/app/page.tsx | 116 ++--------------- pkgs/ui/src/app/theme/themes.ts | 13 ++ pkgs/ui/src/components/sidebar/index.tsx | 122 ++++++++++++++++++ pkgs/ui/tailwind.config.js | 16 +-- 13 files changed, 362 insertions(+), 206 deletions(-) delete mode 100644 pkgs/ui/prettier.config.js create mode 100644 pkgs/ui/public/logo.svg delete mode 100644 pkgs/ui/src/app/dashboard/page.tsx create mode 100644 pkgs/ui/src/app/theme/themes.ts create mode 100644 pkgs/ui/src/components/sidebar/index.tsx diff --git a/pkgs/ui/next.config.js b/pkgs/ui/next.config.js index a35bfad7f..bd7fd9ea2 100644 --- a/pkgs/ui/next.config.js +++ b/pkgs/ui/next.config.js @@ -1,6 +1,7 @@ /** @type {import('next').NextConfig} */ const nextConfig = { output: "export", + images: { unoptimized: true }, }; module.exports = nextConfig; diff --git a/pkgs/ui/nix/pdefs.nix b/pkgs/ui/nix/pdefs.nix index 7a09b4e54..69d2bf90b 100644 --- a/pkgs/ui/nix/pdefs.nix +++ b/pkgs/ui/nix/pdefs.nix @@ -895,6 +895,37 @@ version = "5.14.3"; }; }; + "@mui/icons-material" = { + "5.14.3" = { + depInfo = { + "@babel/runtime" = { + descriptor = "^7.22.6"; + pin = "7.22.6"; + runtime = true; + }; + }; + fetchInfo = { + narHash = "sha256-wmY7EzOahWuCF2g5vpcOeFZ8+iJKwyFLHsQiXh1R2jY="; + type = "tarball"; + url = "https://registry.npmjs.org/@mui/icons-material/-/icons-material-5.14.3.tgz"; + }; + ident = "@mui/icons-material"; + ltype = "file"; + peerInfo = { + "@mui/material" = { + descriptor = "^5.0.0"; + }; + "@types/react" = { + descriptor = "^17.0.0 || ^18.0.0"; + optional = true; + }; + react = { + descriptor = "^17.0.0 || ^18.0.0"; + }; + }; + version = "5.14.3"; + }; + }; "@mui/material" = { "5.14.3" = { depInfo = { @@ -6643,6 +6674,11 @@ pin = "11.11.0"; runtime = true; }; + "@mui/icons-material" = { + descriptor = "^5.14.3"; + pin = "5.14.3"; + runtime = true; + }; "@mui/material" = { descriptor = "^5.14.3"; pin = "5.14.3"; @@ -6693,6 +6729,14 @@ pin = "8.4.27"; runtime = true; }; + prettier = { + descriptor = "^3.0.1"; + pin = "3.0.1"; + }; + prettier-plugin-tailwindcss = { + descriptor = "^0.4.1"; + pin = "0.4.1"; + }; react = { descriptor = "18.2.0"; pin = "18.2.0"; @@ -6856,6 +6900,9 @@ "node_modules/@mui/core-downloads-tracker" = { key = "@mui/core-downloads-tracker/5.14.3"; }; + "node_modules/@mui/icons-material" = { + key = "@mui/icons-material/5.14.3"; + }; "node_modules/@mui/material" = { key = "@mui/material/5.14.3"; }; @@ -7727,6 +7774,14 @@ "node_modules/prelude-ls" = { key = "prelude-ls/1.2.1"; }; + "node_modules/prettier" = { + dev = true; + key = "prettier/3.0.1"; + }; + "node_modules/prettier-plugin-tailwindcss" = { + dev = true; + key = "prettier-plugin-tailwindcss/0.4.1"; + }; "node_modules/prop-types" = { key = "prop-types/15.8.1"; }; @@ -9062,6 +9117,102 @@ version = "1.2.1"; }; }; + prettier = { + "3.0.1" = { + binInfo = { + binPairs = { + prettier = "bin/prettier.cjs"; + }; + }; + fetchInfo = { + narHash = "sha256-rgaO4WYmjoHtlOu8SnOau8b/O9lIEDtt26ovEY7qseY="; + type = "tarball"; + url = "https://registry.npmjs.org/prettier/-/prettier-3.0.1.tgz"; + }; + ident = "prettier"; + ltype = "file"; + treeInfo = { }; + version = "3.0.1"; + }; + }; + prettier-plugin-tailwindcss = { + "0.4.1" = { + fetchInfo = { + narHash = "sha256-39DJn6lvrLmDYTN/lXXuWzMC9pLI4+HNrhnHlYuOMRM="; + type = "tarball"; + url = "https://registry.npmjs.org/prettier-plugin-tailwindcss/-/prettier-plugin-tailwindcss-0.4.1.tgz"; + }; + ident = "prettier-plugin-tailwindcss"; + ltype = "file"; + peerInfo = { + "@ianvs/prettier-plugin-sort-imports" = { + descriptor = "*"; + optional = true; + }; + "@prettier/plugin-pug" = { + descriptor = "*"; + optional = true; + }; + "@shopify/prettier-plugin-liquid" = { + descriptor = "*"; + optional = true; + }; + "@shufo/prettier-plugin-blade" = { + descriptor = "*"; + optional = true; + }; + "@trivago/prettier-plugin-sort-imports" = { + descriptor = "*"; + optional = true; + }; + prettier = { + descriptor = "^2.2 || ^3.0"; + }; + prettier-plugin-astro = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-css-order = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-import-sort = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-jsdoc = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-marko = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-organize-attributes = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-organize-imports = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-style-order = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-svelte = { + descriptor = "*"; + optional = true; + }; + prettier-plugin-twig-melody = { + descriptor = "*"; + optional = true; + }; + }; + treeInfo = { }; + version = "0.4.1"; + }; + }; prop-types = { "15.8.1" = { depInfo = { diff --git a/pkgs/ui/package-lock.json b/pkgs/ui/package-lock.json index b683984ac..9a36af942 100644 --- a/pkgs/ui/package-lock.json +++ b/pkgs/ui/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@emotion/react": "^11.11.1", "@emotion/styled": "^11.11.0", + "@mui/icons-material": "^5.14.3", "@mui/material": "^5.14.3", "@types/node": "20.4.7", "@types/react": "18.2.18", @@ -497,6 +498,31 @@ "url": "https://opencollective.com/mui" } }, + "node_modules/@mui/icons-material": { + "version": "5.14.3", + "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-5.14.3.tgz", + "integrity": "sha512-XkxWPhageu1OPUm2LWjo5XqeQ0t2xfGe8EiLkRW9oz2LHMMZmijvCxulhgquUVTF1DnoSh+3KoDLSsoAFtVNVw==", + "dependencies": { + "@babel/runtime": "^7.22.6" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui" + }, + "peerDependencies": { + "@mui/material": "^5.0.0", + "@types/react": "^17.0.0 || ^18.0.0", + "react": "^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/@mui/material": { "version": "5.14.3", "resolved": "https://registry.npmjs.org/@mui/material/-/material-5.14.3.tgz", diff --git a/pkgs/ui/package.json b/pkgs/ui/package.json index 1e4c073da..4919fc8ec 100644 --- a/pkgs/ui/package.json +++ b/pkgs/ui/package.json @@ -14,6 +14,7 @@ "dependencies": { "@emotion/react": "^11.11.1", "@emotion/styled": "^11.11.0", + "@mui/icons-material": "^5.14.3", "@mui/material": "^5.14.3", "@types/node": "20.4.7", "@types/react": "18.2.18", diff --git a/pkgs/ui/prettier.config.js b/pkgs/ui/prettier.config.js deleted file mode 100644 index a0d9c69fd..000000000 --- a/pkgs/ui/prettier.config.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - plugins: [require("prettier-plugin-tailwindcss")], - tailwindFunctions: ['clsx', 'cx'], -}; diff --git a/pkgs/ui/public/logo.svg b/pkgs/ui/public/logo.svg new file mode 100644 index 000000000..e66bb0848 --- /dev/null +++ b/pkgs/ui/public/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/pkgs/ui/src/app/dashboard/page.tsx b/pkgs/ui/src/app/dashboard/page.tsx deleted file mode 100644 index 1670f0fa5..000000000 --- a/pkgs/ui/src/app/dashboard/page.tsx +++ /dev/null @@ -1,25 +0,0 @@ -import { DashboardCard } from "../../components/card"; -import { Grid } from "@mui/material"; -import { Button } from "@mui/material"; - -export default function Dashboard() { - return ( - - - - - Hallo Mike ! - - - - Server Stats - - - Network Stats - - - - ); -} diff --git a/pkgs/ui/src/app/globals.css b/pkgs/ui/src/app/globals.css index fd81e8858..b5c61c956 100644 --- a/pkgs/ui/src/app/globals.css +++ b/pkgs/ui/src/app/globals.css @@ -1,27 +1,3 @@ @tailwind base; @tailwind components; @tailwind utilities; - -:root { - --foreground-rgb: 0, 0, 0; - --background-start-rgb: 214, 219, 220; - --background-end-rgb: 255, 255, 255; -} - -@media (prefers-color-scheme: dark) { - :root { - --foreground-rgb: 255, 255, 255; - --background-start-rgb: 0, 0, 0; - --background-end-rgb: 0, 0, 0; - } -} - -body { - color: rgb(var(--foreground-rgb)); - background: linear-gradient( - to bottom, - transparent, - rgb(var(--background-end-rgb)) - ) - rgb(var(--background-start-rgb)); -} diff --git a/pkgs/ui/src/app/layout.tsx b/pkgs/ui/src/app/layout.tsx index 00f196872..49106047b 100644 --- a/pkgs/ui/src/app/layout.tsx +++ b/pkgs/ui/src/app/layout.tsx @@ -1,10 +1,15 @@ +"use client"; + import "./globals.css"; -import type { Metadata } from "next"; import localFont from "next/font/local"; import * as React from "react"; +import { CssBaseline, ThemeProvider } from "@mui/material"; +import { ChangeEvent, useState } from "react"; + import { StyledEngineProvider } from "@mui/material/styles"; -import cx from "classnames"; -// import { tw } from "../utils/tailwind"; + +import { darkTheme, lightTheme } from "./theme/themes"; +import { Sidebar } from "@/components/sidebar"; const roboto = localFont({ src: [ @@ -13,49 +18,42 @@ const roboto = localFont({ weight: "400", style: "normal", }, - // { - // path: "./Roboto-Italic.woff2", - // weight: "400", - // style: "italic", - // }, - // { - // path: "./Roboto-Bold.woff2", - // weight: "700", - // style: "normal", - // }, - // { - // path: "./Roboto-BoldItalic.woff2", - // weight: "700", - // style: "italic", - // }, ], }); -export const metadata: Metadata = { - title: "Create Next App", - description: "Generated by create next app", -}; - export default function RootLayout({ children, }: { children: React.ReactNode; }) { + let [useDarkTheme, setUseDarkTheme] = useState(false); + let [theme, setTheme] = useState(useDarkTheme ? darkTheme : lightTheme); + + const changeThemeHandler = (target: ChangeEvent, currentValue: boolean) => { + setUseDarkTheme(currentValue); + setTheme(currentValue ? darkTheme : lightTheme); + }; + return ( + + Clan.lol + + + + - - {children} - + + + +
+ +
+
{children}
+
+
+ + ); diff --git a/pkgs/ui/src/app/page.tsx b/pkgs/ui/src/app/page.tsx index 5cb8b624d..4918af51a 100644 --- a/pkgs/ui/src/app/page.tsx +++ b/pkgs/ui/src/app/page.tsx @@ -1,112 +1,14 @@ -import Image from "next/image"; +import { Button } from "@mui/material"; -export default function Home() { +export default function Dashboard() { return ( -
-
-

- Get started by editing  - src/app/page.tsx -

-
- - Vercel Logo - -
+
+ ); } diff --git a/pkgs/ui/src/app/theme/themes.ts b/pkgs/ui/src/app/theme/themes.ts new file mode 100644 index 000000000..fd845dec0 --- /dev/null +++ b/pkgs/ui/src/app/theme/themes.ts @@ -0,0 +1,13 @@ +import { createTheme } from "@mui/material/styles"; + +export const darkTheme = createTheme({ + palette: { + mode: "dark", + }, +}); + +export const lightTheme = createTheme({ + palette: { + mode: "light", + }, +}); diff --git a/pkgs/ui/src/components/sidebar/index.tsx b/pkgs/ui/src/components/sidebar/index.tsx new file mode 100644 index 000000000..8418af748 --- /dev/null +++ b/pkgs/ui/src/components/sidebar/index.tsx @@ -0,0 +1,122 @@ +import { + Divider, + List, + ListItem, + ListItemButton, + ListItemIcon, + ListItemText, +} from "@mui/material"; +import Image from "next/image"; +import { ReactNode } from "react"; + +import DashboardIcon from "@mui/icons-material/Dashboard"; +import DevicesIcon from "@mui/icons-material/Devices"; +import LanIcon from "@mui/icons-material/Lan"; +import AppsIcon from "@mui/icons-material/Apps"; +import DesignServicesIcon from "@mui/icons-material/DesignServices"; +import BackupIcon from "@mui/icons-material/Backup"; +import Link from "next/link"; + +type MenuEntry = { + icon: ReactNode; + label: string; + to: string; +} & { + subMenuEntries?: MenuEntry[]; +}; + +const menuEntries: MenuEntry[] = [ + { + icon: , + label: "Dashoard", + to: "/", + }, + { + icon: , + label: "Devices", + to: "/nodes", + }, + { + icon: , + label: "Applications", + to: "/applications", + }, + { + icon: , + label: "Network", + to: "/network", + }, + { + icon: , + label: "Templates", + to: "/templates", + }, + { + icon: , + label: "Backups", + to: "/backups", + }, +]; + +export function Sidebar() { + return ( + + ); +} diff --git a/pkgs/ui/tailwind.config.js b/pkgs/ui/tailwind.config.js index 8cbc3bbf4..4e6d13509 100644 --- a/pkgs/ui/tailwind.config.js +++ b/pkgs/ui/tailwind.config.js @@ -1,22 +1,16 @@ /** @type {import('tailwindcss').Config} */ module.exports = { + corePlugins: { + preflight: false, + }, content: [ "./src/pages/**/*.{js,ts,jsx,tsx,mdx}", "./src/components/**/*.{js,ts,jsx,tsx,mdx}", "./src/app/**/*.{js,ts,jsx,tsx,mdx}", ], - important: "#root", + important: "#__next", theme: { - extend: { - backgroundImage: { - "gradient-radial": "radial-gradient(var(--tw-gradient-stops))", - "gradient-conic": - "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))", - }, - }, + extend: {}, }, plugins: [], - corePlugins: { - preflight: false, - }, }; From d90f4bc449e6cce10924efe7c6e097dd4ea46da2 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 5 Aug 2023 13:51:33 +0200 Subject: [PATCH 03/36] init: layout --- pkgs/ui/prettier.config.js | 4 ++++ pkgs/ui/src/app/dashboard/page.tsx | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 pkgs/ui/prettier.config.js create mode 100644 pkgs/ui/src/app/dashboard/page.tsx diff --git a/pkgs/ui/prettier.config.js b/pkgs/ui/prettier.config.js new file mode 100644 index 000000000..a0d9c69fd --- /dev/null +++ b/pkgs/ui/prettier.config.js @@ -0,0 +1,4 @@ +module.exports = { + plugins: [require("prettier-plugin-tailwindcss")], + tailwindFunctions: ['clsx', 'cx'], +}; diff --git a/pkgs/ui/src/app/dashboard/page.tsx b/pkgs/ui/src/app/dashboard/page.tsx new file mode 100644 index 000000000..1670f0fa5 --- /dev/null +++ b/pkgs/ui/src/app/dashboard/page.tsx @@ -0,0 +1,25 @@ +import { DashboardCard } from "../../components/card"; +import { Grid } from "@mui/material"; +import { Button } from "@mui/material"; + +export default function Dashboard() { + return ( + + + + + Hallo Mike ! + + + + Server Stats + + + Network Stats + + + + ); +} From d8cb8069f56abff409304e9c7fba07b06a8e866a Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Aug 2023 17:26:24 +0200 Subject: [PATCH 04/36] init: layout with simple side navigation --- pkgs/ui/prettier.config.js | 4 ---- pkgs/ui/src/app/dashboard/page.tsx | 25 ------------------------- 2 files changed, 29 deletions(-) delete mode 100644 pkgs/ui/prettier.config.js delete mode 100644 pkgs/ui/src/app/dashboard/page.tsx diff --git a/pkgs/ui/prettier.config.js b/pkgs/ui/prettier.config.js deleted file mode 100644 index a0d9c69fd..000000000 --- a/pkgs/ui/prettier.config.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - plugins: [require("prettier-plugin-tailwindcss")], - tailwindFunctions: ['clsx', 'cx'], -}; diff --git a/pkgs/ui/src/app/dashboard/page.tsx b/pkgs/ui/src/app/dashboard/page.tsx deleted file mode 100644 index 1670f0fa5..000000000 --- a/pkgs/ui/src/app/dashboard/page.tsx +++ /dev/null @@ -1,25 +0,0 @@ -import { DashboardCard } from "../../components/card"; -import { Grid } from "@mui/material"; -import { Button } from "@mui/material"; - -export default function Dashboard() { - return ( - - - - - Hallo Mike ! - - - - Server Stats - - - Network Stats - - - - ); -} From f77cd7551369cce5d28e7b7ee65c8b2f14cbba1e Mon Sep 17 00:00:00 2001 From: Clan Merge Bot Date: Mon, 7 Aug 2023 00:00:08 +0000 Subject: [PATCH 05/36] update flake lock - 2023-08-07T00:00+00:00 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'disko': 'github:nix-community/disko/4015740375676402a2ee6adebc3c30ea625b9a94' (2023-07-30) → 'github:nix-community/disko/493b347d8fffa6912afb8d89b91703cd40ff6038' (2023-08-06) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9e1960bc196baf6881340d53dccb203a951745a2' (2023-08-01) → 'github:NixOS/nixpkgs/5faab29808a2d72f4ee0c44c8e850e4e6ada972f' (2023-08-05) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 3749ac0e4..8ff664dfe 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1690739034, - "narHash": "sha256-roW02IaiQ3gnEEDMCDWL5YyN+C4nBf/te6vfL7rG0jk=", + "lastModified": 1691339339, + "narHash": "sha256-wNiTX1c3kZy7BSxWodbn+mem1zCx1wIsdDRDFcIfOkc=", "owner": "nix-community", "repo": "disko", - "rev": "4015740375676402a2ee6adebc3c30ea625b9a94", + "rev": "493b347d8fffa6912afb8d89b91703cd40ff6038", "type": "github" }, "original": { @@ -98,11 +98,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1690881714, - "narHash": "sha256-h/nXluEqdiQHs1oSgkOOWF+j8gcJMWhwnZ9PFabN6q0=", + "lastModified": 1691276849, + "narHash": "sha256-RNnrzxhW38SOFIF6TY/WaX7VB3PCkYFEeRE5YZU+wHw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9e1960bc196baf6881340d53dccb203a951745a2", + "rev": "5faab29808a2d72f4ee0c44c8e850e4e6ada972f", "type": "github" }, "original": { From 0533948085d0df47d4b807ba2c7c45fc4f822e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 13:44:15 +0200 Subject: [PATCH 06/36] fix rotating keys with sops --- pkgs/clan-cli/clan_cli/secrets/groups.py | 4 +- pkgs/clan-cli/clan_cli/secrets/machines.py | 4 +- pkgs/clan-cli/clan_cli/secrets/secrets.py | 72 ++++++++++++++++++---- pkgs/clan-cli/clan_cli/secrets/sops.py | 37 ++++++++++- pkgs/clan-cli/clan_cli/secrets/users.py | 17 +---- pkgs/clan-cli/tests/test_secrets.py | 31 ++++++++++ 6 files changed, 131 insertions(+), 34 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/groups.py b/pkgs/clan-cli/clan_cli/secrets/groups.py index 264c43b7d..c94866ebe 100644 --- a/pkgs/clan-cli/clan_cli/secrets/groups.py +++ b/pkgs/clan-cli/clan_cli/secrets/groups.py @@ -102,12 +102,12 @@ def add_group_argument(parser: argparse.ArgumentParser) -> None: def add_secret_command(args: argparse.Namespace) -> None: secrets.allow_member( - secrets.groups_folder(args.group), sops_machines_folder(), args.group + secrets.groups_folder(args.secret), sops_groups_folder(), args.group ) def remove_secret_command(args: argparse.Namespace) -> None: - secrets.disallow_member(secrets.groups_folder(args.group), args.group) + secrets.disallow_member(secrets.groups_folder(args.secret), args.group) def register_groups_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/machines.py b/pkgs/clan-cli/clan_cli/secrets/machines.py index 9f7692aab..d7a2ffb0a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/machines.py +++ b/pkgs/clan-cli/clan_cli/secrets/machines.py @@ -25,12 +25,12 @@ def remove_command(args: argparse.Namespace) -> None: def add_secret_command(args: argparse.Namespace) -> None: secrets.allow_member( - secrets.machines_folder(args.group), sops_machines_folder(), args.machine + secrets.machines_folder(args.secret), sops_machines_folder(), args.machine ) def remove_secret_command(args: argparse.Namespace) -> None: - secrets.disallow_member(secrets.machines_folder(args.group), args.machine) + secrets.disallow_member(secrets.machines_folder(args.secret), args.machine) def register_machines_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 52c62c6f2..9e7344ea1 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -11,8 +11,8 @@ from typing import IO from .. import tty from ..errors import ClanError from ..nix import nix_shell -from .folders import list_objects, sops_secrets_folder -from .sops import SopsKey, encrypt_file, ensure_sops_key, read_key +from .folders import list_objects, sops_secrets_folder, sops_users_folder +from .sops import SopsKey, encrypt_file, ensure_sops_key, read_key, update_keys from .types import VALID_SECRET_NAME, secret_name_type @@ -54,6 +54,9 @@ def set_command(args: argparse.Namespace) -> None: else: encrypt_secret(key, sops_secrets_folder() / args.secret, sys.stdin) + # make sure we add ourselves to the key + allow_member(users_folder(args.secret), sops_users_folder(), key.username) + def remove_command(args: argparse.Namespace) -> None: secret: str = args.secret @@ -67,6 +70,51 @@ def add_secret_argument(parser: argparse.ArgumentParser) -> None: parser.add_argument("secret", help="the name of the secret", type=secret_name_type) +def machines_folder(group: str) -> Path: + return sops_secrets_folder() / group / "machines" + + +def users_folder(group: str) -> Path: + return sops_secrets_folder() / group / "users" + + +def groups_folder(group: str) -> Path: + return sops_secrets_folder() / group / "groups" + + +def collect_keys_for_type(folder: Path) -> list[str]: + if not folder.exists(): + return [] + keys = [] + for p in folder.iterdir(): + if not p.is_symlink(): + continue + try: + target = p.resolve() + except FileNotFoundError: + tty.warn(f"Ignoring broken symlink {p}") + continue + kind = target.parent.name + if folder.name != kind: + tty.warn(f"Expected {p} to point to {folder} but points to {target.parent}") + continue + keys.append(read_key(target)) + return keys + + +def collect_keys_for_path(path: Path) -> list[str]: + keys = [] + keys += collect_keys_for_type(path / "machines") + keys += collect_keys_for_type(path / "users") + groups = path / "groups" + if not groups.is_dir(): + return keys + for group in groups.iterdir(): + keys += collect_keys_for_type(group / "machines") + keys += collect_keys_for_type(group / "users") + return keys + + def allow_member(group_folder: Path, source_folder: Path, name: str) -> None: source = source_folder / name if not source.exists(): @@ -80,12 +128,20 @@ def allow_member(group_folder: Path, source_folder: Path, name: str) -> None: ) os.remove(user_target) user_target.symlink_to(source) + update_keys(group_folder.parent, collect_keys_for_path(group_folder.parent)) def disallow_member(group_folder: Path, name: str) -> None: target = group_folder / name if not target.exists(): raise ClanError(f"{name} does not exist in group in {group_folder}") + + keys = collect_keys_for_path(group_folder.parent) + + if len(keys) < 2: + raise ClanError( + f"Cannot remove {name} from {group_folder.parent.name}. No keys left. Use 'clan secrets remove {name}' to remove the secret." + ) os.remove(target) if len(os.listdir(group_folder)) == 0: @@ -94,17 +150,7 @@ def disallow_member(group_folder: Path, name: str) -> None: if len(os.listdir(group_folder.parent)) == 0: os.rmdir(group_folder.parent) - -def machines_folder(group: str) -> Path: - return sops_secrets_folder() / group / "machines" - - -def users_folder(group: str) -> Path: - return sops_secrets_folder() / group / "users" - - -def groups_folder(group: str) -> Path: - return sops_secrets_folder() / group / "groups" + update_keys(target.parent.parent, collect_keys_for_path(group_folder.parent)) def register_secrets_parser(subparser: argparse._SubParsersAction) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index 61b80f5f6..4d41c5a33 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -2,9 +2,10 @@ import json import os import shutil import subprocess +from contextlib import contextmanager from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO +from typing import IO, Iterator from .. import tty from ..dirs import user_config_dir @@ -14,8 +15,9 @@ from .folders import sops_users_folder class SopsKey: - def __init__(self, pubkey: str) -> None: + def __init__(self, pubkey: str, username: str) -> None: self.pubkey = pubkey + self.username = username def get_public_key(privkey: str) -> str: @@ -51,7 +53,7 @@ def get_user_name(user: str) -> str: def ensure_user(pub_key: str) -> SopsKey: - key = SopsKey(pub_key) + key = SopsKey(pub_key, username="") users_folder = sops_users_folder() # Check if the public key already exists for any user @@ -60,6 +62,7 @@ def ensure_user(pub_key: str) -> SopsKey: if not user.is_dir(): continue if read_key(user) == pub_key: + key.username = user.name return key # Find a unique user name if the public key is not found @@ -76,6 +79,8 @@ def ensure_user(pub_key: str) -> SopsKey: # Add the public key for the user write_key(users_folder / username, pub_key, False) + key.username = username + return key @@ -100,6 +105,32 @@ def ensure_sops_key() -> SopsKey: return ensure_user(get_public_key(path.read_text())) +@contextmanager +def sops_manifest(keys: list[str]) -> Iterator[Path]: + with NamedTemporaryFile(delete=False, mode="w") as manifest: + json.dump( + dict(creation_rules=[dict(key_groups=[dict(age=keys)])]), manifest, indent=2 + ) + manifest.flush() + yield Path(manifest.name) + + +def update_keys(secret_path: Path, keys: list[str]) -> None: + with sops_manifest(keys) as manifest: + cmd = nix_shell( + ["sops"], + [ + "sops", + "--config", + str(manifest), + "updatekeys", + "--yes", + str(secret_path / "secret"), + ], + ) + subprocess.run(cmd, check=True) + + def encrypt_file(secret_path: Path, content: IO[str], keys: list[str]) -> None: folder = secret_path.parent folder.mkdir(parents=True, exist_ok=True) diff --git a/pkgs/clan-cli/clan_cli/secrets/users.py b/pkgs/clan-cli/clan_cli/secrets/users.py index eccf8a444..ef0c29ecd 100644 --- a/pkgs/clan-cli/clan_cli/secrets/users.py +++ b/pkgs/clan-cli/clan_cli/secrets/users.py @@ -29,12 +29,12 @@ def remove_command(args: argparse.Namespace) -> None: def add_secret_command(args: argparse.Namespace) -> None: secrets.allow_member( - secrets.groups_folder(args.group), sops_users_folder(), args.group + secrets.users_folder(args.secret), sops_users_folder(), args.user ) def remove_secret_command(args: argparse.Namespace) -> None: - secrets.disallow_member(secrets.groups_folder(args.group), args.group) + secrets.disallow_member(secrets.users_folder(args.secret), args.user) def register_users_parser(parser: argparse.ArgumentParser) -> None: @@ -74,21 +74,10 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None: ) add_secret_parser.set_defaults(func=add_secret_command) - add_secret_parser = subparser.add_parser( - "add-secret", help="allow a machine to access a secret" - ) - add_secret_parser.add_argument( - "user", help="the name of the group", type=user_name_type - ) - add_secret_parser.add_argument( - "secret", help="the name of the secret", type=secret_name_type - ) - add_secret_parser.set_defaults(func=add_secret_command) - remove_secret_parser = subparser.add_parser( "remove-secret", help="remove a user's access to a secret" ) - add_secret_parser.add_argument( + remove_secret_parser.add_argument( "user", help="the name of the group", type=user_name_type ) remove_secret_parser.add_argument( diff --git a/pkgs/clan-cli/tests/test_secrets.py b/pkgs/clan-cli/tests/test_secrets.py index b9425abf9..311666337 100644 --- a/pkgs/clan-cli/tests/test_secrets.py +++ b/pkgs/clan-cli/tests/test_secrets.py @@ -23,6 +23,9 @@ class SecretCli: PUBKEY = "age1dhwqzkah943xzc34tc3dlmfayyevcmdmxzjezdgdy33euxwf59vsp3vk3c" PRIVKEY = "AGE-SECRET-KEY-1KF8E3SR3TTGL6M476SKF7EEMR4H9NF7ZWYSLJUAK8JX276JC7KUSSURKFK" +PUBKEY_2 = "age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62" +PRIVKEY_2 = "AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ" + def _test_identities( what: str, clan_flake: Path, capsys: pytest.CaptureFixture @@ -123,6 +126,34 @@ def test_secrets( cli.run(["list"]) assert capsys.readouterr().out == "key\n" + cli.run(["machines", "add", "machine1", PUBKEY]) + cli.run(["machines", "add-secret", "machine1", "key"]) + + with mock_env(SOPS_AGE_KEY=PRIVKEY, SOPS_AGE_KEY_FILE=""): + capsys.readouterr() + cli.run(["get", "key"]) + assert capsys.readouterr().out == "foo" + cli.run(["machines", "remove-secret", "machine1", "key"]) + + cli.run(["users", "add", "user1", PUBKEY_2]) + cli.run(["users", "add-secret", "user1", "key"]) + with mock_env(SOPS_AGE_KEY=PRIVKEY_2, SOPS_AGE_KEY_FILE=""): + capsys.readouterr() + cli.run(["get", "key"]) + assert capsys.readouterr().out == "foo" + cli.run(["users", "remove-secret", "user1", "key"]) + + with pytest.raises(ClanError): # does not exist yet + cli.run(["groups", "add-secret", "admin-group", "key"]) + cli.run(["groups", "add-user", "admin-group", "user1"]) + cli.run(["groups", "add-secret", "admin-group", "key"]) + + with mock_env(SOPS_AGE_KEY=PRIVKEY_2, SOPS_AGE_KEY_FILE=""): + capsys.readouterr() + cli.run(["get", "key"]) + assert capsys.readouterr().out == "foo" + cli.run(["groups", "remove-secret", "admin-group", "key"]) + cli.run(["remove", "key"]) capsys.readouterr() # empty the buffer From 3ea17e156665f01a6b53670aae8c17fbae8b279e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 13:51:00 +0200 Subject: [PATCH 07/36] devshell: fix linking pre-commit --- devShell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devShell.nix b/devShell.nix index 4a71b7afa..ae61a5636 100644 --- a/devShell.nix +++ b/devShell.nix @@ -15,7 +15,7 @@ self'.packages.merge-after-ci ]; shellHook = '' - ln -sf ../../scripts/pre-commit .git/hooks/pre-commit + ln -sf ../../scripts/pre-commit "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ''; }; }; From fbd1c3cff48e4fb4dcd8f7d264042993552ff3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 13:53:08 +0200 Subject: [PATCH 08/36] ci: actually check pull_request merge instead of push --- .gitea/workflows/check.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitea/workflows/check.yaml b/.gitea/workflows/check.yaml index 314910485..73083a8ee 100644 --- a/.gitea/workflows/check.yaml +++ b/.gitea/workflows/check.yaml @@ -1,6 +1,8 @@ name: build on: + pull_request: push: + branches: main jobs: test: runs-on: nix From 6d1de943e1e85c7b13c56a7587c5fee2b74fe83e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 14:03:26 +0200 Subject: [PATCH 09/36] move out non-toplevel flake modules from flake.nix --- flake.nix | 5 ----- pkgs/flake-module.nix | 6 ++++++ templates/flake-module.nix | 5 ++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/flake.nix b/flake.nix index cb5feadd4..c52398e17 100644 --- a/flake.nix +++ b/flake.nix @@ -25,14 +25,9 @@ ./devShell.nix ./formatter.nix ./templates/flake-module.nix - ./templates/python-project/flake-module.nix ./pkgs/flake-module.nix - ./pkgs/clan-cli/flake-module.nix - ./pkgs/installer/flake-module.nix - ./pkgs/ui/flake-module.nix - ./lib/flake-module.nix ({ self, lib, ... }: { flake.nixosModules = lib.mapAttrs (_: nix: { imports = [ nix ]; }) (self.lib.findNixFiles ./nixosModules); diff --git a/pkgs/flake-module.nix b/pkgs/flake-module.nix index d0a2f0613..d0d9aae1c 100644 --- a/pkgs/flake-module.nix +++ b/pkgs/flake-module.nix @@ -1,4 +1,10 @@ { ... }: { + imports = [ + ./clan-cli/flake-module.nix + ./installer/flake-module.nix + ./ui/flake-module.nix + ]; + perSystem = { pkgs, config, ... }: { packages = { tea-create-pr = pkgs.callPackage ./tea-create-pr { }; diff --git a/templates/flake-module.nix b/templates/flake-module.nix index 22b1070dd..a7149430d 100644 --- a/templates/flake-module.nix +++ b/templates/flake-module.nix @@ -1,4 +1,7 @@ -{ ... }: { +{ + imports = [ + ./python-project/flake-module.nix + ]; flake.templates = { new-clan = { description = "Initialize a new clan flake"; From 4296ef92a93d7b804d5bd62a732db0fd4a4061e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 14:11:04 +0200 Subject: [PATCH 10/36] rename clan to clan-cli It's already confusing enough to have so many things called clan-cli --- pkgs/clan-cli/.envrc | 2 +- pkgs/clan-cli/flake-module.nix | 10 +++++----- pkgs/clan-cli/shell.nix | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkgs/clan-cli/.envrc b/pkgs/clan-cli/.envrc index 359803c60..00f84d526 100644 --- a/pkgs/clan-cli/.envrc +++ b/pkgs/clan-cli/.envrc @@ -7,4 +7,4 @@ if type nix_direnv_watch_file &>/dev/null; then else direnv watch flake-module.nix fi -use flake .#clan --builders '' +use flake .#clan-cli --builders '' diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index dc603fbb0..4a0370344 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -1,15 +1,15 @@ { self, ... }: { perSystem = { self', pkgs, ... }: { - devShells.clan = pkgs.callPackage ./shell.nix { + devShells.clan-cli = pkgs.callPackage ./shell.nix { inherit self; - inherit (self'.packages) clan; + inherit (self'.packages) clan-cli; }; packages = { - clan = pkgs.python3.pkgs.callPackage ./default.nix { + clan-cli = pkgs.python3.pkgs.callPackage ./default.nix { inherit self; zerotierone = self'.packages.zerotierone; }; - default = self'.packages.clan; + default = self'.packages.clan-cli; ## Optional dependencies for clan cli, we re-expose them here to make sure they all build. inherit (pkgs) @@ -27,7 +27,7 @@ ## End optional dependencies }; - checks = self'.packages.clan.tests // { + checks = self'.packages.clan-cli.tests // { # check if the `clan config` example jsonschema and data is valid clan-config-example-schema-valid = pkgs.runCommand "clan-config-example-schema-valid" { } '' echo "Checking that example-schema.json is valid" diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index d3c658e8b..cc6490a7c 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -1,9 +1,9 @@ -{ self, clan, pkgs }: +{ self, clan-cli, pkgs }: let pythonWithDeps = pkgs.python3.withPackages ( ps: - clan.propagatedBuildInputs - ++ clan.devDependencies + clan-cli.propagatedBuildInputs + ++ clan-cli.devDependencies ++ [ ps.pip ] From 4cf82f35961c74eda8487004d5fd89418e35de18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 14:20:45 +0200 Subject: [PATCH 11/36] also rename project/pypi name --- pkgs/clan-cli/default.nix | 2 +- pkgs/clan-cli/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 67aefb4f3..f9623c890 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -31,7 +31,7 @@ let checkPython = python3.withPackages (_ps: dependencies ++ testDependencies); in python3.pkgs.buildPythonPackage { - name = "clan"; + name = "clan-cli"; src = lib.cleanSource ./.; format = "pyproject"; nativeBuildInputs = [ diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 38ad09c74..f84481b6e 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "setuptools" ] build-backend = "setuptools.build_meta" [project] -name = "clan" +name = "clan-cli" description = "cLAN CLI tool" dynamic = [ "version" ] scripts = { clan = "clan_cli:main" } From 1d1452ddd59b82a39cfb2cff8de36c5444578de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 15:48:19 +0200 Subject: [PATCH 12/36] add import-sops command to secrets --- pkgs/clan-cli/clan_cli/secrets/__init__.py | 4 ++ pkgs/clan-cli/clan_cli/secrets/import_sops.py | 51 +++++++++++++++++++ pkgs/clan-cli/clan_cli/secrets/secrets.py | 26 ++++------ pkgs/clan-cli/clan_cli/secrets/sops.py | 17 +++++-- 4 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 pkgs/clan-cli/clan_cli/secrets/import_sops.py diff --git a/pkgs/clan-cli/clan_cli/secrets/__init__.py b/pkgs/clan-cli/clan_cli/secrets/__init__.py index ff76ee00f..8bb0efebf 100644 --- a/pkgs/clan-cli/clan_cli/secrets/__init__.py +++ b/pkgs/clan-cli/clan_cli/secrets/__init__.py @@ -2,6 +2,7 @@ import argparse from .groups import register_groups_parser +from .import_sops import register_import_sops_parser from .machines import register_machines_parser from .secrets import register_secrets_parser from .users import register_users_parser @@ -25,4 +26,7 @@ def register_parser(parser: argparse.ArgumentParser) -> None: machines_parser = subparser.add_parser("machines", help="manage machines") register_machines_parser(machines_parser) + import_sops_parser = subparser.add_parser("import-sops", help="import a sops file") + register_import_sops_parser(import_sops_parser) + register_secrets_parser(subparser) diff --git a/pkgs/clan-cli/clan_cli/secrets/import_sops.py b/pkgs/clan-cli/clan_cli/secrets/import_sops.py new file mode 100644 index 000000000..36c369dd0 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/secrets/import_sops.py @@ -0,0 +1,51 @@ +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from ..errors import ClanError +from ..nix import nix_shell +from .secrets import encrypt_secret + + +def import_sops(args: argparse.Namespace) -> None: + file = Path(args.sops_file) + file_type = file.suffix + + try: + file.read_text() + except OSError as e: + raise ClanError(f"Could not read file {file}: {e}") from e + if file_type == ".yaml": + cmd = ["sops"] + if args.input_type: + cmd += ["--input-type", args.input_type] + cmd += ["--output-type", "json", "--decrypt", args.sops_file] + cmd = nix_shell(["sops"], cmd) + try: + res = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) + except subprocess.CalledProcessError as e: + raise ClanError(f"Could not import sops file {file}: {e}") from e + secrets = json.loads(res.stdout) + for k, v in secrets.items(): + if not isinstance(v, str): + print( + f"WARNING: {k} is not a string but {type(v)}, skipping", + file=sys.stderr, + ) + encrypt_secret(k, v) + + +def register_import_sops_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "sops_file", + type=str, + help="the sops file to import (- for stdin)", + ) + parser.add_argument( + "input_type", + type=str, + help="the input type of the sops file (yaml, json, ...)", + ) + parser.set_defaults(func=import_sops) diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 9e7344ea1..7a821a5db 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -2,17 +2,15 @@ import argparse import getpass import os import shutil -import subprocess import sys from io import StringIO from pathlib import Path -from typing import IO +from typing import IO, Union from .. import tty from ..errors import ClanError -from ..nix import nix_shell from .folders import list_objects, sops_secrets_folder, sops_users_folder -from .sops import SopsKey, encrypt_file, ensure_sops_key, read_key, update_keys +from .sops import decrypt_file, encrypt_file, ensure_sops_key, read_key, update_keys from .types import VALID_SECRET_NAME, secret_name_type @@ -28,12 +26,11 @@ def get_command(args: argparse.Namespace) -> None: secret_path = sops_secrets_folder() / secret / "secret" if not secret_path.exists(): raise ClanError(f"Secret '{secret}' does not exist") - cmd = nix_shell(["sops"], ["sops", "--decrypt", str(secret_path)]) - res = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True) - print(res.stdout, end="") + print(decrypt_file(secret_path), end="") -def encrypt_secret(key: SopsKey, secret: Path, value: IO[str]) -> None: +def encrypt_secret(secret: Path, value: Union[IO[str], str]) -> None: + key = ensure_sops_key() keys = set([key.pubkey]) for kind in ["users", "machines", "groups"]: if not (sops_secrets_folder() / kind).is_dir(): @@ -42,20 +39,19 @@ def encrypt_secret(key: SopsKey, secret: Path, value: IO[str]) -> None: keys.add(k) encrypt_file(secret / "secret", value, list(sorted(keys))) + # make sure we add ourselves to the key + allow_member(users_folder(secret.name), sops_users_folder(), key.username) + def set_command(args: argparse.Namespace) -> None: - key = ensure_sops_key() secret_value = os.environ.get("SOPS_NIX_SECRET") if secret_value: - encrypt_secret(key, sops_secrets_folder() / args.secret, StringIO(secret_value)) + encrypt_secret(sops_secrets_folder() / args.secret, StringIO(secret_value)) elif tty.is_interactive(): secret = getpass.getpass(prompt="Paste your secret: ") - encrypt_secret(key, sops_secrets_folder() / args.secret, StringIO(secret)) + encrypt_secret(sops_secrets_folder() / args.secret, StringIO(secret)) else: - encrypt_secret(key, sops_secrets_folder() / args.secret, sys.stdin) - - # make sure we add ourselves to the key - allow_member(users_folder(args.secret), sops_users_folder(), key.username) + encrypt_secret(sops_secrets_folder() / args.secret, sys.stdin) def remove_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index 4d41c5a33..aee7f90f3 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -5,7 +5,7 @@ import subprocess from contextlib import contextmanager from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO, Iterator +from typing import IO, Iterator, Union from .. import tty from ..dirs import user_config_dir @@ -131,7 +131,9 @@ def update_keys(secret_path: Path, keys: list[str]) -> None: subprocess.run(cmd, check=True) -def encrypt_file(secret_path: Path, content: IO[str], keys: list[str]) -> None: +def encrypt_file( + secret_path: Path, content: Union[IO[str], str], keys: list[str] +) -> None: folder = secret_path.parent folder.mkdir(parents=True, exist_ok=True) @@ -139,7 +141,10 @@ def encrypt_file(secret_path: Path, content: IO[str], keys: list[str]) -> None: with NamedTemporaryFile(delete=False) as f: try: with open(f.name, "w") as fd: - shutil.copyfileobj(content, fd) + if isinstance(content, str): + fd.write(content) + else: + shutil.copyfileobj(content, fd) args = ["sops"] for key in keys: args.extend(["--age", key]) @@ -157,6 +162,12 @@ def encrypt_file(secret_path: Path, content: IO[str], keys: list[str]) -> None: pass +def decrypt_file(secret_path: Path) -> str: + cmd = nix_shell(["sops"], ["sops", "--decrypt", str(secret_path)]) + res = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True) + return res.stdout + + def write_key(path: Path, publickey: str, overwrite: bool) -> None: path.mkdir(parents=True, exist_ok=True) try: From d8c2df2e72bdf4e8af71494e5de11117e24a629f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 16:28:38 +0200 Subject: [PATCH 13/36] add test for import-sops command --- pkgs/clan-cli/clan_cli/secrets/import_sops.py | 15 ++++++------ pkgs/clan-cli/tests/conftest.py | 2 +- pkgs/clan-cli/tests/data/secrets.yaml | 23 +++++++++++++++++++ pkgs/clan-cli/tests/root.py | 22 ++++++++++++++++++ pkgs/clan-cli/tests/test_secrets.py | 18 ++++++++++++--- 5 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 pkgs/clan-cli/tests/data/secrets.yaml create mode 100644 pkgs/clan-cli/tests/root.py diff --git a/pkgs/clan-cli/clan_cli/secrets/import_sops.py b/pkgs/clan-cli/clan_cli/secrets/import_sops.py index 36c369dd0..4bce9a344 100644 --- a/pkgs/clan-cli/clan_cli/secrets/import_sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/import_sops.py @@ -6,7 +6,7 @@ from pathlib import Path from ..errors import ClanError from ..nix import nix_shell -from .secrets import encrypt_secret +from .secrets import encrypt_secret, sops_secrets_folder def import_sops(args: argparse.Namespace) -> None: @@ -34,18 +34,19 @@ def import_sops(args: argparse.Namespace) -> None: f"WARNING: {k} is not a string but {type(v)}, skipping", file=sys.stderr, ) - encrypt_secret(k, v) + continue + encrypt_secret(sops_secrets_folder() / k, v) def register_import_sops_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--input_type", + type=str, + help="the input type of the sops file (yaml, json, ...)", + ) parser.add_argument( "sops_file", type=str, help="the sops file to import (- for stdin)", ) - parser.add_argument( - "input_type", - type=str, - help="the input type of the sops file (yaml, json, ...)", - ) parser.set_defaults(func=import_sops) diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index 14a0cf031..481347645 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -3,4 +3,4 @@ import sys sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) -pytest_plugins = ["temporary_dir", "clan_flake"] +pytest_plugins = ["temporary_dir", "clan_flake", "root"] diff --git a/pkgs/clan-cli/tests/data/secrets.yaml b/pkgs/clan-cli/tests/data/secrets.yaml new file mode 100644 index 000000000..3bc636597 --- /dev/null +++ b/pkgs/clan-cli/tests/data/secrets.yaml @@ -0,0 +1,23 @@ +secret-key: ENC[AES256_GCM,data:gjX4OmCUdd3TlA4p,iv:3yZVpyd6FqkITQY0nU2M1iubmzvkR6PfkK2m/s6nQh8=,tag:Abgp9xkiFFylZIyAlap6Ew==,type:str] +nested: + secret-key: ENC[AES256_GCM,data:iUMgDhhIjwvd7wL4,iv:jiJIrh12dSu/sXX+z9ITVoEMNDMjwIlFBnyv40oN4LE=,tag:G9VmAa66Km1sc7JEhW5AvA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0eWdRVjlydXlXOVZFQ3lO + bzU1eG9Iam5Ka29Sdlo0cHJ4b1R6bjdNSzBjCkgwRndCbWZQWHlDU0x1cWRmaGVt + N29lbjR6UjN0L2RhaXEzSG9zQmRsZGsKLS0tIEdsdWgxSmZwU3BWUDVxVWRSSC9M + eVZ6bjgwZnR2TTM5MkRYZWNFSFplQWsKmSzv12/dftL9jx2y35UZUGVK6xWdatE8 + BGJiCvMlp0BQNrh2s/+YaEaBa48w8LL79U/XJnEZ+ZUwxmlbSTn6Hg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2023-08-08T14:27:20Z" + mac: ENC[AES256_GCM,data:iRWWX+L5Q5nKn3fBCLaWoz/mvqGnNnRd93gJmYXDZbRjFoHa9IFJZst5QDIDa1ZRYUe6G0/+lV5SBi+vwRm1pHysJ3c0ZWYjBP+e1jw3jLXxLV5gACsDC8by+6rFUCho0Xgu+Nqu2ehhNenjQQnCvDH5ivWbW70KFT5ynNgR9Tw=,iv:RYnnbLMC/hNfMwWPreMq9uvY0khajwQTZENO/P34ckY=,tag:Xi1PS5vM1c+sRkroHkPn1Q==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.3 diff --git a/pkgs/clan-cli/tests/root.py b/pkgs/clan-cli/tests/root.py new file mode 100644 index 000000000..5855b523e --- /dev/null +++ b/pkgs/clan-cli/tests/root.py @@ -0,0 +1,22 @@ +from pathlib import Path + +import pytest + +TEST_ROOT = Path(__file__).parent.resolve() +PROJECT_ROOT = TEST_ROOT.parent + + +@pytest.fixture +def project_root() -> Path: + """ + Root directory of the tests + """ + return PROJECT_ROOT + + +@pytest.fixture +def test_root() -> Path: + """ + Root directory of the tests + """ + return TEST_ROOT diff --git a/pkgs/clan-cli/tests/test_secrets.py b/pkgs/clan-cli/tests/test_secrets.py index 311666337..2e8ca612c 100644 --- a/pkgs/clan-cli/tests/test_secrets.py +++ b/pkgs/clan-cli/tests/test_secrets.py @@ -104,9 +104,7 @@ def test_groups(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: assert len(groups) == 0 -def test_secrets( - clan_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: cli = SecretCli() capsys.readouterr() # empty the buffer cli.run(["list"]) @@ -159,3 +157,17 @@ def test_secrets( capsys.readouterr() # empty the buffer cli.run(["list"]) assert capsys.readouterr().out == "" + + +def test_import_sops( + test_root: Path, clan_flake: Path, capsys: pytest.CaptureFixture +) -> None: + cli = SecretCli() + + with mock_env(SOPS_AGE_KEY=PRIVKEY_2): + # To edit: + # SOPS_AGE_KEY=AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ sops --age age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 ./data/secrets.yaml + cli.run(["import-sops", str(test_root.joinpath("data", "secrets.yaml"))]) + capsys.readouterr() + cli.run(["get", "secret-key"]) + assert capsys.readouterr().out == "secret-value" From 35f2d6a76e440fcb05c206a1b65ae37051b68305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 16:40:32 +0200 Subject: [PATCH 14/36] add dummy sops manifest --- pkgs/clan-cli/clan_cli/secrets/sops.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index aee7f90f3..dc9595b2b 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -138,14 +138,17 @@ def encrypt_file( folder.mkdir(parents=True, exist_ok=True) # hopefully /tmp is written to an in-memory file to avoid leaking secrets - with NamedTemporaryFile(delete=False) as f: + with NamedTemporaryFile(delete=False) as dummy_manifest_file, NamedTemporaryFile( + delete=False + ) as f: try: with open(f.name, "w") as fd: if isinstance(content, str): fd.write(content) else: shutil.copyfileobj(content, fd) - args = ["sops"] + # we pass an empty manifest to pick up existing configuration of the user + args = ["sops", "--config", dummy_manifest_file.name] for key in keys: args.extend(["--age", key]) args.extend(["-i", "--encrypt", str(f.name)]) From efa0546da55d2eea59958feb6439bc8cba41c6c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 17:24:40 +0200 Subject: [PATCH 15/36] sops: make symlinks relative --- pkgs/clan-cli/clan_cli/secrets/groups.py | 2 +- pkgs/clan-cli/clan_cli/secrets/secrets.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/groups.py b/pkgs/clan-cli/clan_cli/secrets/groups.py index c94866ebe..1deeb46fc 100644 --- a/pkgs/clan-cli/clan_cli/secrets/groups.py +++ b/pkgs/clan-cli/clan_cli/secrets/groups.py @@ -60,7 +60,7 @@ def add_member(group_folder: Path, source_folder: Path, name: str) -> None: f"Cannot add user {name}. {user_target} exists but is not a symlink" ) os.remove(user_target) - user_target.symlink_to(source) + user_target.symlink_to(os.path.relpath(source, user_target.parent)) def remove_member(group_folder: Path, name: str) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 7a821a5db..319b92e63 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -123,7 +123,8 @@ def allow_member(group_folder: Path, source_folder: Path, name: str) -> None: f"Cannot add user {name}. {user_target} exists but is not a symlink" ) os.remove(user_target) - user_target.symlink_to(source) + + user_target.symlink_to(os.path.relpath(source, user_target.parent)) update_keys(group_folder.parent, collect_keys_for_path(group_folder.parent)) From 6f9aaef56a191dfb085a6ef9ee77a6da1a154f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 18:46:37 +0200 Subject: [PATCH 16/36] add option to import sops secrets with groups,users,machines,prefixes --- pkgs/clan-cli/clan_cli/secrets/import_sops.py | 47 ++++++++- pkgs/clan-cli/clan_cli/secrets/secrets.py | 99 ++++++++++++------- pkgs/clan-cli/tests/test_secrets.py | 21 +++- 3 files changed, 127 insertions(+), 40 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/import_sops.py b/pkgs/clan-cli/clan_cli/secrets/import_sops.py index 4bce9a344..a83556063 100644 --- a/pkgs/clan-cli/clan_cli/secrets/import_sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/import_sops.py @@ -29,20 +29,61 @@ def import_sops(args: argparse.Namespace) -> None: raise ClanError(f"Could not import sops file {file}: {e}") from e secrets = json.loads(res.stdout) for k, v in secrets.items(): + k = args.prefix + k if not isinstance(v, str): print( f"WARNING: {k} is not a string but {type(v)}, skipping", file=sys.stderr, ) continue - encrypt_secret(sops_secrets_folder() / k, v) + if (sops_secrets_folder() / k).exists(): + print( + f"WARNING: {k} already exists, skipping", + file=sys.stderr, + ) + continue + encrypt_secret( + sops_secrets_folder() / k, + v, + add_groups=args.group, + add_machines=args.machine, + add_users=args.user, + ) def register_import_sops_parser(parser: argparse.ArgumentParser) -> None: parser.add_argument( - "--input_type", + "--input-type", type=str, - help="the input type of the sops file (yaml, json, ...)", + default=None, + help="the input type of the sops file (yaml, json, ...). If not specified, it will be guessed from the file extension", + ) + parser.add_argument( + "--group", + type=str, + action="append", + default=[], + help="the group to import the secrets to", + ) + parser.add_argument( + "--machine", + type=str, + action="append", + default=[], + help="the machine to import the secrets to", + ) + parser.add_argument( + "--user", + type=str, + action="append", + default=[], + help="the user to import the secrets to", + ) + parser.add_argument( + "--prefix", + type=str, + default="", + help="the prefix to use for the secret names", ) parser.add_argument( "sops_file", diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 319b92e63..420fc6726 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -9,50 +9,52 @@ from typing import IO, Union from .. import tty from ..errors import ClanError -from .folders import list_objects, sops_secrets_folder, sops_users_folder +from .folders import ( + list_objects, + sops_groups_folder, + sops_machines_folder, + sops_secrets_folder, + sops_users_folder, +) from .sops import decrypt_file, encrypt_file, ensure_sops_key, read_key, update_keys from .types import VALID_SECRET_NAME, secret_name_type -def list_command(args: argparse.Namespace) -> None: - list_objects( - sops_secrets_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None - ) - - -def get_command(args: argparse.Namespace) -> None: - secret: str = args.secret - ensure_sops_key() - secret_path = sops_secrets_folder() / secret / "secret" - if not secret_path.exists(): - raise ClanError(f"Secret '{secret}' does not exist") - print(decrypt_file(secret_path), end="") - - -def encrypt_secret(secret: Path, value: Union[IO[str], str]) -> None: +def encrypt_secret( + secret: Path, + value: Union[IO[str], str], + add_users: list[str] = [], + add_machines: list[str] = [], + add_groups: list[str] = [], +) -> None: key = ensure_sops_key() - keys = set([key.pubkey]) + keys = set([]) + + for user in add_users: + allow_member(users_folder(secret.name), sops_users_folder(), user, False) + + for machine in add_machines: + allow_member( + machines_folder(secret.name), sops_machines_folder(), machine, False + ) + + for group in add_groups: + allow_member(groups_folder(secret.name), sops_groups_folder(), group, False) + for kind in ["users", "machines", "groups"]: if not (sops_secrets_folder() / kind).is_dir(): continue k = read_key(sops_secrets_folder() / kind) keys.add(k) + + if key.pubkey not in keys: + keys.add(key.pubkey) + allow_member( + users_folder(secret.name), sops_users_folder(), key.username, False + ) + encrypt_file(secret / "secret", value, list(sorted(keys))) - # make sure we add ourselves to the key - allow_member(users_folder(secret.name), sops_users_folder(), key.username) - - -def set_command(args: argparse.Namespace) -> None: - secret_value = os.environ.get("SOPS_NIX_SECRET") - if secret_value: - encrypt_secret(sops_secrets_folder() / args.secret, StringIO(secret_value)) - elif tty.is_interactive(): - secret = getpass.getpass(prompt="Paste your secret: ") - encrypt_secret(sops_secrets_folder() / args.secret, StringIO(secret)) - else: - encrypt_secret(sops_secrets_folder() / args.secret, sys.stdin) - def remove_command(args: argparse.Namespace) -> None: secret: str = args.secret @@ -111,7 +113,9 @@ def collect_keys_for_path(path: Path) -> list[str]: return keys -def allow_member(group_folder: Path, source_folder: Path, name: str) -> None: +def allow_member( + group_folder: Path, source_folder: Path, name: str, do_update_keys: bool = True +) -> None: source = source_folder / name if not source.exists(): raise ClanError(f"{name} does not exist in {source_folder}") @@ -125,7 +129,8 @@ def allow_member(group_folder: Path, source_folder: Path, name: str) -> None: os.remove(user_target) user_target.symlink_to(os.path.relpath(source, user_target.parent)) - update_keys(group_folder.parent, collect_keys_for_path(group_folder.parent)) + if do_update_keys: + update_keys(group_folder.parent, collect_keys_for_path(group_folder.parent)) def disallow_member(group_folder: Path, name: str) -> None: @@ -150,6 +155,32 @@ def disallow_member(group_folder: Path, name: str) -> None: update_keys(target.parent.parent, collect_keys_for_path(group_folder.parent)) +def list_command(args: argparse.Namespace) -> None: + list_objects( + sops_secrets_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None + ) + + +def get_command(args: argparse.Namespace) -> None: + secret: str = args.secret + ensure_sops_key() + secret_path = sops_secrets_folder() / secret / "secret" + if not secret_path.exists(): + raise ClanError(f"Secret '{secret}' does not exist") + print(decrypt_file(secret_path), end="") + + +def set_command(args: argparse.Namespace) -> None: + secret_value = os.environ.get("SOPS_NIX_SECRET") + if secret_value: + encrypt_secret(sops_secrets_folder() / args.secret, StringIO(secret_value)) + elif tty.is_interactive(): + secret = getpass.getpass(prompt="Paste your secret: ") + encrypt_secret(sops_secrets_folder() / args.secret, StringIO(secret)) + else: + encrypt_secret(sops_secrets_folder() / args.secret, sys.stdin) + + def register_secrets_parser(subparser: argparse._SubParsersAction) -> None: parser_list = subparser.add_parser("list", help="list secrets") parser_list.set_defaults(func=list_command) diff --git a/pkgs/clan-cli/tests/test_secrets.py b/pkgs/clan-cli/tests/test_secrets.py index 2e8ca612c..d944f816a 100644 --- a/pkgs/clan-cli/tests/test_secrets.py +++ b/pkgs/clan-cli/tests/test_secrets.py @@ -26,6 +26,9 @@ PRIVKEY = "AGE-SECRET-KEY-1KF8E3SR3TTGL6M476SKF7EEMR4H9NF7ZWYSLJUAK8JX276JC7KUSS PUBKEY_2 = "age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62" PRIVKEY_2 = "AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ" +PUBKEY_3 = "age1dhuh9xtefhgpr2sjjf7gmp9q2pr37z92rv4wsadxuqdx48989g7qj552qp" +PRIVKEY_3 = "AGE-SECRET-KEY-169N3FT32VNYQ9WYJMLUSVTMA0TTZGVJF7YZWS8AHTWJ5RR9VGR7QCD8SKF" + def _test_identities( what: str, clan_flake: Path, capsys: pytest.CaptureFixture @@ -110,11 +113,11 @@ def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: cli.run(["list"]) assert capsys.readouterr().out == "" - with pytest.raises(ClanError): # does not exist yet - cli.run(["get", "nonexisting"]) with mock_env( SOPS_NIX_SECRET="foo", SOPS_AGE_KEY_FILE=str(clan_flake / ".." / "age.key") ): + with pytest.raises(ClanError): # does not exist yet + cli.run(["get", "nonexisting"]) cli.run(["set", "key"]) capsys.readouterr() cli.run(["get", "key"]) @@ -165,9 +168,21 @@ def test_import_sops( cli = SecretCli() with mock_env(SOPS_AGE_KEY=PRIVKEY_2): + cli.run(["machines", "add", "machine1", PUBKEY]) + cli.run(["users", "add", "user1", PUBKEY_3]) + # To edit: # SOPS_AGE_KEY=AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ sops --age age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 ./data/secrets.yaml - cli.run(["import-sops", str(test_root.joinpath("data", "secrets.yaml"))]) + cli.run( + [ + "import-sops", + "--user", + "user1", + "--machine", + "machine1", + str(test_root.joinpath("data", "secrets.yaml")), + ] + ) capsys.readouterr() cli.run(["get", "secret-key"]) assert capsys.readouterr().out == "secret-value" From 504adb4f6d36ed097b5d1766647780c6db25ab6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 19:03:45 +0200 Subject: [PATCH 17/36] make error message nicer when members are not found --- pkgs/clan-cli/clan_cli/secrets/groups.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/groups.py b/pkgs/clan-cli/clan_cli/secrets/groups.py index 1deeb46fc..c2763090a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/groups.py +++ b/pkgs/clan-cli/clan_cli/secrets/groups.py @@ -48,10 +48,21 @@ def list_command(args: argparse.Namespace) -> None: print(f) +def list_directory(directory: Path) -> str: + if not directory.exists(): + return "{directory} does not exist" + msg = f"\n{directory} contains:" + for f in directory.iterdir(): + msg += f"\n {f}" + return msg + + def add_member(group_folder: Path, source_folder: Path, name: str) -> None: source = source_folder / name if not source.exists(): - raise ClanError(f"{name} does not exist in {source_folder}") + msg = f"{name} does not exist in {source_folder}" + msg += list_directory(source_folder) + raise ClanError(msg) group_folder.mkdir(parents=True, exist_ok=True) user_target = group_folder / name if user_target.exists(): @@ -66,7 +77,9 @@ def add_member(group_folder: Path, source_folder: Path, name: str) -> None: def remove_member(group_folder: Path, name: str) -> None: target = group_folder / name if not target.exists(): - raise ClanError(f"{name} does not exist in group in {group_folder}") + msg = f"{name} does not exist in group in {group_folder}" + msg += list_directory(group_folder) + raise ClanError(msg) os.remove(target) if len(os.listdir(group_folder)) == 0: From 5ee620b77bf62bb03bf3ccc99c52b9a63cfa3e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 19:07:38 +0200 Subject: [PATCH 18/36] only print names when listing existing users --- pkgs/clan-cli/clan_cli/secrets/groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/groups.py b/pkgs/clan-cli/clan_cli/secrets/groups.py index c2763090a..a5f8acd8e 100644 --- a/pkgs/clan-cli/clan_cli/secrets/groups.py +++ b/pkgs/clan-cli/clan_cli/secrets/groups.py @@ -53,7 +53,7 @@ def list_directory(directory: Path) -> str: return "{directory} does not exist" msg = f"\n{directory} contains:" for f in directory.iterdir(): - msg += f"\n {f}" + msg += f"\n {f.name}" return msg From 298853290938786d4eb165d534e2dab1dc752af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 8 Aug 2023 19:40:35 +0200 Subject: [PATCH 19/36] allow to set groups/admins/users when setting secrets --- pkgs/clan-cli/clan_cli/secrets/secrets.py | 41 ++++++++++++++++++----- pkgs/clan-cli/tests/test_secrets.py | 18 +++++++++- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 420fc6726..db869b07c 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -3,7 +3,6 @@ import getpass import os import shutil import sys -from io import StringIO from pathlib import Path from typing import IO, Union @@ -171,14 +170,19 @@ def get_command(args: argparse.Namespace) -> None: def set_command(args: argparse.Namespace) -> None: - secret_value = os.environ.get("SOPS_NIX_SECRET") - if secret_value: - encrypt_secret(sops_secrets_folder() / args.secret, StringIO(secret_value)) + env_value = os.environ.get("SOPS_NIX_SECRET") + secret_value: Union[str, IO[str]] = sys.stdin + if env_value: + secret_value = env_value elif tty.is_interactive(): - secret = getpass.getpass(prompt="Paste your secret: ") - encrypt_secret(sops_secrets_folder() / args.secret, StringIO(secret)) - else: - encrypt_secret(sops_secrets_folder() / args.secret, sys.stdin) + secret_value = getpass.getpass(prompt="Paste your secret: ") + encrypt_secret( + sops_secrets_folder() / args.secret, + secret_value, + args.user, + args.machine, + args.group, + ) def register_secrets_parser(subparser: argparse._SubParsersAction) -> None: @@ -191,6 +195,27 @@ def register_secrets_parser(subparser: argparse._SubParsersAction) -> None: parser_set = subparser.add_parser("set", help="set a secret") add_secret_argument(parser_set) + parser_set.add_argument( + "--group", + type=str, + action="append", + default=[], + help="the group to import the secrets to", + ) + parser_set.add_argument( + "--machine", + type=str, + action="append", + default=[], + help="the machine to import the secrets to", + ) + parser_set.add_argument( + "--user", + type=str, + action="append", + default=[], + help="the user to import the secrets to", + ) parser_set.set_defaults(func=set_command) parser_delete = subparser.add_parser("remove", help="remove a secret") diff --git a/pkgs/clan-cli/tests/test_secrets.py b/pkgs/clan-cli/tests/test_secrets.py index d944f816a..50c9f7b70 100644 --- a/pkgs/clan-cli/tests/test_secrets.py +++ b/pkgs/clan-cli/tests/test_secrets.py @@ -122,6 +122,11 @@ def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: capsys.readouterr() cli.run(["get", "key"]) assert capsys.readouterr().out == "foo" + capsys.readouterr() + cli.run(["users", "list"]) + users = capsys.readouterr().out.rstrip().split("\n") + assert len(users) == 1, f"users: {users}" + owner = users[0] capsys.readouterr() # empty the buffer cli.run(["list"]) @@ -147,8 +152,12 @@ def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: with pytest.raises(ClanError): # does not exist yet cli.run(["groups", "add-secret", "admin-group", "key"]) cli.run(["groups", "add-user", "admin-group", "user1"]) + cli.run(["groups", "add-user", "admin-group", owner]) cli.run(["groups", "add-secret", "admin-group", "key"]) + capsys.readouterr() # empty the buffer + cli.run(["set", "--group", "admin-group", "key2"]) + with mock_env(SOPS_AGE_KEY=PRIVKEY_2, SOPS_AGE_KEY_FILE=""): capsys.readouterr() cli.run(["get", "key"]) @@ -156,6 +165,7 @@ def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: cli.run(["groups", "remove-secret", "admin-group", "key"]) cli.run(["remove", "key"]) + cli.run(["remove", "key2"]) capsys.readouterr() # empty the buffer cli.run(["list"]) @@ -169,7 +179,8 @@ def test_import_sops( with mock_env(SOPS_AGE_KEY=PRIVKEY_2): cli.run(["machines", "add", "machine1", PUBKEY]) - cli.run(["users", "add", "user1", PUBKEY_3]) + cli.run(["users", "add", "user1", PUBKEY_2]) + cli.run(["users", "add", "user2", PUBKEY_3]) # To edit: # SOPS_AGE_KEY=AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ sops --age age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 ./data/secrets.yaml @@ -183,6 +194,11 @@ def test_import_sops( str(test_root.joinpath("data", "secrets.yaml")), ] ) + capsys.readouterr() + cli.run(["users", "list"]) + users = sorted(capsys.readouterr().out.rstrip().split()) + assert users == ["user1", "user2"] + capsys.readouterr() cli.run(["get", "secret-key"]) assert capsys.readouterr().out == "secret-value" From 68905fc23308e43091e966aac7ff26252dc113de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 10:17:11 +0200 Subject: [PATCH 20/36] don't add user to a secret if they already can access the secret If the user is part of a group we don't need to add them explicitly --- pkgs/clan-cli/clan_cli/secrets/secrets.py | 82 ++++++++--------- pkgs/clan-cli/clan_cli/secrets/sops.py | 8 +- pkgs/clan-cli/tests/conftest.py | 2 +- pkgs/clan-cli/tests/helpers/secret_cli.py | 14 +++ pkgs/clan-cli/tests/test_import_sops.py | 46 ++++++++++ pkgs/clan-cli/tests/test_keys.py | 31 +++++++ pkgs/clan-cli/tests/test_secrets.py | 103 +++++++--------------- 7 files changed, 168 insertions(+), 118 deletions(-) create mode 100644 pkgs/clan-cli/tests/helpers/secret_cli.py create mode 100644 pkgs/clan-cli/tests/test_import_sops.py create mode 100644 pkgs/clan-cli/tests/test_keys.py diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index db869b07c..4c3c1715c 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -19,6 +19,39 @@ from .sops import decrypt_file, encrypt_file, ensure_sops_key, read_key, update_ from .types import VALID_SECRET_NAME, secret_name_type +def collect_keys_for_type(folder: Path) -> set[str]: + if not folder.exists(): + return set() + keys = set() + for p in folder.iterdir(): + if not p.is_symlink(): + continue + try: + target = p.resolve() + except FileNotFoundError: + tty.warn(f"Ignoring broken symlink {p}") + continue + kind = target.parent.name + if folder.name != kind: + tty.warn(f"Expected {p} to point to {folder} but points to {target.parent}") + continue + keys.add(read_key(target)) + return keys + + +def collect_keys_for_path(path: Path) -> set[str]: + keys = set([]) + keys.update(collect_keys_for_type(path / "machines")) + keys.update(collect_keys_for_type(path / "users")) + groups = path / "groups" + if not groups.is_dir(): + return keys + for group in groups.iterdir(): + keys.update(collect_keys_for_type(group / "machines")) + keys.update(collect_keys_for_type(group / "users")) + return keys + + def encrypt_secret( secret: Path, value: Union[IO[str], str], @@ -40,11 +73,7 @@ def encrypt_secret( for group in add_groups: allow_member(groups_folder(secret.name), sops_groups_folder(), group, False) - for kind in ["users", "machines", "groups"]: - if not (sops_secrets_folder() / kind).is_dir(): - continue - k = read_key(sops_secrets_folder() / kind) - keys.add(k) + keys = collect_keys_for_path(secret) if key.pubkey not in keys: keys.add(key.pubkey) @@ -52,6 +81,7 @@ def encrypt_secret( users_folder(secret.name), sops_users_folder(), key.username, False ) + breakpoint() encrypt_file(secret / "secret", value, list(sorted(keys))) @@ -79,39 +109,6 @@ def groups_folder(group: str) -> Path: return sops_secrets_folder() / group / "groups" -def collect_keys_for_type(folder: Path) -> list[str]: - if not folder.exists(): - return [] - keys = [] - for p in folder.iterdir(): - if not p.is_symlink(): - continue - try: - target = p.resolve() - except FileNotFoundError: - tty.warn(f"Ignoring broken symlink {p}") - continue - kind = target.parent.name - if folder.name != kind: - tty.warn(f"Expected {p} to point to {folder} but points to {target.parent}") - continue - keys.append(read_key(target)) - return keys - - -def collect_keys_for_path(path: Path) -> list[str]: - keys = [] - keys += collect_keys_for_type(path / "machines") - keys += collect_keys_for_type(path / "users") - groups = path / "groups" - if not groups.is_dir(): - return keys - for group in groups.iterdir(): - keys += collect_keys_for_type(group / "machines") - keys += collect_keys_for_type(group / "users") - return keys - - def allow_member( group_folder: Path, source_folder: Path, name: str, do_update_keys: bool = True ) -> None: @@ -129,7 +126,10 @@ def allow_member( user_target.symlink_to(os.path.relpath(source, user_target.parent)) if do_update_keys: - update_keys(group_folder.parent, collect_keys_for_path(group_folder.parent)) + update_keys( + group_folder.parent, + list(sorted(collect_keys_for_path(group_folder.parent))), + ) def disallow_member(group_folder: Path, name: str) -> None: @@ -151,7 +151,9 @@ def disallow_member(group_folder: Path, name: str) -> None: if len(os.listdir(group_folder.parent)) == 0: os.rmdir(group_folder.parent) - update_keys(target.parent.parent, collect_keys_for_path(group_folder.parent)) + update_keys( + target.parent.parent, list(sorted(collect_keys_for_path(group_folder.parent))) + ) def list_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index dc9595b2b..80650ee40 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -138,9 +138,7 @@ def encrypt_file( folder.mkdir(parents=True, exist_ok=True) # hopefully /tmp is written to an in-memory file to avoid leaking secrets - with NamedTemporaryFile(delete=False) as dummy_manifest_file, NamedTemporaryFile( - delete=False - ) as f: + with sops_manifest(keys) as manifest, NamedTemporaryFile(delete=False) as f: try: with open(f.name, "w") as fd: if isinstance(content, str): @@ -148,9 +146,7 @@ def encrypt_file( else: shutil.copyfileobj(content, fd) # we pass an empty manifest to pick up existing configuration of the user - args = ["sops", "--config", dummy_manifest_file.name] - for key in keys: - args.extend(["--age", key]) + args = ["sops", "--config", str(manifest)] args.extend(["-i", "--encrypt", str(f.name)]) cmd = nix_shell(["sops"], args) subprocess.run(cmd, check=True) diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index 481347645..7bde26d77 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -3,4 +3,4 @@ import sys sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) -pytest_plugins = ["temporary_dir", "clan_flake", "root"] +pytest_plugins = ["temporary_dir", "clan_flake", "root", "test_keys"] diff --git a/pkgs/clan-cli/tests/helpers/secret_cli.py b/pkgs/clan-cli/tests/helpers/secret_cli.py new file mode 100644 index 000000000..d43408d05 --- /dev/null +++ b/pkgs/clan-cli/tests/helpers/secret_cli.py @@ -0,0 +1,14 @@ +import argparse + +from clan_cli.secrets import register_parser + + +class SecretCli: + def __init__(self) -> None: + self.parser = argparse.ArgumentParser() + register_parser(self.parser) + + def run(self, args: list[str]) -> argparse.Namespace: + parsed = self.parser.parse_args(args) + parsed.func(parsed) + return parsed diff --git a/pkgs/clan-cli/tests/test_import_sops.py b/pkgs/clan-cli/tests/test_import_sops.py new file mode 100644 index 000000000..b578dd0f4 --- /dev/null +++ b/pkgs/clan-cli/tests/test_import_sops.py @@ -0,0 +1,46 @@ +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from environment import mock_env +from secret_cli import SecretCli + +if TYPE_CHECKING: + from test_keys import KeyPair + + +def test_import_sops( + test_root: Path, + clan_flake: Path, + capsys: pytest.CaptureFixture, + test_keys: list["KeyPair"], +) -> None: + cli = SecretCli() + + with mock_env(SOPS_AGE_KEY=test_keys[1].privkey): + cli.run(["machines", "add", "machine1", test_keys[0].pubkey]) + cli.run(["users", "add", "user1", test_keys[1].pubkey]) + cli.run(["users", "add", "user2", test_keys[2].pubkey]) + cli.run(["groups", "add-user", "group1", "user1"]) + cli.run(["groups", "add-user", "group1", "user2"]) + + # To edit: + # SOPS_AGE_KEY=AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ sops --age age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 ./data/secrets.yaml + cli.run( + [ + "import-sops", + "--group", + "group1", + "--machine", + "machine1", + str(test_root.joinpath("data", "secrets.yaml")), + ] + ) + capsys.readouterr() + cli.run(["users", "list"]) + users = sorted(capsys.readouterr().out.rstrip().split()) + assert users == ["user1", "user2"] + + capsys.readouterr() + cli.run(["get", "secret-key"]) + assert capsys.readouterr().out == "secret-value" diff --git a/pkgs/clan-cli/tests/test_keys.py b/pkgs/clan-cli/tests/test_keys.py new file mode 100644 index 000000000..518a2becd --- /dev/null +++ b/pkgs/clan-cli/tests/test_keys.py @@ -0,0 +1,31 @@ +import pytest + + +class KeyPair: + def __init__(self, pubkey: str, privkey: str) -> None: + self.pubkey = pubkey + self.privkey = privkey + + +KEYS = [ + KeyPair( + "age1dhwqzkah943xzc34tc3dlmfayyevcmdmxzjezdgdy33euxwf59vsp3vk3c", + "AGE-SECRET-KEY-1KF8E3SR3TTGL6M476SKF7EEMR4H9NF7ZWYSLJUAK8JX276JC7KUSSURKFK", + ), + KeyPair( + "age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62", + "AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ", + ), + KeyPair( + "age1dhuh9xtefhgpr2sjjf7gmp9q2pr37z92rv4wsadxuqdx48989g7qj552qp", + "AGE-SECRET-KEY-169N3FT32VNYQ9WYJMLUSVTMA0TTZGVJF7YZWS8AHTWJ5RR9VGR7QCD8SKF", + ), +] + + +@pytest.fixture +def test_keys() -> list[KeyPair]: + """ + Root directory of the tests + """ + return KEYS diff --git a/pkgs/clan-cli/tests/test_secrets.py b/pkgs/clan-cli/tests/test_secrets.py index 50c9f7b70..9aac1e68b 100644 --- a/pkgs/clan-cli/tests/test_secrets.py +++ b/pkgs/clan-cli/tests/test_secrets.py @@ -1,45 +1,30 @@ -import argparse import os from pathlib import Path +from typing import TYPE_CHECKING import pytest from environment import mock_env +from secret_cli import SecretCli from clan_cli.errors import ClanError -from clan_cli.secrets import register_parser - -class SecretCli: - def __init__(self) -> None: - self.parser = argparse.ArgumentParser() - register_parser(self.parser) - - def run(self, args: list[str]) -> argparse.Namespace: - parsed = self.parser.parse_args(args) - parsed.func(parsed) - return parsed - - -PUBKEY = "age1dhwqzkah943xzc34tc3dlmfayyevcmdmxzjezdgdy33euxwf59vsp3vk3c" -PRIVKEY = "AGE-SECRET-KEY-1KF8E3SR3TTGL6M476SKF7EEMR4H9NF7ZWYSLJUAK8JX276JC7KUSSURKFK" - -PUBKEY_2 = "age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62" -PRIVKEY_2 = "AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ" - -PUBKEY_3 = "age1dhuh9xtefhgpr2sjjf7gmp9q2pr37z92rv4wsadxuqdx48989g7qj552qp" -PRIVKEY_3 = "AGE-SECRET-KEY-169N3FT32VNYQ9WYJMLUSVTMA0TTZGVJF7YZWS8AHTWJ5RR9VGR7QCD8SKF" +if TYPE_CHECKING: + from test_keys import KeyPair def _test_identities( - what: str, clan_flake: Path, capsys: pytest.CaptureFixture + what: str, + clan_flake: Path, + capsys: pytest.CaptureFixture, + test_keys: list["KeyPair"], ) -> None: cli = SecretCli() sops_folder = clan_flake / "sops" - cli.run([what, "add", "foo", PUBKEY]) + cli.run([what, "add", "foo", test_keys[0].pubkey]) assert (sops_folder / what / "foo" / "key.json").exists() with pytest.raises(ClanError): - cli.run([what, "add", "foo", PUBKEY]) + cli.run([what, "add", "foo", test_keys[0].pubkey]) cli.run( [ @@ -47,7 +32,7 @@ def _test_identities( "add", "-f", "foo", - PRIVKEY, + test_keys[0].privkey, ] ) capsys.readouterr() # empty the buffer @@ -68,15 +53,21 @@ def _test_identities( assert "foo" not in out.out -def test_users(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: - _test_identities("users", clan_flake, capsys) +def test_users( + clan_flake: Path, capsys: pytest.CaptureFixture, test_keys: list["KeyPair"] +) -> None: + _test_identities("users", clan_flake, capsys, test_keys) -def test_machines(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: - _test_identities("machines", clan_flake, capsys) +def test_machines( + clan_flake: Path, capsys: pytest.CaptureFixture, test_keys: list["KeyPair"] +) -> None: + _test_identities("machines", clan_flake, capsys, test_keys) -def test_groups(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: +def test_groups( + clan_flake: Path, capsys: pytest.CaptureFixture, test_keys: list["KeyPair"] +) -> None: cli = SecretCli() capsys.readouterr() # empty the buffer cli.run(["groups", "list"]) @@ -86,13 +77,13 @@ def test_groups(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: cli.run(["groups", "add-machine", "group1", "machine1"]) with pytest.raises(ClanError): # user does not exist yet cli.run(["groups", "add-user", "groupb1", "user1"]) - cli.run(["machines", "add", "machine1", PUBKEY]) + cli.run(["machines", "add", "machine1", test_keys[0].pubkey]) cli.run(["groups", "add-machine", "group1", "machine1"]) # Should this fail? cli.run(["groups", "add-machine", "group1", "machine1"]) - cli.run(["users", "add", "user1", PUBKEY]) + cli.run(["users", "add", "user1", test_keys[0].pubkey]) cli.run(["groups", "add-user", "group1", "user1"]) capsys.readouterr() # empty the buffer @@ -107,7 +98,9 @@ def test_groups(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: assert len(groups) == 0 -def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: +def test_secrets( + clan_flake: Path, capsys: pytest.CaptureFixture, test_keys: list["KeyPair"] +) -> None: cli = SecretCli() capsys.readouterr() # empty the buffer cli.run(["list"]) @@ -132,18 +125,18 @@ def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: cli.run(["list"]) assert capsys.readouterr().out == "key\n" - cli.run(["machines", "add", "machine1", PUBKEY]) + cli.run(["machines", "add", "machine1", test_keys[0].pubkey]) cli.run(["machines", "add-secret", "machine1", "key"]) - with mock_env(SOPS_AGE_KEY=PRIVKEY, SOPS_AGE_KEY_FILE=""): + with mock_env(SOPS_AGE_KEY=test_keys[1].privkey, SOPS_AGE_KEY_FILE=""): capsys.readouterr() cli.run(["get", "key"]) assert capsys.readouterr().out == "foo" cli.run(["machines", "remove-secret", "machine1", "key"]) - cli.run(["users", "add", "user1", PUBKEY_2]) + cli.run(["users", "add", "user1", test_keys[1].pubkey]) cli.run(["users", "add-secret", "user1", "key"]) - with mock_env(SOPS_AGE_KEY=PRIVKEY_2, SOPS_AGE_KEY_FILE=""): + with mock_env(SOPS_AGE_KEY=test_keys[1].privkey, SOPS_AGE_KEY_FILE=""): capsys.readouterr() cli.run(["get", "key"]) assert capsys.readouterr().out == "foo" @@ -158,7 +151,7 @@ def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: capsys.readouterr() # empty the buffer cli.run(["set", "--group", "admin-group", "key2"]) - with mock_env(SOPS_AGE_KEY=PRIVKEY_2, SOPS_AGE_KEY_FILE=""): + with mock_env(SOPS_AGE_KEY=test_keys[1].privkey, SOPS_AGE_KEY_FILE=""): capsys.readouterr() cli.run(["get", "key"]) assert capsys.readouterr().out == "foo" @@ -170,35 +163,3 @@ def test_secrets(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: capsys.readouterr() # empty the buffer cli.run(["list"]) assert capsys.readouterr().out == "" - - -def test_import_sops( - test_root: Path, clan_flake: Path, capsys: pytest.CaptureFixture -) -> None: - cli = SecretCli() - - with mock_env(SOPS_AGE_KEY=PRIVKEY_2): - cli.run(["machines", "add", "machine1", PUBKEY]) - cli.run(["users", "add", "user1", PUBKEY_2]) - cli.run(["users", "add", "user2", PUBKEY_3]) - - # To edit: - # SOPS_AGE_KEY=AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ sops --age age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 ./data/secrets.yaml - cli.run( - [ - "import-sops", - "--user", - "user1", - "--machine", - "machine1", - str(test_root.joinpath("data", "secrets.yaml")), - ] - ) - capsys.readouterr() - cli.run(["users", "list"]) - users = sorted(capsys.readouterr().out.rstrip().split()) - assert users == ["user1", "user2"] - - capsys.readouterr() - cli.run(["get", "secret-key"]) - assert capsys.readouterr().out == "secret-value" From 3d5a37ad03e7a7e946dfc1c98bc12fb8065ea538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 10:21:59 +0200 Subject: [PATCH 21/36] check for breakpoint() function in code --- pkgs/clan-cli/clan_cli/secrets/secrets.py | 1 - pkgs/clan-cli/default.nix | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 4c3c1715c..ef998d44b 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -81,7 +81,6 @@ def encrypt_secret( users_folder(secret.name), sops_users_folder(), key.username, False ) - breakpoint() encrypt_file(secret / "secret", value, list(sorted(keys))) diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index f9623c890..9205d0dc8 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -79,6 +79,11 @@ python3.pkgs.buildPythonPackage { ''; checkPhase = '' PYTHONPATH= $out/bin/clan --help + if grep --include \*.py -q "breakpoint()" $out; then + echo "breakpoint() found in $out:" + grep --include \*.py -Rn "breakpoint()" $out + exit 1 + fi ''; meta.mainProgram = "clan"; } From 9189c31def38a3841a612da2bdef18eee14bf841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 10:33:54 +0200 Subject: [PATCH 22/36] improve error message if users don't exist --- pkgs/clan-cli/clan_cli/secrets/secrets.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index ef998d44b..7282ebdc5 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -108,12 +108,23 @@ def groups_folder(group: str) -> Path: return sops_secrets_folder() / group / "groups" +def list_directory(directory: Path) -> str: + if not directory.exists(): + return "{directory} does not exist" + msg = f"\n{directory} contains:" + for f in directory.iterdir(): + msg += f"\n {f.name}" + return msg + + def allow_member( group_folder: Path, source_folder: Path, name: str, do_update_keys: bool = True ) -> None: source = source_folder / name if not source.exists(): - raise ClanError(f"{name} does not exist in {source_folder}") + msg = f"{name} does not exist in {source_folder}" + msg += list_directory(source_folder) + raise ClanError(msg) group_folder.mkdir(parents=True, exist_ok=True) user_target = group_folder / name if user_target.exists(): @@ -134,7 +145,9 @@ def allow_member( def disallow_member(group_folder: Path, name: str) -> None: target = group_folder / name if not target.exists(): - raise ClanError(f"{name} does not exist in group in {group_folder}") + msg = f"{name} does not exist in group in {group_folder}" + msg += list_directory(group_folder) + raise ClanError(msg) keys = collect_keys_for_path(group_folder.parent) From 7d39f8346c11de4df795ee26874d8dcc2e8d2a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 10:36:46 +0200 Subject: [PATCH 23/36] clan-cli: fix grep in breakpoint check --- pkgs/clan-cli/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 9205d0dc8..f9569ef78 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -79,7 +79,7 @@ python3.pkgs.buildPythonPackage { ''; checkPhase = '' PYTHONPATH= $out/bin/clan --help - if grep --include \*.py -q "breakpoint()" $out; then + if grep --include \*.py -Rq "breakpoint()" $out; then echo "breakpoint() found in $out:" grep --include \*.py -Rn "breakpoint()" $out exit 1 From daf1058312ecc0b81ea731afc5cf78cd47e431a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 10:50:48 +0200 Subject: [PATCH 24/36] clan-cli: fix secrets test --- pkgs/clan-cli/tests/test_secrets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/tests/test_secrets.py b/pkgs/clan-cli/tests/test_secrets.py index 9aac1e68b..3b555f6e3 100644 --- a/pkgs/clan-cli/tests/test_secrets.py +++ b/pkgs/clan-cli/tests/test_secrets.py @@ -128,7 +128,7 @@ def test_secrets( cli.run(["machines", "add", "machine1", test_keys[0].pubkey]) cli.run(["machines", "add-secret", "machine1", "key"]) - with mock_env(SOPS_AGE_KEY=test_keys[1].privkey, SOPS_AGE_KEY_FILE=""): + with mock_env(SOPS_AGE_KEY=test_keys[0].privkey, SOPS_AGE_KEY_FILE=""): capsys.readouterr() cli.run(["get", "key"]) assert capsys.readouterr().out == "foo" From 119d68bdcd2d44df091d62efc5adab1398481aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 13:05:26 +0200 Subject: [PATCH 25/36] clan-cli: add deploykit --- pkgs/clan-cli/clan_cli/deploy/__init__.py | 821 ++++++++++++++++++++++ 1 file changed, 821 insertions(+) create mode 100644 pkgs/clan-cli/clan_cli/deploy/__init__.py diff --git a/pkgs/clan-cli/clan_cli/deploy/__init__.py b/pkgs/clan-cli/clan_cli/deploy/__init__.py new file mode 100644 index 000000000..189967404 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/deploy/__init__.py @@ -0,0 +1,821 @@ +# Adapted from https://github.com/numtide/deploykit + +import fcntl +import logging +import math +import os +import select +import shlex +import subprocess +import sys +import time +from contextlib import ExitStack, contextmanager +from enum import Enum +from pathlib import Path +from shlex import quote +from threading import Thread +from typing import ( + IO, + Any, + Callable, + Dict, + Generic, + Iterator, + List, + Literal, + Optional, + Tuple, + TypeVar, + Union, + overload, +) + +# https://no-color.org +DISABLE_COLOR = not sys.stderr.isatty() or os.environ.get("NO_COLOR", "") != "" + + +def ansi_color(color: int) -> str: + return f"\x1b[{color}m" + + +class CommandFormatter(logging.Formatter): + """ + print errors in red and warnings in yellow + """ + + def __init__(self) -> None: + super().__init__( + "%(prefix_color)s[%(command_prefix)s]%(color_reset)s %(color)s%(message)s%(color_reset)s" + ) + self.hostnames: List[str] = [] + self.hostname_color_offset = 1 # first host shouldn't get agressive red + + def formatMessage(self, record: logging.LogRecord) -> str: + colorcode = 0 + if record.levelno == logging.ERROR: + colorcode = 31 # red + if record.levelno == logging.WARN: + colorcode = 33 # yellow + + color, prefix_color, color_reset = "", "", "" + if not DISABLE_COLOR: + command_prefix = getattr(record, "command_prefix", "") + color = ansi_color(colorcode) + prefix_color = ansi_color(self.hostname_colorcode(command_prefix)) + color_reset = "\x1b[0m" + + setattr(record, "color", color) + setattr(record, "prefix_color", prefix_color) + setattr(record, "color_reset", color_reset) + + return super().formatMessage(record) + + def hostname_colorcode(self, hostname: str) -> int: + try: + index = self.hostnames.index(hostname) + except ValueError: + self.hostnames += [hostname] + index = self.hostnames.index(hostname) + return 31 + (index + self.hostname_color_offset) % 7 + + +def setup_loggers() -> Tuple[logging.Logger, logging.Logger]: + # If we use the default logger here (logging.error etc) or a logger called + # "deploykit", then cmdlog messages are also posted on the default logger. + # To avoid this message duplication, we set up a main and command logger + # and use a "deploykit" main logger. + kitlog = logging.getLogger("deploykit.main") + kitlog.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + ch.setFormatter(logging.Formatter()) + + kitlog.addHandler(ch) + + # use specific logger for command outputs + cmdlog = logging.getLogger("deploykit.command") + cmdlog.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + ch.setFormatter(CommandFormatter()) + + cmdlog.addHandler(ch) + return (kitlog, cmdlog) + + +# loggers for: general deploykit, command output +kitlog, cmdlog = setup_loggers() + +info = kitlog.info +warn = kitlog.warning +error = kitlog.error + + +@contextmanager +def _pipe() -> Iterator[Tuple[IO[str], IO[str]]]: + (pipe_r, pipe_w) = os.pipe() + read_end = os.fdopen(pipe_r, "r") + write_end = os.fdopen(pipe_w, "w") + + try: + fl = fcntl.fcntl(read_end, fcntl.F_GETFL) + fcntl.fcntl(read_end, fcntl.F_SETFL, fl | os.O_NONBLOCK) + + yield (read_end, write_end) + finally: + read_end.close() + write_end.close() + + +FILE = Union[None, int] + +# Seconds until a message is printed when _run produces no output. +NO_OUTPUT_TIMEOUT = 20 + + +class HostKeyCheck(Enum): + # Strictly check ssh host keys, prompt for unknown ones + STRICT = 0 + # Trust on ssh keys on first use + TOFU = 1 + # Do not check ssh host keys + NONE = 2 + + +class DeployHost: + def __init__( + self, + host: str, + user: Optional[str] = None, + port: Optional[int] = None, + key: Optional[str] = None, + forward_agent: bool = False, + command_prefix: Optional[str] = None, + host_key_check: HostKeyCheck = HostKeyCheck.STRICT, + meta: Dict[str, Any] = {}, + verbose_ssh: bool = False, + ) -> None: + """ + Creates a DeployHost + @host the hostname to connect to via ssh + @port the port to connect to via ssh + @forward_agent: wheter to forward ssh agent + @command_prefix: string to prefix each line of the command output with, defaults to host + @host_key_check: wether to check ssh host keys + @verbose_ssh: Enables verbose logging on ssh connections + @meta: meta attributes associated with the host. Those can be accessed in custom functions passed to `run_function` + """ + self.host = host + self.user = user + self.port = port + self.key = key + if command_prefix: + self.command_prefix = command_prefix + else: + self.command_prefix = host + self.forward_agent = forward_agent + self.host_key_check = host_key_check + self.meta = meta + self.verbose_ssh = verbose_ssh + + def _prefix_output( + self, + displayed_cmd: str, + print_std_fd: Optional[IO[str]], + print_err_fd: Optional[IO[str]], + stdout: Optional[IO[str]], + stderr: Optional[IO[str]], + timeout: float = math.inf, + ) -> Tuple[str, str]: + rlist = [] + if print_std_fd is not None: + rlist.append(print_std_fd) + if print_err_fd is not None: + rlist.append(print_err_fd) + if stdout is not None: + rlist.append(stdout) + + if stderr is not None: + rlist.append(stderr) + + print_std_buf = "" + print_err_buf = "" + stdout_buf = "" + stderr_buf = "" + + start = time.time() + last_output = time.time() + while len(rlist) != 0: + r, _, _ = select.select(rlist, [], [], min(timeout, NO_OUTPUT_TIMEOUT)) + + def print_from( + print_fd: IO[str], print_buf: str, is_err: bool = False + ) -> Tuple[float, str]: + read = os.read(print_fd.fileno(), 4096) + if len(read) == 0: + rlist.remove(print_fd) + print_buf += read.decode("utf-8") + if (read == b"" and len(print_buf) != 0) or "\n" in print_buf: + # print and empty the print_buf, if the stream is draining, + # but there is still something in the buffer or on newline. + lines = print_buf.rstrip("\n").split("\n") + for line in lines: + if not is_err: + cmdlog.info( + line, extra=dict(command_prefix=self.command_prefix) + ) + pass + else: + cmdlog.error( + line, extra=dict(command_prefix=self.command_prefix) + ) + print_buf = "" + last_output = time.time() + return (last_output, print_buf) + + if print_std_fd in r and print_std_fd is not None: + (last_output, print_std_buf) = print_from( + print_std_fd, print_std_buf, is_err=False + ) + if print_err_fd in r and print_err_fd is not None: + (last_output, print_err_buf) = print_from( + print_err_fd, print_err_buf, is_err=True + ) + + now = time.time() + elapsed = now - start + if now - last_output > NO_OUTPUT_TIMEOUT: + elapsed_msg = time.strftime("%H:%M:%S", time.gmtime(elapsed)) + cmdlog.warn( + f"still waiting for '{displayed_cmd}' to finish... ({elapsed_msg} elapsed)", + extra=dict(command_prefix=self.command_prefix), + ) + + def handle_fd(fd: Optional[IO[Any]]) -> str: + if fd and fd in r: + read = os.read(fd.fileno(), 4096) + if len(read) == 0: + rlist.remove(fd) + else: + return read.decode("utf-8") + return "" + + stdout_buf += handle_fd(stdout) + stderr_buf += handle_fd(stderr) + + if now - last_output >= timeout: + break + return stdout_buf, stderr_buf + + def _run( + self, + cmd: List[str], + displayed_cmd: str, + shell: bool, + stdout: FILE = None, + stderr: FILE = None, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, + timeout: float = math.inf, + ) -> subprocess.CompletedProcess[str]: + with ExitStack() as stack: + read_std_fd, write_std_fd = (None, None) + read_err_fd, write_err_fd = (None, None) + + if stdout is None or stderr is None: + read_std_fd, write_std_fd = stack.enter_context(_pipe()) + read_err_fd, write_err_fd = stack.enter_context(_pipe()) + + if stdout is None: + stdout_read = None + stdout_write = write_std_fd + elif stdout == subprocess.PIPE: + stdout_read, stdout_write = stack.enter_context(_pipe()) + else: + raise Exception(f"unsupported value for stdout parameter: {stdout}") + + if stderr is None: + stderr_read = None + stderr_write = write_err_fd + elif stderr == subprocess.PIPE: + stderr_read, stderr_write = stack.enter_context(_pipe()) + else: + raise Exception(f"unsupported value for stderr parameter: {stderr}") + + env = os.environ.copy() + env.update(extra_env) + + with subprocess.Popen( + cmd, + text=True, + shell=shell, + stdout=stdout_write, + stderr=stderr_write, + env=env, + cwd=cwd, + ) as p: + if write_std_fd is not None: + write_std_fd.close() + if write_err_fd is not None: + write_err_fd.close() + if stdout == subprocess.PIPE: + assert stdout_write is not None + stdout_write.close() + if stderr == subprocess.PIPE: + assert stderr_write is not None + stderr_write.close() + + start = time.time() + stdout_data, stderr_data = self._prefix_output( + displayed_cmd, + read_std_fd, + read_err_fd, + stdout_read, + stderr_read, + timeout, + ) + try: + ret = p.wait(timeout=max(0, timeout - (time.time() - start))) + except subprocess.TimeoutExpired: + p.kill() + raise + if ret != 0: + if check: + raise subprocess.CalledProcessError( + ret, cmd=cmd, output=stdout_data, stderr=stderr_data + ) + else: + cmdlog.warning( + f"[Command failed: {ret}] {displayed_cmd}", + extra=dict(command_prefix=self.command_prefix), + ) + return subprocess.CompletedProcess( + cmd, ret, stdout=stdout_data, stderr=stderr_data + ) + raise RuntimeError("unreachable") + + def run_local( + self, + cmd: Union[str, List[str]], + stdout: FILE = None, + stderr: FILE = None, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, + timeout: float = math.inf, + ) -> subprocess.CompletedProcess[str]: + """ + Command to run locally for the host + + @cmd the commmand to run + @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE + @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE + @extra_env environment variables to override whe running the command + @cwd current working directory to run the process in + @timeout: Timeout in seconds for the command to complete + + @return subprocess.CompletedProcess result of the command + """ + shell = False + if isinstance(cmd, str): + cmd = [cmd] + shell = True + displayed_cmd = " ".join(cmd) + cmdlog.info( + f"$ {displayed_cmd}", extra=dict(command_prefix=self.command_prefix) + ) + return self._run( + cmd, + displayed_cmd, + shell=shell, + stdout=stdout, + stderr=stderr, + extra_env=extra_env, + cwd=cwd, + check=check, + timeout=timeout, + ) + + def run( + self, + cmd: Union[str, List[str]], + stdout: FILE = None, + stderr: FILE = None, + become_root: bool = False, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, + verbose_ssh: bool = False, + timeout: float = math.inf, + ) -> subprocess.CompletedProcess[str]: + """ + Command to run on the host via ssh + + @cmd the commmand to run + @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE + @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE + @become_root if the ssh_user is not root than sudo is prepended + @extra_env environment variables to override whe running the command + @cwd current working directory to run the process in + @verbose_ssh: Enables verbose logging on ssh connections + @timeout: Timeout in seconds for the command to complete + + @return subprocess.CompletedProcess result of the ssh command + """ + sudo = "" + if become_root and self.user != "root": + sudo = "sudo -- " + vars = [] + for k, v in extra_env.items(): + vars.append(f"{shlex.quote(k)}={shlex.quote(v)}") + + displayed_cmd = "" + export_cmd = "" + if vars: + export_cmd = f"export {' '.join(vars)}; " + displayed_cmd += export_cmd + if isinstance(cmd, list): + displayed_cmd += " ".join(cmd) + else: + displayed_cmd += cmd + cmdlog.info( + f"$ {displayed_cmd}", extra=dict(command_prefix=self.command_prefix) + ) + + if self.user is not None: + ssh_target = f"{self.user}@{self.host}" + else: + ssh_target = self.host + + ssh_opts = ["-A"] if self.forward_agent else [] + if self.port: + ssh_opts.extend(["-p", str(self.port)]) + if self.key: + ssh_opts.extend(["-i", self.key]) + + if self.host_key_check != HostKeyCheck.STRICT: + ssh_opts.extend(["-o", "StrictHostKeyChecking=no"]) + if self.host_key_check == HostKeyCheck.NONE: + ssh_opts.extend(["-o", "UserKnownHostsFile=/dev/null"]) + if verbose_ssh or self.verbose_ssh: + ssh_opts.extend(["-v"]) + + bash_cmd = export_cmd + bash_args = [] + if isinstance(cmd, list): + bash_cmd += 'exec "$@"' + bash_args += cmd + else: + bash_cmd += cmd + # FIXME we assume bash to be present here? Should be documented... + ssh_cmd = ( + ["ssh", ssh_target] + + ssh_opts + + [ + "--", + f"{sudo}bash -c {quote(bash_cmd)} -- {' '.join(map(quote, bash_args))}", + ] + ) + return self._run( + ssh_cmd, + displayed_cmd, + shell=False, + stdout=stdout, + stderr=stderr, + cwd=cwd, + check=check, + timeout=timeout, + ) + + +T = TypeVar("T") + + +class HostResult(Generic[T]): + def __init__(self, host: DeployHost, result: Union[T, Exception]) -> None: + self.host = host + self._result = result + + @property + def error(self) -> Optional[Exception]: + """ + Returns an error if the command failed + """ + if isinstance(self._result, Exception): + return self._result + return None + + @property + def result(self) -> T: + """ + Unwrap the result + """ + if isinstance(self._result, Exception): + raise self._result + return self._result + + +DeployResults = List[HostResult[subprocess.CompletedProcess[str]]] + + +def _worker( + func: Callable[[DeployHost], T], + host: DeployHost, + results: List[HostResult[T]], + idx: int, +) -> None: + try: + results[idx] = HostResult(host, func(host)) + except Exception as e: + kitlog.exception(e) + results[idx] = HostResult(host, e) + + +class DeployGroup: + def __init__(self, hosts: List[DeployHost]) -> None: + self.hosts = hosts + + def _run_local( + self, + cmd: Union[str, List[str]], + host: DeployHost, + results: DeployResults, + stdout: FILE = None, + stderr: FILE = None, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, + verbose_ssh: bool = False, + timeout: float = math.inf, + ) -> None: + try: + proc = host.run_local( + cmd, + stdout=stdout, + stderr=stderr, + extra_env=extra_env, + cwd=cwd, + check=check, + timeout=timeout, + ) + results.append(HostResult(host, proc)) + except Exception as e: + kitlog.exception(e) + results.append(HostResult(host, e)) + + def _run_remote( + self, + cmd: Union[str, List[str]], + host: DeployHost, + results: DeployResults, + stdout: FILE = None, + stderr: FILE = None, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, + verbose_ssh: bool = False, + timeout: float = math.inf, + ) -> None: + try: + proc = host.run( + cmd, + stdout=stdout, + stderr=stderr, + extra_env=extra_env, + cwd=cwd, + check=check, + verbose_ssh=verbose_ssh, + timeout=timeout, + ) + results.append(HostResult(host, proc)) + except Exception as e: + kitlog.exception(e) + results.append(HostResult(host, e)) + + def _reraise_errors(self, results: List[HostResult[Any]]) -> None: + errors = 0 + for result in results: + e = result.error + if e: + cmdlog.error( + f"failed with: {e}", + extra=dict(command_prefix=result.host.command_prefix), + ) + errors += 1 + if errors > 0: + raise Exception( + f"{errors} hosts failed with an error. Check the logs above" + ) + + def _run( + self, + cmd: Union[str, List[str]], + local: bool = False, + stdout: FILE = None, + stderr: FILE = None, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, + verbose_ssh: bool = False, + timeout: float = math.inf, + ) -> DeployResults: + results: DeployResults = [] + threads = [] + for host in self.hosts: + fn = self._run_local if local else self._run_remote + thread = Thread( + target=fn, + kwargs=dict( + results=results, + cmd=cmd, + host=host, + stdout=stdout, + stderr=stderr, + extra_env=extra_env, + cwd=cwd, + check=check, + verbose_ssh=verbose_ssh, + timeout=timeout, + ), + ) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + if check: + self._reraise_errors(results) + + return results + + def run( + self, + cmd: Union[str, List[str]], + stdout: FILE = None, + stderr: FILE = None, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, + verbose_ssh: bool = False, + timeout: float = math.inf, + ) -> DeployResults: + """ + Command to run on the remote host via ssh + @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE + @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE + @cwd current working directory to run the process in + @verbose_ssh: Enables verbose logging on ssh connections + @timeout: Timeout in seconds for the command to complete + + @return a lists of tuples containing DeployNode and the result of the command for this DeployNode + """ + return self._run( + cmd, + stdout=stdout, + stderr=stderr, + extra_env=extra_env, + cwd=cwd, + check=check, + verbose_ssh=verbose_ssh, + timeout=timeout, + ) + + def run_local( + self, + cmd: Union[str, List[str]], + stdout: FILE = None, + stderr: FILE = None, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, + timeout: float = math.inf, + ) -> DeployResults: + """ + Command to run locally for each host in the group in parallel + @cmd the commmand to run + @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE + @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE + @cwd current working directory to run the process in + @extra_env environment variables to override whe running the command + @timeout: Timeout in seconds for the command to complete + + @return a lists of tuples containing DeployNode and the result of the command for this DeployNode + """ + return self._run( + cmd, + local=True, + stdout=stdout, + stderr=stderr, + extra_env=extra_env, + cwd=cwd, + check=check, + timeout=timeout, + ) + + def run_function( + self, func: Callable[[DeployHost], T], check: bool = True + ) -> List[HostResult[T]]: + """ + Function to run for each host in the group in parallel + + @func the function to call + """ + threads = [] + results: List[HostResult[T]] = [ + HostResult(h, Exception(f"No result set for thread {i}")) + for (i, h) in enumerate(self.hosts) + ] + for i, host in enumerate(self.hosts): + thread = Thread( + target=_worker, + args=(func, host, results, i), + ) + threads.append(thread) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + if check: + self._reraise_errors(results) + return results + + def filter(self, pred: Callable[[DeployHost], bool]) -> "DeployGroup": + """Return a new DeployGroup with the results filtered by the predicate""" + return DeployGroup(list(filter(pred, self.hosts))) + + +@overload +def run( + cmd: Union[List[str], str], + text: Literal[True] = ..., + stdout: FILE = ..., + stderr: FILE = ..., + extra_env: Dict[str, str] = ..., + cwd: Union[None, str, Path] = ..., + check: bool = ..., +) -> subprocess.CompletedProcess[str]: + ... + + +@overload +def run( + cmd: Union[List[str], str], + text: Literal[False], + stdout: FILE = ..., + stderr: FILE = ..., + extra_env: Dict[str, str] = ..., + cwd: Union[None, str, Path] = ..., + check: bool = ..., +) -> subprocess.CompletedProcess[bytes]: + ... + + +def run( + cmd: Union[List[str], str], + text: bool = True, + stdout: FILE = None, + stderr: FILE = None, + extra_env: Dict[str, str] = {}, + cwd: Union[None, str, Path] = None, + check: bool = True, +) -> subprocess.CompletedProcess[Any]: + """ + Run command locally + + @cmd if this parameter is a string the command is interpreted as a shell command, + otherwise if it is a list, than the first list element is the command + and the remaining list elements are passed as arguments to the + command. + @text when true, file objects for stdout and stderr are opened in text mode. + @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE + @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE + @extra_env environment variables to override whe running the command + @cwd current working directory to run the process in + @check If check is true, and the process exits with a non-zero exit code, a + CalledProcessError exception will be raised. Attributes of that exception + hold the arguments, the exit code, and stdout and stderr if they were + captured. + """ + if isinstance(cmd, list): + info("$ " + " ".join(cmd)) + else: + info(f"$ {cmd}") + env = os.environ.copy() + env.update(extra_env) + + return subprocess.run( + cmd, + stdout=stdout, + stderr=stderr, + env=env, + cwd=cwd, + check=check, + shell=not isinstance(cmd, list), + text=text, + ) From 2483e2981953a5298209cf8d2b532e5e35189da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 13:56:24 +0200 Subject: [PATCH 26/36] users: seperate cli from library --- pkgs/clan-cli/clan_cli/secrets/users.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/users.py b/pkgs/clan-cli/clan_cli/secrets/users.py index ef0c29ecd..b39d00307 100644 --- a/pkgs/clan-cli/clan_cli/secrets/users.py +++ b/pkgs/clan-cli/clan_cli/secrets/users.py @@ -15,16 +15,24 @@ def add_user(name: str, key: str, force: bool) -> None: write_key(sops_users_folder() / name, key, force) -def list_command(args: argparse.Namespace) -> None: +def remove_user(name: str) -> None: + remove_object(sops_users_folder(), name) + + +def list_users() -> None: list_objects(sops_users_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None) +def list_command(args: argparse.Namespace) -> None: + list_users() + + def add_command(args: argparse.Namespace) -> None: add_user(args.user, args.key, args.force) def remove_command(args: argparse.Namespace) -> None: - remove_object(sops_users_folder(), args.user) + remove_user(args.user) def add_secret_command(args: argparse.Namespace) -> None: From c535cf78a1b7c07d2425634a96ab9188e197e82f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 14:10:04 +0200 Subject: [PATCH 27/36] add secret api for machines/users/secrets --- pkgs/clan-cli/clan_cli/secrets/folders.py | 8 +++++--- pkgs/clan-cli/clan_cli/secrets/machines.py | 16 ++++++++++++++-- pkgs/clan-cli/clan_cli/secrets/secrets.py | 15 +++++++++++---- pkgs/clan-cli/clan_cli/secrets/users.py | 8 +++++--- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/folders.py b/pkgs/clan-cli/clan_cli/secrets/folders.py index b0f487bbf..f9e8d31ea 100644 --- a/pkgs/clan-cli/clan_cli/secrets/folders.py +++ b/pkgs/clan-cli/clan_cli/secrets/folders.py @@ -24,12 +24,14 @@ sops_machines_folder = gen_sops_subfolder("machines") sops_groups_folder = gen_sops_subfolder("groups") -def list_objects(path: Path, is_valid: Callable[[str], bool]) -> None: +def list_objects(path: Path, is_valid: Callable[[str], bool]) -> list[str]: + objs: list[str] = [] if not path.exists(): - return + return objs for f in os.listdir(path): if is_valid(f): - print(f) + objs.append(f) + return objs def remove_object(path: Path, name: str) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/machines.py b/pkgs/clan-cli/clan_cli/secrets/machines.py index d7a2ffb0a..330f2ad64 100644 --- a/pkgs/clan-cli/clan_cli/secrets/machines.py +++ b/pkgs/clan-cli/clan_cli/secrets/machines.py @@ -11,12 +11,24 @@ from .types import ( ) +def add_machine(name: str, key: str, force: bool) -> None: + write_key(sops_machines_folder() / name, key, force) + + +def remove_machine(name: str) -> None: + remove_object(sops_machines_folder(), name) + + +def list_machines() -> list[str]: + return list_objects(sops_machines_folder(), lambda x: validate_hostname(x)) + + def list_command(args: argparse.Namespace) -> None: - list_objects(sops_machines_folder(), lambda x: validate_hostname(x)) + print("\n".join(list_machines())) def add_command(args: argparse.Namespace) -> None: - write_key(sops_machines_folder() / args.machine, args.key, args.force) + add_machine(args.machine, args.key, args.force) def remove_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 7282ebdc5..4cc78e03c 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -84,14 +84,17 @@ def encrypt_secret( encrypt_file(secret / "secret", value, list(sorted(keys))) -def remove_command(args: argparse.Namespace) -> None: - secret: str = args.secret +def remove_secret(secret: str) -> None: path = sops_secrets_folder() / secret if not path.exists(): raise ClanError(f"Secret '{secret}' does not exist") shutil.rmtree(path) +def remove_command(args: argparse.Namespace) -> None: + remove_secret(args.secret) + + def add_secret_argument(parser: argparse.ArgumentParser) -> None: parser.add_argument("secret", help="the name of the secret", type=secret_name_type) @@ -168,12 +171,16 @@ def disallow_member(group_folder: Path, name: str) -> None: ) -def list_command(args: argparse.Namespace) -> None: - list_objects( +def list_secrets() -> list[str]: + return list_objects( sops_secrets_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None ) +def list_command(args: argparse.Namespace) -> None: + print("\n".join(list_secrets())) + + def get_command(args: argparse.Namespace) -> None: secret: str = args.secret ensure_sops_key() diff --git a/pkgs/clan-cli/clan_cli/secrets/users.py b/pkgs/clan-cli/clan_cli/secrets/users.py index b39d00307..b5c48d216 100644 --- a/pkgs/clan-cli/clan_cli/secrets/users.py +++ b/pkgs/clan-cli/clan_cli/secrets/users.py @@ -19,12 +19,14 @@ def remove_user(name: str) -> None: remove_object(sops_users_folder(), name) -def list_users() -> None: - list_objects(sops_users_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None) +def list_users() -> list[str]: + return list_objects( + sops_users_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None + ) def list_command(args: argparse.Namespace) -> None: - list_users() + print("\n".join(list_users())) def add_command(args: argparse.Namespace) -> None: From 6c169b0bed739adedfdf4170bf83d8f4b6216f75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 14:22:56 +0200 Subject: [PATCH 28/36] print nothing if secret collections are empty --- pkgs/clan-cli/clan_cli/secrets/machines.py | 22 ++++++++++++++++------ pkgs/clan-cli/clan_cli/secrets/secrets.py | 4 +++- pkgs/clan-cli/clan_cli/secrets/users.py | 18 +++++++++++++----- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/machines.py b/pkgs/clan-cli/clan_cli/secrets/machines.py index 330f2ad64..ebb9fb8a2 100644 --- a/pkgs/clan-cli/clan_cli/secrets/machines.py +++ b/pkgs/clan-cli/clan_cli/secrets/machines.py @@ -23,8 +23,20 @@ def list_machines() -> list[str]: return list_objects(sops_machines_folder(), lambda x: validate_hostname(x)) +def add_secret(machine: str, secret: str) -> None: + secrets.allow_member( + secrets.machines_folder(secret), sops_machines_folder(), machine + ) + + +def remove_secret(machine: str, secret: str) -> None: + secrets.disallow_member(secrets.machines_folder(secret), machine) + + def list_command(args: argparse.Namespace) -> None: - print("\n".join(list_machines())) + lst = list_machines() + if len(lst) > 0: + print("\n".join(lst)) def add_command(args: argparse.Namespace) -> None: @@ -32,17 +44,15 @@ def add_command(args: argparse.Namespace) -> None: def remove_command(args: argparse.Namespace) -> None: - remove_object(sops_machines_folder(), args.machine) + remove_machine(args.machine) def add_secret_command(args: argparse.Namespace) -> None: - secrets.allow_member( - secrets.machines_folder(args.secret), sops_machines_folder(), args.machine - ) + add_secret(args.machine, args.secret) def remove_secret_command(args: argparse.Namespace) -> None: - secrets.disallow_member(secrets.machines_folder(args.secret), args.machine) + remove_secret(args.machine, args.secret) def register_machines_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index 4cc78e03c..ed59022a8 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -178,7 +178,9 @@ def list_secrets() -> list[str]: def list_command(args: argparse.Namespace) -> None: - print("\n".join(list_secrets())) + lst = list_secrets() + if len(lst) > 0: + print("\n".join(lst)) def get_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/users.py b/pkgs/clan-cli/clan_cli/secrets/users.py index b5c48d216..25cf28ae2 100644 --- a/pkgs/clan-cli/clan_cli/secrets/users.py +++ b/pkgs/clan-cli/clan_cli/secrets/users.py @@ -25,8 +25,18 @@ def list_users() -> list[str]: ) +def add_secret(user: str, secret: str) -> None: + secrets.allow_member(secrets.users_folder(secret), sops_users_folder(), user) + + +def remove_secret(user: str, secret: str) -> None: + secrets.disallow_member(secrets.users_folder(secret), user) + + def list_command(args: argparse.Namespace) -> None: - print("\n".join(list_users())) + lst = list_users() + if len(lst) > 0: + print("\n".join(lst)) def add_command(args: argparse.Namespace) -> None: @@ -38,13 +48,11 @@ def remove_command(args: argparse.Namespace) -> None: def add_secret_command(args: argparse.Namespace) -> None: - secrets.allow_member( - secrets.users_folder(args.secret), sops_users_folder(), args.user - ) + add_secret(args.user, args.secret) def remove_secret_command(args: argparse.Namespace) -> None: - secrets.disallow_member(secrets.users_folder(args.secret), args.user) + remove_secret(args.user, args.secret) def register_users_parser(parser: argparse.ArgumentParser) -> None: From caa1c0dfd81c78917f11cdac83cea6fa4aafc701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 15:16:35 +0200 Subject: [PATCH 29/36] pre-commit: fix weird bug where commit_files is undefined in bash --- scripts/pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pre-commit b/scripts/pre-commit index e24104482..61b1010ea 100755 --- a/scripts/pre-commit +++ b/scripts/pre-commit @@ -18,7 +18,7 @@ log() { } # If the commit has no files, skip everything as there is nothing to format -if [[ ${#commit_files} = 0 ]]; then +if [[ -z ${commit_files+x} ]] || [[ ${#commit_files} = 0 ]]; then log "no files to format" exit 0 fi From e103a4186c8aff9fd7fef8e1fe25d235cd93a7b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 15:06:32 +0200 Subject: [PATCH 30/36] add api for secret groups and decrypting secrets --- pkgs/clan-cli/clan_cli/secrets/groups.py | 54 ++++++++++++++++------- pkgs/clan-cli/clan_cli/secrets/secrets.py | 9 ++-- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/groups.py b/pkgs/clan-cli/clan_cli/secrets/groups.py index a5f8acd8e..07c9c8625 100644 --- a/pkgs/clan-cli/clan_cli/secrets/groups.py +++ b/pkgs/clan-cli/clan_cli/secrets/groups.py @@ -23,29 +23,51 @@ def users_folder(group: str) -> Path: return sops_groups_folder() / group / "users" -# TODO: make this a tree -def list_command(args: argparse.Namespace) -> None: +class Group: + def __init__(self, name: str, machines: list[str], users: list[str]) -> None: + self.name = name + self.machines = machines + self.users = users + + +def list_groups() -> list[Group]: + groups = [] folder = sops_groups_folder() if not folder.exists(): - return + return groups - for group in os.listdir(folder): - group_folder = folder / group + for name in os.listdir(folder): + group_folder = folder / name if not group_folder.is_dir(): continue - print(group) - machines = machines_folder(group) - if machines.is_dir(): - print("machines:") - for f in machines.iterdir(): + machines_path = machines_folder(name) + machines = [] + if machines_path.is_dir(): + for f in machines_path.iterdir(): if validate_hostname(f.name): - print(f.name) - users = users_folder(group) - if users.is_dir(): - print("users:") - for f in users.iterdir(): + machines.append(f.name) + users_path = users_folder(name) + users = [] + if users_path.is_dir(): + for f in users_path.iterdir(): if VALID_USER_NAME.match(f.name): - print(f) + users.append(f.name) + groups.append(Group(name, machines, users)) + return groups + + +def list_command(args: argparse.Namespace) -> None: + for group in list_groups(): + print(group.name) + if group.machines: + print("machines:") + for machine in group.machines: + print(f" {machine}") + if group.users: + print("users:") + for user in group.users: + print(f" {user}") + print() def list_directory(directory: Path) -> str: diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index ed59022a8..72ac68569 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -183,13 +183,16 @@ def list_command(args: argparse.Namespace) -> None: print("\n".join(lst)) -def get_command(args: argparse.Namespace) -> None: - secret: str = args.secret +def decrypt_secret(secret: str) -> str: ensure_sops_key() secret_path = sops_secrets_folder() / secret / "secret" if not secret_path.exists(): raise ClanError(f"Secret '{secret}' does not exist") - print(decrypt_file(secret_path), end="") + return decrypt_file(secret_path) + + +def get_command(args: argparse.Namespace) -> None: + print(decrypt_secret(args.secret), end="") def set_command(args: argparse.Namespace) -> None: From 1f79a610d4a3d7920fa78f51af501ce6e537ca0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 15:23:27 +0200 Subject: [PATCH 31/36] groups: add more api --- pkgs/clan-cli/clan_cli/secrets/groups.py | 44 +++++++++++++++++------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/groups.py b/pkgs/clan-cli/clan_cli/secrets/groups.py index 07c9c8625..2d13614ad 100644 --- a/pkgs/clan-cli/clan_cli/secrets/groups.py +++ b/pkgs/clan-cli/clan_cli/secrets/groups.py @@ -31,7 +31,7 @@ class Group: def list_groups() -> list[Group]: - groups = [] + groups: list[Group] = [] folder = sops_groups_folder() if not folder.exists(): return groups @@ -111,38 +111,56 @@ def remove_member(group_folder: Path, name: str) -> None: os.rmdir(group_folder.parent) +def add_user(group: str, name: str) -> None: + add_member(users_folder(group), sops_users_folder(), name) + + def add_user_command(args: argparse.Namespace) -> None: - add_member(users_folder(args.group), sops_users_folder(), args.user) + add_user(args.group, args.user) + + +def remove_user(group: str, name: str) -> None: + remove_member(users_folder(group), name) def remove_user_command(args: argparse.Namespace) -> None: - remove_member(users_folder(args.group), args.user) + remove_user(args.group, args.user) + + +def add_machine(group: str, name: str) -> None: + add_member(machines_folder(group), sops_machines_folder(), name) def add_machine_command(args: argparse.Namespace) -> None: - add_member( - machines_folder(args.group), - sops_machines_folder(), - args.machine, - ) + add_machine(args.group, args.machine) + + +def remove_machine(group: str, name: str) -> None: + remove_member(machines_folder(group), name) def remove_machine_command(args: argparse.Namespace) -> None: - remove_member(machines_folder(args.group), args.machine) + remove_machine(args.group, args.machine) def add_group_argument(parser: argparse.ArgumentParser) -> None: parser.add_argument("group", help="the name of the secret", type=group_name_type) +def add_secret(group: str, name: str) -> None: + secrets.allow_member(secrets.groups_folder(name), sops_groups_folder(), group) + + def add_secret_command(args: argparse.Namespace) -> None: - secrets.allow_member( - secrets.groups_folder(args.secret), sops_groups_folder(), args.group - ) + add_secret(args.group, args.secret) + + +def remove_secret(group: str, name: str) -> None: + secrets.disallow_member(secrets.groups_folder(name), group) def remove_secret_command(args: argparse.Namespace) -> None: - secrets.disallow_member(secrets.groups_folder(args.secret), args.group) + remove_secret(args.group, args.secret) def register_groups_parser(parser: argparse.ArgumentParser) -> None: From 8fea55da32f9f2c348c42cca616c486ba3486764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 15:40:31 +0200 Subject: [PATCH 32/36] move ssh cli to cli submodule --- pkgs/clan-cli/clan_cli/__init__.py | 5 ++- .../clan_cli/{deploy => ssh}/__init__.py | 44 +++++++++---------- pkgs/clan-cli/clan_cli/{ssh.py => ssh/cli.py} | 2 +- pkgs/clan-cli/tests/test_clan_ssh.py | 8 ++-- 4 files changed, 30 insertions(+), 29 deletions(-) rename pkgs/clan-cli/clan_cli/{deploy => ssh}/__init__.py (96%) rename pkgs/clan-cli/clan_cli/{ssh.py => ssh/cli.py} (98%) diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index acb72c483..32144ec64 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -1,8 +1,9 @@ import argparse import sys -from . import admin, secrets, ssh +from . import admin, secrets from .errors import ClanError +from .ssh import cli as ssh_cli has_argcomplete = True try: @@ -27,7 +28,7 @@ def main() -> None: # warn(f"The config command does not work in the nix sandbox: {e}") parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine") - ssh.register_parser(parser_ssh) + ssh_cli.register_parser(parser_ssh) parser_secrets = subparsers.add_parser("secrets", help="manage secrets") secrets.register_parser(parser_secrets) diff --git a/pkgs/clan-cli/clan_cli/deploy/__init__.py b/pkgs/clan-cli/clan_cli/ssh/__init__.py similarity index 96% rename from pkgs/clan-cli/clan_cli/deploy/__init__.py rename to pkgs/clan-cli/clan_cli/ssh/__init__.py index 189967404..8b7165c80 100644 --- a/pkgs/clan-cli/clan_cli/deploy/__init__.py +++ b/pkgs/clan-cli/clan_cli/ssh/__init__.py @@ -144,7 +144,7 @@ class HostKeyCheck(Enum): NONE = 2 -class DeployHost: +class Host: def __init__( self, host: str, @@ -158,7 +158,7 @@ class DeployHost: verbose_ssh: bool = False, ) -> None: """ - Creates a DeployHost + Creates a Host @host the hostname to connect to via ssh @port the port to connect to via ssh @forward_agent: wheter to forward ssh agent @@ -495,7 +495,7 @@ T = TypeVar("T") class HostResult(Generic[T]): - def __init__(self, host: DeployHost, result: Union[T, Exception]) -> None: + def __init__(self, host: Host, result: Union[T, Exception]) -> None: self.host = host self._result = result @@ -518,12 +518,12 @@ class HostResult(Generic[T]): return self._result -DeployResults = List[HostResult[subprocess.CompletedProcess[str]]] +Results = List[HostResult[subprocess.CompletedProcess[str]]] def _worker( - func: Callable[[DeployHost], T], - host: DeployHost, + func: Callable[[Host], T], + host: Host, results: List[HostResult[T]], idx: int, ) -> None: @@ -534,15 +534,15 @@ def _worker( results[idx] = HostResult(host, e) -class DeployGroup: - def __init__(self, hosts: List[DeployHost]) -> None: +class Group: + def __init__(self, hosts: List[Host]) -> None: self.hosts = hosts def _run_local( self, cmd: Union[str, List[str]], - host: DeployHost, - results: DeployResults, + host: Host, + results: Results, stdout: FILE = None, stderr: FILE = None, extra_env: Dict[str, str] = {}, @@ -569,8 +569,8 @@ class DeployGroup: def _run_remote( self, cmd: Union[str, List[str]], - host: DeployHost, - results: DeployResults, + host: Host, + results: Results, stdout: FILE = None, stderr: FILE = None, extra_env: Dict[str, str] = {}, @@ -621,8 +621,8 @@ class DeployGroup: check: bool = True, verbose_ssh: bool = False, timeout: float = math.inf, - ) -> DeployResults: - results: DeployResults = [] + ) -> Results: + results: Results = [] threads = [] for host in self.hosts: fn = self._run_local if local else self._run_remote @@ -662,7 +662,7 @@ class DeployGroup: check: bool = True, verbose_ssh: bool = False, timeout: float = math.inf, - ) -> DeployResults: + ) -> Results: """ Command to run on the remote host via ssh @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE @@ -671,7 +671,7 @@ class DeployGroup: @verbose_ssh: Enables verbose logging on ssh connections @timeout: Timeout in seconds for the command to complete - @return a lists of tuples containing DeployNode and the result of the command for this DeployNode + @return a lists of tuples containing Host and the result of the command for this Host """ return self._run( cmd, @@ -693,7 +693,7 @@ class DeployGroup: cwd: Union[None, str, Path] = None, check: bool = True, timeout: float = math.inf, - ) -> DeployResults: + ) -> Results: """ Command to run locally for each host in the group in parallel @cmd the commmand to run @@ -703,7 +703,7 @@ class DeployGroup: @extra_env environment variables to override whe running the command @timeout: Timeout in seconds for the command to complete - @return a lists of tuples containing DeployNode and the result of the command for this DeployNode + @return a lists of tuples containing Host and the result of the command for this Host """ return self._run( cmd, @@ -717,7 +717,7 @@ class DeployGroup: ) def run_function( - self, func: Callable[[DeployHost], T], check: bool = True + self, func: Callable[[Host], T], check: bool = True ) -> List[HostResult[T]]: """ Function to run for each host in the group in parallel @@ -745,9 +745,9 @@ class DeployGroup: self._reraise_errors(results) return results - def filter(self, pred: Callable[[DeployHost], bool]) -> "DeployGroup": - """Return a new DeployGroup with the results filtered by the predicate""" - return DeployGroup(list(filter(pred, self.hosts))) + def filter(self, pred: Callable[[Host], bool]) -> "Group": + """Return a new Group with the results filtered by the predicate""" + return Group(list(filter(pred, self.hosts))) @overload diff --git a/pkgs/clan-cli/clan_cli/ssh.py b/pkgs/clan-cli/clan_cli/ssh/cli.py similarity index 98% rename from pkgs/clan-cli/clan_cli/ssh.py rename to pkgs/clan-cli/clan_cli/ssh/cli.py index 6c8bd22ec..f7966d083 100644 --- a/pkgs/clan-cli/clan_cli/ssh.py +++ b/pkgs/clan-cli/clan_cli/ssh/cli.py @@ -3,7 +3,7 @@ import json import subprocess from typing import Optional -from .nix import nix_shell +from ..nix import nix_shell def ssh( diff --git a/pkgs/clan-cli/tests/test_clan_ssh.py b/pkgs/clan-cli/tests/test_clan_ssh.py index ed1d366e3..3c6a37ce9 100644 --- a/pkgs/clan-cli/tests/test_clan_ssh.py +++ b/pkgs/clan-cli/tests/test_clan_ssh.py @@ -6,7 +6,7 @@ import pytest_subprocess.fake_process from environment import mock_env from pytest_subprocess import utils -import clan_cli.ssh +from clan_cli.ssh import cli def test_no_args( @@ -40,7 +40,7 @@ def test_ssh_no_pass(fp: pytest_subprocess.fake_process.FakeProcess) -> None: fp.any(), ] fp.register(cmd) - clan_cli.ssh.ssh( + cli.ssh( host=host, user=user, ) @@ -64,7 +64,7 @@ def test_ssh_with_pass(fp: pytest_subprocess.fake_process.FakeProcess) -> None: fp.any(), ] fp.register(cmd) - clan_cli.ssh.ssh( + cli.ssh( host=host, user=user, password="XXX", @@ -75,5 +75,5 @@ def test_ssh_with_pass(fp: pytest_subprocess.fake_process.FakeProcess) -> None: def test_qrcode_scan(fp: pytest_subprocess.fake_process.FakeProcess) -> None: cmd: list[Union[str, utils.Any]] = [fp.any()] fp.register(cmd, stdout="https://test.test") - result = clan_cli.ssh.qrcode_scan("test.png") + result = cli.qrcode_scan("test.png") assert result == "https://test.test" From d977499ffeffc3e53311b8688321d47f5ed403c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 15:51:10 +0200 Subject: [PATCH 33/36] rename cli tests to have a common suffix rename test_import_sops -> test_import_sops --- .../tests/{test_keys.py => age_keys.py} | 2 +- pkgs/clan-cli/tests/conftest.py | 2 +- .../{test_clan_admin.py => test_admin_cli.py} | 0 ...import_sops.py => test_import_sops_cli.py} | 12 +++---- .../{test_secrets.py => test_secrets_cli.py} | 36 +++++++++---------- .../{test_clan_ssh.py => test_ssh_cli.py} | 1 + 6 files changed, 27 insertions(+), 26 deletions(-) rename pkgs/clan-cli/tests/{test_keys.py => age_keys.py} (95%) rename pkgs/clan-cli/tests/{test_clan_admin.py => test_admin_cli.py} (100%) rename pkgs/clan-cli/tests/{test_import_sops.py => test_import_sops_cli.py} (78%) rename pkgs/clan-cli/tests/{test_secrets.py => test_secrets_cli.py} (78%) rename pkgs/clan-cli/tests/{test_clan_ssh.py => test_ssh_cli.py} (99%) diff --git a/pkgs/clan-cli/tests/test_keys.py b/pkgs/clan-cli/tests/age_keys.py similarity index 95% rename from pkgs/clan-cli/tests/test_keys.py rename to pkgs/clan-cli/tests/age_keys.py index 518a2becd..5a0e038ad 100644 --- a/pkgs/clan-cli/tests/test_keys.py +++ b/pkgs/clan-cli/tests/age_keys.py @@ -24,7 +24,7 @@ KEYS = [ @pytest.fixture -def test_keys() -> list[KeyPair]: +def age_keys() -> list[KeyPair]: """ Root directory of the tests """ diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index 7bde26d77..ec743b128 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -3,4 +3,4 @@ import sys sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) -pytest_plugins = ["temporary_dir", "clan_flake", "root", "test_keys"] +pytest_plugins = ["temporary_dir", "clan_flake", "root", "age_keys"] diff --git a/pkgs/clan-cli/tests/test_clan_admin.py b/pkgs/clan-cli/tests/test_admin_cli.py similarity index 100% rename from pkgs/clan-cli/tests/test_clan_admin.py rename to pkgs/clan-cli/tests/test_admin_cli.py diff --git a/pkgs/clan-cli/tests/test_import_sops.py b/pkgs/clan-cli/tests/test_import_sops_cli.py similarity index 78% rename from pkgs/clan-cli/tests/test_import_sops.py rename to pkgs/clan-cli/tests/test_import_sops_cli.py index b578dd0f4..73a6a1349 100644 --- a/pkgs/clan-cli/tests/test_import_sops.py +++ b/pkgs/clan-cli/tests/test_import_sops_cli.py @@ -6,21 +6,21 @@ from environment import mock_env from secret_cli import SecretCli if TYPE_CHECKING: - from test_keys import KeyPair + from age_keys import KeyPair def test_import_sops( test_root: Path, clan_flake: Path, capsys: pytest.CaptureFixture, - test_keys: list["KeyPair"], + age_keys: list["KeyPair"], ) -> None: cli = SecretCli() - with mock_env(SOPS_AGE_KEY=test_keys[1].privkey): - cli.run(["machines", "add", "machine1", test_keys[0].pubkey]) - cli.run(["users", "add", "user1", test_keys[1].pubkey]) - cli.run(["users", "add", "user2", test_keys[2].pubkey]) + with mock_env(SOPS_AGE_KEY=age_keys[1].privkey): + cli.run(["machines", "add", "machine1", age_keys[0].pubkey]) + cli.run(["users", "add", "user1", age_keys[1].pubkey]) + cli.run(["users", "add", "user2", age_keys[2].pubkey]) cli.run(["groups", "add-user", "group1", "user1"]) cli.run(["groups", "add-user", "group1", "user2"]) diff --git a/pkgs/clan-cli/tests/test_secrets.py b/pkgs/clan-cli/tests/test_secrets_cli.py similarity index 78% rename from pkgs/clan-cli/tests/test_secrets.py rename to pkgs/clan-cli/tests/test_secrets_cli.py index 3b555f6e3..1d5ec3dae 100644 --- a/pkgs/clan-cli/tests/test_secrets.py +++ b/pkgs/clan-cli/tests/test_secrets_cli.py @@ -9,22 +9,22 @@ from secret_cli import SecretCli from clan_cli.errors import ClanError if TYPE_CHECKING: - from test_keys import KeyPair + from age_keys import KeyPair def _test_identities( what: str, clan_flake: Path, capsys: pytest.CaptureFixture, - test_keys: list["KeyPair"], + age_keys: list["KeyPair"], ) -> None: cli = SecretCli() sops_folder = clan_flake / "sops" - cli.run([what, "add", "foo", test_keys[0].pubkey]) + cli.run([what, "add", "foo", age_keys[0].pubkey]) assert (sops_folder / what / "foo" / "key.json").exists() with pytest.raises(ClanError): - cli.run([what, "add", "foo", test_keys[0].pubkey]) + cli.run([what, "add", "foo", age_keys[0].pubkey]) cli.run( [ @@ -32,7 +32,7 @@ def _test_identities( "add", "-f", "foo", - test_keys[0].privkey, + age_keys[0].privkey, ] ) capsys.readouterr() # empty the buffer @@ -54,19 +54,19 @@ def _test_identities( def test_users( - clan_flake: Path, capsys: pytest.CaptureFixture, test_keys: list["KeyPair"] + clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("users", clan_flake, capsys, test_keys) + _test_identities("users", clan_flake, capsys, age_keys) def test_machines( - clan_flake: Path, capsys: pytest.CaptureFixture, test_keys: list["KeyPair"] + clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("machines", clan_flake, capsys, test_keys) + _test_identities("machines", clan_flake, capsys, age_keys) def test_groups( - clan_flake: Path, capsys: pytest.CaptureFixture, test_keys: list["KeyPair"] + clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: cli = SecretCli() capsys.readouterr() # empty the buffer @@ -77,13 +77,13 @@ def test_groups( cli.run(["groups", "add-machine", "group1", "machine1"]) with pytest.raises(ClanError): # user does not exist yet cli.run(["groups", "add-user", "groupb1", "user1"]) - cli.run(["machines", "add", "machine1", test_keys[0].pubkey]) + cli.run(["machines", "add", "machine1", age_keys[0].pubkey]) cli.run(["groups", "add-machine", "group1", "machine1"]) # Should this fail? cli.run(["groups", "add-machine", "group1", "machine1"]) - cli.run(["users", "add", "user1", test_keys[0].pubkey]) + cli.run(["users", "add", "user1", age_keys[0].pubkey]) cli.run(["groups", "add-user", "group1", "user1"]) capsys.readouterr() # empty the buffer @@ -99,7 +99,7 @@ def test_groups( def test_secrets( - clan_flake: Path, capsys: pytest.CaptureFixture, test_keys: list["KeyPair"] + clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: cli = SecretCli() capsys.readouterr() # empty the buffer @@ -125,18 +125,18 @@ def test_secrets( cli.run(["list"]) assert capsys.readouterr().out == "key\n" - cli.run(["machines", "add", "machine1", test_keys[0].pubkey]) + cli.run(["machines", "add", "machine1", age_keys[0].pubkey]) cli.run(["machines", "add-secret", "machine1", "key"]) - with mock_env(SOPS_AGE_KEY=test_keys[0].privkey, SOPS_AGE_KEY_FILE=""): + with mock_env(SOPS_AGE_KEY=age_keys[0].privkey, SOPS_AGE_KEY_FILE=""): capsys.readouterr() cli.run(["get", "key"]) assert capsys.readouterr().out == "foo" cli.run(["machines", "remove-secret", "machine1", "key"]) - cli.run(["users", "add", "user1", test_keys[1].pubkey]) + cli.run(["users", "add", "user1", age_keys[1].pubkey]) cli.run(["users", "add-secret", "user1", "key"]) - with mock_env(SOPS_AGE_KEY=test_keys[1].privkey, SOPS_AGE_KEY_FILE=""): + with mock_env(SOPS_AGE_KEY=age_keys[1].privkey, SOPS_AGE_KEY_FILE=""): capsys.readouterr() cli.run(["get", "key"]) assert capsys.readouterr().out == "foo" @@ -151,7 +151,7 @@ def test_secrets( capsys.readouterr() # empty the buffer cli.run(["set", "--group", "admin-group", "key2"]) - with mock_env(SOPS_AGE_KEY=test_keys[1].privkey, SOPS_AGE_KEY_FILE=""): + with mock_env(SOPS_AGE_KEY=age_keys[1].privkey, SOPS_AGE_KEY_FILE=""): capsys.readouterr() cli.run(["get", "key"]) assert capsys.readouterr().out == "foo" diff --git a/pkgs/clan-cli/tests/test_clan_ssh.py b/pkgs/clan-cli/tests/test_ssh_cli.py similarity index 99% rename from pkgs/clan-cli/tests/test_clan_ssh.py rename to pkgs/clan-cli/tests/test_ssh_cli.py index 3c6a37ce9..884157116 100644 --- a/pkgs/clan-cli/tests/test_clan_ssh.py +++ b/pkgs/clan-cli/tests/test_ssh_cli.py @@ -6,6 +6,7 @@ import pytest_subprocess.fake_process from environment import mock_env from pytest_subprocess import utils +import clan_cli from clan_cli.ssh import cli From 9438a9eb5f14c5ae65d25047fc34c6a0d4ba33f3 Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 9 Aug 2023 12:10:27 +0200 Subject: [PATCH 34/36] clan-config: move jsonschema lib to clanLib --- lib/default.nix | 31 +++++++++---------- .../schema-lib.nix => lib/jsonschema.nix | 0 pkgs/clan-cli/clan_cli/config/__init__.py | 5 ++- pkgs/clan-cli/shell.nix | 2 +- .../tests/config/test_parseOption.nix | 2 +- .../tests/config/test_parseOptions.nix | 2 +- 6 files changed, 22 insertions(+), 20 deletions(-) rename pkgs/clan-cli/clan_cli/config/schema-lib.nix => lib/jsonschema.nix (100%) diff --git a/lib/default.nix b/lib/default.nix index b0c2432fa..292d46023 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -1,17 +1,16 @@ { lib, ... }: -let - clanLib = { - findNixFiles = folder: - lib.mapAttrs' - (name: type: - if - type == "directory" - then - lib.nameValuePair name "${folder}/${name}" - else - lib.nameValuePair (lib.removeSuffix ".nix" name) "${folder}/${name}" - ) - (builtins.readDir folder); - }; -in -clanLib +{ + findNixFiles = folder: + lib.mapAttrs' + (name: type: + if + type == "directory" + then + lib.nameValuePair name "${folder}/${name}" + else + lib.nameValuePair (lib.removeSuffix ".nix" name) "${folder}/${name}" + ) + (builtins.readDir folder); + + jsonschema = import ./jsonschema.nix { inherit lib; }; +} diff --git a/pkgs/clan-cli/clan_cli/config/schema-lib.nix b/lib/jsonschema.nix similarity index 100% rename from pkgs/clan-cli/clan_cli/config/schema-lib.nix rename to lib/jsonschema.nix diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index 707115e2a..c36e80c33 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -1,6 +1,7 @@ # !/usr/bin/env python3 import argparse import json +import os import subprocess import sys from pathlib import Path @@ -8,6 +9,8 @@ from typing import Any, Optional, Type, Union from clan_cli.errors import ClanError +CLAN_FLAKE = os.getenv("CLAN_FLAKE") + class Kwargs: def __init__(self) -> None: @@ -27,7 +30,7 @@ def schema_from_module_file( nix_expr = f""" let lib = import ; - slib = import {__file__}/../schema-lib.nix {{inherit lib;}}; + slib = import {CLAN_FLAKE}/lib/jsonschema.nix {{inherit lib;}}; in slib.parseModule {absolute_path} """ diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index cc6490a7c..61a98ffba 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -9,7 +9,7 @@ let ] ); checkScript = pkgs.writeScriptBin "check" '' - nix build -f . tests -L "$@" + nix build .#checks.${pkgs.system}.{treefmt,clan-mypy,clan-pytest} -L "$@" ''; in pkgs.mkShell { diff --git a/pkgs/clan-cli/tests/config/test_parseOption.nix b/pkgs/clan-cli/tests/config/test_parseOption.nix index b3e6173b5..7ea927630 100644 --- a/pkgs/clan-cli/tests/config/test_parseOption.nix +++ b/pkgs/clan-cli/tests/config/test_parseOption.nix @@ -1,7 +1,7 @@ # tests for the nixos options to jsonschema converter # run these tests via `nix-unit ./test.nix` { lib ? (import { }).lib -, slib ? import ../../clan_cli/config/schema-lib.nix { inherit lib; } +, slib ? import ../../../../lib/jsonschema.nix { inherit lib; } }: let description = "Test Description"; diff --git a/pkgs/clan-cli/tests/config/test_parseOptions.nix b/pkgs/clan-cli/tests/config/test_parseOptions.nix index 4787d9d95..14b8d31ac 100644 --- a/pkgs/clan-cli/tests/config/test_parseOptions.nix +++ b/pkgs/clan-cli/tests/config/test_parseOptions.nix @@ -1,7 +1,7 @@ # tests for the nixos options to jsonschema converter # run these tests via `nix-unit ./test.nix` { lib ? (import { }).lib -, slib ? import ../../clan_cli/config/schema-lib.nix { inherit lib; } +, slib ? import ../../../../lib/jsonschema.nix { inherit lib; } }: let evaledOptions = From 7262208a4c47cb6ed9f3854532352cc6df74909f Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 9 Aug 2023 16:05:16 +0200 Subject: [PATCH 35/36] clanLib.jsonschema: move tests from pkgs/clan-cli --- flake.nix | 1 + lib/default.nix | 2 +- lib/flake-module.nix | 3 ++ .../default.nix} | 0 .../jsonschema}/example-data.json | 0 .../jsonschema}/example-interface.nix | 0 .../jsonschema}/example-schema.json | 0 lib/jsonschema/flake-module.nix | 29 +++++++++++++++++++ .../tests/config => lib/jsonschema}/test.nix | 2 +- .../jsonschema}/test_parseOption.nix | 2 +- .../jsonschema}/test_parseOptions.nix | 2 +- pkgs/clan-cli/flake-module.nix | 25 +--------------- 12 files changed, 38 insertions(+), 28 deletions(-) rename lib/{jsonschema.nix => jsonschema/default.nix} (100%) rename {pkgs/clan-cli/tests/config => lib/jsonschema}/example-data.json (100%) rename {pkgs/clan-cli/tests/config => lib/jsonschema}/example-interface.nix (100%) rename {pkgs/clan-cli/tests/config => lib/jsonschema}/example-schema.json (100%) create mode 100644 lib/jsonschema/flake-module.nix rename {pkgs/clan-cli/tests/config => lib/jsonschema}/test.nix (76%) rename {pkgs/clan-cli/tests/config => lib/jsonschema}/test_parseOption.nix (98%) rename {pkgs/clan-cli/tests/config => lib/jsonschema}/test_parseOptions.nix (87%) diff --git a/flake.nix b/flake.nix index c52398e17..50d88a32e 100644 --- a/flake.nix +++ b/flake.nix @@ -22,6 +22,7 @@ "aarch64-linux" ]; imports = [ + # ./checks/flake-module.nix ./devShell.nix ./formatter.nix ./templates/flake-module.nix diff --git a/lib/default.nix b/lib/default.nix index 292d46023..855c3d75b 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -12,5 +12,5 @@ ) (builtins.readDir folder); - jsonschema = import ./jsonschema.nix { inherit lib; }; + jsonschema = import ./jsonschema { inherit lib; }; } diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 38d84a837..2738bef8d 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -1,5 +1,8 @@ { lib , ... }: { + imports = [ + ./jsonschema/flake-module.nix + ]; flake.lib = import ./default.nix { inherit lib; }; } diff --git a/lib/jsonschema.nix b/lib/jsonschema/default.nix similarity index 100% rename from lib/jsonschema.nix rename to lib/jsonschema/default.nix diff --git a/pkgs/clan-cli/tests/config/example-data.json b/lib/jsonschema/example-data.json similarity index 100% rename from pkgs/clan-cli/tests/config/example-data.json rename to lib/jsonschema/example-data.json diff --git a/pkgs/clan-cli/tests/config/example-interface.nix b/lib/jsonschema/example-interface.nix similarity index 100% rename from pkgs/clan-cli/tests/config/example-interface.nix rename to lib/jsonschema/example-interface.nix diff --git a/pkgs/clan-cli/tests/config/example-schema.json b/lib/jsonschema/example-schema.json similarity index 100% rename from pkgs/clan-cli/tests/config/example-schema.json rename to lib/jsonschema/example-schema.json diff --git a/lib/jsonschema/flake-module.nix b/lib/jsonschema/flake-module.nix new file mode 100644 index 000000000..26fc76cf4 --- /dev/null +++ b/lib/jsonschema/flake-module.nix @@ -0,0 +1,29 @@ +{ + perSystem = { pkgs, self', ... }: { + checks = { + + # check if the `clan config` example jsonschema and data is valid + lib-jsonschema-example-valid = pkgs.runCommand "lib-jsonschema-example-valid" { } '' + echo "Checking that example-schema.json is valid" + ${pkgs.check-jsonschema}/bin/check-jsonschema \ + --check-metaschema ${./.}/example-schema.json + + echo "Checking that example-data.json is valid according to example-schema.json" + ${pkgs.check-jsonschema}/bin/check-jsonschema \ + --schemafile ${./.}/example-schema.json \ + ${./.}/example-data.json + + touch $out + ''; + + # check if the `clan config` nix jsonschema converter unit tests succeed + lib-jsonschema-nix-unit-tests = pkgs.runCommand "lib-jsonschema-nix-unit-tests" { } '' + export NIX_PATH=nixpkgs=${pkgs.path} + ${self'.packages.nix-unit}/bin/nix-unit \ + ${./.}/test.nix \ + --eval-store $(realpath .) + touch $out + ''; + }; + }; +} diff --git a/pkgs/clan-cli/tests/config/test.nix b/lib/jsonschema/test.nix similarity index 76% rename from pkgs/clan-cli/tests/config/test.nix rename to lib/jsonschema/test.nix index 8a39ed248..34e05274b 100644 --- a/pkgs/clan-cli/tests/config/test.nix +++ b/lib/jsonschema/test.nix @@ -1,6 +1,6 @@ # run these tests via `nix-unit ./test.nix` { lib ? (import { }).lib -, slib ? import ../../clan_cli/config/schema-lib.nix { inherit lib; } +, slib ? import ./. { inherit lib; } }: { parseOption = import ./test_parseOption.nix { inherit lib slib; }; diff --git a/pkgs/clan-cli/tests/config/test_parseOption.nix b/lib/jsonschema/test_parseOption.nix similarity index 98% rename from pkgs/clan-cli/tests/config/test_parseOption.nix rename to lib/jsonschema/test_parseOption.nix index 7ea927630..7adb3d660 100644 --- a/pkgs/clan-cli/tests/config/test_parseOption.nix +++ b/lib/jsonschema/test_parseOption.nix @@ -1,7 +1,7 @@ # tests for the nixos options to jsonschema converter # run these tests via `nix-unit ./test.nix` { lib ? (import { }).lib -, slib ? import ../../../../lib/jsonschema.nix { inherit lib; } +, slib ? import ./. { inherit lib; } }: let description = "Test Description"; diff --git a/pkgs/clan-cli/tests/config/test_parseOptions.nix b/lib/jsonschema/test_parseOptions.nix similarity index 87% rename from pkgs/clan-cli/tests/config/test_parseOptions.nix rename to lib/jsonschema/test_parseOptions.nix index 14b8d31ac..c635286de 100644 --- a/pkgs/clan-cli/tests/config/test_parseOptions.nix +++ b/lib/jsonschema/test_parseOptions.nix @@ -1,7 +1,7 @@ # tests for the nixos options to jsonschema converter # run these tests via `nix-unit ./test.nix` { lib ? (import { }).lib -, slib ? import ../../../../lib/jsonschema.nix { inherit lib; } +, slib ? import ./. { inherit lib; } }: let evaledOptions = diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 4a0370344..30581d2e5 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -27,30 +27,7 @@ ## End optional dependencies }; - checks = self'.packages.clan-cli.tests // { - # check if the `clan config` example jsonschema and data is valid - clan-config-example-schema-valid = pkgs.runCommand "clan-config-example-schema-valid" { } '' - echo "Checking that example-schema.json is valid" - ${pkgs.check-jsonschema}/bin/check-jsonschema \ - --check-metaschema ${./.}/tests/config/example-schema.json - - echo "Checking that example-data.json is valid according to example-schema.json" - ${pkgs.check-jsonschema}/bin/check-jsonschema \ - --schemafile ${./.}/tests/config/example-schema.json \ - ${./.}/tests/config/example-data.json - - touch $out - ''; - - # check if the `clan config` nix jsonschema converter unit tests succeed - clan-config-nix-unit-tests = pkgs.runCommand "clan-edit-unit-tests" { } '' - export NIX_PATH=nixpkgs=${pkgs.path} - ${self'.packages.nix-unit}/bin/nix-unit \ - ${./.}/tests/config/test.nix \ - --eval-store $(realpath .) - touch $out - ''; - }; + checks = self'.packages.clan-cli.tests; }; } From f8f9cd457ae26752c8304603bbfe481a89abcf99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 9 Aug 2023 16:23:15 +0200 Subject: [PATCH 36/36] add test for local commands --- pkgs/clan-cli/tests/test_ssh_local.py | 97 +++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 pkgs/clan-cli/tests/test_ssh_local.py diff --git a/pkgs/clan-cli/tests/test_ssh_local.py b/pkgs/clan-cli/tests/test_ssh_local.py new file mode 100644 index 000000000..b0073101f --- /dev/null +++ b/pkgs/clan-cli/tests/test_ssh_local.py @@ -0,0 +1,97 @@ +import subprocess + +from clan_cli.ssh import Group, Host, run + + +def test_run() -> None: + p = run("echo hello") + assert p.stdout is None + + +def test_run_failure() -> None: + p = run("exit 1", check=False) + assert p.returncode == 1 + + try: + p = run("exit 1") + except Exception: + pass + else: + assert False, "Command should have raised an error" + + +hosts = Group([Host("some_host")]) + + +def test_run_environment() -> None: + p1 = run("echo $env_var", stdout=subprocess.PIPE, extra_env=dict(env_var="true")) + assert p1.stdout == "true\n" + + p2 = hosts.run_local( + "echo $env_var", extra_env=dict(env_var="true"), stdout=subprocess.PIPE + ) + assert p2[0].result.stdout == "true\n" + + p3 = hosts.run_local( + ["env"], extra_env=dict(env_var="true"), stdout=subprocess.PIPE + ) + assert "env_var=true" in p3[0].result.stdout + + +def test_run_non_shell() -> None: + p = run(["echo", "$hello"], stdout=subprocess.PIPE) + assert p.stdout == "$hello\n" + + +def test_run_stderr_stdout() -> None: + p = run("echo 1; echo 2 >&2", stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert p.stdout == "1\n" + assert p.stderr == "2\n" + + +def test_run_local() -> None: + hosts.run_local("echo hello") + + +def test_timeout() -> None: + try: + hosts.run_local("sleep 10", timeout=0.01) + except Exception: + pass + else: + assert False, "should have raised TimeoutExpired" + + +def test_run_function() -> None: + def some_func(h: Host) -> bool: + p = h.run_local("echo hello", stdout=subprocess.PIPE) + return p.stdout == "hello\n" + + res = hosts.run_function(some_func) + assert res[0].result + + +def test_run_exception() -> None: + try: + hosts.run_local("exit 1") + except Exception: + pass + else: + assert False, "should have raised Exception" + + +def test_run_function_exception() -> None: + def some_func(h: Host) -> None: + h.run_local("exit 1") + + try: + hosts.run_function(some_func) + except Exception: + pass + else: + assert False, "should have raised Exception" + + +def test_run_local_non_shell() -> None: + p2 = hosts.run_local(["echo", "1"], stdout=subprocess.PIPE) + assert p2[0].result.stdout == "1\n"