From c9e640a74d7ddacd7f1ed534d179c13bb09a5e5d Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 28 Nov 2023 13:23:48 +0100 Subject: [PATCH 1/2] add dummy backups cli --- pkgs/clan-cli/clan_cli/__init__.py | 7 ++- pkgs/clan-cli/clan_cli/backups/__init__.py | 25 ++++++++++ pkgs/clan-cli/clan_cli/backups/create.py | 34 +++++++++++++ pkgs/clan-cli/clan_cli/backups/list.py | 56 ++++++++++++++++++++++ pkgs/clan-cli/clan_cli/backups/restore.py | 27 +++++++++++ 5 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 pkgs/clan-cli/clan_cli/backups/__init__.py create mode 100644 pkgs/clan-cli/clan_cli/backups/create.py create mode 100644 pkgs/clan-cli/clan_cli/backups/list.py create mode 100644 pkgs/clan-cli/clan_cli/backups/restore.py diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 597955023..190d2834d 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -5,7 +5,7 @@ from pathlib import Path from types import ModuleType from typing import Any, Optional, Sequence -from . import config, flakes, machines, secrets, vms, webui +from . import config, flakes, machines, secrets, vms, webui, backups from .custom_logger import setup_logging from .dirs import get_clan_flake_toplevel from .ssh import cli as ssh_cli @@ -64,6 +64,11 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser: subparsers = parser.add_subparsers() + parser_backups = subparsers.add_parser( + "backups", help="manage backups of clan machines" + ) + backups.register_parser(parser_backups) + parser_flake = subparsers.add_parser( "flakes", help="create a clan flake inside the current directory" ) diff --git a/pkgs/clan-cli/clan_cli/backups/__init__.py b/pkgs/clan-cli/clan_cli/backups/__init__.py new file mode 100644 index 000000000..b7ce53971 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/backups/__init__.py @@ -0,0 +1,25 @@ +# !/usr/bin/env python3 +import argparse + +from .list import register_list_parser +from .create import register_create_parser +from .restore import register_restore_parser + + +# takes a (sub)parser and configures it +def register_parser(parser: argparse.ArgumentParser) -> None: + subparser = parser.add_subparsers( + title="command", + description="the command to run", + help="the command to run", + required=True, + ) + + list_parser = subparser.add_parser("list", help="list backups") + register_list_parser(list_parser) + + create_parser = subparser.add_parser("create", help="create backups") + register_create_parser(create_parser) + + restore_parser = subparser.add_parser("restore", help="restore backups") + register_restore_parser(restore_parser) diff --git a/pkgs/clan-cli/clan_cli/backups/create.py b/pkgs/clan-cli/clan_cli/backups/create.py new file mode 100644 index 000000000..48b08db5a --- /dev/null +++ b/pkgs/clan-cli/clan_cli/backups/create.py @@ -0,0 +1,34 @@ +import argparse +import pprint +from pathlib import Path +from typing import Optional + +from ..errors import ClanError + + +def create_backup(flake_dir: Path, machine: Optional[str] = None, provider: Optional[str] = None) -> None: + if machine is None: + # TODO get all machines here + machines = [ "machine1", "machine2" ] + else: + machines = [ machine ] + + if provider is None: + # TODO get all providers here + providers = [ "provider1", "provider2" ] + else: + providers = [ provider ] + + print("would create backups for machines: ", machines, " with providers: ", providers) + + +def create_command(args: argparse.Namespace) -> None: + if args.flake is None: + raise ClanError("Could not find clan flake toplevel directory") + create_backup(Path(args.flake), machine=args.machine, provider=args.provider) + + +def register_create_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--machine", type=str, help="machine in the flake to create backups of") + parser.add_argument("--provider", type=str, help="backup provider to use") + parser.set_defaults(func=create_command) diff --git a/pkgs/clan-cli/clan_cli/backups/list.py b/pkgs/clan-cli/clan_cli/backups/list.py new file mode 100644 index 000000000..641e330d0 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/backups/list.py @@ -0,0 +1,56 @@ +import argparse +import pprint +from pathlib import Path +from typing import Optional + +from ..errors import ClanError + + +def list_backups(flake_dir: Path, machine: Optional[str] = None, provider: Optional[str] = None) -> dict[str, dict[str, list[str]]]: + dummy_data = { + "testhostname": { + "borgbackup": [ + "2021-01-01T00:00:00Z", + "2022-01-01T00:00:00Z", + "2023-01-01T00:00:00Z", + ], + "restic" : [ + "2021-01-01T00:00:00Z", + "2022-01-01T00:00:00Z", + "2023-01-01T00:00:00Z", + ], + }, + "another host": { + "borgbackup": [ + "2021-01-01T00:00:00Z", + "2022-01-01T00:00:00Z", + "2023-01-01T00:00:00Z", + ], + }, + } + + if provider is not None: + new_data = {} + for machine_ in dummy_data: + if provider in dummy_data[machine_]: + new_data[machine_] = {provider: dummy_data[machine_][provider]} + dummy_data = new_data + + if machine is None: + return dummy_data + else: + return {machine: dummy_data[machine]} + +def list_command(args: argparse.Namespace) -> None: + if args.flake is None: + raise ClanError("Could not find clan flake toplevel directory") + backups = list_backups(Path(args.flake), machine=args.machine, provider=args.provider) + if len(backups) > 0: + pp = pprint.PrettyPrinter(depth=4) + pp.pprint(backups) + + +def register_list_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--machine", type=str, help="machine in the flake to show backups of") + parser.add_argument("--provider", type=str, help="backup provider to filter by") + parser.set_defaults(func=list_command) diff --git a/pkgs/clan-cli/clan_cli/backups/restore.py b/pkgs/clan-cli/clan_cli/backups/restore.py new file mode 100644 index 000000000..ae311566a --- /dev/null +++ b/pkgs/clan-cli/clan_cli/backups/restore.py @@ -0,0 +1,27 @@ +import argparse +import pprint +from pathlib import Path +from typing import Optional + +from ..errors import ClanError + + +def restore_backup(flake_dir: Path, machine: str, provider: str, backup_id: str, service: Optional[str] = None) -> None: + if service is None: + print("would restore backup", machine, provider, backup_id) + else: + print("would restore backup", machine, provider, backup_id, "of service:", service) + + +def restore_command(args: argparse.Namespace) -> None: + if args.flake is None: + raise ClanError("Could not find clan flake toplevel directory") + restore_backup(Path(args.flake), machine=args.machine, provider=args.provider, backup_id=args.backup_id, service=args.service) + + +def register_restore_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument("machine", type=str, help="machine in the flake to create backups of") + parser.add_argument("provider", type=str, help="backup provider to use") + parser.add_argument("backup_id", type=str, help="id of the backup to restore") + parser.add_argument("--service", type=str, help="name of the service to restore") + parser.set_defaults(func=restore_command) From 83dd430d03482ca796337e60be70ba95673b9ac5 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 4 Dec 2023 17:05:37 +0100 Subject: [PATCH 2/2] implement backup cli for borgbackup --- checks/borgbackup/default.nix | 2 +- clanModules/borgbackup.nix | 21 ++++++---- nixosModules/clanCore/backups.nix | 20 ++++++--- nixosModules/clanCore/zerotier/default.nix | 2 + pkgs/clan-cli/clan_cli/__init__.py | 2 +- pkgs/clan-cli/clan_cli/backups/__init__.py | 2 +- pkgs/clan-cli/clan_cli/backups/create.py | 47 +++++++++++++--------- pkgs/clan-cli/clan_cli/backups/list.py | 33 +++++++++------ pkgs/clan-cli/clan_cli/backups/restore.py | 27 ++++++++++--- pkgs/clan-cli/tests/test_backups.py | 22 ++++++++++ 10 files changed, 125 insertions(+), 53 deletions(-) create mode 100644 pkgs/clan-cli/tests/test_backups.py diff --git a/checks/borgbackup/default.nix b/checks/borgbackup/default.nix index 9bf7aa831..c8cb0c077 100644 --- a/checks/borgbackup/default.nix +++ b/checks/borgbackup/default.nix @@ -16,7 +16,7 @@ { clanCore.machineName = "machine"; clanCore.clanDir = ./.; - clanCore.state."/etc/state" = { }; + clanCore.state.testState.folders = [ "/etc/state" ]; environment.etc.state.text = "hello world"; clan.borgbackup = { enable = true; diff --git a/clanModules/borgbackup.nix b/clanModules/borgbackup.nix index fa5550cd5..933889c6a 100644 --- a/clanModules/borgbackup.nix +++ b/clanModules/borgbackup.nix @@ -33,7 +33,7 @@ in config = lib.mkIf cfg.enable { services.borgbackup.jobs = lib.mapAttrs (_: dest: { - paths = map (state: state.folder) (lib.attrValues config.clanCore.state); + paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state)); exclude = [ "*.pyc" ]; @@ -58,16 +58,23 @@ in clanCore.backups.providers.borgbackup = { list = '' ${lib.concatMapStringsSep "\n" (dest: '' - echo listing backups for ${dest} - borg-job-${dest} list - '') cfg.destinations} + ( + export BORG_REPO=${lib.escapeShellArg dest.repo} + export BORG_RSH=${lib.escapeShellArg dest.rsh} + ${lib.getExe config.services.borgbackup.package} list + ) + '') (lib.attrValues cfg.destinations)} ''; start = '' - ${lib.concatMapStringsSep "\n" (dest: '' - systemctl start borgbackup-job-${dest} - '') cfg.destinations} + ssh ${config.clan.networking.deploymentAddress} -- ' + ${lib.concatMapStringsSep "\n" (dest: '' + systemctl start borgbackup-job-${dest.name} + '') (lib.attrValues cfg.destinations)} + ' ''; + restore = '' + ''; }; }; } diff --git a/nixosModules/clanCore/backups.nix b/nixosModules/clanCore/backups.nix index 3e905029a..61f2b2224 100644 --- a/nixosModules/clanCore/backups.nix +++ b/nixosModules/clanCore/backups.nix @@ -3,15 +3,22 @@ options.clanCore.state = lib.mkOption { default = { }; type = lib.types.attrsOf - (lib.types.submodule ({ name, ... }: { + (lib.types.submodule ({ ... }: { options = { - folder = lib.mkOption { - type = lib.types.str; - default = name; + folders = lib.mkOption { + type = lib.types.listOf lib.types.str; description = '' Folder where state resides in ''; }; + restoreScript = lib.mkOption { + type = lib.types.str; + default = ":"; + description = '' + script to restore the service after the state dir was restored from a backup + ''; + + }; }; })); }; @@ -32,10 +39,11 @@ script to list backups ''; }; - delete = lib.mkOption { + restore = lib.mkOption { type = lib.types.str; description = '' - script to delete a backup + script to restore a backup + should take an optional service name as argument ''; }; start = lib.mkOption { diff --git a/nixosModules/clanCore/zerotier/default.nix b/nixosModules/clanCore/zerotier/default.nix index c263a621f..8cd04eb0f 100644 --- a/nixosModules/clanCore/zerotier/default.nix +++ b/nixosModules/clanCore/zerotier/default.nix @@ -147,6 +147,8 @@ in --network-id "$facts/zerotier-network-id" ''; }; + clanCore.state.zerotier.folders = [ "/var/lib/zerotier-one" ]; + environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ]; }) (lib.mkIf (config.clanCore.secretsUploadDirectory != null && !cfg.controller.enable && cfg.networkId != null) { diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 190d2834d..da58cae2a 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -5,7 +5,7 @@ from pathlib import Path from types import ModuleType from typing import Any, Optional, Sequence -from . import config, flakes, machines, secrets, vms, webui, backups +from . import backups, config, flakes, machines, secrets, vms, webui from .custom_logger import setup_logging from .dirs import get_clan_flake_toplevel from .ssh import cli as ssh_cli diff --git a/pkgs/clan-cli/clan_cli/backups/__init__.py b/pkgs/clan-cli/clan_cli/backups/__init__.py index b7ce53971..b060d45ea 100644 --- a/pkgs/clan-cli/clan_cli/backups/__init__.py +++ b/pkgs/clan-cli/clan_cli/backups/__init__.py @@ -1,8 +1,8 @@ # !/usr/bin/env python3 import argparse -from .list import register_list_parser from .create import register_create_parser +from .list import register_list_parser from .restore import register_restore_parser diff --git a/pkgs/clan-cli/clan_cli/backups/create.py b/pkgs/clan-cli/clan_cli/backups/create.py index 48b08db5a..5bbe5e8a8 100644 --- a/pkgs/clan-cli/clan_cli/backups/create.py +++ b/pkgs/clan-cli/clan_cli/backups/create.py @@ -1,34 +1,45 @@ import argparse -import pprint -from pathlib import Path +import json +import subprocess from typing import Optional from ..errors import ClanError +from ..machines.machines import Machine -def create_backup(flake_dir: Path, machine: Optional[str] = None, provider: Optional[str] = None) -> None: - if machine is None: - # TODO get all machines here - machines = [ "machine1", "machine2" ] - else: - machines = [ machine ] - +def create_backup(machine: Machine, provider: Optional[str] = None) -> None: + backup_scripts = json.loads( + machine.eval_nix(f"nixosConfigurations.{machine.name}.config.clanCore.backups") + ) if provider is None: - # TODO get all providers here - providers = [ "provider1", "provider2" ] + for provider in backup_scripts["providers"]: + proc = subprocess.run( + ["bash", "-c", backup_scripts["providers"][provider]["start"]], + ) + if proc.returncode != 0: + raise ClanError("failed to start backup") + else: + print("successfully started backup") else: - providers = [ provider ] - - print("would create backups for machines: ", machines, " with providers: ", providers) + if provider not in backup_scripts["providers"]: + raise ClanError(f"provider {provider} not found") + proc = subprocess.run( + ["bash", "-c", backup_scripts["providers"][provider]["start"]], + ) + if proc.returncode != 0: + raise ClanError("failed to start backup") + else: + print("successfully started backup") def create_command(args: argparse.Namespace) -> None: - if args.flake is None: - raise ClanError("Could not find clan flake toplevel directory") - create_backup(Path(args.flake), machine=args.machine, provider=args.provider) + machine = Machine(name=args.machine, flake_dir=args.flake) + create_backup(machine=machine, provider=args.provider) def register_create_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--machine", type=str, help="machine in the flake to create backups of") + parser.add_argument( + "machine", type=str, help="machine in the flake to create backups of" + ) parser.add_argument("--provider", type=str, help="backup provider to use") parser.set_defaults(func=create_command) diff --git a/pkgs/clan-cli/clan_cli/backups/list.py b/pkgs/clan-cli/clan_cli/backups/list.py index 641e330d0..c7fe84579 100644 --- a/pkgs/clan-cli/clan_cli/backups/list.py +++ b/pkgs/clan-cli/clan_cli/backups/list.py @@ -6,25 +6,27 @@ from typing import Optional from ..errors import ClanError -def list_backups(flake_dir: Path, machine: Optional[str] = None, provider: Optional[str] = None) -> dict[str, dict[str, list[str]]]: +def list_backups( + flake_dir: Path, machine: str, provider: Optional[str] = None +) -> dict[str, dict[str, list[dict[str, str]]]]: dummy_data = { "testhostname": { "borgbackup": [ - "2021-01-01T00:00:00Z", - "2022-01-01T00:00:00Z", - "2023-01-01T00:00:00Z", + {"date": "2021-01-01T00:00:00Z", "id": "1"}, + {"date": "2022-01-01T00:00:00Z", "id": "2"}, + {"date": "2023-01-01T00:00:00Z", "id": "3"}, ], - "restic" : [ - "2021-01-01T00:00:00Z", - "2022-01-01T00:00:00Z", - "2023-01-01T00:00:00Z", + "restic": [ + {"date": "2021-01-01T00:00:00Z", "id": "1"}, + {"date": "2022-01-01T00:00:00Z", "id": "2"}, + {"date": "2023-01-01T00:00:00Z", "id": "3"}, ], }, "another host": { "borgbackup": [ - "2021-01-01T00:00:00Z", - "2022-01-01T00:00:00Z", - "2023-01-01T00:00:00Z", + {"date": "2021-01-01T00:00:00Z", "id": "1"}, + {"date": "2022-01-01T00:00:00Z", "id": "2"}, + {"date": "2023-01-01T00:00:00Z", "id": "3"}, ], }, } @@ -41,16 +43,21 @@ def list_backups(flake_dir: Path, machine: Optional[str] = None, provider: Optio else: return {machine: dummy_data[machine]} + def list_command(args: argparse.Namespace) -> None: if args.flake is None: raise ClanError("Could not find clan flake toplevel directory") - backups = list_backups(Path(args.flake), machine=args.machine, provider=args.provider) + backups = list_backups( + Path(args.flake), machine=args.machine, provider=args.provider + ) if len(backups) > 0: pp = pprint.PrettyPrinter(depth=4) pp.pprint(backups) def register_list_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--machine", type=str, help="machine in the flake to show backups of") + parser.add_argument( + "machine", type=str, help="machine in the flake to show backups of" + ) parser.add_argument("--provider", type=str, help="backup provider to filter by") parser.set_defaults(func=list_command) diff --git a/pkgs/clan-cli/clan_cli/backups/restore.py b/pkgs/clan-cli/clan_cli/backups/restore.py index ae311566a..64102fc0a 100644 --- a/pkgs/clan-cli/clan_cli/backups/restore.py +++ b/pkgs/clan-cli/clan_cli/backups/restore.py @@ -1,26 +1,41 @@ import argparse -import pprint from pathlib import Path from typing import Optional from ..errors import ClanError -def restore_backup(flake_dir: Path, machine: str, provider: str, backup_id: str, service: Optional[str] = None) -> None: +def restore_backup( + flake_dir: Path, + machine: str, + provider: str, + backup_id: str, + service: Optional[str] = None, +) -> None: if service is None: - print("would restore backup", machine, provider, backup_id) + print("would restore backup", machine, provider, backup_id) else: - print("would restore backup", machine, provider, backup_id, "of service:", service) + print( + "would restore backup", machine, provider, backup_id, "of service:", service + ) def restore_command(args: argparse.Namespace) -> None: if args.flake is None: raise ClanError("Could not find clan flake toplevel directory") - restore_backup(Path(args.flake), machine=args.machine, provider=args.provider, backup_id=args.backup_id, service=args.service) + restore_backup( + Path(args.flake), + machine=args.machine, + provider=args.provider, + backup_id=args.backup_id, + service=args.service, + ) def register_restore_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument("machine", type=str, help="machine in the flake to create backups of") + parser.add_argument( + "machine", type=str, help="machine in the flake to create backups of" + ) parser.add_argument("provider", type=str, help="backup provider to use") parser.add_argument("backup_id", type=str, help="id of the backup to restore") parser.add_argument("--service", type=str, help="name of the service to restore") diff --git a/pkgs/clan-cli/tests/test_backups.py b/pkgs/clan-cli/tests/test_backups.py new file mode 100644 index 000000000..4129fa70f --- /dev/null +++ b/pkgs/clan-cli/tests/test_backups.py @@ -0,0 +1,22 @@ +import logging + +from cli import Cli +from fixtures_flakes import FlakeForTest + +log = logging.getLogger(__name__) + + +def test_backups( + test_flake: FlakeForTest, +) -> None: + cli = Cli() + + cli.run( + [ + "--flake", + str(test_flake.path), + "backups", + "list", + "testhostname", + ] + )