From ed03e3320a5760a796c26402dda8af51ac4747db Mon Sep 17 00:00:00 2001 From: pinpox Date: Fri, 13 Jun 2025 11:16:51 +0200 Subject: [PATCH] Migrate postgresql to clanServices --- checks/flake-module.nix | 2 +- clanModules/postgresql/README.md | 2 + clanServices/postgresql/default.nix | 236 ++++++++++++++++++ clanServices/postgresql/flake-module.nix | 18 ++ .../postgresql/tests/vm}/default.nix | 108 ++++---- .../tests/vm/sops/users/admin/key.json | 4 + docs/mkdocs.yml | 3 +- 7 files changed, 329 insertions(+), 44 deletions(-) create mode 100644 clanServices/postgresql/default.nix create mode 100644 clanServices/postgresql/flake-module.nix rename {checks/postgresql => clanServices/postgresql/tests/vm}/default.nix (52%) create mode 100644 clanServices/postgresql/tests/vm/sops/users/admin/key.json diff --git a/checks/flake-module.nix b/checks/flake-module.nix index c87131108..f624a14f8 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -48,7 +48,7 @@ in container = self.clanLib.test.containerTest ./container nixosTestArgs; zt-tcp-relay = self.clanLib.test.containerTest ./zt-tcp-relay nixosTestArgs; matrix-synapse = self.clanLib.test.containerTest ./matrix-synapse nixosTestArgs; - postgresql = self.clanLib.test.containerTest ./postgresql nixosTestArgs; + postgresql-legacy = self.clanLib.test.containerTest ./postgresql nixosTestArgs; dummy-inventory-test = import ./dummy-inventory-test nixosTestArgs; dummy-inventory-test-from-flake = import ./dummy-inventory-test-from-flake nixosTestArgs; diff --git a/clanModules/postgresql/README.md b/clanModules/postgresql/README.md index 86108a33b..efc33efef 100644 --- a/clanModules/postgresql/README.md +++ b/clanModules/postgresql/README.md @@ -1,3 +1,5 @@ --- description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance." +categories = ["Database"] +features = ["inventory", "deprecated"] --- diff --git a/clanServices/postgresql/default.nix b/clanServices/postgresql/default.nix new file mode 100644 index 000000000..1abe7e5f5 --- /dev/null +++ b/clanServices/postgresql/default.nix @@ -0,0 +1,236 @@ +{ ... }: +{ + _class = "clan.service"; + manifest.name = "clan-core/postgresql"; + manifest.description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance."; + manifest.categories = [ "Database" ]; + + roles.default = { + interface = + { lib, ... }: + { + options = { + databases = lib.mkOption { + description = "Databases to create"; + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + type = lib.types.str; + default = name; + description = "Database name."; + }; + service = lib.mkOption { + type = lib.types.str; + default = name; + description = "Service name that we associate with the database."; + }; + # set to false, in case the upstream module uses ensureDatabase option + create.enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Create the database if it does not exist."; + }; + create.options = lib.mkOption { + description = "Options to pass to the CREATE DATABASE command."; + type = lib.types.lazyAttrsOf lib.types.str; + default = { }; + example = { + TEMPLATE = "template0"; + LC_COLLATE = "C"; + LC_CTYPE = "C"; + ENCODING = "UTF8"; + OWNER = "foo"; + }; + }; + restore.stopOnRestore = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "List of systemd services to stop before restoring the database."; + }; + }; + } + ) + ); + }; + users = lib.mkOption { + description = "Users to create"; + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options.name = lib.mkOption { + description = "User name"; + type = lib.types.str; + default = name; + }; + } + ) + ); + }; + }; + }; + + perInstance = + { settings, ... }: + { + nixosModule = + { config, pkgs, lib, ... }: + let + createDatabaseState = + db: + let + folder = "/var/backup/postgres/${db.name}"; + current = "${folder}/pg-dump"; + compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd"; + in + { + folders = [ folder ]; + preBackupScript = '' + export PATH=${ + lib.makeBinPath [ + config.services.postgresql.package + config.systemd.package + pkgs.coreutils + pkgs.util-linux + pkgs.zstd + ] + } + while [[ "$(systemctl is-active postgresql)" == activating ]]; do + sleep 1 + done + + mkdir -p "${folder}" + runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp" + mv "${current}.tmp" ${current} + ''; + postRestoreScript = '' + export PATH=${ + lib.makeBinPath [ + config.services.postgresql.package + config.systemd.package + pkgs.coreutils + pkgs.util-linux + pkgs.zstd + pkgs.gnugrep + ] + } + while [[ "$(systemctl is-active postgresql)" == activating ]]; do + sleep 1 + done + echo "Waiting for postgres to be ready..." + while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do + if ! systemctl is-active postgresql; then exit 1; fi + sleep 0.1 + done + + if [[ -e "${current}" ]]; then + ( + systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore} + trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT + + mkdir -p "${folder}" + if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then + runuser -u postgres -- dropdb "${db.name}" + fi + runuser -u postgres -- pg_restore -C -d postgres "${current}" + ) + else + echo No database backup found, skipping restore + fi + ''; + }; + + createDatabase = db: '' + CREATE DATABASE "${db.name}" ${ + lib.concatStringsSep " " ( + lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options + ) + } + ''; + + userClauses = lib.mapAttrsToList ( + _: user: + ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' '' + ) settings.users; + + databaseClauses = lib.mapAttrsToList ( + name: db: + lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} '' + ) settings.databases; + in + { + services.postgresql.settings = { + wal_level = "replica"; + max_wal_senders = 3; + }; + + services.postgresql.enable = true; + # We are duplicating a bit the upstream module but allow to create databases with options + systemd.services.postgresql.postStart = '' + PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}" + + while ! $PSQL -d postgres -c "" 2> /dev/null; do + if ! kill -0 "$MAINPID"; then exit 1; fi + sleep 0.1 + done + ${lib.concatStringsSep "\n" userClauses} + ${lib.concatStringsSep "\n" databaseClauses} + ''; + + clan.core.state = lib.mapAttrs' ( + _: db: lib.nameValuePair db.service (createDatabaseState db) + ) settings.databases; + + environment.systemPackages = builtins.map ( + db: + let + folder = "/var/backup/postgres/${db.name}"; + current = "${folder}/pg-dump"; + in + pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" '' + export PATH=${ + lib.makeBinPath [ + config.services.postgresql.package + config.systemd.package + pkgs.coreutils + pkgs.util-linux + pkgs.zstd + pkgs.gnugrep + ] + } + while [[ "$(systemctl is-active postgresql)" == activating ]]; do + sleep 1 + done + echo "Waiting for postgres to be ready..." + while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do + if ! systemctl is-active postgresql; then exit 1; fi + sleep 0.1 + done + + if [[ -e "${current}" ]]; then + ( + ${lib.optionalString (db.restore.stopOnRestore != [ ]) '' + systemctl stop ${builtins.toString db.restore.stopOnRestore} + trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT + ''} + + mkdir -p "${folder}" + if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then + runuser -u postgres -- dropdb "${db.name}" + fi + runuser -u postgres -- pg_restore -C -d postgres "${current}" + ) + else + echo No database backup found, skipping restore + fi + '' + ) (builtins.attrValues settings.databases); + }; + }; + }; +} diff --git a/clanServices/postgresql/flake-module.nix b/clanServices/postgresql/flake-module.nix new file mode 100644 index 000000000..54c360f1c --- /dev/null +++ b/clanServices/postgresql/flake-module.nix @@ -0,0 +1,18 @@ +{ lib, self, ... }: +{ + clan.modules = { + postgresql = lib.modules.importApply ./default.nix { }; + }; + + perSystem = + { pkgs, ... }: + { + checks = lib.optionalAttrs (pkgs.stdenv.isLinux) { + postgresql = import ./tests/vm/default.nix { + inherit pkgs; + clan-core = self; + nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { }; + }; + }; + }; +} diff --git a/checks/postgresql/default.nix b/clanServices/postgresql/tests/vm/default.nix similarity index 52% rename from checks/postgresql/default.nix rename to clanServices/postgresql/tests/vm/default.nix index ac45ef5a9..32ea457bb 100644 --- a/checks/postgresql/default.nix +++ b/clanServices/postgresql/tests/vm/default.nix @@ -1,58 +1,81 @@ -({ - name = "postgresql"; +{ + pkgs, + nixosLib, + clan-core, + ... +}: - nodes.machine = - { self, config, ... }: - { - imports = [ - self.nixosModules.clanCore - self.clanModules.postgresql - self.clanModules.localbackup - ]; - clan.postgresql.users.test = { }; - clan.postgresql.databases.test.create.options.OWNER = "test"; - clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ]; - clan.localbackup.targets.hdd.directory = "/mnt/external-disk"; - clan.core.settings.directory = ./.; +nixosLib.runTest ( + { ... }: + { + imports = [ + clan-core.modules.nixosVmTest.clanTest + ]; - systemd.services.sample-service = { - wantedBy = [ "multi-user.target" ]; - script = '' - while true; do - echo "Hello, world!" - sleep 5 - done - ''; + hostPkgs = pkgs; + + name = "postgresql"; + + clan = { + directory = ./.; + modules."@clan/postgresql" = ../../default.nix; + inventory = { + machines.machine = { }; + + instances = { + postgresql-test = { + module.name = "@clan/postgresql"; + roles.default.machines."machine".settings = { + users.test = { }; + databases.test = { + create.options.OWNER = "test"; + restore.stopOnRestore = [ "sample-service" ]; + }; + }; + }; + }; }; - - environment.systemPackages = [ config.services.postgresql.package ]; }; - testScript = - { nodes, ... }: - '' + + nodes = { + machine = + { config, ... }: + { + systemd.services.sample-service = { + wantedBy = [ "multi-user.target" ]; + script = '' + while true; do + echo "Hello, world!" + sleep 5 + done + ''; + }; + + environment.systemPackages = [ config.services.postgresql.package ]; + }; + }; + + testScript = '' start_all() machine.wait_for_unit("postgresql") machine.wait_for_unit("sample-service") # Create a test table machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test") - machine.succeed("/run/current-system/sw/bin/localbackup-create >&2") timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip()) - machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }") + # Create backup file directory + machine.succeed("mkdir -p /var/backup/postgres/test") + + # Create a test table and do a manual backup machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'") + machine.succeed("runuser -u postgres -- pg_dump test -Fc -c > /var/backup/postgres/test/pg-dump") + + # Drop the table to verify restore works machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'") - machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }") - machine.succeed("rm -rf /var/backup/postgres") - - machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2") - machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }") - - machine.succeed(""" - set -x - ${nodes.machine.clan.core.state.test.postRestoreCommand} - """) + # Run the post-restore command (the script created by the service) + machine.succeed("postgres-db-restore-command-test") machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2") machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2") @@ -67,7 +90,8 @@ # check if restore works if the database does not exist machine.succeed("runuser -u postgres -- dropdb test") - machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}") + machine.succeed("postgres-db-restore-command-test") machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2") ''; -}) + } +) diff --git a/clanServices/postgresql/tests/vm/sops/users/admin/key.json b/clanServices/postgresql/tests/vm/sops/users/admin/key.json new file mode 100644 index 000000000..e408aa96b --- /dev/null +++ b/clanServices/postgresql/tests/vm/sops/users/admin/key.json @@ -0,0 +1,4 @@ +{ + "publickey": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg", + "type": "age" +} diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 3d41d0ed1..4dfc89a11 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -93,8 +93,9 @@ nav: - reference/clanServices/importer.md - reference/clanServices/localsend.md - reference/clanServices/mycelium.md + - reference/clanServices/postgresql.md - reference/clanServices/sshd.md - - reference/clanServices/users.md + - reference/clanServices/user-password.md - reference/clanServices/hello-world.md - reference/clanServices/wifi.md - reference/clanServices/zerotier.md