Migrate postgresql to clanServices

checks/flake-module.nix

@@ -48,7 +48,7 @@ in
               container = self.clanLib.test.containerTest ./container nixosTestArgs;
               zt-tcp-relay = self.clanLib.test.containerTest ./zt-tcp-relay nixosTestArgs;
               matrix-synapse = self.clanLib.test.containerTest ./matrix-synapse nixosTestArgs;
-              postgresql = self.clanLib.test.containerTest ./postgresql nixosTestArgs;
+              postgresql-legacy = self.clanLib.test.containerTest ./postgresql nixosTestArgs;
 
               dummy-inventory-test = import ./dummy-inventory-test nixosTestArgs;
               dummy-inventory-test-from-flake = import ./dummy-inventory-test-from-flake nixosTestArgs;

clanModules/postgresql/README.md

@@ -1,3 +1,5 @@
 ---
 description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance."
+categories = ["Database"]
+features = ["inventory", "deprecated"]
 ---

clanServices/postgresql/default.nix

@@ -0,0 +1,236 @@
+{ ... }:
+{
+  _class = "clan.service";
+  manifest.name = "clan-core/postgresql";
+  manifest.description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.";
+  manifest.categories = [ "Database" ];
+
+  roles.default = {
+    interface =
+      { lib, ... }:
+      {
+        options = {
+          databases = lib.mkOption {
+            description = "Databases to create";
+            default = { };
+            type = lib.types.attrsOf (
+              lib.types.submodule (
+                { name, ... }:
+                {
+                  options = {
+                    name = lib.mkOption {
+                      type = lib.types.str;
+                      default = name;
+                      description = "Database name.";
+                    };
+                    service = lib.mkOption {
+                      type = lib.types.str;
+                      default = name;
+                      description = "Service name that we associate with the database.";
+                    };
+                    # set to false, in case the upstream module uses ensureDatabase option
+                    create.enable = lib.mkOption {
+                      type = lib.types.bool;
+                      default = true;
+                      description = "Create the database if it does not exist.";
+                    };
+                    create.options = lib.mkOption {
+                      description = "Options to pass to the CREATE DATABASE command.";
+                      type = lib.types.lazyAttrsOf lib.types.str;
+                      default = { };
+                      example = {
+                        TEMPLATE = "template0";
+                        LC_COLLATE = "C";
+                        LC_CTYPE = "C";
+                        ENCODING = "UTF8";
+                        OWNER = "foo";
+                      };
+                    };
+                    restore.stopOnRestore = lib.mkOption {
+                      type = lib.types.listOf lib.types.str;
+                      default = [ ];
+                      description = "List of systemd services to stop before restoring the database.";
+                    };
+                  };
+                }
+              )
+            );
+          };
+          users = lib.mkOption {
+            description = "Users to create";
+            default = { };
+            type = lib.types.attrsOf (
+              lib.types.submodule (
+                { name, ... }:
+                {
+                  options.name = lib.mkOption {
+                    description = "User name";
+                    type = lib.types.str;
+                    default = name;
+                  };
+                }
+              )
+            );
+          };
+        };
+      };
+
+    perInstance =
+      { settings, ... }:
+      {
+        nixosModule =
+          { config, pkgs, lib, ... }:
+          let
+            createDatabaseState =
+              db:
+              let
+                folder = "/var/backup/postgres/${db.name}";
+                current = "${folder}/pg-dump";
+                compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
+              in
+              {
+                folders = [ folder ];
+                preBackupScript = ''
+                  export PATH=${
+                    lib.makeBinPath [
+                      config.services.postgresql.package
+                      config.systemd.package
+                      pkgs.coreutils
+                      pkgs.util-linux
+                      pkgs.zstd
+                    ]
+                  }
+                  while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+                    sleep 1
+                  done
+
+                  mkdir -p "${folder}"
+                  runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
+                  mv "${current}.tmp" ${current}
+                '';
+                postRestoreScript = ''
+                  export PATH=${
+                    lib.makeBinPath [
+                      config.services.postgresql.package
+                      config.systemd.package
+                      pkgs.coreutils
+                      pkgs.util-linux
+                      pkgs.zstd
+                      pkgs.gnugrep
+                    ]
+                  }
+                  while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+                    sleep 1
+                  done
+                  echo "Waiting for postgres to be ready..."
+                  while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+                    if ! systemctl is-active postgresql; then exit 1; fi
+                    sleep 0.1
+                  done
+
+                  if [[ -e "${current}" ]]; then
+                    (
+                      systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+                      trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+                      mkdir -p "${folder}"
+                      if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+                        runuser -u postgres -- dropdb "${db.name}"
+                      fi
+                      runuser -u postgres -- pg_restore -C -d postgres "${current}"
+                    )
+                  else
+                    echo No database backup found, skipping restore
+                  fi
+                '';
+              };
+
+            createDatabase = db: ''
+              CREATE DATABASE "${db.name}" ${
+                lib.concatStringsSep " " (
+                  lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
+                )
+              }
+            '';
+
+            userClauses = lib.mapAttrsToList (
+              _: user:
+              ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
+            ) settings.users;
+            
+            databaseClauses = lib.mapAttrsToList (
+              name: db:
+              lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
+            ) settings.databases;
+          in
+          {
+            services.postgresql.settings = {
+              wal_level = "replica";
+              max_wal_senders = 3;
+            };
+
+            services.postgresql.enable = true;
+            # We are duplicating a bit the upstream module but allow to create databases with options
+            systemd.services.postgresql.postStart = ''
+              PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
+
+              while ! $PSQL -d postgres -c "" 2> /dev/null; do
+                if ! kill -0 "$MAINPID"; then exit 1; fi
+                sleep 0.1
+              done
+              ${lib.concatStringsSep "\n" userClauses}
+              ${lib.concatStringsSep "\n" databaseClauses}
+            '';
+
+            clan.core.state = lib.mapAttrs' (
+              _: db: lib.nameValuePair db.service (createDatabaseState db)
+            ) settings.databases;
+
+            environment.systemPackages = builtins.map (
+              db:
+              let
+                folder = "/var/backup/postgres/${db.name}";
+                current = "${folder}/pg-dump";
+              in
+              pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
+                export PATH=${
+                  lib.makeBinPath [
+                    config.services.postgresql.package
+                    config.systemd.package
+                    pkgs.coreutils
+                    pkgs.util-linux
+                    pkgs.zstd
+                    pkgs.gnugrep
+                  ]
+                }
+                while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+                  sleep 1
+                done
+                echo "Waiting for postgres to be ready..."
+                while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+                  if ! systemctl is-active postgresql; then exit 1; fi
+                  sleep 0.1
+                done
+
+                if [[ -e "${current}" ]]; then
+                  (
+                    ${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
+                      systemctl stop ${builtins.toString db.restore.stopOnRestore}
+                      trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
+                    ''}
+
+                    mkdir -p "${folder}"
+                    if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+                      runuser -u postgres -- dropdb "${db.name}"
+                    fi
+                    runuser -u postgres -- pg_restore -C -d postgres "${current}"
+                  )
+                else
+                  echo No database backup found, skipping restore
+                fi
+              ''
+            ) (builtins.attrValues settings.databases);
+          };
+      };
+  };
+}

clanServices/postgresql/flake-module.nix

@@ -0,0 +1,18 @@
+{ lib, self, ... }:
+{
+  clan.modules = {
+    postgresql = lib.modules.importApply ./default.nix { };
+  };
+
+  perSystem =
+    { pkgs, ... }:
+    {
+      checks = lib.optionalAttrs (pkgs.stdenv.isLinux) {
+        postgresql = import ./tests/vm/default.nix {
+          inherit pkgs;
+          clan-core = self;
+          nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
+        };
+      };
+    };
+}

clanServices/postgresql/tests/vm/default.nix

@@ -1,58 +1,81 @@
-({
+{
-  name = "postgresql";
+  pkgs,
+  nixosLib,
+  clan-core,
+  ...
+}:
 
-  nodes.machine =
+nixosLib.runTest (
-    { self, config, ... }:
+  { ... }:
-    {
+  {
-      imports = [
+    imports = [
-        self.nixosModules.clanCore
+      clan-core.modules.nixosVmTest.clanTest
-        self.clanModules.postgresql
+    ];
-        self.clanModules.localbackup
-      ];
-      clan.postgresql.users.test = { };
-      clan.postgresql.databases.test.create.options.OWNER = "test";
-      clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
-      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
-      clan.core.settings.directory = ./.;
 
-      systemd.services.sample-service = {
+    hostPkgs = pkgs;
-        wantedBy = [ "multi-user.target" ];
+
-        script = ''
+    name = "postgresql";
-          while true; do
+
-            echo "Hello, world!"
+    clan = {
-            sleep 5
+      directory = ./.;
-          done
+      modules."@clan/postgresql" = ../../default.nix;
-        '';
+      inventory = {
+        machines.machine = { };
+
+        instances = {
+          postgresql-test = {
+            module.name = "@clan/postgresql";
+            roles.default.machines."machine".settings = {
+              users.test = { };
+              databases.test = {
+                create.options.OWNER = "test";
+                restore.stopOnRestore = [ "sample-service" ];
+              };
+            };
+          };
+        };
       };
-
-      environment.systemPackages = [ config.services.postgresql.package ];
     };
-  testScript =
+
-    { nodes, ... }:
+    nodes = {
-    ''
+      machine =
+        { config, ... }:
+        {
+          systemd.services.sample-service = {
+            wantedBy = [ "multi-user.target" ];
+            script = ''
+              while true; do
+                echo "Hello, world!"
+                sleep 5
+              done
+            '';
+          };
+
+          environment.systemPackages = [ config.services.postgresql.package ];
+        };
+    };
+
+    testScript = ''
       start_all()
       machine.wait_for_unit("postgresql")
       machine.wait_for_unit("sample-service")
       # Create a test table
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
 
-      machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
       timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
 
-      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+      # Create backup file directory
+      machine.succeed("mkdir -p /var/backup/postgres/test")
+
+      # Create a test table and do a manual backup
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+      machine.succeed("runuser -u postgres -- pg_dump test -Fc -c > /var/backup/postgres/test/pg-dump")
+
+      # Drop the table to verify restore works
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
-      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
 
-      machine.succeed("rm -rf /var/backup/postgres")
+      # Run the post-restore command (the script created by the service)
-
+      machine.succeed("postgres-db-restore-command-test")
-      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
-      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
-
-      machine.succeed("""
-      set -x
-      ${nodes.machine.clan.core.state.test.postRestoreCommand}
-      """)
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
 
@@ -67,7 +90,8 @@
 
       # check if restore works if the database does not exist
       machine.succeed("runuser -u postgres -- dropdb test")
-      machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
+      machine.succeed("postgres-db-restore-command-test")
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
     '';
-})
+  }
+)

clanServices/postgresql/tests/vm/sops/users/admin/key.json

@@ -0,0 +1,4 @@
+{
+  "publickey": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+  "type": "age"
+}

docs/mkdocs.yml

@@ -93,8 +93,9 @@ nav:
           - reference/clanServices/importer.md
           - reference/clanServices/localsend.md
           - reference/clanServices/mycelium.md
+          - reference/clanServices/postgresql.md
           - reference/clanServices/sshd.md
-          - reference/clanServices/users.md
+          - reference/clanServices/user-password.md
           - reference/clanServices/hello-world.md
           - reference/clanServices/wifi.md
           - reference/clanServices/zerotier.md