yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Migrate postgresql to clanServices

Browse Source
This commit is contained in:
pinpox
2025-06-13 11:16:51 +02:00
parent eabf7f86fc
commit ed03e3320a
7 changed files with 329 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
checks/flake-module.nix
View File

@@ -48,7 +48,7 @@ in
container = self.clanLib.test.containerTest ./container nixosTestArgs;
zt-tcp-relay = self.clanLib.test.containerTest ./zt-tcp-relay nixosTestArgs;
matrix-synapse = self.clanLib.test.containerTest ./matrix-synapse nixosTestArgs;
postgresql = self.clanLib.test.containerTest ./postgresql nixosTestArgs;
postgresql-legacy = self.clanLib.test.containerTest ./postgresql nixosTestArgs;
dummy-inventory-test = import ./dummy-inventory-test nixosTestArgs;
dummy-inventory-test-from-flake = import ./dummy-inventory-test-from-flake nixosTestArgs;

2
clanModules/postgresql/README.md
View File

@@ -1,3 +1,5 @@
---
description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance."
categories = ["Database"]
features = ["inventory", "deprecated"]
---

236
clanServices/postgresql/default.nix Normal file
View File

@@ -0,0 +1,236 @@
{ ... }:
{
_class = "clan.service";
manifest.name = "clan-core/postgresql";
manifest.description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.";
manifest.categories = [ "Database" ];
roles.default = {
interface =
{ lib, ... }:
{
options = {
databases = lib.mkOption {
description = "Databases to create";
default = { };
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options = {
name = lib.mkOption {
type = lib.types.str;
default = name;
description = "Database name.";
};
service = lib.mkOption {
type = lib.types.str;
default = name;
description = "Service name that we associate with the database.";
};
# set to false, in case the upstream module uses ensureDatabase option
create.enable = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Create the database if it does not exist.";
};
create.options = lib.mkOption {
description = "Options to pass to the CREATE DATABASE command.";
type = lib.types.lazyAttrsOf lib.types.str;
default = { };
example = {
TEMPLATE = "template0";
LC_COLLATE = "C";
LC_CTYPE = "C";
ENCODING = "UTF8";
OWNER = "foo";
};
};
restore.stopOnRestore = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "List of systemd services to stop before restoring the database.";
};
};
}
)
);
};
users = lib.mkOption {
description = "Users to create";
default = { };
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options.name = lib.mkOption {
description = "User name";
type = lib.types.str;
default = name;
};
}
)
);
};
};
};
perInstance =
{ settings, ... }:
{
nixosModule =
{ config, pkgs, lib, ... }:
let
createDatabaseState =
db:
let
folder = "/var/backup/postgres/${db.name}";
current = "${folder}/pg-dump";
compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
in
{
folders = [ folder ];
preBackupScript = ''
export PATH=${
lib.makeBinPath [
config.services.postgresql.package
config.systemd.package
pkgs.coreutils
pkgs.util-linux
pkgs.zstd
]
}
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
sleep 1
done
mkdir -p "${folder}"
runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
mv "${current}.tmp" ${current}
'';
postRestoreScript = ''
export PATH=${
lib.makeBinPath [
config.services.postgresql.package
config.systemd.package
pkgs.coreutils
pkgs.util-linux
pkgs.zstd
pkgs.gnugrep
]
}
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
sleep 1
done
echo "Waiting for postgres to be ready..."
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
if ! systemctl is-active postgresql; then exit 1; fi
sleep 0.1
done
if [[ -e "${current}" ]]; then
(
systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
mkdir -p "${folder}"
if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
runuser -u postgres -- dropdb "${db.name}"
fi
runuser -u postgres -- pg_restore -C -d postgres "${current}"
)
else
echo No database backup found, skipping restore
fi
'';
};
createDatabase = db: ''
CREATE DATABASE "${db.name}" ${
lib.concatStringsSep " " (
lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
)
}
'';
userClauses = lib.mapAttrsToList (
_: user:
''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
) settings.users;
databaseClauses = lib.mapAttrsToList (
name: db:
lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
) settings.databases;
in
{
services.postgresql.settings = {
wal_level = "replica";
max_wal_senders = 3;
};
services.postgresql.enable = true;
# We are duplicating a bit the upstream module but allow to create databases with options
systemd.services.postgresql.postStart = ''
PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
while ! $PSQL -d postgres -c "" 2> /dev/null; do
if ! kill -0 "$MAINPID"; then exit 1; fi
sleep 0.1
done
${lib.concatStringsSep "\n" userClauses}
${lib.concatStringsSep "\n" databaseClauses}
'';
clan.core.state = lib.mapAttrs' (
_: db: lib.nameValuePair db.service (createDatabaseState db)
) settings.databases;
environment.systemPackages = builtins.map (
db:
let
folder = "/var/backup/postgres/${db.name}";
current = "${folder}/pg-dump";
in
pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
export PATH=${
lib.makeBinPath [
config.services.postgresql.package
config.systemd.package
pkgs.coreutils
pkgs.util-linux
pkgs.zstd
pkgs.gnugrep
]
}
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
sleep 1
done
echo "Waiting for postgres to be ready..."
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
if ! systemctl is-active postgresql; then exit 1; fi
sleep 0.1
done
if [[ -e "${current}" ]]; then
(
${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
systemctl stop ${builtins.toString db.restore.stopOnRestore}
trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
''}
mkdir -p "${folder}"
if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
runuser -u postgres -- dropdb "${db.name}"
fi
runuser -u postgres -- pg_restore -C -d postgres "${current}"
)
else
echo No database backup found, skipping restore
fi
''
) (builtins.attrValues settings.databases);
};
};
};
}

18
clanServices/postgresql/flake-module.nix Normal file
View File

@@ -0,0 +1,18 @@
{ lib, self, ... }:
{
clan.modules = {
postgresql = lib.modules.importApply ./default.nix { };
};
perSystem =
{ pkgs, ... }:
{
checks = lib.optionalAttrs (pkgs.stdenv.isLinux) {
postgresql = import ./tests/vm/default.nix {
inherit pkgs;
clan-core = self;
nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
};
};
};
}

108
checks/postgresql/default.nix → clanServices/postgresql/tests/vm/default.nix
View File

@@ -1,58 +1,81 @@
({
name = "postgresql";
{
pkgs,
nixosLib,
clan-core,
...
}:
nodes.machine =
{ self, config, ... }:
{
imports = [
self.nixosModules.clanCore
self.clanModules.postgresql
self.clanModules.localbackup
];
clan.postgresql.users.test = { };
clan.postgresql.databases.test.create.options.OWNER = "test";
clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
clan.core.settings.directory = ./.;
nixosLib.runTest (
{ ... }:
{
imports = [
clan-core.modules.nixosVmTest.clanTest
];
systemd.services.sample-service = {
wantedBy = [ "multi-user.target" ];
script = ''
while true; do
echo "Hello, world!"
sleep 5
done
'';
hostPkgs = pkgs;
name = "postgresql";
clan = {
directory = ./.;
modules."@clan/postgresql" = ../../default.nix;
inventory = {
machines.machine = { };
instances = {
postgresql-test = {
module.name = "@clan/postgresql";
roles.default.machines."machine".settings = {
users.test = { };
databases.test = {
create.options.OWNER = "test";
restore.stopOnRestore = [ "sample-service" ];
};
};
};
};
};
environment.systemPackages = [ config.services.postgresql.package ];
};
testScript =
{ nodes, ... }:
''
nodes = {
machine =
{ config, ... }:
{
systemd.services.sample-service = {
wantedBy = [ "multi-user.target" ];
script = ''
while true; do
echo "Hello, world!"
sleep 5
done
'';
};
environment.systemPackages = [ config.services.postgresql.package ];
};
};
testScript = ''
start_all()
machine.wait_for_unit("postgresql")
machine.wait_for_unit("sample-service")
# Create a test table
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
# Create backup file directory
machine.succeed("mkdir -p /var/backup/postgres/test")
# Create a test table and do a manual backup
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
machine.succeed("runuser -u postgres -- pg_dump test -Fc -c > /var/backup/postgres/test/pg-dump")
# Drop the table to verify restore works
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
machine.succeed("rm -rf /var/backup/postgres")
machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
machine.succeed("""
set -x
${nodes.machine.clan.core.state.test.postRestoreCommand}
""")
# Run the post-restore command (the script created by the service)
machine.succeed("postgres-db-restore-command-test")
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
@@ -67,7 +90,8 @@
# check if restore works if the database does not exist
machine.succeed("runuser -u postgres -- dropdb test")
machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
machine.succeed("postgres-db-restore-command-test")
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
'';
})
}
)

4
clanServices/postgresql/tests/vm/sops/users/admin/key.json Normal file
View File

@@ -0,0 +1,4 @@
{
"publickey": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
"type": "age"
}

3
docs/mkdocs.yml
View File

@@ -93,8 +93,9 @@ nav:
- reference/clanServices/importer.md
- reference/clanServices/localsend.md
- reference/clanServices/mycelium.md
- reference/clanServices/postgresql.md
- reference/clanServices/sshd.md
- reference/clanServices/users.md
- reference/clanServices/user-password.md
- reference/clanServices/hello-world.md
- reference/clanServices/wifi.md
- reference/clanServices/zerotier.md