Migrate postgresql to clan.core
This commit is contained in:
@@ -19,10 +19,21 @@ let
|
|||||||
nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
|
nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
imports = filter pathExists [
|
imports =
|
||||||
|
let
|
||||||
|
clanCoreModulesDir = ../nixosModules/clanCore;
|
||||||
|
getClanCoreTestModules =
|
||||||
|
let
|
||||||
|
moduleNames = attrNames (builtins.readDir clanCoreModulesDir);
|
||||||
|
testPaths = map (
|
||||||
|
moduleName: clanCoreModulesDir + "/${moduleName}/tests/flake-module.nix"
|
||||||
|
) moduleNames;
|
||||||
|
in
|
||||||
|
filter pathExists testPaths;
|
||||||
|
in
|
||||||
|
getClanCoreTestModules
|
||||||
|
++ filter pathExists [
|
||||||
./backups/flake-module.nix
|
./backups/flake-module.nix
|
||||||
../nixosModules/clanCore/machine-id/tests/flake-module.nix
|
|
||||||
../nixosModules/clanCore/state-version/tests/flake-module.nix
|
|
||||||
./devshell/flake-module.nix
|
./devshell/flake-module.nix
|
||||||
./flash/flake-module.nix
|
./flash/flake-module.nix
|
||||||
./impure/flake-module.nix
|
./impure/flake-module.nix
|
||||||
@@ -88,7 +99,6 @@ in
|
|||||||
nixos-test-container = self.clanLib.test.containerTest ./container nixosTestArgs;
|
nixos-test-container = self.clanLib.test.containerTest ./container nixosTestArgs;
|
||||||
nixos-test-zt-tcp-relay = self.clanLib.test.containerTest ./zt-tcp-relay nixosTestArgs;
|
nixos-test-zt-tcp-relay = self.clanLib.test.containerTest ./zt-tcp-relay nixosTestArgs;
|
||||||
nixos-test-matrix-synapse = self.clanLib.test.containerTest ./matrix-synapse nixosTestArgs;
|
nixos-test-matrix-synapse = self.clanLib.test.containerTest ./matrix-synapse nixosTestArgs;
|
||||||
nixos-test-postgresql = self.clanLib.test.containerTest ./postgresql nixosTestArgs;
|
|
||||||
nixos-test-user-firewall-iptables = self.clanLib.test.containerTest ./user-firewall/iptables.nix nixosTestArgs;
|
nixos-test-user-firewall-iptables = self.clanLib.test.containerTest ./user-firewall/iptables.nix nixosTestArgs;
|
||||||
nixos-test-user-firewall-nftables = self.clanLib.test.containerTest ./user-firewall/nftables.nix nixosTestArgs;
|
nixos-test-user-firewall-nftables = self.clanLib.test.containerTest ./user-firewall/nftables.nix nixosTestArgs;
|
||||||
|
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
({
|
|
||||||
name = "postgresql";
|
|
||||||
|
|
||||||
nodes.machine =
|
|
||||||
{ self, config, ... }:
|
|
||||||
{
|
|
||||||
imports = [
|
|
||||||
self.nixosModules.clanCore
|
|
||||||
self.clanModules.postgresql
|
|
||||||
self.clanModules.localbackup
|
|
||||||
];
|
|
||||||
clan.postgresql.users.test = { };
|
|
||||||
clan.postgresql.databases.test.create.options.OWNER = "test";
|
|
||||||
clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
|
|
||||||
clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
|
|
||||||
clan.core.settings.directory = ./.;
|
|
||||||
|
|
||||||
systemd.services.sample-service = {
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
script = ''
|
|
||||||
while true; do
|
|
||||||
echo "Hello, world!"
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
environment.systemPackages = [ config.services.postgresql.package ];
|
|
||||||
};
|
|
||||||
testScript =
|
|
||||||
{ nodes, ... }:
|
|
||||||
''
|
|
||||||
start_all()
|
|
||||||
machine.wait_for_unit("postgresql")
|
|
||||||
machine.wait_for_unit("sample-service")
|
|
||||||
# Create a test table
|
|
||||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
|
|
||||||
|
|
||||||
machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
|
|
||||||
timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
|
|
||||||
|
|
||||||
machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
|
||||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
|
|
||||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
|
|
||||||
machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
|
||||||
|
|
||||||
machine.succeed("rm -rf /var/backup/postgres")
|
|
||||||
|
|
||||||
machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
|
|
||||||
machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
|
||||||
|
|
||||||
machine.succeed("""
|
|
||||||
set -x
|
|
||||||
${nodes.machine.clan.core.state.test.postRestoreCommand}
|
|
||||||
""")
|
|
||||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
|
|
||||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
|
|
||||||
|
|
||||||
timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
|
|
||||||
assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
|
|
||||||
|
|
||||||
# Check that the table is still there
|
|
||||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
|
|
||||||
output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
|
|
||||||
owner = output.split("\n")[1]
|
|
||||||
assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
|
|
||||||
|
|
||||||
# check if restore works if the database does not exist
|
|
||||||
machine.succeed("runuser -u postgres -- dropdb test")
|
|
||||||
machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
|
|
||||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
|
|
||||||
'';
|
|
||||||
})
|
|
||||||
@@ -61,7 +61,6 @@ in
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
imports = [
|
imports = [
|
||||||
../postgresql
|
|
||||||
(lib.mkRemovedOptionModule [
|
(lib.mkRemovedOptionModule [
|
||||||
"clan"
|
"clan"
|
||||||
"matrix-synapse"
|
"matrix-synapse"
|
||||||
@@ -106,15 +105,16 @@ in
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
clan.postgresql.users.matrix-synapse = { };
|
clan.core.postgresql.enable = true;
|
||||||
clan.postgresql.databases.matrix-synapse.create.options = {
|
clan.core.postgresql.users.matrix-synapse = { };
|
||||||
|
clan.core.postgresql.databases.matrix-synapse.create.options = {
|
||||||
TEMPLATE = "template0";
|
TEMPLATE = "template0";
|
||||||
LC_COLLATE = "C";
|
LC_COLLATE = "C";
|
||||||
LC_CTYPE = "C";
|
LC_CTYPE = "C";
|
||||||
ENCODING = "UTF8";
|
ENCODING = "UTF8";
|
||||||
OWNER = "matrix-synapse";
|
OWNER = "matrix-synapse";
|
||||||
};
|
};
|
||||||
clan.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
|
clan.core.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
|
||||||
|
|
||||||
clan.core.vars.generators =
|
clan.core.vars.generators =
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,224 +1,9 @@
|
|||||||
|
{ lib, ... }:
|
||||||
{
|
{
|
||||||
pkgs,
|
imports = [
|
||||||
lib,
|
(lib.mkRemovedOptionModule [
|
||||||
config,
|
"clan"
|
||||||
...
|
"postgresql"
|
||||||
}:
|
] "The postgresql module has been migrated to a clan core option. Use clan.core.postgresql instead")
|
||||||
let
|
];
|
||||||
createDatabaseState =
|
|
||||||
db:
|
|
||||||
let
|
|
||||||
folder = "/var/backup/postgres/${db.name}";
|
|
||||||
current = "${folder}/pg-dump";
|
|
||||||
compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
|
|
||||||
in
|
|
||||||
{
|
|
||||||
folders = [ folder ];
|
|
||||||
preBackupScript = ''
|
|
||||||
export PATH=${
|
|
||||||
lib.makeBinPath [
|
|
||||||
config.services.postgresql.package
|
|
||||||
config.systemd.package
|
|
||||||
pkgs.coreutils
|
|
||||||
pkgs.util-linux
|
|
||||||
pkgs.zstd
|
|
||||||
]
|
|
||||||
}
|
|
||||||
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
mkdir -p "${folder}"
|
|
||||||
runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
|
|
||||||
mv "${current}.tmp" ${current}
|
|
||||||
'';
|
|
||||||
postRestoreScript = ''
|
|
||||||
export PATH=${
|
|
||||||
lib.makeBinPath [
|
|
||||||
config.services.postgresql.package
|
|
||||||
config.systemd.package
|
|
||||||
pkgs.coreutils
|
|
||||||
pkgs.util-linux
|
|
||||||
pkgs.zstd
|
|
||||||
pkgs.gnugrep
|
|
||||||
]
|
|
||||||
}
|
|
||||||
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
echo "Waiting for postgres to be ready..."
|
|
||||||
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
|
|
||||||
if ! systemctl is-active postgresql; then exit 1; fi
|
|
||||||
sleep 0.1
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ -e "${current}" ]]; then
|
|
||||||
(
|
|
||||||
systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
|
|
||||||
trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
|
|
||||||
|
|
||||||
mkdir -p "${folder}"
|
|
||||||
if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
|
|
||||||
runuser -u postgres -- dropdb "${db.name}"
|
|
||||||
fi
|
|
||||||
runuser -u postgres -- pg_restore -C -d postgres "${current}"
|
|
||||||
)
|
|
||||||
else
|
|
||||||
echo No database backup found, skipping restore
|
|
||||||
fi
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
createDatabase = db: ''
|
|
||||||
CREATE DATABASE "${db.name}" ${
|
|
||||||
lib.concatStringsSep " " (
|
|
||||||
lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
|
|
||||||
)
|
|
||||||
}
|
|
||||||
'';
|
|
||||||
cfg = config.clan.postgresql;
|
|
||||||
|
|
||||||
userClauses = lib.mapAttrsToList (
|
|
||||||
_: user:
|
|
||||||
''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
|
|
||||||
) cfg.users;
|
|
||||||
databaseClauses = lib.mapAttrsToList (
|
|
||||||
name: db:
|
|
||||||
lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
|
|
||||||
) cfg.databases;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
options.clan.postgresql = {
|
|
||||||
# we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
|
|
||||||
databases = lib.mkOption {
|
|
||||||
description = "Databases to create";
|
|
||||||
default = { };
|
|
||||||
type = lib.types.attrsOf (
|
|
||||||
lib.types.submodule (
|
|
||||||
{ name, ... }:
|
|
||||||
{
|
|
||||||
options = {
|
|
||||||
name = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = name;
|
|
||||||
description = "Database name.";
|
|
||||||
};
|
|
||||||
service = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = name;
|
|
||||||
description = "Service name that we associate with the database.";
|
|
||||||
};
|
|
||||||
# set to false, in case the upstream module uses ensureDatabase option
|
|
||||||
create.enable = lib.mkOption {
|
|
||||||
type = lib.types.bool;
|
|
||||||
default = true;
|
|
||||||
description = "Create the database if it does not exist.";
|
|
||||||
};
|
|
||||||
create.options = lib.mkOption {
|
|
||||||
description = "Options to pass to the CREATE DATABASE command.";
|
|
||||||
type = lib.types.lazyAttrsOf lib.types.str;
|
|
||||||
default = { };
|
|
||||||
example = {
|
|
||||||
TEMPLATE = "template0";
|
|
||||||
LC_COLLATE = "C";
|
|
||||||
LC_CTYPE = "C";
|
|
||||||
ENCODING = "UTF8";
|
|
||||||
OWNER = "foo";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
restore.stopOnRestore = lib.mkOption {
|
|
||||||
type = lib.types.listOf lib.types.str;
|
|
||||||
default = [ ];
|
|
||||||
description = "List of systemd services to stop before restoring the database.";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
)
|
|
||||||
);
|
|
||||||
};
|
|
||||||
users = lib.mkOption {
|
|
||||||
description = "Users to create";
|
|
||||||
default = { };
|
|
||||||
type = lib.types.attrsOf (
|
|
||||||
lib.types.submodule (
|
|
||||||
{ name, ... }:
|
|
||||||
{
|
|
||||||
options.name = lib.mkOption {
|
|
||||||
description = "User name";
|
|
||||||
type = lib.types.str;
|
|
||||||
default = name;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
)
|
|
||||||
);
|
|
||||||
};
|
|
||||||
};
|
|
||||||
config = {
|
|
||||||
services.postgresql.settings = {
|
|
||||||
wal_level = "replica";
|
|
||||||
max_wal_senders = 3;
|
|
||||||
};
|
|
||||||
|
|
||||||
services.postgresql.enable = true;
|
|
||||||
# We are duplicating a bit the upstream module but allow to create databases with options
|
|
||||||
systemd.services.postgresql.postStart = ''
|
|
||||||
PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
|
|
||||||
|
|
||||||
while ! $PSQL -d postgres -c "" 2> /dev/null; do
|
|
||||||
if ! kill -0 "$MAINPID"; then exit 1; fi
|
|
||||||
sleep 0.1
|
|
||||||
done
|
|
||||||
${lib.concatStringsSep "\n" userClauses}
|
|
||||||
${lib.concatStringsSep "\n" databaseClauses}
|
|
||||||
'';
|
|
||||||
|
|
||||||
clan.core.state = lib.mapAttrs' (
|
|
||||||
_: db: lib.nameValuePair db.service (createDatabaseState db)
|
|
||||||
) config.clan.postgresql.databases;
|
|
||||||
|
|
||||||
environment.systemPackages = builtins.map (
|
|
||||||
db:
|
|
||||||
let
|
|
||||||
folder = "/var/backup/postgres/${db.name}";
|
|
||||||
current = "${folder}/pg-dump";
|
|
||||||
in
|
|
||||||
pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
|
|
||||||
export PATH=${
|
|
||||||
lib.makeBinPath [
|
|
||||||
config.services.postgresql.package
|
|
||||||
config.systemd.package
|
|
||||||
pkgs.coreutils
|
|
||||||
pkgs.util-linux
|
|
||||||
pkgs.zstd
|
|
||||||
pkgs.gnugrep
|
|
||||||
]
|
|
||||||
}
|
|
||||||
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
echo "Waiting for postgres to be ready..."
|
|
||||||
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
|
|
||||||
if ! systemctl is-active postgresql; then exit 1; fi
|
|
||||||
sleep 0.1
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ -e "${current}" ]]; then
|
|
||||||
(
|
|
||||||
${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
|
|
||||||
systemctl stop ${builtins.toString db.restore.stopOnRestore}
|
|
||||||
trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
|
|
||||||
''}
|
|
||||||
|
|
||||||
mkdir -p "${folder}"
|
|
||||||
if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
|
|
||||||
runuser -u postgres -- dropdb "${db.name}"
|
|
||||||
fi
|
|
||||||
runuser -u postgres -- pg_restore -C -d postgres "${current}"
|
|
||||||
)
|
|
||||||
else
|
|
||||||
echo No database backup found, skipping restore
|
|
||||||
fi
|
|
||||||
''
|
|
||||||
) (builtins.attrValues config.clan.postgresql.databases);
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ in
|
|||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../postgresql
|
|
||||||
(lib.mkRemovedOptionModule [
|
(lib.mkRemovedOptionModule [
|
||||||
"clan"
|
"clan"
|
||||||
"vaultwarden"
|
"vaultwarden"
|
||||||
@@ -57,15 +56,17 @@ in
|
|||||||
|
|
||||||
config = {
|
config = {
|
||||||
|
|
||||||
clan.postgresql.users.vaultwarden = { };
|
clan.core.postgresql.enable = true;
|
||||||
clan.postgresql.databases.vaultwarden.create.options = {
|
|
||||||
|
clan.core.postgresql.users.vaultwarden = { };
|
||||||
|
clan.core.postgresql.databases.vaultwarden.create.options = {
|
||||||
TEMPLATE = "template0";
|
TEMPLATE = "template0";
|
||||||
LC_COLLATE = "C";
|
LC_COLLATE = "C";
|
||||||
LC_CTYPE = "C";
|
LC_CTYPE = "C";
|
||||||
ENCODING = "UTF8";
|
ENCODING = "UTF8";
|
||||||
OWNER = "vaultwarden";
|
OWNER = "vaultwarden";
|
||||||
};
|
};
|
||||||
clan.postgresql.databases.vaultwarden.restore.stopOnRestore = [ "vaultwarden" ];
|
clan.core.postgresql.databases.vaultwarden.restore.stopOnRestore = [ "vaultwarden" ];
|
||||||
|
|
||||||
services.nginx = {
|
services.nginx = {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
|||||||
@@ -171,6 +171,7 @@ nav:
|
|||||||
- reference/clan.core/deployment.md
|
- reference/clan.core/deployment.md
|
||||||
- reference/clan.core/facts.md
|
- reference/clan.core/facts.md
|
||||||
- reference/clan.core/networking.md
|
- reference/clan.core/networking.md
|
||||||
|
- reference/clan.core/postgresql.md
|
||||||
- reference/clan.core/settings.md
|
- reference/clan.core/settings.md
|
||||||
- reference/clan.core/sops.md
|
- reference/clan.core/sops.md
|
||||||
- reference/clan.core/state.md
|
- reference/clan.core/state.md
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
++ lib.optionals (_class == "nixos") [
|
++ lib.optionals (_class == "nixos") [
|
||||||
./nixos-facter.nix
|
./nixos-facter.nix
|
||||||
./vm.nix
|
./vm.nix
|
||||||
|
./postgresql
|
||||||
./machine-id
|
./machine-id
|
||||||
./state-version
|
./state-version
|
||||||
./wayland-proxy-virtwl.nix
|
./wayland-proxy-virtwl.nix
|
||||||
|
|||||||
236
nixosModules/clanCore/postgresql/default.nix
Normal file
236
nixosModules/clanCore/postgresql/default.nix
Normal file
@@ -0,0 +1,236 @@
|
|||||||
|
{
|
||||||
|
lib,
|
||||||
|
config,
|
||||||
|
pkgs,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
|
||||||
|
cfg = config.clan.core.postgresql;
|
||||||
|
|
||||||
|
createDatabaseState =
|
||||||
|
db:
|
||||||
|
let
|
||||||
|
folder = "/var/backup/postgres/${db.name}";
|
||||||
|
current = "${folder}/pg-dump";
|
||||||
|
compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
|
||||||
|
in
|
||||||
|
{
|
||||||
|
folders = [ folder ];
|
||||||
|
preBackupScript = ''
|
||||||
|
export PATH=${
|
||||||
|
lib.makeBinPath [
|
||||||
|
config.services.postgresql.package
|
||||||
|
config.systemd.package
|
||||||
|
pkgs.coreutils
|
||||||
|
pkgs.util-linux
|
||||||
|
pkgs.zstd
|
||||||
|
]
|
||||||
|
}
|
||||||
|
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
mkdir -p "${folder}"
|
||||||
|
runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
|
||||||
|
mv "${current}.tmp" ${current}
|
||||||
|
'';
|
||||||
|
postRestoreScript = ''
|
||||||
|
export PATH=${
|
||||||
|
lib.makeBinPath [
|
||||||
|
config.services.postgresql.package
|
||||||
|
config.systemd.package
|
||||||
|
pkgs.coreutils
|
||||||
|
pkgs.util-linux
|
||||||
|
pkgs.zstd
|
||||||
|
pkgs.gnugrep
|
||||||
|
]
|
||||||
|
}
|
||||||
|
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "Waiting for postgres to be ready..."
|
||||||
|
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
|
||||||
|
if ! systemctl is-active postgresql; then exit 1; fi
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -e "${current}" ]]; then
|
||||||
|
(
|
||||||
|
systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
|
||||||
|
trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
|
||||||
|
|
||||||
|
mkdir -p "${folder}"
|
||||||
|
if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
|
||||||
|
runuser -u postgres -- dropdb "${db.name}"
|
||||||
|
fi
|
||||||
|
runuser -u postgres -- pg_restore -C -d postgres "${current}"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
echo No database backup found, skipping restore
|
||||||
|
fi
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
createDatabase = db: ''
|
||||||
|
CREATE DATABASE "${db.name}" ${
|
||||||
|
lib.concatStringsSep " " (
|
||||||
|
lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
|
||||||
|
)
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
|
||||||
|
userClauses = lib.mapAttrsToList (
|
||||||
|
_: user:
|
||||||
|
''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
|
||||||
|
) cfg.users;
|
||||||
|
databaseClauses = lib.mapAttrsToList (
|
||||||
|
name: db:
|
||||||
|
lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
|
||||||
|
) cfg.databases;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
options.clan.core.postgresql = {
|
||||||
|
|
||||||
|
enable = lib.mkEnableOption "Whether to enable PostgreSQL Server";
|
||||||
|
|
||||||
|
# we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
|
||||||
|
databases = lib.mkOption {
|
||||||
|
description = "Databases to create";
|
||||||
|
default = { };
|
||||||
|
type = lib.types.attrsOf (
|
||||||
|
lib.types.submodule (
|
||||||
|
{ name, ... }:
|
||||||
|
{
|
||||||
|
options = {
|
||||||
|
name = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = name;
|
||||||
|
description = "Database name.";
|
||||||
|
};
|
||||||
|
service = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = name;
|
||||||
|
description = "Service name that we associate with the database.";
|
||||||
|
};
|
||||||
|
# set to false, in case the upstream module uses ensureDatabase option
|
||||||
|
create.enable = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = true;
|
||||||
|
description = "Create the database if it does not exist.";
|
||||||
|
};
|
||||||
|
create.options = lib.mkOption {
|
||||||
|
description = "Options to pass to the CREATE DATABASE command.";
|
||||||
|
type = lib.types.lazyAttrsOf lib.types.str;
|
||||||
|
default = { };
|
||||||
|
example = {
|
||||||
|
TEMPLATE = "template0";
|
||||||
|
LC_COLLATE = "C";
|
||||||
|
LC_CTYPE = "C";
|
||||||
|
ENCODING = "UTF8";
|
||||||
|
OWNER = "foo";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
restore.stopOnRestore = lib.mkOption {
|
||||||
|
type = lib.types.listOf lib.types.str;
|
||||||
|
default = [ ];
|
||||||
|
description = "List of systemd services to stop before restoring the database.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
};
|
||||||
|
users = lib.mkOption {
|
||||||
|
description = "Users to create";
|
||||||
|
default = { };
|
||||||
|
type = lib.types.attrsOf (
|
||||||
|
lib.types.submodule (
|
||||||
|
{ name, ... }:
|
||||||
|
{
|
||||||
|
options.name = lib.mkOption {
|
||||||
|
description = "User name";
|
||||||
|
type = lib.types.str;
|
||||||
|
default = name;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkIf (config.clan.core.postgresql.enable) {
|
||||||
|
|
||||||
|
clan.core.settings.state-version.enable = true;
|
||||||
|
|
||||||
|
# services.postgresql.package = lib.mkDefault pkgs.postgresql_16;
|
||||||
|
|
||||||
|
services.postgresql.enable = true;
|
||||||
|
|
||||||
|
services.postgresql.settings = {
|
||||||
|
wal_level = "replica";
|
||||||
|
max_wal_senders = 3;
|
||||||
|
};
|
||||||
|
|
||||||
|
# We are duplicating a bit the upstream module but allow to create databases with options
|
||||||
|
systemd.services.postgresql.postStart = ''
|
||||||
|
PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
|
||||||
|
|
||||||
|
while ! $PSQL -d postgres -c "" 2> /dev/null; do
|
||||||
|
if ! kill -0 "$MAINPID"; then exit 1; fi
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
${lib.concatStringsSep "\n" userClauses}
|
||||||
|
${lib.concatStringsSep "\n" databaseClauses}
|
||||||
|
'';
|
||||||
|
|
||||||
|
clan.core.state = lib.mapAttrs' (
|
||||||
|
_: db: lib.nameValuePair db.service (createDatabaseState db)
|
||||||
|
) config.clan.core.postgresql.databases;
|
||||||
|
|
||||||
|
environment.systemPackages = builtins.map (
|
||||||
|
db:
|
||||||
|
let
|
||||||
|
folder = "/var/backup/postgres/${db.name}";
|
||||||
|
current = "${folder}/pg-dump";
|
||||||
|
in
|
||||||
|
pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
|
||||||
|
export PATH=${
|
||||||
|
lib.makeBinPath [
|
||||||
|
config.services.postgresql.package
|
||||||
|
config.systemd.package
|
||||||
|
pkgs.coreutils
|
||||||
|
pkgs.util-linux
|
||||||
|
pkgs.zstd
|
||||||
|
pkgs.gnugrep
|
||||||
|
]
|
||||||
|
}
|
||||||
|
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "Waiting for postgres to be ready..."
|
||||||
|
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
|
||||||
|
if ! systemctl is-active postgresql; then exit 1; fi
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -e "${current}" ]]; then
|
||||||
|
(
|
||||||
|
${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
|
||||||
|
systemctl stop ${builtins.toString db.restore.stopOnRestore}
|
||||||
|
trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
|
||||||
|
''}
|
||||||
|
|
||||||
|
mkdir -p "${folder}"
|
||||||
|
if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
|
||||||
|
runuser -u postgres -- dropdb "${db.name}"
|
||||||
|
fi
|
||||||
|
runuser -u postgres -- pg_restore -C -d postgres "${current}"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
echo No database backup found, skipping restore
|
||||||
|
fi
|
||||||
|
''
|
||||||
|
) (builtins.attrValues config.clan.core.postgresql.databases);
|
||||||
|
};
|
||||||
|
}
|
||||||
106
nixosModules/clanCore/postgresql/tests/flake-module.nix
Normal file
106
nixosModules/clanCore/postgresql/tests/flake-module.nix
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
{ self, ... }:
|
||||||
|
{
|
||||||
|
perSystem =
|
||||||
|
{ ... }:
|
||||||
|
{
|
||||||
|
clan.nixosTests.postgresql = {
|
||||||
|
|
||||||
|
name = "service-postgresql";
|
||||||
|
|
||||||
|
clan = {
|
||||||
|
directory = ./.;
|
||||||
|
|
||||||
|
# Workaround until we can use nodes.machine = { };
|
||||||
|
modules."@clan/importer" = ../../../../clanServices/importer;
|
||||||
|
|
||||||
|
inventory = {
|
||||||
|
machines.machine = { };
|
||||||
|
instances.importer = {
|
||||||
|
module.name = "@clan/importer";
|
||||||
|
module.input = "self";
|
||||||
|
roles.default.tags.all = { };
|
||||||
|
roles.default.extraModules = [
|
||||||
|
{
|
||||||
|
|
||||||
|
imports = [
|
||||||
|
# self.nixosModules.clanCore
|
||||||
|
self.clanModules.localbackup
|
||||||
|
];
|
||||||
|
|
||||||
|
clan.core.postgresql.enable = true;
|
||||||
|
clan.core.postgresql.users.test = { };
|
||||||
|
clan.core.postgresql.databases.test.create.options.OWNER = "test";
|
||||||
|
clan.core.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
|
||||||
|
clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
|
||||||
|
clan.core.settings.directory = ./.;
|
||||||
|
|
||||||
|
systemd.services.sample-service = {
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
script = ''
|
||||||
|
while true; do
|
||||||
|
echo "Hello, world!"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# TODO: Broken. Use instead of importer after fixing.
|
||||||
|
# nodes.machine = { };
|
||||||
|
|
||||||
|
testScript =
|
||||||
|
|
||||||
|
{ nodes, ... }:
|
||||||
|
|
||||||
|
''
|
||||||
|
start_all()
|
||||||
|
machine.wait_for_unit("postgresql")
|
||||||
|
machine.wait_for_unit("sample-service")
|
||||||
|
# Create a test table
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
|
||||||
|
|
||||||
|
machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
|
||||||
|
timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
|
||||||
|
|
||||||
|
# import time
|
||||||
|
# time.sleep(5400000)
|
||||||
|
|
||||||
|
machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
|
||||||
|
machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
||||||
|
|
||||||
|
machine.succeed("rm -rf /var/backup/postgres")
|
||||||
|
|
||||||
|
machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
|
||||||
|
machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
||||||
|
|
||||||
|
machine.succeed("""
|
||||||
|
set -x
|
||||||
|
${nodes.machine.clan.core.state.test.postRestoreCommand}
|
||||||
|
""")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
|
||||||
|
|
||||||
|
timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
|
||||||
|
assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
|
||||||
|
|
||||||
|
# Check that the table is still there
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
|
||||||
|
output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
|
||||||
|
owner = output.split("\n")[1]
|
||||||
|
assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
|
||||||
|
|
||||||
|
# check if restore works if the database does not exist
|
||||||
|
machine.succeed("runuser -u postgres -- dropdb test")
|
||||||
|
machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user