yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'Expand backup and restore capabilities w.r.t. postgresql.' (#1582) from synapse into main

Browse Source
This commit is contained in:
clan-bot
2024-06-10 13:24:08 +00:00
parent dd2bd2f989 66bdc61e3d
commit a1acf0b05d
14 changed files with 411 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
clanModules/borgbackup/default.nix
View File

@@ -6,6 +6,27 @@
}:
let
cfg = config.clan.borgbackup;
preBackupScript = ''
declare -A preCommandErrors
${lib.concatMapStringsSep "\n" (
state:
lib.optionalString (state.preBackupCommand != null) ''
echo "Running pre-backup command for ${state.name}"
if ! ( ${state.preBackupCommand} ) then
preCommandErrors["${state.name}"]=1
fi
''
) (lib.attrValues config.clanCore.state)}
if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
echo "PreBackupCommand failed for the following services:"
for state in "''${!preCommandErrors[@]}"; do
echo " $state"
done
exit 1
fi
'';
in
{
options.clan.borgbackup.destinations = lib.mkOption {
@@ -50,17 +71,26 @@ in
];
config = lib.mkIf (cfg.destinations != { }) {
systemd.services = lib.mapAttrs' (
_: dest:
lib.nameValuePair "borgbackup-job-${dest.name}" {
# since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
serviceConfig.ExecStartPre = [
(''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}'')
];
}
) cfg.destinations;
services.borgbackup.jobs = lib.mapAttrs (_: dest: {
paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
paths = lib.unique (
lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state))
);
exclude = [ "*.pyc" ];
repo = dest.repo;
environment.BORG_RSH = dest.rsh;
compression = "auto,zstd";
startAt = "*-*-* 01:00:00";
persistentTimer = true;
preHook = ''
set -x
'';
encryption = {
mode = "repokey";

1
clanModules/flake-module.nix
View File

@@ -12,6 +12,7 @@
localsend = ./localsend;
matrix-synapse = ./matrix-synapse;
moonlight = ./moonlight;
postgresql = ./postgresql;
root-password = ./root-password;
sshd = ./sshd;
sunshine = ./sunshine;

62
clanModules/localbackup/default.nix
View File

@@ -6,7 +6,10 @@
}:
let
cfg = config.clan.localbackup;
rsnapshotConfig = target: states: ''
uniqueFolders = lib.unique (
lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clanCore.state)
);
rsnapshotConfig = target: ''
config_version 1.2
snapshot_root ${target.directory}
sync_first 1
@@ -17,12 +20,6 @@ let
cmd_logger ${pkgs.inetutils}/bin/logger
cmd_du ${pkgs.coreutils}/bin/du
cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff
${lib.optionalString (target.preBackupHook != null) ''
cmd_preexec ${pkgs.writeShellScript "preexec.sh" ''
set -efu -o pipefail
${target.preBackupHook}
''}
''}
${lib.optionalString (target.postBackupHook != null) ''
cmd_postexec ${pkgs.writeShellScript "postexec.sh" ''
@@ -31,11 +28,9 @@ let
''}
''}
retain snapshot ${builtins.toString config.clan.localbackup.snapshots}
${lib.concatMapStringsSep "\n" (state: ''
${lib.concatMapStringsSep "\n" (folder: ''
backup ${folder} ${config.networking.hostName}/
'') state.folders}
'') states}
${lib.concatMapStringsSep "\n" (folder: ''
backup ${folder} ${config.networking.hostName}/
'') uniqueFolders}
'';
in
{
@@ -129,14 +124,30 @@ in
]
}
${lib.concatMapStringsSep "\n" (target: ''
(
${mountHook target}
echo "Creating backup '${target.name}'"
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
)
'') (builtins.attrValues cfg.targets)}
'')
${mountHook target}
set -x
echo "Creating backup '${target.name}'"
${lib.optionalString (target.preBackupHook != null) ''
(
${target.preBackupHook}
)
''}
declare -A preCommandErrors
${lib.concatMapStringsSep "\n" (
state:
lib.optionalString (state.preBackupCommand != null) ''
echo "Running pre-backup command for ${state.name}"
if ! ( ${state.preBackupCommand} ) then
preCommandErrors["${state.name}"]=1
fi
''
) (builtins.attrValues config.clanCore.state)}
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
'') (builtins.attrValues cfg.targets)}'')
(pkgs.writeShellScriptBin "localbackup-list" ''
set -efu -o pipefail
export PATH=${
@@ -167,6 +178,14 @@ in
pkgs.gawk
]
}
if [[ "''${NAME:-}" == "" ]]; then
echo "No backup name given via NAME environment variable"
exit 1
fi
if [[ "''${FOLDERS:-}" == "" ]]; then
echo "No folders given via FOLDERS environment variable"
exit 1
fi
name=$(awk -F'::' '{print $1}' <<< $NAME)
backupname=''${NAME#$name::}
@@ -182,8 +201,9 @@ in
exit 1
fi
IFS=';' read -ra FOLDER <<< "$FOLDERS"
IFS=':' read -ra FOLDER <<< "''$FOLDERS"
for folder in "''${FOLDER[@]}"; do
mkdir -p "$folder"
rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
done
'')

82
clanModules/matrix-synapse/default.nix
View File

@@ -6,16 +6,35 @@
}:
let
cfg = config.clan.matrix-synapse;
nginx-vhost = "matrix.${config.clan.matrix-synapse.domain}";
element-web =
pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; }
''
cp -r ${pkgs.element-web} $out
chmod -R u+w $out
jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
> $out/config.json < ${pkgs.element-web}/config.json
ln -s $out/config.json $out/config.${nginx-vhost}.json
'';
in
{
options.clan.matrix-synapse = {
enable = lib.mkEnableOption "Enable matrix-synapse";
domain = lib.mkOption {
type = lib.types.str;
description = "The domain name of the matrix server";
example = "example.com";
};
};
config = lib.mkIf cfg.enable {
imports = [
(lib.mkRemovedOptionModule [
"clan"
"matrix-synapse"
"enable"
] "Importing the module will already enable the service.")
../postgresql
];
config = {
services.matrix-synapse = {
enable = true;
settings = {
@@ -49,16 +68,27 @@ in
}
];
};
extraConfigFiles = [ "/var/lib/matrix-synapse/registration_shared_secret.yaml" ];
extraConfigFiles = [ "/run/synapse-registration-shared-secret.yaml" ];
};
systemd.tmpfiles.settings."synapse" = {
"/run/synapse-registration-shared-secret.yaml" = {
C.argument =
config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
z = {
mode = "0400";
user = "matrix-synapse";
};
};
};
clan.postgresql.users.matrix-synapse = { };
clan.postgresql.databases.matrix-synapse.create.options = {
TEMPLATE = "template0";
LC_COLLATE = "C";
LC_CTYPE = "C";
ENCODING = "UTF8";
OWNER = "matrix-synapse";
};
systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
"+${pkgs.writeScript "copy_registration_shared_secret" ''
#!/bin/sh
cp ${config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path} /var/lib/matrix-synapse/registration_shared_secret.yaml
chown matrix-synapse:matrix-synapse /var/lib/matrix-synapse/registration_shared_secret.yaml
chmod 600 /var/lib/matrix-synapse/registration_shared_secret.yaml
''}"
];
clanCore.facts.services."matrix-synapse" = {
secret."synapse-registration_shared_secret" = { };
@@ -71,23 +101,6 @@ in
'';
};
services.postgresql.enable = true;
# we need to use both ensusureDatabases and initialScript, because the former runs everytime but with the wrong collation
services.postgresql = {
ensureDatabases = [ "matrix-synapse" ];
ensureUsers = [
{
name = "matrix-synapse";
ensureDBOwnership = true;
}
];
initialScript = pkgs.writeText "synapse-init.sql" ''
CREATE DATABASE "matrix-synapse"
TEMPLATE template0
LC_COLLATE = "C"
LC_CTYPE = "C";
'';
};
services.nginx = {
enable = true;
virtualHosts = {
@@ -102,7 +115,7 @@ in
return 200 '${
builtins.toJSON {
"m.homeserver" = {
"base_url" = "https://matrix.${cfg.domain}";
"base_url" = "https://${nginx-vhost}";
};
"m.identity_server" = {
"base_url" = "https://vector.im";
@@ -111,15 +124,12 @@ in
}';
'';
};
"matrix.${cfg.domain}" = {
${nginx-vhost} = {
forceSSL = true;
enableACME = true;
locations."/_matrix" = {
proxyPass = "http://localhost:8008";
};
locations."/test".extraConfig = ''
return 200 "Hello, world!";
'';
locations."/_matrix".proxyPass = "http://localhost:8008";
locations."/_synapse".proxyPass = "http://localhost:8008";
locations."/".root = element-web;
};
};
};

2
clanModules/postgresql/README.md Normal file
View File

@@ -0,0 +1,2 @@
A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.
---

166
clanModules/postgresql/default.nix Normal file
View File

@@ -0,0 +1,166 @@
{
pkgs,
lib,
config,
...
}:
let
createDatatbaseState =
db:
let
folder = "/var/backup/postgres/${db.name}";
current = "${folder}/pg-dump";
compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
in
{
folders = [ folder ];
preBackupCommand = ''
export PATH=${
lib.makeBinPath [
config.services.postgresql.package
config.systemd.package
pkgs.coreutils
pkgs.util-linux
pkgs.zstd
]
}
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
sleep 1
done
mkdir -p "${folder}"
runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
mv "${current}.tmp" ${current}
'';
postRestoreCommand = ''
export PATH=${
lib.makeBinPath [
config.services.postgresql.package
config.systemd.package
pkgs.coreutils
pkgs.util-linux
pkgs.zstd
]
}
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
sleep 1
done
echo "Waiting for postgres to be ready..."
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
if ! systemctl is-active postgresql; then exit 1; fi
sleep 0.1
done
if [[ -e "${current}" ]]; then
(
systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
mkdir -p "${folder}"
runuser -u postgres -- dropdb "${db.name}"
runuser -u postgres -- pg_restore -C -d postgres "${current}"
)
else
echo No database backup found, skipping restore
fi
'';
};
createDatabase = db: ''
CREATE DATABASE "${db.name}" ${
lib.concatStringsSep " " (
lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
)
}
'';
cfg = config.clan.postgresql;
userClauses = lib.mapAttrsToList (
_: user:
''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
) cfg.users;
databaseClauses = lib.mapAttrsToList (
name: db:
lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
) cfg.databases;
in
{
options.clan.postgresql = {
# we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
databases = lib.mkOption {
default = { };
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options = {
name = lib.mkOption {
type = lib.types.str;
default = name;
};
# set to false, in case the upstream module uses ensureDatabase option
create.enable = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Create the database if it does not exist.";
};
create.options = lib.mkOption {
type = lib.types.lazyAttrsOf lib.types.str;
default = { };
example = {
TEMPLATE = "template0";
LC_COLLATE = "C";
LC_CTYPE = "C";
ENCODING = "UTF8";
OWNER = "foo";
};
};
restore.stopOnRestore = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "List of services to stop before restoring the database.";
};
};
}
)
);
};
users = lib.mkOption {
default = { };
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options.name = lib.mkOption {
type = lib.types.str;
default = name;
};
}
)
);
};
};
config = {
services.postgresql.settings = {
wal_level = "replica";
max_wal_senders = 3;
};
services.postgresql.enable = true;
# We are duplicating a bit the upstream module but allow to create databases with options
systemd.services.postgresql.postStart = ''
PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
while ! $PSQL -d postgres -c "" 2> /dev/null; do
if ! kill -0 "$MAINPID"; then exit 1; fi
sleep 0.1
done
${lib.concatStringsSep "\n" userClauses}
${lib.concatStringsSep "\n" databaseClauses}
'';
clanCore.state = lib.mapAttrs' (
_: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
) config.clan.postgresql.databases;
};
}