Merge remote-tracking branch 'origin/main' into rework-installation

This commit is contained in:
Jörg Thalheim
2024-08-21 13:33:27 +02:00
196 changed files with 10069 additions and 2432 deletions

View File

@@ -0,0 +1,6 @@
---
description = "A dynamic DNS service to update domain IPs"
---

View File

@@ -0,0 +1,265 @@
{
config,
pkgs,
lib,
...
}:
let
name = "dyndns";
cfg = config.clan.${name};
# We dedup secrets if they have the same provider + base domain
secret_id = opt: "${name}-${opt.provider}-${opt.domain}";
secret_path =
opt: config.clan.core.facts.services."${secret_id opt}".secret."${secret_id opt}".path;
# We check that a secret has not been set in extraSettings.
extraSettingsSafe =
opt:
if (builtins.hasAttr opt.secret_field_name opt.extraSettings) then
throw "Please do not set ${opt.secret_field_name} in extraSettings, it is automatically set by the dyndns module."
else
opt.extraSettings;
/*
We go from:
{home.example.com:{value:{domain:example.com,host:home, provider:namecheap}}}
To:
{settings: [{domain: example.com, host: home, provider: namecheap, password: dyndns-namecheap-example.com}]}
*/
service_config = {
settings = builtins.catAttrs "value" (
builtins.attrValues (
lib.mapAttrs (_: opt: {
value =
(extraSettingsSafe opt)
// {
domain = opt.domain;
provider = opt.provider;
}
// {
"${opt.secret_field_name}" = secret_id opt;
};
}) cfg.settings
)
);
};
secret_generator = _: opt: {
name = secret_id opt;
value = {
secret.${secret_id opt} = { };
generator.prompt = "Dyndns passphrase for ${secret_id opt}";
generator.script = ''
echo "$prompt_value" > $secrets/${secret_id opt}
'';
};
};
in
{
options.clan.${name} = {
user = lib.mkOption {
type = lib.types.str;
default = name;
description = "User to run the service as";
};
group = lib.mkOption {
type = lib.types.str;
default = name;
description = "Group to run the service as";
};
server = {
enable = lib.mkEnableOption "dyndns webserver";
domain = lib.mkOption {
type = lib.types.str;
description = "Domain to serve the webservice on";
};
port = lib.mkOption {
type = lib.types.int;
default = 54805;
description = "Port to listen on";
};
};
period = lib.mkOption {
type = lib.types.int;
default = 5;
description = "Domain update period in minutes";
};
settings = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule (
{ ... }:
{
options = {
provider = lib.mkOption {
example = "namecheap";
type = lib.types.str;
description = "The dyndns provider to use";
};
domain = lib.mkOption {
type = lib.types.str;
example = "example.com";
description = "The top level domain to update.";
};
secret_field_name = lib.mkOption {
example = [
"password"
"api_key"
];
type = lib.types.enum [
"password"
"token"
"api_key"
];
default = "password";
description = "The field name for the secret";
};
# TODO: Ideally we would create a gigantic list of all possible settings / types
# optimally we would have a way to generate the options from the source code
extraSettings = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
description = ''
Extra settings for the provider.
Provider specific settings: https://github.com/qdm12/ddns-updater#configuration
'';
};
};
}
)
);
default = [ ];
description = "Configuration for which domains to update";
};
};
imports = [
(lib.mkRemovedOptionModule [
"clan"
"dyndns"
"enable"
] "Just define clan.dyndns.settings to enable it")
];
config = lib.mkMerge [
(lib.mkIf (cfg.settings != { }) {
clan.core.facts.services = lib.mapAttrs' secret_generator cfg.settings;
users.groups.${cfg.group} = { };
users.users.${cfg.user} = {
group = cfg.group;
isSystemUser = true;
description = "User for ${name} service";
home = "/var/lib/${name}";
createHome = true;
};
networking.firewall.allowedTCPPorts = lib.mkIf cfg.server.enable [
80
443
];
services.nginx = lib.mkIf cfg.server.enable {
enable = true;
virtualHosts = {
"${cfg.server.domain}" = {
forceSSL = true;
enableACME = true;
locations."/" = {
proxyPass = "http://localhost:${toString cfg.server.port}";
};
};
};
};
systemd.services.${name} = {
path = [ ];
description = "Dynamic DNS updater";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
environment = {
MYCONFIG = "${builtins.toJSON service_config}";
SERVER_ENABLED = if cfg.server.enable then "yes" else "no";
PERIOD = "${toString cfg.period}m";
LISTENING_ADDRESS = ":${toString cfg.server.port}";
};
serviceConfig =
let
pyscript = pkgs.writers.writePyPy3Bin "test.py" { libraries = [ ]; } ''
import json
from pathlib import Path
import os
cred_dir = Path(os.getenv("CREDENTIALS_DIRECTORY"))
config_str = os.getenv("MYCONFIG")
def get_credential(name):
secret_p = cred_dir / name
with open(secret_p, 'r') as f:
return f.read().strip()
config = json.loads(config_str)
print(f"Config: {config}")
for attrset in config["settings"]:
if "password" in attrset:
attrset['password'] = get_credential(attrset['password'])
elif "token" in attrset:
attrset['token'] = get_credential(attrset['token'])
elif "api_key" in attrset:
attrset['api_key'] = get_credential(attrset['api_key'])
else:
raise ValueError(f"Missing secret field in {attrset}")
# create directory data if it does not exist
data_dir = Path('data')
data_dir.mkdir(mode=0o770, exist_ok=True)
# Write the config with secrets back
config_path = data_dir / 'config.json'
with open(config_path, 'w') as f:
f.write(json.dumps(config, indent=4))
# Set file permissions to read and write
# only by the user and group
config_path.chmod(0o660)
# Set file permissions to read
# and write only by the user and group
for file in data_dir.iterdir():
file.chmod(0o660)
'';
in
{
ExecStartPre = lib.getExe pyscript;
ExecStart = lib.getExe pkgs.ddns-updater;
LoadCredential = lib.mapAttrsToList (_: opt: "${secret_id opt}:${secret_path opt}") cfg.settings;
User = cfg.user;
Group = cfg.group;
NoNewPrivileges = true;
PrivateTmp = true;
ProtectSystem = "strict";
ReadOnlyPaths = "/";
PrivateDevices = "yes";
ProtectKernelModules = "yes";
ProtectKernelTunables = "yes";
WorkingDirectory = "/var/lib/${name}";
ReadWritePaths = [
"/proc/self"
"/var/lib/${name}"
];
Restart = "always";
RestartSec = 60;
};
};
})
];
}

View File

@@ -4,18 +4,23 @@
borgbackup = ./borgbackup;
borgbackup-static = ./borgbackup-static;
deltachat = ./deltachat;
dyndns = ./dyndns;
ergochat = ./ergochat;
garage = ./garage;
golem-provider = ./golem-provider;
iwd = ./iwd;
localbackup = ./localbackup;
localsend = ./localsend;
single-disk = ./single-disk;
matrix-synapse = ./matrix-synapse;
moonlight = ./moonlight;
mumble = ./mumble;
packages = ./packages;
postgresql = ./postgresql;
root-password = ./root-password;
single-disk = ./single-disk;
sshd = ./sshd;
sunshine = ./sunshine;
static-hosts = ./static-hosts;
sunshine = ./sunshine;
syncthing = ./syncthing;
syncthing-static-peers = ./syncthing-static-peers;
thelounge = ./thelounge;

View File

@@ -0,0 +1,10 @@
---
description = "S3-compatible object store for small self-hosted geo-distributed deployments"
---
This module generates garage specific keys automatically.
When using garage in a distributed deployment the `rpc_key` between connected instances must be shared.
This is currently still a manual process.
Options: [NixosModuleOptions](https://search.nixos.org/options?channel=unstable&size=50&sort=relevance&type=packages&query=garage)
Documentation: https://garagehq.deuxfleurs.fr/

View File

@@ -0,0 +1,33 @@
{ config, pkgs, ... }:
{
systemd.services.garage.serviceConfig = {
LoadCredential = [
"rpc_secret_path:${config.clan.core.vars.generators.garage.files.rpc_secret.path}"
"admin_token_path:${config.clan.core.vars.generators.garage.files.admin_token.path}"
"metrics_token_path:${config.clan.core.vars.generators.garage.files.metrics_token.path}"
];
Environment = [
"GARAGE_ALLOW_WORLD_READABLE_SECRETS=true"
"GARAGE_RPC_SECRET_FILE=%d/rpc_secret_path"
"GARAGE_ADMIN_TOKEN_FILE=%d/admin_token_path"
"GARAGE_METRICS_TOKEN_FILE=%d/metrics_token_path"
];
};
clan.core.vars.generators.garage = {
files.rpc_secret = { };
files.admin_token = { };
files.metrics_token = { };
runtimeInputs = [
pkgs.coreutils
pkgs.openssl
];
script = ''
openssl rand -hex -out $out/rpc_secret 32
openssl rand -base64 -out $out/admin_token 32
openssl rand -base64 -out $out/metrics_token 32
'';
};
clan.core.state.garage.folders = [ config.services.garage.settings.metadata_dir ];
}

View File

@@ -0,0 +1,7 @@
---
description = "Golem Provider for the Golem Network, an open-source and decentralized platform where everyone can use and share each other's computing power without relying on centralized entities like cloud computing corporations"
---
By running a golem provider your machine's compute resources are offered via the golem network which will allow other members to execute compute tasks on your machine. If this happens, you will be compensated with GLM, an ERC20 token.
More about golem providers: https://docs.golem.network/docs/golem/overview

View File

@@ -0,0 +1,34 @@
{ config, pkgs, ... }:
let
cfg = config.clan.golem-provider;
yagna = pkgs.callPackage ../../pkgs/yagna { };
accountFlag = if cfg.account != null then "--account ${cfg.account}" else "";
in
{
imports = [ ./interface.nix ];
users.users.golem = {
isSystemUser = true;
home = "/var/lib/golem";
group = "golem";
createHome = true;
};
users.groups.golem = { };
environment.systemPackages = [ yagna ];
systemd.services.golem-provider = {
description = "Golem Provider";
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
serviceConfig = {
Type = "simple";
ExecStart = "${yagna}/bin/golemsp run --no-interactive ${accountFlag}";
Restart = "always";
RestartSec = "5";
User = "golem";
Group = "golem";
};
};
}

View File

@@ -0,0 +1,20 @@
{ lib, ... }:
let
inherit (lib) mkOption;
inherit (lib.types) nullOr str;
in
{
options.clan.golem-provider = {
account = mkOption {
type = nullOr str;
description = ''
Ethereum address for payouts.
Leave empty to automatically generate a new address upon first start.
'';
default = null;
};
};
}

View File

@@ -0,0 +1,4 @@
{ ... }:
{
imports = [ ../. ];
}

View File

@@ -0,0 +1,6 @@
---
description = "Automatically provisions wifi credentials"
---

View File

@@ -0,0 +1,83 @@
{ lib, config, ... }:
let
cfg = config.clan.iwd;
secret_path = ssid: config.clan.core.facts.services."iwd.${ssid}".secret."iwd.${ssid}".path;
secret_generator = name: value: {
name = "iwd.${value.ssid}";
value =
let
secret_name = "iwd.${value.ssid}";
in
{
secret.${secret_name} = { };
generator.prompt = "Wifi password for '${value.ssid}'";
generator.script = ''
config="
[Security]
Passphrase=$prompt_value
"
echo "$config" > $secrets/${secret_name}
'';
};
};
in
{
options.clan.iwd = {
networks = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options = {
ssid = lib.mkOption {
type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
default = name;
description = "The name of the wifi network";
};
};
}
)
);
default = { };
description = "Wifi networks to predefine";
};
};
imports = [
(lib.mkRemovedOptionModule [
"clan"
"iwd"
"enable"
] "Just define clan.iwd.networks to enable it")
];
config = lib.mkMerge [
(lib.mkIf (cfg.networks != { }) {
# Systemd tmpfiles rule to create /var/lib/iwd/example.psk file
systemd.tmpfiles.rules = lib.mapAttrsToList (
_: value: "C /var/lib/iwd/${value.ssid}.psk 0600 root root - ${secret_path value.ssid}"
) cfg.networks;
clan.core.facts.services = lib.mapAttrs' secret_generator cfg.networks;
# TODO: restart the iwd.service if something changes
})
{
# disable wpa supplicant
networking.wireless.enable = false;
# Use iwd instead of wpa_supplicant. It has a user friendly CLI
networking.wireless.iwd = {
enable = true;
settings = {
Network = {
EnableIPv6 = true;
RoutePriorityOffset = 300;
};
Settings.AutoConnect = true;
};
};
}
];
}

View File

@@ -169,6 +169,11 @@ in
];
};
networking.firewall.allowedTCPPorts = [
80
443
];
services.nginx = {
enable = true;
virtualHosts = {

View File

@@ -0,0 +1,14 @@
---
description = "Open Source, Low Latency, High Quality Voice Chat."
categories = ["chat", "voice"]
---
The mumble clan module gives you:
- True low latency voice communication.
- Secure, authenticated encryption.
- Free software.
- Backed by a large and active open-source community.
This all set up in a way that allows peer-to-peer hosting.
Every machine inside the clan can be a host for mumble,
and thus it doesn't matter who in the network is online - as long as two people are online they are able to chat with each other.

View File

@@ -0,0 +1,104 @@
{
lib,
config,
pkgs,
...
}:
let
clanDir = config.clan.core.clanDir;
machineDir = clanDir + "/machines/";
machinesFileSet = builtins.readDir machineDir;
machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
machineJson = builtins.toJSON machines;
certificateMachinePath = machines: machineDir + "/${machines}" + "/facts/mumble-cert";
certificatesUnchecked = builtins.map (
machine:
let
fullPath = certificateMachinePath machine;
in
if builtins.pathExists fullPath then machine else null
) machines;
certificate = lib.filter (machine: machine != null) certificatesUnchecked;
machineCert = builtins.map (
machine: (lib.nameValuePair machine (builtins.readFile (certificateMachinePath machine)))
) certificate;
machineCertJson = builtins.toJSON machineCert;
in
{
options.clan.services.mumble = {
user = lib.mkOption {
type = lib.types.str;
default = "alice";
description = "The user mumble should be set up for.";
};
};
config = {
services.murmur = {
enable = true;
logDays = -1;
registerName = config.clan.core.machineName;
openFirewall = true;
bonjour = true;
sslKey = config.clan.core.facts.services.mumble.secret.mumble-key.path;
sslCert = config.clan.core.facts.services.mumble.public.mumble-cert.path;
};
clan.core.state.mumble.folders = [
"/var/lib/mumble"
"/var/lib/murmur"
];
systemd.tmpfiles.rules = [
"d '/var/lib/mumble' 0770 '${config.clan.services.mumble.user}' 'users' - -"
];
environment.systemPackages =
let
mumbleCfgDir = "/var/lib/mumble";
mumbleDatabasePath = "${mumbleCfgDir}/mumble.sqlite";
mumbleCfgPath = "/var/lib/mumble/mumble_settings.json";
populate-channels = pkgs.writers.writePython3 "mumble-populate-channels" {
libraries = [
pkgs.python3Packages.cryptography
pkgs.python3Packages.pyopenssl
];
flakeIgnore = [
# We don't live in the dark ages anymore.
# Languages like Python that are whitespace heavy will overrun
# 79 characters..
"E501"
];
} (builtins.readFile ./mumble-populate-channels.py);
mumble = pkgs.writeShellScriptBin "mumble" ''
set -xeu
mkdir -p ${mumbleCfgDir}
pushd "${mumbleCfgDir}"
XDG_DATA_HOME=${mumbleCfgDir}
XDG_DATA_DIR=${mumbleCfgDir}
${populate-channels} --ensure-config '${mumbleCfgPath}' --db-location ${mumbleDatabasePath}
echo ${machineCertJson}
${populate-channels} --machines '${machineJson}' --username ${config.clan.core.machineName} --db-location ${mumbleDatabasePath}
${populate-channels} --servers '${machineCertJson}' --username ${config.clan.core.machineName} --db-location ${mumbleDatabasePath} --cert True
${pkgs.mumble}/bin/mumble --config ${mumbleCfgPath} "$@"
popd
'';
in
[ mumble ];
clan.core.facts.services.mumble = {
secret.mumble-key = { };
public.mumble-cert = { };
generator.path = [
pkgs.coreutils
pkgs.openssl
];
generator.script = ''
openssl genrsa -out $secrets/mumble-key 2048
openssl req -new -x509 -key $secrets/mumble-key -out $facts/mumble-cert
'';
};
};
}

View File

@@ -0,0 +1,249 @@
import argparse
import json
import os
import sqlite3
def ensure_config(path: str, db_path: str) -> None:
# Default JSON structure if the file doesn't exist
default_json = {
"misc": {
"audio_wizard_has_been_shown": True,
"database_location": db_path,
"viewed_server_ping_consent_message": True,
},
"settings_version": 1,
}
# Check if the file exists
if os.path.exists(path):
with open(path) as file:
data = json.load(file)
else:
data = default_json
# Create the file with default JSON structure
with open(path, "w") as file:
json.dump(data, file, indent=4)
# TODO: make sure to only update the diff
updated_data = {**default_json, **data}
# Write the modified JSON object back to the file
with open(path, "w") as file:
json.dump(updated_data, file, indent=4)
def initialize_database(db_location: str) -> None:
"""
Initializes the database. If the database or the servers table does not exist, it creates them.
:param db_location: The path to the SQLite database
"""
conn = sqlite3.connect(db_location)
try:
cursor = conn.cursor()
# Create the servers table if it doesn't exist
cursor.execute("""
CREATE TABLE IF NOT EXISTS servers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
hostname TEXT NOT NULL,
port INTEGER NOT NULL,
username TEXT NOT NULL,
password TEXT NOT NULL,
url TEXT
)
""")
# Commit the changes
conn.commit()
except sqlite3.Error as e:
print(f"An error occurred while initializing the database: {e}")
finally:
conn.close()
def initialize_certificates(
db_location: str, hostname: str, port: str, digest: str
) -> None:
# Connect to the SQLite database
conn = sqlite3.connect(db_location)
try:
# Create a cursor object
cursor = conn.cursor()
# TODO: check if cert already there
# if server_check(cursor, name, hostname):
# print(
# f"Server with name '{name}' and hostname '{hostname}' already exists."
# )
# return
# SQL command to insert data into the servers table
insert_query = """
INSERT INTO cert (hostname, port, digest)
VALUES (?, ?, ?)
"""
# Data to be inserted
data = (hostname, port, digest)
# Execute the insert command with the provided data
cursor.execute(insert_query, data)
# Commit the changes
conn.commit()
print("Data has been successfully inserted.")
except sqlite3.Error as e:
print(f"An error occurred: {e}")
finally:
# Close the connection
conn.close()
pass
def calculate_digest(cert: str) -> str:
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
cert = cert.strip()
cert = cert.encode("utf-8")
cert = x509.load_pem_x509_certificate(cert, default_backend())
digest = cert.fingerprint(hashes.SHA1()).hex()
return digest
def server_check(cursor: str, name: str, hostname: str) -> bool:
"""
Check if a server with the given name and hostname already exists.
:param cursor: The database cursor
:param name: The name of the server
:param hostname: The hostname of the server
:return: True if the server exists, False otherwise
"""
check_query = """
SELECT 1 FROM servers WHERE name = ? AND hostname = ?
"""
cursor.execute(check_query, (name, hostname))
return cursor.fetchone() is not None
def insert_server(
name: str,
hostname: str,
port: str,
username: str,
password: str,
url: str,
db_location: str,
) -> None:
"""
Inserts a new server record into the servers table.
:param name: The name of the server
:param hostname: The hostname of the server
:param port: The port number
:param username: The username
:param password: The password
:param url: The URL
"""
# Connect to the SQLite database
conn = sqlite3.connect(db_location)
try:
# Create a cursor object
cursor = conn.cursor()
if server_check(cursor, name, hostname):
print(
f"Server with name '{name}' and hostname '{hostname}' already exists."
)
return
# SQL command to insert data into the servers table
insert_query = """
INSERT INTO servers (name, hostname, port, username, password, url)
VALUES (?, ?, ?, ?, ?, ?)
"""
# Data to be inserted
data = (name, hostname, port, username, password, url)
# Execute the insert command with the provided data
cursor.execute(insert_query, data)
# Commit the changes
conn.commit()
print("Data has been successfully inserted.")
except sqlite3.Error as e:
print(f"An error occurred: {e}")
finally:
# Close the connection
conn.close()
if __name__ == "__main__":
port = 64738
password = ""
url = None
parser = argparse.ArgumentParser(
prog="initialize_mumble",
)
subparser = parser.add_subparsers(dest="certificates")
# cert_parser = subparser.add_parser("certificates")
parser.add_argument("--cert")
parser.add_argument("--digest")
parser.add_argument("--machines")
parser.add_argument("--servers")
parser.add_argument("--username")
parser.add_argument("--db-location")
parser.add_argument("--ensure-config")
args = parser.parse_args()
print(args)
if args.ensure_config:
ensure_config(args.ensure_config, args.db_location)
print("Initialized config")
exit(0)
if args.servers:
print(args.servers)
servers = json.loads(f"{args.servers}")
db_location = args.db_location
for server in servers:
digest = calculate_digest(server.get("value"))
name = server.get("name")
initialize_certificates(db_location, name, port, digest)
print("Initialized certificates")
exit(0)
initialize_database(args.db_location)
# Insert the server into the database
print(args.machines)
machines = json.loads(f"{args.machines}")
print(machines)
print(list(machines))
for machine in list(machines):
print(f"Inserting {machine}.")
insert_server(
machine,
machine,
port,
args.username,
password,
url,
args.db_location,
)

View File

@@ -0,0 +1,42 @@
{ pkgs, self, ... }:
pkgs.nixosTest {
name = "mumble";
nodes.peer1 =
{ ... }:
{
imports = [
self.nixosModules.mumble
self.inputs.clan-core.nixosModules.clanCore
{
config = {
clan.core.machineName = "peer1";
clan.core.clanDir = ./.;
documentation.enable = false;
};
}
];
};
nodes.peer2 =
{ ... }:
{
imports = [
self.nixosModules.mumble
self.inputs.clan-core.nixosModules.clanCore
{
config = {
clan.core.machineName = "peer2";
clan.core.clanDir = ./.;
documentation.enable = false;
};
}
];
};
testScript = ''
start_all()
'';
}