{ roles.telegraf.perInstance = { settings, ... }: { nixosModule = { config, pkgs, lib, ... }: let auth_user = "prometheus"; in { warnings = lib.optionals (settings.allowAllInterfaces != null) [ "monitoring.settings.allowAllInterfaces is deprecated and and has no effect. Please remove it from your inventory." "The monitoring service will now always listen on all interfaces over https." ] ++ (lib.optionals (settings.interfaces != null) [ "monitoring.settings.interfaces is deprecated and and has no effect. Please remove it from your inventory." "The monitoring service will now always listen on all interfaces over https." ]); networking.firewall.allowedTCPPorts = [ 9273 9990 ]; clan.core.vars.generators."telegraf-certs" = { files.crt = { restartUnits = [ "telegraf.service" ]; deploy = true; secret = false; }; files.key = { mode = "0600"; restartUnits = [ "telegraf.service" ]; }; runtimeInputs = [ pkgs.openssl ]; # TODO: Implement automated certificate rotation instead of using a 100-year expiration script = '' openssl req -x509 -nodes -newkey rsa:4096 \ -days 36500 \ -keyout "$out"/key \ -out "$out"/crt \ -subj "/C=US/ST=CA/L=San Francisco/O=Example Corp/OU=IT/CN=example.com" ''; }; clan.core.vars.generators."telegraf" = { files.password.restartUnits = [ "telegraf.service" ]; files.password-env.restartUnits = [ "telegraf.service" ]; files.miniserve-auth.restartUnits = [ "telegraf.service" ]; dependencies = [ "telegraf-certs" ]; runtimeInputs = [ pkgs.coreutils pkgs.xkcdpass pkgs.mkpasswd ]; script = '' PASSWORD=$(xkcdpass --numwords 4 --delimiter - --count 1 | tr -d "\n") echo "BASIC_AUTH_PWD=$PASSWORD" > "$out"/password-env echo "${auth_user}:$PASSWORD" > "$out"/miniserve-auth echo "$PASSWORD" | tr -d "\n" > "$out"/password ''; }; systemd.services.telegraf-json = { enable = true; wantedBy = [ "multi-user.target" ]; after = [ "telegraf.service" ]; requires = [ "telegraf.service" ]; serviceConfig = { LoadCredential = [ "auth_file_path:${config.clan.core.vars.generators.telegraf.files.miniserve-auth.path}" "telegraf_crt_path:${config.clan.core.vars.generators.telegraf-certs.files.crt.path}" "telegraf_key_path:${config.clan.core.vars.generators.telegraf-certs.files.key.path}" ]; Environment = [ "AUTH_FILE_PATH=%d/auth_file_path" "CRT_PATH=%d/telegraf_crt_path" "KEY_PATH=%d/telegraf_key_path" ]; Restart = "on-failure"; User = "telegraf"; Group = "telegraf"; RuntimeDirectory = "telegraf-www"; }; script = "${pkgs.miniserve}/bin/miniserve -p 9990 /run/telegraf-www --auth-file \"$AUTH_FILE_PATH\" --tls-cert \"$CRT_PATH\" --tls-key \"$KEY_PATH\""; }; systemd.services.telegraf = { serviceConfig = { LoadCredential = [ "telegraf_crt_path:${config.clan.core.vars.generators.telegraf-certs.files.crt.path}" "telegraf_key_path:${config.clan.core.vars.generators.telegraf-certs.files.key.path}" ]; Environment = [ "CRT_PATH=%d/telegraf_crt_path" "KEY_PATH=%d/telegraf_key_path" ]; }; }; services.telegraf = { enable = true; environmentFiles = [ (builtins.toString config.clan.core.vars.generators.telegraf.files.password-env.path) ]; extraConfig = { agent.interval = "60s"; inputs = { diskio = { }; kernel_vmstat = { }; system = { }; mem = { }; systemd_units = { }; swap = { }; exec = let nixosSystems = pkgs.writeShellScript "current-system" '' printf "nixos_systems,current_system=%s,booted_system=%s,current_kernel=%s,booted_kernel=%s present=0\n" \ "$(readlink /run/current-system)" "$(readlink /run/booted-system)" \ "$(basename $(echo /run/current-system/kernel-modules/lib/modules/*))" \ "$(basename $(echo /run/booted-system/kernel-modules/lib/modules/*))" ''; in [ { # Expose the path to current-system as metric. We use # this to check if the machine is up-to-date. commands = [ nixosSystems ]; data_format = "influx"; } ]; }; # sadly there doesn'T seem to exist a telegraf http_client output plugin outputs.prometheus_client = { listen = ":9273"; metric_version = 2; basic_username = "${auth_user}"; basic_password = "$${BASIC_AUTH_PWD}"; tls_cert = "$${CRT_PATH}"; tls_key = "$${KEY_PATH}"; }; outputs.file = { files = [ "/run/telegraf-www/telegraf.json" ]; data_format = "json"; json_timestamp_units = "1s"; }; }; }; }; }; }