diff --git a/clanServices/monitoring/default.nix b/clanServices/monitoring/default.nix index 1369d8ca2..96e8292e3 100644 --- a/clanServices/monitoring/default.nix +++ b/clanServices/monitoring/default.nix @@ -24,5 +24,12 @@ }; }; - imports = [ ./telegraf.nix ]; + # roles.prometheus = { + # interface = { lib, ... }: { }; + # }; + + imports = [ + ./telegraf.nix + ./prometheus.nix + ]; } diff --git a/clanServices/monitoring/prometheus.nix b/clanServices/monitoring/prometheus.nix new file mode 100644 index 000000000..7ac8f0514 --- /dev/null +++ b/clanServices/monitoring/prometheus.nix @@ -0,0 +1,182 @@ +{ + roles.prometheus.perInstance = + { settings, roles, ... }: + { + nixosModule = + { pkgs, lib, ... }: + { + + # imports = [ + # # ./matrix-alertmanager.nix + # # ./irc-alertmanager.nix + # # ./rules.nix + # ]; + + services.prometheus = { + + # webExternalUrl = "https://prometheus.thalheim.io"; + extraFlags = [ "--storage.tsdb.retention.time=30d" ]; + scrapeConfigs = [ + { + job_name = "telegraf"; + scrape_interval = "60s"; + metrics_path = "/metrics"; + static_configs = [ + + (map (host: { + + labels.host = host; + # labels.org = "TODO"; + targets = [ "${host}.clan:9273" ]; + + }) lib.attrNames roles.telegraf.machines) + + # { + # # labels.host = "rauter.r:9273"; + # # labels.org = "TODO"; + # targets = map (host: "${host}.clan:9273") lib.attrNames roles.telegraf.machines; + # } + ]; + } + # { + # job_name = "gitea"; + # scrape_interval = "60s"; + # metrics_path = "/metrics"; + # + # scheme = "https"; + # static_configs = [ { targets = [ "git.thalheim.io:443" ]; } ]; + # } + ]; + alertmanagers = [ { static_configs = [ { targets = [ "localhost:9093" ]; } ]; } ]; + }; + services.prometheus.alertmanager = { + enable = true; + # environmentFile = config.sops.secrets.alertmanager.path; + # webExternalUrl = "https://alertmanager.thalheim.io"; + # listenAddress = "[::1]"; + # configuration = { + # global = { + # # The smarthost and SMTP sender used for mail notifications. + # smtp_smarthost = "mail.thalheim.io:587"; + # smtp_from = "alertmanager@thalheim.io"; + # smtp_auth_username = "alertmanager@thalheim.io"; + # smtp_auth_password = "$SMTP_PASSWORD"; + # }; + # route = { + # receiver = "default"; + # routes = [ + # { + # group_by = [ "host" ]; + # match_re.org = "krebs"; + # group_wait = "5m"; + # group_interval = "5m"; + # repeat_interval = "4h"; + # receiver = "krebs"; + # } + # { + # group_by = [ "host" ]; + # match_re.org = "nixos-wiki"; + # group_wait = "5m"; + # group_interval = "5m"; + # repeat_interval = "4h"; + # receiver = "nixos-wiki"; + # } + # { + # group_by = [ "host" ]; + # match_re.org = "numtide"; + # group_wait = "5m"; + # group_interval = "5m"; + # repeat_interval = "4h"; + # receiver = "numtide"; + # } + # { + # group_by = [ "host" ]; + # match_re.org = "clan-lol"; + # group_wait = "5m"; + # group_interval = "5m"; + # repeat_interval = "4h"; + # receiver = "clan-lol"; + # } + # { + # group_by = [ "host" ]; + # match_re.org = "dave"; + # group_wait = "5m"; + # group_interval = "5m"; + # repeat_interval = "4h"; + # receiver = "dave"; + # } + # { + # group_by = [ "host" ]; + # group_wait = "30s"; + # group_interval = "2m"; + # repeat_interval = "2h"; + # receiver = "all"; + # } + # ]; + # }; + # receivers = [ + # { + # name = "krebs"; + # webhook_configs = [ + # { + # url = "http://127.0.0.1:9223/"; + # max_alerts = 5; + # } + # ]; + # } + # { + # name = "numtide"; + # webhook_configs = [ + # # TODO + # #{ + # # send_resolved = true; + # # url = "https://chat.ntd.one/plugins/alertmanager/api/webhook?token='xxxxxxxxxxxxxxxxxxx-yyyyyyy'"; + # #} + # ]; + # } + # { + # name = "nixos-wiki"; + # webhook_configs = [ + # { + # url = "http://localhost:9088/alert"; + # max_alerts = 5; + # } + # ]; + # } + # { + # name = "clan-lol"; + # webhook_configs = [ + # # TODO + # #{ + # # url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U"; + # # max_alerts = 5; + # #} + # ]; + # } + # { + # name = "dave"; + # telegram_configs = [ + # { + # chat_id = 42927997; + # bot_token = "$TELEGRAM_BOT_TOKEN"; + # } + # ]; + # } + # { + # name = "all"; + # # pushover_configs = [ + # # { + # # user_key = "$PUSHOVER_USER_KEY"; + # # token = "$PUSHOVER_TOKEN"; + # # priority = "0"; + # # } + # # ]; + # } + # { name = "default"; } + # ]; + # }; + }; + + }; + }; +}