Add prometheus role to monitoring

clanServices/monitoring/default.nix

@@ -24,5 +24,12 @@
       };
   };
 
-  imports = [ ./telegraf.nix ];
+  # roles.prometheus = {
+  #   interface = { lib, ... }: { };
+  # };
+
+  imports = [
+    ./telegraf.nix
+    ./prometheus.nix
+  ];
 }

clanServices/monitoring/prometheus.nix

@@ -0,0 +1,182 @@
+{
+  roles.prometheus.perInstance =
+    { settings, roles, ... }:
+    {
+      nixosModule =
+        { pkgs, lib, ... }:
+        {
+
+          # imports = [
+          #   # ./matrix-alertmanager.nix
+          #   # ./irc-alertmanager.nix
+          #   # ./rules.nix
+          # ];
+
+          services.prometheus = {
+
+            # webExternalUrl = "https://prometheus.thalheim.io";
+            extraFlags = [ "--storage.tsdb.retention.time=30d" ];
+            scrapeConfigs = [
+              {
+                job_name = "telegraf";
+                scrape_interval = "60s";
+                metrics_path = "/metrics";
+                static_configs = [
+
+                  (map (host: {
+
+                    labels.host = host;
+                    # labels.org = "TODO";
+                    targets = [ "${host}.clan:9273" ];
+
+                  }) lib.attrNames roles.telegraf.machines)
+
+                  # {
+                  #   # labels.host = "rauter.r:9273";
+                  #   # labels.org = "TODO";
+                  #   targets = map (host: "${host}.clan:9273") lib.attrNames roles.telegraf.machines;
+                  # }
+                ];
+              }
+              # {
+              #   job_name = "gitea";
+              #   scrape_interval = "60s";
+              #   metrics_path = "/metrics";
+              #
+              #   scheme = "https";
+              #   static_configs = [ { targets = [ "git.thalheim.io:443" ]; } ];
+              # }
+            ];
+            alertmanagers = [ { static_configs = [ { targets = [ "localhost:9093" ]; } ]; } ];
+          };
+          services.prometheus.alertmanager = {
+            enable = true;
+            # environmentFile = config.sops.secrets.alertmanager.path;
+            # webExternalUrl = "https://alertmanager.thalheim.io";
+            # listenAddress = "[::1]";
+            # configuration = {
+            #   global = {
+            #     # The smarthost and SMTP sender used for mail notifications.
+            #     smtp_smarthost = "mail.thalheim.io:587";
+            #     smtp_from = "alertmanager@thalheim.io";
+            #     smtp_auth_username = "alertmanager@thalheim.io";
+            #     smtp_auth_password = "$SMTP_PASSWORD";
+            #   };
+            #   route = {
+            #     receiver = "default";
+            #     routes = [
+            #       {
+            #         group_by = [ "host" ];
+            #         match_re.org = "krebs";
+            #         group_wait = "5m";
+            #         group_interval = "5m";
+            #         repeat_interval = "4h";
+            #         receiver = "krebs";
+            #       }
+            #       {
+            #         group_by = [ "host" ];
+            #         match_re.org = "nixos-wiki";
+            #         group_wait = "5m";
+            #         group_interval = "5m";
+            #         repeat_interval = "4h";
+            #         receiver = "nixos-wiki";
+            #       }
+            #       {
+            #         group_by = [ "host" ];
+            #         match_re.org = "numtide";
+            #         group_wait = "5m";
+            #         group_interval = "5m";
+            #         repeat_interval = "4h";
+            #         receiver = "numtide";
+            #       }
+            #       {
+            #         group_by = [ "host" ];
+            #         match_re.org = "clan-lol";
+            #         group_wait = "5m";
+            #         group_interval = "5m";
+            #         repeat_interval = "4h";
+            #         receiver = "clan-lol";
+            #       }
+            #       {
+            #         group_by = [ "host" ];
+            #         match_re.org = "dave";
+            #         group_wait = "5m";
+            #         group_interval = "5m";
+            #         repeat_interval = "4h";
+            #         receiver = "dave";
+            #       }
+            #       {
+            #         group_by = [ "host" ];
+            #         group_wait = "30s";
+            #         group_interval = "2m";
+            #         repeat_interval = "2h";
+            #         receiver = "all";
+            #       }
+            #     ];
+            #   };
+            #   receivers = [
+            #     {
+            #       name = "krebs";
+            #       webhook_configs = [
+            #         {
+            #           url = "http://127.0.0.1:9223/";
+            #           max_alerts = 5;
+            #         }
+            #       ];
+            #     }
+            #     {
+            #       name = "numtide";
+            #       webhook_configs = [
+            #         # TODO
+            #         #{
+            #         #  send_resolved = true;
+            #         #  url = "https://chat.ntd.one/plugins/alertmanager/api/webhook?token='xxxxxxxxxxxxxxxxxxx-yyyyyyy'";
+            #         #}
+            #       ];
+            #     }
+            #     {
+            #       name = "nixos-wiki";
+            #       webhook_configs = [
+            #         {
+            #           url = "http://localhost:9088/alert";
+            #           max_alerts = 5;
+            #         }
+            #       ];
+            #     }
+            #     {
+            #       name = "clan-lol";
+            #       webhook_configs = [
+            #         # TODO
+            #         #{
+            #         #  url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
+            #         #  max_alerts = 5;
+            #         #}
+            #       ];
+            #     }
+            #     {
+            #       name = "dave";
+            #       telegram_configs = [
+            #         {
+            #           chat_id = 42927997;
+            #           bot_token = "$TELEGRAM_BOT_TOKEN";
+            #         }
+            #       ];
+            #     }
+            #     {
+            #       name = "all";
+            #       # pushover_configs = [
+            #       #   {
+            #       #     user_key = "$PUSHOVER_USER_KEY";
+            #       #     token = "$PUSHOVER_TOKEN";
+            #       #     priority = "0";
+            #       #   }
+            #       # ];
+            #     }
+            #     { name = "default"; }
+            #   ];
+            # };
+          };
+
+        };
+    };
+}