Add favicon

clanServices/monitoring/alert-rules.nix

@@ -0,0 +1,24 @@
+{ lib }:
+lib.mapAttrsToList
+  (name: opts: {
+    alert = name;
+    expr = opts.condition;
+    for = opts.time or "2m";
+    labels = { };
+    annotations.description = opts.description;
+  })
+  {
+
+    # TODO Remove this alert, just for testing
+    "Filesystem > = 10%" = {
+      condition = ''disk_used_percent{fstype!~"tmpfs|vfat|devtmpfs|efivarfs"} > 10'';
+      time = "1m";
+      description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 90% space left on its filesystem.";
+    };
+
+    filesystem_full_80percent = {
+      condition = ''disk_used_percent{fstype!~"tmpfs|vfat|devtmpfs|efivarfs"} > 80'';
+      time = "1m";
+      description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 20% space left on its filesystem.";
+    };
+  }

clanServices/monitoring/default.nix

@@ -24,5 +24,48 @@
       };
   };
 
-  imports = [ ./telegraf.nix ];
+  roles.prometheus = {
+    description = "Prometheus monitoring daemon. Will collect metrics from all hosts with the telegraf role";
+    interface =
+      { lib, ... }:
+      {
+        options.webExternalUrl = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          example = "https://prometheus.tld";
+          description = "The URL under which Prometheus is externally reachable";
+        };
+      };
+  };
+
+  imports = [
+    ./telegraf.nix
+    ./prometheus.nix
+  ];
+
+  perMachine.nixosModule =
+    { pkgs, ... }:
+    {
+      clan.core.vars.generators."prometheus" = {
+
+        share = true;
+
+        files.password.restartUnits = [
+          "telegraf.service"
+          "prometheus.service"
+        ];
+
+        files.password-env.restartUnits = [ "telegraf.service" ];
+
+        runtimeInputs = [
+          pkgs.coreutils
+          pkgs.xkcdpass
+        ];
+
+        script = ''
+          xkcdpass --numwords 6 --delimiter - --count 1 | tr -d "\n" > $out/password
+          printf 'BASIC_AUTH_PWD=%s\n' "$(cat $out/password)" > $out/password-env
+        '';
+      };
+    };
 }

clanServices/monitoring/prometheus-consoles/favicon.svg

@@ -0,0 +1,11 @@
+<svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.dev/svgjs" width="1000" height="1000"><g clip-path="url(#SvgjsClipPath1007)"><rect width="1000" height="1000" fill="#ffffff"></rect><g transform="matrix(5.132341080724394,0,0,5.132341080724394,217.38764012391061,149.97935090550055)"><svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.dev/svgjs" width="110.13" height="136.39"><svg id="Layer_1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 110.13 136.39">
+  <defs>
+    <style>
+      .cls-1 {
+        fill: #231f20;
+      }
+    </style>
+  <clipPath id="SvgjsClipPath1007"><rect width="1000" height="1000" x="0" y="0" rx="350" ry="350"></rect></clipPath></defs>
+  <path class="cls-1" d="M88.27,30.81h16.69c1.77,0,3.21-1.44,3.21-3.21v-12.84c0-1.77-1.44-3.21-3.21-3.21h-5.26c-1.7,0-3.08-1.38-3.08-3.08V3.21c0-1.77-1.44-3.21-3.21-3.21h-47.49c-1.77,0-3.21,1.44-3.21,3.21v5.26c0,1.7-1.38,3.08-3.08,3.08h-5.26c-1.77,0-3.21,1.44-3.21,3.21v5.26c0,1.7-1.38,3.08-3.08,3.08h-5.26c-1.77,0-3.21,1.44-3.21,3.21,0,0-.77-1.95-.77,34.47,0,32.56.77,29.7.77,29.7,0,1.77,1.44,3.21,3.21,3.21h5.26c1.7,0,3.08,1.38,3.08,3.08v5.39c0,1.7,1.38,3.08,3.08,3.08h5.39c1.7,0,3.08,1.38,3.08,3.08v5.26c0,1.77,1.44,3.21,3.21,3.21h46.21c1.77,0,3.21-1.44,3.21-3.21v-5.26c0-1.7,1.38-3.08,3.08-3.08h8.5c1.77,0,3.21-1.44,3.21-3.21v-15.3c0-1.77-1.44-3.21-3.21-3.21h-19.93c-1.77,0-3.21,1.44-3.21,3.21v7.73c0,1.7-1.38,3.08-3.08,3.08h-23.36c-1.7,0-3.08-1.38-3.08-3.08v-7.83c0-1.77-1.44-3.21-3.21-3.21h-7.83c-1.7,0-2.66.25-3.08-3.08-.13-1.07-.2-2.38-.3-4.13-.25-4.41-.47-2.64-.47-15.89,0-18.52.48-23.85.77-26.42s1.38-3.08,3.08-3.08h7.83c1.77,0,3.21-1.44,3.21-3.21v-5.26c0-1.7,1.38-3.08,3.08-3.08h24.65c1.7,0,3.08,1.38,3.08,3.08v5.26c0,1.77,1.44,3.21,3.21,3.21Z"></path>
+  <path class="cls-1" d="M28.49,113.03h-3.79c-.74,0-1.33-.6-1.33-1.33v-3.79c0-1.47-1.19-2.67-2.67-2.67h-10.24c-1.47,0-2.67,1.19-2.67,2.67v3.79c0,.74-.6,1.33-1.33,1.33h-3.79c-1.47,0-2.67,1.19-2.67,2.67v10.24c0,1.47,1.19,2.67,2.67,2.67h3.79c.74,0,1.33.6,1.33,1.33v3.79c0,1.47,1.19,2.67,2.67,2.67h10.24c1.47,0,2.67-1.19,2.67-2.67v-3.79c0-.74.6-1.33,1.33-1.33h3.79c1.47,0,2.67-1.19,2.67-2.67v-10.24c0-1.47-1.19-2.67-2.67-2.67Z"></path>
+</svg></svg></g></g></svg>

clanServices/monitoring/prometheus-consoles/index.html

@@ -0,0 +1,249 @@
+<!DOCTYPE html> <html>
+<head>
+  <meta charset="utf-8">
+  <title>Clan status</title>
+  <link rel="icon" type="image/png" href="favicon-48x48.png" sizes="48x48" />
+  <link rel="icon" type="image/svg+xml" href="favicon.svg" />
+  <link rel="shortcut icon" href="favicon.ico" />
+  <link rel="apple-touch-icon" sizes="180x180" href="apple-touch-icon.png" />
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <style>
+ :root {
+  --dark: rgb(22, 35, 36);
+  --light: rgb(229, 231, 235);
+}
+
+body {
+  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+  margin: 0;
+  padding: 20px;
+  background: var(--dark);
+}
+.container {
+  max-width: 1400px;
+  margin: 0 auto;
+  background: var(--light);
+  padding: 30px;
+  border-radius: 8px;
+  box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+}
+h1 {
+  margin-top: 0;
+  color: #333;
+  border-bottom: 2px solid var(--dark);
+  padding-bottom: 10px;
+}
+h2 {
+  color: #555;
+  margin-top: 30px;
+}
+table {
+  width: 100%;
+  border-collapse: collapse;
+  margin: 20px 0;
+}
+th {
+  background: var(--dark);
+  color: var(--light);
+  padding: 12px;
+  text-align: left;
+  font-weight: 600;
+}
+td {
+  padding: 10px 12px;
+  border-bottom: 1px solid #ddd;
+}
+tr:hover {
+  background: var(--light);
+}
+.status-up {
+  color: #28a745;
+  font-weight: bold;
+}
+.status-down {
+  color: #dc3545;
+  font-weight: bold;
+}
+.alert-success {
+  background: #d4edda;
+  color: #155724;
+  padding: 12px;
+  border-radius: 4px;
+  border: 1px solid #c3e6cb;
+}
+.grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
+  gap: 20px;
+  margin: 20px 0;
+}
+.card {
+  border: 1px solid #ddd;
+  border-radius: 4px;
+  padding: 15px;
+}
+.metric-value {
+  font-size: 1.2em;
+  font-weight: bold;
+  color: var(--dark);
+}
+
+  </style>
+
+</head>
+<body>
+  <div class="container">
+    <h1>Clan Status</h1>
+
+
+    <h2>Instances</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Host</th>
+          <th>Status</th>
+          <th>CPU Usage</th>
+          <th>Memory Usage</th>
+          <th>Disk Usage</th>
+        </tr>
+      </thead>
+      <tbody>
+        {{ range query "up" | sortByLabel "instance" }}
+        {{ $hostname := reReplaceAll "\\..*" "" .Labels.instance }}
+        <tr>
+          <td>{{ $hostname }}</td>
+          <td>
+            {{ if eq .Value 1.0 }}
+              <span class="status-up">UP</span>
+            {{ else }}
+              <span class="status-down">DOWN</span>
+            {{ end }}
+          </td>
+          <td>
+            {{ $cpuQuery := query (printf "100 - cpu_usage_idle{cpu=\"cpu-total\",host=\"%s\"}" $hostname) }}
+            {{ if $cpuQuery }}
+              {{ with $cpuQuery | first }}
+                <span class="metric-value">{{ . | value | printf "%.1f" }}%</span>
+              {{ end }}
+            {{ else }}
+              N/A
+            {{ end }}
+          </td>
+          <td>
+            {{ $memQuery := query (printf "(1 - (mem_available{host=\"%s\"} / mem_total{host=\"%s\"})) * 100" $hostname $hostname) }}
+            {{ if $memQuery }}
+              {{ with $memQuery | first }}
+                <span class="metric-value">{{ . | value | printf "%.1f" }}%</span>
+              {{ end }}
+            {{ else }}
+              N/A
+            {{ end }}
+          </td>
+          <td>
+            {{ $diskQuery := query (printf "(1 - (disk_free{host=\"%s\",path=\"/\"} / disk_total{host=\"%s\",path=\"/\"})) * 100" $hostname $hostname) }}
+            {{ if $diskQuery }}
+              {{ with $diskQuery | first }}
+                <span class="metric-value">{{ . | value | printf "%.1f" }}%</span>
+              {{ end }}
+            {{ else }}
+              N/A
+            {{ end }}
+          </td>
+        </tr>
+        {{ end }}
+      </tbody>
+    </table>
+
+    <h2>Services</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Service</th>
+          <th>Host</th>
+          <th>State</th>
+        </tr>
+      </thead>
+      <tbody>
+		  <!--     <tr> -->
+		  <!--       <td>Vaultwarden</td> -->
+		  <!--       <td>kiwi</td> -->
+		  <!--       <td> -->
+		  <!--           <span class="status-up">UP</span> -->
+		  <!-- </td> -->
+		  <!--     </tr> -->
+      </tbody>
+    </table>
+
+    <!-- <h2>NixOS Systems</h2> -->
+    <!-- <table> -->
+    <!--   <thead> -->
+    <!--     <tr> -->
+    <!--       <th>Host</th> -->
+    <!--       <th>Booted System</th> -->
+    <!--       <th>Current System</th> -->
+    <!--       <th>Booted Kernel</th> -->
+    <!--       <th>Current Kernel</th> -->
+    <!--     </tr> -->
+    <!--   </thead> -->
+    <!--   <tbody> -->
+    <!--     {{ range query "nixos_systems_present" | sortByLabel "host" }} -->
+    <!--     <tr> -->
+    <!--       <td>{{ .Labels.host }}</td> -->
+    <!--       <td style="font-family: monospace; font-size: 0.85em;">{{ .Labels.booted_system }}</td> -->
+    <!--       <td style="font-family: monospace; font-size: 0.85em;">{{ .Labels.current_system }}</td> -->
+    <!--       <td>{{ .Labels.booted_kernel }}</td> -->
+    <!--       <td>{{ .Labels.current_kernel }}</td> -->
+    <!--     </tr> -->
+    <!--     {{ end }} -->
+    <!--   </tbody> -->
+    <!-- </table> -->
+
+    <h2>Failed Systemd Units</h2>
+    {{ $failedUnits := query "systemd_units_sub_code{sub=\"failed\"}" }}
+    {{ if $failedUnits }}
+    <table>
+      <thead>
+        <tr>
+          <th>Host</th>
+          <th>Unit</th>
+        </tr>
+      </thead>
+      <tbody>
+        {{ range $failedUnits | sortByLabel "host" }}
+        <tr>
+          <td>{{ .Labels.host }}</td>
+          <td style="color: #dc3545;">{{ .Labels.name }}</td>
+        </tr>
+        {{ end }}
+      </tbody>
+    </table>
+    {{ else }}
+    <div class="alert-success">No failed systemd units</div>
+    {{ end }}
+
+    <h2>Active Alerts</h2>
+    {{ with query "ALERTS{alertstate=\"firing\"}" }}
+    <table>
+      <thead>
+        <tr>
+          <th>Host</th>
+          <th>Alert</th>
+          <th>Value</th>
+        </tr>
+      </thead>
+      <tbody>
+        {{ range . }}
+        <tr>
+          <td>{{ or .Labels.host .Labels.instance }}</td>
+          <td>{{ .Labels.alertname }}</td>
+          <td>{{ .Value }}</td>
+        </tr>
+        {{ end }}
+      </tbody>
+    </table>
+    {{ else }}
+    <div class="alert-success">No active alerts</div>
+    {{ end }}
+  </div>
+</body>
+</html>

clanServices/monitoring/prometheus-consoles/style.css

@@ -0,0 +1,80 @@
+ :root {
+  --dark: rgb(22, 35, 36);
+  --light: rgb(229, 231, 235);
+}
+
+body {
+  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+  margin: 0;
+  padding: 20px;
+  background: var(--dark);
+}
+.container {
+  max-width: 1400px;
+  margin: 0 auto;
+  background: var(--light);
+  padding: 30px;
+  border-radius: 8px;
+  box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+}
+h1 {
+  margin-top: 0;
+  color: #333;
+  border-bottom: 2px solid var(--dark);
+  padding-bottom: 10px;
+}
+h2 {
+  color: #555;
+  margin-top: 30px;
+}
+table {
+  width: 100%;
+  border-collapse: collapse;
+  margin: 20px 0;
+}
+th {
+  background: var(--dark);
+  color: var(--light);
+  padding: 12px;
+  text-align: left;
+  font-weight: 600;
+}
+td {
+  padding: 10px 12px;
+  border-bottom: 1px solid #ddd;
+}
+tr:hover {
+  background: var(--light);
+}
+.status-up {
+  color: #28a745;
+  font-weight: bold;
+}
+.status-down {
+  color: #dc3545;
+  font-weight: bold;
+}
+.alert-success {
+  background: #d4edda;
+  color: #155724;
+  padding: 12px;
+  border-radius: 4px;
+  border: 1px solid #c3e6cb;
+}
+.grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
+  gap: 20px;
+  margin: 20px 0;
+}
+.card {
+  border: 1px solid #ddd;
+  border-radius: 4px;
+  padding: 15px;
+}
+.metric-value {
+  font-size: 1.2em;
+  font-weight: bold;
+  color: var(--dark);
+}
+

clanServices/monitoring/prometheus.nix

@@ -0,0 +1,83 @@
+{
+  roles.prometheus.perInstance =
+    {
+      settings,
+      instanceName,
+      roles,
+      ...
+    }:
+    {
+
+      nixosModule =
+        {
+          config,
+          lib,
+          pkgs,
+          ...
+        }:
+        {
+
+          systemd.services.prometheus = {
+            serviceConfig = {
+              LoadCredential = "password:${config.clan.core.vars.generators.prometheus.files.password.path}";
+              BindReadOnlyPaths = "%d/password:/etc/prometheus/password";
+            };
+          };
+
+          services.prometheus = {
+
+            enable = true;
+
+            # TODO what do we set here? do we even need something?
+            # TODO this should be a export
+            # "https://prometheus.${config.clan.core.settings.tld}";
+            webExternalUrl = settings.webExternalUrl;
+
+            # Configure console templates and libraries paths
+            extraFlags = [
+              "--storage.tsdb.retention.time=30d"
+              "--web.console.templates=${./prometheus-consoles}"
+              "--web.console.libraries=${./prometheus-consoles}"
+            ];
+
+            ruleFiles = [
+              (pkgs.writeText "prometheus-rules.yml" (
+                builtins.toJSON {
+                  groups = [
+                    {
+                      name = "alerting-rules";
+                      rules = import ./alert-rules.nix { inherit lib; };
+                    }
+                  ];
+                }
+              ))
+            ];
+
+            scrapeConfigs = [
+              {
+                job_name = "telegraf";
+                scrape_interval = "60s";
+                metrics_path = "/metrics";
+                basic_auth.username = "prometheus";
+                basic_auth.password_file = "/etc/prometheus/password";
+
+                static_configs = [
+                  {
+                    # Scrape all machines with the `telegraf` role
+                    # https://prometheus:<password>@<host>.<tld>:9273/metrics
+
+                    # scheme = "https";
+                    # scheme = "http";
+
+                    targets = map (m: "${m}.${config.clan.core.settings.tld}:9273") (
+                      lib.attrNames roles.telegraf.machines
+                    );
+                    labels.type = instanceName;
+                  }
+                ];
+              }
+            ];
+          };
+        };
+    };
+}

clanServices/monitoring/telegraf.nix

@@ -1,128 +1,32 @@
 {
   roles.telegraf.perInstance =
-    { settings, ... }:
+    { ... }:
     {
 
       nixosModule =
         {
           config,
           pkgs,
-          lib,
           ...
         }:
-        let
-          auth_user = "prometheus";
-        in
         {
-          warnings =
-            lib.optionals (settings.allowAllInterfaces != null) [
-              "monitoring.settings.allowAllInterfaces is deprecated and and has no effect. Please remove it from your inventory."
-              "The monitoring service will now always listen on all interfaces over https."
-            ]
-            ++ (lib.optionals (settings.interfaces != null) [
-              "monitoring.settings.interfaces is deprecated and and has no effect. Please remove it from your inventory."
-              "The monitoring service will now always listen on all interfaces over https."
-            ]);
 
-          networking.firewall.allowedTCPPorts = [
+          networking.firewall.allowedTCPPorts = [ 9273 ];
-            9273
-            9990
-          ];
-
-          clan.core.vars.generators."telegraf-certs" = {
-            files.crt = {
-              restartUnits = [ "telegraf.service" ];
-              deploy = true;
-              secret = false;
-            };
-            files.key = {
-              mode = "0600";
-              restartUnits = [ "telegraf.service" ];
-            };
-
-            runtimeInputs = [
-              pkgs.openssl
-            ];
-
-            # TODO: Implement automated certificate rotation instead of using a 100-year expiration
-            script = ''
-              openssl req -x509 -nodes -newkey rsa:4096 \
-                -days 36500 \
-                -keyout "$out"/key \
-                -out "$out"/crt \
-                -subj "/C=US/ST=CA/L=San Francisco/O=Example Corp/OU=IT/CN=example.com"
-            '';
-          };
-
-          clan.core.vars.generators."telegraf" = {
-            files.password.restartUnits = [ "telegraf.service" ];
-            files.password-env.restartUnits = [ "telegraf.service" ];
-            files.miniserve-auth.restartUnits = [ "telegraf.service" ];
-
-            dependencies = [ "telegraf-certs" ];
-
-            runtimeInputs = [
-              pkgs.coreutils
-              pkgs.xkcdpass
-              pkgs.mkpasswd
-            ];
-
-            script = ''
-              PASSWORD=$(xkcdpass --numwords 4 --delimiter - --count 1 | tr -d "\n")
-              echo "BASIC_AUTH_PWD=$PASSWORD" > "$out"/password-env
-              echo "${auth_user}:$PASSWORD" > "$out"/miniserve-auth
-              echo "$PASSWORD" | tr -d "\n" > "$out"/password
-            '';
-          };
-
-          systemd.services.telegraf-json = {
-            enable = true;
-            wantedBy = [ "multi-user.target" ];
-            after = [ "telegraf.service" ];
-            requires = [ "telegraf.service" ];
-            serviceConfig = {
-              LoadCredential = [
-                "auth_file_path:${config.clan.core.vars.generators.telegraf.files.miniserve-auth.path}"
-                "telegraf_crt_path:${config.clan.core.vars.generators.telegraf-certs.files.crt.path}"
-                "telegraf_key_path:${config.clan.core.vars.generators.telegraf-certs.files.key.path}"
-              ];
-              Environment = [
-                "AUTH_FILE_PATH=%d/auth_file_path"
-                "CRT_PATH=%d/telegraf_crt_path"
-                "KEY_PATH=%d/telegraf_key_path"
-              ];
-              Restart = "on-failure";
-              User = "telegraf";
-              Group = "telegraf";
-              RuntimeDirectory = "telegraf-www";
-            };
-            script = "${pkgs.miniserve}/bin/miniserve -p 9990 /run/telegraf-www --auth-file \"$AUTH_FILE_PATH\" --tls-cert \"$CRT_PATH\" --tls-key \"$KEY_PATH\"";
-          };
-
-          systemd.services.telegraf = {
-            serviceConfig = {
-              LoadCredential = [
-                "telegraf_crt_path:${config.clan.core.vars.generators.telegraf-certs.files.crt.path}"
-                "telegraf_key_path:${config.clan.core.vars.generators.telegraf-certs.files.key.path}"
-              ];
-              Environment = [
-                "CRT_PATH=%d/telegraf_crt_path"
-                "KEY_PATH=%d/telegraf_key_path"
-              ];
-            };
-          };
 
           services.telegraf = {
             enable = true;
-            environmentFiles = [
+            environmentFiles = [ config.clan.core.vars.generators.prometheus.files.password-env.path ];
-              (builtins.toString config.clan.core.vars.generators.telegraf.files.password-env.path)
-            ];
 
             extraConfig = {
               agent.interval = "60s";
               inputs = {
 
+                # More input plugins available at:
+                # https://github.com/influxdata/telegraf/tree/master/plugins/inputs
                 diskio = { };
+                disk = { };
+                cpu = { };
+                processes = { };
                 kernel_vmstat = { };
                 system = { };
                 mem = { };
@@ -147,20 +51,12 @@
                     }
                   ];
               };
-              # sadly there doesn'T seem to exist a telegraf http_client output plugin
+              # sadly there doesn't seem to exist a telegraf http_client output plugin
               outputs.prometheus_client = {
                 listen = ":9273";
                 metric_version = 2;
-                basic_username = "${auth_user}";
+                basic_username = "prometheus";
                 basic_password = "$${BASIC_AUTH_PWD}";
-                tls_cert = "$${CRT_PATH}";
-                tls_key = "$${KEY_PATH}";
-              };
-
-              outputs.file = {
-                files = [ "/run/telegraf-www/telegraf.json" ];
-                data_format = "json";
-                json_timestamp_units = "1s";
               };
             };
           };