Compare commits
4 Commits
6a6a371256
...
monitoring
| Author | SHA1 | Date | |
|---|---|---|---|
|
bdaff0a8a4 | ||
|
|
fabbfcaab6 | ||
|
|
98cfaac849 | ||
|
|
decb91a529 |
24
clanServices/monitoring/alert-rules.nix
Normal file
24
clanServices/monitoring/alert-rules.nix
Normal file
@@ -0,0 +1,24 @@
|
||||
{ lib }:
|
||||
lib.mapAttrsToList
|
||||
(name: opts: {
|
||||
alert = name;
|
||||
expr = opts.condition;
|
||||
for = opts.time or "2m";
|
||||
labels = { };
|
||||
annotations.description = opts.description;
|
||||
})
|
||||
{
|
||||
|
||||
# TODO Remove this alert, just for testing
|
||||
"Filesystem > = 10%" = {
|
||||
condition = ''disk_used_percent{fstype!~"tmpfs|vfat|devtmpfs|efivarfs"} > 10'';
|
||||
time = "1m";
|
||||
description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 90% space left on its filesystem.";
|
||||
};
|
||||
|
||||
filesystem_full_80percent = {
|
||||
condition = ''disk_used_percent{fstype!~"tmpfs|vfat|devtmpfs|efivarfs"} > 80'';
|
||||
time = "1m";
|
||||
description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 20% space left on its filesystem.";
|
||||
};
|
||||
}
|
||||
@@ -24,5 +24,48 @@
|
||||
};
|
||||
};
|
||||
|
||||
imports = [ ./telegraf.nix ];
|
||||
roles.prometheus = {
|
||||
description = "Prometheus monitoring daemon. Will collect metrics from all hosts with the telegraf role";
|
||||
interface =
|
||||
{ lib, ... }:
|
||||
{
|
||||
options.webExternalUrl = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "https://prometheus.tld";
|
||||
description = "The URL under which Prometheus is externally reachable";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
imports = [
|
||||
./telegraf.nix
|
||||
./prometheus.nix
|
||||
];
|
||||
|
||||
perMachine.nixosModule =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
clan.core.vars.generators."prometheus" = {
|
||||
|
||||
share = true;
|
||||
|
||||
files.password.restartUnits = [
|
||||
"telegraf.service"
|
||||
"prometheus.service"
|
||||
];
|
||||
|
||||
files.password-env.restartUnits = [ "telegraf.service" ];
|
||||
|
||||
runtimeInputs = [
|
||||
pkgs.coreutils
|
||||
pkgs.xkcdpass
|
||||
];
|
||||
|
||||
script = ''
|
||||
xkcdpass --numwords 6 --delimiter - --count 1 | tr -d "\n" > $out/password
|
||||
printf 'BASIC_AUTH_PWD=%s\n' "$(cat $out/password)" > $out/password-env
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
BIN
clanServices/monitoring/prometheus-consoles/favicon.ico
Normal file
BIN
clanServices/monitoring/prometheus-consoles/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
11
clanServices/monitoring/prometheus-consoles/favicon.svg
Normal file
11
clanServices/monitoring/prometheus-consoles/favicon.svg
Normal file
@@ -0,0 +1,11 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.dev/svgjs" width="1000" height="1000"><g clip-path="url(#SvgjsClipPath1007)"><rect width="1000" height="1000" fill="#ffffff"></rect><g transform="matrix(5.132341080724394,0,0,5.132341080724394,217.38764012391061,149.97935090550055)"><svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.dev/svgjs" width="110.13" height="136.39"><svg id="Layer_1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 110.13 136.39">
|
||||
<defs>
|
||||
<style>
|
||||
.cls-1 {
|
||||
fill: #231f20;
|
||||
}
|
||||
</style>
|
||||
<clipPath id="SvgjsClipPath1007"><rect width="1000" height="1000" x="0" y="0" rx="350" ry="350"></rect></clipPath></defs>
|
||||
<path class="cls-1" d="M88.27,30.81h16.69c1.77,0,3.21-1.44,3.21-3.21v-12.84c0-1.77-1.44-3.21-3.21-3.21h-5.26c-1.7,0-3.08-1.38-3.08-3.08V3.21c0-1.77-1.44-3.21-3.21-3.21h-47.49c-1.77,0-3.21,1.44-3.21,3.21v5.26c0,1.7-1.38,3.08-3.08,3.08h-5.26c-1.77,0-3.21,1.44-3.21,3.21v5.26c0,1.7-1.38,3.08-3.08,3.08h-5.26c-1.77,0-3.21,1.44-3.21,3.21,0,0-.77-1.95-.77,34.47,0,32.56.77,29.7.77,29.7,0,1.77,1.44,3.21,3.21,3.21h5.26c1.7,0,3.08,1.38,3.08,3.08v5.39c0,1.7,1.38,3.08,3.08,3.08h5.39c1.7,0,3.08,1.38,3.08,3.08v5.26c0,1.77,1.44,3.21,3.21,3.21h46.21c1.77,0,3.21-1.44,3.21-3.21v-5.26c0-1.7,1.38-3.08,3.08-3.08h8.5c1.77,0,3.21-1.44,3.21-3.21v-15.3c0-1.77-1.44-3.21-3.21-3.21h-19.93c-1.77,0-3.21,1.44-3.21,3.21v7.73c0,1.7-1.38,3.08-3.08,3.08h-23.36c-1.7,0-3.08-1.38-3.08-3.08v-7.83c0-1.77-1.44-3.21-3.21-3.21h-7.83c-1.7,0-2.66.25-3.08-3.08-.13-1.07-.2-2.38-.3-4.13-.25-4.41-.47-2.64-.47-15.89,0-18.52.48-23.85.77-26.42s1.38-3.08,3.08-3.08h7.83c1.77,0,3.21-1.44,3.21-3.21v-5.26c0-1.7,1.38-3.08,3.08-3.08h24.65c1.7,0,3.08,1.38,3.08,3.08v5.26c0,1.77,1.44,3.21,3.21,3.21Z"></path>
|
||||
<path class="cls-1" d="M28.49,113.03h-3.79c-.74,0-1.33-.6-1.33-1.33v-3.79c0-1.47-1.19-2.67-2.67-2.67h-10.24c-1.47,0-2.67,1.19-2.67,2.67v3.79c0,.74-.6,1.33-1.33,1.33h-3.79c-1.47,0-2.67,1.19-2.67,2.67v10.24c0,1.47,1.19,2.67,2.67,2.67h3.79c.74,0,1.33.6,1.33,1.33v3.79c0,1.47,1.19,2.67,2.67,2.67h10.24c1.47,0,2.67-1.19,2.67-2.67v-3.79c0-.74.6-1.33,1.33-1.33h3.79c1.47,0,2.67-1.19,2.67-2.67v-10.24c0-1.47-1.19-2.67-2.67-2.67Z"></path>
|
||||
</svg></svg></g></g></svg>
|
||||
|
After Width: | Height: | Size: 2.3 KiB |
249
clanServices/monitoring/prometheus-consoles/index.html
Normal file
249
clanServices/monitoring/prometheus-consoles/index.html
Normal file
@@ -0,0 +1,249 @@
|
||||
<!DOCTYPE html> <html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Clan status</title>
|
||||
<link rel="icon" type="image/png" href="favicon-48x48.png" sizes="48x48" />
|
||||
<link rel="icon" type="image/svg+xml" href="favicon.svg" />
|
||||
<link rel="shortcut icon" href="favicon.ico" />
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="apple-touch-icon.png" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<style>
|
||||
:root {
|
||||
--dark: rgb(22, 35, 36);
|
||||
--light: rgb(229, 231, 235);
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
background: var(--dark);
|
||||
}
|
||||
.container {
|
||||
max-width: 1400px;
|
||||
margin: 0 auto;
|
||||
background: var(--light);
|
||||
padding: 30px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
||||
}
|
||||
h1 {
|
||||
margin-top: 0;
|
||||
color: #333;
|
||||
border-bottom: 2px solid var(--dark);
|
||||
padding-bottom: 10px;
|
||||
}
|
||||
h2 {
|
||||
color: #555;
|
||||
margin-top: 30px;
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin: 20px 0;
|
||||
}
|
||||
th {
|
||||
background: var(--dark);
|
||||
color: var(--light);
|
||||
padding: 12px;
|
||||
text-align: left;
|
||||
font-weight: 600;
|
||||
}
|
||||
td {
|
||||
padding: 10px 12px;
|
||||
border-bottom: 1px solid #ddd;
|
||||
}
|
||||
tr:hover {
|
||||
background: var(--light);
|
||||
}
|
||||
.status-up {
|
||||
color: #28a745;
|
||||
font-weight: bold;
|
||||
}
|
||||
.status-down {
|
||||
color: #dc3545;
|
||||
font-weight: bold;
|
||||
}
|
||||
.alert-success {
|
||||
background: #d4edda;
|
||||
color: #155724;
|
||||
padding: 12px;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #c3e6cb;
|
||||
}
|
||||
.grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
|
||||
gap: 20px;
|
||||
margin: 20px 0;
|
||||
}
|
||||
.card {
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
padding: 15px;
|
||||
}
|
||||
.metric-value {
|
||||
font-size: 1.2em;
|
||||
font-weight: bold;
|
||||
color: var(--dark);
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>Clan Status</h1>
|
||||
|
||||
|
||||
<h2>Instances</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Host</th>
|
||||
<th>Status</th>
|
||||
<th>CPU Usage</th>
|
||||
<th>Memory Usage</th>
|
||||
<th>Disk Usage</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{ range query "up" | sortByLabel "instance" }}
|
||||
{{ $hostname := reReplaceAll "\\..*" "" .Labels.instance }}
|
||||
<tr>
|
||||
<td>{{ $hostname }}</td>
|
||||
<td>
|
||||
{{ if eq .Value 1.0 }}
|
||||
<span class="status-up">UP</span>
|
||||
{{ else }}
|
||||
<span class="status-down">DOWN</span>
|
||||
{{ end }}
|
||||
</td>
|
||||
<td>
|
||||
{{ $cpuQuery := query (printf "100 - cpu_usage_idle{cpu=\"cpu-total\",host=\"%s\"}" $hostname) }}
|
||||
{{ if $cpuQuery }}
|
||||
{{ with $cpuQuery | first }}
|
||||
<span class="metric-value">{{ . | value | printf "%.1f" }}%</span>
|
||||
{{ end }}
|
||||
{{ else }}
|
||||
N/A
|
||||
{{ end }}
|
||||
</td>
|
||||
<td>
|
||||
{{ $memQuery := query (printf "(1 - (mem_available{host=\"%s\"} / mem_total{host=\"%s\"})) * 100" $hostname $hostname) }}
|
||||
{{ if $memQuery }}
|
||||
{{ with $memQuery | first }}
|
||||
<span class="metric-value">{{ . | value | printf "%.1f" }}%</span>
|
||||
{{ end }}
|
||||
{{ else }}
|
||||
N/A
|
||||
{{ end }}
|
||||
</td>
|
||||
<td>
|
||||
{{ $diskQuery := query (printf "(1 - (disk_free{host=\"%s\",path=\"/\"} / disk_total{host=\"%s\",path=\"/\"})) * 100" $hostname $hostname) }}
|
||||
{{ if $diskQuery }}
|
||||
{{ with $diskQuery | first }}
|
||||
<span class="metric-value">{{ . | value | printf "%.1f" }}%</span>
|
||||
{{ end }}
|
||||
{{ else }}
|
||||
N/A
|
||||
{{ end }}
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>Services</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Service</th>
|
||||
<th>Host</th>
|
||||
<th>State</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<!-- <tr> -->
|
||||
<!-- <td>Vaultwarden</td> -->
|
||||
<!-- <td>kiwi</td> -->
|
||||
<!-- <td> -->
|
||||
<!-- <span class="status-up">UP</span> -->
|
||||
<!-- </td> -->
|
||||
<!-- </tr> -->
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<!-- <h2>NixOS Systems</h2> -->
|
||||
<!-- <table> -->
|
||||
<!-- <thead> -->
|
||||
<!-- <tr> -->
|
||||
<!-- <th>Host</th> -->
|
||||
<!-- <th>Booted System</th> -->
|
||||
<!-- <th>Current System</th> -->
|
||||
<!-- <th>Booted Kernel</th> -->
|
||||
<!-- <th>Current Kernel</th> -->
|
||||
<!-- </tr> -->
|
||||
<!-- </thead> -->
|
||||
<!-- <tbody> -->
|
||||
<!-- {{ range query "nixos_systems_present" | sortByLabel "host" }} -->
|
||||
<!-- <tr> -->
|
||||
<!-- <td>{{ .Labels.host }}</td> -->
|
||||
<!-- <td style="font-family: monospace; font-size: 0.85em;">{{ .Labels.booted_system }}</td> -->
|
||||
<!-- <td style="font-family: monospace; font-size: 0.85em;">{{ .Labels.current_system }}</td> -->
|
||||
<!-- <td>{{ .Labels.booted_kernel }}</td> -->
|
||||
<!-- <td>{{ .Labels.current_kernel }}</td> -->
|
||||
<!-- </tr> -->
|
||||
<!-- {{ end }} -->
|
||||
<!-- </tbody> -->
|
||||
<!-- </table> -->
|
||||
|
||||
<h2>Failed Systemd Units</h2>
|
||||
{{ $failedUnits := query "systemd_units_sub_code{sub=\"failed\"}" }}
|
||||
{{ if $failedUnits }}
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Host</th>
|
||||
<th>Unit</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{ range $failedUnits | sortByLabel "host" }}
|
||||
<tr>
|
||||
<td>{{ .Labels.host }}</td>
|
||||
<td style="color: #dc3545;">{{ .Labels.name }}</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
</tbody>
|
||||
</table>
|
||||
{{ else }}
|
||||
<div class="alert-success">No failed systemd units</div>
|
||||
{{ end }}
|
||||
|
||||
<h2>Active Alerts</h2>
|
||||
{{ with query "ALERTS{alertstate=\"firing\"}" }}
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Host</th>
|
||||
<th>Alert</th>
|
||||
<th>Value</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{ range . }}
|
||||
<tr>
|
||||
<td>{{ or .Labels.host .Labels.instance }}</td>
|
||||
<td>{{ .Labels.alertname }}</td>
|
||||
<td>{{ .Value }}</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
</tbody>
|
||||
</table>
|
||||
{{ else }}
|
||||
<div class="alert-success">No active alerts</div>
|
||||
{{ end }}
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
80
clanServices/monitoring/prometheus-consoles/style.css
Normal file
80
clanServices/monitoring/prometheus-consoles/style.css
Normal file
@@ -0,0 +1,80 @@
|
||||
:root {
|
||||
--dark: rgb(22, 35, 36);
|
||||
--light: rgb(229, 231, 235);
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
background: var(--dark);
|
||||
}
|
||||
.container {
|
||||
max-width: 1400px;
|
||||
margin: 0 auto;
|
||||
background: var(--light);
|
||||
padding: 30px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
||||
}
|
||||
h1 {
|
||||
margin-top: 0;
|
||||
color: #333;
|
||||
border-bottom: 2px solid var(--dark);
|
||||
padding-bottom: 10px;
|
||||
}
|
||||
h2 {
|
||||
color: #555;
|
||||
margin-top: 30px;
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin: 20px 0;
|
||||
}
|
||||
th {
|
||||
background: var(--dark);
|
||||
color: var(--light);
|
||||
padding: 12px;
|
||||
text-align: left;
|
||||
font-weight: 600;
|
||||
}
|
||||
td {
|
||||
padding: 10px 12px;
|
||||
border-bottom: 1px solid #ddd;
|
||||
}
|
||||
tr:hover {
|
||||
background: var(--light);
|
||||
}
|
||||
.status-up {
|
||||
color: #28a745;
|
||||
font-weight: bold;
|
||||
}
|
||||
.status-down {
|
||||
color: #dc3545;
|
||||
font-weight: bold;
|
||||
}
|
||||
.alert-success {
|
||||
background: #d4edda;
|
||||
color: #155724;
|
||||
padding: 12px;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #c3e6cb;
|
||||
}
|
||||
.grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
|
||||
gap: 20px;
|
||||
margin: 20px 0;
|
||||
}
|
||||
.card {
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
padding: 15px;
|
||||
}
|
||||
.metric-value {
|
||||
font-size: 1.2em;
|
||||
font-weight: bold;
|
||||
color: var(--dark);
|
||||
}
|
||||
|
||||
83
clanServices/monitoring/prometheus.nix
Normal file
83
clanServices/monitoring/prometheus.nix
Normal file
@@ -0,0 +1,83 @@
|
||||
{
|
||||
roles.prometheus.perInstance =
|
||||
{
|
||||
settings,
|
||||
instanceName,
|
||||
roles,
|
||||
...
|
||||
}:
|
||||
{
|
||||
|
||||
nixosModule =
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
|
||||
systemd.services.prometheus = {
|
||||
serviceConfig = {
|
||||
LoadCredential = "password:${config.clan.core.vars.generators.prometheus.files.password.path}";
|
||||
BindReadOnlyPaths = "%d/password:/etc/prometheus/password";
|
||||
};
|
||||
};
|
||||
|
||||
services.prometheus = {
|
||||
|
||||
enable = true;
|
||||
|
||||
# TODO what do we set here? do we even need something?
|
||||
# TODO this should be a export
|
||||
# "https://prometheus.${config.clan.core.settings.tld}";
|
||||
webExternalUrl = settings.webExternalUrl;
|
||||
|
||||
# Configure console templates and libraries paths
|
||||
extraFlags = [
|
||||
"--storage.tsdb.retention.time=30d"
|
||||
"--web.console.templates=${./prometheus-consoles}"
|
||||
"--web.console.libraries=${./prometheus-consoles}"
|
||||
];
|
||||
|
||||
ruleFiles = [
|
||||
(pkgs.writeText "prometheus-rules.yml" (
|
||||
builtins.toJSON {
|
||||
groups = [
|
||||
{
|
||||
name = "alerting-rules";
|
||||
rules = import ./alert-rules.nix { inherit lib; };
|
||||
}
|
||||
];
|
||||
}
|
||||
))
|
||||
];
|
||||
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "telegraf";
|
||||
scrape_interval = "60s";
|
||||
metrics_path = "/metrics";
|
||||
basic_auth.username = "prometheus";
|
||||
basic_auth.password_file = "/etc/prometheus/password";
|
||||
|
||||
static_configs = [
|
||||
{
|
||||
# Scrape all machines with the `telegraf` role
|
||||
# https://prometheus:<password>@<host>.<tld>:9273/metrics
|
||||
|
||||
# scheme = "https";
|
||||
# scheme = "http";
|
||||
|
||||
targets = map (m: "${m}.${config.clan.core.settings.tld}:9273") (
|
||||
lib.attrNames roles.telegraf.machines
|
||||
);
|
||||
labels.type = instanceName;
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,128 +1,32 @@
|
||||
{
|
||||
roles.telegraf.perInstance =
|
||||
{ settings, ... }:
|
||||
{ ... }:
|
||||
{
|
||||
|
||||
nixosModule =
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
auth_user = "prometheus";
|
||||
in
|
||||
{
|
||||
warnings =
|
||||
lib.optionals (settings.allowAllInterfaces != null) [
|
||||
"monitoring.settings.allowAllInterfaces is deprecated and and has no effect. Please remove it from your inventory."
|
||||
"The monitoring service will now always listen on all interfaces over https."
|
||||
]
|
||||
++ (lib.optionals (settings.interfaces != null) [
|
||||
"monitoring.settings.interfaces is deprecated and and has no effect. Please remove it from your inventory."
|
||||
"The monitoring service will now always listen on all interfaces over https."
|
||||
]);
|
||||
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
9273
|
||||
9990
|
||||
];
|
||||
|
||||
clan.core.vars.generators."telegraf-certs" = {
|
||||
files.crt = {
|
||||
restartUnits = [ "telegraf.service" ];
|
||||
deploy = true;
|
||||
secret = false;
|
||||
};
|
||||
files.key = {
|
||||
mode = "0600";
|
||||
restartUnits = [ "telegraf.service" ];
|
||||
};
|
||||
|
||||
runtimeInputs = [
|
||||
pkgs.openssl
|
||||
];
|
||||
|
||||
# TODO: Implement automated certificate rotation instead of using a 100-year expiration
|
||||
script = ''
|
||||
openssl req -x509 -nodes -newkey rsa:4096 \
|
||||
-days 36500 \
|
||||
-keyout "$out"/key \
|
||||
-out "$out"/crt \
|
||||
-subj "/C=US/ST=CA/L=San Francisco/O=Example Corp/OU=IT/CN=example.com"
|
||||
'';
|
||||
};
|
||||
|
||||
clan.core.vars.generators."telegraf" = {
|
||||
files.password.restartUnits = [ "telegraf.service" ];
|
||||
files.password-env.restartUnits = [ "telegraf.service" ];
|
||||
files.miniserve-auth.restartUnits = [ "telegraf.service" ];
|
||||
|
||||
dependencies = [ "telegraf-certs" ];
|
||||
|
||||
runtimeInputs = [
|
||||
pkgs.coreutils
|
||||
pkgs.xkcdpass
|
||||
pkgs.mkpasswd
|
||||
];
|
||||
|
||||
script = ''
|
||||
PASSWORD=$(xkcdpass --numwords 4 --delimiter - --count 1 | tr -d "\n")
|
||||
echo "BASIC_AUTH_PWD=$PASSWORD" > "$out"/password-env
|
||||
echo "${auth_user}:$PASSWORD" > "$out"/miniserve-auth
|
||||
echo "$PASSWORD" | tr -d "\n" > "$out"/password
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services.telegraf-json = {
|
||||
enable = true;
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "telegraf.service" ];
|
||||
requires = [ "telegraf.service" ];
|
||||
serviceConfig = {
|
||||
LoadCredential = [
|
||||
"auth_file_path:${config.clan.core.vars.generators.telegraf.files.miniserve-auth.path}"
|
||||
"telegraf_crt_path:${config.clan.core.vars.generators.telegraf-certs.files.crt.path}"
|
||||
"telegraf_key_path:${config.clan.core.vars.generators.telegraf-certs.files.key.path}"
|
||||
];
|
||||
Environment = [
|
||||
"AUTH_FILE_PATH=%d/auth_file_path"
|
||||
"CRT_PATH=%d/telegraf_crt_path"
|
||||
"KEY_PATH=%d/telegraf_key_path"
|
||||
];
|
||||
Restart = "on-failure";
|
||||
User = "telegraf";
|
||||
Group = "telegraf";
|
||||
RuntimeDirectory = "telegraf-www";
|
||||
};
|
||||
script = "${pkgs.miniserve}/bin/miniserve -p 9990 /run/telegraf-www --auth-file \"$AUTH_FILE_PATH\" --tls-cert \"$CRT_PATH\" --tls-key \"$KEY_PATH\"";
|
||||
};
|
||||
|
||||
systemd.services.telegraf = {
|
||||
serviceConfig = {
|
||||
LoadCredential = [
|
||||
"telegraf_crt_path:${config.clan.core.vars.generators.telegraf-certs.files.crt.path}"
|
||||
"telegraf_key_path:${config.clan.core.vars.generators.telegraf-certs.files.key.path}"
|
||||
];
|
||||
Environment = [
|
||||
"CRT_PATH=%d/telegraf_crt_path"
|
||||
"KEY_PATH=%d/telegraf_key_path"
|
||||
];
|
||||
};
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 9273 ];
|
||||
|
||||
services.telegraf = {
|
||||
enable = true;
|
||||
environmentFiles = [
|
||||
(builtins.toString config.clan.core.vars.generators.telegraf.files.password-env.path)
|
||||
];
|
||||
environmentFiles = [ config.clan.core.vars.generators.prometheus.files.password-env.path ];
|
||||
|
||||
extraConfig = {
|
||||
agent.interval = "60s";
|
||||
inputs = {
|
||||
|
||||
# More input plugins available at:
|
||||
# https://github.com/influxdata/telegraf/tree/master/plugins/inputs
|
||||
diskio = { };
|
||||
disk = { };
|
||||
cpu = { };
|
||||
processes = { };
|
||||
kernel_vmstat = { };
|
||||
system = { };
|
||||
mem = { };
|
||||
@@ -147,20 +51,12 @@
|
||||
}
|
||||
];
|
||||
};
|
||||
# sadly there doesn'T seem to exist a telegraf http_client output plugin
|
||||
# sadly there doesn't seem to exist a telegraf http_client output plugin
|
||||
outputs.prometheus_client = {
|
||||
listen = ":9273";
|
||||
metric_version = 2;
|
||||
basic_username = "${auth_user}";
|
||||
basic_username = "prometheus";
|
||||
basic_password = "$${BASIC_AUTH_PWD}";
|
||||
tls_cert = "$${CRT_PATH}";
|
||||
tls_key = "$${KEY_PATH}";
|
||||
};
|
||||
|
||||
outputs.file = {
|
||||
files = [ "/run/telegraf-www/telegraf.json" ];
|
||||
data_format = "json";
|
||||
json_timestamp_units = "1s";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user