Merge branch 'grafana'

Systemüberwachung eingerichtet

4 files changed, 344 insertions, 0 deletions

diff --git a/README.md b/README.md
index 4bce6f7..45c4763 100644
--- a/README.md
+++ b/README.md
@@ -137,6 +137,15 @@ ssh tharos
 nix run .#deploy-tharos -- switch
 ```
 
+### Systemüberwachung und Statistiken
+
+Alle Metriken werden gesammelt und dargestellt auf <https://grafana.heimfeld.hamburg>.
+
+Dashboards werden deklarativ verwaltet in [tharos/dashboards](./tharos/dashboards).
+
+Grafische Bearbeitung ist möglich [in der virtuellen Maschine](#konfiguration-lokal-ausprobieren).
+Danach muss die JSON-Konfiguration exportiert und von Hand auf das Wesentliche reduziert werden.
+
 ## Architektur
 
 Die Systeme laufen auf NixOS und werden mit Nix verwaltet.
diff --git a/tharos/dashboards/system.json b/tharos/dashboards/system.json
new file mode 100644
index 0000000..6b8ec35
--- /dev/null
+++ b/tharos/dashboards/system.json
@@ -0,0 +1,195 @@
+{
+  "editable": true,
+  "id": 1,
+  "panels": [
+    {
+      "id": 1,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "bars"
+          },
+          "unit": "decbytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "legend": {
+          "showLegend": false
+        }
+      },
+      "pluginVersion": "12.0.5",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "node_memory_MemTotal_bytes",
+          "legendFormat": "Total",
+          "range": true,
+          "refId": "total"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
+          "refId": "used"
+        }
+      ],
+      "title": "Arbeitsspeicher",
+      "transformations": [
+        {
+          "id": "configFromData",
+          "options": {
+            "applyTo": {
+              "id": "byName",
+              "options": "Value"
+            },
+            "configRefId": "total",
+            "mappings": [
+              {
+                "fieldName": "Total",
+                "handlerKey": "max"
+              }
+            ]
+          }
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "bars",
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            }
+          },
+          "max": 1,
+          "unit": "percentunit"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "legend": {
+          "showLegend": true
+        }
+      },
+      "pluginVersion": "12.0.5",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum by(mode) (rate(node_cpu_seconds_total{mode!~\"idle\"}[$__rate_interval])) / scalar(count(count by(cpu) (node_cpu_seconds_total)))",
+          "legendFormat": "{{mode}}"
+        }
+      ],
+      "title": "Prozessorlast"
+    },
+    {
+      "id": 3,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "bars",
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            }
+          },
+          "max": 1,
+          "unit": "percentunit"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "legend": {
+          "showLegend": true
+        }
+      },
+      "pluginVersion": "12.0.5",
+      "repeat": "CPU",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "rate(node_cpu_seconds_total{cpu=\"$CPU\", mode!=\"idle\"}[$__rate_interval])",
+          "legendFormat": "{{mode}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Prozessor $CPU",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 41,
+  "templating": {
+    "list": [
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "allowCustomValue": false,
+        "definition": "label_values(node_cpu_seconds_total,cpu)",
+        "includeAll": true,
+        "name": "CPU",
+        "query": {
+          "qryType": 1,
+          "query": "label_values(node_cpu_seconds_total,cpu)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 2,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "title": "System",
+  "uid": "d6a74067-e6b9-4460-8119-d205c606ef68",
+  "version": 1
+}
diff --git a/tharos/grafana.nix b/tharos/grafana.nix
new file mode 100644
index 0000000..be723c6
--- /dev/null
+++ b/tharos/grafana.nix
@@ -0,0 +1,106 @@
+{ self, lib, ... }:
+{
+  flake.machines.tharos = {
+    nixos =
+      { config, ... }:
+      let
+        grafana = {
+          domain = config.services.grafana.settings.server.domain;
+          port = config.services.grafana.settings.server.http_port;
+          database = config.services.grafana.settings.database;
+        };
+      in
+      {
+        services.grafana = {
+          enable = true;
+          openFirewall = true;
+          settings = {
+            server = {
+              domain = "grafana.${self.domain}";
+              root_url = "https://${grafana.domain}";
+            };
+            database = {
+              type = "postgres";
+              user = grafana.database.name;
+              host =
+                let
+                  nextcloud = config.services.nextcloud;
+                in
+                lib.mkIf nextcloud.enable nextcloud.config.dbhost;
+            };
+            security = {
+              disable_initial_admin_creation = true;
+            };
+            "auth.anonymous" = {
+              enabled = true;
+              org_role = "Viewer";
+            };
+            users.default_language = "de-DE";
+            dashboards.default_home_dashboard_path = "${./dashboards + "/system.json"}";
+          };
+          provision = {
+            enable = true;
+            datasources.settings = {
+              prune = true;
+              datasources = [
+                {
+                  name = "Prometheus";
+                  type = "prometheus";
+                  access = "proxy";
+                  url = "http://localhost:${toString config.services.prometheus.port}";
+                }
+              ];
+            };
+            dashboards.settings.providers = [
+              { options.path = ./dashboards; }
+            ];
+          };
+        };
+        services.postgresql = {
+          enable = true;
+          ensureDatabases = [ grafana.database.name ];
+          ensureUsers = [
+            {
+              name = grafana.database.user;
+              ensureDBOwnership = true;
+            }
+          ];
+        };
+        services.caddy = {
+          virtualHosts.${grafana.domain}.extraConfig = ''
+            reverse_proxy http://localhost:${toString grafana.port}
+          '';
+        };
+      };
+
+    vm =
+      { pkgs, config, ... }:
+      let
+        grafana = {
+          domain = config.services.grafana.settings.server.domain;
+          port = config.services.grafana.settings.server.http_port;
+        };
+      in
+      {
+        services.grafana = {
+          settings.server = {
+            domain = lib.mkForce "grafana.localhost";
+            root_url = lib.mkForce "http://${grafana.domain}:${toString grafana.port}";
+          };
+          settings."auth.anonymous" = {
+            org_role = lib.mkForce "Admin";
+          };
+        };
+
+        services.caddy.virtualHosts = {
+          "http://${grafana.domain}:${toString config.virtualisation.exposedPorts.http.port}".extraConfig =
+            config.services.caddy.virtualHosts.${grafana.domain}.extraConfig;
+        };
+        services.getty.helpLine = lib.mkBefore ''
+          Grafana: http://${grafana.domain}:${
+            with config.virtualisation; toString (portOffset + exposedPorts.http.port)
+          }
+        '';
+      };
+  };
+}
diff --git a/tharos/prometheus.nix b/tharos/prometheus.nix
new file mode 100644
index 0000000..48d7a85
--- /dev/null
+++ b/tharos/prometheus.nix
@@ -0,0 +1,34 @@
+{ self, lib, ... }:
+{
+  flake.machines.tharos = {
+    nixos =
+      { config, ... }:
+      {
+        services.prometheus = {
+          enable = true;
+          exporters.node = {
+            enable = true;
+            openFirewall = true;
+            enabledCollectors = [ ];
+          };
+          globalConfig.scrape_interval = "10s";
+          scrapeConfigs = [
+            {
+              job_name = "node";
+              static_configs = [
+                {
+                  targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+                }
+              ];
+            }
+          ];
+        };
+      };
+
+    vm =
+      { ... }:
+      {
+        services.prometheus.globalConfig.scrape_interval = lib.mkForce "5s";
+      };
+  };
+}