setup grafana db

flake.nix

@@ -134,6 +134,12 @@
                 owner = "grafana";
                 group = "grafana";
               };
+              postgres-grafana = {
+                file = ./secrets/postgres-grafana.age;
+                mode = "440";
+                owner = "grafana";
+                group = "grafana";
+              };
             };
           }
           ./hosts/monitoring
@@ -160,6 +166,12 @@
                 group = "postgres";
                 mode = "0400";
               };
+              postgres-grafana = {
+                file = ./secrets/postgres-grafana.age;
+                owner = "postgres";
+                group = "postgres";
+                mode = "0400";
+              };
             };
           }
           ./hosts/sql

hosts/matrix/nginx.nix

@@ -18,8 +18,9 @@
           proxy_http_version 1.1;
         '';
       };
-      "/metrics" = {
+      "/_synapse/metrics" = {
-        return = "204 \"🔍️\"";
+        proxyPass = "http://[::1]:9009";
+        recommendedProxySettings = true;
         extraConfig = ''
           allow 2001:678:760:cccb::14;
           allow 195.160.173.14;

hosts/monitoring/grafana.nix

@@ -0,0 +1,48 @@
+{ config, ... }:
+
+# Dashboards:
+# - Synapse: https://github.com/element-hq/synapse/tree/master/contrib/grafana
+
+{
+  services = {
+    grafana = {
+      enable = true;
+      settings = {
+        server.http_addr = "::1";
+        database = {
+          type = "postgres";
+          name = "grafana";
+          user = "grafana";
+          host = "sql.berlin.ccc.de:5432";
+          password = "$__file{${config.age.secrets.postgres-grafana.path}}";
+        };
+        security = {
+          secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}";
+          admin_user = "xengi";
+          admin_password = "$__file{${config.age.secrets.grafana_admin_password.path}}";
+          admin_email = "cccb-grafana@xengi.de";
+        };
+        analytics = {
+          reporting_enabled = false;
+          feedback_links_enabled = false;
+        };
+      };
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
+            jsonData = {
+              httpMethod = "GET";
+              prometheusType = "Prometheus";
+              cacheLevel = "High";
+            };
+          }
+        ];
+      };
+    };
+  };
+}
+

hosts/monitoring/prometheus.nix

@@ -0,0 +1,100 @@
+{ config, pkgs, lib, ... }:
+
+{
+  services.prometheus = {
+    enable = true;
+    retentionTime = "14d";
+    listenAddress = "[::1]";
+    exporters = {
+      node = {
+        enable = true;
+        listenAddress = config.services.prometheus.listenAddress;
+      };
+      nginx = {
+        enable = true;
+        listenAddress = config.services.prometheus.listenAddress;
+      };
+      #postgres = {};
+    };
+    scrapeConfigs = [
+      {
+        job_name = "synapse";
+        scrape_interval = "15s";
+        metrics_path = "/_synapse/metrics";
+        static_configs = [{ targets = [ "matrix.berlin.ccc.de:443" ]; }];
+      }
+      {
+        job_name = "node";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = [
+              "matrix.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "md.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "sql.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "${config.services.prometheus.exporters.node.listenAddress}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "nginx";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = [
+              "matrix.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
+              "md.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
+              "${config.services.prometheus.exporters.nginx.listenAddress}:${toString config.services.prometheus.exporters.nginx.port}"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "postgres";
+        scrape_interval = "15s";
+        static_configs = [{ targets = [ "sql.berlin.ccc.de:${config.services.prometheus.exporters.postgres.port}" ]; }];
+      }
+    ];
+    ruleFiles = [
+      # https://github.com/element-hq/synapse/tree/master/contrib/prometheus
+      (pkgs.writeText "prom-synapse-rules.yaml" ''
+        groups:
+        - name: synapse
+          rules:
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_client_sent_edus_total + 0'
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "Query"
+            expr: 'sum(synapse_federation_client_sent_queries) by (job)'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_server_received_edus_total + 0'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_server_received_pdus_total + 0'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "Query"
+            expr: 'sum(synapse_federation_server_received_queries) by (job)'
+          - record: 'synapse_federation_transaction_queue_pending'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_transaction_queue_pending_edus + 0'
+          - record: 'synapse_federation_transaction_queue_pending'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+      '')
+    ];
+  };
+}

hosts/sql/postgres.nix

@@ -7,6 +7,7 @@ let
   entries = [
     (mkEntry "matrix-synapse" 25) # matrix.berlin.ccc.de
     (mkEntry "hedgedoc" 26) # md.berlin.ccc.de
+    (MkEntry "grafana" 14) # monitoring.berlin.ccc.de
   ];
   mkEntry = name: octet: {
     user = {

secrets/postgres-grafana.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 uH+n1w DYVhwEzWwZyr3IH7Cl2qa0g2Zg7Yrmeyn0Pf+Gqt7HU
+kDQGApNi5sfgIWXdW4LH6Kzy2mrvFN8yk5uWUYT4oXY
+-> ssh-ed25519 EvLbWw K/A2Gy0eVr1u+44r4XsHV2N6lmUHIu8cV+g3IkjHe0M
+HiMAzVd2NV7bbi20J+VEjsiyaevm7iEtec4igRA2ySQ
+-> ssh-ed25519 dM+fLQ 5F6iAC+GHRsF489WhDjjkLgcnOr+ywU3sWtoYuYMZyg
+C8by0wcMpQwZmI4FaNkNc6rpzAKZPrzS7cm+CwFl3Rw
+-> ssh-ed25519 jxWM2Q idZ3x0aaYoH71CgL6NQ1qDf233yXkH9oNZhwzasgOxc
+ED0t4oArzmVAIS5+KU6cCENnEUO4kuwBNhuMYl183vE
+-> ssh-ed25519 /yCUCg 98BSIjgr6S+QHlbZ8SP+PFBePaCXel+2n+rCLAITU38
+HSum+YkvuOhZZKJfgJ/KO2cTi04r/JuBXbSk9CD0jFs
+-> ssh-ed25519 FGp51g CvBv6/QnajCnWLJb4VXA7KE6jQ/1VrUZWlwvQDcGWn0
+WEKU5CrdH/ZuTj1MYgeoX/oM7qqYqa3kbCi2AIKvQJQ
+-> ssh-ed25519 I2FcBQ 3msOE4meBfv7DgSAAgbuVIYQM6VrR666P6Ay1GfAtm0
+N0/JVT/IOTNamPfjVuUbb8H/vYMXsdAViiKWMHapu1M
+-> ssh-ed25519 Iapucg y2M94CuOpTskf5aHoqewoMwtjJ/+XUdojaTY5A+RAl8
+9oG5ux0F/Y1hz4ZMZvHvvgcopOJ3SRVg4RRCZeT+RDQ
+--- AGVfxKHT0uVsWIFMvRHrAQwBiCzke/xWNb6pg6mxlVE
+ךv_<76>ñ<EFBFBD>L›ŠÜ|Ù"ý󰀎.$EEuAÂÝÊKÞ7´J‚›¹øýæó+Ñ[ظ×å<cö½ê©®)ªÛe¥-@—ÜÒ§@]c¬íóüñ0«Ý<C2AB>¡€äÔU͹Þ
+Ël±R‚

secrets/secrets.nix

@@ -38,6 +38,7 @@ in
 
   "postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ];
   "postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ];
+  "postgres-grafana.age".publicKeys = users ++ [ _sql _monitoring ];
 
   "www-staging-htpasswd.age".publicKeys = users ++ [ _www ];
 }