setup grafana db

hosts/matrix/nginx.nix

@@ -18,8 +18,9 @@
           proxy_http_version 1.1;
         '';
       };
-      "/metrics" = {
-        return = "204 \"🔍️\"";
+      "/_synapse/metrics" = {
+        proxyPass = "http://[::1]:9009";
+        recommendedProxySettings = true;
         extraConfig = ''
           allow 2001:678:760:cccb::14;
           allow 195.160.173.14;

hosts/monitoring/grafana.nix

@@ -0,0 +1,48 @@
+{ config, ... }:
+
+# Dashboards:
+# - Synapse: https://github.com/element-hq/synapse/tree/master/contrib/grafana
+
+{
+  services = {
+    grafana = {
+      enable = true;
+      settings = {
+        server.http_addr = "::1";
+        database = {
+          type = "postgres";
+          name = "grafana";
+          user = "grafana";
+          host = "sql.berlin.ccc.de:5432";
+          password = "$__file{${config.age.secrets.postgres-grafana.path}}";
+        };
+        security = {
+          secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}";
+          admin_user = "xengi";
+          admin_password = "$__file{${config.age.secrets.grafana_admin_password.path}}";
+          admin_email = "cccb-grafana@xengi.de";
+        };
+        analytics = {
+          reporting_enabled = false;
+          feedback_links_enabled = false;
+        };
+      };
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
+            jsonData = {
+              httpMethod = "GET";
+              prometheusType = "Prometheus";
+              cacheLevel = "High";
+            };
+          }
+        ];
+      };
+    };
+  };
+}
+

hosts/monitoring/prometheus.nix

@@ -0,0 +1,100 @@
+{ config, pkgs, lib, ... }:
+
+{
+  services.prometheus = {
+    enable = true;
+    retentionTime = "14d";
+    listenAddress = "[::1]";
+    exporters = {
+      node = {
+        enable = true;
+        listenAddress = config.services.prometheus.listenAddress;
+      };
+      nginx = {
+        enable = true;
+        listenAddress = config.services.prometheus.listenAddress;
+      };
+      #postgres = {};
+    };
+    scrapeConfigs = [
+      {
+        job_name = "synapse";
+        scrape_interval = "15s";
+        metrics_path = "/_synapse/metrics";
+        static_configs = [{ targets = [ "matrix.berlin.ccc.de:443" ]; }];
+      }
+      {
+        job_name = "node";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = [
+              "matrix.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "md.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "sql.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "${config.services.prometheus.exporters.node.listenAddress}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "nginx";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = [
+              "matrix.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
+              "md.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
+              "${config.services.prometheus.exporters.nginx.listenAddress}:${toString config.services.prometheus.exporters.nginx.port}"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "postgres";
+        scrape_interval = "15s";
+        static_configs = [{ targets = [ "sql.berlin.ccc.de:${config.services.prometheus.exporters.postgres.port}" ]; }];
+      }
+    ];
+    ruleFiles = [
+      # https://github.com/element-hq/synapse/tree/master/contrib/prometheus
+      (pkgs.writeText "prom-synapse-rules.yaml" ''
+        groups:
+        - name: synapse
+          rules:
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_client_sent_edus_total + 0'
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "Query"
+            expr: 'sum(synapse_federation_client_sent_queries) by (job)'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_server_received_edus_total + 0'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_server_received_pdus_total + 0'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "Query"
+            expr: 'sum(synapse_federation_server_received_queries) by (job)'
+          - record: 'synapse_federation_transaction_queue_pending'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_transaction_queue_pending_edus + 0'
+          - record: 'synapse_federation_transaction_queue_pending'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+      '')
+    ];
+  };
+}

hosts/sql/postgres.nix

@@ -7,6 +7,7 @@ let
   entries = [
     (mkEntry "matrix-synapse" 25) # matrix.berlin.ccc.de
     (mkEntry "hedgedoc" 26) # md.berlin.ccc.de
+    (MkEntry "grafana" 14) # monitoring.berlin.ccc.de
   ];
   mkEntry = name: octet: {
     user = {