setup grafana db

flake.nix

@@ -134,6 +134,12 @@
                 owner = "grafana";
                 group = "grafana";
               };
+              postgres-grafana = {
+                file = ./secrets/postgres-grafana.age;
+                mode = "440";
+                owner = "grafana";
+                group = "grafana";
+              };
             };
           }
           ./hosts/monitoring
@@ -160,6 +166,12 @@
                 group = "postgres";
                 mode = "0400";
               };
+              postgres-grafana = {
+                file = ./secrets/postgres-grafana.age;
+                owner = "postgres";
+                group = "postgres";
+                mode = "0400";
+              };
             };
           }
           ./hosts/sql

hosts/matrix/nginx.nix

@@ -18,8 +18,9 @@
           proxy_http_version 1.1;
         '';
       };
-      "/metrics" = {
-        return = "204 \"🔍️\"";
+      "/_synapse/metrics" = {
+        proxyPass = "http://[::1]:9009";
+        recommendedProxySettings = true;
         extraConfig = ''
           allow 2001:678:760:cccb::14;
           allow 195.160.173.14;

hosts/monitoring/grafana.nix

@@ -13,13 +13,14 @@
           type = "postgres";
           name = "grafana";
           user = "grafana";
-          host = "/run/postgresql";
+          host = "sql.berlin.ccc.de:5432";
+          password = "$__file{${config.age.secrets.postgres-grafana.path}}";
         };
         security = {
           secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}";
           admin_user = "xengi";
           admin_password = "$__file{${config.age.secrets.grafana_admin_password.path}}";
-          admin_email = "grafana@xengi.de";
+          admin_email = "cccb-grafana@xengi.de";
         };
         analytics = {
           reporting_enabled = false;
@@ -42,17 +43,6 @@
         ];
       };
     };
-
-    postgresql = {
-      ensureUsers = [
-        {
-          name = config.services.grafana.settings.database.user;
-          ensureDBOwnership = true;
-        }
-      ];
-      ensureDatabases = [
-        config.services.grafana.settings.database.name
-      ];
-    };
   };
 }
+

hosts/monitoring/prometheus.nix

@@ -20,31 +20,42 @@
       {
         job_name = "synapse";
         scrape_interval = "15s";
-        static_configs = [
-          {
-            targets = lib.pipe config.services.matrix-synapse.settings.listeners [
-              (lib.filter (l: l.type == "metrics"))
-              builtins.head
-              (l: [ "[${builtins.head l.bind_addresses}]:${toString l.port}" ])
-            ];
-          }
-        ];
+        metrics_path = "/_synapse/metrics";
+        static_configs = [{ targets = [ "matrix.berlin.ccc.de:443" ]; }];
       }
       {
         job_name = "node";
         scrape_interval = "15s";
         static_configs = [
-          { targets = [ "${config.services.prometheus.exporters.node.listenAddress}:${toString config.services.prometheus.exporters.node.port}" ]; }
+          {
+            targets = [
+              "matrix.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "md.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "sql.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
+              "${config.services.prometheus.exporters.node.listenAddress}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+          }
         ];
       }
       {
         job_name = "nginx";
         scrape_interval = "15s";
         static_configs = [
-          { targets = [ "${config.services.prometheus.exporters.nginx.listenAddress}:${toString config.services.prometheus.exporters.nginx.port}" ]; }
+          {
+            targets = [
+              "matrix.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
+              "md.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
+              "${config.services.prometheus.exporters.nginx.listenAddress}:${toString config.services.prometheus.exporters.nginx.port}"
             ];
           }
         ];
+      }
+      {
+        job_name = "postgres";
+        scrape_interval = "15s";
+        static_configs = [{ targets = [ "sql.berlin.ccc.de:${config.services.prometheus.exporters.postgres.port}" ]; }];
+      }
+    ];
     ruleFiles = [
       # https://github.com/element-hq/synapse/tree/master/contrib/prometheus
       (pkgs.writeText "prom-synapse-rules.yaml" ''

hosts/sql/postgres.nix

@@ -7,6 +7,7 @@ let
   entries = [
     (mkEntry "matrix-synapse" 25) # matrix.berlin.ccc.de
     (mkEntry "hedgedoc" 26) # md.berlin.ccc.de
+    (MkEntry "grafana" 14) # monitoring.berlin.ccc.de
   ];
   mkEntry = name: octet: {
     user = {

secrets/postgres-grafana.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 uH+n1w DYVhwEzWwZyr3IH7Cl2qa0g2Zg7Yrmeyn0Pf+Gqt7HU
+kDQGApNi5sfgIWXdW4LH6Kzy2mrvFN8yk5uWUYT4oXY
+-> ssh-ed25519 EvLbWw K/A2Gy0eVr1u+44r4XsHV2N6lmUHIu8cV+g3IkjHe0M
+HiMAzVd2NV7bbi20J+VEjsiyaevm7iEtec4igRA2ySQ
+-> ssh-ed25519 dM+fLQ 5F6iAC+GHRsF489WhDjjkLgcnOr+ywU3sWtoYuYMZyg
+C8by0wcMpQwZmI4FaNkNc6rpzAKZPrzS7cm+CwFl3Rw
+-> ssh-ed25519 jxWM2Q idZ3x0aaYoH71CgL6NQ1qDf233yXkH9oNZhwzasgOxc
+ED0t4oArzmVAIS5+KU6cCENnEUO4kuwBNhuMYl183vE
+-> ssh-ed25519 /yCUCg 98BSIjgr6S+QHlbZ8SP+PFBePaCXel+2n+rCLAITU38
+HSum+YkvuOhZZKJfgJ/KO2cTi04r/JuBXbSk9CD0jFs
+-> ssh-ed25519 FGp51g CvBv6/QnajCnWLJb4VXA7KE6jQ/1VrUZWlwvQDcGWn0
+WEKU5CrdH/ZuTj1MYgeoX/oM7qqYqa3kbCi2AIKvQJQ
+-> ssh-ed25519 I2FcBQ 3msOE4meBfv7DgSAAgbuVIYQM6VrR666P6Ay1GfAtm0
+N0/JVT/IOTNamPfjVuUbb8H/vYMXsdAViiKWMHapu1M
+-> ssh-ed25519 Iapucg y2M94CuOpTskf5aHoqewoMwtjJ/+XUdojaTY5A+RAl8
+9oG5ux0F/Y1hz4ZMZvHvvgcopOJ3SRVg4RRCZeT+RDQ
+--- AGVfxKHT0uVsWIFMvRHrAQwBiCzke/xWNb6pg6mxlVE
+ךv_<76>ñ<EFBFBD>L›ŠÜ|Ù"ý󰀎.$EEuAÂÝÊKÞ7´J‚›¹øýæó+Ñ[ظ×å<cö½ê©®)ªÛe¥-@—ÜÒ§@]c¬íóüñ0«Ý<C2AB>¡€äÔU͹Þ
+Ël±R‚

secrets/secrets.nix

@@ -38,6 +38,7 @@ in
 
   "postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ];
   "postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ];
+  "postgres-grafana.age".publicKeys = users ++ [ _sql _monitoring ];
 
   "www-staging-htpasswd.age".publicKeys = users ++ [ _www ];
 }