improve all the things

README.md

@@ -1,5 +1,11 @@
 # Nix based CCCB infra
 
+## Folder structure
+
+- `./services` holds generic service configuration that is shared between hosts
+- `./hosts` holds host specific configuration
+- `./secrets` holds age encrypted secrets using [agenix](https://github.com/ryantm/agenix)
+
 ## Admin handbook
 
 ### Update a container
@@ -7,7 +13,7 @@
 ```shell
 ssh <container>
 cd /etc/nixos
-nix run .#apps.nixos-diff   # Show what changes would be applied
+nix run .#apps.nixos-diff   # git pull + build + diff wth running config
 nixos-rebuild switch        # Apply changes
 ```
 
@@ -15,12 +21,13 @@ nixos-rebuild switch        # Apply changes
 
 Production:
 
-- [Matrix](./README.matrix.md)
+- [Matrix (matrix.berlin.ccc.de)](./README.matrix.md)
 
 Testing:
 
-- [Hedgedoc](./README.hedgedoc.md)
+- [Hedgedoc (md.berlin.ccc.de)](./README.hedgedoc.md)
-- [Postgres](./README.postgres.md)
+- [Postgres (sql.berlin.ccc.de)](./README.postgres.md)
+- [Grafana/Prometheus (monitoring.berlin.ccc.de)](./README.monitoring.md)
 
 ---
 

README.monitoring.md

@@ -0,0 +1,10 @@
+# Monitoring
+
+## Grafana
+
+## Prometheus
+
+---
+
+Build with ❤️ and ❄️.
+

flake.nix

@@ -100,6 +100,16 @@
         modules = [
           agenix.nixosModules.default
           { environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
+          {
+            age.secrets = {
+              hedgedoc-env = {
+                file = ./secrets/hedgedoc-env.age;
+                mode = "440";
+                owner = "hedgedoc";
+                group = "hedgedoc";
+              };
+            };
+          }
           ./hosts/md
         ];
       };
@@ -124,11 +134,37 @@
                 group = "postgres";
                 mode = "0400";
               };
+              postgres-grafana = {
+                file = ./secrets/postgres-grafana.age;
+                owner = "postgres";
+                group = "postgres";
+                mode = "0400";
+              };
             };
           }
           ./hosts/sql
         ];
       };
+      nixosConfigurations."monitoring" = nixpkgs.lib.nixosSystem {
+        #system = "x86_64-linux";
+        #pkgs = import nixpkgs { inherit system; };
+        inherit system;
+        modules = [
+          agenix.nixosModules.default
+          { environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
+          {
+            age.secrets = {
+              postgres-grafana = {
+                file = ./secrets/postgres-grafana.age;
+                owner = "postgres";
+                group = "postgres";
+                mode = "0400";
+              };
+            };
+          }
+          ./hosts/monitoring
+        ];
+      };
     };
   #);
 }

hosts/common.nix

@@ -8,6 +8,7 @@
 {
   imports = [
     (modulesPath + "/virtualisation/proxmox-lxc.nix")
+    ../../services/node-exporter.nix
   ];
 
   systemd.suppressedSystemUnits = [

hosts/monitoring/default.nix

@@ -0,0 +1,41 @@
+{ ... }:
+
+{
+  imports = [
+    ../common.nix
+    ../../services/openssh.nix
+    ../../services/nginx.nix
+    ./nginx.nix
+    ./prometheus.nix
+    ./grafana.nix
+  ];
+
+  networking = {
+    hostName = "monitoring";
+    firewall = {
+      allowedTCPPorts = [
+        80 # HTTP/1
+        443 # HTTP/2
+      ];
+      allowedUDPPorts = [
+        443 # HTTP/3
+      ];
+    };
+  };
+
+  services = {
+    openssh.banner = ''
+                                    __
+                                 __/\ \__                __
+        ___ ___     ___     ___ /\_\ \ ,_\   ___   _ __ /\_\    ___      __
+      /' __` __`\  / __`\ /' _ `\/\ \ \ \/  / __`\/\`'__\/\ \ /' _ `\  /'_ `\
+      /\ \/\ \/\ \/\ \L\ \/\ \/\ \ \ \ \ \_/\ \L\ \ \ \/ \ \ \/\ \/\ \/\ \L\ \
+      \ \_\ \_\ \_\ \____/\ \_\ \_\ \_\ \__\ \____/\ \_\  \ \_\ \_\ \_\ \____ \
+       \/_/\/_/\/_/\/___/  \/_/\/_/\/_/\/__/\/___/  \/_/   \/_/\/_/\/_/\/___L\ \
+                                                                         /\____/
+                                                                         \_/__/
+    '';
+  };
+
+  system.stateVersion = "25.11";
+}

hosts/monitoring/grafana.nix

@@ -11,9 +11,10 @@
         server.http_addr = "::1";
         database = {
           type = "postgres";
+          host = "sql.berlin.ccc.de";
           name = "grafana";
           user = "grafana";
-          host = "/run/postgresql";
+          password = "$__file{${config.age.secrets.postgres_grafana.path}}";
         };
         security = {
           secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}";
@@ -42,17 +43,5 @@
         ];
       };
     };
-
-    postgresql = {
-      ensureUsers = [
-        {
-          name = config.services.grafana.settings.database.user;
-          ensureDBOwnership = true;
-        }
-      ];
-      ensureDatabases = [
-        config.services.grafana.settings.database.name
-      ];
-    };
   };
 }

hosts/monitoring/nginx.nix

@@ -0,0 +1,29 @@
+{ config, pkgs, ... }:
+
+{
+  services.nginx = {
+    upstreams.grafana.servers."localhost:3000" = {};
+    virtualHosts."${config.networking.hostName}.${config.networking.domain}" = {
+      default = true;
+      quic = true;
+      kTLS = true;
+      forceSSL = true;
+      enableACME = true;
+      #auth_basic "Administrator’s Area";
+      #auth_basic_user_file ${config.age.secrets.grafana-basic-auth.path};
+      locations = {
+        #"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot;
+        "/" = {
+          recommendedProxySettings = true;
+          proxyPass = "http://grafana";
+        };
+        "/api/live/" = {
+          recommendedProxySettings = true;
+          proxyWebsockets = true;
+          proxyPass = "http://grafana";
+        };
+      };
+    };
+  };
+}
+

hosts/monitoring/prometheus.nix

@@ -5,43 +5,42 @@
     enable = true;
     retentionTime = "14d";
     listenAddress = "[::1]";
-    exporters = {
-      node = {
-        enable = true;
-        listenAddress = config.services.prometheus.listenAddress;
-      };
-      nginx = {
-        enable = true;
-        listenAddress = config.services.prometheus.listenAddress;
-      };
-      #postgres = {};
-    };
     scrapeConfigs = [
+      {
+        job_name = "hedgedoc";
+        scrape_interval = "15s";
+        scheme = "https";
+        static_configs = [{ targets = ["md.berlin.ccc.de:443"]; }];
+      }
       {
         job_name = "synapse";
         scrape_interval = "15s";
-        static_configs = [
+        static_configs = [{ targets = ["matrix.berlin.ccc.de:9009"]; }];
-          {
-            targets = lib.pipe config.services.matrix-synapse.settings.listeners [
-              (lib.filter (l: l.type == "metrics"))
-              builtins.head
-              (l: [ "[${builtins.head l.bind_addresses}]:${toString l.port}" ])
-            ];
-          }
-        ];
       }
       {
         job_name = "node";
         scrape_interval = "15s";
         static_configs = [
-          { targets = [ "${config.services.prometheus.exporters.node.listenAddress}:${toString config.services.prometheus.exporters.node.port}" ]; }
+          {
+            targets = [
+              "matrix.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+              "md.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+              "postgres.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+              "monitoring:${toString config.services.prometheus.exporters.node.port}"
+            ];
+          }
         ];
       }
       {
         job_name = "nginx";
         scrape_interval = "15s";
         static_configs = [
-          { targets = [ "${config.services.prometheus.exporters.nginx.listenAddress}:${toString config.services.prometheus.exporters.nginx.port}" ]; }
+          {
+            targets = [
+              "monitoring:${toString config.services.prometheus.exporters.nginx.port}"
+              "matrix:${toString config.services.prometheus.exporters.nginx.port}"
+            ];
+          }
         ];
       }
     ];

secrets/secrets.nix

@@ -17,17 +17,22 @@ let
   _matrix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIApAkkhHLj918co/wUGuyW8WCPYHxsNM4uo32XDEu7VV root@matrix";
   _md = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdFkdEEDXo8+k5YZpI1O2GqZlxcpCDtxqVun35duITm root@md";
   _sql = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPcSXjDSyVVVdJbpheOhT0fIuOGFk+jsHhjrAVnBNLQV root@sql";
+  _mon = "";
 in
 {
+  "pushover_app_token.age".publicKeys = users ++ [ _matrix ];
+  "pushover_user_key.age".publicKeys = users ++ [ _matrix ];
   "matrix_admin_password.age".publicKeys = users;
   "draupnir_access_token.age".publicKeys = users ++ [ _matrix ];
   "matrix_signing_key.age".publicKeys = users ++ [ _matrix ];
   "matrix_registration_shared_secret.age".publicKeys = users ++ [ _matrix ];
-  "pushover_app_token.age".publicKeys = users ++ [ _matrix ];
-  "pushover_user_key.age".publicKeys = users ++ [ _matrix ];
   "grafana_admin_password.age".publicKeys = users ++ [ _matrix ];
   "grafana_secret_key.age".publicKeys = users ++ [ _matrix ];
+
+  "hedgedoc-env.age".publicKeys = users ++ [ _md ];
+
   "postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ];
   "postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ];
+  "postgres-grafana.age".publicKeys = users ++ [ _sql _mon ];
 }
 

services/node-exporter.nix

@@ -0,0 +1,13 @@
+{ ... }:
+
+{
+  services.prometheus.exporters.node = {
+    enable = true;
+    openFirewall = true;
+    firewallRules = ''
+      ip saddr 195.160.173.14 tcp dport 9187 counter accept
+      ip6 saddr 2001:678:760:cccb::14 tcp dport 9187 counter accept
+    '';
+  };
+}
+