improve all the things

README.md

@@ -1,5 +1,11 @@
 # Nix based CCCB infra
 
+## Folder structure
+
+- `./services` holds generic service configuration that is shared between hosts
+- `./hosts` holds host specific configuration
+- `./secrets` holds age encrypted secrets using [agenix](https://github.com/ryantm/agenix)
+
 ## Admin handbook
 
 ### Update a container
@@ -7,7 +13,7 @@
 ```shell
 ssh <container>
 cd /etc/nixos
-nix run .#apps.nixos-diff   # Show what changes would be applied
+nix run .#apps.nixos-diff   # git pull + build + diff wth running config
 nixos-rebuild switch        # Apply changes
 ```
 
@@ -15,12 +21,13 @@ nixos-rebuild switch        # Apply changes
 
 Production:
 
-- [Matrix](./README.matrix.md)
+- [Matrix (matrix.berlin.ccc.de)](./README.matrix.md)
 
 Testing:
 
-- [Hedgedoc](./README.hedgedoc.md)
-- [Postgres](./README.postgres.md)
+- [Hedgedoc (md.berlin.ccc.de)](./README.hedgedoc.md)
+- [Postgres (sql.berlin.ccc.de)](./README.postgres.md)
+- [Grafana/Prometheus (monitoring.berlin.ccc.de)](./README.monitoring.md)
 
 ---
 

README.monitoring.md

@@ -0,0 +1,10 @@
+# Monitoring
+
+## Grafana
+
+## Prometheus
+
+---
+
+Build with ❤️ and ❄️.
+

flake.nix

@@ -100,6 +100,16 @@
         modules = [
           agenix.nixosModules.default
           { environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
+          {
+            age.secrets = {
+              hedgedoc-env = {
+                file = ./secrets/hedgedoc-env.age;
+                mode = "440";
+                owner = "hedgedoc";
+                group = "hedgedoc";
+              };
+            };
+          }
           ./hosts/md
         ];
       };
@@ -124,11 +134,37 @@
                 group = "postgres";
                 mode = "0400";
               };
+              postgres-grafana = {
+                file = ./secrets/postgres-grafana.age;
+                owner = "postgres";
+                group = "postgres";
+                mode = "0400";
+              };
             };
           }
           ./hosts/sql
         ];
       };
+      nixosConfigurations."monitoring" = nixpkgs.lib.nixosSystem {
+        #system = "x86_64-linux";
+        #pkgs = import nixpkgs { inherit system; };
+        inherit system;
+        modules = [
+          agenix.nixosModules.default
+          { environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
+          {
+            age.secrets = {
+              postgres-grafana = {
+                file = ./secrets/postgres-grafana.age;
+                owner = "postgres";
+                group = "postgres";
+                mode = "0400";
+              };
+            };
+          }
+          ./hosts/monitoring
+        ];
+      };
     };
   #);
 }

hosts/common.nix

@@ -8,6 +8,7 @@
 {
   imports = [
     (modulesPath + "/virtualisation/proxmox-lxc.nix")
+    ../../services/node-exporter.nix
   ];
 
   systemd.suppressedSystemUnits = [

hosts/monitoring/default.nix

@@ -0,0 +1,41 @@
+{ ... }:
+
+{
+  imports = [
+    ../common.nix
+    ../../services/openssh.nix
+    ../../services/nginx.nix
+    ./nginx.nix
+    ./prometheus.nix
+    ./grafana.nix
+  ];
+
+  networking = {
+    hostName = "monitoring";
+    firewall = {
+      allowedTCPPorts = [
+        80 # HTTP/1
+        443 # HTTP/2
+      ];
+      allowedUDPPorts = [
+        443 # HTTP/3
+      ];
+    };
+  };
+
+  services = {
+    openssh.banner = ''
+                                    __
+                                 __/\ \__                __
+        ___ ___     ___     ___ /\_\ \ ,_\   ___   _ __ /\_\    ___      __
+      /' __` __`\  / __`\ /' _ `\/\ \ \ \/  / __`\/\`'__\/\ \ /' _ `\  /'_ `\
+      /\ \/\ \/\ \/\ \L\ \/\ \/\ \ \ \ \ \_/\ \L\ \ \ \/ \ \ \/\ \/\ \/\ \L\ \
+      \ \_\ \_\ \_\ \____/\ \_\ \_\ \_\ \__\ \____/\ \_\  \ \_\ \_\ \_\ \____ \
+       \/_/\/_/\/_/\/___/  \/_/\/_/\/_/\/__/\/___/  \/_/   \/_/\/_/\/_/\/___L\ \
+                                                                         /\____/
+                                                                         \_/__/
+    '';
+  };
+
+  system.stateVersion = "25.11";
+}

hosts/monitoring/grafana.nix

@@ -0,0 +1,47 @@
+{ config, ... }:
+
+# Dashboards:
+# - Synapse: https://github.com/element-hq/synapse/tree/master/contrib/grafana
+
+{
+  services = {
+    grafana = {
+      enable = true;
+      settings = {
+        server.http_addr = "::1";
+        database = {
+          type = "postgres";
+          host = "sql.berlin.ccc.de";
+          name = "grafana";
+          user = "grafana";
+          password = "$__file{${config.age.secrets.postgres_grafana.path}}";
+        };
+        security = {
+          secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}";
+          admin_user = "xengi";
+          admin_password = "$__file{${config.age.secrets.grafana_admin_password.path}}";
+          admin_email = "grafana@xengi.de";
+        };
+        analytics = {
+          reporting_enabled = false;
+          feedback_links_enabled = false;
+        };
+      };
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
+            jsonData = {
+              httpMethod = "GET";
+              prometheusType = "Prometheus";
+              cacheLevel = "High";
+            };
+          }
+        ];
+      };
+    };
+  };
+}

hosts/monitoring/nginx.nix

@@ -0,0 +1,29 @@
+{ config, pkgs, ... }:
+
+{
+  services.nginx = {
+    upstreams.grafana.servers."localhost:3000" = {};
+    virtualHosts."${config.networking.hostName}.${config.networking.domain}" = {
+      default = true;
+      quic = true;
+      kTLS = true;
+      forceSSL = true;
+      enableACME = true;
+      #auth_basic "Administrator’s Area";
+      #auth_basic_user_file ${config.age.secrets.grafana-basic-auth.path};
+      locations = {
+        #"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot;
+        "/" = {
+          recommendedProxySettings = true;
+          proxyPass = "http://grafana";
+        };
+        "/api/live/" = {
+          recommendedProxySettings = true;
+          proxyWebsockets = true;
+          proxyPass = "http://grafana";
+        };
+      };
+    };
+  };
+}
+

hosts/monitoring/prometheus.nix

@@ -0,0 +1,88 @@
+{ config, pkgs, lib, ... }:
+
+{
+  services.prometheus = {
+    enable = true;
+    retentionTime = "14d";
+    listenAddress = "[::1]";
+    scrapeConfigs = [
+      {
+        job_name = "hedgedoc";
+        scrape_interval = "15s";
+        scheme = "https";
+        static_configs = [{ targets = ["md.berlin.ccc.de:443"]; }];
+      }
+      {
+        job_name = "synapse";
+        scrape_interval = "15s";
+        static_configs = [{ targets = ["matrix.berlin.ccc.de:9009"]; }];
+      }
+      {
+        job_name = "node";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = [
+              "matrix.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+              "md.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+              "postgres.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+              "monitoring:${toString config.services.prometheus.exporters.node.port}"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "nginx";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = [
+              "monitoring:${toString config.services.prometheus.exporters.nginx.port}"
+              "matrix:${toString config.services.prometheus.exporters.nginx.port}"
+            ];
+          }
+        ];
+      }
+    ];
+    ruleFiles = [
+      # https://github.com/element-hq/synapse/tree/master/contrib/prometheus
+      (pkgs.writeText "prom-synapse-rules.yaml" ''
+        groups:
+        - name: synapse
+          rules:
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_client_sent_edus_total + 0'
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+          - record: 'synapse_federation_client_sent'
+            labels:
+              type: "Query"
+            expr: 'sum(synapse_federation_client_sent_queries) by (job)'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_server_received_edus_total + 0'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_server_received_pdus_total + 0'
+          - record: 'synapse_federation_server_received'
+            labels:
+              type: "Query"
+            expr: 'sum(synapse_federation_server_received_queries) by (job)'
+          - record: 'synapse_federation_transaction_queue_pending'
+            labels:
+              type: "EDU"
+            expr: 'synapse_federation_transaction_queue_pending_edus + 0'
+          - record: 'synapse_federation_transaction_queue_pending'
+            labels:
+              type: "PDU"
+            expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+      '')
+    ];
+  };
+}

secrets/secrets.nix

@@ -17,17 +17,22 @@ let
   _matrix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIApAkkhHLj918co/wUGuyW8WCPYHxsNM4uo32XDEu7VV root@matrix";
   _md = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdFkdEEDXo8+k5YZpI1O2GqZlxcpCDtxqVun35duITm root@md";
   _sql = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPcSXjDSyVVVdJbpheOhT0fIuOGFk+jsHhjrAVnBNLQV root@sql";
+  _mon = "";
 in
 {
+  "pushover_app_token.age".publicKeys = users ++ [ _matrix ];
+  "pushover_user_key.age".publicKeys = users ++ [ _matrix ];
   "matrix_admin_password.age".publicKeys = users;
   "draupnir_access_token.age".publicKeys = users ++ [ _matrix ];
   "matrix_signing_key.age".publicKeys = users ++ [ _matrix ];
   "matrix_registration_shared_secret.age".publicKeys = users ++ [ _matrix ];
-  "pushover_app_token.age".publicKeys = users ++ [ _matrix ];
-  "pushover_user_key.age".publicKeys = users ++ [ _matrix ];
   "grafana_admin_password.age".publicKeys = users ++ [ _matrix ];
   "grafana_secret_key.age".publicKeys = users ++ [ _matrix ];
+
+  "hedgedoc-env.age".publicKeys = users ++ [ _md ];
+
   "postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ];
   "postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ];
+  "postgres-grafana.age".publicKeys = users ++ [ _sql _mon ];
 }
 

services/node-exporter.nix

@@ -0,0 +1,13 @@
+{ ... }:
+
+{
+  services.prometheus.exporters.node = {
+    enable = true;
+    openFirewall = true;
+    firewallRules = ''
+      ip saddr 195.160.173.14 tcp dport 9187 counter accept
+      ip6 saddr 2001:678:760:cccb::14 tcp dport 9187 counter accept
+    '';
+  };
+}
+