diff --git a/README.md b/README.md index 4410e01..fb55bd4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Nix based CCCB infra +## Folder structure + +- `./services` holds generic service configuration that is shared between hosts +- `./hosts` holds host specific configuration +- `./secrets` holds age encrypted secrets using [agenix](https://github.com/ryantm/agenix) + ## Admin handbook ### Update a container @@ -7,7 +13,7 @@ ```shell ssh cd /etc/nixos -nix run .#apps.nixos-diff # Show what changes would be applied +nix run .#apps.nixos-diff # git pull + build + diff wth running config nixos-rebuild switch # Apply changes ``` @@ -15,12 +21,13 @@ nixos-rebuild switch # Apply changes Production: -- [Matrix](./README.matrix.md) +- [Matrix (matrix.berlin.ccc.de)](./README.matrix.md) Testing: -- [Hedgedoc](./README.hedgedoc.md) -- [Postgres](./README.postgres.md) +- [Hedgedoc (md.berlin.ccc.de)](./README.hedgedoc.md) +- [Postgres (sql.berlin.ccc.de)](./README.postgres.md) +- [Grafana/Prometheus (monitoring.berlin.ccc.de)](./README.monitoring.md) --- diff --git a/README.monitoring.md b/README.monitoring.md new file mode 100644 index 0000000..cc0224f --- /dev/null +++ b/README.monitoring.md @@ -0,0 +1,10 @@ +# Monitoring + +## Grafana + +## Prometheus + +--- + +Build with ❤️ and ❄️. + diff --git a/flake.nix b/flake.nix index 6d6c0f4..b22ae31 100644 --- a/flake.nix +++ b/flake.nix @@ -100,6 +100,16 @@ modules = [ agenix.nixosModules.default { environment.systemPackages = [ (agenix.packages.${system}.default) ]; } + { + age.secrets = { + hedgedoc-env = { + file = ./secrets/hedgedoc-env.age; + mode = "440"; + owner = "hedgedoc"; + group = "hedgedoc"; + }; + }; + } ./hosts/md ]; }; @@ -124,11 +134,37 @@ group = "postgres"; mode = "0400"; }; + postgres-grafana = { + file = ./secrets/postgres-grafana.age; + owner = "postgres"; + group = "postgres"; + mode = "0400"; + }; }; } ./hosts/sql ]; }; + nixosConfigurations."monitoring" = nixpkgs.lib.nixosSystem { + #system = "x86_64-linux"; + #pkgs = import nixpkgs { inherit system; }; + inherit system; + modules = [ + agenix.nixosModules.default + { environment.systemPackages = [ (agenix.packages.${system}.default) ]; } + { + age.secrets = { + postgres-grafana = { + file = ./secrets/postgres-grafana.age; + owner = "postgres"; + group = "postgres"; + mode = "0400"; + }; + }; + } + ./hosts/monitoring + ]; + }; }; #); } diff --git a/hosts/common.nix b/hosts/common.nix index e795c64..42d42e9 100644 --- a/hosts/common.nix +++ b/hosts/common.nix @@ -8,6 +8,7 @@ { imports = [ (modulesPath + "/virtualisation/proxmox-lxc.nix") + ../../services/node-exporter.nix ]; systemd.suppressedSystemUnits = [ diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix new file mode 100644 index 0000000..10f7e18 --- /dev/null +++ b/hosts/monitoring/default.nix @@ -0,0 +1,41 @@ +{ ... }: + +{ + imports = [ + ../common.nix + ../../services/openssh.nix + ../../services/nginx.nix + ./nginx.nix + ./prometheus.nix + ./grafana.nix + ]; + + networking = { + hostName = "monitoring"; + firewall = { + allowedTCPPorts = [ + 80 # HTTP/1 + 443 # HTTP/2 + ]; + allowedUDPPorts = [ + 443 # HTTP/3 + ]; + }; + }; + + services = { + openssh.banner = '' + __ + __/\ \__ __ + ___ ___ ___ ___ /\_\ \ ,_\ ___ _ __ /\_\ ___ __ + /' __` __`\ / __`\ /' _ `\/\ \ \ \/ / __`\/\`'__\/\ \ /' _ `\ /'_ `\ + /\ \/\ \/\ \/\ \L\ \/\ \/\ \ \ \ \ \_/\ \L\ \ \ \/ \ \ \/\ \/\ \/\ \L\ \ + \ \_\ \_\ \_\ \____/\ \_\ \_\ \_\ \__\ \____/\ \_\ \ \_\ \_\ \_\ \____ \ + \/_/\/_/\/_/\/___/ \/_/\/_/\/_/\/__/\/___/ \/_/ \/_/\/_/\/_/\/___L\ \ + /\____/ + \_/__/ + ''; + }; + + system.stateVersion = "25.11"; +} diff --git a/services/grafana.nix b/hosts/monitoring/grafana.nix similarity index 79% rename from services/grafana.nix rename to hosts/monitoring/grafana.nix index b14e43c..e64080e 100644 --- a/services/grafana.nix +++ b/hosts/monitoring/grafana.nix @@ -11,9 +11,10 @@ server.http_addr = "::1"; database = { type = "postgres"; + host = "sql.berlin.ccc.de"; name = "grafana"; user = "grafana"; - host = "/run/postgresql"; + password = "$__file{${config.age.secrets.postgres_grafana.path}}"; }; security = { secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}"; @@ -42,17 +43,5 @@ ]; }; }; - - postgresql = { - ensureUsers = [ - { - name = config.services.grafana.settings.database.user; - ensureDBOwnership = true; - } - ]; - ensureDatabases = [ - config.services.grafana.settings.database.name - ]; - }; }; } diff --git a/hosts/monitoring/nginx.nix b/hosts/monitoring/nginx.nix new file mode 100644 index 0000000..31dddfe --- /dev/null +++ b/hosts/monitoring/nginx.nix @@ -0,0 +1,29 @@ +{ config, pkgs, ... }: + +{ + services.nginx = { + upstreams.grafana.servers."localhost:3000" = {}; + virtualHosts."${config.networking.hostName}.${config.networking.domain}" = { + default = true; + quic = true; + kTLS = true; + forceSSL = true; + enableACME = true; + #auth_basic "Administrator’s Area"; + #auth_basic_user_file ${config.age.secrets.grafana-basic-auth.path}; + locations = { + #"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot; + "/" = { + recommendedProxySettings = true; + proxyPass = "http://grafana"; + }; + "/api/live/" = { + recommendedProxySettings = true; + proxyWebsockets = true; + proxyPass = "http://grafana"; + }; + }; + }; + }; +} + diff --git a/services/prometheus.nix b/hosts/monitoring/prometheus.nix similarity index 70% rename from services/prometheus.nix rename to hosts/monitoring/prometheus.nix index d8c18b8..efb3e1f 100644 --- a/services/prometheus.nix +++ b/hosts/monitoring/prometheus.nix @@ -5,43 +5,42 @@ enable = true; retentionTime = "14d"; listenAddress = "[::1]"; - exporters = { - node = { - enable = true; - listenAddress = config.services.prometheus.listenAddress; - }; - nginx = { - enable = true; - listenAddress = config.services.prometheus.listenAddress; - }; - #postgres = {}; - }; scrapeConfigs = [ + { + job_name = "hedgedoc"; + scrape_interval = "15s"; + scheme = "https"; + static_configs = [{ targets = ["md.berlin.ccc.de:443"]; }]; + } { job_name = "synapse"; scrape_interval = "15s"; - static_configs = [ - { - targets = lib.pipe config.services.matrix-synapse.settings.listeners [ - (lib.filter (l: l.type == "metrics")) - builtins.head - (l: [ "[${builtins.head l.bind_addresses}]:${toString l.port}" ]) - ]; - } - ]; + static_configs = [{ targets = ["matrix.berlin.ccc.de:9009"]; }]; } { job_name = "node"; scrape_interval = "15s"; static_configs = [ - { targets = [ "${config.services.prometheus.exporters.node.listenAddress}:${toString config.services.prometheus.exporters.node.port}" ]; } + { + targets = [ + "matrix.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}" + "md.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}" + "postgres.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}" + "monitoring:${toString config.services.prometheus.exporters.node.port}" + ]; + } ]; } { job_name = "nginx"; scrape_interval = "15s"; static_configs = [ - { targets = [ "${config.services.prometheus.exporters.nginx.listenAddress}:${toString config.services.prometheus.exporters.nginx.port}" ]; } + { + targets = [ + "monitoring:${toString config.services.prometheus.exporters.nginx.port}" + "matrix:${toString config.services.prometheus.exporters.nginx.port}" + ]; + } ]; } ]; diff --git a/secrets/secrets.nix b/secrets/secrets.nix index e8853ac..38abf38 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -17,17 +17,22 @@ let _matrix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIApAkkhHLj918co/wUGuyW8WCPYHxsNM4uo32XDEu7VV root@matrix"; _md = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdFkdEEDXo8+k5YZpI1O2GqZlxcpCDtxqVun35duITm root@md"; _sql = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPcSXjDSyVVVdJbpheOhT0fIuOGFk+jsHhjrAVnBNLQV root@sql"; + _mon = ""; in { + "pushover_app_token.age".publicKeys = users ++ [ _matrix ]; + "pushover_user_key.age".publicKeys = users ++ [ _matrix ]; "matrix_admin_password.age".publicKeys = users; "draupnir_access_token.age".publicKeys = users ++ [ _matrix ]; "matrix_signing_key.age".publicKeys = users ++ [ _matrix ]; "matrix_registration_shared_secret.age".publicKeys = users ++ [ _matrix ]; - "pushover_app_token.age".publicKeys = users ++ [ _matrix ]; - "pushover_user_key.age".publicKeys = users ++ [ _matrix ]; "grafana_admin_password.age".publicKeys = users ++ [ _matrix ]; "grafana_secret_key.age".publicKeys = users ++ [ _matrix ]; + + "hedgedoc-env.age".publicKeys = users ++ [ _md ]; + "postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ]; "postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ]; + "postgres-grafana.age".publicKeys = users ++ [ _sql _mon ]; } diff --git a/services/node-exporter.nix b/services/node-exporter.nix new file mode 100644 index 0000000..a29114c --- /dev/null +++ b/services/node-exporter.nix @@ -0,0 +1,13 @@ +{ ... }: + +{ + services.prometheus.exporters.node = { + enable = true; + openFirewall = true; + firewallRules = '' + ip saddr 195.160.173.14 tcp dport 9187 counter accept + ip6 saddr 2001:678:760:cccb::14 tcp dport 9187 counter accept + ''; + }; +} +