Merge branch 'main' of ssh://git.berlin.ccc.de/xengi/infra

2026-09-11 19:37:40 +02:00 · 2026-09-11 19:37:40 +02:00 · f7c03f7fe7
commit f7c03f7fe7
parent 2751358bf2 cd48d3c530
25 changed files with 449 additions and 79 deletions
--- a/.forgejo/workflows/forgejo.yaml
+++ b/.forgejo/workflows/forgejo.yaml
@ -0,0 +1,37 @@
+name: Test Forgejo setup
+
+on:
+  workflow_dispatch:
+
+env:
+  TEST_URL: berlin.ccc.de
+
+jobs:
+  build:
+    runs-on: debian-stable
+    name: Test setup
+    steps:
+      - name: Check env
+        run: env
+      - name: Install tools
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends curl ca-certificates inetutils-ping iproute2 git
+      - name: Check resolv.conf
+        run: cat /etc/resolv.conf
+      - name: Check IP config
+        run: ip address
+      - name: Check IPv4 route
+        run: ip -4 route
+      - name: Check IPv6 route
+        run: ip -6 route
+      - name: Check IPv4 internet
+        run: ping -c4 1.1.1.1
+      - name: Check IPv6 internet
+        run: ping6 -c4 2606:4700:4700::1111
+      - name: Check IPv4 HTTP
+        run: curl -4 -vfSLo /dev/null ${{ env.TEST_URL }}
+      - name: Check IPv6 HTTP
+        run: curl -6 -vfSLo /dev/null ${{ env.TEST_URL }}
+      - name: Check out the repository
+        run: git clone ${{ env.FORGEJO_SERVER_URL }}/${{ env.FORGEJO_REPOSITORY }} .
--- a/.forgejo/workflows/nix.yaml
+++ b/.forgejo/workflows/nix.yaml
@ -0,0 +1,22 @@
+name: Test Nix flake
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  nix:
+    runs-on: nix-latest
+    name: Test nix flake
+    steps:
+      - name: Checkout repository
+        run: git clone ${{ env.FORGEJO_SERVER_URL }}/${{ env.FORGEJO_REPOSITORY }} .
+      - name: Run formatter
+        run: nix fmt
+      - name: Check flake
+        run: nix flake check
+      - name: Show metadata
+        run: nix flake metadata
+      - name: Show content
+        run: nix flake show
+
--- a/README.md
+++ b/README.md
@ -1,4 +1,6 @@
 [![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page)
+[![Forgejo](https://git.berlin.ccc.de/xengi/infra/actions/workflows/forgejo.yaml/badge.svg)](https://git.berlin.ccc.de/xengi/infra/actions/workflows/forgejo.yaml)
+[![Nix](https://git.berlin.ccc.de/xengi/infra/actions/workflows/nix.yaml/badge.svg)](https://git.berlin.ccc.de/xengi/infra/actions/workflows/nix.yaml)

 # Nix based CCCB infra

--- a/flake.nix
+++ b/flake.nix
@ -23,7 +23,7 @@
    in
    {
      formatter.${system} = pkgs.nixfmt-tree;
-      apps = {
+      apps.${system} = {
        nixos-diff = {
          type = "app";
          program = "${pkgs.writeShellScript "nixos-diff.sh" ''
@ -293,6 +293,32 @@
          ./hosts/git
        ];
      };
+      nixosConfigurations."forgejo-runner" = nixpkgs.lib.nixosSystem {
+        #system = "x86_64-linux";
+        #pkgs = import nixpkgs { inherit system; };
+        inherit system;
+        modules = [
+          agenix.nixosModules.default
+          { environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
+          {
+            age.secrets = {
+              forgejo-runner-token = {
+                file = ./secrets/forgejo-runner-token-snowden.age;
+                owner = "root";
+                group = "root";
+                mode = "0444";
+              };
+              forgejo-runner-token-env = {
+                file = ./secrets/forgejo-runner-token-snowden-env.age;
+                owner = "root";
+                group = "root";
+                mode = "0444";
+              };
+            };
+          }
+          ./hosts/forgejo-runner
+        ];
+      };
    };
  #);
 }
--- a/hosts/common.nix
+++ b/hosts/common.nix
@ -37,7 +37,7 @@
    gc = {
      automatic = true;
      dates = "weekly";
-      options = "--delete-older-than 14d";
+      options = "--delete-older-than 7d";
    };
  };

--- a/hosts/forgejo-runner/default.nix
+++ b/hosts/forgejo-runner/default.nix
@ -0,0 +1,35 @@
+{ ... }:
+
+{
+  imports = [
+    ../common.nix
+    ../../services/openssh.nix
+    ../../services/prometheus-node.nix
+    ./podman.nix
+    ./forgejo-runner.nix
+  ];
+
+  networking = {
+    hostName = "forgejo-runner";
+    firewall = {
+      allowedTCPPorts = [
+        22 # SSH
+      ];
+    };
+  };
+
+  users.users.root.openssh.authorizedKeys.keys = [
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMn0SP19A5C8PqdH+99ki3TILozj/U4tBSQxcRRepN21 samuel@pluto"
+  ];
+
+  environment.etc."ssh/banner".text = ''
+     _ __   __  __    ___     ___      __   _ __
+    /\`'__\/\ \/\ \ /' _ `\ /' _ `\  /'__`\/\`'__\
+    \ \ \/ \ \ \_\ \/\ \/\ \/\ \/\ \/\  __/\ \ \/
+     \ \_\  \ \____/\ \_\ \_\ \_\ \_\ \____\\ \_\
+      \/_/   \/___/  \/_/\/_/\/_/\/_/\/____/ \/_/
+  '';
+  services.openssh.settings.Banner = "/etc/ssh/banner";
+
+  system.stateVersion = "26.05";
+}
--- a/hosts/forgejo-runner/forgejo-runner.nix
+++ b/hosts/forgejo-runner/forgejo-runner.nix
@ -0,0 +1,91 @@
+{ config, pkgs, ... }:
+
+{
+  services.gitea-actions-runner = {
+    package = pkgs.forgejo-runner;
+    instances.snowden = {
+      enable = true;
+      name = "snowden";
+      url = "https://git.berlin.ccc.de";
+      labels = [
+        "debian-stable:docker://docker.io/debian:stable-slim"
+        "alpine-latest:docker://docker.io/alpine:latest"
+        "nix-latest:docker://docker.io/nixos/nix:latest"
+        "buildkit:docker://docker.io/moby/buildkit:rootless"
+      ];
+      tokenFile = config.age.secrets.forgejo-runner-token-env.path;
+      settings = {
+        runner.labels = {
+          debian-stable = {
+            backend = "docker";
+            backend-options = {
+              image = "docker.io/debian:stable-slim";
+              platform = "linux/amd64";
+            };
+          };
+          alpine-latest = {
+            backend = "docker";
+            backend-options = {
+              image = "docker.io/alpine:latest";
+              platform = "linux/amd64";
+            };
+          };
+          buildkit = {
+            backend = "docker";
+            backend-options = {
+              image = "docker.io/moby/buildkit:rootless";
+              platform = "linux/amd64";
+            };
+          };
+        };
+        server.connections.forgejo = {
+          url = "https://git.berlin.ccc.de/";
+          uuid = "78cbc26a-09d3-4226-b239-9c2ee146a63f";
+          token_url = "file://${config.age.secrets.forgejo-runner-token.path}";
+        };
+        container = {
+          enable_ipv6 = true;
+          network = "forgejo";
+          privileged = true;
+          force_pull = true;
+          docker_host = "unix:///run/podman/podman.sock";
+        };
+      };
+    };
+  };
+
+  systemd = {
+    services."forgejo-runner-container-updater" = {
+      description = "Update forgejo runner container images";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      serviceConfig.Type = "oneshot";
+      script = ''
+        set -uo pipefail
+        failed=0
+        images=(
+          docker.io/library/debian:stable-slim
+          docker.io/library/alpine:latest
+          docker.io/nixos/nix:latest
+          docker.io/moby/buildkit:latest
+        )
+        for image in "''${images[@]}"; do
+          echo "Pulling $image..."
+          if ! ${pkgs.podman}/bin/podman pull "$image"; then
+            echo "Failed to pull $image"
+            failed=1
+          fi
+        done
+        exit $failed
+      '';
+    };
+    timers."forgejo-runner-container-updater" = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "daily";
+        RandomizedDelaySec = "30m";
+        Persistent = true;
+      };
+    };
+  };
+}
--- a/hosts/forgejo-runner/podman.nix
+++ b/hosts/forgejo-runner/podman.nix
@ -0,0 +1,36 @@
+{ ... }:
+
+{
+  boot.kernel.sysctl = {
+    "net.ipv6.conf.all.forwarding" = true;
+    "net.ipv4.conf.all.forwarding" = true;
+  };
+
+  networking.firewall.interfaces.podman1 = {
+    allowedTCPPorts = [ 53 ];
+    allowedUDPPorts = [ 53 ];
+  };
+
+  virtualisation.podman = {
+    enable = true;
+    dockerSocket.enable = true;
+    autoPrune = {
+      enable = true;
+      dates = "weekly";
+    };
+    defaultNetwork.settings = {
+      dns_enabled = true;
+      ipv6_enabled = true;
+      subnets = [
+        {
+          gateway = "10.88.0.1";
+          subnet = "10.88.0.0/16";
+        }
+        {
+          gateway = "fd10:88::1";
+          subnet = "fd10:88::/64";
+        }
+      ];
+    };
+  };
+}
--- a/hosts/git/default.nix
+++ b/hosts/git/default.nix
@ -40,5 +40,5 @@
  '';
  services.openssh.settings.Banner = "/etc/ssh/banner";

-  system.stateVersion = "25.11";
+  system.stateVersion = "26.05";
 }
--- a/hosts/git/forgejo.nix
+++ b/hosts/git/forgejo.nix
@ -26,7 +26,6 @@ in
        oauth2.JWT_SECRET = lib.mkDefault config.age.secrets.forgejo-oauth2-jwt-secret.path;
      };
      settings = {
-        security.GLOBAL_TWO_FACTOR_REQUIREMENT = "all";
        session.COOKIE_SECURE = true;
        log.LEVEL = "Warn";
        server = {
@ -54,19 +53,15 @@ in
          SENDMAIL_PATH = "${pkgs.msmtp}/bin/msmtp";
          SENDMAIL_ARGS = "--";
        };
-        cors = {
-          ENABLED = false;
-          ALLOW_DOMAIN = "https://git.berlin.ccc.de";
-          ALLOW_CREDENTIALS = true;
-        };
        metrics.ENABLED = true;
+        "cron.archive_cleanup" = {
+          ENABLED = true;
+          RUN_AT_START = true;
+          SCHEDULE = "@every 4h";
+          OLDER_THAN = "4h";
+        };
      };
      lfs.enable = true;
-      dump = {
-        enable = true;
-        type = "tar.zst";
-        interval = "daily";
-      };
      database = {
        createDatabase = false;
        host = "sql.${config.networking.domain}";
@ -78,30 +73,43 @@ in
      };
    };

-    nginx.virtualHosts."${fqdn}" = {
-      quic = true;
-      kTLS = true;
-      forceSSL = true;
-      enableACME = true;
-      extraConfig = ''
-        merge_slashes off; # Required to handle URL-encoded slashes
+    nginx = {
+      appendHttpConfig = ''
+        limit_req_zone $binary_remote_addr zone=forgejo_archive:10m rate=2r/s;
      '';
-      locations = {
-        "/" = {
-          recommendedProxySettings = true;
-          proxyWebsockets = true;
-          proxyPass = "http://unix:${config.services.forgejo.settings.server.HTTP_ADDR}";
-          extraConfig = ''
-            client_max_body_size 512M;
+      virtualHosts."${fqdn}" = {
+        quic = true;
+        kTLS = true;
+        forceSSL = true;
+        enableACME = true;
+        extraConfig = ''
+          merge_slashes off; # Required to handle URL-encoded slashes
+        '';
+        locations = {
+          "/" = {
+            recommendedProxySettings = true;
+            proxyWebsockets = true;
+            proxyPass = "http://unix:${config.services.forgejo.settings.server.HTTP_ADDR}";
+            extraConfig = ''
+              client_max_body_size 512M;
+              proxy_set_header Forwarded "$proxy_forwarded_for_element;$proxy_forwarded_by_element;host=$host;proto=$scheme";
+            '';
+          };
+          "/metrics" = {
+            recommendedProxySettings = true;
+            proxyPass = "http://unix:${config.services.forgejo.settings.server.HTTP_ADDR}";
+            extraConfig = ''
+              allow 195.160.173.14;
+              allow 2001:678:760:cccb::14;
+              deny all;
+            '';
+          };
+          "~ ^/[^/]+/[^/]+/archive/".extraConfig = ''
+            limit_req zone=forgejo_archive burst=10 nodelay;
          '';
-        };
-        "/metrics" = {
-          recommendedProxySettings = true;
-          proxyPass = "http://unix:${config.services.forgejo.settings.server.HTTP_ADDR}";
-          extraConfig = ''
-            allow 195.160.173.14;
-            allow 2001:678:760:cccb::14;
-            deny all;
+          "= /robots.txt".extraConfig = ''
+            default_type text/plain;
+            return 200 "User-agent: *\nDisallow: /*/*/archive/\n";
          '';
        };
      };
--- a/hosts/matrix/synapse.nix
+++ b/hosts/matrix/synapse.nix
@ -82,6 +82,7 @@
            client_max_body_size 64M;
            proxy_set_header X-Request-ID $request_id;
            proxy_http_version 1.1;
+            proxy_set_header Forwarded "$proxy_forwarded_for_element;$proxy_forwarded_by_element;host=$host;proto=$scheme";
          '';
        };
        "/_synapse/metrics" = {
--- a/hosts/md/hedgedoc.nix
+++ b/hosts/md/hedgedoc.nix
@ -34,12 +34,16 @@ in
          recommendedProxySettings = true;
          extraConfig = ''
            add_header Content-Security-Policy "frame-src 'self'; default-src 'self'; script-src 'self'; img-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; form-action 'self'; upgrade-insecure-requests;" always;
+            proxy_set_header Forwarded "$proxy_forwarded_for_element;$proxy_forwarded_by_element;host=$host;proto=$scheme";
          '';
        };
        "/socket.io/" = {
          proxyPass = "http://${cfg.host}:${toString cfg.port}";
          proxyWebsockets = true;
          recommendedProxySettings = true;
+          extraConfig = ''
+            proxy_set_header Forwarded "$proxy_forwarded_for_element;$proxy_forwarded_by_element;host=$host;proto=$scheme";
+          '';
        };
        "/metrics" = {
          proxyPass = "http://${cfg.host}:${toString cfg.port}";
--- a/hosts/monitoring/grafana.nix
+++ b/hosts/monitoring/grafana.nix
@ -83,11 +83,17 @@ in
          basicAuthFile = config.age.secrets.grafana_basic_auth.path;
          proxyPass = "http://unix:/run/grafana/grafana.sock";
          recommendedProxySettings = true;
+          extraConfig = ''
+            proxy_set_header Forwarded "$proxy_forwarded_for_element;$proxy_forwarded_by_element;host=$host;proto=$scheme";
+          '';
        };
        "/api/live/" = {
          proxyPass = "http://unix:/run/grafana/grafana.sock";
          recommendedProxySettings = true;
          proxyWebsockets = true;
+          extraConfig = ''
+            proxy_set_header Forwarded "$proxy_forwarded_for_element;$proxy_forwarded_by_element;host=$host;proto=$scheme";
+          '';
        };
      };
    };
--- a/hosts/monitoring/prometheus.nix
+++ b/hosts/monitoring/prometheus.nix
@ -7,7 +7,6 @@

 {
  networking.extraHosts = ''
-    2001:678:560:23:3036:24ff:fecf:965b mqtt.club.berlin.ccc.de
    2001:678:560:23:be24:11ff:fe37:a49a forgejo-runner.club.berlin.ccc.de
  '';

@ -33,7 +32,9 @@
        scrape_interval = "15s";
        static_configs = [
          {
-            targets = [ "sql.${config.networking.domain}:${toString config.services.prometheus.exporters.postgres.port}" ];
+            targets = [
+              "sql.${config.networking.domain}:${toString config.services.prometheus.exporters.postgres.port}"
+            ];
          }
        ];
      }
@ -50,7 +51,6 @@
              "www.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
              "dav.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
              "git.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
-              "mqtt.club.berlin.ccc.de:9100"
              "forgejo-runner.club.berlin.ccc.de:9100"
            ];
          }
--- a/hosts/sql/prometheus-postgres.nix
+++ b/hosts/sql/prometheus-postgres.nix
@ -7,9 +7,9 @@
    runAsLocalSuperUser = true;
    firewallRules = ''
      ip6 saddr 2001:678:760:cccb::14/128 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept comment "Allow prometheus on monitoring.berlin.ccc.de"
-      ip6 saddr 2a03:4000:3b:28b::1/128 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept comment "Allow uptime-kuma on mon.xengi.de"
+      ip6 saddr 2a0a:4cc0:c2:b028::/64 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept comment "Allow uptime-kuma on mon.xengi.de"
      ip saddr 195.160.173.14/32 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept comment "Allow prometheus on monitoring.berlin.ccc.de"
-      ip saddr 185.163.117.133/32 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept comment "Allow uptime-kuma on mon.xengi.de"
+      ip saddr 159.195.204.204/32 tcp dport ${toString config.services.prometheus.exporters.postgres.port} accept comment "Allow uptime-kuma on mon.xengi.de"
    '';
  };
 }
--- a/hosts/www/caldav-export.py
+++ b/hosts/www/caldav-export.py
@ -1,5 +1,4 @@
 import argparse
-import os
 import re
 from pathlib import Path

@ -12,7 +11,12 @@ def safe_filename(name: str) -> str:
    return re.sub(r"[^a-zA-Z0-9._-]+", "_", name)


-def export_calendar(url: str, username: str, password: str, calendar: str) -> Calendar:
+def export_calendar(
+    url: str,
+    username: str,
+    password: str,
+    calendar: str,
+) -> Calendar:
    client = DAVClient(
        url=url,
        username=username,
@ -58,13 +62,45 @@ def export_calendar(url: str, username: str, password: str, calendar: str) -> Ca

        cal = Calendar.from_ical(data)

-        for component in cal.walk():
-            if component.name != "VCALENDAR":
-                result.add_component(component)
+        # Only copy top-level components.
+        #
+        # Do NOT use cal.walk() here. walk() also returns the
+        # STANDARD/DAYLIGHT components inside VTIMEZONE.
+        for component in cal.subcomponents:
+            result.add_component(component)

    return result


+def add_normalized_components(
+    target: Calendar,
+    source: Calendar,
+    timezones: dict[str, object],
+) -> None:
+    """
+    Add components from source to target while collecting VTIMEZONE
+    components separately.
+
+    Exactly one VTIMEZONE is retained for each TZID.
+    """
+
+    for component in source.subcomponents:
+        if component.name == "VTIMEZONE":
+            tzid = component.get("TZID")
+
+            if tzid is None:
+                print("  warning: ignoring VTIMEZONE without TZID")
+                continue
+
+            tzid = str(tzid)
+
+            if tzid not in timezones:
+                timezones[tzid] = component
+
+        else:
+            target.add_component(component)
+
+
 def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("config")
@ -81,12 +117,16 @@ def main():
    output_dir = Path(args.output)
    output_dir.mkdir(parents=True, exist_ok=True)

+    username = config["username"]
+    password = config["password"]
+
    combined = Calendar()
    combined.add("prodid", "-//Combined calendar//")
    combined.add("version", "2.0")

-    username = config["username"]
-    password = config["password"]
+    # TZID -> VTIMEZONE component
+    timezones = {}
+
    for entry in config["calendars"]:
        cal = export_calendar(
            entry["url"],
@ -97,19 +137,37 @@ def main():

        filename = output_dir / f"{safe_filename(entry['calendar'])}.ics"

+        # Individual calendar export stays as-is.
        with open(filename, "wb") as f:
            f.write(cal.to_ical())

        print(f"  -> {filename}")

-        for component in cal.walk():
-            if component.name != "VCALENDAR":
-                combined.add_component(component)
+        # Normalize components for the combined calendar.
+        add_normalized_components(
+            combined,
+            cal,
+            timezones,
+        )
+
+    # Add each VTIMEZONE exactly once.
+    #
+    # Put them before events/components in the resulting VCALENDAR.
+    normalized = Calendar()
+    normalized.add("prodid", "-//Combined calendar//")
+    normalized.add("version", "2.0")
+
+    for tzid, timezone in timezones.items():
+        print(f"  timezone: {tzid}")
+        normalized.add_component(timezone)
+
+    for component in combined.subcomponents:
+        normalized.add_component(component)

    combined_file = output_dir / "all.ics"

    with open(combined_file, "wb") as f:
-        f.write(combined.to_ical())
+        f.write(normalized.to_ical())

    print()
    print(f"Combined export: {combined_file}")
--- a/hosts/www/openssh.nix
+++ b/hosts/www/openssh.nix
@ -8,8 +8,8 @@
      isSystemUser = true;
      group = "deploy";
      openssh.authorizedKeys.keys = [
-        "command=\"${pkgs.rsync}/bin/rsync --server -vlogDtpre.iLsfxCIvu . /srv/http/www/\",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM35LG+xuDaoHQ2bYD4eRc3P6Cl3JVYntoP5Gu9R+mZC deploy@www.berlin.ccc.de production"
-        "command=\"${pkgs.rsync}/bin/rsync --server -vlogDtpre.iLsfxCIvu . /srv/http/www-staging/\",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKjjLtnGf7w5D/ON+P2HpMZ5HA2fWp5YSQMGMuu5CjUt deploy@www.berlin.ccc.de staging"
+        "command=\"${pkgs.rsync}/bin/rsync --server -vlogDtpre.iLsfxCIvu . /srv/http/www/\",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPTUmrD0RMS0atfreGYjobnxBhppqo5XDEOUkLgUMWpD deploy@www.berlin.ccc.de production"
+        "command=\"${pkgs.rsync}/bin/rsync --server -vlogDtpre.iLsfxCIvu . /srv/http/www-staging/\",restrict ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIenjEIMaOclgb+vg/LZf2P/eF9J1MaAHpSlYpmtKToB deploy@www.berlin.ccc.de staging"
      ];
    };
    groups.deploy = { };
--- a/secrets/forgejo-runner-token-snowden-env.age
+++ b/secrets/forgejo-runner-token-snowden-env.age
@ -1,19 +1,21 @@
 age-encryption.org/v1
-> ssh-ed25519 uH+n1w +KvGqAaNROyE3bbbjwH5UlsC8EEuJsVkvCx58ac+sE4
-E+ZptaFcETE4hYPeHeLTN6Nv15dURwEyvpnkTAZC3y0
-> ssh-ed25519 EvLbWw 7SaKZybSLPBWWR5c7x+77I+3o2VSoAWXEkM402guJzw
-i/2BmlC/w4AwQ9FXRPkUF4IqWjDrQIYupNtwa5FTr48
-> ssh-ed25519 dM+fLQ ihhDunwhNoIzkgFu+GlyFXmpFnh7CPUdbqHZG1GrBk0
-2kuiVHb5xR2dZR2K3hnxEE27W8ZTULKSg7jI7Tq1AR0
-> ssh-ed25519 jxWM2Q v6m2gYWR59bnMT0+5bJ6is0n1v+erNJIIdReaxzdYiI
-IYL86COa5MQG9hFGadaMbr7tphadgsckYvchf3B1rVU
-> ssh-ed25519 /yCUCg z1FrBS/4kUxxOQl1BWcSQczVhrQ3r/zswGhR1MIBNWo
-8Ivi+R3gThGkSQe4QmyFIurExpRZCyDjwVNKSYe/fDo
-> ssh-ed25519 FGp51g afHNk8lS2oJWDa5YmokpE+kD30eeSZfGb8VFc0r/H2A
-pDaCkHvgFeNn3X/wDPe3yAPRV+L/g64dXEEpN+85h10
-> ssh-ed25519 b5OiTQ I9iDVd4KcKpH+nwFNeyGSMIegsux2rDzuB+LnIYCBgk
-BbptMjkV3HIisjidsfPBYLU1J1vna2QpWPiR+hkiBe0
-> ssh-ed25519 HaNCag BUbf27o6Ph2p78kpaZO7CsCtpd6wxcBIGk4kUidsrzE
-VoFu53x9AGdLDFei5C6D3Oa9UKBEdfxNh+5Nc3gGANw
--- yCHltWgEm8iJbqSTj4sd6MEkUc/KCSQCOvCqWoYgxl8
-îvSîZÂ#Ò¹à¤àFY	fÊ‹«Ê¤ý÷Àp%ÀIç†¥Š¢³“À<E2809C>îEÄÙÀSœjû8NMdm5‡šNádüD
Rø¢Ð’`d\7ñ„Æì»’
+-> ssh-ed25519 uH+n1w bp1EvLVub9v6ppLzHkYez8PgU1Mlnm/2GaYn+tVdjgE
+4bbbmYqqIkYB3uug+Y2MTlW2r7URu3QYRjfHs6JppDs
+-> ssh-ed25519 EvLbWw QWeFZk5fNASwR3iIutHhQp1zO4LFMjc4ZaiM7ch7yiU
+LI7VFU5xlzYJO7AEzxFSBJTLjG3IKy0wcRIZ4m2m5rw
+-> ssh-ed25519 dM+fLQ 1U6xgX/S5KLO5H/bgd5yQphWcUbnvOgiPrYHPMzIdHc
+Tdy1HWJsH06d+g568JG5t76Lj0VISk3XX4/Jxt2FsvY
+-> ssh-ed25519 jxWM2Q A9AVs+ovW2czIBXjVMB0urdHk66jlAXD0L3TjVmdtk4
+BR+3DIrI162I3AJv5JSYwxmANoxfqe+E54ESnAfufSA
+-> ssh-ed25519 /yCUCg C7Qv+0ZX1+G2aikHtqnw22DUB1MDXsI8tQY/JdwZ7h4
+PpVG78Ob9RNC9KF0OOtsvciwTWhpAU3lXyfsjLUTFl8
+-> ssh-ed25519 FGp51g Xx+aciETooUoFrACeRGHrJH3E7onCLIC+aKWnLiUDXI
+kdOUehdreH4UMXB2feOM1hD18uyRQ8223woqfEMkFmc
+-> ssh-ed25519 b5OiTQ 7fREKJIt+m6WDc20ouEFkLWWmcYPBVafKRHt4niu1w8
+q3Vs8wKlVLfqtPIe/dUteXxTADOOq0vcjGtIQe4PO8Q
+-> ssh-ed25519 HaNCag WZfGqjFrLiEhDPDPrQsbAkdI1A1G6mygFEMvDwJSEDI
+lhQOzJbxIrRWcmSiC9Po6M+NKFrErEB2ZHpFKyQ70U
+-> ssh-ed25519 cToZIA QqrOxRVOxcrAU4nOoNVhvn9veQk8jSeJ+uYOMlq3+Hw
+p+uPIgeZ/kegqQzufkWNk35ZfglE+63KOOYT/VDA1pw
+--- WYlIa+76j+TDCYx9LhYRfwNIw1JNLe7HxsSvEeFmDS4
+l»ÄM|æ¦íSÃ¡ë††ÖÉÙc°ûÛ|¹Ïý2Rÿ›’åd-ØŒ©¾z7ã—¥s\A"vz,.]Ü¹{A«ÚÁ–¥áüÙ©ðÒ
--- a/secrets/forgejo-runner-token-snowden.age
+++ b/secrets/forgejo-runner-token-snowden.age
--- a/secrets/id_ed25519_www-production.age
+++ b/secrets/id_ed25519_www-production.age
--- a/secrets/id_ed25519_www-staging.age
+++ b/secrets/id_ed25519_www-staging.age
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@ -24,6 +24,7 @@ let
  _monitoring = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINmRWdAUur0lb08NiB6ZWLrGmCeELRV30ElxRLfVJGPB root@monitoring";
  _dav = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICArHbX8OlNOv8HCWyyFvyi60d6MRFYe+apK0iGJ7yIM root@dav";
  _git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII93S1RUXVbH6mQksk9c5fXP8avSKXEUHBH0a7/ZbZY5 root@git";
+  _runner = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKFC7hGJC7cZ2NlrPEP9dhX1+vBNpoTFVNTKcyJX+EVP root@forgejo-runner";
 in
 {
  "matrix_admin_password.age".publicKeys = xengi;
@ -55,18 +56,38 @@ in
    _sql
    _monitoring
  ];
-  "postgres-forgejo.age".publicKeys = xengi ++ kaythxbye ++ [ _sql _git ];
+  "postgres-forgejo.age".publicKeys =
+    xengi
+    ++ kaythxbye
+    ++ [
+      _sql
+      _git
+    ];
  "postgres-extkuma.age".publicKeys = xengi ++ [ _sql ];
  "postgres-baikal.age".publicKeys = xengi ++ [
    _sql
    _dav
  ];
  "www-staging-htpasswd.age".publicKeys = xengi ++ [ _www ];
+  "id_ed25519_www-staging.age".publicKeys = xengi;
+  "id_ed25519_www-production.age".publicKeys = xengi;
  "caldav-export-config.age".publicKeys = xengi ++ [ _www ];
  "radicale_htpasswd.age".publicKeys = xengi ++ [ _dav ];
  "forgejo-internal-token.age".publicKeys = xengi ++ kaythxbye ++ [ _git ];
-  "forgejo-runner-token-snowden-env.age".publicKeys = xengi ++ kaythxbye ++ [ _git ];
-  "forgejo-runner-token-snowden.age".publicKeys = xengi ++ kaythxbye ++ [ _git ];
+  "forgejo-runner-token-snowden-env.age".publicKeys =
+    xengi
+    ++ kaythxbye
+    ++ [
+      _git
+      _runner
+    ];
+  "forgejo-runner-token-snowden.age".publicKeys =
+    xengi
+    ++ kaythxbye
+    ++ [
+      _git
+      _runner
+    ];
  "forgejo-secret-key.age".publicKeys = xengi ++ kaythxbye ++ [ _git ];
  "forgejo-oauth2-jwt-secret.age".publicKeys = xengi ++ kaythxbye ++ [ _git ];
 }
--- a/services/nginx.nix
+++ b/services/nginx.nix
@ -17,5 +17,26 @@
    recommendedOptimisation = true;
    recommendedGzipSettings = true;
    recommendedBrotliSettings = true;
+    commonHttpConfig = ''
+      map $remote_addr $proxy_forwarded_for_element {
+        # IPv4 addresses can be sent as-is
+        ~^[0-9.]+$          "for=$remote_addr:$remote_port";
+
+        # IPv6 addresses need to be bracketed and quoted
+        ~^[0-9A-Fa-f:.]+$   "for=\"[$remote_addr]:$remote_port\"";
+
+        default             "for=unknown";
+      }
+
+      map $server_addr $proxy_forwarded_by_element {
+        # IPv4 addresses can be sent as-is
+        ~^[0-9.]+$        "by=$server_addr:$server_port";
+
+        # IPv6 addresses need to be bracketed and quoted
+        ~^[0-9A-Fa-f:.]+$ "by=\"[$server_addr]:$server_port\"";
+
+        default           "by=unknown";
+      }
+    '';
  };
 }
--- a/services/prometheus-nginx.nix
+++ b/services/prometheus-nginx.nix
@ -6,9 +6,9 @@
    openFirewall = true;
    firewallRules = ''
      ip6 saddr 2001:678:760:cccb::14/128 tcp dport ${toString config.services.prometheus.exporters.nginx.port} accept comment "Allow prometheus on monitoring.berlin.ccc.de"
-      ip6 saddr 2a03:4000:3b:28b::1/128 tcp dport ${toString config.services.prometheus.exporters.nginx.port} accept comment "Allow uptime-kuma on mon.xengi.de"
+      ip6 saddr 2a0a:4cc0:c2:b028::/64 tcp dport ${toString config.services.prometheus.exporters.nginx.port} accept comment "Allow uptime-kuma on mon.xengi.de"
      ip saddr 195.160.173.14/32 tcp dport ${toString config.services.prometheus.exporters.nginx.port} accept comment "Allow prometheus on monitoring.berlin.ccc.de"
-      ip saddr 185.163.117.133/32 tcp dport ${toString config.services.prometheus.exporters.nginx.port} accept comment "Allow uptime-kuma on mon.xengi.de"
+      ip saddr 159.195.204.204/32 tcp dport ${toString config.services.prometheus.exporters.nginx.port} accept comment "Allow uptime-kuma on mon.xengi.de"
    '';
  };
 }
--- a/services/prometheus-node.nix
+++ b/services/prometheus-node.nix
@ -6,9 +6,9 @@
    openFirewall = true;
    firewallRules = ''
      ip6 saddr 2001:678:760:cccb::14/128 tcp dport ${toString config.services.prometheus.exporters.node.port} accept comment "Allow prometheus on monitoring.berlin.ccc.de"
-      ip6 saddr 2a03:4000:3b:28b::1/128 tcp dport ${toString config.services.prometheus.exporters.node.port} accept comment "Allow uptime-kuma on mon.xengi.de"
+      ip6 saddr 2a0a:4cc0:c2:b028::/64 tcp dport ${toString config.services.prometheus.exporters.node.port} accept comment "Allow uptime-kuma on mon.xengi.de"
      ip saddr 195.160.173.14/32 tcp dport ${toString config.services.prometheus.exporters.node.port} accept comment "Allow prometheus on monitoring.berlin.ccc.de"
-      ip saddr 185.163.117.133/32 tcp dport ${toString config.services.prometheus.exporters.node.port} accept comment "Allow uptime-kuma on mon.xengi.de"
+      ip saddr 159.195.204.204/32 tcp dport ${toString config.services.prometheus.exporters.node.port} accept comment "Allow uptime-kuma on mon.xengi.de"
    '';
    enabledCollectors = [ ];
    disabledCollectors = [ ];