hyperhive/nix/host-modules/hive-forge/default.nix

{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.forge;
  gatewayCfg = config.services.hyperhive.gateway;
  hyperhiveDomain = config.services.hyperhive.domain;
  swarmDomain = config.services.hyperhive.swarm.domain;
  tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
  networkCfg = config.services.hyperhive.network;

  # Forgejo's name for the login source. A constant, not an option: it
  # is the key this module's own idempotency check looks up, so making
  # it configurable would buy nothing and add a way for the lookup and
  # the row to disagree. Duplicated in ./service.nix, which builds
  # `sso.redirectUri` from it; keep the two equal.
  ssoSourceName = "authelia";

  # `url` is the half of the authelia module that exists on EVERY hive —
  # null when no SSO provider is configured anywhere, which the
  # assertion below turns into an eval failure rather than a discovery
  # request to `null/.well-known/...`.
  autheliaCfg = config.services.hyperhive.swarm.authelia;
  deployCfg = config.services.hyperhive.deploy;
  autheliaUrl = autheliaCfg.url;
  autheliaDiscoveryUrl = "${toString autheliaUrl}/.well-known/openid-configuration";

  # The forge's OIDC client secret, in a host tmpfs directory bound read-only
  # into the container at the same path: `swarm-bao-forge-oidc` writes it,
  # `forgejo-sso-source` loads it as a credential. After a reboot it is absent
  # until the fetch succeeds; logins still work then, because forgejo keeps
  # the secret in its login-source row.
  oidcSecretDir = "/run/hive-forge-oidc";
  oidcSecretPath = "${oidcSecretDir}/${cfg.sso.clientId}.secret";

  # Where the fetched secret used to be written, on the container's persistent
  # root. Removed by the tmpfiles rule below.
  oldOidcSecretPath = "/var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/${cfg.sso.clientId}.secret";

  # Where swarm-secret-publish in authelia's container leaves this client's
  # secret.
  # The forge publisher's store role grants read on this path.
  oidcStorePath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";

  # The swarm-controller's forge account. Same name as
  # `swarm-controller.nix`'s existing `queueClientId` — the account is
  # provisioned to *match* that identity, not invented independently —
  # "swarm controller having one identity with stuff derived from it is
  # the right shape" was the swarm-level design call this account
  # follows. A constant, not an option:
  # nothing here makes the name configurable without also updating
  # `swarm-controller.nix`'s own constant, so a shared option would
  # invite the two to drift rather than prevent it.
  swarmControllerForgeUser = "swarm-controller";

  # The mint's record that it has minted and published the controller's
  # token: the token's name, not its value. Inside the forge container, under
  # forgejo's own state dir, so it survives a reboot.
  swarmControllerTokenNamePath = "/var/lib/forgejo/swarm-controller-token-name";

  # Where the minted token itself used to be written. The mint removes it once
  # it has published a token.
  oldSwarmControllerTokenPath = "/var/lib/forgejo/swarm-controller-token";

  # Where the mint puts the token: `controllerForgeTokenLeaf` in
  # ../swarm-bao.nix, under the KV mount.
  controllerTokenStorePath = "secret/swarm/controller/swarm-controller/forge/token";

  baoCfg = config.services.hyperhive.swarm.bao;
  storeRetry = import ../lib/store-retry.nix { };
  storeRefreshTimer = import ../lib/store-refresh-timer.nix { };
  atomicWriteSecret = import ../lib/atomic-write-secret.nix { };
  refreshConsumer = import ../lib/refresh-consumer.nix { };
  havePublisherIdentity =
    deployCfg.bao.forgePublisherClientCertFile != null
    && deployCfg.bao.forgePublisherClientKeyFile != null;

  caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };

  # The container's journal forwarder pushes to its own receiver on the swarm
  # collector, the route `serviceProducers.hive-forge` in
  # ../swarm-otel-service.nix declares, as a client of its own rather than the
  # login source's. The port suffix is elided on 443, as in ../swarm-bao.nix.
  forwarderClientId = cfg.otel.clientId;
  gatewayPortSuffix = lib.optionalString (
    gatewayCfg.httpsPort != 443
  ) ":${toString gatewayCfg.httpsPort}";
  forwarderEndpoint = "https://${config.services.hyperhive.swarm.otel.domain}${gatewayPortSuffix}/hive-forge";
  forwarderUnit = "swarm-journal-forwarder-oidc";
  # In the host netns: distinct from every other collector's there, which the
  # assertion below checks.
  forwarderTelemetryPort = 8898;

  # The forge host's store identity, handed into the container as system
  # credentials (credential id → host source) for the forwarder's fetch unit
  # and the controller-token mint. `swarm-bao-forge-oidc` on the host keeps
  # reading the same files directly.
  storeFiles = lib.filterAttrs (_: source: source != null) {
    forge-bao-cert = deployCfg.bao.forgePublisherClientCertFile;
    forge-bao-key = deployCfg.bao.forgePublisherClientKeyFile;
    forge-bao-server-ca = deployCfg.bao.serverCaFile;
  };
  # nixos-container expands `extraFlags` unquoted, so a source path must be
  # one word.
  oneWord = path: builtins.match ".*[[:space:]].*" path == null;
  forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
  mintUnit = "forgejo-swarm-controller-account";
  mintCredential = id: "/run/credentials/${mintUnit}.service/${id}";

  # ROOT_URL forgejo advertises in clone links + outbound URLs.
  # `cfg.domain` doubles as both the forgejo `DOMAIN` setting AND the
  # gateway vhost server-name, so ROOT_URL just uses it directly. The
  # gateway always terminates TLS (self-signed is the implicit floor when
  # neither `tls.certDir` nor ACME is configured), so the forge is always
  # advertised over `https` on `httpsPort` — the canonical 443 elides the
  # port suffix. Operators can still override via `cfg.rootUrl` for
  # bespoke shapes. Both bindings are duplicated in ./service.nix, which
  # builds `sso.redirectUri` from them; keep the two equal.
  defaultRootUrl =
    let
      portSuffix = if gatewayCfg.httpsPort == 443 then "" else ":${toString gatewayCfg.httpsPort}";
    in
    "https://${cfg.domain}${portSuffix}/";
  effectiveRootUrl = if cfg.rootUrl != null then cfg.rootUrl else defaultRootUrl;

  # `actionsFromSelf` resolves `uses:` against this forge, which then has to
  # hold every action a workflow in `.forgejo/workflows/` names.
  actionsFromSelf = deployCfg.forgejo.actionsFromSelf;
  actionMirrors =
    map
      (name: {
        upstream = "https://github.com/actions/${name}";
        dest = "actions/${name}";
      })
      [
        "checkout"
        "upload-artifact"
      ];
  # Minus any dest the operator already declared, which would duplicate it.
  effectiveMirrors =
    deployCfg.forgejo.mirrors
    ++ lib.optionals actionsFromSelf (
      lib.filter (a: !(lib.any (m: m.dest == a.dest) deployCfg.forgejo.mirrors)) actionMirrors
    );

  # Share host netns — forgejo's HTTP / SSH listeners then look
  # exactly like a host-side service, no port forwarding dance,
  # and agent containers (which also share host netns) reach it
  # via plain `localhost`.
  privateNetwork = false;
in
{
  # Private Forgejo in a `hive-forge` nixos-container, shared host
  # netns, on the one host with `deploy.forgejo.enable`. Agents reach it
  # at `forge.<domain>` via the gateway. State at
  # `/var/lib/nixos-containers/hive-forge/var/lib/forgejo/` survives
  # restart. See `docs/networking/gateway.md::hive-forge container shape`.

  imports = [
    ./service.nix
    (lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "forgejo" "sso" "clientSecretFile" ] ''
      The forge's OIDC client secret is fetched from the swarm secret store
      (swarm-bao-forge-oidc) into a fixed runtime path. Remove this
      definition.
    '')
    (lib.mkRemovedOptionModule
      [ "services" "hyperhive" "deploy" "forgejo" "hostSwarmControllerTokenFile" ]
      ''
        The forge publishes the swarm-controller's forge token to the swarm
        secret store (forgejo-swarm-controller-account), and the controller
        reads it there. Remove this definition; the file it named is read by
        nothing now and can be deleted.
      ''
    )
  ];

  # What ./service.nix declares is what the forge IS from any hive's point of
  # view: the names and ports it answers on, the URLs it advertises, and the
  # client id it is registered under. What lives here is what the host running
  # it decides — which build it runs, how it is served, what it mirrors, and
  # where its host-local secrets sit. Same rule as ../swarm-victorialogs.nix,
  # and the renames are in ../deploy.nix with the rest.
  options.services.hyperhive.deploy.forgejo = {
    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.forgejo;
      defaultText = lib.literalExpression "pkgs.forgejo";
      description = ''
        Forgejo package to run inside the container. Defaults to
        `pkgs.forgejo` (the latest release line) rather than the
        nixpkgs-module default of `pkgs.forgejo-lts`, because LTS
        lags far behind on schema and the DB easily ends up "newer
        than the binary" if the operator ever ran a non-LTS forgejo
        against the same state dir. Override to `pkgs.forgejo-lts`
        if you actively want the slower release train.
      '';
    };

    openFirewall = lib.mkOption {
      type = lib.types.bool;
      default = false;
      example = true;
      description = ''
        Open `httpPort` + `sshPort` in the host firewall. Off by
        default (secure-by-default): agent containers reach the forge
        at `forge.<domain>` via the gateway (not directly), and the
        host reaches it on loopback — so the firewall opens only
        matter for access from outside the host. Flip to `true` when
        you want the operator's browser or external git clients to
        hit the forge directly.
      '';
    };

    mirrors = lib.mkOption {
      type = lib.types.listOf (
        lib.types.submodule {
          options = {
            upstream = lib.mkOption {
              type = lib.types.str;
              example = "https://github.com/actions/checkout";
              description = "Upstream clone URL to mirror from.";
            };
            dest = lib.mkOption {
              type = lib.types.str;
              example = "actions/checkout";
              description = ''
                Local `<owner>/<repo>` the pull-mirror is created at. The
                `<owner>` org is auto-created if missing. Keep mirror dests
                in their own orgs (e.g. `actions/*`) — separate from the
                hive-c0re-managed namespaces (config/shared/agents/core) so
                the seed never collides with core's own provisioning.
              '';
            };
          };
        }
      );
      default = [ ];
      example = lib.literalExpression ''
        [ { upstream = "https://github.com/actions/checkout"; dest = "actions/checkout"; } ]
      '';
      description = ''
        General-purpose Forgejo **pull-mirrors** to auto-seed on the local
        forge. Each entry is created as a real Forgejo pull-mirror (it
        re-syncs from `upstream` out-of-band), not a one-off pushed clone —
        so a host-resolver blip leaves a *stale* mirror, never a hard
        failure on whatever reads it.

        With {option}`services.hyperhive.deploy.forgejo.actionsFromSelf`
        enabled (the default on a host that runs the swarm-controller),
        `actions/checkout` and `actions/upload-artifact` mirrors are appended
        to this list.
      '';
    };

    actionsFromSelf = lib.mkOption {
      type = lib.types.bool;
      default = false;
      example = false;
      description = ''
        Resolve a workflow's `uses: <org>/<action>@vN` against this forge
        instead of `data.forgejo.org`, and add `actions/checkout` and
        `actions/upload-artifact` pull-mirrors to
        {option}`services.hyperhive.deploy.forgejo.mirrors` so those steps
        have something to resolve to. A CI job's actions then need no
        external DNS; only the mirrors' re-sync does.

        Defaults to on when this host also runs the swarm-controller, which
        seeds the mirrors (set by `glue-forge-controller.nix`), and off
        otherwise. Setting it `true` on a host without
        the controller leaves both actions unresolved: nothing seeds the
        mirrors, and the warning about unseeded mirrors fires.

        Any other action a workflow names has to be mirrored here too. Set to
        `false` to resolve actions from `data.forgejo.org` and seed no mirror.
      '';
    };

    effectiveMirrors = lib.mkOption {
      type = lib.types.listOf lib.types.attrs;
      internal = true;
      readOnly = true;
      description = ''
        {option}`mirrors` plus the `actions/*` mirrors {option}`actionsFromSelf`
        adds; what `glue-forge-controller.nix` forwards to the controller.
      '';
    };
  };

  config = lib.mkIf deployCfg.forgejo.enable {
    # This service's own gateway surface: the vhost that fronts it and
    # the name the hive resolver answers for. Declared here rather than
    # in the gateway so the forge's public face lives with the forge —
    # the gateway adds `listen`, the certificate and the security headers
    # (`gateway.vhosts`) and never needs to know this service by name.
    services.hyperhive.gateway.localNames = [ cfg.domain ];
    services.hyperhive.gateway.enable = lib.mkDefault true;

    # The forge container resolves the rest of the hive through dnsmasq.
    services.hyperhive.gateway.dns.enable = lib.mkDefault true;

    # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for
    # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB
    # clones). SSH stays direct on `forge.sshPort`. See
    # `docs/networking/gateway.md`.
    services.hyperhive.gateway.vhosts = {
      "${cfg.domain}" = {
        locations = {
          "/" = {
            proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/";
            proxyWebsockets = true;
            extraConfig = ''
              proxy_buffering off;
              client_max_body_size 1G;
              proxy_read_timeout 1h;
              proxy_send_timeout 1h;
            '';
          };

          # ⚠️ EXACT match, and that is what makes this safe. Forgejo
          # serves `/metrics` on the same listener the `/` prefix above
          # already proxies, so without a more specific location the
          # endpoint would ride that catch-all to anyone. `= /metrics`
          # outranks the `/` prefix in nginx, so this location — and its
          # auth — is the one that runs.
          #
          # The collector requests exactly this URL, declared as
          # `publishedScrapeTargets.forgejo` in ../swarm-otel-service.nix;
          # the two must agree, since that string is also the token's audience.
          "= /metrics" = {
            proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/metrics";
            extraConfig = ''
              auth_request /__forge_metrics_authz;
            '';
          };

          # ⚠️ Forgejo strips trailing slashes before routing, so `/metrics/`
          # reaches the same unauthenticated handler — and the exact match
          # above does not take it, leaving it to the `/` catch-all. nginx
          # merges slashes and decodes `%2F` before matching, so this one
          # location also takes `/metrics//` and `/metrics%2F`.
          "= /metrics/".return = "404";

          # The subrequest. Same implementation and header set as
          # `swarm-ui.nix` uses, for the same reason: `X-Original-URL`
          # and `X-Original-Method` are what authelia's `auth-request`
          # implementation reads.
          #
          # ⚠️ NO `error_page 401 =302` here, and its absence is the
          # whole point. The swarm UI redirects an unauthenticated
          # browser to a login page; a scraper handed that 302 would
          # follow it and parse an HTML page as metrics. A machine-facing
          # location lets the 401 reach the client unchanged.
          "= /__forge_metrics_authz" = {
            # By NAME, not `127.0.0.1`: this gateway and authelia are not
            # required to share a host, so the loopback literal encoded a
            # co-location nobody had agreed to. The name resolves here
            # today and resolves off-host later without this line moving.
            proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request";
            # nixpkgs appends its OWN `Host $host` after extraConfig,
            # which would override verifiedProxyTo's — see the comment
            # on verifiedProxyTo in hive-gateway/vhost-lib.nix.
            recommendedProxySettings = false;
            extraConfig = ''
              internal;
              ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain}
              proxy_pass_request_body off;
              proxy_set_header Content-Length "";
              proxy_set_header X-Original-Method $request_method;
              proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
              proxy_set_header X-Forwarded-Proto $scheme;
              proxy_set_header X-Forwarded-Host $http_host;
              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            '';
          };
        };
      };
    };

    assertions = [
      {
        assertion = cfg.rootUrl == null || lib.hasSuffix "/" cfg.rootUrl;
        message = ''
          services.hyperhive.swarm.forge.rootUrl must end with "/". forgejo's
          ROOT_URL contract requires a trailing slash for correct
          relative-link generation; without it forgejo emits URLs like
          `https://forge.example.com.user.id` instead of
          `https://forge.example.com/user.id`. Got: ${toString cfg.rootUrl}
        '';
      }
      {
        # `cfg.domain` can't be empty — would render `.<hive>` shaped
        # garbage as both server_name (nginx wildcard catch-all) and
        # /etc/hosts entry (invalid). The default derives a non-empty
        # `forge.<domain>`, but an operator-set empty string should fail
        # loud.
        assertion = cfg.domain != "";
        message = ''
          services.hyperhive.swarm.forge.domain = "" is rejected. The
          rendered URLs would be invalid (nginx wildcard catch-all
          for an empty server_name, /etc/hosts rejects empty entries).
          Either leave at default (auto-derives to
          "forge.<services.hyperhive.domain>"), or set a non-empty
          hostname like "forge.example.com" or "git.internal".
        '';
      }
      {
        # Each mirror dest must be exactly `<owner>/<repo>` — the seed
        # splits on the single slash to create the org + repo.
        assertion = lib.all (m: lib.length (lib.splitString "/" m.dest) == 2) effectiveMirrors;
        message = ''
          Every services.hyperhive.deploy.forgejo.mirrors[].dest must be exactly
          "<owner>/<repo>" (one slash). Got: ${lib.concatMapStringsSep ", " (m: m.dest) effectiveMirrors}
        '';
      }
      {
        # Keep mirror orgs out of the hyperhive-managed namespaces
        # (config/shared/agents/core) so the seed never races / collides
        # with the provisioning of those orgs.
        assertion = lib.all (
          m:
          !(lib.elem (builtins.elemAt (lib.splitString "/" m.dest) 0) [
            "config"
            "shared"
            "agents"
            "core"
          ])
        ) effectiveMirrors;
        message = ''
          services.hyperhive.deploy.forgejo.mirrors[].dest must not place a mirror
          in a hyperhive-managed org (config / shared / agents / core) —
          those are provisioned by hyperhive itself and a mirror there would
          collide. Use a dedicated org (e.g. "actions/checkout").
        '';
      }
      {
        assertion = lib.all oneWord (lib.attrValues storeFiles);
        message = ''
          The forge host's store identity paths must not contain whitespace:
          nspawn's --load-credential receives them through nixos-container's
          unquoted extra flags.
        '';
      }
      {
        assertion =
          (config.services.hyperhive.swarm.otel.serviceProducers.hive-forge.clientId or null)
          == forwarderClientId;
        message = ''
          services.hyperhive.swarm.otel.serviceProducers has no entry
          "hive-forge" with clientId "${forwarderClientId}", so the swarm
          collector has no receiver for the forge's journal forwarder.
          Declare it in swarm-otel-service.nix.
        '';
      }
      {
        assertion =
          !(lib.elem forwarderTelemetryPort [
            config.services.hyperhive.otel.telemetryPort
            config.services.hyperhive.swarm.otel.telemetryPort
            config.services.hyperhive.swarm.bao.otel.telemetryPort
            config.services.hyperhive.swarm.nats.otel.telemetryPort
          ]);
        message = ''
          The forge's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
          collides with another collector's in the host netns: the hive
          collector's, the swarm collector's, the store forwarder's or the
          queue forwarder's.
        '';
      }
    ];

    # `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
    # in self-signed mode (see the hive-ca-trust helper), so the CA bind
    # source exists before nspawn sets the mount up. nspawn also fails to
    # start it when a credential source is missing, so where this host mints
    # the leaf the container pulls the minting unit in and waits for it, as
    # ../swarm-builder.nix's does.
    systemd.services."container@hive-forge" = lib.mkMerge [
      caTrust.containerOrdering
      {
        after = [ "swarm-bao-pki.service" ];
        wants = [ "swarm-bao-pki.service" ];
      }
    ];

    containers.hive-forge = {
      autoStart = true;
      ephemeral = false;
      # No `--link-journal=host`: the journal stays in the guest, where the
      # container's own forwarder reads it (../../container-modules/swarm-journal-forwarder.nix).
      extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
      inherit privateNetwork;
      # Self-signed mode: bind the hive's trust bundle read-only so forgejo
      # can trust the gateway's self-signed leaf for outbound webhook
      # delivery. The bind-mount, the `container@` ordering and the bundle
      # the container's consumers read all come from the hive-ca-trust
      # helper — see the container's `imports` for the consumption half.
      #
      # The OIDC secret's directory: a container's `/run` is an nspawn tmpfs
      # the host cannot see into. The host directory comes from the tmpfiles
      # rule below, so it exists before the container starts.
      bindMounts = caTrust.bindMount // {
        ${oidcSecretDir} = {
          hostPath = oidcSecretDir;
          isReadOnly = true;
        };
      };
      config =
        { pkgs, ... }:
        let
          # Build a custom static-root that is the standard forgejo data
          # output with our theme CSS added. Using STATIC_ROOT_PATH instead
          # of tmpfiles / bind-mounts means the theme is always present in
          # the nix store — no separate hive-forge container rebuild needed,
          # and no persistent-state directory involved.
          staticRootWithTheme = pkgs.runCommand "forgejo-static-with-theme" { } ''
            cp -r --no-preserve=mode,ownership ${deployCfg.forgejo.package.data}/. $out/
            mkdir -p $out/public/assets/css
            cp ${./theme-catppuccin-vibec0re.css} \
               $out/public/assets/css/theme-catppuccin-vibec0re.css
            # Replace the default Forgejo logo + favicon with the hyperhive
            # mark. Files in public/assets/img/ are served before built-ins.
            mkdir -p $out/public/assets/img
            cp ${../../../branding/hyperhive.svg} $out/public/assets/img/logo.svg
            cp ${../../../branding/hyperhive.svg} $out/public/assets/img/favicon.svg
            cp ${../../../branding/hyperhive.png} $out/public/assets/img/logo.png
            cp ${../../../branding/hyperhive.png} $out/public/assets/img/favicon.png
            cp ${../../../branding/hyperhive.png} $out/public/assets/img/avatar_default.png
          '';
        in
        {
          imports = [
            ./../../container-modules/swarm-container.nix
            ./../../container-modules/swarm-journal-forwarder.nix
            # Ask the hive's dnsmasq, not the host's resolvers. The swarm
            # names — the controller's webhook endpoint, the SSO issuer —
            # resolve to the bridge and have no public records, so a
            # container using the host's resolvers gets "no such host" for
            # every one of them while the same name resolves fine one
            # container over.
            #
            # Every unit that makes an outbound call is a consumer, for the
            # same reason the trust bundle below names them: an unordered
            # resolver write is a race that only shows up on a cold boot. The
            # forwarder and its secret fetch dial the swarm collector, authelia
            # and the store by name, and the controller-token mint the store.
            (import ./../../container-modules/swarm-container-resolver.nix {
              inherit (networkCfg) bridgeIp;
              dnsConsumers = [
                "forgejo.service"
                "forgejo-sso-source.service"
                "opentelemetry-collector.service"
                "${forwarderUnit}.service"
                "${mintUnit}.service"
              ];
            })

            # Forgejo is Go, and `SSL_CERT_FILE` *replaces* the default store
            # rather than adding to it — so it needs the system CAs and the
            # store's root concatenated, not the root alone, or public mirror
            # fetches lose every anchor they had.
            #
            # EVERY unit that makes an outbound HTTPS call is a consumer, not
            # just the obvious one: `forgejo-sso-source` fetches the issuer's
            # `.well-known/openid-configuration` under the store's root, and it
            # once shipped without the trust its sibling had. The collector is
            # Go too, for the swarm collector's and authelia's names.
            (caTrust.trustBundle {
              inherit pkgs;
              name = "hive-forge";
              consumers = [
                "forgejo"
                "forgejo-sso-source"
                "opentelemetry-collector"
              ];
            })
          ];

          system.stateVersion = "25.11";
          services.hyperhive.swarmContainer = { inherit privateNetwork; };

          # Authenticated as the forwarder's own OIDC client, with the forge
          # host's store identity: one identity for the whole container. It is
          # a system credential, so the fetch unit loads it.
          services.hyperhive.swarmJournalForwarder = {
            enable = true;
            serviceName = "forgejo";
            endpoint = forwarderEndpoint;
            clientId = forwarderClientId;
            tokenUrl = "${toString autheliaUrl}/api/oidc/token";
            telemetryPort = forwarderTelemetryPort;
            store = {
              package = deployCfg.bao.package;
              addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
              secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
              clientCertFile = forwarderCredential "forge-bao-cert";
              clientKeyFile = forwarderCredential "forge-bao-key";
              caFile =
                if storeFiles ? forge-bao-server-ca then forwarderCredential "forge-bao-server-ca" else null;
            };
          };
          systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;

          services.forgejo = {
            enable = true;
            package = deployCfg.forgejo.package;
            database.type = "sqlite3";
            lfs.enable = true;
            settings = {
              DEFAULT.APP_NAME = "HyperHive";
              server = {
                DOMAIN = cfg.domain;
                ROOT_URL = effectiveRootUrl;
                HTTP_PORT = cfg.httpPort;
                START_SSH_SERVER = true;
                SSH_PORT = cfg.sshPort;
                SSH_LISTEN_PORT = cfg.sshPort;
                BUILTIN_SSH_SERVER_USER = "git";
                DISABLE_SSH = false;
                # Point forgejo at our extended static root that includes
                # the custom theme CSS baked straight into the nix store.
                STATIC_ROOT_PATH = staticRootWithTheme;
              };
              # Local sign-up off. An agent's account is made by
              # swarm-controller over the admin API, a human's by their
              # first authelia login (`oauth2_client` below), which this
              # does not block: forgejo's auto-registration checks only
              # `ALLOW_ONLY_INTERNAL_REGISTRATION`
              # (`routers/web/auth/oauth.go`, 16.0.5).
              service = {
                DISABLE_REGISTRATION = true;
                REQUIRE_SIGNIN_VIEW = false;
              };
              # A first login through the authelia source creates the
              # account, named after authelia's `preferred_username` — the
              # authelia username, not the opaque `sub`.
              #
              # ⚠️ `ACCOUNT_LINKING = login`, never `auto`. A login whose
              # name is already taken does not create an account; `auto`
              # would then hand the SSO user the existing one. Agents,
              # `core` and `swarm-controller` all have local accounts here,
              # the last two site admins, and `swarmctl user add` refuses
              # none of those names. `login` asks for the existing
              # account's own password instead, which an agent's or a
              # service account's never leaves the forge.
              oauth2_client = {
                ENABLE_AUTO_REGISTRATION = true;
                USERNAME = "preferred_username";
                ACCOUNT_LINKING = "login";
              };
              repository = {
                DEFAULT_BRANCH = "main";
                DEFAULT_PRIVATE = "private";
              };
              # Not an option: a swarm-integrated, auto-deployed forge
              # always has metrics, behind the protected `= /metrics`
              # location on its gateway vhost.
              #
              # No `TOKEN` here on purpose. Forgejo can guard this itself
              # with a static bearer, but the swarm authenticates the
              # scraper at the gateway, so a second credential system per
              # service would buy nothing and would be the one that stops
              # getting rotated.
              metrics.ENABLED = true;
              # The two per-dimension breakdowns, alongside the endpoint
              # itself: `gitea_issues_by_label{label=…}` and
              # `gitea_issues_by_repository{repository=…}`. Off by default
              # upstream because they are the only metrics here whose series
              # count grows with the CONTENT of the forge rather than with
              # its feature set — so the question is cardinality, and it was
              # measured rather than assumed.
              #
              # Measured on this instance: ~19 distinct labels and ~29 repos
              # ⇒ **well under 100 series**, against a store that took +2234
              # from nine host scrapers. Two orders of magnitude below the
              # cost that already argued against shipping a `process`
              # scraper, so the breakdowns are affordable here.
              #
              # ⚠️ Both grow with repo and label count, not with traffic. A
              # forge that grew to thousands of repos would want this
              # revisited — that is a real trigger, unlike a time-based one:
              # `count(gitea_issues_by_repository)` answers it directly.
              metrics.ENABLED_ISSUE_BY_LABEL = true;
              metrics.ENABLED_ISSUE_BY_REPOSITORY = true;
              # Repo migrations / pull-mirrors fetch from the source
              # URL *inside* Forgejo. hyperhive code is synced from
              # `localhost` (and the host LAN), which Forgejo's
              # migration guard blocks by default ("cannot import from
              # disallowed hosts"). Allow loopback + RFC-1918 sources
              # so an in-hive mirror of the hyperhive repo works.
              migrations.ALLOW_LOCALNETWORKS = true;
              # Forgejo's docs say an empty `ALLOWED_DOMAINS` allows
              # every domain, but that's not true in practice on the
              # versions we've hit this on — the migration guard still
              # rejects genuinely public hosts ("cannot import from
              # disallowed hosts") unless the wildcard is set
              # explicitly (a known upstream doc/behavior mismatch,
              # tracked upstream in the go-gitea project). Public-domain
              # pull/push mirrors (agents' personal repos synced to
              # forge.darkest.space, etc.) were failing every sync
              # attempt without this.
              migrations.ALLOWED_DOMAINS = "*";
              # Forgejo's webhook SSRF allow-list, a strict whitelist.
              # `external` (public unicast only) stays, or user-repo webhooks to
              # github/slack break; both hyperhive hosts are added because their
              # webhooks target a private gateway IP `external` denies. The
              # swarm's host was the missing one, and every swarm-controller
              # delivery was refused for it.
              #
              # Not gated on `deploy.swarm-controller.enable`: in a spread deployment the
              # forge and the swarm host are different machines, so that
              # per-host flag is false exactly where this entry is needed.
              webhook.ALLOWED_HOST_LIST = lib.concatStringsSep "," (
                lib.unique (
                  [
                    "external"
                    hyperhiveDomain
                  ]
                  ++ lib.optional (swarmDomain != null) swarmDomain
                )
              );
              log.LEVEL = "Warn";
              # Pinned explicitly rather than left to upstream's default
              # (currently `bleve`, a separate full-text index Forgejo
              # builds/maintains itself). `db` searches the database
              # directly instead — no second store that can silently
              # disagree with it. Found via a false-positive search bug:
              # `hive-forge list --search` returned matches with zero
              # literal occurrences of the search term, consistent with a
              # bleve index that drifted stale against the DB with
              # nothing to detect the drift. At this repo's scale (~3.4k
              # issues+PRs) a DB scan costs nothing worth noticing, and
              # it's the same substring semantics `--search`'s own doc
              # already promises — bleve's relevance ranking buys nothing
              # we use here. Config-only (part of the forgejo unit, so a
              # rebuild restarts it for free); the old bleve index
              # directory is left on disk, unused, rather than risk
              # deleting the wrong thing.
              indexer.ISSUE_INDEXER_TYPE = "db";
              ui = {
                DEFAULT_THEME = "catppuccin-vibec0re";
                THEMES = "catppuccin-vibec0re,forgejo-auto,forgejo-light,forgejo-dark,gitea-auto,gitea-light,gitea-dark";
              };
              # Point forgejo at the GPG key generated by the
              # forgejo-gpg-init service below. SIGNING_KEY = "default"
              # resolves via the forgejo process's git config
              # (`user.signingkey`) — which forgejo-gpg-init sets to the
              # generated key — not by scanning GNUPGHOME. GNUPGHOME is
              # the keyring forgejo signs from; must be absolute +
              # writeable by the forgejo user.
              "repository.signing" = {
                SIGNING_KEY = "default";
                GNUPGHOME = "/var/lib/forgejo/.gnupg";
              };
              # Enable Forgejo Actions so the runner registration token
              # API endpoint is available. Without this the endpoint
              # returns "runner registration token not found" regardless
              # of token scopes. swarm-controller's `forge::runner_token`
              # reads it.
              actions.ENABLED = true;
              # With `actionsFromSelf`, resolve `uses: <org>/<action>@vN` from
              # THIS instance (the seeded `actions/*` pull-mirrors)
              # instead of the upstream default `data.forgejo.org` — keeps
              # the checkout step on loopback, immune to a host-resolver
              # blip. `self` = forgejo expands actions against its
              # own ROOT_URL.
              actions.DEFAULT_ACTIONS_URL = lib.mkIf actionsFromSelf "self";
              # F3 (federation) computes its data dir relative to the
              # forgejo binary, which lands in the read-only nix
              # store and crashes anything that touches the F3
              # subsystem — including `forgejo admin user create`,
              # which init-ses F3 even when ENABLED=false. Pin the
              # path absolute alongside the disable so the init
              # resolution succeeds before the flag is checked.
              "F3" = {
                ENABLED = false;
                PATH = "/var/lib/forgejo/data/f3";
              };
            };
          };
          environment.systemPackages = [
            pkgs.forgejo
            pkgs.gnupg
          ];

          # Forgejo's local Actions-artifact storage defaults to
          # `{APP_DATA_PATH}/actions_artifacts` (=
          # `/var/lib/forgejo/data/actions_artifacts`), but Forgejo does not
          # pre-create that directory. The artifact endpoint ingests the
          # chunked upload, then the merge-chunks step does an `lstat` on a
          # tmp dir under it and fails:
          #   Error merge chunks: lstat
          #   /var/lib/forgejo/data/actions_artifacts/tmpNNN: no such file or
          #   directory
          # so every `upload-artifact` step dies after the build succeeds.
          # Pre-create the dir (forgejo-owned) so uploads actually persist.
          # `actions.ENABLED = true` registers the endpoints; this gives them
          # somewhere to write.
          systemd.tmpfiles.rules = [
            "d /var/lib/forgejo/data 0750 forgejo forgejo - -"
            "d /var/lib/forgejo/data/actions_artifacts 0750 forgejo forgejo - -"
          ];

          # Ensure Forgejo has a usable GPG signing key so UI merges / CRUD
          # commits are signed instead of erroring "does not have a signing
          # key". This service (a) generates a key in forgejo's persistent
          # keyring iff one isn't already present — keyed on the actual
          # secret key, NOT a stamp file, so a partial state wipe that loses
          # the key still regenerates it — and (b) points the forgejo user's
          # git config at it (`user.signingkey` + commit/tag gpgsign), which
          # is how `SIGNING_KEY = "default"` actually resolves. Runs as the
          # forgejo user before forgejo on each start; idempotent (the keygen
          # is guarded, the git-config is a cheap re-set).
          systemd.services.forgejo-gpg-init = {
            description = "ensure Forgejo's GPG signing key + git signing config";
            # Start before forgejo so the key + signing config are ready when
            # forgejo reads repository.signing on startup.
            wantedBy = [ "forgejo.service" ];
            before = [ "forgejo.service" ];
            serviceConfig = {
              Type = "oneshot";
              User = "forgejo";
              Group = "forgejo";
              # Pin the journal identity (else it's the `script` store-path wrapper).
              SyslogIdentifier = "forgejo-gpg-init";
            };
            # GNUPGHOME = the keyring forgejo signs from; HOME so
            # `git config --global` lands where the forgejo process reads it.
            environment = {
              GNUPGHOME = "/var/lib/forgejo/.gnupg";
              HOME = "/var/lib/forgejo";
            };
            path = [
              pkgs.gnupg
              pkgs.git
              pkgs.gnugrep
              pkgs.gawk
              pkgs.coreutils
            ];
            script = ''
              set -euo pipefail
              mkdir -p "$GNUPGHOME"
              chmod 700 "$GNUPGHOME"

              # Generate only if no secret key is present (key-based guard,
              # not a stamp — a stamp can outlive the key after a state wipe
              # and wrongly suppress regeneration).
              if ! gpg --list-secret-keys --with-colons 2>/dev/null | grep -q '^sec:'; then
                printf '%s\n' \
                  '%no-protection' \
                  'Key-Type: RSA' \
                  'Key-Length: 4096' \
                  'Name-Real: HyperHive Forge' \
                  'Name-Email: forgejo@hive' \
                  'Expire-Date: 0' \
                  | gpg --batch --gen-key
              fi

              # Point git (hence Forgejo's SIGNING_KEY="default") at the key.
              KEYID=$(gpg --list-secret-keys --keyid-format long --with-colons \
                | awk -F: '/^sec:/ { print $5; exit }')
              if [ -n "$KEYID" ]; then
                git config --global user.signingkey "$KEYID"
                git config --global commit.gpgsign true
                git config --global tag.gpgsign true
              fi
            '';
          };

          # Register authelia as an OIDC login source.
          #
          # ⚠️ Ordered AFTER forgejo, unlike forgejo-gpg-init above, and
          # the difference is not stylistic: a login source is a row in
          # forgejo's database, and on a fresh hive that database does
          # not exist until forgejo has started and run its migrations.
          # Running first would either fail or let the CLI initialise a
          # schema behind the server's back.
          #
          # Idempotency is by QUERY, not by a stamp file — same reason
          # spelled out for the GPG key above: a stamp survives a state
          # wipe that took the thing it claims exists, and then suppresses
          # the repair.
          systemd.services.forgejo-sso-source = {
            description = "register authelia as Forgejo's OIDC login source";
            after = [ "forgejo.service" ];
            requires = [ "forgejo.service" ];
            wantedBy = [ "multi-user.target" ];
            serviceConfig = {
              Type = "oneshot";
              RemainAfterExit = true;
              User = "forgejo";
              Group = "forgejo";
              SyslogIdentifier = "forgejo-sso-source";
              # Absent until `swarm-bao-forge-oidc` on the host has fetched
              # it, and this unit fails to start until then; that unit starts
              # it again once the file is written.
              LoadCredential = [ "oidc-client-secret:${oidcSecretPath}" ];
            };
            # `FORGEJO_CUSTOM` (the forgejo-prefixed name, not `GITEA_CUSTOM`)
            # is how the CLI finds the app.ini upstream's module wrote.
            #
            # This unit is also a trust-bundle consumer (declared in the
            # container's `imports` above, which is where `SSL_CERT_FILE`
            # comes from): registering the login source makes an **outbound
            # HTTPS call** to `<issuer>/.well-known/openid-configuration`,
            # served under the store's `pki` root, which the default store has
            # never heard of.
            # It once shipped without the trust `forgejo.service` had, and
            # failed every single time with `x509: certificate signed by
            # unknown authority`. The trust belongs to every process that
            # makes the call, not to the obvious consumer.
            environment = {
              FORGEJO_CUSTOM = "/var/lib/forgejo/custom";
            };
            path = [
              deployCfg.forgejo.package
              pkgs.coreutils
              pkgs.gnugrep
            ];
            script = ''
              set -euo pipefail

              secret=$(cat "$CREDENTIALS_DIRECTORY/oidc-client-secret")
              if [ -z "$secret" ]; then
                echo "empty OIDC client secret at ${oidcSecretPath}" >&2
                exit 1
              fi

              # `--secret` is the only input forgejo offers: there is no
              # --secret-file and no env var, though its sibling
              # `forgejo-cli actions register` has both. So the value is
              # on this argv for the length of one exec, inside this
              # container, where forgejo already stores it at rest in the
              # login-source row — root and forgejo are the only
              # principals here, and root can read the resting copy
              # anyway. Accepted deliberately (see docs/swarm/), not
              # overlooked; upstream gap filed.
              # An ARRAY, not a string. A string of arguments has to be
              # word-split at the call site to become argv, and splitting
              # does not honour the quotes inside the value — the shell
              # already finished quote removal by then, so
              # `--scopes 'openid profile email groups'` arrives as four
              # words with two stray apostrophes, and forgejo rejects the
              # last three as unexpected arguments. An array carries the
              # boundaries instead of re-deriving them from whitespace.
              args=(
                --provider openidConnect
                --key ${lib.escapeShellArg cfg.sso.clientId}
                --auto-discover-url ${lib.escapeShellArg autheliaDiscoveryUrl}
                --scopes ${lib.escapeShellArg "openid profile email groups"}
              )

              if forgejo admin auth list | grep -q "[[:space:]]${ssoSourceName}[[:space:]]"; then
                id=$(forgejo admin auth list \
                  | grep "[[:space:]]${ssoSourceName}[[:space:]]" \
                  | cut -f1)
                forgejo admin auth update-oauth --id "$id" \
                  --name ${lib.escapeShellArg ssoSourceName} \
                  --secret "$secret" "''${args[@]}"
                echo "updated OIDC login source ${ssoSourceName} (id $id)"
              else
                forgejo admin auth add-oauth \
                  --name ${lib.escapeShellArg ssoSourceName} \
                  --secret "$secret" "''${args[@]}"
                echo "added OIDC login source ${ssoSourceName}"
              fi

              # Assert the EFFECT, not the command. The bug this replaced
              # was a malformed argv: the unit ran the right verb, forgejo
              # rejected it, and every check that read the rendered script
              # still passed. A registration that does not appear in the
              # source list did not happen, whatever the exit code said.
              if ! forgejo admin auth list | grep -q "[[:space:]]${ssoSourceName}[[:space:]]"; then
                echo "login source ${ssoSourceName} is absent after registration" >&2
                exit 1
              fi
            '';
          };

          # Provision the swarm-controller's forge account + access token.
          # Unconditional (not gated on any SSO/co-location option): forge
          # is a swarm-wide singleton, so this account exists wherever
          # forge does, regardless of which host (if any) actually runs
          # swarm-controller — the design requirement was explicit that
          # this must work even when forge and swarm-controller don't
          # share a host.
          #
          # The token goes straight to the swarm secret store, under the forge
          # host's store identity, and never onto disk. Forgejo does not wait
          # for this unit.
          systemd.services.${mintUnit} = {
            description = "provision the swarm-controller's forge account + access token";
            after = [ "forgejo.service" ];
            requires = [ "forgejo.service" ];
            wantedBy = [ "multi-user.target" ];
            # ../lib/store-retry.nix: a store that is sealed or unreachable is
            # retried, and so is a forge that is not answering yet.
            inherit (storeRetry) startLimitBurst startLimitIntervalSec;
            serviceConfig = storeRetry.serviceConfig // {
              Type = "oneshot";
              RemainAfterExit = true;
              User = "forgejo";
              Group = "forgejo";
              SyslogIdentifier = mintUnit;
              LoadCredential = lib.attrNames storeFiles;
              # The forge's answers pass through a temp file.
              PrivateTmp = true;
              # Each forge call is bounded by curl's own timeouts, and a sealed
              # store answers on the port and never answers the write.
              TimeoutStartSec = "3min";
            };
            # `FORGEJO_CUSTOM`, same reason as `forgejo-sso-source` above:
            # every `forgejo admin` invocation needs it to find the app.ini
            # the upstream module wrote, not just the auth-source verb.
            environment = {
              FORGEJO_CUSTOM = "/var/lib/forgejo/custom";
              BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
            }
            // lib.optionalAttrs havePublisherIdentity {
              BAO_CLIENT_CERT = mintCredential "forge-bao-cert";
              BAO_CLIENT_KEY = mintCredential "forge-bao-key";
            }
            // lib.optionalAttrs (storeFiles ? forge-bao-server-ca) {
              BAO_CACERT = mintCredential "forge-bao-server-ca";
            };
            path = [
              deployCfg.forgejo.package
              deployCfg.bao.package
              pkgs.coreutils
              pkgs.curl
              pkgs.gnugrep
              pkgs.jq
            ];
            script = ''
              set -euo pipefail

              # Idempotent by query, same reasoning as the SSO unit: assert
              # the effect, not the command's exit code. `--admin`: the
              # controller creates repos in the agents org on the swarm's
              # behalf, which needs org-admin standing to do at all. Only
              # applies to a fresh account — `forgejo admin` has no
              # in-place "promote to admin" verb (checked: `admin user`
              # offers create/list/change-password/delete/
              # generate-access-token/must-change-password/reset-mfa and
              # nothing else), and mara's call was not to delete+recreate
              # an already-provisioned non-admin account to work around
              # that (every hive from before this fix has one) — she
              # promotes those by hand instead.
              if ! forgejo admin user list | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
                forgejo admin user create \
                  --username ${lib.escapeShellArg swarmControllerForgeUser} \
                  --email ${lib.escapeShellArg "${swarmControllerForgeUser}@hyperhive.local"} \
                  --admin --random-password --must-change-password=false
                echo "created forge account ${swarmControllerForgeUser}"
              fi
              if ! forgejo admin user list | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
                echo "forge account ${swarmControllerForgeUser} absent after creation" >&2
                exit 1
              fi

              # Minted once. The marker holds the published token's name and is
              # written last, so a marker present means the store has a token
              # and every earlier one is revoked. Deleting it forces a re-mint,
              # the step for a store that lost the token or for new scopes.
              name_path=${lib.escapeShellArg swarmControllerTokenNamePath}
              if [ -s "$name_path" ]; then
                echo "the forge token $(< "$name_path") for ${swarmControllerForgeUser} is already in the swarm secret store"
                exit 0
              fi
            ''
            + (
              if !havePublisherIdentity then
                ''
                  echo "the forge's host has no forge-publisher store identity, so the swarm-controller's forge token cannot be published: set services.hyperhive.deploy.bao.forgePublisherClientCertFile and forgePublisherClientKeyFile (they default to the leaf the store mints only where services.hyperhive.deploy.bao.enable is set on this host)" >&2
                  exit 1
                ''
              else
                ''
                  err="$(mktemp)"
                  body="$(mktemp)"
                  trap 'rm -f "$err" "$body"' EXIT

                  bao_failed() {
                    echo "$1" >&2
                    if [ -s "$err" ]; then
                      cat "$err" >&2
                    else
                      echo "bao failed without writing a diagnostic." >&2
                    fi
                    exit 1
                  }

                  # The revocation below goes through the forge's admin API,
                  # which refuses a token of an account that is not a site
                  # admin; each retry would then leave one more token live.
                  if ! forgejo admin user list --admin | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
                    echo "forge account ${swarmControllerForgeUser} is not a site admin; promote it by hand. No token minted." >&2
                    exit 1
                  fi

                  # Before the mint, so a store that is down fails the attempt
                  # before forgejo holds a token nothing can read.
                  # `-token-only` answers on stdout and skips the token helper, a
                  # `sh` this unit's `path` does not carry.
                  if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
                    bao_failed "could not log in to swarm-bao with the forge publisher's certificate; no forge token minted for ${swarmControllerForgeUser}."
                  fi
                  export BAO_TOKEN

                  # `write:admin`: the controller's `CreateForgeUser` jobq node
                  # (`swarm-controller::forge::Client::ensure_agent_user`) and
                  # the revocation below call the admin API.
                  #
                  # Timestamped: forgejo refuses a name the account already has
                  # a token under, and the earlier tokens are still there until
                  # the revocation below. `hive-c0re::forge::users::mint_token`
                  # names its tokens the same way.
                  name="swarm-controller-boot-$(date +%s)"
                  out=$(forgejo admin user generate-access-token \
                    --username ${lib.escapeShellArg swarmControllerForgeUser} \
                    --token-name "$name" \
                    --scopes "write:repository,write:organization,write:issue,read:user,write:admin")
                  token=$(printf '%s' "$out" | grep -oE '[0-9a-f]{32,}' | head -n1)
                  if [ -z "$token" ]; then
                    echo "no token-shaped word in forgejo's generate-access-token output" >&2
                    exit 1
                  fi

                  # On stdin: the token is never an argument of any process.
                  if ! printf '%s' "$token" | bao kv put ${lib.escapeShellArg controllerTokenStorePath} value=- >/dev/null 2>"$err"; then
                    bao_failed "swarm-bao refused the write to ${controllerTokenStorePath}; the next attempt mints another token and revokes this one."
                  fi
                  echo "published the forge token $name for ${swarmControllerForgeUser} to ${controllerTokenStorePath}"

                  # One call to the forge's API as the new token. Prints the
                  # HTTP status; the body lands in "$body". The token reaches
                  # curl as a header file, never as an argument.
                  forge() {
                    local method="$1" path="$2"
                    curl --silent --show-error --connect-timeout 5 --max-time 15 \
                      -H @<(printf 'Authorization: token %s\n' "$token") \
                      -X "$method" -o "$body" -w '%{http_code}' \
                      "http://127.0.0.1:${toString cfg.httpPort}/api/v1$path"
                  }

                  # Every other token this unit minted: one whose write to the
                  # store failed, and the one the store held before the marker
                  # was lost. The bare `swarm-controller-boot` is the name the
                  # unit minted under before the timestamp suffix. A failure
                  # here leaves the marker unwritten, so the retry mints again
                  # and repeats the revocation.
                  stale=()
                  page=1
                  while :; do
                    status="$(forge GET "/admin/users/${swarmControllerForgeUser}/tokens?limit=50&page=$page")"
                    if [ "$status" != 200 ]; then
                      echo "the forge answered $status to listing ${swarmControllerForgeUser}'s tokens:" >&2
                      cat "$body" >&2
                      exit 1
                    fi
                    [ "$(jq length "$body")" -gt 0 ] || break
                    while IFS= read -r n; do
                      case "$n" in
                        "$name") ;;
                        swarm-controller-boot | swarm-controller-boot-*) stale+=("$n") ;;
                      esac
                    done < <(jq -r '.[].name' "$body")
                    page=$((page + 1))
                  done
                  for n in "''${stale[@]}"; do
                    status="$(forge DELETE "/admin/users/${swarmControllerForgeUser}/tokens/$n")"
                    case "$status" in
                      204 | 404) ;;
                      *)
                        echo "the forge answered $status to revoking ${swarmControllerForgeUser}'s token $n:" >&2
                        cat "$body" >&2
                        exit 1
                        ;;
                    esac
                  done
                  echo "revoked ''${#stale[@]} earlier forge token(s) of ${swarmControllerForgeUser}"

                  rm -f ${lib.escapeShellArg oldSwarmControllerTokenPath}
                  printf '%s\n' "$name" >"$name_path.tmp"
                  mv -f "$name_path.tmp" "$name_path"
                ''
            );
          };
        };
    };

    # The client this login source authenticates as is registered by
    # ../glue-forge-oidc-client.nix, wherever authelia runs.
    #
    # The forge's OIDC client secret, out of the swarm secret store under the
    # forge publisher's leaf — the one route, wherever authelia runs. Same
    # shape as ../hive-matrix.nix's `swarm-bao-matrix-oidc`: the login and the
    # read both fail loudly, and `storeRetry` retries until the publisher in
    # authelia's container has written the secret.
    #
    # On the host, which reads that leaf from its own files. A host with no
    # publisher leaf fails here, naming the options, rather than rendering no
    # unit.
    systemd.services.swarm-bao-forge-oidc = {
      description = "fetch the forge's OIDC client secret from the swarm secret store";
      # Ordered after the store where it runs here
      # (../glue-bao-readers-store-order.nix); elsewhere the timeout below
      # bounds the attempt.
      before = [ "container@hive-forge.service" ];
      wantedBy = [
        "multi-user.target"
        "container@hive-forge.service"
      ];
      path = [
        deployCfg.bao.package
        pkgs.coreutils
        pkgs.systemd
      ];
      # ../lib/store-retry.nix. The container waits for one attempt; a secret a
      # later attempt lands re-runs `forgejo-sso-source` (below).
      inherit (storeRetry) startLimitBurst startLimitIntervalSec;
      serviceConfig = storeRetry.serviceConfig // {
        # No `RemainAfterExit`: the timer below starts this again, and
        # starting an active unit is a no-op.
        Type = "oneshot";
        SyslogIdentifier = "swarm-bao-forge-oidc";
        TimeoutStartSec = 30;
      };
      environment = {
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
      }
      // lib.optionalAttrs havePublisherIdentity {
        BAO_CLIENT_CERT = deployCfg.bao.forgePublisherClientCertFile;
        BAO_CLIENT_KEY = deployCfg.bao.forgePublisherClientKeyFile;
      }
      // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
        BAO_CACERT = deployCfg.bao.serverCaFile;
      };
      script =
        if !havePublisherIdentity then
          ''
            echo "the forge's host has no forge-publisher store identity, so it cannot read the forge's OIDC client secret: set services.hyperhive.deploy.bao.forgePublisherClientCertFile and forgePublisherClientKeyFile (they default to the leaf the store mints only where services.hyperhive.deploy.bao.enable is set on this host)" >&2
            exit 1
          ''
        else
          ''
            set -euo pipefail

            ${atomicWriteSecret}
            ${refreshConsumer}

            err="$(mktemp)"
            trap 'rm -f "$err"' EXIT

            # `-token-only` answers on stdout and skips the token helper, a
            # `sh` this unit's `path` does not carry.
            if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
              echo "could not log in to swarm-bao with the forge publisher's certificate; leaving the forge's OIDC client secret as it is." >&2
              if [ -s "$err" ]; then
                cat "$err" >&2
              else
                echo "bao failed without writing a diagnostic." >&2
              fi
              exit 1
            fi
            export BAO_TOKEN

            if ! secret="$(bao kv get -field=value ${lib.escapeShellArg oidcStorePath} 2>"$err")"; then
              echo "swarm-bao did not return ${oidcStorePath}; the forge has no OIDC client secret yet." >&2
              if [ -s "$err" ]; then
                cat "$err" >&2
              else
                echo "bao failed without writing a diagnostic." >&2
              fi
              exit 1
            fi

            if [ -z "$secret" ]; then
              echo "swarm-bao returned an empty ${oidcStorePath}; the forge has no OIDC client secret yet." >&2
              exit 1
            fi

            # Root-owned: the container's systemd reads it for `LoadCredential`.
            install -d -m 0700 ${lib.escapeShellArg oidcSecretDir}
            if secret_differs ${lib.escapeShellArg oidcSecretPath} "$secret"; then
              atomic_write_secret 0400 "" ${lib.escapeShellArg oidcSecretPath} "$secret"
            fi

            # The login source is a row `forgejo-sso-source` writes from the
            # file each time it runs.
            refresh_consumer hive-forge forgejo-sso-source.service ${lib.escapeShellArg oidcSecretPath}
          '';
    };

    systemd.timers.swarm-bao-forge-oidc = storeRefreshTimer "re-read the forge's OIDC client secret from the swarm secret store";

    systemd.tmpfiles.rules = [
      # The OIDC secret's directory, which the container binds at start.
      "d ${oidcSecretDir} 0700 root root -"
      # Unused: nothing writes or reads these host paths, and they may hold a
      # copy of the controller's forge token or of the forge's OIDC secret.
      "r /var/lib/hyperhive-forge/swarm-controller.token"
      "r ${oldOidcSecretPath}"
    ];

    networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
      allowedTCPPorts = [
        cfg.httpPort
        cfg.sshPort
      ];
    };

    # The mirrors reach a controller on THIS host only (see
    # ../glue-forge-controller.nix); a controller elsewhere reads its own
    # host's list, so the warning below says so at eval rather than leaving
    # it to be noticed as a missing repo.
    services.hyperhive.deploy.forgejo.effectiveMirrors = effectiveMirrors;
    warnings = lib.optional (effectiveMirrors != [ ] && !deployCfg.swarm-controller.enable) ''
      services.hyperhive.deploy.forgejo.mirrors (or the actions/* mirrors
      deploy.forgejo.actionsFromSelf adds) is set on a host that does not run the
      swarm-controller. The controller seeds mirrors from its own host's list,
      so these are not seeded: declare them on the swarm-controller's host.
    '';
  };
}