{ pkgs, lib, config, ... }: let cfg = config.services.hyperhive.swarm.forge; gatewayCfg = config.services.hyperhive.gateway; hyperhiveDomain = config.services.hyperhive.domain; swarmDomain = config.services.hyperhive.swarm.domain; tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; networkCfg = config.services.hyperhive.network; # Forgejo's name for the login source. A constant, not an option: it # is the key this module's own idempotency check looks up, so making # it configurable would buy nothing and add a way for the lookup and # the row to disagree. Duplicated in ./service.nix, which builds # `sso.redirectUri` from it; keep the two equal. ssoSourceName = "authelia"; # `url` is the half of the authelia module that exists on EVERY hive — # null when no SSO provider is configured anywhere, which the # assertion below turns into an eval failure rather than a discovery # request to `null/.well-known/...`. autheliaCfg = config.services.hyperhive.swarm.authelia; deployCfg = config.services.hyperhive.deploy; autheliaUrl = autheliaCfg.url; autheliaDiscoveryUrl = "${toString autheliaUrl}/.well-known/openid-configuration"; # The all-local case: this host runs BOTH the forge and the swarm's # authelia, so the secret can be moved without an operator. The other # two cases (swarm side via swarmctl, remote hive) leave # `clientSecretFile` to be set explicitly — see docs/swarm/. ssoLocal = deployCfg.authelia.enable; # Where the plaintext lands inside the forge container. Under # /var/lib rather than /run: the forge may start before the delivery # unit on a later boot, and a secret that evaporates on reboot turns a # working login into an intermittent one. forgeSecretPath = "/var/lib/forgejo-oidc/${cfg.sso.clientId}.secret"; # The swarm-controller's forge account. Same name as # `swarm-controller.nix`'s existing `queueClientId` — the account is # provisioned to *match* that identity, not invented independently — # "swarm controller having one identity with stuff derived from it is # the right shape" was the swarm-level design call this account # follows. A constant, not an option: # nothing here makes the name configurable without also updating # `swarm-controller.nix`'s own constant, so a shared option would # invite the two to drift rather than prevent it. swarmControllerForgeUser = "swarm-controller"; # Where the minted token lands inside the forge container — under # forgejo's own state dir for the same "survives a reboot" reason as # `forgeSecretPath` above. swarmControllerTokenPath = "/var/lib/forgejo/swarm-controller-token"; caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # ROOT_URL forgejo advertises in clone links + outbound URLs. # `cfg.domain` doubles as both the forgejo `DOMAIN` setting AND the # gateway vhost server-name, so ROOT_URL just uses it directly. The # gateway always terminates TLS (self-signed is the implicit floor when # neither `tls.certDir` nor ACME is configured), so the forge is always # advertised over `https` on `httpsPort` — the canonical 443 elides the # port suffix. Operators can still override via `cfg.rootUrl` for # bespoke shapes. Both bindings are duplicated in ./service.nix, which # builds `sso.redirectUri` from them; keep the two equal. defaultRootUrl = let portSuffix = if gatewayCfg.httpsPort == 443 then "" else ":${toString gatewayCfg.httpsPort}"; in "https://${cfg.domain}${portSuffix}/"; effectiveRootUrl = if cfg.rootUrl != null then cfg.rootUrl else defaultRootUrl; # When CI is enabled, the runner needs `actions/checkout` resolvable # without external DNS (hive-ci shares the host netns, so a host-resolver # blip otherwise reds every `actions/checkout@vN` fetch from # data.forgejo.org). Auto-append a pull-mirror of it and point # forgejo's DEFAULT_ACTIONS_URL at this instance so `uses:` resolves local. ciEnabled = config.services.hyperhive.deploy.forgejo.ci.enable; actionCheckoutMirror = { upstream = "https://github.com/actions/checkout"; dest = "actions/checkout"; }; # Auto-append the actions/checkout mirror only when CI is on AND the # operator hasn't already declared that dest themselves (else CI-on + # an explicit `actions/checkout` entry would duplicate it). effectiveMirrors = deployCfg.forgejo.mirrors ++ lib.optional ( ciEnabled && !(lib.any (m: m.dest == actionCheckoutMirror.dest) deployCfg.forgejo.mirrors) ) actionCheckoutMirror; # Share host netns — forgejo's HTTP / SSH listeners then look # exactly like a host-side service, no port forwarding dance, # and agent containers (which also share host netns) reach it # via plain `localhost`. privateNetwork = false; in { # Private Forgejo in a `hive-forge` nixos-container, shared host # netns, on the one host with `deploy.forgejo.enable`. Agents reach it # at `forge.` via the gateway. State at # `/var/lib/nixos-containers/hive-forge/var/lib/forgejo/` survives # restart. See `docs/networking/gateway.md::hive-forge container shape`. imports = [ ./service.nix ]; # What ./service.nix declares is what the forge IS from any hive's point of # view: the names and ports it answers on, the URLs it advertises, and the # client id it is registered under. What lives here is what the host running # it decides — which build it runs, how it is served, what it mirrors, and # where its host-local secrets sit. Same rule as ./swarm-victorialogs.nix, # and the renames are in ./deploy.nix with the rest. options.services.hyperhive.deploy.forgejo = { package = lib.mkOption { type = lib.types.package; default = pkgs.forgejo; defaultText = lib.literalExpression "pkgs.forgejo"; description = '' Forgejo package to run inside the container. Defaults to `pkgs.forgejo` (the latest release line) rather than the nixpkgs-module default of `pkgs.forgejo-lts`, because LTS lags far behind on schema and the DB easily ends up "newer than the binary" if the operator ever ran a non-LTS forgejo against the same state dir. Override to `pkgs.forgejo-lts` if you actively want the slower release train. ''; }; openFirewall = lib.mkOption { type = lib.types.bool; default = false; example = true; description = '' Open `httpPort` + `sshPort` in the host firewall. Off by default (secure-by-default): agent containers reach the forge at `forge.` via the gateway (not directly), and the host reaches it on loopback — so the firewall opens only matter for access from outside the host. Flip to `true` when you want the operator's browser or external git clients to hit the forge directly. **Breaking change**: this used to default to `true`. If you relied on the old default for external reach, add `services.hyperhive.deploy.forgejo.openFirewall = true;` to your host config before rebuilding. ''; }; mirrors = lib.mkOption { type = lib.types.listOf ( lib.types.submodule { options = { upstream = lib.mkOption { type = lib.types.str; example = "https://github.com/actions/checkout"; description = "Upstream clone URL to mirror from."; }; dest = lib.mkOption { type = lib.types.str; example = "actions/checkout"; description = '' Local `/` the pull-mirror is created at. The `` org is auto-created if missing. Keep mirror dests in their own orgs (e.g. `actions/*`) — separate from the hive-c0re-managed namespaces (config/shared/agents/core) so the seed never collides with core's own provisioning. ''; }; }; } ); default = [ ]; example = lib.literalExpression '' [ { upstream = "https://github.com/actions/checkout"; dest = "actions/checkout"; } ] ''; description = '' General-purpose Forgejo **pull-mirrors** to auto-seed on the local forge. Each entry is created as a real Forgejo pull-mirror (it re-syncs from `upstream` out-of-band), not a one-off pushed clone — so a host-resolver blip leaves a *stale* mirror, never a hard failure on whatever reads it. When `services.hyperhive.deploy.forgejo.ci.enable` is set, an `actions/checkout` mirror is auto-appended to this list and forgejo's `DEFAULT_ACTIONS_URL` is pointed at this instance, so CI `uses: actions/checkout@vN` steps resolve entirely on loopback with no external DNS on the critical path (the seed/re-sync needs external DNS, but that's off the CI path). ''; }; # The other half of the SSO pair: the client *id* is swarm-wide (it has # to match authelia's register), the secret is a path on this host. The # design note for the login source itself is with the id. sso.clientSecretFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; example = "/var/lib/hyperhive/forge-oidc-secret"; description = '' Path **inside the forge container** holding the client secret's plaintext. A path, never a value: an OIDC client secret has two holders in two containers (authelia keeps a hash, this forge needs the plaintext), and a literal written here would be rendered into the world-readable nix store. Required when `enable` is set — deliberately no fallback. A forge that boots with SSO half-configured presents as a login button that always fails, which is harder to diagnose than an eval error. ''; }; hostSwarmControllerTokenFile = lib.mkOption { type = lib.types.str; default = "/var/lib/hyperhive-forge/swarm-controller.token"; description = '' Host path where this forge deposits the freshly-minted forge access token for the swarm's `swarm-controller` account (see `systemd.services.forgejo-swarm-controller-account` inside the forge container, and `hive-forge-swarm-controller-token` on the host, which copies the token out). Same role for this token as `services.hyperhive.deploy.authelia.hostClientSecretDir` plays for the OIDC secret: a **host**-local path (not inside any container), read directly by `swarm-controller.nix`'s `LoadCredential` when the controller runs on this same host. On any other host the token has to get there somehow — copy it out of this path with whatever secret management this deployment already uses, the same shape `swarm.nix`'s own `clientSecretFile` documents for the analogous cross-host case. ''; }; }; config = lib.mkIf deployCfg.forgejo.enable { # This service's own gateway surface: the vhost that fronts it and # the name the hive resolver answers for. Declared here rather than # in the gateway so the forge's public face lives with the forge — # the gateway supplies the primitives (`lib.listen`, `lib.tlsFor`, # `lib.securityHeaders`) and never needs to know this service by # name. services.hyperhive.gateway.localNames = [ cfg.domain ]; services.hyperhive.gateway.enable = lib.mkDefault true; # The forge container resolves the rest of the hive through dnsmasq. services.hyperhive.gateway.dns.enable = lib.mkDefault true; # The metrics endpoint, declared once: the collector both scrapes this # URL and derives from it the audience its token is minted for. # # ⚠️ Written as the exact URL a collector requests, because that is # what authelia compares against — this string agreeing with the # `location` block above it is the whole mechanism. A near miss is a # correctly minted token refused at the target. services.hyperhive.swarm.otel.publishedScrapeTargets = { forgejo = "https://${cfg.domain}/metrics"; }; # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB # clones). SSH stays direct on `forge.sshPort`. See # `docs/networking/gateway.md`. services.nginx.virtualHosts = { "${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; locations = { "/" = { proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/"; proxyWebsockets = true; extraConfig = '' proxy_buffering off; client_max_body_size 1G; proxy_read_timeout 1h; proxy_send_timeout 1h; ''; }; # ⚠️ EXACT match, and that is what makes this safe. Forgejo # serves `/metrics` on the same listener the `/` prefix above # already proxies, so without a more specific location the # endpoint would ride that catch-all to anyone. `= /metrics` # outranks the `/` prefix in nginx, so this location — and its # auth — is the one that runs. "= /metrics" = { proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/metrics"; extraConfig = '' auth_request /__forge_metrics_authz; ''; }; # The subrequest. Same implementation and header set as # `swarm-ui.nix` uses, for the same reason: `X-Original-URL` # and `X-Original-Method` are what authelia's `auth-request` # implementation reads. # # ⚠️ NO `error_page 401 =302` here, and its absence is the # whole point. The swarm UI redirects an unauthenticated # browser to a login page; a scraper handed that 302 would # follow it and parse an HTML page as metrics. A machine-facing # location lets the 401 reach the client unchanged. "= /__forge_metrics_authz" = { # By NAME, not `127.0.0.1`: this gateway and authelia are not # required to share a host, so the loopback literal encoded a # co-location nobody had agreed to. The name resolves here # today and resolves off-host later without this line moving. proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request"; # nixpkgs appends its OWN `Host $host` after extraConfig, # which would override verifiedProxyTo's — see the comment # on verifiedProxyTo in hive-gateway/vhost-lib.nix. recommendedProxySettings = false; extraConfig = '' internal; ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain} proxy_pass_request_body off; proxy_set_header Content-Length ""; proxy_set_header X-Original-Method $request_method; proxy_set_header X-Original-URL $scheme://$http_host$request_uri; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; ''; }; }; }; }; assertions = [ { # Fail at EVAL, not at boot. The alternative failure is a login # button that always 401s, three layers from the missing file. assertion = deployCfg.forgejo.sso.clientSecretFile != null; message = '' The forge's SSO login source requires services.hyperhive.deploy.forgejo.sso.clientSecretFile — the path (inside the forge container) holding the OIDC client secret's plaintext. On a hive that also runs the swarm's authelia this is wired up for you. Set it explicitly when authelia lives on another host: see docs/swarm/ for which secret goes where. ''; } { assertion = cfg.rootUrl == null || lib.hasSuffix "/" cfg.rootUrl; message = '' services.hyperhive.swarm.forge.rootUrl must end with "/". forgejo's ROOT_URL contract requires a trailing slash for correct relative-link generation; without it forgejo emits URLs like `https://forge.example.com.user.id` instead of `https://forge.example.com/user.id`. Got: ${toString cfg.rootUrl} ''; } { # `cfg.domain` can't be empty — would render `.` shaped # garbage as both server_name (nginx wildcard catch-all) and # /etc/hosts entry (invalid). The default derives a non-empty # `forge.`, but an operator-set empty string should fail # loud. assertion = cfg.domain != ""; message = '' services.hyperhive.swarm.forge.domain = "" is rejected. The rendered URLs would be invalid (nginx wildcard catch-all for an empty server_name, /etc/hosts rejects empty entries). Either leave at default (auto-derives to "forge."), or set a non-empty hostname like "forge.example.com" or "git.internal". ''; } { # Each mirror dest must be exactly `/` — the seed # splits on the single slash to create the org + repo. assertion = lib.all (m: lib.length (lib.splitString "/" m.dest) == 2) effectiveMirrors; message = '' Every services.hyperhive.deploy.forgejo.mirrors[].dest must be exactly "/" (one slash). Got: ${lib.concatMapStringsSep ", " (m: m.dest) effectiveMirrors} ''; } { # Keep mirror orgs out of the hyperhive-managed namespaces # (config/shared/agents/core) so the seed never races / collides # with the provisioning of those orgs. assertion = lib.all ( m: !(lib.elem (builtins.elemAt (lib.splitString "/" m.dest) 0) [ "config" "shared" "agents" "core" ]) ) effectiveMirrors; message = '' services.hyperhive.deploy.forgejo.mirrors[].dest must not place a mirror in a hyperhive-managed org (config / shared / agents / core) — those are provisioned by hyperhive itself and a mirror there would collide. Use a dedicated org (e.g. "actions/checkout"). ''; } ]; # `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service` # in self-signed mode (see the hive-ca-trust helper), so the CA bind # source exists before nspawn sets the mount up. systemd.services."container@hive-forge" = caTrust.containerOrdering; containers.hive-forge = { autoStart = true; ephemeral = false; # Journal files on the host, not inside the container: nixpkgs hardcodes # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. extraFlags = [ "--link-journal=host" ]; inherit privateNetwork; # Self-signed mode: bind the public hive CA cert read-only so forgejo # can trust the gateway's self-signed leaf for outbound webhook # delivery. The bind-mount, the `container@` ordering and the bundle # the container's consumers read all come from the hive-ca-trust # helper — see the container's `imports` for the consumption half. bindMounts = caTrust.bindMount; config = { pkgs, ... }: let # Build a custom static-root that is the standard forgejo data # output with our theme CSS added. Using STATIC_ROOT_PATH instead # of tmpfiles / bind-mounts means the theme is always present in # the nix store — no separate hive-forge container rebuild needed, # and no persistent-state directory involved. staticRootWithTheme = pkgs.runCommand "forgejo-static-with-theme" { } '' cp -r --no-preserve=mode,ownership ${deployCfg.forgejo.package.data}/. $out/ mkdir -p $out/public/assets/css cp ${./theme-catppuccin-vibec0re.css} \ $out/public/assets/css/theme-catppuccin-vibec0re.css # Replace the default Forgejo logo + favicon with the hyperhive # mark. Files in public/assets/img/ are served before built-ins. mkdir -p $out/public/assets/img cp ${../../../branding/hyperhive.svg} $out/public/assets/img/logo.svg cp ${../../../branding/hyperhive.svg} $out/public/assets/img/favicon.svg cp ${../../../branding/hyperhive.png} $out/public/assets/img/logo.png cp ${../../../branding/hyperhive.png} $out/public/assets/img/favicon.png cp ${../../../branding/hyperhive.png} $out/public/assets/img/avatar_default.png ''; in { imports = [ ./../../container-modules/swarm-container.nix # Ask the hive's dnsmasq, not the host's resolvers. The swarm # names — the controller's webhook endpoint, the SSO issuer — # resolve to the bridge and have no public records, so a # container using the host's resolvers gets "no such host" for # every one of them while the same name resolves fine one # container over. # # Both units that make an outbound call are consumers, for the # same reason the trust bundle below names both: an unordered # resolver write is a race that only shows up on a cold boot. (import ./../../container-modules/swarm-container-resolver.nix { inherit (networkCfg) bridgeIp; dnsConsumers = [ "forgejo.service" ] ++ lib.optional ssoLocal "forgejo-sso-source.service"; }) # Forgejo is Go, and `SSL_CERT_FILE` *replaces* the default store # rather than adding to it — so it needs the system CAs and the # hive CA concatenated, not the CA alone, or public mirror fetches # lose every anchor they had. # # BOTH units that make an outbound HTTPS call are consumers, not # just the obvious one: `forgejo-sso-source` fetches the issuer's # `.well-known/openid-configuration` over the swarm CA, and it # once shipped without the trust its sibling had. `optional` # rather than a flat list because that unit only exists when SSO # is on — naming an absent unit would order nothing and quietly # define a serviceless one. (caTrust.trustBundle { inherit pkgs; name = "hive-forge"; consumers = [ "forgejo" "forgejo-sso-source" ]; }) ]; system.stateVersion = "25.11"; services.hyperhive.swarmContainer = { inherit privateNetwork; }; services.forgejo = { enable = true; package = deployCfg.forgejo.package; database.type = "sqlite3"; lfs.enable = true; settings = { DEFAULT.APP_NAME = "HyperHive"; server = { DOMAIN = cfg.domain; ROOT_URL = effectiveRootUrl; HTTP_PORT = cfg.httpPort; START_SSH_SERVER = true; SSH_PORT = cfg.sshPort; SSH_LISTEN_PORT = cfg.sshPort; BUILTIN_SSH_SERVER_USER = "git"; DISABLE_SSH = false; # Point forgejo at our extended static root that includes # the custom theme CSS baked straight into the nix store. STATIC_ROOT_PATH = staticRootWithTheme; }; # Local sign-up off. An agent's account is made by # swarm-controller over the admin API, a human's by their # first authelia login (`oauth2_client` below), which this # does not block: forgejo's auto-registration checks only # `ALLOW_ONLY_INTERNAL_REGISTRATION` # (`routers/web/auth/oauth.go`, 16.0.5). service = { DISABLE_REGISTRATION = true; REQUIRE_SIGNIN_VIEW = false; }; # A first login through the authelia source creates the # account, named after authelia's `preferred_username` — the # authelia username, not the opaque `sub`. # # ⚠️ `ACCOUNT_LINKING = login`, never `auto`. A login whose # name is already taken does not create an account; `auto` # would then hand the SSO user the existing one. Agents, # `core` and `swarm-controller` all have local accounts here, # the last two site admins, and `swarmctl user add` refuses # none of those names. `login` asks for the existing # account's own password instead, which an agent's or a # service account's never leaves the forge. oauth2_client = { ENABLE_AUTO_REGISTRATION = true; USERNAME = "preferred_username"; ACCOUNT_LINKING = "login"; }; repository = { DEFAULT_BRANCH = "main"; DEFAULT_PRIVATE = "private"; }; # Not an option: a swarm-integrated, auto-deployed forge # always has metrics, behind the protected `= /metrics` # location on its gateway vhost. # # No `TOKEN` here on purpose. Forgejo can guard this itself # with a static bearer, but the swarm authenticates the # scraper at the gateway, so a second credential system per # service would buy nothing and would be the one that stops # getting rotated. metrics.ENABLED = true; # The two per-dimension breakdowns, alongside the endpoint # itself: `gitea_issues_by_label{label=…}` and # `gitea_issues_by_repository{repository=…}`. Off by default # upstream because they are the only metrics here whose series # count grows with the CONTENT of the forge rather than with # its feature set — so the question is cardinality, and it was # measured rather than assumed. # # Measured on this instance: ~19 distinct labels and ~29 repos # ⇒ **well under 100 series**, against a store that took +2234 # from nine host scrapers. Two orders of magnitude below the # cost that already argued against shipping a `process` # scraper, so the breakdowns are affordable here. # # ⚠️ Both grow with repo and label count, not with traffic. A # forge that grew to thousands of repos would want this # revisited — that is a real trigger, unlike a time-based one: # `count(gitea_issues_by_repository)` answers it directly. metrics.ENABLED_ISSUE_BY_LABEL = true; metrics.ENABLED_ISSUE_BY_REPOSITORY = true; # Repo migrations / pull-mirrors fetch from the source # URL *inside* Forgejo. hyperhive code is synced from # `localhost` (and the host LAN), which Forgejo's # migration guard blocks by default ("cannot import from # disallowed hosts"). Allow loopback + RFC-1918 sources # so an in-hive mirror of the hyperhive repo works. migrations.ALLOW_LOCALNETWORKS = true; # Forgejo's docs say an empty `ALLOWED_DOMAINS` allows # every domain, but that's not true in practice on the # versions we've hit this on — the migration guard still # rejects genuinely public hosts ("cannot import from # disallowed hosts") unless the wildcard is set # explicitly (a known upstream doc/behavior mismatch, # tracked upstream in the go-gitea project). Public-domain # pull/push mirrors (agents' personal repos synced to # forge.darkest.space, etc.) were failing every sync # attempt without this. migrations.ALLOWED_DOMAINS = "*"; # Forgejo's webhook SSRF allow-list, a strict whitelist. # `external` (public unicast only) stays, or user-repo webhooks to # github/slack break; both hyperhive hosts are added because their # webhooks target a private gateway IP `external` denies. The # swarm's host was the missing one, and every swarm-controller # delivery was refused for it. # # Not gated on `deploy.swarm-ui.enable`: in a spread deployment the # forge and the swarm host are different machines, so that # per-host flag is false exactly where this entry is needed. webhook.ALLOWED_HOST_LIST = lib.concatStringsSep "," ( lib.unique ( [ "external" hyperhiveDomain ] ++ lib.optional (swarmDomain != null) swarmDomain ) ); log.LEVEL = "Warn"; # Pinned explicitly rather than left to upstream's default # (currently `bleve`, a separate full-text index Forgejo # builds/maintains itself). `db` searches the database # directly instead — no second store that can silently # disagree with it. Found via a false-positive search bug: # `hive-forge list --search` returned matches with zero # literal occurrences of the search term, consistent with a # bleve index that drifted stale against the DB with # nothing to detect the drift. At this repo's scale (~3.4k # issues+PRs) a DB scan costs nothing worth noticing, and # it's the same substring semantics `--search`'s own doc # already promises — bleve's relevance ranking buys nothing # we use here. Config-only (part of the forgejo unit, so a # rebuild restarts it for free); the old bleve index # directory is left on disk, unused, rather than risk # deleting the wrong thing. indexer.ISSUE_INDEXER_TYPE = "db"; ui = { DEFAULT_THEME = "catppuccin-vibec0re"; THEMES = "catppuccin-vibec0re,forgejo-auto,forgejo-light,forgejo-dark,gitea-auto,gitea-light,gitea-dark"; }; # Point forgejo at the GPG key generated by the # forgejo-gpg-init service below. SIGNING_KEY = "default" # resolves via the forgejo process's git config # (`user.signingkey`) — which forgejo-gpg-init sets to the # generated key — not by scanning GNUPGHOME. GNUPGHOME is # the keyring forgejo signs from; must be absolute + # writeable by the forgejo user. "repository.signing" = { SIGNING_KEY = "default"; GNUPGHOME = "/var/lib/forgejo/.gnupg"; }; # Enable Forgejo Actions so the runner registration token # API endpoint is available. Without this the endpoint # returns "runner registration token not found" regardless # of token scopes. Required by `hive-ci-register.service` # in the hive-ci container. actions.ENABLED = true; # When CI is enabled, resolve `uses: /@vN` from # THIS instance (the seeded `actions/checkout` pull-mirror) # instead of the upstream default `data.forgejo.org` — keeps # the checkout step on loopback, immune to a host-resolver # blip. `self` = forgejo expands actions against its # own ROOT_URL. actions.DEFAULT_ACTIONS_URL = lib.mkIf ciEnabled "self"; # F3 (federation) computes its data dir relative to the # forgejo binary, which lands in the read-only nix # store and crashes anything that touches the F3 # subsystem — including `forgejo admin user create`, # which init-ses F3 even when ENABLED=false. Pin the # path absolute alongside the disable so the init # resolution succeeds before the flag is checked. "F3" = { ENABLED = false; PATH = "/var/lib/forgejo/data/f3"; }; }; }; environment.systemPackages = [ pkgs.forgejo pkgs.gnupg ]; # Forgejo's local Actions-artifact storage defaults to # `{APP_DATA_PATH}/actions_artifacts` (= # `/var/lib/forgejo/data/actions_artifacts`), but Forgejo does not # pre-create that directory. The artifact endpoint ingests the # chunked upload, then the merge-chunks step does an `lstat` on a # tmp dir under it and fails: # Error merge chunks: lstat # /var/lib/forgejo/data/actions_artifacts/tmpNNN: no such file or # directory # so every `upload-artifact` step dies after the build succeeds. # Pre-create the dir (forgejo-owned) so uploads actually persist. # `actions.ENABLED = true` registers the endpoints; this gives them # somewhere to write. systemd.tmpfiles.rules = [ "d /var/lib/forgejo/data 0750 forgejo forgejo - -" "d /var/lib/forgejo/data/actions_artifacts 0750 forgejo forgejo - -" ]; # Ensure Forgejo has a usable GPG signing key so UI merges / CRUD # commits are signed instead of erroring "does not have a signing # key". This service (a) generates a key in forgejo's persistent # keyring iff one isn't already present — keyed on the actual # secret key, NOT a stamp file, so a partial state wipe that loses # the key still regenerates it — and (b) points the forgejo user's # git config at it (`user.signingkey` + commit/tag gpgsign), which # is how `SIGNING_KEY = "default"` actually resolves. Runs as the # forgejo user before forgejo on each start; idempotent (the keygen # is guarded, the git-config is a cheap re-set). systemd.services.forgejo-gpg-init = { description = "ensure Forgejo's GPG signing key + git signing config"; # Start before forgejo so the key + signing config are ready when # forgejo reads repository.signing on startup. wantedBy = [ "forgejo.service" ]; before = [ "forgejo.service" ]; serviceConfig = { Type = "oneshot"; User = "forgejo"; Group = "forgejo"; # Pin the journal identity (else it's the `script` store-path wrapper). SyslogIdentifier = "forgejo-gpg-init"; }; # GNUPGHOME = the keyring forgejo signs from; HOME so # `git config --global` lands where the forgejo process reads it. environment = { GNUPGHOME = "/var/lib/forgejo/.gnupg"; HOME = "/var/lib/forgejo"; }; path = [ pkgs.gnupg pkgs.git pkgs.gnugrep pkgs.gawk pkgs.coreutils ]; script = '' set -euo pipefail mkdir -p "$GNUPGHOME" chmod 700 "$GNUPGHOME" # Generate only if no secret key is present (key-based guard, # not a stamp — a stamp can outlive the key after a state wipe # and wrongly suppress regeneration). if ! gpg --list-secret-keys --with-colons 2>/dev/null | grep -q '^sec:'; then printf '%s\n' \ '%no-protection' \ 'Key-Type: RSA' \ 'Key-Length: 4096' \ 'Name-Real: HyperHive Forge' \ 'Name-Email: forgejo@hive' \ 'Expire-Date: 0' \ | gpg --batch --gen-key fi # Point git (hence Forgejo's SIGNING_KEY="default") at the key. KEYID=$(gpg --list-secret-keys --keyid-format long --with-colons \ | awk -F: '/^sec:/ { print $5; exit }') if [ -n "$KEYID" ]; then git config --global user.signingkey "$KEYID" git config --global commit.gpgsign true git config --global tag.gpgsign true fi ''; }; # Register authelia as an OIDC login source. # # ⚠️ Ordered AFTER forgejo, unlike forgejo-gpg-init above, and # the difference is not stylistic: a login source is a row in # forgejo's database, and on a fresh hive that database does # not exist until forgejo has started and run its migrations. # Running first would either fail or let the CLI initialise a # schema behind the server's back. # # Idempotency is by QUERY, not by a stamp file — same reason # spelled out for the GPG key above: a stamp survives a state # wipe that took the thing it claims exists, and then suppresses # the repair. systemd.services.forgejo-sso-source = { description = "register authelia as Forgejo's OIDC login source"; after = [ "forgejo.service" ]; requires = [ "forgejo.service" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; User = "forgejo"; Group = "forgejo"; SyslogIdentifier = "forgejo-sso-source"; }; # `FORGEJO_CUSTOM` (not `GITEA_CUSTOM` — forgejo renamed it) # is how the CLI finds the app.ini upstream's module wrote. # # This unit is also a trust-bundle consumer (declared in the # container's `imports` above, which is where `SSL_CERT_FILE` # comes from): registering the login source makes an **outbound # HTTPS call** to `/.well-known/openid-configuration`, # served under the swarm CA the default store has never heard of. # It once shipped without the trust `forgejo.service` had, and # failed every single time with `x509: certificate signed by # unknown authority`. The trust belongs to every process that # makes the call, not to the obvious consumer. environment = { FORGEJO_CUSTOM = "/var/lib/forgejo/custom"; }; path = [ deployCfg.forgejo.package pkgs.coreutils pkgs.gnugrep ]; script = '' set -euo pipefail secret=$(cat ${lib.escapeShellArg deployCfg.forgejo.sso.clientSecretFile}) if [ -z "$secret" ]; then echo "empty OIDC client secret at ${deployCfg.forgejo.sso.clientSecretFile}" >&2 exit 1 fi # `--secret` is the only input forgejo offers: there is no # --secret-file and no env var, though its sibling # `forgejo-cli actions register` has both. So the value is # on this argv for the length of one exec, inside this # container, where forgejo already stores it at rest in the # login-source row — root and forgejo are the only # principals here, and root can read the resting copy # anyway. Accepted deliberately (see docs/swarm/), not # overlooked; upstream gap filed. # An ARRAY, not a string. A string of arguments has to be # word-split at the call site to become argv, and splitting # does not honour the quotes inside the value — the shell # already finished quote removal by then, so # `--scopes 'openid profile email groups'` arrives as four # words with two stray apostrophes, and forgejo rejects the # last three as unexpected arguments. An array carries the # boundaries instead of re-deriving them from whitespace. args=( --provider openidConnect --key ${lib.escapeShellArg cfg.sso.clientId} --auto-discover-url ${lib.escapeShellArg autheliaDiscoveryUrl} --scopes ${lib.escapeShellArg "openid profile email groups"} ) if forgejo admin auth list | grep -q "[[:space:]]${ssoSourceName}[[:space:]]"; then id=$(forgejo admin auth list \ | grep "[[:space:]]${ssoSourceName}[[:space:]]" \ | cut -f1) forgejo admin auth update-oauth --id "$id" \ --name ${lib.escapeShellArg ssoSourceName} \ --secret "$secret" "''${args[@]}" echo "updated OIDC login source ${ssoSourceName} (id $id)" else forgejo admin auth add-oauth \ --name ${lib.escapeShellArg ssoSourceName} \ --secret "$secret" "''${args[@]}" echo "added OIDC login source ${ssoSourceName}" fi # Assert the EFFECT, not the command. The bug this replaced # was a malformed argv: the unit ran the right verb, forgejo # rejected it, and every check that read the rendered script # still passed. A registration that does not appear in the # source list did not happen, whatever the exit code said. if ! forgejo admin auth list | grep -q "[[:space:]]${ssoSourceName}[[:space:]]"; then echo "login source ${ssoSourceName} is absent after registration" >&2 exit 1 fi ''; }; # Provision the swarm-controller's forge account + access token. # Unconditional (not gated on any SSO/co-location option): forge # is a swarm-wide singleton, so this account exists wherever # forge does, regardless of which host (if any) actually runs # swarm-controller — the design requirement was explicit that # this must work even when forge and swarm-controller don't # share a host. systemd.services.forgejo-swarm-controller-account = { description = "provision the swarm-controller's forge account + access token"; after = [ "forgejo.service" ]; requires = [ "forgejo.service" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; User = "forgejo"; Group = "forgejo"; SyslogIdentifier = "forgejo-swarm-controller-account"; }; # `FORGEJO_CUSTOM`, same reason as `forgejo-sso-source` above: # every `forgejo admin` invocation needs it to find the app.ini # the upstream module wrote, not just the auth-source verb. environment = { FORGEJO_CUSTOM = "/var/lib/forgejo/custom"; }; path = [ deployCfg.forgejo.package pkgs.coreutils pkgs.gnugrep ]; script = '' set -euo pipefail # Idempotent by query, same reasoning as the SSO unit: assert # the effect, not the command's exit code. `--admin`: the # controller creates repos in the agents org on the swarm's # behalf, which needs org-admin standing to do at all. Only # applies to a fresh account — `forgejo admin` has no # in-place "promote to admin" verb (checked: `admin user` # offers create/list/change-password/delete/ # generate-access-token/must-change-password/reset-mfa and # nothing else), and mara's call was not to delete+recreate # an already-provisioned non-admin account to work around # that (every hive from before this fix has one) — she # promotes those by hand instead. if ! forgejo admin user list | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then forgejo admin user create \ --username ${lib.escapeShellArg swarmControllerForgeUser} \ --email ${lib.escapeShellArg "${swarmControllerForgeUser}@hyperhive.local"} \ --admin --random-password --must-change-password=false echo "created forge account ${swarmControllerForgeUser}" fi if ! forgejo admin user list | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then echo "forge account ${swarmControllerForgeUser} absent after creation" >&2 exit 1 fi token_path=${lib.escapeShellArg swarmControllerTokenPath} # Token minting is idempotent by the token FILE's existence, # not a separate stamp — same reasoning `hive-forge-oidc-secret` # (below) uses for its own delivery: the file is both the # record of "already done" and the thing that has to survive, # so a state wipe that deletes it correctly triggers a # re-mint instead of being silently masked by a stamp that # outlived what it claims exists. if [ ! -s "$token_path" ]; then # `write:admin` added for `swarm-controller::forge::Client:: # ensure_agent_user` (the `CreateForgeUser` jobq node) — # Forgejo's `admin_create_user` HTTP endpoint refused every # call with "token does not have at least one of required # scope(s): [write:admin]" without it, an already-deployed # swarm hitting this the moment that node shipped. A host # whose token was minted before this scope was added won't # pick it up automatically — the `[ ! -s "$token_path" ]` # guard above only mints when the file is absent, by design # (see its own comment) — so an existing deployment needs # its token file deleted to force a re-mint with the new # scope. # # Token name carries a timestamp suffix rather than the # fixed `swarm-controller-boot` this used to be — a real # incident, not a hypothetical: deleting the token *file* # (the remediation above) doesn't delete the token Forgejo # already has under that name, so re-running this unit hit # "Command error: access token name has been used already" # and failed outright. Same fix `hive-c0re::forge::users:: # mint_token` already uses for its own agent tokens, for # the identical reason — a monotonic suffix means a re-mint # can never collide with whatever this account already has. out=$(forgejo admin user generate-access-token \ --username ${lib.escapeShellArg swarmControllerForgeUser} \ --token-name "swarm-controller-boot-$(date +%s)" \ --scopes "write:repository,write:organization,write:issue,read:user,write:admin") token=$(printf '%s' "$out" | grep -oE '[0-9a-f]{32,}' | head -n1) if [ -z "$token" ]; then echo "no token-shaped word in forgejo's generate-access-token output" >&2 exit 1 fi umask 0177 printf '%s' "$token" > "$token_path" echo "minted forge access token for ${swarmControllerForgeUser}" fi ''; }; }; }; # The client this login source authenticates as is registered by # ../glue-forge-oidc-client.nix, wherever authelia runs. When authelia # is here too, this host minted the secret, so it can say where the # forge will find it. services.hyperhive.deploy.forgejo.sso.clientSecretFile = lib.mkIf ssoLocal ( lib.mkDefault forgeSecretPath ); # The delivery. It runs on the HOST because that is the only place # both container trees are addressable: they share this host's # network namespace, which makes them feel co-located, but their # filesystem roots are separate — the forge cannot open a path inside # authelia's tree however local the port looks. # # ⚠️ Deliberately a copy and not a `bindMounts` entry. # nixos-container refuses to start when a bind source is missing, and # this secret does not exist until authelia's first boot has minted # it — so binding it would make the forge wait on a file that waits # on a container that starts after it. On a fresh hive that is a # permanent stall presenting as "the forge is broken", several layers # from its cause. systemd.services.hive-forge-oidc-secret = lib.mkIf ssoLocal { description = "deliver the forge's OIDC client secret from authelia"; after = [ "container@${autheliaCfg.machine}.service" ]; requires = [ "container@${autheliaCfg.machine}.service" ]; before = [ "container@hive-forge.service" ]; wantedBy = [ "container@hive-forge.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = "hive-forge-oidc-secret"; # Longer than the 120s bounded wait below. `DefaultTimeoutStartSec` # is 90s, so without this systemd kills the unit at 90 — before it # can emit the message naming the file it was waiting for. TimeoutStartSec = "180s"; }; path = [ pkgs.coreutils ]; script = '' set -euo pipefail src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${cfg.sso.clientId}.secret"} dst=${lib.escapeShellArg "/var/lib/nixos-containers/hive-forge${forgeSecretPath}"} # authelia's container is up, but its first-boot generator may # still be minting. Bounded wait, then fail: a silent skip here # produces a forge whose SSO button 401s, which is the failure # this whole design is trying not to ship. for _ in $(seq 1 60); do [ -s "$src" ] && break sleep 2 done if [ ! -s "$src" ]; then echo "authelia has not minted $src after 120s" >&2 exit 1 fi # The owning uid is DISCOVERED, not assumed: read it off the # forge container's own state dir. Whatever uid maps to forgejo # inside that container is by definition the one that owns the # directory it was created with, and hardcoding a number here # would be a second place for it to be wrong. uid=$(stat -c %u /var/lib/nixos-containers/hive-forge/var/lib/forgejo) install -D -m 0400 -o "$uid" -g "$uid" "$src" "$dst" ''; }; # The reverse direction of the unit above: collect the # swarm-controller's freshly-minted forge token OUT of the container # onto a host path other units/hosts can read. Same "only the host can # see both trees" reasoning, same ordering constraint — this can only # depend on the container being up (`container@hive-forge.service`), # not on the specific in-container oneshot that mints the token # (containers run their own systemd instance, invisible to this one by # unit name) — so it polls the same bounded way # `hive-forge-oidc-secret` does while waiting on authelia above. # # Unconditional, like the in-container unit it collects from: forge is # a swarm-wide singleton, so the token always gets minted and always # gets collected here, regardless of whether swarm-controller runs on # this host, another host, or nowhere in this swarm at all. systemd.services.hive-forge-swarm-controller-token = { description = "collect the swarm-controller's forge token onto the host"; after = [ "container@hive-forge.service" ]; wantedBy = [ "container@hive-forge.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = "hive-forge-swarm-controller-token"; # Longer than the 120s bounded wait below. `DefaultTimeoutStartSec` # is 90s, so without this systemd kills the unit at 90 — before it # can emit the message naming the file it was waiting for. TimeoutStartSec = "180s"; }; path = [ pkgs.coreutils ]; script = '' set -euo pipefail src=${lib.escapeShellArg "/var/lib/nixos-containers/hive-forge${swarmControllerTokenPath}"} dst=${lib.escapeShellArg deployCfg.forgejo.hostSwarmControllerTokenFile} for _ in $(seq 1 60); do [ -s "$src" ] && break sleep 2 done if [ ! -s "$src" ]; then echo "forge has not minted $src after 120s" >&2 exit 1 fi install -D -m 0400 "$src" "$dst" ''; }; networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall { allowedTCPPorts = [ cfg.httpPort cfg.sshPort ]; }; # Forward the declared pull-mirrors to the swarm-controller, which seeds # them in its swarm-wide forge-objects pass # (`swarm-controller/src/forge/objects.rs`), beside the orgs they live in. # JSON-encoded list of { upstream, dest }; `[]` when nothing to seed. # # ⚠️ This host's `mirrors` reach a controller on THIS host only. A # controller elsewhere reads its own host's list, so mirrors declared # here would never be seeded; the warning below says so at eval rather # than leaving it to be noticed as a missing repo. systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS = lib.mkIf deployCfg.swarm-controller.enable (builtins.toJSON effectiveMirrors); warnings = lib.optional (effectiveMirrors != [ ] && !deployCfg.swarm-controller.enable) '' services.hyperhive.deploy.forgejo.mirrors (or the actions/checkout mirror deploy.forgejo.ci.enable adds) is set on a host that does not run the swarm-controller. The controller seeds mirrors from its own host's list, so these are not seeded: declare them on the swarm-controller's host. ''; }; }