Watch
0
0
Fork
You've already forked hyperhive
0

forge: its own netns, HTTP on a unix socket, git over HTTPS only

The hive-forge container leaves the host netns and listens on no TCP port: Forgejo serves HTTP on a unix socket the gateway proxies to, and the built-in SSH server is off. The httpPort, sshPort and openFirewall options are removed with a message each.
This commit is contained in:
maia 2026-11-22 19:44:19 +01:00 • committed by mara
commit fc70f538ac
3 changed files with 153 additions and 117 deletions

View file

@ -527,10 +527,6 @@ in
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "matrix" "allowEncryption" ]
[ "services" "hyperhive" "deploy" "matrix" "allowEncryption" ]

View file

@ -95,8 +95,7 @@ let
forwarderClientId = cfg.otel.clientId;
forwarderEndpoint = "https://${config.services.hyperhive.swarm.otel.domain}/hive-forge";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
# Inside the forge's own netns, so no other collector's port can collide.
forwarderTelemetryPort = 8898;
# The forge host's store identity, handed into the container as system
@ -156,15 +155,22 @@ let
lib.optionals actionsFromSelf actionMirrors ++ hyperhiveMirrors
);
# Share host netns — forgejo's HTTP / SSH listeners then look
# exactly like a host-side service, no port forwarding dance,
# and agent containers (which also share host netns) reach it
# via plain `localhost`.
privateNetwork = false;
# The forge's own netns, on the hive bridge. It listens on no TCP port at
# all: the gateway reaches it through `socketPath`, and nothing else does
# (git goes over HTTPS through the gateway; there is no SSH server).
privateNetwork = true;
# The socket directory, shared with the host so nginx can reach in. Fixed,
# not an option: the directory's mode is what keeps the socket private, so
# it holds this one socket and nothing else. Same shape as
# ../hive-matrix.nix.
socketDir = "/run/hive-forge";
socketPath = "${socketDir}/forgejo.sock";
nginxGid = config.ids.gids.nginx;
in
{
# Private Forgejo in a `hive-forge` nixos-container, shared host
# netns, on the one host with `deploy.forgejo.enable`. Agents reach it
# Private Forgejo in a `hive-forge` nixos-container on the hive bridge,
# on the one host with `deploy.forgejo.enable`. Agents reach it
# at `forge.<domain>` via the gateway. State at
# `/var/lib/nixos-containers/hive-forge/var/lib/forgejo/` survives
# restart. See `docs/4-implementation/networking/gateway.md::hive-forge container shape`.
@ -174,6 +180,16 @@ in
# Set on every host, not only the forge's: the controller's host forwards the
# list whether or not the forge runs beside it.
{ services.hyperhive.deploy.forgejo.effectiveMirrors = effectiveMirrors; }
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "forgejo" "httpPort" ] ''
The forge listens on a unix socket inside its own network namespace, so
it has no TCP port to name. Remove this definition.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "forgejo" "openFirewall" ] ''
The forge has no host port to open: the gateway reaches it through a
unix socket, and git goes over HTTPS through the gateway's vhost, whose
firewall switch is `services.hyperhive.gateway.openFirewall`. Remove
this definition.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "forgejo" "sso" "clientSecretFile" ] ''
The forge's OIDC client secret is fetched from the swarm secret store
(swarm-bao-forge-oidc) into a fixed runtime path. Remove this
@ -197,17 +213,6 @@ in
# where its host-local secrets sit. Same rule as ../swarm-victorialogs.nix,
# and the renames are in ../deploy.nix with the rest.
options.services.hyperhive.deploy.forgejo = {
httpPort = lib.mkOption {
type = lib.types.port;
default = 3000;
description = ''
TCP port the forge serves HTTP on. Default 3000 sits outside
hyperhive's claimed ranges (dashboard 7000, every agent in
8100..8999 via FNV-1a hash). Change this if you already have
another forgejo bound to 3000.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.forgejo;
@ -223,21 +228,6 @@ in
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Open `httpPort` + `sshPort` in the host firewall. Off by
default (secure-by-default): agent containers reach the forge
at `forge.<domain>` via the gateway (not directly), and the
host reaches it on loopback — so the firewall opens only
matter for access from outside the host. Flip to `true` when
you want the operator's browser or external git clients to
hit the forge directly.
'';
};
mirrors = lib.mkOption {
type = lib.types.listOf (
lib.types.submodule {
@ -349,13 +339,13 @@ in
# `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for
# git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB
# clones). SSH stays direct on `forge.sshPort`. See
# `docs/4-implementation/networking/gateway.md`.
# clones). Git goes over HTTPS through this vhost; there is no SSH
# server. See `docs/4-implementation/networking/gateway.md`.
services.hyperhive.gateway.vhosts = {
"${cfg.domain}" = {
locations = {
"/" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/";
proxyPass = "http://unix:${socketPath}:/";
proxyWebsockets = true;
extraConfig = ''
proxy_buffering off;
@ -376,7 +366,7 @@ in
# `publishedScrapeTargets.forgejo` in ../swarm-otel-service.nix;
# the two must agree, since that string is also the token's audience.
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/metrics";
proxyPass = "http://unix:${socketPath}:/metrics";
extraConfig = ''
auth_request /__forge_metrics_authz;
'';
@ -500,20 +490,6 @@ in
Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
config.services.hyperhive.otel.telemetryPort
config.services.hyperhive.deploy.bao.otel.telemetryPort
config.services.hyperhive.deploy.nats.otel.telemetryPort
]);
message = ''
The forge's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
@ -538,6 +514,7 @@ in
# container's own forwarder reads it (../../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
# Self-signed mode: bind the hive's trust bundle read-only so forgejo
# can trust the gateway's self-signed leaf for outbound webhook
# delivery. The bind-mount, the `container@` ordering and the bundle
@ -547,11 +524,19 @@ in
# The OIDC secret's directory: a container's `/run` is an nspawn tmpfs
# the host cannot see into. The host directory comes from the tmpfiles
# rule below, so it exists before the container starts.
#
# The socket directory: a bind mount rather than a path both sides
# happen to know, because a container's `/run` is an nspawn tmpfs the host
# cannot see into. The tmpfiles rule below creates it host-side first.
bindMounts = caTrust.bindMount // {
${oidcSecretDir} = {
hostPath = oidcSecretDir;
isReadOnly = true;
};
${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
};
config =
{ pkgs, ... }:
@ -631,6 +616,19 @@ in
system.stateVersion = "25.11";
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# The gateway's dnsmasq on the bridge answers every name in
# `gateway.localNames` with the bridge IP and forwards the rest
# upstream; the address itself comes from the gateway's DHCP.
networking.nameservers = [ networkCfg.bridgeIp ];
networking.interfaces.eth0.useDHCP = true;
# The group of the socket directory. A container has its own user
# database, so only the gid crosses; the name is local. Forgejo is a
# member, which lets it create its socket in the directory and keeps
# every other uid out of it.
users.groups.gateway-nginx.gid = nginxGid;
users.users.forgejo.extraGroups = [ "gateway-nginx" ];
# Authenticated as the forwarder's own OIDC client, with the forge
# host's store identity: one identity for the whole container. It is
# a system credential, so the fetch unit loads it.
@ -663,12 +661,18 @@ in
server = {
DOMAIN = cfg.domain;
ROOT_URL = effectiveRootUrl;
HTTP_PORT = deployCfg.forgejo.httpPort;
START_SSH_SERVER = true;
SSH_PORT = cfg.sshPort;
SSH_LISTEN_PORT = cfg.sshPort;
BUILTIN_SSH_SERVER_USER = "git";
DISABLE_SSH = false;
# A unix socket, not a port. `0666` is safe because the
# directory it sits in is `0770` for the gateway's group only
# (the tmpfiles rule on the host). Forgejo's primary group is
# its own, so a narrower mode would shut nginx out.
PROTOCOL = "http+unix";
HTTP_ADDR = socketPath;
UNIX_SOCKET_PERMISSION = "666";
# No SSH: git goes over HTTPS through the gateway. With the
# built-in server off and `DISABLE_SSH` set, no clone URL
# offers it.
START_SSH_SERVER = false;
DISABLE_SSH = true;
# Point forgejo at our extended static root that includes
# the custom theme CSS baked straight into the nix store.
STATIC_ROOT_PATH = staticRootWithTheme;
@ -858,6 +862,10 @@ in
# `ProtectProc = "invisible"` stays, so other processes remain hidden.
systemd.services.forgejo.serviceConfig.ProcSubset = lib.mkForce "all";
# Upstream's unit mounts everything but the state directory
# read-only, and the socket lives outside it.
systemd.services.forgejo.serviceConfig.ReadWritePaths = [ socketDir ];
# Ensure Forgejo has a usable GPG signing key so UI merges / CRUD
# commits are signed instead of erroring "does not have a signing
# key". This service (a) generates a key in forgejo's persistent
@ -1228,9 +1236,10 @@ in
forge() {
local method="$1" path="$2"
curl --silent --show-error --connect-timeout 5 --max-time 15 \
--unix-socket ${lib.escapeShellArg socketPath} \
-H @<(printf 'Authorization: token %s\n' "$token") \
-X "$method" -o "$body" -w '%{http_code}' \
"http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/api/v1$path"
"http://localhost/api/v1$path"
}
# Every other token this unit minted: one whose write to the
@ -1385,6 +1394,13 @@ in
systemd.timers.swarm-bao-forge-oidc = storeRefreshTimer "re-read the forge's OIDC client secret from the swarm secret store";
systemd.tmpfiles.rules = [
# The socket directory, created host-side before the container starts:
# nixos-container refuses to start when a bind source is missing.
# Root-owned and group nginx, `0770`: forgejo creates its socket here
# through the group (its uid is allocated inside the container, so the
# host cannot name it), and the gateway traverses it the same way.
# Everything else is kept out.
"d ${socketDir} 0770 root ${toString nginxGid} - -"
# The OIDC secret's directory, which the container binds at start.
"d ${oidcSecretDir} 0700 root root -"
# Unused: nothing writes or reads these host paths, and they may hold a
@ -1393,13 +1409,6 @@ in
"r ${oldOidcSecretPath}"
];
networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
allowedTCPPorts = [
deployCfg.forgejo.httpPort
cfg.sshPort
];
};
# The mirrors reach a controller on THIS host only (see
# ../glue-forge-controller.nix); a controller elsewhere reads its own
# host's list, so the warning below says so at eval rather than leaving

View file

@ -29,17 +29,19 @@ in
# flake + every agent's config repo (and the `internal/*` repos) — but
# only one host runs it: `deploy.forgejo.enable`, in ../deploy.nix. What
# follows is what every hive needs to reach it, wherever it runs.
options.services.hyperhive.swarm.forge = {
sshPort = lib.mkOption {
type = lib.types.port;
default = 2222;
description = ''
TCP port the forge's built-in SSH server listens on. Kept off
22 so it doesn't clash with the host's openssh. Agents push
with `ssh -p <sshPort> git@<domain>:<owner>/<repo>.git`.
'';
};
imports = [
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "forge" "sshPort" ] ''
The forge runs no SSH server: git goes over HTTPS through the gateway
(`https://<domain>/<owner>/<repo>.git`). Remove this definition, and
point any remote that used `ssh -p <sshPort>` at the HTTPS URL.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "forge" "httpPort" ] ''
The forge listens on a unix socket inside its own network namespace, so
it has no TCP port to name. Remove this definition.
'')
];
options.services.hyperhive.swarm.forge = {
domain = lib.mkOption {
type = lib.types.str;
# Under the SWARM domain, not this hive's: a swarm runs one forge