forge: its own netns, HTTP on a unix socket, git over HTTPS only
The hive-forge container leaves the host netns and listens on no TCP port: Forgejo serves HTTP on a unix socket the gateway proxies to, and the built-in SSH server is off. The httpPort, sshPort and openFirewall options are removed with a message each.
This commit is contained in:
parent
e1434e677b
commit
fc70f538ac
3 changed files with 153 additions and 117 deletions
|
|
@ -527,10 +527,6 @@ in
|
|||
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
|
||||
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
|
||||
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "matrix" "allowEncryption" ]
|
||||
[ "services" "hyperhive" "deploy" "matrix" "allowEncryption" ]
|
||||
|
|
|
|||
|
|
@ -95,8 +95,7 @@ let
|
|||
forwarderClientId = cfg.otel.clientId;
|
||||
forwarderEndpoint = "https://${config.services.hyperhive.swarm.otel.domain}/hive-forge";
|
||||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||
# In the host netns: distinct from every other collector's there, which the
|
||||
# assertion below checks.
|
||||
# Inside the forge's own netns, so no other collector's port can collide.
|
||||
forwarderTelemetryPort = 8898;
|
||||
|
||||
# The forge host's store identity, handed into the container as system
|
||||
|
|
@ -156,15 +155,22 @@ let
|
|||
lib.optionals actionsFromSelf actionMirrors ++ hyperhiveMirrors
|
||||
);
|
||||
|
||||
# Share host netns — forgejo's HTTP / SSH listeners then look
|
||||
# exactly like a host-side service, no port forwarding dance,
|
||||
# and agent containers (which also share host netns) reach it
|
||||
# via plain `localhost`.
|
||||
privateNetwork = false;
|
||||
# The forge's own netns, on the hive bridge. It listens on no TCP port at
|
||||
# all: the gateway reaches it through `socketPath`, and nothing else does
|
||||
# (git goes over HTTPS through the gateway; there is no SSH server).
|
||||
privateNetwork = true;
|
||||
|
||||
# The socket directory, shared with the host so nginx can reach in. Fixed,
|
||||
# not an option: the directory's mode is what keeps the socket private, so
|
||||
# it holds this one socket and nothing else. Same shape as
|
||||
# ../hive-matrix.nix.
|
||||
socketDir = "/run/hive-forge";
|
||||
socketPath = "${socketDir}/forgejo.sock";
|
||||
nginxGid = config.ids.gids.nginx;
|
||||
in
|
||||
{
|
||||
# Private Forgejo in a `hive-forge` nixos-container, shared host
|
||||
# netns, on the one host with `deploy.forgejo.enable`. Agents reach it
|
||||
# Private Forgejo in a `hive-forge` nixos-container on the hive bridge,
|
||||
# on the one host with `deploy.forgejo.enable`. Agents reach it
|
||||
# at `forge.<domain>` via the gateway. State at
|
||||
# `/var/lib/nixos-containers/hive-forge/var/lib/forgejo/` survives
|
||||
# restart. See `docs/4-implementation/networking/gateway.md::hive-forge container shape`.
|
||||
|
|
@ -174,6 +180,16 @@ in
|
|||
# Set on every host, not only the forge's: the controller's host forwards the
|
||||
# list whether or not the forge runs beside it.
|
||||
{ services.hyperhive.deploy.forgejo.effectiveMirrors = effectiveMirrors; }
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "forgejo" "httpPort" ] ''
|
||||
The forge listens on a unix socket inside its own network namespace, so
|
||||
it has no TCP port to name. Remove this definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "forgejo" "openFirewall" ] ''
|
||||
The forge has no host port to open: the gateway reaches it through a
|
||||
unix socket, and git goes over HTTPS through the gateway's vhost, whose
|
||||
firewall switch is `services.hyperhive.gateway.openFirewall`. Remove
|
||||
this definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "forgejo" "sso" "clientSecretFile" ] ''
|
||||
The forge's OIDC client secret is fetched from the swarm secret store
|
||||
(swarm-bao-forge-oidc) into a fixed runtime path. Remove this
|
||||
|
|
@ -197,17 +213,6 @@ in
|
|||
# where its host-local secrets sit. Same rule as ../swarm-victorialogs.nix,
|
||||
# and the renames are in ../deploy.nix with the rest.
|
||||
options.services.hyperhive.deploy.forgejo = {
|
||||
httpPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 3000;
|
||||
description = ''
|
||||
TCP port the forge serves HTTP on. Default 3000 sits outside
|
||||
hyperhive's claimed ranges (dashboard 7000, every agent in
|
||||
8100..8999 via FNV-1a hash). Change this if you already have
|
||||
another forgejo bound to 3000.
|
||||
'';
|
||||
};
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = pkgs.forgejo;
|
||||
|
|
@ -223,21 +228,6 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
openFirewall = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
example = true;
|
||||
description = ''
|
||||
Open `httpPort` + `sshPort` in the host firewall. Off by
|
||||
default (secure-by-default): agent containers reach the forge
|
||||
at `forge.<domain>` via the gateway (not directly), and the
|
||||
host reaches it on loopback — so the firewall opens only
|
||||
matter for access from outside the host. Flip to `true` when
|
||||
you want the operator's browser or external git clients to
|
||||
hit the forge directly.
|
||||
'';
|
||||
};
|
||||
|
||||
mirrors = lib.mkOption {
|
||||
type = lib.types.listOf (
|
||||
lib.types.submodule {
|
||||
|
|
@ -349,13 +339,13 @@ in
|
|||
|
||||
# `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for
|
||||
# git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB
|
||||
# clones). SSH stays direct on `forge.sshPort`. See
|
||||
# `docs/4-implementation/networking/gateway.md`.
|
||||
# clones). Git goes over HTTPS through this vhost; there is no SSH
|
||||
# server. See `docs/4-implementation/networking/gateway.md`.
|
||||
services.hyperhive.gateway.vhosts = {
|
||||
"${cfg.domain}" = {
|
||||
locations = {
|
||||
"/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/";
|
||||
proxyPass = "http://unix:${socketPath}:/";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = ''
|
||||
proxy_buffering off;
|
||||
|
|
@ -376,7 +366,7 @@ in
|
|||
# `publishedScrapeTargets.forgejo` in ../swarm-otel-service.nix;
|
||||
# the two must agree, since that string is also the token's audience.
|
||||
"= /metrics" = {
|
||||
proxyPass = "http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/metrics";
|
||||
proxyPass = "http://unix:${socketPath}:/metrics";
|
||||
extraConfig = ''
|
||||
auth_request /__forge_metrics_authz;
|
||||
'';
|
||||
|
|
@ -500,20 +490,6 @@ in
|
|||
Declare it in swarm-otel-service.nix.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
!(lib.elem forwarderTelemetryPort [
|
||||
config.services.hyperhive.otel.telemetryPort
|
||||
config.services.hyperhive.deploy.bao.otel.telemetryPort
|
||||
config.services.hyperhive.deploy.nats.otel.telemetryPort
|
||||
]);
|
||||
message = ''
|
||||
The forge's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
||||
collides with another collector's in the host netns: the hive
|
||||
collector's, the store forwarder's or the
|
||||
queue forwarder's.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
|
||||
|
|
@ -538,6 +514,7 @@ in
|
|||
# container's own forwarder reads it (../../container-modules/swarm-journal-forwarder.nix).
|
||||
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
|
||||
inherit privateNetwork;
|
||||
hostBridge = networkCfg.bridgeName;
|
||||
# Self-signed mode: bind the hive's trust bundle read-only so forgejo
|
||||
# can trust the gateway's self-signed leaf for outbound webhook
|
||||
# delivery. The bind-mount, the `container@` ordering and the bundle
|
||||
|
|
@ -547,11 +524,19 @@ in
|
|||
# The OIDC secret's directory: a container's `/run` is an nspawn tmpfs
|
||||
# the host cannot see into. The host directory comes from the tmpfiles
|
||||
# rule below, so it exists before the container starts.
|
||||
#
|
||||
# The socket directory: a bind mount rather than a path both sides
|
||||
# happen to know, because a container's `/run` is an nspawn tmpfs the host
|
||||
# cannot see into. The tmpfiles rule below creates it host-side first.
|
||||
bindMounts = caTrust.bindMount // {
|
||||
${oidcSecretDir} = {
|
||||
hostPath = oidcSecretDir;
|
||||
isReadOnly = true;
|
||||
};
|
||||
${socketDir} = {
|
||||
hostPath = socketDir;
|
||||
isReadOnly = false;
|
||||
};
|
||||
};
|
||||
config =
|
||||
{ pkgs, ... }:
|
||||
|
|
@ -631,6 +616,19 @@ in
|
|||
system.stateVersion = "25.11";
|
||||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||||
|
||||
# The gateway's dnsmasq on the bridge answers every name in
|
||||
# `gateway.localNames` with the bridge IP and forwards the rest
|
||||
# upstream; the address itself comes from the gateway's DHCP.
|
||||
networking.nameservers = [ networkCfg.bridgeIp ];
|
||||
networking.interfaces.eth0.useDHCP = true;
|
||||
|
||||
# The group of the socket directory. A container has its own user
|
||||
# database, so only the gid crosses; the name is local. Forgejo is a
|
||||
# member, which lets it create its socket in the directory and keeps
|
||||
# every other uid out of it.
|
||||
users.groups.gateway-nginx.gid = nginxGid;
|
||||
users.users.forgejo.extraGroups = [ "gateway-nginx" ];
|
||||
|
||||
# Authenticated as the forwarder's own OIDC client, with the forge
|
||||
# host's store identity: one identity for the whole container. It is
|
||||
# a system credential, so the fetch unit loads it.
|
||||
|
|
@ -663,12 +661,18 @@ in
|
|||
server = {
|
||||
DOMAIN = cfg.domain;
|
||||
ROOT_URL = effectiveRootUrl;
|
||||
HTTP_PORT = deployCfg.forgejo.httpPort;
|
||||
START_SSH_SERVER = true;
|
||||
SSH_PORT = cfg.sshPort;
|
||||
SSH_LISTEN_PORT = cfg.sshPort;
|
||||
BUILTIN_SSH_SERVER_USER = "git";
|
||||
DISABLE_SSH = false;
|
||||
# A unix socket, not a port. `0666` is safe because the
|
||||
# directory it sits in is `0770` for the gateway's group only
|
||||
# (the tmpfiles rule on the host). Forgejo's primary group is
|
||||
# its own, so a narrower mode would shut nginx out.
|
||||
PROTOCOL = "http+unix";
|
||||
HTTP_ADDR = socketPath;
|
||||
UNIX_SOCKET_PERMISSION = "666";
|
||||
# No SSH: git goes over HTTPS through the gateway. With the
|
||||
# built-in server off and `DISABLE_SSH` set, no clone URL
|
||||
# offers it.
|
||||
START_SSH_SERVER = false;
|
||||
DISABLE_SSH = true;
|
||||
# Point forgejo at our extended static root that includes
|
||||
# the custom theme CSS baked straight into the nix store.
|
||||
STATIC_ROOT_PATH = staticRootWithTheme;
|
||||
|
|
@ -858,6 +862,10 @@ in
|
|||
# `ProtectProc = "invisible"` stays, so other processes remain hidden.
|
||||
systemd.services.forgejo.serviceConfig.ProcSubset = lib.mkForce "all";
|
||||
|
||||
# Upstream's unit mounts everything but the state directory
|
||||
# read-only, and the socket lives outside it.
|
||||
systemd.services.forgejo.serviceConfig.ReadWritePaths = [ socketDir ];
|
||||
|
||||
# Ensure Forgejo has a usable GPG signing key so UI merges / CRUD
|
||||
# commits are signed instead of erroring "does not have a signing
|
||||
# key". This service (a) generates a key in forgejo's persistent
|
||||
|
|
@ -1228,9 +1236,10 @@ in
|
|||
forge() {
|
||||
local method="$1" path="$2"
|
||||
curl --silent --show-error --connect-timeout 5 --max-time 15 \
|
||||
--unix-socket ${lib.escapeShellArg socketPath} \
|
||||
-H @<(printf 'Authorization: token %s\n' "$token") \
|
||||
-X "$method" -o "$body" -w '%{http_code}' \
|
||||
"http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/api/v1$path"
|
||||
"http://localhost/api/v1$path"
|
||||
}
|
||||
|
||||
# Every other token this unit minted: one whose write to the
|
||||
|
|
@ -1385,6 +1394,13 @@ in
|
|||
systemd.timers.swarm-bao-forge-oidc = storeRefreshTimer "re-read the forge's OIDC client secret from the swarm secret store";
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
# The socket directory, created host-side before the container starts:
|
||||
# nixos-container refuses to start when a bind source is missing.
|
||||
# Root-owned and group nginx, `0770`: forgejo creates its socket here
|
||||
# through the group (its uid is allocated inside the container, so the
|
||||
# host cannot name it), and the gateway traverses it the same way.
|
||||
# Everything else is kept out.
|
||||
"d ${socketDir} 0770 root ${toString nginxGid} - -"
|
||||
# The OIDC secret's directory, which the container binds at start.
|
||||
"d ${oidcSecretDir} 0700 root root -"
|
||||
# Unused: nothing writes or reads these host paths, and they may hold a
|
||||
|
|
@ -1393,13 +1409,6 @@ in
|
|||
"r ${oldOidcSecretPath}"
|
||||
];
|
||||
|
||||
networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
|
||||
allowedTCPPorts = [
|
||||
deployCfg.forgejo.httpPort
|
||||
cfg.sshPort
|
||||
];
|
||||
};
|
||||
|
||||
# The mirrors reach a controller on THIS host only (see
|
||||
# ../glue-forge-controller.nix); a controller elsewhere reads its own
|
||||
# host's list, so the warning below says so at eval rather than leaving
|
||||
|
|
|
|||
|
|
@ -29,17 +29,19 @@ in
|
|||
# flake + every agent's config repo (and the `internal/*` repos) — but
|
||||
# only one host runs it: `deploy.forgejo.enable`, in ../deploy.nix. What
|
||||
# follows is what every hive needs to reach it, wherever it runs.
|
||||
options.services.hyperhive.swarm.forge = {
|
||||
sshPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 2222;
|
||||
description = ''
|
||||
TCP port the forge's built-in SSH server listens on. Kept off
|
||||
22 so it doesn't clash with the host's openssh. Agents push
|
||||
with `ssh -p <sshPort> git@<domain>:<owner>/<repo>.git`.
|
||||
'';
|
||||
};
|
||||
imports = [
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "forge" "sshPort" ] ''
|
||||
The forge runs no SSH server: git goes over HTTPS through the gateway
|
||||
(`https://<domain>/<owner>/<repo>.git`). Remove this definition, and
|
||||
point any remote that used `ssh -p <sshPort>` at the HTTPS URL.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "forge" "httpPort" ] ''
|
||||
The forge listens on a unix socket inside its own network namespace, so
|
||||
it has no TCP port to name. Remove this definition.
|
||||
'')
|
||||
];
|
||||
|
||||
options.services.hyperhive.swarm.forge = {
|
||||
domain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
# Under the SWARM domain, not this hive's: a swarm runs one forge
|
||||
|
|
|
|||
Loading…
Reference in a new issue