vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

nix: move the remaining service containers onto the swarm-container module

Browse source 
hive-ci, hive-forge, hive-matrix, swarm-authelia, swarm-bao, swarm-grafana,
swarm-nats, swarm-otel and swarm-victorialogs now import
./swarm-container.nix and drop their own copies of the stateVersion,
firewall and resolvconf lines. Each binds privateNetwork once in its
top-level let and passes it to both the host attr and the in-container
option, as swarm-victoriametrics already does.

hive-forge (25.11) and swarm-otel (the host's value) keep their own
stateVersion over the module's mkDefault. hive-ci sets privateNetwork =
true and writesOwnResolvConf = false, which leaves its firewall and
resolvconf on, as before. hive-matrix keeps its useHostResolvConf
override and static resolv.conf; its resolvconf mkForce now comes from
the module default.

Every container's system.build.toplevel drvPath and host-side attrs
evaluate identical to the parent commit.

module-eval-swarm-services-switch gains a fixture with all ten service
containers and checks that each one's in-container privateNetwork equals
its host-side value, that the nine on the host netns run no firewall or
resolvconf, that hive-ci keeps both, and that hive-forge keeps its pinned
stateVersion.

Refs #3773
This commit is contained in:
atlas 2026-09-29 19:10:54 +02:00
commit 10d579ecaf
10 changed files with 133 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

25
nix/host-modules/hive-ci.nix
View file

@ -49,6 +49,12 @@ let
consumers = [ "gitea-runner-hive" ];
};
# Private network namespace, attached to the hive bridge so the
# runner reaches the forge via the gateway — and cannot reach
# host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.).
# Requires `deploy.forgejo.behindGateway = true` (asserted in the
# config block below). See docs/networking/network.md.
privateNetwork = true;
in
{
# Forgejo Actions runner in a `hive-ci` nixos-container.
@ -235,12 +241,7 @@ in
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# Private network namespace, attached to the hive bridge so the
# runner reaches the forge via the gateway — and cannot reach
# host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.).
# Requires `deploy.forgejo.behindGateway = true` (asserted in the
# options block above). See docs/networking/network.md.
privateNetwork = true;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = {
@ -277,9 +278,17 @@ in
# Assembles system CAs + the hive CA into one bundle and sets
# `SSL_CERT_FILE` on the runner unit. See `caBundleModule` above for
# why the Node variable beside it is not enough.
imports = [ caBundleModule ];
imports = [
../container-modules/swarm-container.nix
caBundleModule
];
system.stateVersion = "26.05";
# Its own netns, so it keeps its own firewall; resolvconf stays on
# and serves `networking.nameservers` below.
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
# Point the forge domain at the bridge IP so the runner can
# reach the forge through the gateway — both for registration /

26
nix/host-modules/hive-forge/default.nix
View file

@ -102,6 +102,12 @@ let
++ lib.optional (
ciEnabled && !(lib.any (m: m.dest == actionCheckoutMirror.dest) deployCfg.forgejo.mirrors)
) actionCheckoutMirror;
# Share host netns — forgejo's HTTP / SSH listeners then look
# exactly like a host-side service, no port forwarding dance,
# and agent containers (which also share host netns) reach it
# via plain `localhost`.
privateNetwork = false;
in
{
# Private Forgejo in a `hive-forge` nixos-container, shared host
@ -665,11 +671,7 @@ in
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# Share host netns — forgejo's HTTP / SSH listeners then look
# exactly like a host-side service, no port forwarding dance,
# and agent containers (which also share host netns) reach it
# via plain `localhost`.
privateNetwork = false;
inherit privateNetwork;
# Self-signed mode: bind the public hive CA cert read-only so forgejo
# can trust the gateway's self-signed leaf for outbound webhook
# delivery. The bind-mount, the `container@` ordering and the bundle
@ -701,6 +703,7 @@ in
in
{
imports = [
./../../container-modules/swarm-container.nix
# Ask the hive's dnsmasq, not the host's resolvers. The swarm
# names — the controller's webhook endpoint, the SSO issuer —
# resolve to the bridge and have no public records, so a
@ -742,18 +745,7 @@ in
];
system.stateVersion = "25.11";
# Shared host netns: this container's own firewall.service
# would rewrite the HOST ruleset (flush nixos-fw, drop the
# host's nixos-nat-* chains) at every boot — killing the
# bridge DHCP/DNS holes and agent NAT. The host firewall owns
# all filtering; never run one in here.
networking.firewall.enable = false;
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy doesn't cross
# the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
services.hyperhive.swarmContainer = { inherit privateNetwork; };
services.forgejo = {
enable = true;

28
nix/host-modules/hive-matrix.nix
View file

@ -378,6 +378,9 @@ let
install -m 644 ${fluffychat-web-imaging}/Imaging.wasm $out/Imaging.wasm
'';
});
# Shared host netns — agents reach tuwunel at localhost:<port>.
privateNetwork = false;
in
{
# Private matrix-tuwunel homeserver wrapped in a nixos-container,
@ -1196,8 +1199,7 @@ in
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# Shared host netns — agents reach tuwunel at localhost:<port>.
privateNetwork = false;
inherit privateNetwork;
# Read-only bind of the host-managed appservice registration; tuwunel
# reads it via systemd LoadCredential below (not directly).
#
@ -1219,6 +1221,7 @@ in
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
# tuwunel's rustls verifier resolves through `rustls-native-certs`
# → `openssl-probe`, which reads `SSL_CERT_FILE` — so the
# openssl-shaped variable is the lever despite tuwunel linking no
@ -1239,14 +1242,7 @@ in
})
];
system.stateVersion = "26.05";
# Shared host netns: this container's own firewall.service
# would rewrite the HOST ruleset (flush nixos-fw, drop the
# host's nixos-nat-* chains) at every boot — killing the
# bridge DHCP/DNS holes and agent NAT. The host firewall owns
# all filtering; never run one in here.
networking.firewall.enable = false;
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Swarm-internal trust reaches tuwunel at RUNTIME, not via
# `security.pki.certificateFiles`. That option is read when the
@ -1275,13 +1271,11 @@ in
# This container always shares the host netns
# (`privateNetwork = false`), so it reaches `bridgeIp` regardless
# of agent-container isolation. See `docs/networking/network.md`.
networking = {
# resolvconf is taken out of the loop entirely; the static
# `environment.etc."resolv.conf"` below is the sole source of
# the resolver file (no `nameservers` — nothing would read it).
useHostResolvConf = lib.mkForce false;
resolvconf.enable = lib.mkForce false;
};
# resolvconf is off through `writesOwnResolvConf` (its default);
# the static `environment.etc."resolv.conf"` below is the sole
# source of the resolver file (no `nameservers` — nothing would
# read it).
networking.useHostResolvConf = lib.mkForce false;
# resolvconf is disabled above, so write the static resolver file
# explicitly — NixOS won't synthesise one from `nameservers` once

21
nix/host-modules/swarm-authelia.nix
View file

@ -384,6 +384,10 @@ let
chmod 0600 ${lib.escapeShellArg "${clientsFile}.tmp"}
mv ${lib.escapeShellArg "${clientsFile}.tmp"} ${lib.escapeShellArg clientsFile}
'';
# Shared host netns, like the forge and matrix containers: the
# gateway reaches authelia at 127.0.0.1:<port>.
privateNetwork = false;
in
{
# `enable` and both packages moved to `services.hyperhive.deploy.authelia`
@ -1213,9 +1217,7 @@ in
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# Shared host netns, like the forge and matrix containers: the
# gateway reaches authelia at 127.0.0.1:<port>.
privateNetwork = false;
inherit privateNetwork;
# Public trust bundle only, read-only. Empty when the gateway is not
# self-signed, so the whole trust path drops out cleanly.
bindMounts = caTrust.bindMount;
@ -1224,6 +1226,7 @@ in
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "authelia-${instance}.service" ];
@ -1231,7 +1234,7 @@ in
caBundleModule
];
system.stateVersion = "26.05";
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# The authelia binary itself, so an operator who gets a shell
# in here can run `authelia crypto hash generate` to make a
@ -1240,16 +1243,6 @@ in
# through the store path, and nothing puts the CLI on PATH.
environment.systemPackages = [ deployCfg.authelia.package ];
# This container shares the host netns, so its own
# firewall.service would rewrite the HOST ruleset at every
# boot. The host firewall owns all filtering.
networking.firewall.enable = false;
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy doesn't
# cross the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
# authelia's own secrets, generated in-container on first
# boot. They are jwt/session/storage keys — nothing outside
# this container ever reads them, which is what makes

22
nix/host-modules/swarm-bao.nix
View file

@ -1208,6 +1208,12 @@ let
api_addr = "https://${cfg.domain}:${toString cfg.port}";
cluster_addr = "https://${cfg.domain}:${toString (cfg.port + 1)}";
};
# Shared host netns, like every sibling swarm container. Unlike them the
# gateway is NOT the client here (see the no-vhost note at the top), so
# sharing the netns is what lets the store bind the host's own addresses
# rather than a convenience for nginx.
privateNetwork = false;
in
{
# One service, two namespaces, and the split decides who may set what.
@ -3444,11 +3450,7 @@ in
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Shared host netns, like every sibling swarm container. Unlike them the
# gateway is NOT the client here (see the no-vhost note at the top), so
# sharing the netns is what lets the store bind the host's own addresses
# rather than a convenience for nginx.
privateNetwork = false;
inherit privateNetwork;
# Only material an operator has to be able to back up crosses the
# boundary — the store's identity and its seal. The raft state
@ -3513,6 +3515,7 @@ in
{ config, ... }:
{
imports = [
../container-modules/swarm-container.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
# The forwarder resolves two swarm names of its own — the
@ -3539,14 +3542,7 @@ in
})
];
system.stateVersion = "26.05";
# Shares the host netns, so its own firewall.service would rewrite
# the HOST ruleset at every boot. The host firewall owns filtering.
networking.firewall.enable = false;
# The resolver unit imported above owns /etc/resolv.conf; leaving
# resolvconf on would let host-tracking regenerate it empty.
networking.resolvconf.enable = lib.mkForce false;
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# `${tokenGroup}` takes whatever gid this container allocates —
# nothing outside reads it. `${tpmGroup}` must take the PINNED one,

20
nix/host-modules/swarm-grafana.nix
View file

@ -170,6 +170,9 @@ let
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
# Shared host netns, like every sibling swarm container: the gateway
# reaches this at 127.0.0.1:<port>.
privateNetwork = false;
in
{
# `enable` moved to `services.hyperhive.deploy.grafana.enable` — see
@ -703,9 +706,7 @@ in
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# Shared host netns, like every sibling swarm container: the gateway
# reaches this at 127.0.0.1:<port>.
privateNetwork = false;
inherit privateNetwork;
# The socket directory, shared with the host so nginx can reach in.
#
@ -727,6 +728,7 @@ in
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "grafana.service" ];
@ -743,17 +745,7 @@ in
})
];
system.stateVersion = "26.05";
# This container shares the host netns, so its own firewall.service
# would rewrite the HOST ruleset at every boot. The host firewall
# owns all filtering.
networking.firewall.enable = false;
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy doesn't cross
# the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Grafana's `secret_key` has **no default in nixpkgs** and an
# assertion refuses the build without one — which is how the first

39
nix/host-modules/swarm-nats.nix
View file

@ -242,6 +242,19 @@ let
# grants it (./swarm-bao.nix), so the daily timer below has two weeks of
# retries before it lapses.
leafRenewSeconds = 360 * 3600;
# Shared host netns, like every sibling swarm container.
#
# ⚠️ Which is exactly why the server below must refuse everyone
# until the callout responder exists: the queue is on the host's
# own loopback, in reach of every process there and every sibling
# on this netns, and each remote hive in a multi-host swarm dials
# it directly. An unauthenticated interim state would be a hole
# rather than a rough edge.
#
# Not agent containers, though: they have a netns of their own and
# the bridge firewall does not open this port.
privateNetwork = false;
in
{
# The swarm's message queue: one NATS server, reached by every hive at
@ -723,18 +736,7 @@ in
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# Shared host netns, like every sibling swarm container.
#
# ⚠️ Which is exactly why the server below must refuse everyone
# until the callout responder exists: the queue is on the host's
# own loopback, in reach of every process there and every sibling
# on this netns, and each remote hive in a multi-host swarm dials
# it directly. An unauthenticated interim state would be a hole
# rather than a rough edge.
#
# Not agent containers, though: they have a netns of their own and
# the bridge firewall does not open this port.
privateNetwork = false;
inherit privateNetwork;
# The public trust bundle (empty when the gateway is not self-signed)
# and the queue's own TLS leaf, both read-only.
bindMounts = caTrust.bindMount // {
@ -747,6 +749,7 @@ in
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
# The responder introspects authelia BY NAME on every auth
@ -758,17 +761,7 @@ in
caBundleModule
];
system.stateVersion = "26.05";
# Shared host netns: this container's own firewall.service
# would rewrite the HOST ruleset at every boot. The host
# firewall owns all filtering.
networking.firewall.enable = false;
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy does not
# cross the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
services.hyperhive.swarmContainer = { inherit privateNetwork; };
services.nats = {
enable = true;

18
nix/host-modules/swarm-otel.nix
View file

@ -304,6 +304,12 @@ let
# An empty exporter list is not a quiet no-op — the collector rejects it.
# Now always satisfied, as a consequence of the log store always existing.
collectLogs = logExporterNames != [ ];
# Shared host netns, like every sibling swarm service: the hive tier
# reaches this collector, and this collector reaches the metrics
# store, without either crossing a network boundary that would need
# its own trust material.
privateNetwork = false;
in
{
# `enable` moved to `services.hyperhive.deploy.swarm-otel.enable` — see ./deploy.nix.
@ -1140,11 +1146,7 @@ in
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# Shared host netns, like every sibling swarm service: the hive tier
# reaches this collector, and this collector reaches the metrics
# store, without either crossing a network boundary that would need
# its own trust material.
privateNetwork = false;
inherit privateNetwork;
# The upstream credential is operator-provided and lives on the host.
# Read-only, and only when one is configured — binding a path that
@ -1194,6 +1196,7 @@ in
# taken once at boot, so without this the upstream export
# depends on the host's file having been right at that instant.
imports = [
../container-modules/swarm-container.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (config.services.hyperhive.network) bridgeIp;
dnsConsumers = [ "opentelemetry-collector.service" ];
@ -1213,10 +1216,7 @@ in
];
system.stateVersion = config.system.stateVersion;
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact — same reasoning
# as the sibling swarm containers.
networking.resolvconf.enable = lib.mkForce false;
services.hyperhive.swarmContainer = { inherit privateNetwork; };
services.opentelemetry-collector = {
enable = true;

21
nix/host-modules/swarm-victorialogs.nix
View file

@ -57,6 +57,10 @@ let
auth_request_set $target_url $scheme://$http_host$request_uri;
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
'';
# Shared host netns, like every sibling swarm container: the collector
# and Grafana reach this at 127.0.0.1:<port>.
privateNetwork = false;
in
{
# What stays here is what the store IS from any hive's point of view: the
@ -271,31 +275,20 @@ in
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# Shared host netns, like every sibling swarm container: the collector
# and Grafana reach this at 127.0.0.1:<port>.
privateNetwork = false;
inherit privateNetwork;
config =
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "victorialogs.service" ];
})
];
system.stateVersion = "26.05";
# This container shares the host netns, so its own firewall.service
# would rewrite the HOST ruleset at every boot. The host firewall
# owns all filtering.
networking.firewall.enable = false;
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy doesn't cross
# the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
services.hyperhive.swarmContainer = { inherit privateNetwork; };
services.victorialogs = {
enable = true;

48
nix/module-eval/swarm-services-switch.nix
View file

@ -94,6 +94,15 @@ let
deploy.forgejo.enable = true;
deploy.forgejo.mirrors = [ aMirror ];
};
# Every service container at once: the services-only host plus the CI
# runner, the one container on a netns of its own.
serviceContainersWithCi = hive {
deploy.hive-controller.enable = false;
deploy.allSwarmServices = true;
deploy.forgejo.ci.enable = true;
};
serviceContainers = serviceContainersWithCi.containers;
cases = [
{
# `lib.all` over an empty set holds vacuously, so the roster is counted
@ -186,19 +195,46 @@ let
&& !(s ? swarm-bao-queue-agent);
}
{
# Both values come from ../container-modules/swarm-container.nix. Read
# through the metrics store: nothing else in this suite evaluates that
# container's config.
# The in-container option drives the container's firewall, and host
# `false` with container `true` would let the container's
# firewall.service rewrite the HOST ruleset. Counted first so a fixture
# that lost a container can't pass by comparing fewer of them.
name = "every service container restates its host-side privateNetwork";
ok =
lib.length (lib.attrNames serviceContainers) == 10
&& lib.all (
c: c.privateNetwork == (c.config.services.hyperhive.swarmContainer.privateNetwork or null)
) (lib.attrValues serviceContainers);
}
{
# Both values come from ../container-modules/swarm-container.nix.
name = "a service container on the host netns runs no firewall or resolvconf of its own";
ok =
let
c = swarmServicesOnly.containers.swarm-victoriametrics.config;
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
in
!c.networking.firewall.enable && !c.networking.resolvconf.enable;
lib.length shared == 9
&& lib.all (
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
) shared;
}
{
# The `privateNetwork = true` and `writesOwnResolvConf = false` arms:
# NixOS enables both by default, so the module must leave them alone.
name = "a service container on a netns of its own keeps its firewall and resolvconf";
ok =
let
c = serviceContainers.hive-ci.config;
in
c.networking.firewall.enable && c.networking.resolvconf.enable;
}
{
name = "a service container that sets no stateVersion of its own is on 26.05";
ok = swarmServicesOnly.containers.swarm-victoriametrics.config.system.stateVersion == "26.05";
ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";
}
{
name = "a service container that pins its own stateVersion keeps it";
ok = serviceContainers.hive-forge.config.system.stateVersion == "25.11";
}
{
# An operator's explicit `false` beats every `mkDefault` assertion,