vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(#3363): swarm containers write their own resolver file

Browse source 
A swarm service container shares the host netns and force-disables
resolvconf, so it inherits the /etc/resolv.conf nixos-containers copies in
at start (cp --remove-destination, host-side preStart, once per start) and
nothing ever refreshes it. That makes the container's resolver a snapshot of
the host's file at its boot instant.

When that snapshot is wrong the container can never recover, and the symptom
appears arbitrarily far from the cause: swarm-nats-auth cannot resolve
authelia's name, so it denies every auth-callout request and the queue
refuses every client with an authorization violation.

Give each of the four swarm containers a oneshot that writes the resolver
file itself, from the bridge IP, ordered before that container's first DNS
consumer. The shape is the one every agent container already uses.

networking.nameservers cannot do this: resolvconf is its only consumer and
these containers disable it, so setting it renders no file while still
evaluating cleanly. A static environment.etc entry cannot either -- it would
have to survive etc activation landing on the regular file the host already
copied there, which no eval can show.
This commit is contained in:
atlas 2026-08-17 14:49:51 +02:00 • committed by mara
commit 5539819330
5 changed files with 113 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

15
nix/host-modules/swarm-authelia.nix
View file

@ -36,6 +36,7 @@
}:
let
cfg = config.services.hyperhive.swarm.authelia;
networkCfg = config.services.hyperhive.network;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
hyperhiveDomain = hyperhiveCfg.domain;
@ -740,6 +741,13 @@ in
config =
{ ... }:
{
imports = [
(import ./swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "authelia-${instance}.service" ];
})
];
system.stateVersion = "26.05";
# The authelia binary itself, so an operator who gets a shell
@ -753,9 +761,10 @@ in
# firewall.service would rewrite the HOST ruleset at every
# boot. The host firewall owns all filtering.
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact — resolvconf's
# host-tracking would regenerate it to an empty file, since
# the host's copy doesn't cross the boundary after start.
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy doesn't
# cross the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
# authelia's own secrets, generated in-container on first

61
nix/host-modules/swarm-container-resolver.nix Normal file
View file

@ -0,0 +1,61 @@
# The resolver file a swarm service container writes for itself.
#
# Every swarm service container shares the host netns (`privateNetwork =
# false`) and force-disables `resolvconf`, so that the `/etc/resolv.conf`
# `nixos-containers` copies in at start is not regenerated empty. That copy
# is a `cp --remove-destination` in the host-side preStart, run ONCE per
# container start — so the container's resolver is a snapshot of the host's
# file at its boot instant, and stays that snapshot for its whole life.
#
# A snapshot is not a resolver. Anything that makes the host's file wrong at
# that one instant — a resolvconf regeneration mid-deploy, a host that has
# not yet pointed itself at the bridge — leaves the container with a resolver
# it can never recover from, and the symptom surfaces arbitrarily far from
# the cause: a queue refusing every client because the auth-callout responder
# cannot look up its IdP.
#
# So the container writes the file itself, on every boot, from the one
# address that is correct on both sides of a netns boundary (the bridge IP —
# see `hive-gateway/default.nix`, which forces the host to the same value).
{
bridgeIp,
# Units in this container that resolve a name. The caller names them
# because this module cannot know them, and an unordered resolver write
# is a race that only shows up on a cold boot.
dnsConsumers ? [ ],
}:
{ lib, pkgs, ... }:
{
# ⚠️ `networking.nameservers` CANNOT replace this unit. `resolvconf` is its
# only consumer, and these containers disable it — so setting it renders no
# file and changes no behaviour, while still evaluating and deploying
# perfectly cleanly. It reads like a fix and is a no-op.
#
# ⚠️ Nor can a static `environment.etc."resolv.conf"`: that has to survive
# `etc` activation landing on top of the regular file the host already
# copied there, which is a runtime property no eval can demonstrate. This
# oneshot shape is the one every agent container already uses
# (`nix/agent-modules/network.nix`), so it has runtime evidence behind it.
systemd.services.swarm-bridge-dns = {
description = "point resolv.conf at the hive bridge resolver";
wantedBy = [ "multi-user.target" ];
after = [ "local-fs.target" ];
before = [ "network-online.target" ] ++ dnsConsumers;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Pin the journal identity; without it systemd derives one from the
# generated script's store path (an opaque `<hash>-…-start`).
SyslogIdentifier = "swarm-bridge-dns";
};
path = [ pkgs.coreutils ];
script = ''
set -eu
# `rm` first: this is a regular file the host copied in, not something
# to write through, and a leftover symlink would redirect the write.
rm -f /etc/resolv.conf
printf 'nameserver %s\n' ${lib.escapeShellArg bridgeIp} > /etc/resolv.conf
echo "swarm-bridge-dns: resolv.conf -> nameserver ${bridgeIp}"
'';
};
}

15
nix/host-modules/swarm-grafana.nix
View file

@ -14,6 +14,7 @@
}:
let
cfg = config.services.hyperhive.swarm.grafana;
networkCfg = config.services.hyperhive.network;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
tlsCfg = hyperhiveCfg.tls;
@ -351,15 +352,23 @@ in
config =
{ ... }:
{
imports = [
(import ./swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "grafana.service" ];
})
];
system.stateVersion = "26.05";
# This container shares the host netns, so its own firewall.service
# would rewrite the HOST ruleset at every boot. The host firewall
# owns all filtering.
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact — resolvconf's
# host-tracking would regenerate it to an empty file, since the
# host's copy doesn't cross the boundary after start.
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy doesn't cross
# the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
# Self-signed mode: Grafana is Go, and Go's `SSL_CERT_FILE`

19
nix/host-modules/swarm-nats.nix
View file

@ -8,6 +8,7 @@ let
cfg = config.services.hyperhive.swarm.nats;
autheliaCfg = config.services.hyperhive.swarm.authelia;
autheliaUrl = autheliaCfg.url;
networkCfg = config.services.hyperhive.network;
# The account the callout responder authenticates as, and the account
# authorized clients are placed in. Two accounts rather than one: an
@ -436,15 +437,27 @@ in
config =
{ ... }:
{
imports = [
(import ./swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
# The responder introspects authelia BY NAME on every auth
# request, and a responder that cannot resolve it denies
# every client — so it must not start before the resolver
# file exists.
dnsConsumers = [ "swarm-nats-auth.service" ];
})
];
system.stateVersion = "26.05";
# Shared host netns: this container's own firewall.service
# would rewrite the HOST ruleset at every boot. The host
# firewall owns all filtering.
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact — resolvconf's
# host-tracking regenerates it empty, since the host's copy
# does not cross the boundary after start.
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy does not
# cross the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
services.nats = {

15
nix/host-modules/swarm-victoriametrics.nix
View file

@ -17,6 +17,7 @@
}:
let
cfg = config.services.hyperhive.swarm.victoriametrics;
networkCfg = config.services.hyperhive.network;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
swarmDomain = hyperhiveCfg.swarm.domain;
@ -130,15 +131,23 @@ in
config =
{ ... }:
{
imports = [
(import ./swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "victoriametrics.service" ];
})
];
system.stateVersion = "26.05";
# This container shares the host netns, so its own firewall.service
# would rewrite the HOST ruleset at every boot. The host firewall
# owns all filtering.
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact — resolvconf's
# host-tracking would regenerate it to an empty file, since the
# host's copy doesn't cross the boundary after start.
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy doesn't cross
# the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
services.victoriametrics = {