5539819330
A swarm service container shares the host netns and force-disables resolvconf, so it inherits the /etc/resolv.conf nixos-containers copies in at start (cp --remove-destination, host-side preStart, once per start) and nothing ever refreshes it. That makes the container's resolver a snapshot of the host's file at its boot instant. When that snapshot is wrong the container can never recover, and the symptom appears arbitrarily far from the cause: swarm-nats-auth cannot resolve authelia's name, so it denies every auth-callout request and the queue refuses every client with an authorization violation. Give each of the four swarm containers a oneshot that writes the resolver file itself, from the bridge IP, ordered before that container's first DNS consumer. The shape is the one every agent container already uses. networking.nameservers cannot do this: resolvconf is its only consumer and these containers disable it, so setting it renders no file while still evaluating cleanly. A static environment.etc entry cannot either -- it would have to survive etc activation landing on the regular file the host already copied there, which no eval can show.
61 lines
2.9 KiB
Nix
61 lines
2.9 KiB
Nix
# The resolver file a swarm service container writes for itself.
|
|
#
|
|
# Every swarm service container shares the host netns (`privateNetwork =
|
|
# false`) and force-disables `resolvconf`, so that the `/etc/resolv.conf`
|
|
# `nixos-containers` copies in at start is not regenerated empty. That copy
|
|
# is a `cp --remove-destination` in the host-side preStart, run ONCE per
|
|
# container start — so the container's resolver is a snapshot of the host's
|
|
# file at its boot instant, and stays that snapshot for its whole life.
|
|
#
|
|
# A snapshot is not a resolver. Anything that makes the host's file wrong at
|
|
# that one instant — a resolvconf regeneration mid-deploy, a host that has
|
|
# not yet pointed itself at the bridge — leaves the container with a resolver
|
|
# it can never recover from, and the symptom surfaces arbitrarily far from
|
|
# the cause: a queue refusing every client because the auth-callout responder
|
|
# cannot look up its IdP.
|
|
#
|
|
# So the container writes the file itself, on every boot, from the one
|
|
# address that is correct on both sides of a netns boundary (the bridge IP —
|
|
# see `hive-gateway/default.nix`, which forces the host to the same value).
|
|
{
|
|
bridgeIp,
|
|
# Units in this container that resolve a name. The caller names them
|
|
# because this module cannot know them, and an unordered resolver write
|
|
# is a race that only shows up on a cold boot.
|
|
dnsConsumers ? [ ],
|
|
}:
|
|
{ lib, pkgs, ... }:
|
|
{
|
|
# ⚠️ `networking.nameservers` CANNOT replace this unit. `resolvconf` is its
|
|
# only consumer, and these containers disable it — so setting it renders no
|
|
# file and changes no behaviour, while still evaluating and deploying
|
|
# perfectly cleanly. It reads like a fix and is a no-op.
|
|
#
|
|
# ⚠️ Nor can a static `environment.etc."resolv.conf"`: that has to survive
|
|
# `etc` activation landing on top of the regular file the host already
|
|
# copied there, which is a runtime property no eval can demonstrate. This
|
|
# oneshot shape is the one every agent container already uses
|
|
# (`nix/agent-modules/network.nix`), so it has runtime evidence behind it.
|
|
systemd.services.swarm-bridge-dns = {
|
|
description = "point resolv.conf at the hive bridge resolver";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "local-fs.target" ];
|
|
before = [ "network-online.target" ] ++ dnsConsumers;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
# Pin the journal identity; without it systemd derives one from the
|
|
# generated script's store path (an opaque `<hash>-…-start`).
|
|
SyslogIdentifier = "swarm-bridge-dns";
|
|
};
|
|
path = [ pkgs.coreutils ];
|
|
script = ''
|
|
set -eu
|
|
# `rm` first: this is a regular file the host copied in, not something
|
|
# to write through, and a leftover symlink would redirect the write.
|
|
rm -f /etc/resolv.conf
|
|
printf 'nameserver %s\n' ${lib.escapeShellArg bridgeIp} > /etc/resolv.conf
|
|
echo "swarm-bridge-dns: resolv.conf -> nameserver ${bridgeIp}"
|
|
'';
|
|
};
|
|
}
|