mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

run swarm-attic in its own network namespace

Browse source 
the gateway reaches atticd over a unix socket a relay in the container
forwards, and the host unit that creates the cache dials its public name.
This commit is contained in:
atlas 2026-11-02 23:56:41 +01:00 • committed by mara
commit 1d3ab2fae5
2 changed files with 198 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

20
nix/host-modules/lib/attic-init.nix
View file

@ -31,7 +31,8 @@
url,
# The cache's name on the server.
cache,
# Where the host reaches atticd, scheme included.
# Where the host reaches atticd, scheme included: the gateway, which answers
# 502 while atticd is down.
endpoint,
# The server's config file, which the host's `atticadm` reads as well.
configFile,
@ -110,16 +111,23 @@ in
# tokens are signed here with the key the store holds; the cache is created
# through the API by a token that lives for five minutes and is never stored.
provision = ''
# The gateway answers 502 while atticd is down, so only a status below 500
# is atticd answering.
up=0
code=000
for _ in $(seq 1 30); do
if curl -s -o /dev/null --max-time 2 ${q endpoint}; then
up=1
break
fi
code="$(curl -s -o /dev/null --max-time 2 -w '%{http_code}' ${q endpoint} || true)"
case "$code" in
"" | 000 | 5??) ;;
*)
up=1
break
;;
esac
sleep 2
done
if [ "$up" != 1 ]; then
echo "atticd did not answer at ${endpoint}." >&2
echo "atticd did not answer at ${endpoint} (last status $code)." >&2
exit 1
fi

116
nix/host-modules/swarm-attic.nix
View file

@ -10,10 +10,13 @@
# (its address, name and public key), each only while absent
# (./lib/attic-init.nix). Neither needs a manual step.
#
# The container shares the host's network namespace, as ./swarm-victorialogs.nix
# does, and atticd listens on loopback only: the gateway's vhost is the way in,
# and attic's own tokens are the authentication. Storage is local disk under the
# container's state.
# The container has a network namespace of its own on the hive bridge. atticd
# listens on that namespace's loopback only, because it cannot listen on a unix
# socket. A socket-activated relay in the container forwards a unix socket to it,
# and the gateway's nginx proxies to that socket. The host units reach the cache
# as every client does, at its published name through the gateway. Attic's own
# tokens are the authentication. Storage is local disk under the container's
# state.
#
# The two `swarm.attic` options every host reads (./swarm-attic-options.nix)
# default to this cache's names on every host, whether or not it runs here.
@ -45,8 +48,17 @@ let
secretDir = "/run/swarm-attic-secrets";
envFile = "${secretDir}/server.env";
# Where the host units reach atticd: the shared loopback, not the gateway.
endpoint = "http://127.0.0.1:${toString cfg.port}";
# Where the host units reach atticd: the gateway, at the published name.
endpoint = "https://${atticCfg.domain}";
# The unix socket the gateway's nginx connects to. The directory's mode is
# what keeps it private, so it holds this one socket.
socketDir = "/run/swarm-attic";
socketPath = "${socketDir}/atticd.sock";
socketGroup = "gateway-nginx";
# Only the number crosses into the container, which knows the group by name.
nginxGid = config.ids.gids.nginx;
relayUnit = "swarm-attic-relay";
configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" {
listen = "127.0.0.1:${toString cfg.port}";
@ -77,7 +89,7 @@ let
urlPath = "secret/swarm/services/attic/url";
cachePath = "secret/swarm/services/attic/cache";
publicKeyPath = "secret/swarm/services/attic/public-key";
url = "https://${atticCfg.domain}";
url = endpoint;
cache = atticCfg.cache;
inherit
endpoint
@ -100,10 +112,10 @@ let
forwarderClientId = atticCfg.otel.clientId;
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there.
# Loopback ports of the forwarder's own metrics and of its OTLP/HTTP metrics
# receiver, which the cache-size exporter pushes to, in the container's own
# netns.
forwarderTelemetryPort = 8903;
# The forwarder's OTLP/HTTP metrics receiver, which the cache-size exporter
# pushes to. Not 4318: the hive collector holds that in the host netns.
forwarderOtlpPort = 8905;
# The exporter's unit and query. `V` is attic's stored value for a valid
@ -144,6 +156,20 @@ let
};
in
{
# The tokens unit dials the gateway's name, whose leaf the system store lacks
# when the gateway serves a self-signed one. `hostUnit`: it reads the host's
# CA path and waits on `hive-tls-ca.service`. `enable`: `imports` is
# unconditional, so without it a host that runs no cache gets a bundle unit.
imports = [
(caTrust.trustBundle {
inherit pkgs;
name = "swarm-bao-attic-tokens";
consumers = [ "swarm-bao-attic-tokens" ];
hostUnit = true;
inherit (cfg) enable;
})
];
options.services.hyperhive.deploy.attic = {
package = lib.mkOption {
type = lib.types.package;
@ -163,11 +189,9 @@ in
type = lib.types.port;
default = 8910;
description = ''
Loopback port atticd listens on, which the gateway proxies to.
⚠️ Every swarm container shares the host's network namespace, so this
is a swarm-wide claim: two modules picking the same number collide at
runtime with no bind error. No repo-wide check catches it.
Port atticd listens on, on the loopback of the container's own network
namespace. The container's relay forwards the gateway's unix socket to
it; nothing else dials it.
'';
};
@ -276,6 +300,9 @@ in
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# The hive bridge carries the container's DHCP lease and its resolver.
services.hyperhive.network.enable = lib.mkDefault true;
# `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks
# its own tokens, and a nix client cannot follow a login redirect. A store
# path is uploaded as one request that can run for minutes, so the body is
@ -283,7 +310,7 @@ in
services.hyperhive.gateway.vhosts."${atticCfg.domain}" = {
forceSSL = true;
locations."/" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}";
proxyPass = "http://unix:${socketPath}";
extraConfig = ''
client_max_body_size 0;
proxy_request_buffering off;
@ -295,8 +322,12 @@ in
# The key's directory is the container's read-only bind source, so it has
# to exist before the container starts even when the store does not answer.
# The delivery unit creates it too, which covers a switch.
systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ];
# The delivery unit creates it too, which covers a switch. The socket
# directory is the gateway's way in: its group is nginx's.
systemd.tmpfiles.rules = [
"d ${secretDir} 0700 root root -"
"d ${socketDir} 0770 root ${toString nginxGid} - -"
];
# The signing key, created in the store if it holds none and written where
# the container reads it. Before the container and wanted by it: a store
@ -344,9 +375,12 @@ in
# facts are for the hives, which the controller copies them to.
systemd.services.swarm-bao-attic-tokens = {
description = "create the swarm cache, and its tokens and public facts in the secret store, if absent";
# `nginx.service`: the unit dials the cache through the gateway, whose
# resolver answers the name with the bridge address (`localNames` above).
after = [
"container@${machine}.service"
"swarm-bao-attic-secret.service"
"nginx.service"
];
wants = [ "container@${machine}.service" ];
wantedBy = [ "multi-user.target" ];
@ -383,12 +417,20 @@ in
extraFlags = lib.mapAttrsToList (
id: source: "--load-credential=${id}:${source}"
) forwarderStoreFiles;
privateNetwork = false;
privateNetwork = true;
hostBridge = networkCfg.bridgeName;
bindMounts = {
${secretDir} = {
hostPath = secretDir;
isReadOnly = true;
};
# A bind mount, not a path both sides know: a container's `/run` is an
# nspawn tmpfs the host cannot see into. Connecting to a socket is a
# write to it.
${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
}
// caTrust.bindMount;
@ -416,7 +458,41 @@ in
})
];
services.hyperhive.swarmContainer.privateNetwork = false;
services.hyperhive.swarmContainer.privateNetwork = true;
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
# owns resolv.conf, so dhcpcd must not write it over that unit.
networking.interfaces.eth0.useDHCP = true;
networking.dhcpcd.extraConfig = "nohook resolv.conf";
# The socket's group is nginx's, so the gateway can connect to it.
users.groups.${socketGroup}.gid = nginxGid;
systemd.sockets.${relayUnit} = {
description = "unix socket the gateway reaches atticd through";
wantedBy = [ "sockets.target" ];
listenStreams = [ socketPath ];
socketConfig = {
SocketMode = "0660";
SocketUser = "root";
SocketGroup = socketGroup;
};
};
systemd.services.${relayUnit} = {
description = "relay the gateway's connections to atticd";
requires = [ "${relayUnit}.socket" ];
after = [ "${relayUnit}.socket" ];
serviceConfig = {
ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd 127.0.0.1:${toString cfg.port}";
DynamicUser = true;
PrivateTmp = true;
PrivateDevices = true;
NoNewPrivileges = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_UNIX"
];
};
};
# Authenticated as the forwarder's own OIDC client, with the cache's
# store identity: one identity for the whole container. It is a