run swarm-attic in its own network namespace
the gateway reaches atticd over a unix socket a relay in the container forwards, and the host unit that creates the cache dials its public name.
This commit is contained in:
atlas
mara
parent
4440d8fb29
commit
1d3ab2fae5
8 changed files with 198 additions and 41 deletions
|
|
@ -394,9 +394,16 @@ path nobody added or fetched for 14 days goes at the next garbage collection,
|
|||
which runs every 12 hours (`deploy.attic.retentionPeriod`). The controller
|
||||
[rebuilds a closure the cache lost](#rebuild-on-a-cache-miss).
|
||||
|
||||
The gateway terminates TLS with a swarm services certificate and forwards to
|
||||
`127.0.0.1:8910` (`deploy.attic.port`) with no login in front: attic checks its
|
||||
own tokens, and a nix client can't follow a login redirect. The container logs
|
||||
The container sits on the hive bridge in a network namespace of its own. atticd
|
||||
listens on that namespace's loopback (`deploy.attic.port`), and a socket-activated
|
||||
relay forwards `/run/swarm-attic/atticd.sock` to it, a directory only the
|
||||
gateway's nginx and the container can open. The gateway terminates TLS with a
|
||||
swarm services certificate and forwards to that socket with no login in front:
|
||||
attic checks its own tokens, and a nix client can't follow a login redirect.
|
||||
The two host units reach the cache through the gateway, at
|
||||
`https://<swarm.attic.domain>`, like every other client; the host resolves the
|
||||
name through the hive resolver, and where the gateway serves a self-signed
|
||||
certificate `swarm-bao-attic-tokens` trusts the swarm's CA. The container logs
|
||||
to its journal only; its journal doesn't reach the swarm log store.
|
||||
|
||||
### External cache
|
||||
|
|
|
|||
|
|
@ -459,8 +459,9 @@ it. For an external cache, override `swarm.attic.{domain,cache}` and don't set
|
|||
`deploy.attic.enable` anywhere, or set it to `lib.mkForce false` under
|
||||
`allSwarmServices`.
|
||||
|
||||
The container shares the host's network namespace and atticd listens on loopback
|
||||
only. The gateway serves it at `swarm.attic.domain` (default
|
||||
The container sits on the hive bridge in a network namespace of its own, and atticd
|
||||
listens on that namespace's loopback only; the gateway reaches it over a unix socket.
|
||||
The gateway serves it at `swarm.attic.domain` (default
|
||||
`attic.<swarm domain>`, on every host) over TLS, with no login in front, because attic checks
|
||||
its own tokens. Storage is local disk in the container's state.
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@
|
|||
url,
|
||||
# The cache's name on the server.
|
||||
cache,
|
||||
# Where the host reaches atticd, scheme included.
|
||||
# Where the host reaches atticd, scheme included: the gateway, which answers
|
||||
# 502 while atticd is down.
|
||||
endpoint,
|
||||
# The server's config file, which the host's `atticadm` reads as well.
|
||||
configFile,
|
||||
|
|
@ -110,16 +111,23 @@ in
|
|||
# tokens are signed here with the key the store holds; the cache is created
|
||||
# through the API by a token that lives for five minutes and is never stored.
|
||||
provision = ''
|
||||
# The gateway answers 502 while atticd is down, so only a status below 500
|
||||
# is atticd answering.
|
||||
up=0
|
||||
code=000
|
||||
for _ in $(seq 1 30); do
|
||||
if curl -s -o /dev/null --max-time 2 ${q endpoint}; then
|
||||
up=1
|
||||
break
|
||||
fi
|
||||
code="$(curl -s -o /dev/null --max-time 2 -w '%{http_code}' ${q endpoint} || true)"
|
||||
case "$code" in
|
||||
"" | 000 | 5??) ;;
|
||||
*)
|
||||
up=1
|
||||
break
|
||||
;;
|
||||
esac
|
||||
sleep 2
|
||||
done
|
||||
if [ "$up" != 1 ]; then
|
||||
echo "atticd did not answer at ${endpoint}." >&2
|
||||
echo "atticd did not answer at ${endpoint} (last status $code)." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -10,10 +10,13 @@
|
|||
# (its address, name and public key), each only while absent
|
||||
# (./lib/attic-init.nix). Neither needs a manual step.
|
||||
#
|
||||
# The container shares the host's network namespace, as ./swarm-victorialogs.nix
|
||||
# does, and atticd listens on loopback only: the gateway's vhost is the way in,
|
||||
# and attic's own tokens are the authentication. Storage is local disk under the
|
||||
# container's state.
|
||||
# The container has a network namespace of its own on the hive bridge. atticd
|
||||
# listens on that namespace's loopback only, because it cannot listen on a unix
|
||||
# socket. A socket-activated relay in the container forwards a unix socket to it,
|
||||
# and the gateway's nginx proxies to that socket. The host units reach the cache
|
||||
# as every client does, at its published name through the gateway. Attic's own
|
||||
# tokens are the authentication. Storage is local disk under the container's
|
||||
# state.
|
||||
#
|
||||
# The two `swarm.attic` options every host reads (./swarm-attic-options.nix)
|
||||
# default to this cache's names on every host, whether or not it runs here.
|
||||
|
|
@ -45,8 +48,17 @@ let
|
|||
secretDir = "/run/swarm-attic-secrets";
|
||||
envFile = "${secretDir}/server.env";
|
||||
|
||||
# Where the host units reach atticd: the shared loopback, not the gateway.
|
||||
endpoint = "http://127.0.0.1:${toString cfg.port}";
|
||||
# Where the host units reach atticd: the gateway, at the published name.
|
||||
endpoint = "https://${atticCfg.domain}";
|
||||
|
||||
# The unix socket the gateway's nginx connects to. The directory's mode is
|
||||
# what keeps it private, so it holds this one socket.
|
||||
socketDir = "/run/swarm-attic";
|
||||
socketPath = "${socketDir}/atticd.sock";
|
||||
socketGroup = "gateway-nginx";
|
||||
# Only the number crosses into the container, which knows the group by name.
|
||||
nginxGid = config.ids.gids.nginx;
|
||||
relayUnit = "swarm-attic-relay";
|
||||
|
||||
configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" {
|
||||
listen = "127.0.0.1:${toString cfg.port}";
|
||||
|
|
@ -77,7 +89,7 @@ let
|
|||
urlPath = "secret/swarm/services/attic/url";
|
||||
cachePath = "secret/swarm/services/attic/cache";
|
||||
publicKeyPath = "secret/swarm/services/attic/public-key";
|
||||
url = "https://${atticCfg.domain}";
|
||||
url = endpoint;
|
||||
cache = atticCfg.cache;
|
||||
inherit
|
||||
endpoint
|
||||
|
|
@ -100,10 +112,10 @@ let
|
|||
forwarderClientId = atticCfg.otel.clientId;
|
||||
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${machine}";
|
||||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||
# In the host netns: distinct from every other collector's there.
|
||||
# Loopback ports of the forwarder's own metrics and of its OTLP/HTTP metrics
|
||||
# receiver, which the cache-size exporter pushes to, in the container's own
|
||||
# netns.
|
||||
forwarderTelemetryPort = 8903;
|
||||
# The forwarder's OTLP/HTTP metrics receiver, which the cache-size exporter
|
||||
# pushes to. Not 4318: the hive collector holds that in the host netns.
|
||||
forwarderOtlpPort = 8905;
|
||||
|
||||
# The exporter's unit and query. `V` is attic's stored value for a valid
|
||||
|
|
@ -144,6 +156,20 @@ let
|
|||
};
|
||||
in
|
||||
{
|
||||
# The tokens unit dials the gateway's name, whose leaf the system store lacks
|
||||
# when the gateway serves a self-signed one. `hostUnit`: it reads the host's
|
||||
# CA path and waits on `hive-tls-ca.service`. `enable`: `imports` is
|
||||
# unconditional, so without it a host that runs no cache gets a bundle unit.
|
||||
imports = [
|
||||
(caTrust.trustBundle {
|
||||
inherit pkgs;
|
||||
name = "swarm-bao-attic-tokens";
|
||||
consumers = [ "swarm-bao-attic-tokens" ];
|
||||
hostUnit = true;
|
||||
inherit (cfg) enable;
|
||||
})
|
||||
];
|
||||
|
||||
options.services.hyperhive.deploy.attic = {
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
|
|
@ -163,11 +189,9 @@ in
|
|||
type = lib.types.port;
|
||||
default = 8910;
|
||||
description = ''
|
||||
Loopback port atticd listens on, which the gateway proxies to.
|
||||
|
||||
⚠️ Every swarm container shares the host's network namespace, so this
|
||||
is a swarm-wide claim: two modules picking the same number collide at
|
||||
runtime with no bind error. No repo-wide check catches it.
|
||||
Port atticd listens on, on the loopback of the container's own network
|
||||
namespace. The container's relay forwards the gateway's unix socket to
|
||||
it; nothing else dials it.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -276,6 +300,9 @@ in
|
|||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||||
|
||||
# The hive bridge carries the container's DHCP lease and its resolver.
|
||||
services.hyperhive.network.enable = lib.mkDefault true;
|
||||
|
||||
# `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks
|
||||
# its own tokens, and a nix client cannot follow a login redirect. A store
|
||||
# path is uploaded as one request that can run for minutes, so the body is
|
||||
|
|
@ -283,7 +310,7 @@ in
|
|||
services.hyperhive.gateway.vhosts."${atticCfg.domain}" = {
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||||
proxyPass = "http://unix:${socketPath}";
|
||||
extraConfig = ''
|
||||
client_max_body_size 0;
|
||||
proxy_request_buffering off;
|
||||
|
|
@ -295,8 +322,12 @@ in
|
|||
|
||||
# The key's directory is the container's read-only bind source, so it has
|
||||
# to exist before the container starts even when the store does not answer.
|
||||
# The delivery unit creates it too, which covers a switch.
|
||||
systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ];
|
||||
# The delivery unit creates it too, which covers a switch. The socket
|
||||
# directory is the gateway's way in: its group is nginx's.
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${secretDir} 0700 root root -"
|
||||
"d ${socketDir} 0770 root ${toString nginxGid} - -"
|
||||
];
|
||||
|
||||
# The signing key, created in the store if it holds none and written where
|
||||
# the container reads it. Before the container and wanted by it: a store
|
||||
|
|
@ -344,9 +375,12 @@ in
|
|||
# facts are for the hives, which the controller copies them to.
|
||||
systemd.services.swarm-bao-attic-tokens = {
|
||||
description = "create the swarm cache, and its tokens and public facts in the secret store, if absent";
|
||||
# `nginx.service`: the unit dials the cache through the gateway, whose
|
||||
# resolver answers the name with the bridge address (`localNames` above).
|
||||
after = [
|
||||
"container@${machine}.service"
|
||||
"swarm-bao-attic-secret.service"
|
||||
"nginx.service"
|
||||
];
|
||||
wants = [ "container@${machine}.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
|
@ -383,12 +417,20 @@ in
|
|||
extraFlags = lib.mapAttrsToList (
|
||||
id: source: "--load-credential=${id}:${source}"
|
||||
) forwarderStoreFiles;
|
||||
privateNetwork = false;
|
||||
privateNetwork = true;
|
||||
hostBridge = networkCfg.bridgeName;
|
||||
bindMounts = {
|
||||
${secretDir} = {
|
||||
hostPath = secretDir;
|
||||
isReadOnly = true;
|
||||
};
|
||||
# A bind mount, not a path both sides know: a container's `/run` is an
|
||||
# nspawn tmpfs the host cannot see into. Connecting to a socket is a
|
||||
# write to it.
|
||||
${socketDir} = {
|
||||
hostPath = socketDir;
|
||||
isReadOnly = false;
|
||||
};
|
||||
}
|
||||
// caTrust.bindMount;
|
||||
|
||||
|
|
@ -416,7 +458,41 @@ in
|
|||
})
|
||||
];
|
||||
|
||||
services.hyperhive.swarmContainer.privateNetwork = false;
|
||||
services.hyperhive.swarmContainer.privateNetwork = true;
|
||||
|
||||
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
|
||||
# owns resolv.conf, so dhcpcd must not write it over that unit.
|
||||
networking.interfaces.eth0.useDHCP = true;
|
||||
networking.dhcpcd.extraConfig = "nohook resolv.conf";
|
||||
|
||||
# The socket's group is nginx's, so the gateway can connect to it.
|
||||
users.groups.${socketGroup}.gid = nginxGid;
|
||||
systemd.sockets.${relayUnit} = {
|
||||
description = "unix socket the gateway reaches atticd through";
|
||||
wantedBy = [ "sockets.target" ];
|
||||
listenStreams = [ socketPath ];
|
||||
socketConfig = {
|
||||
SocketMode = "0660";
|
||||
SocketUser = "root";
|
||||
SocketGroup = socketGroup;
|
||||
};
|
||||
};
|
||||
systemd.services.${relayUnit} = {
|
||||
description = "relay the gateway's connections to atticd";
|
||||
requires = [ "${relayUnit}.socket" ];
|
||||
after = [ "${relayUnit}.socket" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd 127.0.0.1:${toString cfg.port}";
|
||||
DynamicUser = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
NoNewPrivileges = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_UNIX"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# Authenticated as the forwarder's own OIDC client, with the cache's
|
||||
# store identity: one identity for the whole container. It is a
|
||||
|
|
|
|||
|
|
@ -462,7 +462,7 @@ let
|
|||
&& lib.elem "swarm-bao-attic-policy.service" atticSecret.after
|
||||
&& lib.elem "swarm-bao-attic-policy.service" atticTokens.after
|
||||
&& atticHere.containers.swarm-attic.bindMounts."/run/swarm-attic-secrets".isReadOnly
|
||||
&& !atticHere.containers.swarm-attic.privateNetwork
|
||||
&& atticHere.containers.swarm-attic.privateNetwork
|
||||
&& atticUnit.serviceConfig.EnvironmentFile == "/run/swarm-attic-secrets/server.env"
|
||||
&& lib.hasInfix "--mode monolithic" atticUnit.serviceConfig.ExecStart
|
||||
&& atticDeploy.retentionPeriod == "14 days";
|
||||
|
|
@ -534,7 +534,7 @@ let
|
|||
names = m: m.services.hyperhive.swarm.serviceDomains;
|
||||
in
|
||||
atticVhost.forceSSL
|
||||
&& atticVhost.locations."/".proxyPass == "http://127.0.0.1:8910"
|
||||
&& atticVhost.locations."/".proxyPass == "http://unix:/run/swarm-attic/atticd.sock"
|
||||
&& !(lib.hasInfix "auth_request" atticVhost.locations."/".extraConfig)
|
||||
&& lib.hasInfix "client_max_body_size 0" atticVhost.locations."/".extraConfig
|
||||
&& lib.hasInfix "proxy_request_buffering off" atticVhost.locations."/".extraConfig
|
||||
|
|
|
|||
|
|
@ -174,8 +174,8 @@ let
|
|||
# Every service container at once: the services-only host plus the CI
|
||||
# runner. The CI runner, the review classifier, the notification router, the
|
||||
# builder, the claude login minter, the metrics store, the homeserver,
|
||||
# authelia, Grafana and the collector are the ten containers on a netns of
|
||||
# their own.
|
||||
# authelia, Grafana, the collector and the cache are the eleven containers on
|
||||
# a netns of their own.
|
||||
serviceContainersWithCi = hive {
|
||||
deploy.hive-controller.enable = false;
|
||||
deploy.allSwarmServices = true;
|
||||
|
|
@ -814,14 +814,14 @@ let
|
|||
}
|
||||
{
|
||||
# Both values come from ../container-modules/swarm-container.nix.
|
||||
# Five of the 15 service containers share the host netns (hive-forge, bao,
|
||||
# the queue, the log store, the cache) and ten have one of their own.
|
||||
# Four of the 15 service containers share the host netns (hive-forge, bao,
|
||||
# the queue, the log store) and eleven have one of their own.
|
||||
name = "a service container on the host netns runs no firewall or resolvconf of its own";
|
||||
ok =
|
||||
let
|
||||
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
|
||||
in
|
||||
lib.length shared == 5
|
||||
lib.length shared == 4
|
||||
&& lib.all (
|
||||
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
|
||||
) shared;
|
||||
|
|
@ -992,6 +992,46 @@ let
|
|||
&& lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded
|
||||
&& c.bindMounts.${dir}.hostPath == dir;
|
||||
}
|
||||
{
|
||||
# atticd listens on the container's own loopback, a relay there forwards
|
||||
# the socket nginx connects to, and the host unit that drives the
|
||||
# TCP-only `attic` client dials the published name, not a loopback. The
|
||||
# `cache create` clause and the trust-bundle clauses are controls: they
|
||||
# find something in the same script and unit the absences are claimed of.
|
||||
# `upstreams != []` keeps an empty vhost from passing.
|
||||
name = "the cache runs on a netns of its own, the gateway reaches it over a socket and the host unit dials its public name";
|
||||
ok =
|
||||
let
|
||||
host = serviceContainersWithCi;
|
||||
h = host.services.hyperhive;
|
||||
c = host.containers.swarm-attic;
|
||||
dir = "/run/swarm-attic";
|
||||
sock = "${dir}/atticd.sock";
|
||||
domain = h.swarm.attic.domain;
|
||||
upstreams = map (l: l.proxyPass) (lib.attrValues h.gateway.vhosts.${domain}.locations);
|
||||
relay = c.config.systemd.services.swarm-attic-relay;
|
||||
tokens = host.systemd.services.swarm-bao-attic-tokens;
|
||||
in
|
||||
c.privateNetwork
|
||||
&& c.hostBridge == h.network.bridgeName
|
||||
&& c.hostAddress == null
|
||||
&& c.localAddress == null
|
||||
&& c.config.networking.interfaces.eth0.useDHCP
|
||||
&& c.config.networking.firewall.enable
|
||||
&& upstreams != [ ]
|
||||
&& lib.all (p: p == "http://unix:${sock}") upstreams
|
||||
&& c.config.systemd.sockets.swarm-attic-relay.listenStreams == [ sock ]
|
||||
&& lib.hasSuffix "systemd-socket-proxyd 127.0.0.1:${toString h.deploy.attic.port}" relay.serviceConfig.ExecStart
|
||||
&& c.bindMounts.${dir}.hostPath == dir
|
||||
&& lib.elem "d ${dir} 0770 root ${toString host.ids.gids.nginx} - -" host.systemd.tmpfiles.rules
|
||||
&& lib.hasInfix "cache create" tokens.script
|
||||
&& lib.hasInfix "https://${domain}" tokens.script
|
||||
&& !(lib.hasInfix "127.0.0.1" tokens.script)
|
||||
&& lib.elem "nginx.service" tokens.after
|
||||
&& h.gateway.useSelfSigned
|
||||
&& tokens.environment.SSL_CERT_FILE == "/run/swarm-bao-attic-tokens-ca/trust-bundle.pem"
|
||||
&& lib.elem "swarm-bao-attic-tokens-ca-bundle.service" tokens.requires;
|
||||
}
|
||||
{
|
||||
# On the bridge by DHCP, with every receiver on a socket: the hive's, the
|
||||
# swarm tier's and each service forwarder's. Every gateway location that
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ let
|
|||
publicKeyPath = "secret/swarm/services/attic/public-key";
|
||||
url = "http://cache.example";
|
||||
cache = "grid";
|
||||
endpoint = "http://127.0.0.1:8910";
|
||||
endpoint = "https://cache.example";
|
||||
configFile = "@DIR@/server.toml";
|
||||
envFile = "@DIR@/secrets/server.env";
|
||||
machine = "swarm-attic";
|
||||
|
|
|
|||
|
|
@ -95,8 +95,14 @@ chmod +x "$dir/bin/"*
|
|||
|
||||
cat >"$dir/bin/curl" <<STUB
|
||||
#!$BASH
|
||||
echo curl >>"$calls"
|
||||
[ "\${CURL_FAIL:-0}" != 1 ]
|
||||
echo "curl \$*" >>"$calls"
|
||||
# A curl that cannot connect prints 000 and exits 7; CURL_CODE is the status of
|
||||
# a server that answered.
|
||||
if [ "\${CURL_FAIL:-0}" = 1 ]; then
|
||||
printf 000
|
||||
exit 7
|
||||
fi
|
||||
printf %s "\${CURL_CODE:-200}"
|
||||
STUB
|
||||
printf '#!%s\nexit 0\n' "$BASH" >"$dir/bin/sleep"
|
||||
# A container that is not running has nothing to refresh.
|
||||
|
|
@ -246,6 +252,25 @@ expect "exit status" $? 1
|
|||
expect "nothing minted" "$(n '^atticadm')" 0
|
||||
expect "nothing stored" "$(n '^bao put')" 0
|
||||
|
||||
case="a 502 from the gateway is not atticd answering"
|
||||
reset
|
||||
run "$ENSURE_JWT"
|
||||
: >"$calls"
|
||||
run "$PROVISION" CURL_CODE=502
|
||||
expect "exit status" $? 1
|
||||
expect "nothing minted" "$(n '^atticadm')" 0
|
||||
expect "nothing stored" "$(n '^bao put')" 0
|
||||
expect "the probe asked the gateway's name" "$(n '^curl .*https://cache.example')" 30
|
||||
|
||||
case="a status from atticd below 500 is an answer, a 404 included"
|
||||
reset
|
||||
run "$ENSURE_JWT"
|
||||
: >"$calls"
|
||||
run "$PROVISION" CURL_CODE=404
|
||||
expect "exit status" $? 0
|
||||
expect "the cache is created" "$(n '^attic cache create')" 1
|
||||
expect "one probe" "$(n '^curl')" 1
|
||||
|
||||
case="the client reads its token from a file, not from an argument"
|
||||
reset
|
||||
run "$ENSURE_JWT"
|
||||
|
|
|
|||
Loading…
Reference in a new issue