atlas
mara
parent
ef156d9d23
commit
220b51d0a3
12 changed files with 204 additions and 105 deletions
|
|
@ -61,6 +61,19 @@ in
|
|||
own, and the gateway reaches it there, so it binds no host port. Remove
|
||||
this definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "authelia" "port" ] ''
|
||||
authelia listens on a unix socket in a network namespace of its own, and
|
||||
the gateway reaches it there, so it binds no port. Remove this
|
||||
definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "authelia" "metricsPort" ] ''
|
||||
authelia's metrics listener is a unix socket the gateway scrapes, so it
|
||||
binds no port. Remove this definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "authelia" "bridgePort" ] ''
|
||||
swarm-authelia-bridge listens on a unix socket the gateway proxies to, so
|
||||
it binds no port. Remove this definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
|
|
|
|||
|
|
@ -24,39 +24,6 @@ in
|
|||
# Here is what authelia IS, including `url` and the OIDC client registry
|
||||
# every hive needs as a *client* whether or not it runs the container.
|
||||
options.services.hyperhive.swarm.authelia = {
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 9091;
|
||||
description = ''
|
||||
TCP port authelia listens on. 9091 is upstream's default and
|
||||
sits outside hyperhive's claimed ranges (dashboard 7000, forge
|
||||
3000, every agent in 8100..8999 via FNV-1a hash).
|
||||
'';
|
||||
};
|
||||
|
||||
metricsPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 9959;
|
||||
description = ''
|
||||
TCP port authelia serves its Prometheus metrics on, bound to
|
||||
loopback. Upstream's default, kept so an operator reading
|
||||
authelia's documentation finds what they expect.
|
||||
|
||||
A separate port from {option}`port` because it is a separate
|
||||
listener with a different audience: the main one serves logins and
|
||||
is reachable from the swarm, this one carries no authentication of
|
||||
its own. The swarm collector scrapes it, from whichever host it
|
||||
runs on, as `https://''${domain}/metrics`: the gateway's
|
||||
`= /metrics` location proxies to this port only after authelia
|
||||
has accepted the collector's token for that URL.
|
||||
|
||||
⚠️ Every swarm container shares the host network namespace, so
|
||||
two services defaulting to the same port do not conflict at build
|
||||
time — one simply loses at runtime, with nothing in any log. Check
|
||||
a new value against the others before changing this.
|
||||
'';
|
||||
};
|
||||
|
||||
domain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
# Under the SWARM domain, like the forge and matrix: a swarm has one
|
||||
|
|
@ -426,21 +393,6 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
bridgePort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 9092;
|
||||
description = ''
|
||||
TCP port `swarm-authelia-bridge` listens on, loopback-bound
|
||||
(`127.0.0.1:''${bridgePort}`) — one above authelia's own default
|
||||
`port` (9091), outside hyperhive's other claimed ranges.
|
||||
|
||||
Not published on its own: the authelia vhost proxies
|
||||
`/bridge/` to it, and that is the address `swarm-controller`
|
||||
uses from any host. The bridge answers only
|
||||
`swarm-controller`'s OIDC client.
|
||||
'';
|
||||
};
|
||||
|
||||
otel.clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
|
|
|
|||
|
|
@ -208,8 +208,8 @@ let
|
|||
gatewayCfg.httpsPort != 443
|
||||
) ":${toString gatewayCfg.httpsPort}";
|
||||
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
|
||||
# In the host netns: distinct from every other collector's there, which the
|
||||
# assertion below checks.
|
||||
# Loopback port of the forwarder's own metrics, inside the container's netns:
|
||||
# no other collector shares it. The module requires a value.
|
||||
forwarderTelemetryPort = 8901;
|
||||
# A client this container registers and mints for itself. Its secret goes
|
||||
# through the store like every other forwarder's: the publisher carries it
|
||||
|
|
@ -603,13 +603,31 @@ let
|
|||
mv ${lib.escapeShellArg "${clientsFile}.tmp"} ${lib.escapeShellArg clientsFile}
|
||||
'';
|
||||
|
||||
# Shared host netns, like the forge container: the
|
||||
# gateway reaches authelia at 127.0.0.1:<port>.
|
||||
privateNetwork = false;
|
||||
# A netns of its own on the hive bridge: the container dials the store, the
|
||||
# swarm collector and its own public name through the bridge, and the
|
||||
# gateway reaches the three listeners below over sockets, so authelia binds
|
||||
# no TCP port on any interface.
|
||||
privateNetwork = true;
|
||||
|
||||
# Fixed, not an option: the directory's mode is what keeps the sockets
|
||||
# private, so it holds these three sockets and nothing else.
|
||||
socketDir = "/run/swarm-authelia";
|
||||
mainSocket = "${socketDir}/authelia.sock";
|
||||
metricsSocket = "${socketDir}/metrics.sock";
|
||||
bridgeSocket = "${socketDir}/bridge.sock";
|
||||
# `upstream` forwards the request's own URI, `upstreamAt` replaces it.
|
||||
upstream = socket: "http://unix:${socket}";
|
||||
upstreamAt = socket: path: "http://unix:${socket}:${path}";
|
||||
|
||||
# The gid the gateway's nginx runs as. Only the number crosses into the
|
||||
# container, which knows it as `gateway-nginx`; that group is the primary
|
||||
# group of every process that creates a socket, so nginx can connect.
|
||||
nginxGid = config.ids.gids.nginx;
|
||||
socketGroup = "gateway-nginx";
|
||||
in
|
||||
{
|
||||
# What authelia IS to every hive, where it answers (`url`), the OIDC
|
||||
# register every service checks itself against and its port, is
|
||||
# register every service checks itself against, is
|
||||
# `swarm.authelia` in ./swarm-authelia-service.nix. What the host running
|
||||
# the container decides is here — which two builds it
|
||||
# runs, and two filesystem paths that only exist on the machine running
|
||||
|
|
@ -854,21 +872,6 @@ in
|
|||
it in swarm-otel-service.nix.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
!(lib.elem forwarderTelemetryPort [
|
||||
hyperhiveCfg.otel.telemetryPort
|
||||
hyperhiveCfg.swarm.otel.telemetryPort
|
||||
hyperhiveCfg.swarm.bao.otel.telemetryPort
|
||||
hyperhiveCfg.swarm.nats.otel.telemetryPort
|
||||
]);
|
||||
message = ''
|
||||
authelia's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
||||
collides with another collector's in the host netns: the hive
|
||||
collector's, the swarm collector's, the store forwarder's or the
|
||||
queue forwarder's.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# Authelia's own gateway surface: the vhost that fronts it and the
|
||||
|
|
@ -899,14 +902,14 @@ in
|
|||
# page behind the login it serves.
|
||||
services.hyperhive.gateway.vhosts."${cfg.domain}" = {
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
|
||||
proxyPass = upstreamAt mainSocket "/";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = ''
|
||||
proxy_buffering off;
|
||||
# authelia decides by the ORIGINAL request, not by the hop it
|
||||
# sees — the login redirect and the session cookie's domain
|
||||
# both derive from these. Without them every request looks
|
||||
# like it arrived at 127.0.0.1 over plain http.
|
||||
# like it arrived over a local socket in plain http.
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
proxy_set_header X-Forwarded-Uri $request_uri;
|
||||
|
|
@ -939,21 +942,21 @@ in
|
|||
# page's own XHR. Enumerating individual endpoints would leave the
|
||||
# next one added silently intercepted.
|
||||
locations."/api/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||||
proxyPass = upstream mainSocket;
|
||||
};
|
||||
locations."/.well-known/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||||
proxyPass = upstream mainSocket;
|
||||
};
|
||||
# `swarm-authelia-bridge`, a separate process from authelia, also a
|
||||
# machine surface. The trailing slash on `proxyPass` strips the
|
||||
# prefix, so `/bridge/requests` reaches the bridge as `/requests`.
|
||||
# The bridge itself refuses every client but swarm-controller's.
|
||||
locations."/bridge/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.bridgePort}/";
|
||||
proxyPass = upstreamAt bridgeSocket "/";
|
||||
};
|
||||
|
||||
# Authelia's own metrics, for the swarm collector wherever it runs.
|
||||
# The listener (`metricsPort`) is loopback-only and unauthenticated;
|
||||
# The listener (`metricsSocket`) is unauthenticated and serves at `/`;
|
||||
# this location is the only way in from off the machine. `= /metrics`
|
||||
# outranks the `/` prefix, so its auth is the one that runs.
|
||||
#
|
||||
|
|
@ -961,7 +964,7 @@ in
|
|||
# `publishedScrapeTargets.authelia` in ./swarm-otel-service.nix; the two
|
||||
# must agree, since that string is also the token's audience.
|
||||
locations."= /metrics" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics";
|
||||
proxyPass = upstreamAt metricsSocket "/";
|
||||
extraConfig = ''
|
||||
auth_request /__authelia_metrics_authz;
|
||||
'';
|
||||
|
|
@ -1009,6 +1012,20 @@ in
|
|||
})
|
||||
];
|
||||
|
||||
# The socket directory, created host-side before the container starts:
|
||||
# nixos-container refuses to start when a bind source is missing.
|
||||
#
|
||||
# Root-owned and group nginx, `0770`: the listeners create their sockets
|
||||
# through the group (the authelia user's uid is allocated inside the
|
||||
# container, so the host cannot name it), and the gateway traverses it the
|
||||
# same way. Everything else is kept out.
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${socketDir} 0770 root ${toString nginxGid} - -"
|
||||
];
|
||||
|
||||
# The hive bridge carries the container's DHCP lease and its resolver.
|
||||
services.hyperhive.network.enable = lib.mkDefault true;
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
@ -1018,7 +1035,16 @@ in
|
|||
inherit privateNetwork;
|
||||
# Public trust bundle only, read-only. Empty when the gateway is not
|
||||
# self-signed, so the whole trust path drops out cleanly.
|
||||
bindMounts = caTrust.bindMount;
|
||||
hostBridge = networkCfg.bridgeName;
|
||||
bindMounts = {
|
||||
# A bind mount rather than a path both sides happen to know: a
|
||||
# container's `/run` is an nspawn tmpfs the host cannot see into.
|
||||
${socketDir} = {
|
||||
hostPath = socketDir;
|
||||
isReadOnly = false;
|
||||
};
|
||||
}
|
||||
// caTrust.bindMount;
|
||||
|
||||
config =
|
||||
{ ... }:
|
||||
|
|
@ -1045,6 +1071,16 @@ in
|
|||
|
||||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||||
|
||||
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
|
||||
# owns resolv.conf, so dhcpcd must not write it over that unit.
|
||||
networking.interfaces.eth0.useDHCP = true;
|
||||
networking.dhcpcd.extraConfig = "nohook resolv.conf";
|
||||
|
||||
# The sockets' group is the primary group of the processes that create
|
||||
# them, so that group is nginx's gid. A static group, so every unit
|
||||
# that binds a socket can name it.
|
||||
users.groups.${socketGroup}.gid = nginxGid;
|
||||
|
||||
# Authenticated as the forwarder's own OIDC client, its secret read
|
||||
# from the store with the publisher's identity: one identity for the
|
||||
# whole container. With no identity there is nothing to read it with,
|
||||
|
|
@ -1119,7 +1155,7 @@ in
|
|||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
User = unitName;
|
||||
Group = unitName;
|
||||
Group = socketGroup;
|
||||
StateDirectory = unitName;
|
||||
StateDirectoryMode = "0700";
|
||||
UMask = "0077";
|
||||
|
|
@ -1185,12 +1221,12 @@ in
|
|||
serviceConfig = {
|
||||
ExecStart = "${deployCfg.authelia.bridgePackage}/bin/swarm-authelia-bridge";
|
||||
User = unitName;
|
||||
Group = unitName;
|
||||
Group = socketGroup;
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5s";
|
||||
};
|
||||
environment = {
|
||||
SWARM_AUTHELIA_BRIDGE_BIND = "127.0.0.1:${toString cfg.bridgePort}";
|
||||
SWARM_AUTHELIA_BRIDGE_SOCKET = bridgeSocket;
|
||||
# ONE file, and this is it. The bridge keeps no second,
|
||||
# private users store: a store of its own treated as canonical,
|
||||
# with `users.yml` written as a rendering of it, would be two
|
||||
|
|
@ -1225,10 +1261,22 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# Upstream's unit is `ProtectSystem=strict`, so the socket directory
|
||||
# is writable only by name. A socket left by a killed process would
|
||||
# fail the next `bind`, so both are removed before each start.
|
||||
systemd.services.${unitName}.serviceConfig = {
|
||||
ReadWritePaths = [ socketDir ];
|
||||
ExecStartPre = [ "${pkgs.coreutils}/bin/rm -f ${mainSocket} ${metricsSocket}" ];
|
||||
};
|
||||
|
||||
services.authelia.instances.${instance} = {
|
||||
enable = true;
|
||||
package = deployCfg.authelia.package;
|
||||
|
||||
# Primary group of the unit and its user: the group the sockets are
|
||||
# created in (see `socketGroup`).
|
||||
group = socketGroup;
|
||||
|
||||
# Merged at RUNTIME alongside the nix-generated config, which
|
||||
# is the whole reason the client digests can exist at all:
|
||||
# `settings` below is rendered into the world-readable nix
|
||||
|
|
@ -1272,7 +1320,7 @@ in
|
|||
# where those messages go instead of implying a mail path
|
||||
settings = {
|
||||
theme = "dark";
|
||||
server.address = "tcp://127.0.0.1:${toString cfg.port}";
|
||||
server.address = "unix://${mainSocket}?umask=0007";
|
||||
|
||||
# Let a machine present an OAuth2 access token to the same
|
||||
# `auth_request` endpoint browsers use, so a scraper can be
|
||||
|
|
@ -1305,14 +1353,14 @@ in
|
|||
log.level = "info";
|
||||
|
||||
# Prometheus exposition for the swarm collector to scrape.
|
||||
# Loopback only, like the main listener above and for a
|
||||
# stronger reason: this endpoint has no authentication of its
|
||||
# A socket only the gateway can reach, like the main listener above
|
||||
# and for a stronger reason: this endpoint has no authentication of its
|
||||
# own, and it reports request volumes and outcomes for every
|
||||
# SSO login on the swarm. The collector reaches it through the
|
||||
# vhost's gated `= /metrics` location.
|
||||
telemetry.metrics = {
|
||||
enabled = true;
|
||||
address = "tcp://127.0.0.1:${toString cfg.metricsPort}";
|
||||
address = "unix://${metricsSocket}?umask=0007";
|
||||
};
|
||||
|
||||
# `watch` is load-bearing, not a convenience: authelia reads
|
||||
|
|
|
|||
|
|
@ -692,7 +692,7 @@ let
|
|||
in
|
||||
url autheliaNotColocated == "${autheliaNotColocated.services.hyperhive.swarm.authelia.url}/bridge"
|
||||
&& url autheliaColocated == url autheliaNotColocated
|
||||
&& location.proxyPass == "http://127.0.0.1:${toString authelia.bridgePort}/";
|
||||
&& location.proxyPass == "http://unix:/run/swarm-authelia/bridge.sock:/";
|
||||
}
|
||||
{
|
||||
# Config NAMES the IdP; it never computes where the IdP is. Both arms
|
||||
|
|
|
|||
|
|
@ -236,7 +236,7 @@ let
|
|||
}
|
||||
{
|
||||
# The location the collector's job requests: gated, and proxying to the
|
||||
# loopback listener that has no authentication of its own.
|
||||
# socket listener that has no authentication of its own.
|
||||
name = "authelia's vhost serves /metrics only behind the collector's auth subrequest";
|
||||
ok =
|
||||
let
|
||||
|
|
@ -244,7 +244,7 @@ let
|
|||
locs = autheliaOnly.services.nginx.virtualHosts.${h.swarm.authelia.domain}.locations;
|
||||
authz = locs."= /__authelia_metrics_authz";
|
||||
in
|
||||
locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.authelia.metricsPort}/metrics"
|
||||
locs."= /metrics".proxyPass == "http://unix:/run/swarm-authelia/metrics.sock:/"
|
||||
&& lib.hasInfix "auth_request /__authelia_metrics_authz;" locs."= /metrics".extraConfig
|
||||
&& authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request"
|
||||
&& lib.hasInfix "internal;" authz.extraConfig;
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ let
|
|||
|
||||
# Every service container at once: the services-only host plus the CI
|
||||
# runner. The CI runner, the review classifier, the builder, the claude login
|
||||
# minter, the metrics store and the homeserver are the six containers on a
|
||||
# minter, the metrics store, the homeserver and authelia are the seven containers on a
|
||||
# netns of their own.
|
||||
serviceContainersWithCi = hive {
|
||||
deploy.hive-controller.enable = false;
|
||||
|
|
@ -599,7 +599,7 @@ let
|
|||
let
|
||||
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
|
||||
in
|
||||
lib.length shared == 7
|
||||
lib.length shared == 6
|
||||
&& lib.all (
|
||||
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
|
||||
) shared;
|
||||
|
|
@ -694,6 +694,51 @@ let
|
|||
&& unit.serviceConfig.Group == "gateway-nginx"
|
||||
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
||||
}
|
||||
{
|
||||
# On the bridge by DHCP, with all three listeners on sockets: nothing
|
||||
# the bridge can dial. Every vhost location that forwards goes through
|
||||
# one of the sockets, except the internal authz subrequest, which is
|
||||
# `https://`. The main, bridge and metrics locations are named so an
|
||||
# empty filter can't pass.
|
||||
name = "authelia runs on a netns of its own and the gateway reaches all three listeners over sockets";
|
||||
ok =
|
||||
let
|
||||
host = serviceContainersWithCi;
|
||||
net = host.services.hyperhive.network;
|
||||
c = host.containers.swarm-authelia;
|
||||
dir = "/run/swarm-authelia";
|
||||
sock = n: "${dir}/${n}.sock";
|
||||
locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.authelia.domain}.locations;
|
||||
forwarded = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
|
||||
map (l: l.proxyPass) (lib.attrValues locs)
|
||||
);
|
||||
inner = c.config;
|
||||
settings = inner.services.authelia.instances.swarm.settings;
|
||||
unit = inner.systemd.services.authelia-swarm;
|
||||
bridge = inner.systemd.services.swarm-authelia-bridge;
|
||||
nginxGid = host.ids.gids.nginx;
|
||||
in
|
||||
c.privateNetwork
|
||||
&& c.hostBridge == net.bridgeName
|
||||
&& c.hostAddress == null
|
||||
&& c.localAddress == null
|
||||
&& inner.networking.interfaces.eth0.useDHCP
|
||||
&& inner.networking.firewall.enable
|
||||
&& settings.server.address == "unix://${sock "authelia"}?umask=0007"
|
||||
&& settings.telemetry.metrics.address == "unix://${sock "metrics"}?umask=0007"
|
||||
&& bridge.environment.SWARM_AUTHELIA_BRIDGE_SOCKET == sock "bridge"
|
||||
&& !(bridge.environment ? SWARM_AUTHELIA_BRIDGE_BIND)
|
||||
&& locs."/".proxyPass == "http://unix:${sock "authelia"}:/"
|
||||
&& locs."/bridge/".proxyPass == "http://unix:${sock "bridge"}:/"
|
||||
&& locs."= /metrics".proxyPass == "http://unix:${sock "metrics"}:/"
|
||||
&& lib.all (lib.hasPrefix "http://unix:${dir}/") forwarded
|
||||
&& c.bindMounts.${dir}.hostPath == dir
|
||||
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
||||
&& inner.users.groups.gateway-nginx.gid == nginxGid
|
||||
&& unit.serviceConfig.Group == "gateway-nginx"
|
||||
&& bridge.serviceConfig.Group == "gateway-nginx"
|
||||
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
||||
}
|
||||
{
|
||||
# The DynamicUser layout keeps the history under /var/lib/private; the
|
||||
# static user's StateDirectory is the plain path. The move runs as its
|
||||
|
|
|
|||
Loading…
Reference in a new issue