diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 6f908eed..e0c472cc 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -330,10 +330,11 @@ go there is `deploy.swarm-otel.upstream.signals` (default `[ "metrics" "logs" ]` always get both. It holds the upstream credential, which is why no hive and no agent needs one. -It runs in a `swarm-otel` container. Its `swarm.otel.port` defaults to -`4319` rather than OTLP's usual `4318`, which the hive tier uses — swarm -containers share the host's network namespace, so two collectors on one -port is a coin toss at runtime rather than an error at build time. +It runs in a `swarm-otel` container on the hive bridge, in a network +namespace of its own. It binds no OTLP port: every receiver (one per hive, +the swarm tier's and one per service forwarder) listens on a unix socket in +`/run/swarm-otel`, a directory only the gateway's nginx and the container can +open, and the gateway forwards to them. Every hive's own collector reaches this one by its gateway name, `swarm.otel.domain` (default `otel.`) — the same @@ -345,7 +346,6 @@ gateway either way. | Option | When you'd touch it | | ------------------- | --------------------------------------------------------------------------------------- | | `swarm.otel.domain` | Only to rename it — the default already resolves correctly for every hive in the swarm. | -| `swarm.otel.port` | Only if something else on the services host already claims `4319`. | The `swarm-bao`, `swarm-nats`, `swarm-builder`, `swarm-claude-login`, `swarm-notify-router`, `swarm-review-classifier`, `swarm-ci-runner`, `swarm-grafana`, diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 4ebe201a..1ad52b7b 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -69,6 +69,19 @@ in own, and the gateway reaches it there, so it binds no host port. Remove this definition. '') + (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "port" ] '' + The swarm collector's OTLP receivers listen on unix sockets in a network + namespace of its own, and the gateway reaches them there, so there is no + receiver port range. Remove this definition. + '') + (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "producerPort" ] '' + The swarm-tier receiver listens on a unix socket, so it binds no port. + Remove this definition. + '') + (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "storeProducerPort" ] '' + The store forwarder's receiver listens on a unix socket, so it binds no + port. Remove this definition. + '') (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "authelia" "port" ] '' authelia listens on a unix socket in a network namespace of its own, and the gateway reaches it there, so it binds no port. Remove this diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 98d8440e..d9317a59 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -50,34 +50,6 @@ in ''; }; - port = lib.mkOption { - type = lib.types.port; - default = 4319; - description = '' - First port of this collector's receiver range. Every hive in - {option}`services.hyperhive.swarm.hives` gets its **own** - authenticated receiver — that is what makes the `hive` label - unforgeable — so the range is one port per hive, starting here, in - sorted-name order. - - ⚠️ Internal. No client is ever told a port: a hive reaches its own - receiver as `https://''${domain}/`, and the gateway routes on - that path. So adding a hive, which renumbers the ones after it, is - harmless — nginx is rendered from this same evaluation and moves - with it. - - ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the - hive tier already uses it (`services.hyperhive.swarm.hiveOtel.collectorPort`) - and every swarm container shares the host's network namespace. Two - listeners claiming one port on one host is not a build failure — - it is a runtime coin toss over which one gets it, with nothing in - any log saying so. On the host running this collector, an - assertion rejects a derived range that overlaps any other port - this module or the hive tier declares, which is as far as a module - can see. - ''; - }; - telemetryPort = lib.mkOption { type = lib.types.port; default = 8889; @@ -87,73 +59,8 @@ in that telemetry is being lost, so it is worth keeping rather than switching off. - ⚠️ **Deliberately not 8889's neighbour 8888**, which is the - collector's built-in default and therefore what the hive tier - already binds. Two collectors share a network namespace whenever - they are co-located, and unlike the OTLP port this one appears - nowhere in either config — it is a default inside the binary, so - nothing that compares configured ports can see the clash. The - second collector to start simply dies with - `bind: address already in use`. - ''; - }; - - producerPort = lib.mkOption { - type = lib.types.port; - default = 4390; - description = '' - Port the swarm tier's own OTLP/HTTP receiver for a - **swarm-level** producer (`swarm-controller`'s vcs/jobq counters - today) listens on, at `127.0.0.1`. Reached through the gateway at - `https://''${domain}/''${producerName}/`, same shape as a hive's - own receiver — a swarm-level producer may not share a host with - this collector (the swarm services need not run on the same host - as the swarm controller), so loopback-only reachability is not a - supported shape here, as it is not for a hive. - - Authenticated by `oidc/''${producerName}`, checking for the - audience `services.hyperhive.swarm.controller.queueClientId` - requests — not the per-hive `hiveClientPrefix` namespace, because - a swarm-level producer is explicitly not a hive (see that - option's own doc comment). The `swarm` label this receiver's - samples get is still the one the receiver's own *existence* - establishes, same as before — there is no hive identity being - forged or attributed here, just a caller proving it is the one - principal allowed to push into this pipeline. - - ⚠️ Deliberately NOT derived from `port + (number of hives)`: that - range grows every time a hive is added, and a fixed offset from - it would silently start colliding once the hive count caught up. - Kept as its own reserved value instead, and the port-collision - assertion still rejects a real collision (including one hive - growth eventually causes) rather than starting a collector that - quietly drops one receiver's samples. - ''; - }; - - storeProducerPort = lib.mkOption { - type = lib.types.port; - default = 4391; - description = '' - Port the receiver for the **secret store's own forwarder** listens - on, at `127.0.0.1`. Reached through the gateway at - `https://''${domain}/''${storeProducerName}/`, the same - path-per-producer shape a hive and the swarm-tier producer above - both use. - - Its own receiver, and not a second sender into - {option}`services.hyperhive.swarm.otel.producerPort`, because an - `oidc` authenticator checks exactly ONE audience: that receiver - admits the audience `swarm-controller` registers for itself, and - the store's forwarder is a different principal with a client of its - own. Sharing the route would mean sharing that client — which is - the thing the store exists to make unnecessary. - - ⚠️ A reserved value rather than an offset from - {option}`services.hyperhive.swarm.otel.port`, for the reason stated - on `producerPort`: that range grows with the hive count and would - eventually walk into any fixed offset from it. The port-collision - assertion covers this port too. + Bound on the container's own loopback, so no other collector + shares it; only this collector scrapes it. ''; }; @@ -170,14 +77,6 @@ in the forwarder requests. ''; }; - port = lib.mkOption { - type = lib.types.port; - description = '' - Loopback port of the producer's receiver on the collector's - host. In the port-collision assertion with every other - receiver there. - ''; - }; }; } ); @@ -256,13 +155,15 @@ in scrapeTargets = lib.mkOption { type = lib.types.attrsOf lib.types.str; default = { }; - example = lib.literalExpression ''{ forgejo = "127.0.0.1:3000"; }''; + example = lib.literalExpression ''{ collector = "127.0.0.1:8889"; }''; description = '' Prometheus exposition endpoints this collector scrapes, as - ` = ":"`. + ` = ":"`. The collector runs in a network namespace of + its own, so `127.0.0.1` is the collector container's loopback: a target + here is a listener inside that container, such as its own telemetry. A path and query may follow the port — - `"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an + `"127.0.0.1:9100/stats?format=prometheus"` — for an exporter that does not serve `/metrics`. Both are optional and omitted when absent, so a bare `host:port` is scraped exactly as before. @@ -270,11 +171,10 @@ in **A service declares its own entry, from its own module, under its own `enable`.** Do not assemble the list here: an entry then exists only where the service that named it runs, so a target is never - declared on a host that does not serve it. An entry that also needs - the collector on the same host (the loopback address is only honest - then) is declared from a glue module instead, which reads both - switches. Every swarm service is a `publishedScrapeTargets` entry - instead, which a collector on any host scrapes. + declared on a host that does not serve it. A swarm service on the + host is not reachable here, since its loopback is not the collector's: + it is a `publishedScrapeTargets` entry instead, which a collector on + any host scrapes. ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference is a silent gap.** Nothing here puts the collector on the same host. @@ -383,73 +283,59 @@ in # The forwarder in ./swarm-bao.nix. ${cfg.storeProducerName} = { clientId = baoCfg.otel.clientId; - port = cfg.storeProducerPort; }; # The forwarder in ./swarm-nats.nix, which addresses this route by the # queue container's name. swarm-nats = { clientId = hyperhiveCfg.swarm.nats.clientId; - port = lib.mkDefault 4392; }; # The forwarders in ./swarm-builder.nix, ./swarm-claude-login.nix, # ./swarm-notify-router.nix and ./swarm-review-classifier.nix, which # address their routes by container name. swarm-builder = { clientId = hyperhiveCfg.swarm.builder.clientId; - port = lib.mkDefault 4393; }; swarm-claude-login = { clientId = hyperhiveCfg.swarm.claude-login.clientId; - port = lib.mkDefault 4394; }; swarm-review-classifier = { clientId = hyperhiveCfg.swarm.review-classifier.clientId; - port = lib.mkDefault 4395; }; swarm-notify-router = { clientId = hyperhiveCfg.swarm.notify-router.clientId; - port = lib.mkDefault 4404; }; # The forwarders in ./swarm-ci-runner.nix, ./swarm-grafana.nix and # ./swarm-victorialogs.nix, each authenticated as a client of its own. swarm-ci-runner = { clientId = hyperhiveCfg.swarm.ci-runner.otel.clientId; - port = lib.mkDefault 4396; }; swarm-grafana = { clientId = hyperhiveCfg.swarm.grafana.otel.clientId; - port = lib.mkDefault 4397; }; swarm-victorialogs = { clientId = hyperhiveCfg.swarm.victorialogs.otel.clientId; - port = lib.mkDefault 4398; }; # The forwarders in ./hive-forge/default.nix and ./hive-matrix.nix, each # authenticated as a client of its own. hive-forge = { clientId = hyperhiveCfg.swarm.forge.otel.clientId; - port = lib.mkDefault 4399; }; hive-matrix = { clientId = hyperhiveCfg.swarm.matrix.otel.clientId; - port = lib.mkDefault 4400; }; # The forwarder in ./swarm-victoriametrics.nix, which reaches this route # through its host's nginx. swarm-victoriametrics = { clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId; - port = lib.mkDefault 4401; }; # The CI run poller in ./swarm-ci-metrics.nix, a host unit that pushes # its run samples here directly. swarm-ci-metrics = { clientId = hyperhiveCfg.swarm.ci-metrics.clientId; - port = lib.mkDefault 4403; }; # The forwarder in ./swarm-authelia.nix. swarm-authelia = { clientId = hyperhiveCfg.swarm.authelia.otel.clientId; - port = lib.mkDefault 4402; }; }; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 82d0abcb..db1d5557 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -168,16 +168,10 @@ let # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. # - # These ports are internal and appear in no URL: a hive addresses its own - # receiver as a PATH on this collector's single gateway name, and nginx — - # rendered from this same evaluation — is the only thing that ever names - # the port. That is what makes deriving them safe here and unsafe in the - # obvious other place: were a hive told a port, inserting a hive would - # renumber the ones after it and silently move a port a running hive was - # already sending to. - hivePorts = lib.listToAttrs ( - lib.imap0 (i: h: lib.nameValuePair h (cfg.port + i)) (lib.attrNames hyperhiveCfg.swarm.hives) - ); + # A hive addresses its own receiver as a PATH on this collector's single + # gateway name; nginx, rendered from this same evaluation, is the only thing + # that ever names the socket. + hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf; # The swarm's authelia is reached by its gateway name, whose leaf is # issued by the swarm services sub-CA — so this container needs the same @@ -301,11 +295,25 @@ let # Now always satisfied, as a consequence of the log store always existing. collectLogs = logExporterNames != [ ]; - # Shared host netns, like every sibling swarm service: the hive tier - # reaches this collector, and this collector reaches the metrics - # store, without either crossing a network boundary that would need - # its own trust material. - privateNetwork = false; + # A netns of its own on the hive bridge: the container dials authelia, the + # stores and the upstream by name through the bridge, and the gateway reaches + # every OTLP receiver over a socket, so no receiver binds a TCP port on any + # interface. + privateNetwork = true; + + # Fixed, not an option: the directory's mode is what keeps the sockets + # private, so it holds the receivers' sockets and nothing else. + socketDir = "/run/swarm-otel"; + # One socket per receiver, named after its owner (hive, swarm tier or + # service): `otlp/` is already unique, so the socket is too. + socketOf = owner: "${socketDir}/${owner}.sock"; + upstreamOf = owner: "http://unix:${socketOf owner}:/"; + allSockets = lib.attrValues hiveSockets ++ map socketOf ([ swarmTierName ] ++ serviceProducerNames); + + # The gid the gateway's nginx runs as. Only the number crosses into the + # container, which knows it as `gateway-nginx`. + nginxGid = config.ids.gids.nginx; + socketGroup = "gateway-nginx"; storeRetry = import ./lib/store-retry.nix { }; storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in @@ -455,23 +463,23 @@ in # vhost would answer for a service it does not have. services.hyperhive.gateway.localNames = [ cfg.domain ]; - # This host serves the vhost, and the container behind it resolves - # through the hive's dnsmasq. + # This host serves the vhost, and the hive's dnsmasq answers its name. services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; + # The hive bridge carries the container's DHCP lease and its resolver. + services.hyperhive.network.enable = lib.mkDefault true; + # The metrics counterpart to this collector's own journal, which its # receiver ships like any other unit's: the journal says the process is # alive, these counters say whether it is dropping what it receives. - # Loopback works here because `privateNetwork = false` — this container - # shares the host's netns, so `127.0.0.1` is where `telemetryPort` is - # bound. + # Loopback is the container's own: this collector scrapes `telemetryPort` + # inside its netns, and nothing else reads it. services.hyperhive.swarm.otel.scrapeTargets.collector = "127.0.0.1:${toString cfg.telemetryPort}"; # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as - # every sibling swarm service: TLS terminates here, then plain http to - # the co-located container over loopback (shared netns, like the store - # this collector writes to). + # every sibling swarm service: TLS terminates here, then plain http over + # the receiver's unix socket. services.hyperhive.gateway.vhosts."${cfg.domain}" = { locations = # One name for the whole collector, and the hive is a path under @@ -484,16 +492,14 @@ in # serves `/v1/metrics` and knows nothing about hives. Without it # the receiver sees `//v1/metrics` and answers 404 to a # request that authenticated perfectly. - lib.mapAttrs' ( - h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } - ) hivePorts + lib.mapAttrs' (h: _: lib.nameValuePair "/${h}/" { proxyPass = upstreamOf h; }) hiveSockets // { # The swarm-tier producer's own route — same trailing-slash # shape as the per-hive locations above, and load-bearing for # the same reason: it strips `/${producerName}` before the # request reaches the receiver, which knows nothing about the # path it was found at. - "/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/"; + "/${swarmTierName}/".proxyPass = upstreamOf swarmTierName; # There is no swarm-wide inbox, and a closed door is the honest # description of that. Every other route into this collector # belongs to exactly one hive or one swarm-tier producer. @@ -502,7 +508,7 @@ in # Each service forwarder, on its own route for its own audience — # same trailing slashes, same reason. // lib.mapAttrs' ( - name: p: lib.nameValuePair "/${name}/" { proxyPass = "http://127.0.0.1:${toString p.port}/"; } + name: _: lib.nameValuePair "/${name}/" { proxyPass = upstreamOf name; } ) serviceProducers; }; @@ -663,7 +669,18 @@ in lib.mkDefault collectorSecretInContainer ); - systemd.tmpfiles.rules = lib.optionals haveClientIdentity ( + systemd.tmpfiles.rules = [ + # The socket directory, created host-side before the container starts: + # nixos-container refuses to start when a bind source is missing. + # + # Root-owned, group nginx, setgid `2770`: the collector creates its + # sockets here through the group (its uid is allocated inside the + # container, so the host cannot name it), and a socket takes the + # directory's group, so the gateway can connect. Everything else is + # kept out. + "d ${socketDir} 2770 root ${toString nginxGid} - -" + ] + ++ lib.optionals haveClientIdentity ( [ # The bind source must exist when the container starts. "d ${collectorSecretDir} 0700 root root -" @@ -872,44 +889,11 @@ in anything, and the two are scraped by different rules with different trust. - Rename one. A target is either reachable on loopback because it - shares this host, or published and reached with a credential; it + Rename one. A target is either reachable on the collector's own + loopback, or published and reached with a credential; it should not be described as both. ''; } - { - # A port collision between two listeners on one host is a runtime - # coin toss with nothing in any log — the failure this whole - # comment budget exists to prevent. Checked against every port - # reachable from here; a port some other module picks is not. - # - # ⚠️ `cfg.port` is deliberately absent from `others`: it is the - # FIRST element of the derived range, so listing it would make this - # assertion fire on every config. - assertion = - let - derived = lib.attrValues hivePorts; - others = [ - cfg.telemetryPort - cfg.producerPort - config.services.hyperhive.swarm.hiveOtel.collectorPort - ] - ++ map (p: p.port) (lib.attrValues serviceProducers); - all = derived ++ others; - in - lib.length (lib.unique all) == lib.length all; - message = '' - services.hyperhive.swarm.otel: the receiver range starting at - port (${toString cfg.port}, one port per hive in - services.hyperhive.swarm.hives) overlaps another port on this - host. - - Every swarm container shares the host's network namespace, so - two listeners claiming one port is not a build failure — it is - whichever process started first, silently. Move - services.hyperhive.swarm.otel.port to a free range. - ''; - } ]; containers.${cfg.machine} = { @@ -918,19 +902,28 @@ in # No `--link-journal=host`: this container's journal stays in the guest, # where `journald/self` reads it. inherit privateNetwork; + hostBridge = config.services.hyperhive.network.bridgeName; - # The upstream credential is operator-provided and lives on the host. - # Read-only, and only when one is configured — binding a path that - # does not exist makes nixos-container refuse to start the container, - # which is a stall several layers from its cause. bindMounts = + # A bind mount rather than a path both sides happen to know: a + # container's `/run` is an nspawn tmpfs the host cannot see into. + { + ${socketDir} = { + hostPath = socketDir; + isReadOnly = false; + }; + } # The collector's OIDC secret, where the delivery unit writes it. - lib.optionalAttrs haveClientIdentity { + // lib.optionalAttrs haveClientIdentity { ${collectorSecretDir} = { hostPath = collectorSecretDir; isReadOnly = true; }; } + # The upstream credential is operator-provided and lives on the host. + # Read-only, and only when one is configured — binding a path that + # does not exist makes nixos-container refuse to start the container, + # which is a stall several layers from its cause. // lib.optionalAttrs (upstream.headersCredential != null) { ${upstream.headersCredential} = { hostPath = upstream.headersCredential; @@ -996,6 +989,15 @@ in system.stateVersion = config.system.stateVersion; services.hyperhive.swarmContainer = { inherit privateNetwork; }; + # The bridge address comes from the gateway's DHCP. `swarm-bridge-dns` + # owns resolv.conf, so dhcpcd must not write it over that unit. + networking.interfaces.eth0.useDHCP = true; + networking.dhcpcd.extraConfig = "nohook resolv.conf"; + + # The sockets' group is the directory's, so that group is nginx's gid. + # A static group, so the unit below can name it. + users.groups.${socketGroup}.gid = nginxGid; + services.opentelemetry-collector = { enable = true; package = pkgs.opentelemetry-collector-contrib; @@ -1018,22 +1020,24 @@ in # reveals which one matched. receivers = lib.mapAttrs' ( - h: p: + h: sock: lib.nameValuePair "otlp/${h}" { protocols.http = { - endpoint = "127.0.0.1:${toString p}"; + endpoint = sock; + transport = "unix"; auth.authenticator = "oidc/${h}"; }; } - ) hivePorts + ) hiveSockets # The swarm tier's OWN receiver — authenticated exactly # like a hive's, just against a different identity (the - # swarm-level producer is not a hive; see - # `producerPort`'s description for what its audience - # checks and why). + # swarm-level producer is not a hive). Its authenticator checks + # the audience `swarm.controller.queueClientId` requests, not + # the per-hive `hiveClientPrefix` namespace. // { "otlp/${swarmTierName}".protocols.http = { - endpoint = "127.0.0.1:${toString cfg.producerPort}"; + endpoint = socketOf swarmTierName; + transport = "unix"; auth.authenticator = "oidc/${swarmTierName}"; }; } @@ -1041,10 +1045,11 @@ in # principal of its own — a receiver per audience, because an # `oidc` authenticator checks one. // lib.mapAttrs' ( - name: p: + name: _: lib.nameValuePair "otlp/${name}" { protocols.http = { - endpoint = "127.0.0.1:${toString p.port}"; + endpoint = socketOf name; + transport = "unix"; auth.authenticator = "oidc/${name}"; }; } @@ -1256,7 +1261,7 @@ in # Same rule, and the journald receiver's `storage:` above # would name a component the collector never starts. [ "file_storage" ] - ++ map (h: "oidc/${h}") (lib.attrNames hivePorts) + ++ map (h: "oidc/${h}") (lib.attrNames hiveSockets) # The swarm-tier producer's own authenticator — unconditional, # same reasoning as its receiver above: `otlp/${swarmTierName}` # always exists, so the authenticator it names must too, or an @@ -1285,7 +1290,7 @@ in processors = [ "resource/${h}" ]; exporters = exporterNames; } - ) hivePorts + ) hiveSockets # Its OWN pipeline, and that separation is the ruling, not a # tidiness choice: every `resource/` above UPSERTS a # `hive` key, so a scraped swarm sample routed through any of @@ -1343,7 +1348,7 @@ in processors = [ "resource/${h}" ]; exporters = logExporterNames; } - ) hivePorts + ) hiveSockets ) # Logs fan out exactly as metrics do: the swarm's store when it # runs, the operator's upstream when one is configured, both @@ -1446,7 +1451,7 @@ in # worth not re-deriving: `issuer_ca_file`, `ca_file` and # `tls.ca_file` are INVALID KEYS for this extension.) } - ) hivePorts + ) hiveSockets # The swarm-tier producer's own authenticator. Same extension # type and same `issuer_url`/no-`issuer_ca_path` shape as every # `oidc/${h}` above — only the audience differs, and @@ -1534,7 +1539,7 @@ in } ]; } - ) hivePorts + ) hiveSockets # The swarm tier's own stamp: `swarm` and deliberately NO # `hive`. A scraped swarm service belongs to the swarm, not to # any one hive, so there is no honest value to put there — and @@ -1623,6 +1628,18 @@ in serviceConfig.RestartSec = 5; } + { + serviceConfig = { + # The unit's user is allocated per start (`DynamicUser`), so + # it reaches the socket directory through this group, and + # `0007` makes each socket `0770`: group nginx can connect. + SupplementaryGroups = [ socketGroup ]; + UMask = "0007"; + # A socket left by a killed process would fail the next `bind`. + ExecStartPre = [ "${pkgs.coreutils}/bin/rm -f ${lib.escapeShellArgs allSockets}" ]; + }; + } + # The credential file is already `NAME=value`, systemd's # EnvironmentFile format — so the secret reaches the process as # an environment variable without being read by nix, written to diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index d032ec64..95f01e18 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -795,7 +795,7 @@ let otel = h.services.hyperhive.swarm.otel; s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings; clientOf = svc: h.services.hyperhive.swarm.${svc.name}.clientId; - ports = map (svc: otel.serviceProducers.${svc.machine}.port) forwarderServices; + socks = map (svc: s.receivers."otlp/${svc.machine}".protocols.http.endpoint) forwarderServices; in !h.services.hyperhive.deploy.builder.enable && lib.all ( @@ -806,9 +806,9 @@ let && lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers && h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/" ) forwarderServices - # Their receivers and the queue's on the collector's host, none sharing a port. + # Their receivers and the queue's on the collector's host, none sharing a socket. && - lib.length (lib.unique (ports ++ [ otel.serviceProducers.swarm-nats.port ])) + lib.length (lib.unique (socks ++ [ s.receivers."otlp/swarm-nats".protocols.http.endpoint ])) == lib.length forwarderServices + 1; } { @@ -837,7 +837,9 @@ let h = shared.baoWithCollector; otel = h.services.hyperhive.swarm.otel; s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings; - ports = map (p: p.port) (lib.attrValues otel.serviceProducers); + socks = map (n: s.receivers."otlp/${n}".protocols.http.endpoint) ( + lib.attrNames otel.serviceProducers + ); in lib.all ( svc: @@ -847,7 +849,7 @@ let && lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers && h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/" ) ownClientForwarders - && lib.length (lib.unique ports) == lib.length ports; + && lib.length (lib.unique socks) == lib.length socks; } { # Grafana's forwarder is a second client beside its browser login, which diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 105bcdd0..8b78bd2a 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -170,9 +170,10 @@ let lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links; # Every service container at once: the services-only host plus the CI - # runner. The CI runner, the review classifier, the builder, the claude login - # minter, the metrics store, the homeserver, authelia and Grafana are the eight - # containers on a netns of their own. + # runner. The CI runner, the review classifier, the notification router, the + # builder, the claude login minter, the metrics store, the homeserver, + # authelia, Grafana and the collector are the ten containers on a netns of + # their own. serviceContainersWithCi = hive { deploy.hive-controller.enable = false; deploy.allSwarmServices = true; @@ -792,6 +793,62 @@ let && lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded && c.bindMounts.${dir}.hostPath == dir; } + { + # On the bridge by DHCP, with every receiver on a socket: the hive's, the + # swarm tier's and each service forwarder's. Every gateway location that + # forwards goes through one of them, and each receiver's socket is the + # one its location names. The three kinds are named so an empty filter + # can't pass. + name = "the swarm collector runs on a netns of its own and the gateway reaches every receiver over a socket"; + ok = + let + host = serviceContainersWithCi; + net = host.services.hyperhive.network; + c = host.containers.swarm-otel; + otel = host.services.hyperhive.swarm.otel; + dir = "/run/swarm-otel"; + sock = n: "${dir}/${n}.sock"; + locs = host.services.nginx.virtualHosts.${otel.domain}.locations; + forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs)); + inner = c.config; + s = inner.services.opentelemetry-collector.settings; + unit = inner.systemd.services.opentelemetry-collector.serviceConfig; + nginxGid = host.ids.gids.nginx; + receivers = lib.filterAttrs (n: _: lib.hasPrefix "otlp/" n) s.receivers; + onSocket = + owner: + let + r = s.receivers."otlp/${owner}".protocols.http; + in + r.transport == "unix" + && r.endpoint == sock owner + && locs."/${owner}/".proxyPass == "http://unix:${sock owner}:/"; + in + c.privateNetwork + && c.hostBridge == net.bridgeName + && c.hostAddress == null + && c.localAddress == null + && inner.networking.interfaces.eth0.useDHCP + && inner.networking.firewall.enable + && onSocket "h1" + && onSocket otel.producerName + && onSocket "swarm-grafana" + && lib.all (n: (receivers.${n}.protocols.http.transport or null) == "unix") ( + lib.attrNames receivers + ) + && + lib.length (lib.attrNames receivers) + == lib.length (lib.attrNames host.services.hyperhive.swarm.hives) + + 1 + + lib.length (lib.attrNames otel.serviceProducers) + && forwarded != [ ] + && lib.all (lib.hasPrefix "http://unix:${dir}/") forwarded + && c.bindMounts.${dir}.hostPath == dir + && builtins.elem "d ${dir} 2770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules + && inner.users.groups.gateway-nginx.gid == nginxGid + && lib.elem "gateway-nginx" unit.SupplementaryGroups + && unit.UMask == "0007"; + } { # The DynamicUser layout keeps the history under /var/lib/private; the # static user's StateDirectory is the plain path. The move runs as its