diff --git a/docs/swarm/services.md b/docs/swarm/services.md index afe352b2..cea50f2e 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -330,11 +330,10 @@ go there is `deploy.swarm-otel.upstream.signals` (default `[ "metrics" "logs" ]` always get both. It holds the upstream credential, which is why no hive and no agent needs one. -It runs in a `swarm-otel` container on the hive bridge, in a network -namespace of its own. It binds no OTLP port: every receiver (one per hive, -the swarm tier's and one per service forwarder) listens on a unix socket in -`/run/swarm-otel`, a directory only the gateway's nginx and the container can -open, and the gateway forwards to them. +It runs in a `swarm-otel` container. Its `swarm.otel.port` defaults to +`4319` rather than OTLP's usual `4318`, which the hive tier uses — swarm +containers share the host's network namespace, so two collectors on one +port is a coin toss at runtime rather than an error at build time. Every hive's own collector reaches this one by its gateway name, `swarm.otel.domain` (default `otel.`) — the same @@ -346,6 +345,7 @@ gateway either way. | Option | When you'd touch it | | ------------------- | --------------------------------------------------------------------------------------- | | `swarm.otel.domain` | Only to rename it — the default already resolves correctly for every hive in the swarm. | +| `swarm.otel.port` | Only if something else on the services host already claims `4319`. | The `swarm-bao`, `swarm-nats`, `swarm-builder`, `swarm-claude-login`, `swarm-notify-router`, `swarm-review-classifier`, `swarm-ci-runner`, `swarm-grafana`, diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 1ad52b7b..4ebe201a 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -69,19 +69,6 @@ in own, and the gateway reaches it there, so it binds no host port. Remove this definition. '') - (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "port" ] '' - The swarm collector's OTLP receivers listen on unix sockets in a network - namespace of its own, and the gateway reaches them there, so there is no - receiver port range. Remove this definition. - '') - (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "producerPort" ] '' - The swarm-tier receiver listens on a unix socket, so it binds no port. - Remove this definition. - '') - (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "otel" "storeProducerPort" ] '' - The store forwarder's receiver listens on a unix socket, so it binds no - port. Remove this definition. - '') (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "authelia" "port" ] '' authelia listens on a unix socket in a network namespace of its own, and the gateway reaches it there, so it binds no port. Remove this diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index d9317a59..98d8440e 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -50,6 +50,34 @@ in ''; }; + port = lib.mkOption { + type = lib.types.port; + default = 4319; + description = '' + First port of this collector's receiver range. Every hive in + {option}`services.hyperhive.swarm.hives` gets its **own** + authenticated receiver — that is what makes the `hive` label + unforgeable — so the range is one port per hive, starting here, in + sorted-name order. + + ⚠️ Internal. No client is ever told a port: a hive reaches its own + receiver as `https://''${domain}/`, and the gateway routes on + that path. So adding a hive, which renumbers the ones after it, is + harmless — nginx is rendered from this same evaluation and moves + with it. + + ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the + hive tier already uses it (`services.hyperhive.swarm.hiveOtel.collectorPort`) + and every swarm container shares the host's network namespace. Two + listeners claiming one port on one host is not a build failure — + it is a runtime coin toss over which one gets it, with nothing in + any log saying so. On the host running this collector, an + assertion rejects a derived range that overlaps any other port + this module or the hive tier declares, which is as far as a module + can see. + ''; + }; + telemetryPort = lib.mkOption { type = lib.types.port; default = 8889; @@ -59,8 +87,73 @@ in that telemetry is being lost, so it is worth keeping rather than switching off. - Bound on the container's own loopback, so no other collector - shares it; only this collector scrapes it. + ⚠️ **Deliberately not 8889's neighbour 8888**, which is the + collector's built-in default and therefore what the hive tier + already binds. Two collectors share a network namespace whenever + they are co-located, and unlike the OTLP port this one appears + nowhere in either config — it is a default inside the binary, so + nothing that compares configured ports can see the clash. The + second collector to start simply dies with + `bind: address already in use`. + ''; + }; + + producerPort = lib.mkOption { + type = lib.types.port; + default = 4390; + description = '' + Port the swarm tier's own OTLP/HTTP receiver for a + **swarm-level** producer (`swarm-controller`'s vcs/jobq counters + today) listens on, at `127.0.0.1`. Reached through the gateway at + `https://''${domain}/''${producerName}/`, same shape as a hive's + own receiver — a swarm-level producer may not share a host with + this collector (the swarm services need not run on the same host + as the swarm controller), so loopback-only reachability is not a + supported shape here, as it is not for a hive. + + Authenticated by `oidc/''${producerName}`, checking for the + audience `services.hyperhive.swarm.controller.queueClientId` + requests — not the per-hive `hiveClientPrefix` namespace, because + a swarm-level producer is explicitly not a hive (see that + option's own doc comment). The `swarm` label this receiver's + samples get is still the one the receiver's own *existence* + establishes, same as before — there is no hive identity being + forged or attributed here, just a caller proving it is the one + principal allowed to push into this pipeline. + + ⚠️ Deliberately NOT derived from `port + (number of hives)`: that + range grows every time a hive is added, and a fixed offset from + it would silently start colliding once the hive count caught up. + Kept as its own reserved value instead, and the port-collision + assertion still rejects a real collision (including one hive + growth eventually causes) rather than starting a collector that + quietly drops one receiver's samples. + ''; + }; + + storeProducerPort = lib.mkOption { + type = lib.types.port; + default = 4391; + description = '' + Port the receiver for the **secret store's own forwarder** listens + on, at `127.0.0.1`. Reached through the gateway at + `https://''${domain}/''${storeProducerName}/`, the same + path-per-producer shape a hive and the swarm-tier producer above + both use. + + Its own receiver, and not a second sender into + {option}`services.hyperhive.swarm.otel.producerPort`, because an + `oidc` authenticator checks exactly ONE audience: that receiver + admits the audience `swarm-controller` registers for itself, and + the store's forwarder is a different principal with a client of its + own. Sharing the route would mean sharing that client — which is + the thing the store exists to make unnecessary. + + ⚠️ A reserved value rather than an offset from + {option}`services.hyperhive.swarm.otel.port`, for the reason stated + on `producerPort`: that range grows with the hive count and would + eventually walk into any fixed offset from it. The port-collision + assertion covers this port too. ''; }; @@ -77,6 +170,14 @@ in the forwarder requests. ''; }; + port = lib.mkOption { + type = lib.types.port; + description = '' + Loopback port of the producer's receiver on the collector's + host. In the port-collision assertion with every other + receiver there. + ''; + }; }; } ); @@ -155,15 +256,13 @@ in scrapeTargets = lib.mkOption { type = lib.types.attrsOf lib.types.str; default = { }; - example = lib.literalExpression ''{ collector = "127.0.0.1:8889"; }''; + example = lib.literalExpression ''{ forgejo = "127.0.0.1:3000"; }''; description = '' Prometheus exposition endpoints this collector scrapes, as - ` = ":"`. The collector runs in a network namespace of - its own, so `127.0.0.1` is the collector container's loopback: a target - here is a listener inside that container, such as its own telemetry. + ` = ":"`. A path and query may follow the port — - `"127.0.0.1:9100/stats?format=prometheus"` — for an + `"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an exporter that does not serve `/metrics`. Both are optional and omitted when absent, so a bare `host:port` is scraped exactly as before. @@ -171,10 +270,11 @@ in **A service declares its own entry, from its own module, under its own `enable`.** Do not assemble the list here: an entry then exists only where the service that named it runs, so a target is never - declared on a host that does not serve it. A swarm service on the - host is not reachable here, since its loopback is not the collector's: - it is a `publishedScrapeTargets` entry instead, which a collector on - any host scrapes. + declared on a host that does not serve it. An entry that also needs + the collector on the same host (the loopback address is only honest + then) is declared from a glue module instead, which reads both + switches. Every swarm service is a `publishedScrapeTargets` entry + instead, which a collector on any host scrapes. ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference is a silent gap.** Nothing here puts the collector on the same host. @@ -283,59 +383,73 @@ in # The forwarder in ./swarm-bao.nix. ${cfg.storeProducerName} = { clientId = baoCfg.otel.clientId; + port = cfg.storeProducerPort; }; # The forwarder in ./swarm-nats.nix, which addresses this route by the # queue container's name. swarm-nats = { clientId = hyperhiveCfg.swarm.nats.clientId; + port = lib.mkDefault 4392; }; # The forwarders in ./swarm-builder.nix, ./swarm-claude-login.nix, # ./swarm-notify-router.nix and ./swarm-review-classifier.nix, which # address their routes by container name. swarm-builder = { clientId = hyperhiveCfg.swarm.builder.clientId; + port = lib.mkDefault 4393; }; swarm-claude-login = { clientId = hyperhiveCfg.swarm.claude-login.clientId; + port = lib.mkDefault 4394; }; swarm-review-classifier = { clientId = hyperhiveCfg.swarm.review-classifier.clientId; + port = lib.mkDefault 4395; }; swarm-notify-router = { clientId = hyperhiveCfg.swarm.notify-router.clientId; + port = lib.mkDefault 4404; }; # The forwarders in ./swarm-ci-runner.nix, ./swarm-grafana.nix and # ./swarm-victorialogs.nix, each authenticated as a client of its own. swarm-ci-runner = { clientId = hyperhiveCfg.swarm.ci-runner.otel.clientId; + port = lib.mkDefault 4396; }; swarm-grafana = { clientId = hyperhiveCfg.swarm.grafana.otel.clientId; + port = lib.mkDefault 4397; }; swarm-victorialogs = { clientId = hyperhiveCfg.swarm.victorialogs.otel.clientId; + port = lib.mkDefault 4398; }; # The forwarders in ./hive-forge/default.nix and ./hive-matrix.nix, each # authenticated as a client of its own. hive-forge = { clientId = hyperhiveCfg.swarm.forge.otel.clientId; + port = lib.mkDefault 4399; }; hive-matrix = { clientId = hyperhiveCfg.swarm.matrix.otel.clientId; + port = lib.mkDefault 4400; }; # The forwarder in ./swarm-victoriametrics.nix, which reaches this route # through its host's nginx. swarm-victoriametrics = { clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId; + port = lib.mkDefault 4401; }; # The CI run poller in ./swarm-ci-metrics.nix, a host unit that pushes # its run samples here directly. swarm-ci-metrics = { clientId = hyperhiveCfg.swarm.ci-metrics.clientId; + port = lib.mkDefault 4403; }; # The forwarder in ./swarm-authelia.nix. swarm-authelia = { clientId = hyperhiveCfg.swarm.authelia.otel.clientId; + port = lib.mkDefault 4402; }; }; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index db1d5557..82d0abcb 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -168,10 +168,16 @@ let # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. # - # A hive addresses its own receiver as a PATH on this collector's single - # gateway name; nginx, rendered from this same evaluation, is the only thing - # that ever names the socket. - hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf; + # These ports are internal and appear in no URL: a hive addresses its own + # receiver as a PATH on this collector's single gateway name, and nginx — + # rendered from this same evaluation — is the only thing that ever names + # the port. That is what makes deriving them safe here and unsafe in the + # obvious other place: were a hive told a port, inserting a hive would + # renumber the ones after it and silently move a port a running hive was + # already sending to. + hivePorts = lib.listToAttrs ( + lib.imap0 (i: h: lib.nameValuePair h (cfg.port + i)) (lib.attrNames hyperhiveCfg.swarm.hives) + ); # The swarm's authelia is reached by its gateway name, whose leaf is # issued by the swarm services sub-CA — so this container needs the same @@ -295,25 +301,11 @@ let # Now always satisfied, as a consequence of the log store always existing. collectLogs = logExporterNames != [ ]; - # A netns of its own on the hive bridge: the container dials authelia, the - # stores and the upstream by name through the bridge, and the gateway reaches - # every OTLP receiver over a socket, so no receiver binds a TCP port on any - # interface. - privateNetwork = true; - - # Fixed, not an option: the directory's mode is what keeps the sockets - # private, so it holds the receivers' sockets and nothing else. - socketDir = "/run/swarm-otel"; - # One socket per receiver, named after its owner (hive, swarm tier or - # service): `otlp/` is already unique, so the socket is too. - socketOf = owner: "${socketDir}/${owner}.sock"; - upstreamOf = owner: "http://unix:${socketOf owner}:/"; - allSockets = lib.attrValues hiveSockets ++ map socketOf ([ swarmTierName ] ++ serviceProducerNames); - - # The gid the gateway's nginx runs as. Only the number crosses into the - # container, which knows it as `gateway-nginx`. - nginxGid = config.ids.gids.nginx; - socketGroup = "gateway-nginx"; + # Shared host netns, like every sibling swarm service: the hive tier + # reaches this collector, and this collector reaches the metrics + # store, without either crossing a network boundary that would need + # its own trust material. + privateNetwork = false; storeRetry = import ./lib/store-retry.nix { }; storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in @@ -463,23 +455,23 @@ in # vhost would answer for a service it does not have. services.hyperhive.gateway.localNames = [ cfg.domain ]; - # This host serves the vhost, and the hive's dnsmasq answers its name. + # This host serves the vhost, and the container behind it resolves + # through the hive's dnsmasq. services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; - # The hive bridge carries the container's DHCP lease and its resolver. - services.hyperhive.network.enable = lib.mkDefault true; - # The metrics counterpart to this collector's own journal, which its # receiver ships like any other unit's: the journal says the process is # alive, these counters say whether it is dropping what it receives. - # Loopback is the container's own: this collector scrapes `telemetryPort` - # inside its netns, and nothing else reads it. + # Loopback works here because `privateNetwork = false` — this container + # shares the host's netns, so `127.0.0.1` is where `telemetryPort` is + # bound. services.hyperhive.swarm.otel.scrapeTargets.collector = "127.0.0.1:${toString cfg.telemetryPort}"; # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as - # every sibling swarm service: TLS terminates here, then plain http over - # the receiver's unix socket. + # every sibling swarm service: TLS terminates here, then plain http to + # the co-located container over loopback (shared netns, like the store + # this collector writes to). services.hyperhive.gateway.vhosts."${cfg.domain}" = { locations = # One name for the whole collector, and the hive is a path under @@ -492,14 +484,16 @@ in # serves `/v1/metrics` and knows nothing about hives. Without it # the receiver sees `//v1/metrics` and answers 404 to a # request that authenticated perfectly. - lib.mapAttrs' (h: _: lib.nameValuePair "/${h}/" { proxyPass = upstreamOf h; }) hiveSockets + lib.mapAttrs' ( + h: p: lib.nameValuePair "/${h}/" { proxyPass = "http://127.0.0.1:${toString p}/"; } + ) hivePorts // { # The swarm-tier producer's own route — same trailing-slash # shape as the per-hive locations above, and load-bearing for # the same reason: it strips `/${producerName}` before the # request reaches the receiver, which knows nothing about the # path it was found at. - "/${swarmTierName}/".proxyPass = upstreamOf swarmTierName; + "/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/"; # There is no swarm-wide inbox, and a closed door is the honest # description of that. Every other route into this collector # belongs to exactly one hive or one swarm-tier producer. @@ -508,7 +502,7 @@ in # Each service forwarder, on its own route for its own audience — # same trailing slashes, same reason. // lib.mapAttrs' ( - name: _: lib.nameValuePair "/${name}/" { proxyPass = upstreamOf name; } + name: p: lib.nameValuePair "/${name}/" { proxyPass = "http://127.0.0.1:${toString p.port}/"; } ) serviceProducers; }; @@ -669,18 +663,7 @@ in lib.mkDefault collectorSecretInContainer ); - systemd.tmpfiles.rules = [ - # The socket directory, created host-side before the container starts: - # nixos-container refuses to start when a bind source is missing. - # - # Root-owned, group nginx, setgid `2770`: the collector creates its - # sockets here through the group (its uid is allocated inside the - # container, so the host cannot name it), and a socket takes the - # directory's group, so the gateway can connect. Everything else is - # kept out. - "d ${socketDir} 2770 root ${toString nginxGid} - -" - ] - ++ lib.optionals haveClientIdentity ( + systemd.tmpfiles.rules = lib.optionals haveClientIdentity ( [ # The bind source must exist when the container starts. "d ${collectorSecretDir} 0700 root root -" @@ -889,11 +872,44 @@ in anything, and the two are scraped by different rules with different trust. - Rename one. A target is either reachable on the collector's own - loopback, or published and reached with a credential; it + Rename one. A target is either reachable on loopback because it + shares this host, or published and reached with a credential; it should not be described as both. ''; } + { + # A port collision between two listeners on one host is a runtime + # coin toss with nothing in any log — the failure this whole + # comment budget exists to prevent. Checked against every port + # reachable from here; a port some other module picks is not. + # + # ⚠️ `cfg.port` is deliberately absent from `others`: it is the + # FIRST element of the derived range, so listing it would make this + # assertion fire on every config. + assertion = + let + derived = lib.attrValues hivePorts; + others = [ + cfg.telemetryPort + cfg.producerPort + config.services.hyperhive.swarm.hiveOtel.collectorPort + ] + ++ map (p: p.port) (lib.attrValues serviceProducers); + all = derived ++ others; + in + lib.length (lib.unique all) == lib.length all; + message = '' + services.hyperhive.swarm.otel: the receiver range starting at + port (${toString cfg.port}, one port per hive in + services.hyperhive.swarm.hives) overlaps another port on this + host. + + Every swarm container shares the host's network namespace, so + two listeners claiming one port is not a build failure — it is + whichever process started first, silently. Move + services.hyperhive.swarm.otel.port to a free range. + ''; + } ]; containers.${cfg.machine} = { @@ -902,28 +918,19 @@ in # No `--link-journal=host`: this container's journal stays in the guest, # where `journald/self` reads it. inherit privateNetwork; - hostBridge = config.services.hyperhive.network.bridgeName; + # The upstream credential is operator-provided and lives on the host. + # Read-only, and only when one is configured — binding a path that + # does not exist makes nixos-container refuse to start the container, + # which is a stall several layers from its cause. bindMounts = - # A bind mount rather than a path both sides happen to know: a - # container's `/run` is an nspawn tmpfs the host cannot see into. - { - ${socketDir} = { - hostPath = socketDir; - isReadOnly = false; - }; - } # The collector's OIDC secret, where the delivery unit writes it. - // lib.optionalAttrs haveClientIdentity { + lib.optionalAttrs haveClientIdentity { ${collectorSecretDir} = { hostPath = collectorSecretDir; isReadOnly = true; }; } - # The upstream credential is operator-provided and lives on the host. - # Read-only, and only when one is configured — binding a path that - # does not exist makes nixos-container refuse to start the container, - # which is a stall several layers from its cause. // lib.optionalAttrs (upstream.headersCredential != null) { ${upstream.headersCredential} = { hostPath = upstream.headersCredential; @@ -989,15 +996,6 @@ in system.stateVersion = config.system.stateVersion; services.hyperhive.swarmContainer = { inherit privateNetwork; }; - # The bridge address comes from the gateway's DHCP. `swarm-bridge-dns` - # owns resolv.conf, so dhcpcd must not write it over that unit. - networking.interfaces.eth0.useDHCP = true; - networking.dhcpcd.extraConfig = "nohook resolv.conf"; - - # The sockets' group is the directory's, so that group is nginx's gid. - # A static group, so the unit below can name it. - users.groups.${socketGroup}.gid = nginxGid; - services.opentelemetry-collector = { enable = true; package = pkgs.opentelemetry-collector-contrib; @@ -1020,24 +1018,22 @@ in # reveals which one matched. receivers = lib.mapAttrs' ( - h: sock: + h: p: lib.nameValuePair "otlp/${h}" { protocols.http = { - endpoint = sock; - transport = "unix"; + endpoint = "127.0.0.1:${toString p}"; auth.authenticator = "oidc/${h}"; }; } - ) hiveSockets + ) hivePorts # The swarm tier's OWN receiver — authenticated exactly # like a hive's, just against a different identity (the - # swarm-level producer is not a hive). Its authenticator checks - # the audience `swarm.controller.queueClientId` requests, not - # the per-hive `hiveClientPrefix` namespace. + # swarm-level producer is not a hive; see + # `producerPort`'s description for what its audience + # checks and why). // { "otlp/${swarmTierName}".protocols.http = { - endpoint = socketOf swarmTierName; - transport = "unix"; + endpoint = "127.0.0.1:${toString cfg.producerPort}"; auth.authenticator = "oidc/${swarmTierName}"; }; } @@ -1045,11 +1041,10 @@ in # principal of its own — a receiver per audience, because an # `oidc` authenticator checks one. // lib.mapAttrs' ( - name: _: + name: p: lib.nameValuePair "otlp/${name}" { protocols.http = { - endpoint = socketOf name; - transport = "unix"; + endpoint = "127.0.0.1:${toString p.port}"; auth.authenticator = "oidc/${name}"; }; } @@ -1261,7 +1256,7 @@ in # Same rule, and the journald receiver's `storage:` above # would name a component the collector never starts. [ "file_storage" ] - ++ map (h: "oidc/${h}") (lib.attrNames hiveSockets) + ++ map (h: "oidc/${h}") (lib.attrNames hivePorts) # The swarm-tier producer's own authenticator — unconditional, # same reasoning as its receiver above: `otlp/${swarmTierName}` # always exists, so the authenticator it names must too, or an @@ -1290,7 +1285,7 @@ in processors = [ "resource/${h}" ]; exporters = exporterNames; } - ) hiveSockets + ) hivePorts # Its OWN pipeline, and that separation is the ruling, not a # tidiness choice: every `resource/` above UPSERTS a # `hive` key, so a scraped swarm sample routed through any of @@ -1348,7 +1343,7 @@ in processors = [ "resource/${h}" ]; exporters = logExporterNames; } - ) hiveSockets + ) hivePorts ) # Logs fan out exactly as metrics do: the swarm's store when it # runs, the operator's upstream when one is configured, both @@ -1451,7 +1446,7 @@ in # worth not re-deriving: `issuer_ca_file`, `ca_file` and # `tls.ca_file` are INVALID KEYS for this extension.) } - ) hiveSockets + ) hivePorts # The swarm-tier producer's own authenticator. Same extension # type and same `issuer_url`/no-`issuer_ca_path` shape as every # `oidc/${h}` above — only the audience differs, and @@ -1539,7 +1534,7 @@ in } ]; } - ) hiveSockets + ) hivePorts # The swarm tier's own stamp: `swarm` and deliberately NO # `hive`. A scraped swarm service belongs to the swarm, not to # any one hive, so there is no honest value to put there — and @@ -1628,18 +1623,6 @@ in serviceConfig.RestartSec = 5; } - { - serviceConfig = { - # The unit's user is allocated per start (`DynamicUser`), so - # it reaches the socket directory through this group, and - # `0007` makes each socket `0770`: group nginx can connect. - SupplementaryGroups = [ socketGroup ]; - UMask = "0007"; - # A socket left by a killed process would fail the next `bind`. - ExecStartPre = [ "${pkgs.coreutils}/bin/rm -f ${lib.escapeShellArgs allSockets}" ]; - }; - } - # The credential file is already `NAME=value`, systemd's # EnvironmentFile format — so the secret reaches the process as # an environment variable without being read by nix, written to diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index 95f01e18..d032ec64 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -795,7 +795,7 @@ let otel = h.services.hyperhive.swarm.otel; s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings; clientOf = svc: h.services.hyperhive.swarm.${svc.name}.clientId; - socks = map (svc: s.receivers."otlp/${svc.machine}".protocols.http.endpoint) forwarderServices; + ports = map (svc: otel.serviceProducers.${svc.machine}.port) forwarderServices; in !h.services.hyperhive.deploy.builder.enable && lib.all ( @@ -806,9 +806,9 @@ let && lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers && h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/" ) forwarderServices - # Their receivers and the queue's on the collector's host, none sharing a socket. + # Their receivers and the queue's on the collector's host, none sharing a port. && - lib.length (lib.unique (socks ++ [ s.receivers."otlp/swarm-nats".protocols.http.endpoint ])) + lib.length (lib.unique (ports ++ [ otel.serviceProducers.swarm-nats.port ])) == lib.length forwarderServices + 1; } { @@ -837,9 +837,7 @@ let h = shared.baoWithCollector; otel = h.services.hyperhive.swarm.otel; s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings; - socks = map (n: s.receivers."otlp/${n}".protocols.http.endpoint) ( - lib.attrNames otel.serviceProducers - ); + ports = map (p: p.port) (lib.attrValues otel.serviceProducers); in lib.all ( svc: @@ -849,7 +847,7 @@ let && lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers && h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/" ) ownClientForwarders - && lib.length (lib.unique socks) == lib.length socks; + && lib.length (lib.unique ports) == lib.length ports; } { # Grafana's forwarder is a second client beside its browser login, which diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 8b78bd2a..105bcdd0 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -170,10 +170,9 @@ let lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links; # Every service container at once: the services-only host plus the CI - # runner. The CI runner, the review classifier, the notification router, the - # builder, the claude login minter, the metrics store, the homeserver, - # authelia, Grafana and the collector are the ten containers on a netns of - # their own. + # runner. The CI runner, the review classifier, the builder, the claude login + # minter, the metrics store, the homeserver, authelia and Grafana are the eight + # containers on a netns of their own. serviceContainersWithCi = hive { deploy.hive-controller.enable = false; deploy.allSwarmServices = true; @@ -793,62 +792,6 @@ let && lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded && c.bindMounts.${dir}.hostPath == dir; } - { - # On the bridge by DHCP, with every receiver on a socket: the hive's, the - # swarm tier's and each service forwarder's. Every gateway location that - # forwards goes through one of them, and each receiver's socket is the - # one its location names. The three kinds are named so an empty filter - # can't pass. - name = "the swarm collector runs on a netns of its own and the gateway reaches every receiver over a socket"; - ok = - let - host = serviceContainersWithCi; - net = host.services.hyperhive.network; - c = host.containers.swarm-otel; - otel = host.services.hyperhive.swarm.otel; - dir = "/run/swarm-otel"; - sock = n: "${dir}/${n}.sock"; - locs = host.services.nginx.virtualHosts.${otel.domain}.locations; - forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs)); - inner = c.config; - s = inner.services.opentelemetry-collector.settings; - unit = inner.systemd.services.opentelemetry-collector.serviceConfig; - nginxGid = host.ids.gids.nginx; - receivers = lib.filterAttrs (n: _: lib.hasPrefix "otlp/" n) s.receivers; - onSocket = - owner: - let - r = s.receivers."otlp/${owner}".protocols.http; - in - r.transport == "unix" - && r.endpoint == sock owner - && locs."/${owner}/".proxyPass == "http://unix:${sock owner}:/"; - in - c.privateNetwork - && c.hostBridge == net.bridgeName - && c.hostAddress == null - && c.localAddress == null - && inner.networking.interfaces.eth0.useDHCP - && inner.networking.firewall.enable - && onSocket "h1" - && onSocket otel.producerName - && onSocket "swarm-grafana" - && lib.all (n: (receivers.${n}.protocols.http.transport or null) == "unix") ( - lib.attrNames receivers - ) - && - lib.length (lib.attrNames receivers) - == lib.length (lib.attrNames host.services.hyperhive.swarm.hives) - + 1 - + lib.length (lib.attrNames otel.serviceProducers) - && forwarded != [ ] - && lib.all (lib.hasPrefix "http://unix:${dir}/") forwarded - && c.bindMounts.${dir}.hostPath == dir - && builtins.elem "d ${dir} 2770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules - && inner.users.groups.gateway-nginx.gid == nginxGid - && lib.elem "gateway-nginx" unit.SupplementaryGroups - && unit.UMask == "0007"; - } { # The DynamicUser layout keeps the history under /var/lib/private; the # static user's StateDirectory is the plain path. The move runs as its