diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 71bed5c2..3ecf2421 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -288,12 +288,13 @@ gateway either way. | `swarm.otel.domain` | Only to rename it — the default already resolves correctly for every hive in the swarm. | | `swarm.otel.port` | Only if something else on the services host already claims `4319`. | -The `swarm-bao` and `swarm-nats` containers each run a collector of their own -that forwards the container's journal here, on a route and receiver per -service (`https:////`), authenticated as that -service's own OIDC client. Their records carry `service.name` (`bao`, -`nats`) and no `hive` label. The queue's forwarder is -`nix/container-modules/swarm-journal-forwarder.nix`. +The `swarm-bao`, `swarm-nats`, `swarm-builder`, `swarm-claude-login` and +`swarm-review-classifier` containers each run a collector of their own that +forwards the container's journal here, on a route and receiver per service +(`https:////`), authenticated as that service's +own OIDC client. Their records carry `service.name` (`bao`, `nats`, `builder`, +`claude-login`, `review-classifier`) and no `hive` label. The forwarder of all +but the store's is `nix/container-modules/swarm-journal-forwarder.nix`. **Both store exporters are unconditional**, and `deploy.victoriametrics.enable` doesn't gate them: that option says this host _runs_ the store, while the swarm diff --git a/nix/host-modules/glue-builder-oidc-client.nix b/nix/host-modules/glue-builder-oidc-client.nix index 1a0e6b00..6d8b72e2 100644 --- a/nix/host-modules/glue-builder-oidc-client.nix +++ b/nix/host-modules/glue-builder-oidc-client.nix @@ -28,6 +28,10 @@ in # `client_credentials`: a daemon authenticating as itself. kind = "machine"; audience = [ clientId ]; + # The container's journal forwarder mints its token as this client, for + # the swarm collector's receiver, which verifies it offline against + # `/jwks.json`; authelia's default opaque handle fails there. + accessTokenSignedResponseAlg = "RS256"; } ]; }; diff --git a/nix/host-modules/glue-claude-login-oidc-client.nix b/nix/host-modules/glue-claude-login-oidc-client.nix index 88dc2af2..6c857dde 100644 --- a/nix/host-modules/glue-claude-login-oidc-client.nix +++ b/nix/host-modules/glue-claude-login-oidc-client.nix @@ -29,6 +29,10 @@ in # `client_credentials`: a daemon authenticating as itself. kind = "machine"; audience = [ clientId ]; + # The container's journal forwarder mints its token as this client, for + # the swarm collector's receiver, which verifies it offline against + # `/jwks.json`; authelia's default opaque handle fails there. + accessTokenSignedResponseAlg = "RS256"; } ]; }; diff --git a/nix/host-modules/glue-review-classifier-oidc-client.nix b/nix/host-modules/glue-review-classifier-oidc-client.nix index 537b736c..58ec8c36 100644 --- a/nix/host-modules/glue-review-classifier-oidc-client.nix +++ b/nix/host-modules/glue-review-classifier-oidc-client.nix @@ -29,6 +29,10 @@ in # `client_credentials`: a daemon authenticating as itself. kind = "machine"; audience = [ clientId ]; + # The container's journal forwarder mints its token as this client, for + # the swarm collector's receiver, which verifies it offline against + # `/jwks.json`; authelia's default opaque handle fails there. + accessTokenSignedResponseAlg = "RS256"; } ]; }; diff --git a/nix/host-modules/swarm-builder.nix b/nix/host-modules/swarm-builder.nix index a37ba55d..3518b1d8 100644 --- a/nix/host-modules/swarm-builder.nix +++ b/nix/host-modules/swarm-builder.nix @@ -41,6 +41,20 @@ let unit = "swarm-builder"; clientId = hyperhiveCfg.swarm.builder.clientId; + # The container's journal forwarder pushes to its own receiver on the swarm + # collector, the route `serviceProducers.swarm-builder` in + # ./swarm-otel-service.nix declares, by the collector's gateway name. The + # port suffix is elided on 443, as in ./swarm-bao.nix. + swarmOtelCfg = hyperhiveCfg.swarm.otel; + gatewayPortSuffix = lib.optionalString ( + gatewayCfg.httpsPort != 443 + ) ":${toString gatewayCfg.httpsPort}"; + forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}"; + forwarderUnit = "swarm-journal-forwarder-oidc"; + # Loopback in the container's own netns, and distinct from the other service + # containers' forwarders all the same. + forwarderTelemetryPort = 8892; + # The daemon reaches authelia, the store and the queue by name over TLS, and # `git` reaches the forge; in self-signed mode their leaves chain to the hive # CA. @@ -48,7 +62,12 @@ let caBundleModule = caTrust.trustBundle { inherit pkgs; name = machine; - consumers = [ unit ]; + consumers = [ + unit + # Go: `SSL_CERT_FILE` replaces its trust store, for the swarm + # collector's and authelia's names. + "opentelemetry-collector" + ]; }; haveClientIdentity = @@ -65,6 +84,7 @@ let # one word. oneWord = path: builtins.match ".*[[:space:]].*" path == null; credential = id: "/run/credentials/${unit}.service/${id}"; + forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}"; # The `nix` the host daemon runs, so client and daemon speak one protocol. nix = config.nix.package; @@ -149,12 +169,9 @@ in # The daemon keeps nothing but `git`'s fetch cache, which a restart may # drop. ephemeral = false; - # Journal files on the host, not inside the container: nixpkgs hardcodes - # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. - extraFlags = [ - "--link-journal=host" - ] - ++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; + # No `--link-journal=host`: the journal stays in the guest, where the + # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix). + extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; inherit privateNetwork; hostBridge = networkCfg.bridgeName; @@ -173,6 +190,7 @@ in { imports = [ ../container-modules/swarm-container.nix + ../container-modules/swarm-journal-forwarder.nix caBundleModule ]; @@ -189,6 +207,29 @@ in systemd.services.nix-daemon.enable = false; systemd.sockets.nix-daemon.enable = false; + # Authenticated as the daemon's own OIDC client, whose secret its + # store identity already reads: one identity for the whole container. + # That identity is a system credential, so the fetch unit loads it + # too; the daemon's `LoadCredential` below is that unit's alone. + services.hyperhive.swarmJournalForwarder = { + enable = true; + serviceName = "builder"; + endpoint = forwarderEndpoint; + inherit clientId; + tokenUrl = "${autheliaCfg.url}/api/oidc/token"; + telemetryPort = forwarderTelemetryPort; + store = { + package = baoDeploy.package; + addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; + secretPath = "secret/swarm/services/${clientId}/oidc/client"; + clientCertFile = forwarderCredential "builder-bao-cert"; + clientKeyFile = forwarderCredential "builder-bao-key"; + caFile = + if storeFiles ? builder-bao-server-ca then forwarderCredential "builder-bao-server-ca" else null; + }; + }; + systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles; + systemd.services.${unit} = { description = "build agent configs for swarm-controller"; wantedBy = [ "multi-user.target" ]; diff --git a/nix/host-modules/swarm-claude-login.nix b/nix/host-modules/swarm-claude-login.nix index 0de53835..4b04d864 100644 --- a/nix/host-modules/swarm-claude-login.nix +++ b/nix/host-modules/swarm-claude-login.nix @@ -44,13 +44,32 @@ let unit = "swarm-claude-login"; clientId = hyperhiveCfg.swarm.claude-login.clientId; + # The container's journal forwarder pushes to its own receiver on the swarm + # collector, the route `serviceProducers.swarm-claude-login` in + # ./swarm-otel-service.nix declares, by the collector's gateway name. The + # port suffix is elided on 443, as in ./swarm-bao.nix. + swarmOtelCfg = hyperhiveCfg.swarm.otel; + gatewayPortSuffix = lib.optionalString ( + gatewayCfg.httpsPort != 443 + ) ":${toString gatewayCfg.httpsPort}"; + forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}"; + forwarderUnit = "swarm-journal-forwarder-oidc"; + # Loopback in the container's own netns, and distinct from the other service + # containers' forwarders all the same. + forwarderTelemetryPort = 8893; + # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; caBundleModule = caTrust.trustBundle { inherit pkgs; name = machine; - consumers = [ unit ]; + consumers = [ + unit + # Go: `SSL_CERT_FILE` replaces its trust store, for the swarm + # collector's and authelia's names. + "opentelemetry-collector" + ]; }; haveClientIdentity = @@ -67,6 +86,7 @@ let # one word. oneWord = path: builtins.match ".*[[:space:]].*" path == null; credential = id: "/run/credentials/${unit}.service/${id}"; + forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}"; privateNetwork = true; in @@ -138,12 +158,9 @@ in # The daemon keeps no state; each login's scratch directory is under # PrivateTmp. ephemeral = false; - # Journal files on the host, not inside the container: nixpkgs hardcodes - # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. - extraFlags = [ - "--link-journal=host" - ] - ++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; + # No `--link-journal=host`: the journal stays in the guest, where the + # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix). + extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; inherit privateNetwork; hostBridge = networkCfg.bridgeName; bindMounts = caTrust.bindMount; @@ -153,6 +170,7 @@ in { imports = [ ../container-modules/swarm-container.nix + ../container-modules/swarm-journal-forwarder.nix caBundleModule ]; @@ -173,6 +191,32 @@ in }; users.groups.${unit} = { }; + # Authenticated as the daemon's own OIDC client, whose secret its + # store identity already reads: one identity for the whole container. + # That identity is a system credential, so the fetch unit loads it + # too; the daemon's `LoadCredential` below is that unit's alone. + services.hyperhive.swarmJournalForwarder = { + enable = true; + serviceName = "claude-login"; + endpoint = forwarderEndpoint; + inherit clientId; + tokenUrl = "${autheliaCfg.url}/api/oidc/token"; + telemetryPort = forwarderTelemetryPort; + store = { + package = baoDeploy.package; + addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; + secretPath = "secret/swarm/services/${clientId}/oidc/client"; + clientCertFile = forwarderCredential "claude-login-bao-cert"; + clientKeyFile = forwarderCredential "claude-login-bao-key"; + caFile = + if storeFiles ? claude-login-bao-server-ca then + forwarderCredential "claude-login-bao-server-ca" + else + null; + }; + }; + systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles; + systemd.services.${unit} = { description = "log agents in to claude for the swarm UI"; wantedBy = [ "multi-user.target" ]; diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 6f6dd8ac..6aad45b3 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -400,6 +400,21 @@ in clientId = hyperhiveCfg.swarm.nats.clientId; port = lib.mkDefault 4392; }; + # The forwarders in ./swarm-builder.nix, ./swarm-claude-login.nix and + # ./swarm-review-classifier.nix, which address their routes by container + # name. + swarm-builder = { + clientId = hyperhiveCfg.swarm.builder.clientId; + port = lib.mkDefault 4393; + }; + swarm-claude-login = { + clientId = hyperhiveCfg.swarm.claude-login.clientId; + port = lib.mkDefault 4394; + }; + swarm-review-classifier = { + clientId = hyperhiveCfg.swarm.review-classifier.clientId; + port = lib.mkDefault 4395; + }; }; # Not under any `enable`: see `publishedScrapeTargets` for why every host diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 9e2f0f09..b226b55a 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -206,16 +206,17 @@ let # `--link-journal=try-guest`, which puts the journal inside the container and # leaves the host with a symlink into that container's transient root: a # reader here cannot follow it, and it dangles the moment the container - # stops. Every container block except `swarm-bao` and `swarm-nats` therefore sets + # stops. Every container block except the `serviceProducers` ones therefore sets # `extraFlags = [ "--link-journal=host" ]`, which the invocation expands # AFTER the hardcoded flag, so the files land here under their own # machine-id subdirectory and are bind-mounted into the guest instead. # # ⚠️ Drop that flag from one of THOSE containers and this receiver silently # stops seeing it — no error, just a unit that never appears in the store. - # `swarm-bao` and `swarm-nats` are the exceptions on purpose: their journals - # stay inside the container for its own collector instead (see - # `serviceProducers`), so this receiver never sees their units. + # The `serviceProducers` containers (`swarm-bao`, `swarm-nats`, + # `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`) are the + # exceptions on purpose: their journals stay inside the container for its own + # collector instead, so this receiver never sees their units. hostJournalDir = "/var/log/journal"; # Each store's OTLP route, by domain. One binding because the same string is diff --git a/nix/host-modules/swarm-review-classifier.nix b/nix/host-modules/swarm-review-classifier.nix index dda38452..54ba25e6 100644 --- a/nix/host-modules/swarm-review-classifier.nix +++ b/nix/host-modules/swarm-review-classifier.nix @@ -46,13 +46,32 @@ let unit = "swarm-review-classifier"; clientId = hyperhiveCfg.swarm.review-classifier.clientId; + # The container's journal forwarder pushes to its own receiver on the swarm + # collector, the route `serviceProducers.swarm-review-classifier` in + # ./swarm-otel-service.nix declares, by the collector's gateway name. The + # port suffix is elided on 443, as in ./swarm-bao.nix. + swarmOtelCfg = hyperhiveCfg.swarm.otel; + gatewayPortSuffix = lib.optionalString ( + gatewayCfg.httpsPort != 443 + ) ":${toString gatewayCfg.httpsPort}"; + forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}"; + forwarderUnit = "swarm-journal-forwarder-oidc"; + # Loopback in the container's own netns, and distinct from the other service + # containers' forwarders all the same. + forwarderTelemetryPort = 8894; + # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; caBundleModule = caTrust.trustBundle { inherit pkgs; name = machine; - consumers = [ unit ]; + consumers = [ + unit + # Go: `SSL_CERT_FILE` replaces its trust store, for the swarm + # collector's and authelia's names. + "opentelemetry-collector" + ]; }; haveClientIdentity = @@ -72,6 +91,7 @@ let # one word. oneWord = path: builtins.match ".*[[:space:]].*" path == null; credential = id: "/run/credentials/${unit}.service/${id}"; + forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}"; # What `claude` exports its metrics with, and the daemon its ACP runs' cost, # where this host runs a hive-tier collector on the bridge. The daemon adds @@ -164,12 +184,9 @@ in autoStart = true; # The daemon keeps no state; its scratch directories are under PrivateTmp. ephemeral = false; - # Journal files on the host, not inside the container: nixpkgs hardcodes - # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. - extraFlags = [ - "--link-journal=host" - ] - ++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; + # No `--link-journal=host`: the journal stays in the guest, where the + # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix). + extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; inherit privateNetwork; hostBridge = networkCfg.bridgeName; bindMounts = caTrust.bindMount; @@ -179,6 +196,7 @@ in { imports = [ ../container-modules/swarm-container.nix + ../container-modules/swarm-journal-forwarder.nix caBundleModule ]; @@ -193,6 +211,32 @@ in # nixpkgs, so the host's allowance does not reach here. nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "claude-code" ]; + # Authenticated as the daemon's own OIDC client, whose secret its + # store identity already reads: one identity for the whole container. + # That identity is a system credential, so the fetch unit loads it + # too; the daemon's `LoadCredential` below is that unit's alone. + services.hyperhive.swarmJournalForwarder = { + enable = true; + serviceName = "review-classifier"; + endpoint = forwarderEndpoint; + inherit clientId; + tokenUrl = "${autheliaCfg.url}/api/oidc/token"; + telemetryPort = forwarderTelemetryPort; + store = { + package = baoDeploy.package; + addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; + secretPath = "secret/swarm/services/${clientId}/oidc/client"; + clientCertFile = forwarderCredential "review-classifier-bao-cert"; + clientKeyFile = forwarderCredential "review-classifier-bao-key"; + caFile = + if storeFiles ? review-classifier-bao-server-ca then + forwarderCredential "review-classifier-bao-server-ca" + else + null; + }; + }; + systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles; + systemd.services.${unit} = { description = "answer the swarm UI's review try-outs"; wantedBy = [ "multi-user.target" ]; diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index 37b55033..556840dc 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -94,6 +94,92 @@ let # The same swarm's IdP host, for the arm that the two agree. autheliaColocated = shared.autheliaOnly; + # The other three queue clients with a container of their own, each forwarding + # its journal as the queue's does. One shared host: the switch that turns on + # every swarm service, which other suites already evaluate. + servicesHost = shared.swarmServicesWithController; + forwarderServices = [ + { + machine = "swarm-builder"; + name = "builder"; + credential = "builder-bao"; + port = 8892; + } + { + machine = "swarm-claude-login"; + name = "claude-login"; + credential = "claude-login-bao"; + port = 8893; + } + { + machine = "swarm-review-classifier"; + name = "review-classifier"; + credential = "review-classifier-bao"; + port = 8894; + } + ]; + forwarderFetchDir = "/run/credentials/swarm-journal-forwarder-oidc.service"; + + forwarderCases = lib.concatMap ( + svc: + let + h = servicesHost; + container = h.containers.${svc.machine}; + c = container.config; + s = c.services.opentelemetry-collector.settings; + otel = h.services.hyperhive.swarm.otel; + fetch = c.systemd.services.swarm-journal-forwarder-oidc; + clientId = h.services.hyperhive.swarm.${svc.name}.clientId; + in + [ + { + # The container is on the bridge, so its identity is a system credential + # rather than a file: the fetch unit loads it, and reads the same client + # secret the daemon's own identity is granted. + name = "the ${svc.name} container forwards its journal to its own route on the swarm collector"; + ok = + c.services.opentelemetry-collector.enable + && s.exporters.otlphttp.endpoint == "https://${otel.domain}/${svc.machine}" + && s.exporters.otlphttp.auth.authenticator == "oauth2client/swarm" + && s.extensions."oauth2client/swarm".client_id == clientId + && s.extensions."oauth2client/swarm".endpoint_params.audience == clientId + && lib.elem "oauth2client/swarm" s.service.extensions + && s.service.pipelines.logs.receivers == [ "journald" ] + && + (lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value == svc.name + && lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential + && fetch.environment.BAO_CLIENT_CERT == "${forwarderFetchDir}/${svc.credential}-cert" + && fetch.environment.BAO_CLIENT_KEY == "${forwarderFetchDir}/${svc.credential}-key" + && lib.elem "${svc.credential}-cert" fetch.serviceConfig.LoadCredential + && lib.elem "${svc.credential}-key" fetch.serviceConfig.LoadCredential + && lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script + # The guest keeps its journal; a host-linked one is empty inside. The + # leaf's credential flag is what remains. + && !(lib.elem "--link-journal=host" container.extraFlags) + && lib.any (lib.hasPrefix "--load-credential=${svc.credential}-cert:") container.extraFlags; + } + { + name = "the ${svc.name} forwarder's metrics port is its own and loopback"; + ok = + let + # Every other collector port on the host, and the queue's forwarder. + taken = [ + 8888 + 8889 + 8890 + h.services.hyperhive.swarm.nats.otel.telemetryPort + ] + ++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices); + reader = lib.head s.service.telemetry.metrics.readers; + in + c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port + && !(lib.elem svc.port taken) + && reader.pull.exporter.prometheus.host == "127.0.0.1" + && reader.pull.exporter.prometheus.port == svc.port; + } + ] + ) forwarderServices; + cases = [ { name = "authelia and nats declare no scrape target on a host with no collector"; @@ -633,6 +719,51 @@ let in c != null && c.audience == [ "swarm-nats" ] && c.accessTokenSignedResponseAlg == "RS256"; } - ]; + { + # The far end for the three: a receiver, authenticator, pipeline entry and + # gateway route each, checking that service's client as audience, beside + # the queue's, on a collector host that runs none of them. + name = "the swarm collector admits the builder, claude login and classifier forwarders on receivers of their own"; + ok = + let + h = shared.baoWithCollector; + otel = h.services.hyperhive.swarm.otel; + s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings; + clientOf = svc: h.services.hyperhive.swarm.${svc.name}.clientId; + ports = map (svc: otel.serviceProducers.${svc.machine}.port) forwarderServices; + in + !h.services.hyperhive.deploy.builder.enable + && lib.all ( + svc: + s.receivers."otlp/${svc.machine}".protocols.http.auth.authenticator == "oidc/${svc.machine}" + && s.extensions."oidc/${svc.machine}".audience == clientOf svc + && lib.elem "oidc/${svc.machine}" s.service.extensions + && lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers + && h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/" + ) forwarderServices + # Four producers' receivers on the collector's host, none sharing a port. + && lib.length (lib.unique (ports ++ [ otel.serviceProducers.swarm-nats.port ])) == 4; + } + { + # The token each forwarder mints must name its own client as audience and + # be a JWT, or the receiver refuses it. + name = "the builder, claude login and classifier clients are registered with their own audience and a signed access token"; + ok = + let + clients = shared.autheliaOnly.services.hyperhive.swarm.authelia.oidc.clients; + ok1 = + id: + let + c = lib.findFirst (c: c.id == id) null clients; + in + c != null + && c.audience == [ id ] + && c.accessTokenSignedResponseAlg == "RS256" + && c.kind == "machine"; + in + lib.all ok1 (map (svc: svc.machine) forwarderServices); + } + ] + ++ forwarderCases; in runGroup "nats-authelia" cases