# `checks.module-eval-core-toggle` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, shared, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive hiveWithController runGroup baoNames baoStream bridgePorts bridgePortsOrNone swarmServiceEnables otelSettings ; inherit (shared) allLocal; bare = shared.bareWithController; # `lib.hasInfix` matches by regex, which refuses a needle that refers to a # store path. Both sides lose their string context here. hasStorePath = needle: hay: lib.hasInfix (builtins.unsafeDiscardStringContext needle) (builtins.unsafeDiscardStringContext hay); # The same stub with the central toggle (`deploy.hive-controller.enable`) # off. Paired with `bare` below to pin the defaults that do not read it: # each is asserted to hold the SAME literal in both, so a future edit # that quietly introduces the dependency — or that changes what the # default renders for a hive with the toggle on — fails here. Reading an # option off this fixture forces that option only, not the config, so the # toggle being off costs nothing. Also the "installs the modules and turns # nothing on" host the swarm-service absences below read: none of the # per-service deployment toggles derives from the hive being on, so it # renders the same absences `bare` does. centralToggleOff = shared.bare; inherit (shared) forgeHere; # Every hive defaults the agents' queue address to the queue's name, so the # only hive without one is a hive told to have none. Also the host still # setting the four removed host options; no case on its queue reads them. noAgentQueue = hiveWithController { deploy.hive-controller.queue.agentNatsUrl = null; c0re.operatorPronouns = "they/them"; c0re.contextWindowTokens.sonnet = 900000; otel.metricIntervalMs = 10000; otel.debug = true; }; # The controller behind a gateway on a non-default TLS port: the only # fixture whose collector URL carries a port. Only the controller's # environment is read off it. controllerAltPort = hive { deploy.swarm-controller.enable = true; gateway.httpsPort = 8443; }; # On the store's host, which mints the runner's store identity. withCi = hive { deploy.bao.enable = true; deploy.ci-runner.enable = true; }; # On the store's host, which mints the classifier's store identity. withClassifier = hive { deploy.bao.enable = true; deploy.review-classifier.enable = true; }; # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is # both necessary and sufficient. `deepSeq` over-specifies this: it keeps # walking *into* the resulting value after the merge already succeeded, # and a package/derivation-shaped value's `override`/`overrideAttrs` # self-reference sends it into nixpkgs' fixpoint machinery and blows the # stack (measured — this is not a hypothetical). forceCiServiceConfigs = let svcs = withCi.containers.swarm-ci-runner.config.systemd.services; vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs); in builtins.foldl' (acc: v: builtins.seq v acc) true vals; # The host fetches that restart their consumer when the store's value # changes (../host-modules/lib/refresh-consumer.nix), each on a fixture # that renders it. storeRefreshed = [ { unit = "swarm-bao-grafana-oidc"; machine = allLocal; } { unit = "swarm-bao-otel-oidc"; machine = allLocal; } { unit = "swarm-bao-matrix-oidc"; machine = allLocal; } { unit = "swarm-bao-forge-oidc"; machine = allLocal; } { unit = "swarm-bao-hive-otel-oidc"; machine = allLocal; } { unit = "swarm-bao-forwarder-oidc"; machine = allLocal; } { unit = "swarm-bao-ci-runner"; machine = withCi; } ]; remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false; cases = [ { # The domain is the stub's swarm domain, which both fixtures share. name = "the forge's publicUrl default does not consult the central toggle"; ok = bare.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local" && centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local"; } { # The controller's token mint follows where the forge runs # (./forge-placement.nix), never the central toggle: a forge host with # the toggle off still mints and publishes it. name = "the forge's controller-token mint does not consult the central toggle"; ok = forgeHere.containers.hive-forge.config.systemd.services ? forgejo-swarm-controller-account; } { name = "a hive that does not host the swarm's shared services runs none of them"; ok = let es = swarmServiceEnables bare; in lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es); } { name = "the all-local mode turns the swarm controller on"; ok = allLocal.services.hyperhive.deploy.swarm-controller.enable; } { # ../host-modules/glue-controller-env.nix: the controller always exports to # the swarm's collector. `controllerAlone` runs no collector of its own # (`allLocal` does, as the presence control); `centralToggleOff` has no # controller, so no unit to set it on. name = "the controller always exports to the swarm's collector"; ok = let env = m: m.systemd.services.swarm-controller.environment; otel = allLocal.services.hyperhive.swarm.otel; clientId = allLocal.services.hyperhive.swarm.controller.queueClientId; # ../host-modules/glue-controller-oidc-client.nix: the collector's # `oidc/swarm` reads only a signed access token. client = lib.findFirst ( c: c.id == clientId ) null allLocal.services.hyperhive.swarm.authelia.oidc.clients; in (env allLocal).OTEL_EXPORTER_OTLP_ENDPOINT or null == "https://${otel.domain}/${otel.producerName}" && (env allLocal).SWARM_CONTROLLER_OTEL_AUDIENCE or null == clientId && client != null && lib.elem clientId client.audience && client.accessTokenSignedResponseAlg == "RS256" && allLocal.services.hyperhive.deploy.swarm-otel.enable && !shared.controllerAlone.services.hyperhive.deploy.swarm-otel.enable && (env shared.controllerAlone).OTEL_EXPORTER_OTLP_ENDPOINT or null == "https://${otel.domain}/${otel.producerName}" && (env shared.controllerAlone).SWARM_CONTROLLER_OTEL_AUDIENCE or null == clientId && (env controllerAltPort).OTEL_EXPORTER_OTLP_ENDPOINT or null == "https://${otel.domain}:8443/${otel.producerName}" && !(centralToggleOff.systemd.services ? swarm-controller); } { # Same file: `swarmctl` is told where the configured authelia is only # where authelia runs here. name = "swarmctl is given the authelia binary and users file only where authelia runs here"; ok = let env = m: m.services.hyperhive.deploy.swarm-controller.swarmctlEnvironment; d = allLocal.services.hyperhive.deploy.authelia; in env allLocal == { SWARMCTL_AUTHELIA_BIN = "${d.package}/bin/authelia"; SWARMCTL_AUTHELIA_USERS_FILE = d.hostUsersFile; } && env shared.controllerAlone == { }; } { # Where the reader puts the files and where the daemon looks for them is # one agreement spanning two modules. Asserted against the option rather # than the literal so moving the directory moves both ends. name = "hive-c0re is told where the agents' queue credential lands"; ok = allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR == toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir; } { # Inside an agent container loopback is the agent itself, so the agents' # address must never be one. By name it is the same string the hive # dials, which ./nats-tls.nix pins for every client. name = "the agents' queue address is the queue's name, never loopback"; ok = let e = allLocal.systemd.services.hive-c0re.environment; in e.HIVE_AGENT_NATS_URL == "tls://nats.t.local:4222" && !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL) && e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL; } { # Agents are handed no address at all: their module defaults to a # literal name, the same on every hive and in every swarm, which this # hive's resolver answers with the bridge IP its collector binds. # hive-c0re itself keeps the address. `HYPERHIVE_OTEL_ENDPOINT` is gone, # and the bridge address next to it is the control that the lookup on # the same attrset finds what is there. name = "the agents' collector address is a fixed name this hive resolves to its bridge"; ok = let e = allLocal.systemd.services.hive-c0re.environment; bridge = allLocal.services.hyperhive.network.bridgeIp; in !(e ? HYPERHIVE_OTEL_ENDPOINT) && e.OTEL_EXPORTER_OTLP_ENDPOINT == "http://${bridge}:4318" && allLocal.services.hyperhive.otel.collector.host == "hive-collector.hive" && lib.elem "/hive-collector.hive/${bridge}" allLocal.services.dnsmasq.settings.address; } { # The agents mint against the swarm's IdP, the same endpoint the hive's # own client uses — a hive-local guess would produce a token the queue # would not accept. name = "the agents' token endpoint is the swarm IdP's"; ok = let e = allLocal.systemd.services.hive-c0re.environment; in lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT && e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT; } { # The absence arm, and what makes the two above able to fail: a hive # with no queue address for its agents must forward neither coordinate, # because half a pair reaches the harness as a partial configuration # rather than as none. name = "a hive with no agent queue address forwards no agent queue coordinates"; ok = let e = noAgentQueue.systemd.services.hive-c0re.environment; in !(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT); } { # Control: `bare` sets none of them, and its serve.json carries the # dashboard port, so the key lookup reads a real document. name = "a definition of a removed host option evaluates and warns, and the c0re ones reach no serve.json"; ok = let warned = m: option: lib.any (w: lib.hasInfix "services.hyperhive.${option} is removed and ignored" w) m.warnings; options = [ "c0re.operatorPronouns" "c0re.contextWindowTokens" "otel.metricIntervalMs" "otel.debug" ]; hasKey = m: key: lib.hasInfix "\"${key}\":" m.environment.etc."hyperhive/serve.json".text; noRemovedKeys = m: hasKey m "dashboard_port" && !(hasKey m "operator_pronouns") && !(hasKey m "context_window_tokens"); in lib.all (warned noAgentQueue) options && !(lib.any (warned bare) options) && noRemovedKeys noAgentQueue && noRemovedKeys bare; } { # Where the store is and where the agent is told it is, one agreement # spanning two modules. Asserted against the hive's own `BAO_ADDR` # rather than a literal, because an agent pointed at a different # spelling of the same store presents a certificate to a listener whose # name it cannot verify. name = "an agent is told the same store address its hive uses"; ok = let e = allLocal.systemd.services.hive-c0re.environment; in e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR; } { # A hive with no certificate of its own can collect no agent's identity, # so forwarding an address would name a store nothing in the container # can reach. The same gate the `BAO_*` pair beside it sits behind. name = "a hive with no store identity forwards no store address to its agents"; ok = !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_ADDR); } { # The gateway's per-name issuer choice. If this ever collapses to a # constant, every swarm-service vhost serves a certificate its CA # is name-constrained out of — which evaluates cleanly and fails in # a browser. name = "a swarm service name gets the swarm-services leaf and the default server does not"; ok = let l = allLocal.services.hyperhive.gateway.lib; in (l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate; } { # A leaf is a key that can SERVE every name in it, so the names it # asks for are that key's blast radius. The forge's host fronts forge # and nothing else; the last conjunct is the control, since # `auth.t.local` is in the swarm's set and an unnarrowed request would # carry it here too. name = "a gateway's services leaf asks for only the swarm names that host fronts"; ok = let s = forgeHere.services.hyperhive.swarm.serviceDomains; u = forgeHere.systemd.services.swarm-services-cert.script; in lib.hasInfix "alt_names=forge.t.local" u && !(lib.hasInfix "auth.t.local" u) && lib.elem "auth.t.local" s; } { # The store's issuing role is built from this list on the store's host, # so it cannot depend on which services the evaluating host runs. The # last conjunct is the control: grafana runs on neither `bare` nor the # forge host, and is still listed. name = "the swarm's service names are the same on hosts running different services"; ok = let s = h: h.services.hyperhive.swarm.serviceDomains; in s forgeHere == s bare && s allLocal == s bare && lib.elem "grafana.t.local" (s bare); } { # The narrowing's floor: no names left means no request, rather than a # request for an empty SAN set that the store would refuse. Its own # vhosts are the hive leaf's, which this host still signs. name = "a host fronting none of the swarm's service names requests no services leaf"; ok = bare.services.hyperhive.deploy.hive-controller.tls.localServiceDomains == [ ] && bare.services.hyperhive.swarm.serviceDomains != [ ] && lib.hasInfix "want_svc=0" bare.systemd.services.swarm-services-cert.script; } { # nixos asserts when a vhost declares both, so this is also a # statement that the `removeAttrs` upstream of it still happens. name = "the swarm UI vhost forces TLS instead of merely adding it"; ok = let v = allLocal.services.nginx.virtualHosts."t.local"; in v.forceSSL && !(v.addSSL or false); } { # `swarmctl --swarm` reads a 401 as "log in again"; a 302 would hand it # the login page instead. The `/` clause is the control: the browser # route still redirects, so a pass means the two routes differ. name = "the controller API answers a refused bearer caller 401 and a browser the login"; ok = let l = allLocal.services.nginx.virtualHosts."t.local".locations; api = l."/api/".extraConfig; refused = l."@swarm_api_unauthorized".extraConfig; in lib.hasInfix "error_page 401 = @swarm_api_unauthorized;" api && !(lib.hasInfix "=302" api) && lib.hasInfix ''proxy_set_header Authorization "";'' api && lib.hasInfix "if ($http_authorization) {\n return 401;\n}" refused && lib.hasInfix "return 302 https://auth.t.local/?rd=$target_url;" refused && lib.hasInfix "error_page 401 =302 https://auth.t.local/" l."/".extraConfig; } { # authelia 4.39.20 refuses a bearer-scoped public client missing any of # these at startup, after a green deploy; and a public client must not # be handed a secret. name = "the swarmctl login client is public, bearer-scoped to the controller API, and gets no secret"; ok = let s = allLocal.containers.swarm-authelia.config.systemd.services.authelia-swarm-secrets.script; entry = lib.last (lib.splitString "client_id: %s\\n' swarmctl" s); client = lib.head (lib.splitString "client_id: %s" entry); in lib.all (line: lib.hasInfix line client) [ "public: true" "token_endpoint_auth_method: none" "require_pkce: true" "pkce_challenge_method: S256" "require_pushed_authorization_requests: true" "consent_mode: explicit" "response_modes: [form_post]" "grant_types: [authorization_code, refresh_token]" "scopes: [offline_access, authelia.bearer.authz]" "audience: [%s]\\n' https://t.local/api/" "' - %s\\n' http://127.0.0.1/callback" ] && !(lib.hasInfix "client_secret" client) && !(lib.hasInfix "mint swarmctl" s) && lib.hasInfix "mint swarm-authelia-bridge" s; } { # swarm-controller checks no caller itself: this location's authelia # subrequest is the operator gate for every route under `/api/`. The # webhook location is the control, the one path to the same socket that # skips the gate, so the first clause reads a string that can lack it. name = "the swarm UI's API location asks authelia before reaching the controller"; ok = let l = allLocal.services.nginx.virtualHosts."t.local".locations; socket = allLocal.services.hyperhive.deploy.swarm-controller.socketPath; in lib.hasInfix "auth_request /__hive_authelia;" l."/api/".extraConfig && lib.hasInfix socket l."/api/".proxyPass && lib.hasInfix socket l."/webhook/forge/".proxyPass && !(lib.hasInfix "auth_request" l."/webhook/forge/".extraConfig); } { # Only nginx (through the clients group) and root reach the controller # socket: the runtime dir is 0750 with that group set. The last two # clauses are the presence controls — the group exists and is not the # daemon's own, so a misspelled name cannot satisfy the rest. name = "the controller socket's directory is 0750 and group-owned by the group nginx joins"; ok = let sc = allLocal.systemd.services.swarm-controller.serviceConfig; group = "swarm-controller-clients"; in sc.RuntimeDirectoryMode == "0750" && lib.hasPrefix "+" sc.ExecStartPost && lib.hasInfix "chgrp ${group} /run/swarm-controller" sc.ExecStartPost && lib.elem group allLocal.systemd.services.nginx.serviceConfig.SupplementaryGroups && allLocal.users.groups ? ${group} && sc.Group != group; } { # nginx serves a path from its longest matching prefix location, so that # one must be the gated `/api/`, and no exact location may take a route # out from under it. name = "the claude login routes reach the controller only through a location that asks authelia"; ok = let l = allLocal.services.nginx.virtualHosts."t.local".locations; route = "/api/agents/a/claude-login/code"; covering = lib.filter ( name: !(lib.hasPrefix "~" name) && !(lib.hasPrefix "= " name) && lib.hasPrefix (prefix name) route ) (lib.attrNames l); prefix = lib.removePrefix "^~ "; longest = lib.foldl' ( a: b: if lib.stringLength (prefix b) > lib.stringLength (prefix a) then b else a ) "" covering; in longest == "/api/" && lib.hasInfix "auth_request /__hive_authelia;" l.${longest}.extraConfig && !(lib.any (name: lib.hasPrefix "= /api/agents" name) (lib.attrNames l)); } { # `auth_request` does not inherit across sibling locations, so each # gated one is read on its own. `/health/` is the control: the one # c0re route that must stay open, so a pass means the block is # per-location rather than everywhere. name = "the dashboard answers on its own name behind authelia, and the hive domain redirects there"; ok = let vh = allLocal.services.nginx.virtualHosts; l = vh."h1.hives.t.local".locations; gated = loc: lib.hasInfix "auth_request /__hive_authelia;" loc.extraConfig; in allLocal.services.hyperhive.swarm.dashboardDomain == "h1.hives.t.local" && vh."h1.hives.t.local".forceSSL && lib.all (p: gated l.${p}) [ "/" "/api/" "/api/docs/" ] && lib.hasInfix "error_page 401 =302 https://auth.t.local/?rd=$target_url;" l."/".extraConfig && !(lib.hasInfix "auth_request" (l."/health/".extraConfig or "")) && l."= /__hive_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request" && lib.hasInfix "agents.conf" vh."h1.hives.t.local".extraConfig && lib.hasInfix "return 301 https://h1.hives.t.local$request_uri;" vh."h1.t.local".locations."/".extraConfig && !(vh."h1.t.local".locations ? "/api/") && lib.elem "h1.hives.t.local" allLocal.services.hyperhive.gateway.localNames && allLocal.systemd.services.hive-c0re.environment.HYPERHIVE_DASHBOARD_DOMAIN == "h1.hives.t.local"; } { # The gateway's catch-all and the hive's two vhosts (../host-modules/ # hive-c0re/vhosts.nix) are declared under `gateway.vhosts` like a # service's, so the gateway's listen set and issuer choice reach them # by the same path. The catch-all keeps its bare `return 444;` (no # headers: it sends no response) and the dashboard keeps its exact # location set. name = "the catch-all and the hive's vhosts render with the gateway's listen set, issuer choice, headers and locations"; ok = let gw = allLocal.services.hyperhive.gateway; vh = allLocal.services.nginx.virtualHosts; listenOf = map (l: { inherit (l) addr port; ssl = l.ssl or false; }); sameListen = v: listenOf v.listen == listenOf gw.lib.listen; sameCert = name: v: v.sslCertificate == (gw.lib.tlsFor name).sslCertificate; catchAll = vh."_"; dash = vh."h1.hives.t.local"; hive = vh."h1.t.local"; in sameListen catchAll && sameCert "_" catchAll && catchAll.addSSL && catchAll.default && catchAll.extraConfig == "return 444;\n" && sameListen dash && sameCert "h1.hives.t.local" dash && dash.forceSSL && !(dash.addSSL or false) && dash.extraConfig == gw.lib.securityHeaders + "include /var/lib/hive-gateway/conf/agents.conf;\n" && lib.attrNames dash.locations == [ "/" "/agent/" "/api/" "/api/docs/" "/health/" "= /__hive_agent_not_found" "= /__hive_agent_unreachable" "= /__hive_authelia" "= /api/docs" ] && sameListen hive && sameCert "h1.t.local" hive && hive.addSSL && hive.extraConfig == gw.lib.securityHeaders && hive.locations ? "/"; } { # The file held bcrypt hashes at 0644; a host carrying one from before # gets it deleted, and nothing names it for nginx. name = "the gateway reads no htpasswd, and tmpfiles removes a leftover one"; ok = lib.elem "r /var/lib/hive-gateway/conf/gateway.htpasswd" allLocal.systemd.tmpfiles.rules && !(lib.any ( v: lib.any (t: lib.hasInfix "auth_basic" t) ( [ (v.extraConfig or "") ] ++ map (loc: loc.extraConfig or "") (lib.attrValues (v.locations or { })) ) ) (lib.attrValues allLocal.services.nginx.virtualHosts)); } { name = "the swarm UI's hive roster carries each hive's dashboard name"; ok = builtins.fromJSON allLocal.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_HIVES == [ { name = "h1"; domain = "h1.t.local"; dashboard = "h1.hives.t.local"; } ]; } { name = "a hive with matrix off serves no matrix discovery endpoint"; ok = !(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations); } { # main got eval-borked twice by this exact class of bug (once on the # unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to # `gitea-actions-runner.nix` adds a plain `serviceConfig.*` # definition that collides with one of ours, and nix refuses to # merge two plain definitions at *host* eval. No other check # instantiates a host with `containers.swarm-ci-runner` actually enabled, so # the collision only surfaces on operator deploy, not in CI. name = "the CI container's unit definitions merge without a priority collision"; ok = forceCiServiceConfigs; } { # It spends money on an operator's key, so no host runs it by default. name = "the review classifier's container exists only where its host enables it"; ok = !(bare.containers ? swarm-review-classifier) && !(centralToggleOff.containers ? swarm-review-classifier) && withClassifier.containers ? swarm-review-classifier; } { # The daemon holds its store leaf, handed in by nspawn without a copy on # disk, and nothing else from the host; a key is read per run and # reaches only the model child's environment. Reading the claude path # also evaluates the container's unfree allowance. name = "the review classifier loads only its store leaf and carries no API or provider key in its unit"; ok = let svc = withClassifier.containers.swarm-review-classifier.config.systemd.services.swarm-review-classifier; env = svc.environment; in lib.elem "review-classifier-bao-key" svc.serviceConfig.LoadCredential && env.BAO_CLIENT_KEY == "/run/credentials/swarm-review-classifier.service/review-classifier-bao-key" && env.SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID == "swarm-review-classifier" && lib.hasSuffix "/bin/claude" env.SWARM_REVIEW_CLASSIFIER_CLAUDE && lib.hasSuffix "/bin/opencode" env.SWARM_REVIEW_CLASSIFIER_OPENCODE && !(env ? ANTHROPIC_API_KEY) && !(env ? ACP_PROVIDER_API_KEY) && lib.elem "--load-credential=review-classifier-bao-key:/var/lib/swarm-bao-pki/review-classifier-key.pem" ( withClassifier.containers.swarm-review-classifier.extraFlags ) # No host unit copies the leaf into the container's root. && !(lib.any (u: lib.hasInfix "nixos-containers/swarm-review-classifier" (u.script or "")) ( lib.attrValues withClassifier.systemd.services )); } { # See the same case in ./builder.nix. name = "the review classifier's container runs the leaf-minting unit before every start"; ok = let c = withClassifier.systemd.services."container@swarm-review-classifier"; in lib.elem "swarm-bao-pki.service" c.wants && lib.elem "swarm-bao-pki.service" c.after; } { # The collector reaches these routes through the gateway now, so each # store needs an ingest location of its own. Without one the write rides # the `/` catch-all: unauthenticated on the metrics store, and into a # browser redirect on the log store. name = "each store's vhost has an authenticated ingest location"; ok = let v = allLocal.services.nginx.virtualHosts; m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null; l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null; in m != null && l != null && lib.hasInfix "auth_request" m.extraConfig && lib.hasInfix "auth_request" l.extraConfig; } { # The metrics vhost is default-deny; the collector's push only lands # because its `metrics_endpoint` path is the vhost's one gated exact # location. The two literals live in different modules, so renaming # either alone breaks ingest while everything still evaluates. name = "the collector's metrics push path is a gated location on the metrics vhost"; ok = let endpoint = (otelSettings allLocal).exporters."otlphttp/victoriametrics".metrics_endpoint; # "https://host/path" -> "/path" path = "/" + lib.concatStringsSep "/" (lib.drop 3 (lib.splitString "/" endpoint)); loc = allLocal.services.nginx.virtualHosts."metrics.t.local".locations."= ${path}" or null; in loc != null && lib.hasInfix "auth_request" loc.extraConfig; } { # The arm that actually protects something. A pusher handed # `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page, # which answers 200 — ingest reporting healthy while storing nothing. # The third clause is the positive control: the log store's browser # location really does redirect, so this says the machine routes differ # rather than that the string is absent from the whole file. name = "the ingest locations answer 401 instead of redirecting a pusher"; ok = let v = allLocal.services.nginx.virtualHosts; m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig; l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig; browser = v."logs.t.local".locations."/".extraConfig; in !(lib.hasInfix "error_page" m) && !(lib.hasInfix "error_page" l) && lib.hasInfix "error_page" browser; } { # The store authenticates nothing itself, so a location that forwards to # it without `auth_request` is open to anything that can reach the # domain. The query location is the control: it shows the filter finds # store-forwarding locations at all. name = "no location on the metrics vhost forwards to the store without auth_request"; ok = let locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations; toStore = lib.filterAttrs ( _: l: l.proxyPass != null && lib.hasPrefix "http://unix:/run/swarm-victoriametrics/" l.proxyPass ) locs; open = lib.filterAttrs (_: l: !(lib.hasInfix "auth_request " l.extraConfig)) toStore; in toStore ? "= /api/v1/query" && open == { } && locs."/".proxyPass == null && locs."/".return == "403"; } { # The read API's check, link by link: every read location names the # switch with no login redirect (a machine reader would get login HTML # as a 200), the switch rewrites to the map's pick, an agent bearer # reaches the agents' responder (listed before the catch-all bearer arm, # since a map takes the first matching regex), any other bearer reaches # Grafana's, each socket bound by its unit, and anything else goes to # authelia. vmui is a browser's, so it redirects to the login. name = "the metrics read API sends an agent bearer to swarm-metrics-agent-auth, another bearer to swarm-metrics-auth and a session to authelia"; ok = let l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations; http = allLocal.services.nginx.appendHttpConfig; unit = allLocal.systemd.services.swarm-metrics-auth; agentUnit = allLocal.systemd.services.swarm-metrics-agent-auth; agentArm = "\"~^Bearer swarm-agent\\.\" /__metrics_agent_auth;"; bearerArm = "\"~^Bearer \" /__metrics_grafana_auth;"; v = allLocal.services.hyperhive.deploy.victoriametrics; logs = allLocal.services.hyperhive.deploy.victorialogs; reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l; in reads ? "= /api/v1/query_range" && lib.all ( r: lib.hasInfix "auth_request /__metrics_read_auth;" r.extraConfig && !(lib.hasInfix "error_page" r.extraConfig) ) (lib.attrValues reads) && lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http && lib.hasInfix agentArm http && lib.hasInfix bearerArm http && !(lib.hasInfix bearerArm (lib.head (lib.splitString agentArm http))) && lib.hasInfix "default /__metrics_authelia;" http && lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;" l."= /__metrics_read_auth".extraConfig && lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig && l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/" && lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart && lib.hasInfix "internal;" l."= /__metrics_agent_auth".extraConfig && l."= /__metrics_agent_auth".proxyPass == "http://unix:/run/swarm-metrics-agent-auth/auth.sock:/" && lib.hasInfix "--socket /run/swarm-metrics-agent-auth/auth.sock" agentUnit.serviceConfig.ExecStart && hasStorePath "${v.agentAuthPackage}/bin/swarm-logs-auth " agentUnit.serviceConfig.ExecStart && !(hasStorePath "${logs.authPackage}" agentUnit.serviceConfig.ExecStart) && hasStorePath "${v.authPackage}/bin/swarm-metrics-auth " unit.serviceConfig.ExecStart && lib.hasInfix "--store-cert-role swarm-metrics-auth" agentUnit.serviceConfig.ExecStart && lib.hasInfix "--verdict-ttl ${toString v.agentAuthVerdictTtl}" agentUnit.serviceConfig.ExecStart && agentUnit.serviceConfig.LoadCredential == unit.serviceConfig.LoadCredential && lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${v.agentAuthRate};" http && lib.hasInfix "limit_req zone=swarm_metrics_agent_auth burst=${toString v.agentAuthBurst} nodelay;" l."= /__metrics_agent_auth".extraConfig && !(lib.hasInfix "limit_req" l."= /__metrics_grafana_auth".extraConfig) && lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig && lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig && l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request"; } { # Grafana's host creates the bearer and its datasource sends it; the # store's responder reads it. Three modules spell the path, and the # store's grants name it twice: if any one drifts, Grafana sends a value # the responder never reads, or one of them is refused by the store. name = "Grafana's datasource sends the bearer swarm-metrics-auth checks, at one store path"; ok = let leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer"; ds = lib.findFirst (d: d.uid == "swarm-victoriametrics") null allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources; bearerUnit = allLocal.systemd.services.swarm-bao-grafana-metrics-bearer; s = allLocal.systemd.services; in ds.jsonData.httpHeaderName1 == "Authorization" && ds.secureJsonData.httpHeaderValue1 == "$__file{/run/swarm-grafana-secrets/metrics-bearer}" && lib.hasInfix "bao kv put -cas=0 secret/${leaf} value=-" bearerUnit.script && lib.hasInfix "/run/swarm-grafana-secrets/metrics-bearer \"Bearer $bearer\"" bearerUnit.script && lib.hasInfix "--store-path ${leaf}" s.swarm-metrics-auth.serviceConfig.ExecStart && lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"read\"]\n}" s.swarm-bao-metrics-auth-policy.script && lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"create\", \"read\"]\n}" s.swarm-bao-grafana-oidc-policy.script; } { # The log store's second responder checks the very bearer the metrics # datasource sends, at the same store path: a different file would be a # bearer the responder never reads. name = "Grafana's logs datasource sends the bearer the log store's responder checks, at one store path"; ok = let datasources = allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources; metrics = lib.findFirst (d: d.uid == "swarm-victoriametrics") null datasources; logs = lib.findFirst (d: d.uid == "swarm-victorialogs") null datasources; leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer"; in logs.secureJsonData.httpHeaderValue1 == metrics.secureJsonData.httpHeaderValue1 && lib.hasInfix "--store-path ${leaf}" allLocal.systemd.services.swarm-logs-grafana-auth.serviceConfig.ExecStart; } { # The responder reads the store as itself, with the leaf the store's # host mints for it. name = "swarm-metrics-auth presents its own store leaf"; ok = let v = allLocal.services.hyperhive.deploy.victoriametrics; lc = allLocal.systemd.services.swarm-metrics-auth.serviceConfig.LoadCredential; in lib.hasSuffix "/metrics-auth.pem" v.authBaoClientCertFile && lib.hasSuffix "/metrics-auth-key.pem" v.authBaoClientKeyFile && builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc && builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc && lib.hasInfix "/metrics-auth.pem ]" allLocal.systemd.services.swarm-bao-pki.script; } { # The read counterpart to the ingest location: an agent queries the log # store with a bearer token, and the `/` catch-all is the browser's # route. Riding it would mean inheriting the login redirect the next # case is about, so the route has to exist separately to be gated # separately. name = "the log store's vhost has an authenticated machine query location"; ok = let q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/" or null; in q != null && lib.hasInfix "auth_request" q.extraConfig; } { # Same trap as the ingest case, on the read side, where it is worse: a # redirected pusher at least stores nothing visibly, while a redirected # *reader* is handed a 200 carrying login HTML and records a query that # succeeded and matched no logs. The browser clause is the positive # control — that location really does redirect to the login host — so a # pass means these two routes differ rather than that the strings are # absent from the whole vhost. name = "the machine query location answers 401 instead of redirecting to a login page"; ok = let v = allLocal.services.nginx.virtualHosts; q = v."logs.t.local".locations."^~ /select/logsql/".extraConfig; browser = v."logs.t.local".locations."/".extraConfig; in !(lib.hasInfix "error_page" q) && !(lib.hasInfix "auth.t.local" q) && lib.hasInfix "error_page" browser && lib.hasInfix "auth.t.local" browser; } { # Read access is deliberately unscoped: an authenticated caller reads # the whole swarm's logs until a permission system exists. Pinned so a # scoping parameter arriving later is a visible diff here rather than a # quiet change of rule — and pinned on `proxyPass` too, because # VictoriaLogs takes its filters as request parameters, which ride an # upstream URI as easily as a directive. The first clause is the # control: it proves the location resolved and that `hasInfix` finds # what is genuinely in this string, so the absences below mean absent # rather than unreadable. name = "the machine query location forwards the caller's query unmodified"; ok = let q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/"; in lib.hasInfix "auth_request" q.extraConfig && !(lib.hasInfix "extra_filters" q.extraConfig) && !(lib.hasInfix "extra_stream_filters" q.extraConfig) && !(lib.hasInfix "$args" q.extraConfig) && !(lib.hasInfix "?" q.proxyPass); } { # `auth_request` takes a fixed URI, so the split between the agents' # responder and authelia is a map plus a rewrite in an internal # location. Every link of that chain is pinned: the map keys on the # agent-token prefix and falls back to authelia, the query location # names the switch, the switch rewrites to the map's pick, and the # agent target reaches the responder's socket, which the unit binds. name = "the machine query location sends agent bearers to swarm-logs-auth and the rest to authelia"; ok = let l = allLocal.services.nginx.virtualHosts."logs.t.local".locations; http = allLocal.services.nginx.appendHttpConfig; unit = allLocal.systemd.services.swarm-logs-auth; grafanaUnit = allLocal.systemd.services.swarm-logs-grafana-auth; logs = allLocal.services.hyperhive.deploy.victorialogs; metrics = allLocal.services.hyperhive.deploy.victoriametrics; in lib.hasInfix "map \$http_authorization \$swarm_logs_auth_target" http && lib.hasInfix "\"~^Bearer swarm-agent\\.\" /__swarm_logs_agent_auth;" http && lib.hasInfix "\"~^Bearer [0-9a-f]{64}$\" /__swarm_logs_grafana_auth;" http && lib.hasInfix "default /__hive_authelia;" http && lib.hasInfix "auth_request /__swarm_logs_auth;" l."^~ /select/logsql/".extraConfig && lib.hasInfix "rewrite ^ \$swarm_logs_auth_target last;" l."= /__swarm_logs_auth".extraConfig && lib.hasInfix "internal;" l."= /__swarm_logs_agent_auth".extraConfig && l."= /__swarm_logs_agent_auth".proxyPass == "http://unix:/run/swarm-logs-auth/auth.sock:/" && lib.hasInfix "--socket /run/swarm-logs-auth/auth.sock" unit.serviceConfig.ExecStart && lib.hasInfix "internal;" l."= /__swarm_logs_grafana_auth".extraConfig && l."= /__swarm_logs_grafana_auth".proxyPass == "http://unix:/run/swarm-logs-grafana-auth/auth.sock:/" && lib.hasInfix "--socket /run/swarm-logs-grafana-auth/auth.sock" grafanaUnit.serviceConfig.ExecStart && hasStorePath "${logs.grafanaAuthPackage}/bin/swarm-metrics-auth " grafanaUnit.serviceConfig.ExecStart && !(hasStorePath "${metrics.authPackage}" grafanaUnit.serviceConfig.ExecStart) && hasStorePath "${logs.authPackage}/bin/swarm-logs-auth " unit.serviceConfig.ExecStart && lib.hasInfix "--store-cert-role swarm-logs-auth" grafanaUnit.serviceConfig.ExecStart && l ? "= /__hive_authelia"; } { # A `DynamicUser` unit's preserved runtime directory lives under # `/run/private/` (0700 root:root), which nginx cannot traverse, so a # socket there answers every proxied request with a 502. Each `unix:` # upstream under `/run//` is matched to the units creating ``. # The first two clauses are the control: the match finds both responders. name = "no unix socket nginx proxies to sits in a DynamicUser's preserved runtime directory"; ok = let prefix = "http://unix:/run/"; upstreamDirs = lib.unique ( map (p: lib.head (lib.splitString "/" (lib.removePrefix prefix p))) ( lib.filter (p: p != null && lib.hasPrefix prefix p) ( lib.concatMap (vh: lib.mapAttrsToList (_: loc: loc.proxyPass) vh.locations) ( lib.attrValues allLocal.services.nginx.virtualHosts ) ) ) ); runtimeDirs = sc: let r = sc.RuntimeDirectory or [ ]; in if lib.isList r then r else lib.splitString " " r; owners = lib.filterAttrs ( _: u: lib.any (d: lib.elem d upstreamDirs) (runtimeDirs u.serviceConfig) ) allLocal.systemd.services; hidden = sc: lib.elem (sc.DynamicUser or false) [ true "yes" ] && !(lib.elem (sc.RuntimeDirectoryPreserve or "no") [ false "no" ]); in owners ? swarm-logs-auth && owners ? swarm-logs-grafana-auth && owners ? swarm-metrics-auth && owners ? swarm-metrics-agent-auth && !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners)); } { # Every agent bearer the responder has not recently granted costs a # store login and read, so its location carries the per-address limit. # The responder gets the grant TTL from the option. The query and # authelia locations carry no limit of their own. name = "agent bearer checks are rate-limited per address and granted for a TTL"; ok = let l = allLocal.services.nginx.virtualHosts."logs.t.local".locations; http = allLocal.services.nginx.appendHttpConfig; unit = allLocal.systemd.services.swarm-logs-auth; v = allLocal.services.hyperhive.deploy.victorialogs; in lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${v.authRate};" http && lib.hasInfix "limit_req zone=swarm_logs_agent_auth burst=${toString v.authBurst} nodelay;" l."= /__swarm_logs_agent_auth".extraConfig && lib.hasInfix "limit_req_status 403;" l."= /__swarm_logs_agent_auth".extraConfig && !(lib.hasInfix "limit_req" l."^~ /select/logsql/".extraConfig) && !(lib.hasInfix "limit_req" l."= /__hive_authelia".extraConfig) && lib.hasInfix "--verdict-ttl 60" unit.serviceConfig.ExecStart; } { # The responder reads the store as itself, with the leaf the store's # host mints for it. name = "swarm-logs-auth presents its own store leaf"; ok = let v = allLocal.services.hyperhive.deploy.victorialogs; lc = allLocal.systemd.services.swarm-logs-auth.serviceConfig.LoadCredential; in lib.hasSuffix "/logs-auth.pem" v.authBaoClientCertFile && lib.hasSuffix "/logs-auth-key.pem" v.authBaoClientKeyFile && builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc && builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc; } { # The absence arm for the case above, and the option's own rule — a # service declares its entry under its own `enable` — made checkable. # Without it, moving the assignment outside the collector's `mkIf` # passes every arm above while handing a collector-less hive a scrape # target for a port nothing binds. name = "a hive with no collector declares no self-scrape target"; ok = bare.services.hyperhive.otel.scrapeTargets == { }; } { # Absence arm, and the one that matters: claiming a name this host does # not serve points every local reader at the wrong machine. name = "a hive that does not run the store claims no name for it"; ok = !(builtins.elem "bao.t.local" (baoNames bare)); } { # ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a # host-wide option, so a block rendered outside the store's own `mkIf` # gives every hive in the swarm a listener — on the port the store # answers on, in front of no store at all. name = "a hive that does not run the store renders no stream passthrough"; ok = baoStream bare == ""; } { # Absence arm for the case above — a hive with no store has no reason to # open the store's port, and opening it would point agents at a host that # answers nothing. name = "a hive that does not run the store opens no bridge port for it"; ok = !(builtins.elem 8200 (bridgePorts bare)); } { # The four blocks below used to be gated on the hive being enabled AND # their own condition. The second half was always the load-bearing one — # none of these conditions is derived from the hive toggle — so these # arms pin what the conjunct was doing: nothing. Each is written against # a host with the hive OFF as well as one with it on, because the way a # dropped conjunct fails is by making something unconditional, and that # shows up as a service appearing where nothing asked for it. name = "no bridge port is opened for an exposeHostPorts nobody set"; ok = !(builtins.elem 5432 (bridgePortsOrNone bare)) && !(builtins.elem 5432 (bridgePortsOrNone centralToggleOff)); } { # `deploy.swarm-controller.enable`, which defaults false and is # deliberately not derived from the hive toggle — a swarm has one # controller, so the host that runs it says so itself. # # Probed by what the daemon needs in order to run, not by # `? swarm-controller`: ./host-modules/glue-hive-tls-controllers.nix defines an # environment key on that unit name, which leaves the attr # present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive # that has a CA — see the comment there. The daemon's system user has no # second definer, so its absence is the unambiguous half. name = "the swarm controller does not run unless this host is told to run it"; ok = let inert = machine: !(machine.users.users ? swarm-controller) && !( (machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart ); in inert bare && inert centralToggleOff; } { # `deploy.swarm-otel.enable`, same shape: the swarm's collector is one # host's job, and the container is the whole of what it renders. name = "the swarm collector container is absent unless this host is told to run it"; ok = !(bare.containers ? swarm-otel) && !(centralToggleOff.containers ? swarm-otel); } { # The swarm UI follows `deploy.swarm-controller.enable` — a sibling # deployment decision, not the hive toggle. `t.local` is the fixtures' # swarm domain, which is the apex the UI claims; the arm below is what # proves this vhost renders at all. name = "the swarm UI vhost is absent unless this host runs the swarm controller"; ok = !(bare.services.nginx.virtualHosts ? "t.local") && !(centralToggleOff.services.nginx.virtualHosts ? "t.local"); } { # The three infrastructure toggles are off by default and asserted by # whoever needs them. With nothing on the host needing them, none of # the three renders — which is also the control for the arm below. name = "the gateway, resolver and bridge are absent where nothing on the host needs them"; ok = !centralToggleOff.services.hyperhive.gateway.enable && !centralToggleOff.services.hyperhive.gateway.dns.enable && !centralToggleOff.services.hyperhive.network.enable && !(centralToggleOff.services.nginx.enable or false) && !(centralToggleOff.services.dnsmasq.enable or false) && !(centralToggleOff.networking.bridges ? hive-br0); } { # hive-c0re asserts all three, and it follows the central toggle — so # an ordinary hive keeps getting them with no opt-in, which is what # this change must not break. name = "an ordinary hive runs the gateway, resolver and bridge because its coordinator needs them"; ok = bare.services.hyperhive.gateway.enable && bare.services.hyperhive.gateway.dns.enable && bare.services.hyperhive.network.enable && bare.services.nginx.enable && bare.services.dnsmasq.enable && bare.networking.bridges ? hive-br0; } { # Refused wherever something turns the name into an identifier, and only # there. The store host is the control that the refusal is not the hive's # alone; the host running neither is the control that it is gated at all. name = "a missing hiveName is refused on a hive and on a store host, not elsewhere"; ok = let refusedHiveName = extra: lib.any (a: !a.assertion && lib.hasInfix "services.hyperhive.hiveName to be set" a.message) (hiveWithController ({ hiveName = null; } // extra)).assertions; in refusedHiveName { } && refusedHiveName { deploy.hive-controller.enable = false; deploy.bao.enable = true; } && !(refusedHiveName { deploy.hive-controller.enable = false; }); } { # The control is a host that defines no such option; the removal message # must not appear for it. name = "a definition of the removed services.hyperhive.ruthless option fails evaluation, pointing at the swarm dashboard"; ok = let refused = m: lib.any ( a: !a.assertion && lib.hasInfix "services.hyperhive.ruthless" a.message && lib.hasInfix "swarm dashboard" a.message ) m.assertions; in refused (hive { ruthless = true; }) && !(refused allLocal); } { # Refused wherever any hyperhive service runs, the hive or a swarm # service alone. The host enabling nothing is the control that it is # gated at all, and has to pass every assertion: it is what an # operator gets from importing the module. name = "a missing swarm.domain is refused on a hive and on a swarm-service host, and a host enabling nothing evaluates"; ok = let noDomain = extra: hiveWithController ({ swarm.domain = null; } // extra); refusedSwarmDomain = extra: lib.any ( a: !a.assertion && lib.hasInfix "services.hyperhive.swarm.domain to be set" a.message ) (noDomain extra).assertions; in refusedSwarmDomain { } && refusedSwarmDomain { deploy.hive-controller.enable = false; deploy.nats.enable = true; } && lib.all (a: a.assertion) (noDomain { deploy.hive-controller.enable = false; }).assertions; } { # Control: a store fetch with no timer still renders `RemainAfterExit`, # so the cases below read a real value rather than a missing one. name = "Grafana's admin-password fetch stays active after it exits and has no timer"; ok = allLocal.systemd.services ? swarm-bao-grafana-admin && remainsAfterExit allLocal.systemd.services.swarm-bao-grafana-admin && !(allLocal.systemd.timers ? swarm-bao-grafana-admin); } { # A timer whose unit does not exist fails to load, so each timer is # gated exactly as its fetch is. name = "a host running none of the store-refreshed fetches renders none of their timers"; ok = lib.all ( { unit, ... }: !(bare.systemd.services ? ${unit}) && !(bare.systemd.timers ? ${unit}) ) storeRefreshed; } { # `or false` so a dropped key fails this case by name instead of # aborting eval. Only the rendered unit is pinned; eval cannot show # that hive-priv still works under the directives. name = "hive-priv locks personality and denies real-time scheduling, kernel logs and module loading"; ok = let sc = allLocal.systemd.services.hive-priv.serviceConfig; in (sc.LockPersonality or false) && (sc.RestrictRealtime or false) && (sc.ProtectKernelLogs or false) && (sc.ProtectKernelModules or false); } ] # Starting a unit that stays `active (exited)` is a no-op, so a timer aimed # at one fires and fetches nothing. ++ map ( { unit, machine }: { name = "${unit} re-runs on a ten-minute timer and does not stay active after it exits"; ok = let s = machine.systemd.services; t = machine.systemd.timers; in s ? ${unit} && t ? ${unit} && !(remainsAfterExit s.${unit}) && lib.elem "timers.target" t.${unit}.wantedBy && t.${unit}.timerConfig.OnUnitInactiveSec == "10min" && (t.${unit}.timerConfig.Unit or "${unit}.service") == "${unit}.service"; } ) storeRefreshed; in runGroup "core-toggle" cases