# `checks.module-eval-swarm-otel-core` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive carriesJournaldSeverity runGroup otelSettings ; # A swarm collector on a host that runs NEITHER store — the fully-spread # shape from docs/swarm/services.md, and the one the old per-host gates made # inexpressible. It is the whole point of the cases below that this hive is # not a degenerate configuration but a supported one. otelNoStores = hive { deploy.swarm-otel.enable = true; deploy.authelia.enable = true; deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; deploy.victoriametrics.enable = false; deploy.victorialogs.enable = false; }; # The collector beside authelia, reading its own OIDC secret out of the # store like every other collector — the cert pair here is not scenery, it # is the arm that would catch the deleted co-located copy unit coming back. otelBaoWithAuthelia = hive { deploy.swarm-otel.enable = true; deploy.authelia.enable = true; deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; }; # The same collector with the IdP on ANOTHER host and a store leaf placed by # hand. Identical to the fixture above in everything the delivery path # reads, which is the point. otelBaoRemoteAuthelia = hive { deploy.swarm-otel.enable = true; swarm.authelia.url = "https://auth.example.invalid"; deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; }; cases = [ { # 🩸 The arm that guards the ruling this slice landed under, the # collector's half of ./swarm-grafana.nix's own. There is ONE delivery # route: the store reader, on every host that runs the collector and # holds a store identity. The negative names the deleted unit rather # than a generic absence, because the way this regresses is someone # re-adding the co-located copy as an optimisation. name = "the collector's OIDC secret has exactly one delivery unit, the store reader, in both topologies"; ok = let local = otelBaoWithAuthelia.systemd.services; remote = otelBaoRemoteAuthelia.systemd.services; in local ? swarm-bao-otel-oidc && remote ? swarm-bao-otel-oidc && !(local ? swarm-otel-oidc-secret) && !(remote ? swarm-otel-oidc-secret); } { # Same 403-not-a-miss reason as grafana's arm above: the reader's grant # covers the `services` prefix, so a path outside it is refused rather # than empty, however correct it reads. name = "the collector's OIDC secret is read from the prefix the publisher writes"; ok = let s = otelBaoRemoteAuthelia.systemd.services.swarm-bao-otel-oidc.script; in lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s && !(lib.hasInfix "secret/swarm/hives/" s); } { # The defect itself. These exporters used to be gated on the stores' # PER-HOST enables, so a collector that did not share a host with them # rendered none at all and dropped everything it received, from every # hive — silently, because an absent exporter is not an error. name = "a collector that hosts neither store still exports to both"; ok = let e = (otelSettings otelNoStores).exporters; in (e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs"); } { # A swarm has one of each store, so the address is a swarm-level name. # A loopback literal here is the co-location assumption written back in, # and it renders, deploys and reports healthy while reaching nothing. name = "the store exporters address the stores by name, never by loopback"; ok = let e = (otelSettings otelNoStores).exporters; m = e."otlphttp/victoriametrics".metrics_endpoint; l = e."otlphttp/victorialogs".logs_endpoint; in !(lib.hasInfix "127.0.0.1" m) && !(lib.hasInfix "127.0.0.1" l) && lib.hasInfix "metrics.t.local" m && lib.hasInfix "logs.t.local" l; } { # `_HOSTNAME` cannot separate machines on its own: a hostname is a # config value two of them can share, and then every stream for a unit # name merges into one. name = "the log stream is keyed by machine, not only by a hostname every container shares"; ok = let l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint; field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l; in field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD"); } { # Defining an exporter and REFERENCING it are two separate lists, and # the second is where the original gate also lived. An exporter no # pipeline names is as silent as one that does not exist — this case # exists because a mutation that restored only the reference-side gate # left every other case here green. name = "every pipeline that has a store exporter defined actually sends to it"; ok = let s = otelSettings otelNoStores; used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines)); in builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used; } { # An authenticator an exporter names but `service.extensions` omits is # INERT — the collector starts clean and pushes unauthenticated until # something at the far end refuses it. Checked as a set relation rather # than by naming the two, so it keeps holding for exporters not written # yet. name = "every exporter authenticator is listed in service.extensions"; ok = let s = otelSettings otelNoStores; named = lib.filter (v: v != null) ( lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters ); in named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } { # The host-journal sibling of ./agent-otel.nix's wiring case, which # carries the full reasoning. Same question, different receiver: this # one reads the HOST's journal rather than a container's, and the two # are unrelated config — a fixed stanza there, a parameterised block # inside `containers.swarm-otel` here — so one losing its parser while # the other keeps one is a real and silent state. # # Contents are not this case's business. The table both receivers import # is asserted once, in ./journald-severity.nix. name = "the swarm collector's journald receiver carries the shared PRIORITY mapping"; ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald; } ]; in runGroup "swarm-otel-core" cases