diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index 24867e6e..dbf62a38 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -1,6 +1,14 @@ # OpenTelemetry wiring for an agent container: the per-agent options the -# meta flake injects (host-driven from `services.hyperhive.otel.*`), and -# the environment every OTLP producer inside the container reads. +# meta flake injects (host-driven from `services.hyperhive.otel.*`), the +# environment every OTLP producer inside the container reads, and the +# collector that forwards this container's journal to the hive. +# +# Two directions, one endpoint. Metrics are PUSHED by processes in here +# that speak OTLP themselves; logs are not pushed by anyone, because +# nothing in a journal has an SDK. The forwarder is what turns the second +# into the first, and it has to run inside the container — a hive-level +# reader cannot see this journal at all (the host-side per-container +# directory is an id-mapped bind mount journald never writes into). # # Claude Code is one producer here, not the owner. `hive-metric` — and # any future in-container exporter — reads the same endpoint, protocol @@ -10,6 +18,7 @@ # options declared below. { lib, + pkgs, config, ... }: @@ -58,6 +67,11 @@ let // lib.optionalAttrs (cfg.metricIntervalMs != null) { OTEL_METRIC_EXPORT_INTERVAL = toString cfg.metricIntervalMs; }; + + # The exporter's NAME picks the wire protocol, so `protocol` has to be + # read here as well as handed to the SDKs above — same expression as the + # swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix. + logExporter = if cfg.protocol == "grpc" then "otlp" else "otlphttp"; in { # OTEL stats export is configured ONCE at host level via @@ -164,5 +178,99 @@ in # while `NIX_REMOTE` — the same value, also set globally — is there. systemd.globalEnvironment = producerEnv; environment.variables = producerEnv; + + # The journal this container writes is reachable from inside it and + # from nowhere else, which is why the forwarder runs here rather than + # the hive reading the container's journal from outside: the host-side + # per-container directory is an id-mapped bind mount and journald + # writes nothing into it. + assertions = [ + { + # Sibling of the swarm collector's storage assertion in + # nix/host-modules/swarm-otel.nix, and it exists because the + # failure is silent at every layer: with a volatile journal the + # receiver below finds an empty directory, reads nothing, and the + # collector starts clean and stays healthy forever. + assertion = + !(lib.elem config.services.journald.storage [ + "volatile" + "none" + ]); + message = '' + hyperhive.otel.enable is on for agent ${userName}, but + services.journald.storage is + "${config.services.journald.storage}" in this container. + + The log forwarder reads /var/log/journal, which journald only + writes when it stores persistently: with "volatile" the journal + lives in /run/log/journal and with "none" there is none at all. + Either way the forwarder would ship nothing while looking + healthy. + ''; + } + ]; + + services.opentelemetry-collector = { + enable = true; + # Contrib, and not a preference: `journald` is a contrib receiver. + # The upstream default build has no way to read a journal at all. + package = pkgs.opentelemetry-collector-contrib; + # Runs `otelcol validate` at build time. ⚠️ A parser, not a wiring + # check — it accepts a pipeline naming a component the build lacks, + # and the collector then dies at startup. Same caveat as both other + # tiers; see nix/host-modules/otel.nix for the measurement. + validateConfigFile = true; + settings = { + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own default + # is the RUNTIME journal (`/run/log/journal`), which in an agent + # container is empty — journald stores persistently here, so + # every entry is under /var/log/journal. Dropping this line + # leaves a collector that validates, starts, reports healthy and + # forwards nothing. + directory = "/var/log/journal"; + # No `units` allowlist, unlike the swarm tier's receiver. That + # one needs one because the host's journal also holds an + # operator's own session; a container's journal is the harness + # and what the harness spawns, so there is no foreign traffic to + # filter out and an allowlist would only be a list to forget to + # update. + }; + + # Identity the hop above cannot supply. A host-side reader can say + # which machine a line came from; only a collector running as the + # agent can say which agent. `hive` and `swarm` are deliberately + # NOT stamped here even though the env above carries them for + # in-process producers: the swarm tier upserts `hive` from the + # receiver that accepted the sample, precisely so the label comes + # from something the sender cannot write. + processors.resource.attributes = [ + { + key = "service.name"; + value = "hyperhive-agent"; + action = "upsert"; + } + { + key = "agent"; + value = userName; + action = "upsert"; + } + ]; + + # The hive's collector, at the same base address every in-process + # producer already exports to. `endpoint` is a BASE the exporter + # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. + exporters.${logExporter} = { + endpoint = cfg.endpoint; + } + // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; + + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ logExporter ]; + }; + }; + }; }; } diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 95ed69e1..65b90c3d 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -532,6 +532,30 @@ in ]; exporters = [ swarmName ]; }; + + # The second signal this tier carries, and the reason it exists + # is that an agent's journal is unreadable from here: the + # host-side per-container journal directory is an id-mapped bind + # mount journald never writes into, so each agent container runs + # its own forwarder (nix/agent-modules/otel.nix) and pushes over + # the `otlp` receiver above. Without this pipeline that receiver + # answers 404 on `/v1/logs` — measured, and indistinguishable + # from a route that was never meant to exist. + # + # One receiver, two pipelines: `otlp` accepts both signals and + # each pipeline takes the one it names. The exporter needs no + # change either — its `endpoint` is a base, and otlphttp appends + # `/v1/logs` to it exactly as it appends `/v1/metrics`. + service.pipelines.logs = { + receivers = [ "otlp" ]; + # `resourcedetection` and NOT `deltatocumulative`: that one is + # a metrics-only processor, and naming it here is a startup + # failure rather than a no-op. Host identity applies to both + # signals for the same reason — a line is only comparable + # across hives once it says which machine collected it. + processors = [ "resourcedetection" ]; + exporters = [ swarmName ]; + }; } // lib.optionalAttrs senderAuth { extensions.${authName} = { diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 090b158c..db813dcd 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1255,6 +1255,28 @@ in exporters = exporterNames; }; } + # A hive's logs, and the receiver is the SAME one its metrics + # arrive on: a receiver is not per-signal, so `otlp/${h}` + # feeds this pipeline and `metrics/${h}` above with no second + # port, credential or audience. Which is also why the `hive` + # label stays trustworthy for logs without any new mechanism + # — it still comes from which receiver accepted the sample. + # + # The senders are agent containers, one forwarder each + # (nix/agent-modules/otel.nix), via their own hive's collector. + # Without this pipeline that whole path terminates here: the + # receiver accepts the push, answers 200, and the records go + # nowhere. + // lib.optionalAttrs collectLogs ( + lib.mapAttrs' ( + h: _: + lib.nameValuePair "logs/${h}" { + receivers = [ "otlp/${h}" ]; + processors = [ "resource/${h}" ]; + exporters = logExporterNames; + } + ) hivePorts + ) # Logs fan out exactly as metrics do: the swarm's store when it # runs, the operator's upstream when one is configured, both # when both. The local store is a destination rather than the