From a00bcb8c0572a507548626d663ce745ffb01bd7d Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 22:20:24 +0100 Subject: [PATCH] merge the self-scrape into each collector's settings instead of defining them twice --- nix/agent-modules/otel.nix | 177 ++++++++++---------- nix/host-modules/swarm-bao.nix | 297 +++++++++++++++++---------------- 2 files changed, 238 insertions(+), 236 deletions(-) diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index 1feaa003..f913ffce 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -215,100 +215,101 @@ in # and the collector then dies at startup. Same caveat as both other # tiers; see nix/host-modules/otel.nix for the measurement. validateConfigFile = true; - settings = { - # `journalctl --follow --lines=0` (what this receiver runs under - # the hood) ships only what's written *after* it starts — every - # collector start has a silent gap at the front. `storage:` below - # persists the read cursor, so a restart resumes from where it - # left off instead of re-opening that gap; `start_at` stays `end` - # (its default) because a cursor already covers every run after - # the first, and `beginning` without one would re-ship the whole - # journal on every restart. The cursor lives in the unit's own - # `StateDirectory` (nixpkgs' module already sets one, `%S`, below - # as `WorkingDirectory` too) — container-lifetime, not the - # host-persisted state bind mount: it indexes /var/log/journal, - # itself fresh on every container start, so the two must share - # a lifetime or the cursor outlives the journal it points into. - # `create_directory` matters here for a build-time-vs-runtime - # reason, not a convenience one: `validateConfigFile` above runs - # `otelcol validate` in the nix build sandbox, before systemd's - # `StateDirectory=` has ever created this path. Without it the - # validator refuses with "directory must exist" and the build fails. - extensions.file_storage = { - directory = "/var/lib/opentelemetry-collector"; - create_directory = true; - }; + settings = lib.mkMerge [ + { + # `journalctl --follow --lines=0` (what this receiver runs under + # the hood) ships only what's written *after* it starts — every + # collector start has a silent gap at the front. `storage:` below + # persists the read cursor, so a restart resumes from where it + # left off instead of re-opening that gap; `start_at` stays `end` + # (its default) because a cursor already covers every run after + # the first, and `beginning` without one would re-ship the whole + # journal on every restart. The cursor lives in the unit's own + # `StateDirectory` (nixpkgs' module already sets one, `%S`, below + # as `WorkingDirectory` too) — container-lifetime, not the + # host-persisted state bind mount: it indexes /var/log/journal, + # itself fresh on every container start, so the two must share + # a lifetime or the cursor outlives the journal it points into. + # `create_directory` matters here for a build-time-vs-runtime + # reason, not a convenience one: `validateConfigFile` above runs + # `otelcol validate` in the nix build sandbox, before systemd's + # `StateDirectory=` has ever created this path. Without it the + # validator refuses with "directory must exist" and the build fails. + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; - receivers.journald = { - # ⚠️ STATED, and it must stay stated: the receiver's own default - # is the RUNTIME journal (`/run/log/journal`), which in an agent - # container is empty — journald stores persistently here, so - # every entry is under /var/log/journal. Dropping this line - # leaves a collector that validates, starts, reports healthy and - # forwards nothing. - directory = "/var/log/journal"; - storage = "file_storage"; - # No `units` allowlist, and no user-session filter like the swarm - # tier's: a container's journal is the harness and what the - # harness spawns, so there is no foreign traffic to filter out and - # an allowlist would only be a list to forget to update. + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own default + # is the RUNTIME journal (`/run/log/journal`), which in an agent + # container is empty — journald stores persistently here, so + # every entry is under /var/log/journal. Dropping this line + # leaves a collector that validates, starts, reports healthy and + # forwards nothing. + directory = "/var/log/journal"; + storage = "file_storage"; + # No `units` allowlist, and no user-session filter like the swarm + # tier's: a container's journal is the harness and what the + # harness spawns, so there is no foreign traffic to filter out and + # an allowlist would only be a list to forget to update. - # journald's PRIORITY carries the level every line already has; - # without this the record reaches VictoriaLogs with - # `severity_text: Unspecified` and the store cannot tell an error - # from a debug line. The table — and the inverted direction that - # makes it worth a file of its own — lives in - # ../journald-severity.nix, shared with the swarm tier's receiver. - operators = import ../journald-severity.nix; - }; + # journald's PRIORITY carries the level every line already has; + # without this the record reaches VictoriaLogs with + # `severity_text: Unspecified` and the store cannot tell an error + # from a debug line. The table — and the inverted direction that + # makes it worth a file of its own — lives in + # ../journald-severity.nix, shared with the swarm tier's receiver. + operators = import ../journald-severity.nix; + }; - # Identity the hop above cannot supply. A host-side reader can say - # which machine a line came from; only a collector running as the - # agent can say which agent. `hive` and `swarm` are deliberately - # NOT stamped here: the swarm tier upserts both from the receiver - # that accepted the sample, precisely so the labels come from - # something the sender cannot write. - processors.resource.attributes = [ - { - key = "service.name"; - value = "hyperhive-agent"; - action = "upsert"; + # Identity the hop above cannot supply. A host-side reader can say + # which machine a line came from; only a collector running as the + # agent can say which agent. `hive` and `swarm` are deliberately + # NOT stamped here: the swarm tier upserts both from the receiver + # that accepted the sample, precisely so the labels come from + # something the sender cannot write. + processors.resource.attributes = [ + { + key = "service.name"; + value = "hyperhive-agent"; + action = "upsert"; + } + { + key = "agent"; + value = userName; + action = "upsert"; + } + ]; + + # The hive's collector, at the same base address every in-process + # producer already exports to. `endpoint` is a BASE the exporter + # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. + exporters.${logExporter} = { + endpoint = cfg.endpoint; } - { - key = "agent"; - value = userName; - action = "upsert"; - } - ]; + // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; - # The hive's collector, at the same base address every in-process - # producer already exports to. `endpoint` is a BASE the exporter - # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. - exporters.${logExporter} = { - endpoint = cfg.endpoint; + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ logExporter ]; + }; + + # An extension configured but not listed here is INERT — the + # journald receiver's `storage: file_storage` above would name a + # component the collector never starts. + service.extensions = [ "file_storage" ]; } - // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; - - service.pipelines.logs = { - receivers = [ "journald" ]; - processors = [ "resource" ]; - exporters = [ logExporter ]; - }; - - # An extension configured but not listed here is INERT — the - # journald receiver's `storage: file_storage` above would name a - # component the collector never starts. - service.extensions = [ "file_storage" ]; - }; - }; - - # Every agent container has a network namespace of its own, so they can all - # use the same port. - services.opentelemetry-collector.settings = import ../collector-self-scrape.nix { - inherit lib; - port = selfTelemetryPort; - exporter = logExporter; - attributes.agent = userName; + # Every agent container has a network namespace of its own, so they can all + # use the same port. + (import ../collector-self-scrape.nix { + inherit lib; + port = selfTelemetryPort; + exporter = logExporter; + attributes.agent = userName; + }) + ]; }; }; } diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 4307c39e..dd5b1bab 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -4438,162 +4438,163 @@ in # caveat as every other tier; see nix/host-modules/otel.nix for # the measurement. validateConfigFile = true; - settings = { - # `journalctl --follow --lines=0` (what the receiver runs under - # the hood) ships only what is written *after* it starts, so - # every collector start has a silent gap at the front. This - # persists the read cursor, so a restart resumes instead of - # re-opening that gap; `start_at` stays `end` (its default), - # since a cursor covers every run after the first and - # `beginning` without one would re-ship the whole journal on - # every restart. `create_directory` is required for a - # build-time reason rather than a convenience one: - # `validateConfigFile` above runs `otelcol validate` in the nix - # sandbox, before systemd's `StateDirectory=` has created the - # path, and the validator refuses with "directory must exist". - extensions.file_storage = { - directory = "/var/lib/opentelemetry-collector"; - create_directory = true; - }; + settings = lib.mkMerge [ + { + # `journalctl --follow --lines=0` (what the receiver runs under + # the hood) ships only what is written *after* it starts, so + # every collector start has a silent gap at the front. This + # persists the read cursor, so a restart resumes instead of + # re-opening that gap; `start_at` stays `end` (its default), + # since a cursor covers every run after the first and + # `beginning` without one would re-ship the whole journal on + # every restart. `create_directory` is required for a + # build-time reason rather than a convenience one: + # `validateConfigFile` above runs `otelcol validate` in the nix + # sandbox, before systemd's `StateDirectory=` has created the + # path, and the validator refuses with "directory must exist". + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; - receivers.journald = { - # ⚠️ STATED, and it must stay stated: the receiver's own - # default is the RUNTIME journal (`/run/log/journal`), which - # in this container is empty — journald stores persistently - # here, so every entry is under /var/log/journal. Dropping - # this line leaves a collector that validates, starts, - # reports healthy and forwards nothing. - directory = "/var/log/journal"; - # ⚠️ `--merge` is a guard: an unqualified - # `journalctl --follow` over a journal with no entry for the - # current boot (a host-linked one) crash-loops the receiver - # every ~2s with "No journal boot entry found for the - # specified boot (+0)". This journal is container-local, so - # the current boot has an entry and the flag changes nothing. - merge = true; - storage = "file_storage"; - }; + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own + # default is the RUNTIME journal (`/run/log/journal`), which + # in this container is empty — journald stores persistently + # here, so every entry is under /var/log/journal. Dropping + # this line leaves a collector that validates, starts, + # reports healthy and forwards nothing. + directory = "/var/log/journal"; + # ⚠️ `--merge` is a guard: an unqualified + # `journalctl --follow` over a journal with no entry for the + # current boot (a host-linked one) crash-loops the receiver + # every ~2s with "No journal boot entry found for the + # specified boot (+0)". This journal is container-local, so + # the current boot has an entry and the flag changes nothing. + merge = true; + storage = "file_storage"; + }; - # The store's scrape, run by this container's own collector. - # - # ⚠️ `metrics_path` and `params` are STATED, and both have to - # be: openbao serves no `/metrics` at all, and - # `/v1/sys/metrics` answers JSON unless the format is asked - # for. A scrape of the default path 404s and a scrape without - # the query parses as nothing — each reads as a dead exporter - # rather than as a wrong request. The spelling matches - # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for - # every other service's target. - # - # No credential, unlike the swarm collector's - # `publishedScrapeTargets` jobs: the target is the loopback - # listener above, inside this container's own netns, which is - # what lets the scrape stay unauthenticated without the API - # listener's client-cert requirement being touched. - receivers.prometheus.config.scrape_configs = [ - { - job_name = "bao"; - static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; - metrics_path = "/v1/sys/metrics"; - params.format = [ "prometheus" ]; - } - ]; + # The store's scrape, run by this container's own collector. + # + # ⚠️ `metrics_path` and `params` are STATED, and both have to + # be: openbao serves no `/metrics` at all, and + # `/v1/sys/metrics` answers JSON unless the format is asked + # for. A scrape of the default path 404s and a scrape without + # the query parses as nothing — each reads as a dead exporter + # rather than as a wrong request. The spelling matches + # ./swarm-otel.nix's `loopbackScrapeConfig`, the renderer for + # every other service's target. + # + # No credential, unlike the swarm collector's + # `publishedScrapeTargets` jobs: the target is the loopback + # listener above, inside this container's own netns, which is + # what lets the scrape stay unauthenticated without the API + # listener's client-cert requirement being touched. + receivers.prometheus.config.scrape_configs = [ + { + job_name = "bao"; + static_configs = [ { targets = [ metricsScrapeTarget ]; } ]; + metrics_path = "/v1/sys/metrics"; + params.format = [ "prometheus" ]; + } + ]; - # Identity the hop above cannot supply: a host-side reader can - # say which machine a line came from, and only a collector - # inside this container can say it was the secret store's. No - # `hive` or `swarm` label — the swarm tier upserts `hive` from - # the receiver that accepted the record, precisely so it comes - # from something the sender cannot write. - # - # `collectorServiceName`, not `cfg.machine`: what the - # dashboard selects on is a display identity, not the - # container/receiver name — see that binding's own comment. - processors.resource.attributes = [ - { - key = "service.name"; - value = collectorServiceName; - action = "upsert"; - } - ]; + # Identity the hop above cannot supply: a host-side reader can + # say which machine a line came from, and only a collector + # inside this container can say it was the secret store's. No + # `hive` or `swarm` label — the swarm tier upserts `hive` from + # the receiver that accepted the record, precisely so it comes + # from something the sender cannot write. + # + # `collectorServiceName`, not `cfg.machine`: what the + # dashboard selects on is a display identity, not the + # container/receiver name — see that binding's own comment. + processors.resource.attributes = [ + { + key = "service.name"; + value = collectorServiceName; + action = "upsert"; + } + ]; - # `otlphttp` unconditionally, unlike the agent forwarder's - # protocol-derived exporter name: the swarm receiver speaks - # OTLP/HTTP protobuf whatever this tier's *upstream* - # protocol is, which is the same reason - # ./hive-c0re/environment.nix pins the protocol it hands out. - # `endpoint` is a BASE the exporter appends `/v1/logs` to. - exporters.otlphttp = { - endpoint = otelFirstHop; - # The receiver at that address accepts an OIDC token and - # nothing else (./swarm-otel.nix's `oidc/*` authenticators), - # so this is what makes the hop deliver rather than collect - # 401s — unconditional, because a forwarder that exports - # unauthenticated has no working mode to fall back to. - auth.authenticator = forwarderAuthName; - }; + # `otlphttp` unconditionally, unlike the agent forwarder's + # protocol-derived exporter name: the swarm receiver speaks + # OTLP/HTTP protobuf whatever this tier's *upstream* + # protocol is, which is the same reason + # ./hive-c0re/environment.nix pins the protocol it hands out. + # `endpoint` is a BASE the exporter appends `/v1/logs` to. + exporters.otlphttp = { + endpoint = otelFirstHop; + # The receiver at that address accepts an OIDC token and + # nothing else (./swarm-otel.nix's `oidc/*` authenticators), + # so this is what makes the hop deliver rather than collect + # 401s — unconditional, because a forwarder that exports + # unauthenticated has no working mode to fall back to. + auth.authenticator = forwarderAuthName; + }; - # The push side, opposite in direction to the receiver's - # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one - # to send. Hence `oauth2client`, and hence a client id and - # secret rather than an issuer and an audience to check. - extensions.${forwarderAuthName} = { - client_id = cfg.otel.clientId; - # A path, never a value — and one systemd resolves at - # runtime, so neither the secret nor the credentials - # directory is rendered into the nix store. - client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; - token_url = "${toString autheliaCfg.url}/api/oidc/token"; - # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. - # Registering it on the client only makes it permissible; a - # token minted without asking carries `aud: []` and the - # receiver refuses it — with a config that reads perfectly - # at both ends. - endpoint_params.audience = cfg.otel.clientId; - }; + # The push side, opposite in direction to the receiver's + # `oidc/*`: those VALIDATE a token arriving, this OBTAINS one + # to send. Hence `oauth2client`, and hence a client id and + # secret rather than an issuer and an audience to check. + extensions.${forwarderAuthName} = { + client_id = cfg.otel.clientId; + # A path, never a value — and one systemd resolves at + # runtime, so neither the secret nor the credentials + # directory is rendered into the nix store. + client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}"; + token_url = "${toString autheliaCfg.url}/api/oidc/token"; + # ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted. + # Registering it on the client only makes it permissible; a + # token minted without asking carries `aud: []` and the + # receiver refuses it — with a config that reads perfectly + # at both ends. + endpoint_params.audience = cfg.otel.clientId; + }; - service.pipelines.logs = { - receivers = [ "journald" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; - # A SECOND pipeline, not a second receiver on the logs one: a - # pipeline is typed, and a prometheus receiver in `logs` is a - # config the collector rejects at startup rather than one that - # quietly ships metrics as log records. - # - # Same `resource` processor and same `otlphttp` exporter, so - # the store's metrics carry the same `service.name` its logs - # do and leave by the same hop — no otel bypass, one exit from - # this container. The exporter treats `endpoint` as a base and - # appends the signal's own path, so `/v1/metrics` here and - # `/v1/logs` above come from the one address. - service.pipelines.metrics = { - receivers = [ "prometheus" ]; - processors = [ "resource" ]; - exporters = [ "otlphttp" ]; - }; + # A SECOND pipeline, not a second receiver on the logs one: a + # pipeline is typed, and a prometheus receiver in `logs` is a + # config the collector rejects at startup rather than one that + # quietly ships metrics as log records. + # + # Same `resource` processor and same `otlphttp` exporter, so + # the store's metrics carry the same `service.name` its logs + # do and leave by the same hop — no otel bypass, one exit from + # this container. The exporter treats `endpoint` as a base and + # appends the signal's own path, so `/v1/metrics` here and + # `/v1/logs` above come from the one address. + service.pipelines.metrics = { + receivers = [ "prometheus" ]; + processors = [ "resource" ]; + exporters = [ "otlphttp" ]; + }; - # An extension configured but not listed here is INERT — the - # receiver's `storage: file_storage` above would name a - # component the collector never starts, and the exporter's - # authenticator would push unauthenticated while the - # collector reports healthy. - service.extensions = [ - "file_storage" - forwarderAuthName - ]; - }; - }; - - # Off the built-in default of `localhost:8888`, which the hive tier - # holds: this container shares the host's netns. - services.opentelemetry-collector.settings = import ../collector-self-scrape.nix { - inherit lib; - port = cfg.otel.telemetryPort; - exporter = "otlphttp"; - attributes.container = collectorServiceName; + # An extension configured but not listed here is INERT — the + # receiver's `storage: file_storage` above would name a + # component the collector never starts, and the exporter's + # authenticator would push unauthenticated while the + # collector reports healthy. + service.extensions = [ + "file_storage" + forwarderAuthName + ]; + } + # Off the built-in default of `localhost:8888`, which the hive tier + # holds: this container shares the host's netns. + (import ../collector-self-scrape.nix { + inherit lib; + port = cfg.otel.telemetryPort; + exporter = "otlphttp"; + attributes.container = collectorServiceName; + }) + ]; }; # How the secret crosses the sandbox. The host unit