diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index da0225c9..666490b5 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -30,7 +30,6 @@ ./swarm-otel.nix ./swarm-snapshot-store.nix ./swarm-ui.nix - ./swarm-victorialogs.nix ./swarm-victoriametrics.nix ./swarm-wireguard.nix ./swarm.nix diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index f87d9c40..4c8cc955 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -32,7 +32,6 @@ let swarmCfg = config.services.hyperhive.swarm; otelCfg = config.services.hyperhive.otel; vmCfg = config.services.hyperhive.swarm.victoriametrics; - vlCfg = config.services.hyperhive.swarm.victorialogs; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; swarmDomain = hyperhiveCfg.swarm.domain; @@ -128,41 +127,13 @@ let # both "the label is absent" and "the label says unknown". swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name; - # Where the journal lives on BOTH sides of the bind mount below — one string - # because the receiver reads the path it is mounted at, and two spellings of - # one path is a mount that succeeds and a receiver that finds nothing. - # - # 🔑 Why the host's directory is enough to see every container: nspawn is - # invoked with `--link-journal=try-guest` for every non-ephemeral container, - # so a container's journal FILES live here, under its own machine-id - # subdirectory, and are bind-mounted into the guest rather than the other way - # round. Measured: `journalctl -D` on this parent directory descends into - # those subdirectories, so one reader covers the host and every container. - hostJournalDir = "/var/log/journal"; - - # The operator-configured upstream, named once: the same exporter carries - # every signal, so metrics and logs both reach it without a second - # definition. - upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"; - upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName; - # One list, read by every pipeline: the per-hive pipelines fan out to # exactly the same destinations as the single pipeline they replace. # Written once because "which exporters" is a property of this tier, not # of which hive a sample came from. - exporterNames = upstreamExporters ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; - - # The same fan-out for logs, and the local store is only ONE of its - # destinations. A deployment that turns the swarm's log store off and keeps - # an upstream endpoint still collects — the store is where logs may be kept, - # not the reason to read the journal at all. - logExporterNames = upstreamExporters ++ lib.optional vlCfg.enable "otlphttp/victorialogs"; - - # Collect when there is anywhere to send it, and only then. A pipeline with - # an empty exporter list is not a quiet no-op — the collector rejects it — - # and reading the journal to drop it on the floor would be worse than not - # reading it. - collectLogs = logExporterNames != [ ]; + exporterNames = + lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") + ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; in { options.services.hyperhive.swarm.otel = { @@ -683,30 +654,7 @@ in } # The public hive CA, read-only — only when something in here # actually verifies a swarm-service name. - // caTrust.bindMount - # The host's journal, read-only, and only when something will read it. - # Read-only is the whole security posture of this mount: the collector - # has no business writing to a journal, and one that cannot write - # cannot corrupt the record it is reporting on. - # - # The mount is the only half we supply. Journal files are - # `0640 root:systemd-journal` and the unit runs `DynamicUser`, so - # reading them needs that group — which upstream's own collector unit - # already grants unconditionally. Adding it here again is not - # harmless: systemd list options CONCATENATE, so a second copy renders - # `[ "systemd-journal" "systemd-journal" ]` and quietly becomes a - # second owner of a fact upstream may later change. - # - # It works across the mount because that gid is FIXED at 62 in - # nixpkgs' `ids.nix`. A per-host allocation would leave the host's - # ownership naming a different group inside the container, and the - # failure would be a receiver that starts cleanly and reads nothing. - // lib.optionalAttrs collectLogs { - ${hostJournalDir} = { - hostPath = hostJournalDir; - isReadOnly = true; - }; - }; + // caTrust.bindMount; config = { ... }: @@ -824,24 +772,6 @@ in }; } ) cfg.publishedScrapeTargets; - } - # The whole journal directory, deliberately unfiltered. - # - # A filter restricted to the swarm's own containers was the - # obvious shape and is the wrong one: the host units are where - # the incidents live — the gateway's nginx, the core daemon, - # dnsmasq — and none of them is a swarm container. Filtering to - # `swarm-*` would exclude the single most-needed source. - # - # Nor would a list of container names stay true: it is a second - # place that has to know which services exist, and it goes stale - # silently the next time one is added. Attribution does not need - # it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID` - # are written by journald rather than by the logging process, so - # a reader can tell the origins apart without this collector - # deciding for them. - // lib.optionalAttrs collectLogs { - journald.directory = hostJournalDir; }; exporters = @@ -867,40 +797,6 @@ in headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; } // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; - } - // lib.optionalAttrs vlCfg.enable { - # `logs_endpoint`, NOT `endpoint`, for exactly the reason the - # metrics exporter above spells out — and the trap is worse - # here, because the two stores' OTLP routes differ. `endpoint` - # is a base the exporter appends `/v1/logs` to; VictoriaLogs - # serves `/insert/opentelemetry/v1/logs`, which is also not - # the metrics store's `/opentelemetry/api/v1/push`. Both - # spellings pass `otelcol validate`. - # - # ⚠️ And the store cannot tell you either: a right path, a - # wrong path and a nonsense path all answer 400. Only its log - # distinguishes them — the real route complains about the - # encoding, everything else says "unsupported path requested". - # 🔑 THE TWO QUERY PARAMETERS ARE NOT TUNING — without the - # first one this pipeline fills the store with records that - # cannot be searched by message text, which is the only - # reason to collect logs at all. - # - # The journald receiver leaves the OTLP *body* empty and - # carries the entry as a map of journal fields, so the store - # has no message to index and writes the literal placeholder - # `missing _msg field` into `_msg` on EVERY record. Nothing - # errors: ingest returns 200, the data is all present, and a - # plain search for a line that is sitting right there returns - # nothing. `_msg_field` tells the store which field carries - # the message. Measured, both with and without. - # - # `_stream_fields` is the difference between one enormous - # stream for the whole host and one per unit per machine — - # both low-cardinality, and both fields journald sets itself. - "otlphttp/victorialogs".logs_endpoint = - "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs" - + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; }; # Moves this collector's self-metrics off the built-in @@ -954,22 +850,6 @@ in processors = [ "resource/${swarmTierName}" ]; exporters = exporterNames; }; - } - # Logs fan out exactly as metrics do: the swarm's store when it - # runs, the operator's upstream when one is configured, both - # when both. The local store is a destination rather than the - # reason to collect, so turning it off leaves a hive that still - # ships its journal to whoever aggregates across swarms. - # - # Same stamp as the scraped swarm services, for the same reason: - # a host's logs belong to the swarm, and there is no honest - # `hive` value to put on them. - // lib.optionalAttrs collectLogs { - "logs/${swarmTierName}" = { - receivers = [ "journald" ]; - processors = [ "resource/${swarmTierName}" ]; - exporters = logExporterNames; - }; }; } // { @@ -1054,13 +934,7 @@ in # any one hive, so there is no honest value to put there — and # an invented one (a sentinel, the local hive's name) would be # queried as though it meant something. - # - # ⚠️ Emitted for the LOG pipeline too, not just the scraped one: - # a processor a pipeline names but the config does not define is - # a collector that refuses to start, and enabling the log store - # without declaring a scrape target is a perfectly ordinary - # config. - // lib.optionalAttrs (cfg.scrapeTargets != { } || collectLogs) { + // lib.optionalAttrs (cfg.scrapeTargets != { }) { "resource/${swarmTierName}".attributes = [ { # The metric LABEL, a different namespace from the diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index a40b7e56..80c3800a 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -71,13 +71,6 @@ in victoriametrics.enable = lib.mkDefault swarmCfg.enableRequiredServices; grafana.enable = lib.mkDefault swarmCfg.enableRequiredServices; - # The log store, deriving from the same switch for the same reason — - # and deliberately in the same commit as the collector pipeline that - # writes to it, never before it. A store nothing writes to is worse - # than no store: it starts, answers queries, and returns nothing, so - # the first person to look concludes there were no logs. - victorialogs.enable = lib.mkDefault swarmCfg.enableRequiredServices; - # The collector that feeds the pair above, and the only tier holding # the upstream credential. Same rule as the rest: once per swarm, # optional, and a hive that is not the service host is a *client* of diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix deleted file mode 100644 index 3e1e4781..00000000 --- a/nix/host-modules/swarm-victorialogs.nix +++ /dev/null @@ -1,168 +0,0 @@ -# The swarm's log store: one VictoriaLogs for the whole swarm, in a -# `swarm-victorialogs` nixos-container beside the metrics store it mirrors. -# -# Why a store at all rather than reading journals directly: an agent can -# verify that a unit was *launched* and never that it is *working*. The -# container journals are not reachable from an agent, so a diagnosis stops at -# the first component that is broken — which is precisely the component whose -# own instrument is least likely to be legible. Collecting logs centrally -# makes the question answerable without host access. -# -# It is the collector that feeds this, not the services directly: one ingest -# point per swarm, same shape as the metrics path. -# -# ⚠️ NO GATEWAY VHOST, and that omission is deliberate rather than unfinished. -# VictoriaLogs' ingest and query endpoints are unauthenticated, exactly like -# the metrics store's — and the metrics store *is* published under a -# resolvable name, which is an open security question rather than a settled -# design. Publishing this one the same way would repeat that before the first -# instance is decided. The reader is Grafana, which is on this host. -{ - pkgs, - lib, - config, - ... -}: -let - cfg = config.services.hyperhive.swarm.victorialogs; - networkCfg = config.services.hyperhive.network; - hyperhiveCfg = config.services.hyperhive; -in -{ - options.services.hyperhive.swarm.victorialogs = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Run the swarm's log store on this host. Off by default and not - derived from {option}`services.hyperhive.enable`: a swarm has one - log store, so enabling it is a decision about swarm topology - rather than about whether hyperhive is installed. - ''; - }; - - package = lib.mkOption { - type = lib.types.package; - default = pkgs.victorialogs; - defaultText = lib.literalExpression "pkgs.victorialogs"; - description = "VictoriaLogs package to run."; - }; - - machine = lib.mkOption { - type = lib.types.str; - readOnly = true; - default = "swarm-victorialogs"; - description = '' - Container name. Read-only: the name appears in host paths and in - `machinectl`, so it is a fact other modules may read rather than a - knob. - ''; - }; - - port = lib.mkOption { - type = lib.types.port; - default = 9428; - description = '' - Loopback port the store listens on. Upstream's default, kept - because there is no reason to move it and a familiar number is - one less thing an operator has to look up. - - ⚠️ Every swarm container shares the host's network namespace, so - this is a swarm-wide claim rather than a per-container one — two - modules picking the same number collide at runtime with no bind - error and nothing in any log. `state/eval-port-collisions.sh` - checks the class. - ''; - }; - - retentionPeriod = lib.mkOption { - type = lib.types.str; - default = "30d"; - example = "90d"; - description = '' - How long log data is kept. - - Deliberately far shorter than the metrics store's retention: logs - are orders of magnitude larger per unit of time, and their value - decays much faster. A log line answers "what happened during that - incident"; a metric answers "is this worse than last quarter". - ''; - }; - }; - - config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { - # This store publishes its own health as prometheus metrics on the same - # listener it serves queries on, so the swarm's collector scrapes it with - # no exporter and no extra port — same arrangement as the metrics store. - # - # Declared here rather than in the collector's module because that is the - # rule the option carries: an entry exists only where the service that - # named it runs. ⚠️ That constrains the TARGET, not the scraper — a - # deployment that splits this container away from the collector's host - # silently drops the entry, and no assertion can see it, because separate - # hosts are separate evaluations. - services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}"; - - containers.${cfg.machine} = { - autoStart = true; - ephemeral = false; - # Shared host netns, like every sibling swarm container: the collector - # and Grafana reach this at 127.0.0.1:. - privateNetwork = false; - - config = - { ... }: - { - imports = [ - (import ./swarm-container-resolver.nix { - inherit (networkCfg) bridgeIp; - dnsConsumers = [ "victorialogs.service" ]; - }) - ]; - - system.stateVersion = "26.05"; - - # This container shares the host netns, so its own firewall.service - # would rewrite the HOST ruleset at every boot. The host firewall - # owns all filtering. - networking.firewall.enable = false; - # resolvconf stays off because the resolver unit imported above - # owns /etc/resolv.conf. Leaving it on would let host-tracking - # regenerate the file empty, since the host's copy doesn't cross - # the boundary after start. - networking.resolvconf.enable = lib.mkForce false; - - services.victorialogs = { - enable = true; - package = cfg.package; - - # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is, - # and it matters more here: upstream's default listens on every - # interface, and this endpoint accepts writes as well as reads - # with no authentication of its own. The bind address is the - # boundary. - listenAddress = "127.0.0.1:${toString cfg.port}"; - - extraOptions = [ "-retentionPeriod=${cfg.retentionPeriod}" ]; - }; - }; - }; - }; - - # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured - # against the pinned build rather than read from docs. The path differs - # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter - # configured by analogy with the metrics one is silently wrong. - # - # ⚠️ And the status code cannot tell you which is which: a wrong path, a - # right path with a wrong body, and a nonsense path ALL answer 400. Only - # the server log distinguishes them — the real route complains about the - # encoding ("json encoding isn't supported ... use protobuf"), while - # anything else logs "unsupported path requested". Verified with a - # nonsense-path control, because a probe where every arm returns the same - # code is not evidence. - # - # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is - # why `listenAddress` above is loopback, why the collector is the only - # writer, and why this module declares no gateway vhost. -}