vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Compare commits

base: vinzenz:012ab5bb37093b28d93059c1f3f3d7786c1520b9
Branches Tags
vinzenz:main
..
compare: vinzenz:79f4132b4f7d306951f34d72534c5332bb49c75c
Branches Tags
vinzenz:main
4 changed files with 5 additions and 307 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nix/host-modules/default.nix
View file

@ -30,7 +30,6 @@
./swarm-otel.nix
./swarm-snapshot-store.nix
./swarm-ui.nix
./swarm-victorialogs.nix
./swarm-victoriametrics.nix
./swarm-wireguard.nix
./swarm.nix

136
nix/host-modules/swarm-otel.nix
View file

@ -32,7 +32,6 @@ let
swarmCfg = config.services.hyperhive.swarm;
otelCfg = config.services.hyperhive.otel;
vmCfg = config.services.hyperhive.swarm.victoriametrics;
vlCfg = config.services.hyperhive.swarm.victorialogs;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
swarmDomain = hyperhiveCfg.swarm.domain;
@ -128,41 +127,13 @@ let
# both "the label is absent" and "the label says unknown".
swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name;
# Where the journal lives on BOTH sides of the bind mount below — one string
# because the receiver reads the path it is mounted at, and two spellings of
# one path is a mount that succeeds and a receiver that finds nothing.
#
# 🔑 Why the host's directory is enough to see every container: nspawn is
# invoked with `--link-journal=try-guest` for every non-ephemeral container,
# so a container's journal FILES live here, under its own machine-id
# subdirectory, and are bind-mounted into the guest rather than the other way
# round. Measured: `journalctl -D` on this parent directory descends into
# those subdirectories, so one reader covers the host and every container.
hostJournalDir = "/var/log/journal";
# The operator-configured upstream, named once: the same exporter carries
# every signal, so metrics and logs both reach it without a second
# definition.
upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp";
upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName;
# One list, read by every pipeline: the per-hive pipelines fan out to
# exactly the same destinations as the single pipeline they replace.
# Written once because "which exporters" is a property of this tier, not
# of which hive a sample came from.
exporterNames = upstreamExporters ++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
# The same fan-out for logs, and the local store is only ONE of its
# destinations. A deployment that turns the swarm's log store off and keeps
# an upstream endpoint still collects — the store is where logs may be kept,
# not the reason to read the journal at all.
logExporterNames = upstreamExporters ++ lib.optional vlCfg.enable "otlphttp/victorialogs";
# Collect when there is anywhere to send it, and only then. A pipeline with
# an empty exporter list is not a quiet no-op — the collector rejects it —
# and reading the journal to drop it on the floor would be worse than not
# reading it.
collectLogs = logExporterNames != [ ];
exporterNames =
lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp")
++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
in
{
options.services.hyperhive.swarm.otel = {
@ -683,30 +654,7 @@ in
}
# The public hive CA, read-only — only when something in here
# actually verifies a swarm-service name.
// caTrust.bindMount
# The host's journal, read-only, and only when something will read it.
# Read-only is the whole security posture of this mount: the collector
# has no business writing to a journal, and one that cannot write
# cannot corrupt the record it is reporting on.
#
# The mount is the only half we supply. Journal files are
# `0640 root:systemd-journal` and the unit runs `DynamicUser`, so
# reading them needs that group — which upstream's own collector unit
# already grants unconditionally. Adding it here again is not
# harmless: systemd list options CONCATENATE, so a second copy renders
# `[ "systemd-journal" "systemd-journal" ]` and quietly becomes a
# second owner of a fact upstream may later change.
#
# It works across the mount because that gid is FIXED at 62 in
# nixpkgs' `ids.nix`. A per-host allocation would leave the host's
# ownership naming a different group inside the container, and the
# failure would be a receiver that starts cleanly and reads nothing.
// lib.optionalAttrs collectLogs {
${hostJournalDir} = {
hostPath = hostJournalDir;
isReadOnly = true;
};
};
// caTrust.bindMount;
config =
{ ... }:
@ -824,24 +772,6 @@ in
};
}
) cfg.publishedScrapeTargets;
}
# The whole journal directory, deliberately unfiltered.
#
# A filter restricted to the swarm's own containers was the
# obvious shape and is the wrong one: the host units are where
# the incidents live — the gateway's nginx, the core daemon,
# dnsmasq — and none of them is a swarm container. Filtering to
# `swarm-*` would exclude the single most-needed source.
#
# Nor would a list of container names stay true: it is a second
# place that has to know which services exist, and it goes stale
# silently the next time one is added. Attribution does not need
# it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID`
# are written by journald rather than by the logging process, so
# a reader can tell the origins apart without this collector
# deciding for them.
// lib.optionalAttrs collectLogs {
journald.directory = hostJournalDir;
};
exporters =
@ -867,40 +797,6 @@ in
headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}";
}
// lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
}
// lib.optionalAttrs vlCfg.enable {
# `logs_endpoint`, NOT `endpoint`, for exactly the reason the
# metrics exporter above spells out — and the trap is worse
# here, because the two stores' OTLP routes differ. `endpoint`
# is a base the exporter appends `/v1/logs` to; VictoriaLogs
# serves `/insert/opentelemetry/v1/logs`, which is also not
# the metrics store's `/opentelemetry/api/v1/push`. Both
# spellings pass `otelcol validate`.
#
# ⚠️ And the store cannot tell you either: a right path, a
# wrong path and a nonsense path all answer 400. Only its log
# distinguishes them — the real route complains about the
# encoding, everything else says "unsupported path requested".
# 🔑 THE TWO QUERY PARAMETERS ARE NOT TUNING — without the
# first one this pipeline fills the store with records that
# cannot be searched by message text, which is the only
# reason to collect logs at all.
#
# The journald receiver leaves the OTLP *body* empty and
# carries the entry as a map of journal fields, so the store
# has no message to index and writes the literal placeholder
# `missing _msg field` into `_msg` on EVERY record. Nothing
# errors: ingest returns 200, the data is all present, and a
# plain search for a line that is sitting right there returns
# nothing. `_msg_field` tells the store which field carries
# the message. Measured, both with and without.
#
# `_stream_fields` is the difference between one enormous
# stream for the whole host and one per unit per machine —
# both low-cardinality, and both fields journald sets itself.
"otlphttp/victorialogs".logs_endpoint =
"http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs"
+ "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT";
};
# Moves this collector's self-metrics off the built-in
@ -954,22 +850,6 @@ in
processors = [ "resource/${swarmTierName}" ];
exporters = exporterNames;
};
}
# Logs fan out exactly as metrics do: the swarm's store when it
# runs, the operator's upstream when one is configured, both
# when both. The local store is a destination rather than the
# reason to collect, so turning it off leaves a hive that still
# ships its journal to whoever aggregates across swarms.
#
# Same stamp as the scraped swarm services, for the same reason:
# a host's logs belong to the swarm, and there is no honest
# `hive` value to put on them.
// lib.optionalAttrs collectLogs {
"logs/${swarmTierName}" = {
receivers = [ "journald" ];
processors = [ "resource/${swarmTierName}" ];
exporters = logExporterNames;
};
};
}
// {
@ -1054,13 +934,7 @@ in
# any one hive, so there is no honest value to put there — and
# an invented one (a sentinel, the local hive's name) would be
# queried as though it meant something.
#
# ⚠️ Emitted for the LOG pipeline too, not just the scraped one:
# a processor a pipeline names but the config does not define is
# a collector that refuses to start, and enabling the log store
# without declaring a scrape target is a perfectly ordinary
# config.
// lib.optionalAttrs (cfg.scrapeTargets != { } || collectLogs) {
// lib.optionalAttrs (cfg.scrapeTargets != { }) {
"resource/${swarmTierName}".attributes = [
{
# The metric LABEL, a different namespace from the

7
nix/host-modules/swarm-required-services.nix
View file

@ -71,13 +71,6 @@ in
victoriametrics.enable = lib.mkDefault swarmCfg.enableRequiredServices;
grafana.enable = lib.mkDefault swarmCfg.enableRequiredServices;
# The log store, deriving from the same switch for the same reason —
# and deliberately in the same commit as the collector pipeline that
# writes to it, never before it. A store nothing writes to is worse
# than no store: it starts, answers queries, and returns nothing, so
# the first person to look concludes there were no logs.
victorialogs.enable = lib.mkDefault swarmCfg.enableRequiredServices;
# The collector that feeds the pair above, and the only tier holding
# the upstream credential. Same rule as the rest: once per swarm,
# optional, and a hive that is not the service host is a *client* of

168
nix/host-modules/swarm-victorialogs.nix
View file

@ -1,168 +0,0 @@
# The swarm's log store: one VictoriaLogs for the whole swarm, in a
# `swarm-victorialogs` nixos-container beside the metrics store it mirrors.
#
# Why a store at all rather than reading journals directly: an agent can
# verify that a unit was *launched* and never that it is *working*. The
# container journals are not reachable from an agent, so a diagnosis stops at
# the first component that is broken — which is precisely the component whose
# own instrument is least likely to be legible. Collecting logs centrally
# makes the question answerable without host access.
#
# It is the collector that feeds this, not the services directly: one ingest
# point per swarm, same shape as the metrics path.
#
# ⚠️ NO GATEWAY VHOST, and that omission is deliberate rather than unfinished.
# VictoriaLogs' ingest and query endpoints are unauthenticated, exactly like
# the metrics store's — and the metrics store *is* published under a
# resolvable name, which is an open security question rather than a settled
# design. Publishing this one the same way would repeat that before the first
# instance is decided. The reader is Grafana, which is on this host.
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.victorialogs;
networkCfg = config.services.hyperhive.network;
hyperhiveCfg = config.services.hyperhive;
in
{
options.services.hyperhive.swarm.victorialogs = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Run the swarm's log store on this host. Off by default and not
derived from {option}`services.hyperhive.enable`: a swarm has one
log store, so enabling it is a decision about swarm topology
rather than about whether hyperhive is installed.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.victorialogs;
defaultText = lib.literalExpression "pkgs.victorialogs";
description = "VictoriaLogs package to run.";
};
machine = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-victorialogs";
description = ''
Container name. Read-only: the name appears in host paths and in
`machinectl`, so it is a fact other modules may read rather than a
knob.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 9428;
description = ''
Loopback port the store listens on. Upstream's default, kept
because there is no reason to move it and a familiar number is
one less thing an operator has to look up.
⚠️ Every swarm container shares the host's network namespace, so
this is a swarm-wide claim rather than a per-container one — two
modules picking the same number collide at runtime with no bind
error and nothing in any log. `state/eval-port-collisions.sh`
checks the class.
'';
};
retentionPeriod = lib.mkOption {
type = lib.types.str;
default = "30d";
example = "90d";
description = ''
How long log data is kept.
Deliberately far shorter than the metrics store's retention: logs
are orders of magnitude larger per unit of time, and their value
decays much faster. A log line answers "what happened during that
incident"; a metric answers "is this worse than last quarter".
'';
};
};
config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) {
# This store publishes its own health as prometheus metrics on the same
# listener it serves queries on, so the swarm's collector scrapes it with
# no exporter and no extra port — same arrangement as the metrics store.
#
# Declared here rather than in the collector's module because that is the
# rule the option carries: an entry exists only where the service that
# named it runs. ⚠️ That constrains the TARGET, not the scraper — a
# deployment that splits this container away from the collector's host
# silently drops the entry, and no assertion can see it, because separate
# hosts are separate evaluations.
services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}";
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Shared host netns, like every sibling swarm container: the collector
# and Grafana reach this at 127.0.0.1:<port>.
privateNetwork = false;
config =
{ ... }:
{
imports = [
(import ./swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "victorialogs.service" ];
})
];
system.stateVersion = "26.05";
# This container shares the host netns, so its own firewall.service
# would rewrite the HOST ruleset at every boot. The host firewall
# owns all filtering.
networking.firewall.enable = false;
# resolvconf stays off because the resolver unit imported above
# owns /etc/resolv.conf. Leaving it on would let host-tracking
# regenerate the file empty, since the host's copy doesn't cross
# the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
services.victorialogs = {
enable = true;
package = cfg.package;
# ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is,
# and it matters more here: upstream's default listens on every
# interface, and this endpoint accepts writes as well as reads
# with no authentication of its own. The bind address is the
# boundary.
listenAddress = "127.0.0.1:${toString cfg.port}";
extraOptions = [ "-retentionPeriod=${cfg.retentionPeriod}" ];
};
};
};
};
# 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured
# against the pinned build rather than read from docs. The path differs
# from the metrics store's `/opentelemetry/api/v1/push`, so an exporter
# configured by analogy with the metrics one is silently wrong.
#
# ⚠️ And the status code cannot tell you which is which: a wrong path, a
# right path with a wrong body, and a nonsense path ALL answer 400. Only
# the server log distinguishes them — the real route complains about the
# encoding ("json encoding isn't supported ... use protobuf"), while
# anything else logs "unsupported path requested". Verified with a
# nonsense-path control, because a probe where every arm returns the same
# code is not evidence.
#
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
# why `listenAddress` above is loopback, why the collector is the only
# writer, and why this module declares no gateway vhost.
}