vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-c0re: replace the agents_crashing banner with a crash-watcher log line + grafana alert

Browse source
This commit is contained in:
damocles 2026-10-12 22:21:04 +02:00 • committed by mara
commit 02831fb5e9
13 changed files with 214 additions and 224 deletions
Download patch file Download diff file Expand all files Collapse all files

98
nix/host-modules/otel.nix
View file

@ -399,6 +399,21 @@ in
# prove this config STARTS, never mind that a sample arrives.
validateConfigFile = true;
settings = {
# The journald receiver's cursor store, same shape as the agent
# tier's own (`nix/agent-modules/otel.nix`): a restart resumes
# from the last-read position instead of re-reading from the
# journal's tail (silently dropping whatever landed while the
# collector was down) or its head (re-sending everything).
# `create_directory` matters at BUILD time: `validateConfigFile`
# runs `otelcol validate` in the nix sandbox, where systemd's
# `StateDirectory=` has never run and the path does not exist
# yet — without it the validator refuses with "directory must
# exist".
extensions.file_storage = {
directory = "/var/lib/opentelemetry-collector";
create_directory = true;
};
receivers = {
otlp.protocols.http.endpoint = listen;
@ -452,6 +467,31 @@ in
system = { };
};
};
# This tier's own host journal, scoped to the one unit whose
# crash-watch warning (`"container crash detected"`,
# `crash_watch.rs`) a swarm-level grafana alert
# (`hh-container-crashed` in `swarm-grafana.nix`) depends on
# reaching VictoriaLogs. Unlike the swarm tier's journald
# receiver (`swarm-otel.nix`) this needs no bind mount: this
# collector is a HOST systemd unit, not a container, so
# `/var/log/journal` is the same filesystem hive-c0re itself
# logs into — no id-mapping gap to cross.
#
# `units`, unlike the swarm tier's unfiltered receiver: this
# collector has no `filter/exclude-user-sessions` processor,
# and a hive host can be someone's workstation too, so an
# allowlist is load-bearing here rather than a tidiness
# preference.
journald = {
directory = "/var/log/journal";
storage = "file_storage";
units = [ "hive-c0re.service" ];
# Severity table shared with the agent and swarm tiers'
# journald receivers — see ../journald-severity.nix for why
# it is its own file.
operators = import ../journald-severity.nix;
};
}
# Only when a service has actually declared a target. An enabled
# `prometheus` receiver with an empty `scrape_configs` renders,
@ -499,8 +539,10 @@ in
// lib.optionalAttrs senderAuth { auth.authenticator = authName; };
# ⚠️ An extension not listed here is INERT: the collector starts
# clean and the exporter naming it sends nothing authenticated.
service.extensions = lib.optional senderAuth authName;
# clean and the exporter naming it sends nothing authenticated —
# and, for `file_storage`, the journald receiver's `storage:`
# above would name a component the collector never starts.
service.extensions = [ "file_storage" ] ++ lib.optional senderAuth authName;
# Host identity, so a sample can say WHICH machine it came from.
# Without this the host's metrics carry `hive` and no host
@ -584,21 +626,29 @@ in
exporters = [ swarmName ];
};
# The second signal this tier carries, and the reason it exists
# is that an agent's journal is unreadable from here: the
# host-side per-container journal directory is an id-mapped bind
# mount journald never writes into, so each agent container runs
# its own forwarder (nix/agent-modules/otel.nix) and pushes over
# the `otlp` receiver above. Without this pipeline that receiver
# answers 404 on `/v1/logs` — measured, and indistinguishable
# from a route that was never meant to exist.
# The second signal this tier carries. Two distinct sources, for
# two distinct reasons:
# - `otlp`: an agent's journal is unreadable from here (the
# host-side per-container journal directory is an id-mapped
# bind mount journald never writes into), so each agent
# container runs its own forwarder (nix/agent-modules/otel.nix)
# and pushes over the `otlp` receiver above. Without this
# pipeline that receiver answers 404 on `/v1/logs` — measured,
# and indistinguishable from a route that was never meant to
# exist.
# - `journald`: this collector's own host journal, scoped to
# `hive-c0re.service` above — no id-mapping gap to cross here,
# since this collector IS the host, unlike an agent.
#
# One receiver, two pipelines: `otlp` accepts both signals and
# One `otlp` receiver, two pipelines: it accepts both signals and
# each pipeline takes the one it names. The exporter needs no
# change either — its `endpoint` is a base, and otlphttp appends
# `/v1/logs` to it exactly as it appends `/v1/metrics`.
service.pipelines.logs = {
receivers = [ "otlp" ];
receivers = [
"otlp"
"journald"
];
# `resourcedetection` and NOT `deltatocumulative`: that one is
# a metrics-only processor, and naming it here is a startup
# failure rather than a no-op. Host identity applies to both
@ -863,6 +913,30 @@ in
docs/swarm/secrets.md.
'';
}
{
# Same guard as ./swarm-otel.nix's sibling journald receiver, and
# the same reason: the receiver reads /var/log/journal, which only
# exists when journald stores persistently. With "volatile" the
# journal lives in /run/log/journal instead and with "none" there
# is none at all — either way `nixos-container`'s analogue for a
# host unit is simpler but no less fatal: the collector's
# `journald` receiver fails to open the directory and the unit
# never starts, with nothing here saying why.
assertion =
!(lib.elem config.services.journald.storage [
"volatile"
"none"
]);
message = ''
services.hyperhive.otel.enable is true, which runs a journald
receiver reading /var/log/journal for hive-c0re's own crash-watch
log line — but services.journald.storage is
"${config.services.journald.storage}" on this host.
Set services.journald.storage = "persistent" (the NixOS
default).
'';
}
];
}
);