vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

container collectors scrape their own telemetry and push it up their hop

Browse source
This commit is contained in:
atlas 2026-10-28 22:01:33 +01:00 • committed by mara
commit 73c966a7eb
5 changed files with 163 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

15
nix/module-eval/agent-otel.nix
View file

@ -21,6 +21,7 @@ let
agent
carriesJournaldSeverity
runGroup
selfScrapes
;
# The log path's three hops, one fixture each. Nothing carries a journal
@ -55,6 +56,20 @@ let
agentSettings = machine: machine.services.opentelemetry-collector.settings;
cases = [
{
# The collector's own counters are the only signal that it is dropping
# telemetry. 8904 keeps agents off 8888, which the collector dashboard
# reads as the hive collector; the exporter follows the wire protocol.
name = "an agent forwarder scrapes its own telemetry endpoint on loopback, off the hive collector's port";
ok =
selfScrapes (agentSettings agentOtel) 8904 { agent = agentOtel.services.hyperhive.agent.user.name; }
&& selfScrapes (agentSettings agentOtelGrpc) 8904 {
agent = agentOtelGrpc.services.hyperhive.agent.user.name;
}
# Control: with the switch off the lookup finds no collector config.
&& !(agentNoOtel.services.opentelemetry-collector.settings ? receivers)
&& (agentSettings agentOtelGrpc).service.pipelines."metrics/self".exporters == [ "otlp" ];
}
{
# The journald receiver's own default directory is the RUNTIME
# journal, and a container that stores persistently leaves that

15
nix/module-eval/bao-otel-collector.nix
View file

@ -22,6 +22,7 @@ let
runGroup
baoSettings
forceServiceConfigs
selfScrapes
;
baoTwoAddresses = hive {
@ -185,6 +186,20 @@ let
&& m.exporters == [ "otlphttp" ]
&& m.exporters == s.service.pipelines.logs.exporters;
}
{
# The store's collector reports its own health through a pipeline of its
# own: the store pipeline above upserts `service.name`, and sharing it
# would relabel these series away from the collector dashboard.
name = "the store's collector scrapes its own telemetry endpoint on loopback";
ok =
let
s = (baoForwarder baoWithCollector).settings;
port = baoWithCollector.services.hyperhive.swarm.bao.otel.telemetryPort;
in
selfScrapes s port { container = "bao"; }
&& s.service.pipelines.metrics.receivers == [ "prometheus" ]
&& s.service.pipelines.metrics.processors == [ "resource" ];
}
{
# Presence control for the case above: both fields are omitted rather
# than defaulted, so a target declared as bare `host:port` renders what

6
nix/module-eval/forge-placement.nix
View file

@ -25,6 +25,7 @@ let
})
hive
runGroup
selfScrapes
;
inherit (shared) bare;
@ -558,11 +559,10 @@ let
ok =
let
c = ciAlone.containers.swarm-ci-runner.config;
reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers;
s = c.services.opentelemetry-collector.settings;
in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == 8895;
&& selfScrapes s 8895 { container = "ci-runner"; };
}
];
in

32
nix/module-eval/lib.nix
View file

@ -208,6 +208,37 @@ let
# the two tier cases differ only in which receiver they hand it.
carriesJournaldSeverity = receiver: lib.elem journaldSeverityParser (receiver.operators or [ ]);
# Whether a container collector's settings carry the self-scrape of
# ../collector-self-scrape.nix: the telemetry endpoint on loopback at `port`,
# a job scraping exactly that address, and a metrics pipeline that reads the
# job and leaves through an exporter the settings define. `attrs` are the
# resource attributes the pipeline stamps.
selfScrapes =
s: port: attrs:
let
job = lib.findFirst (j: j.job_name == "collector") null (
s.receivers."prometheus/self".config.scrape_configs or [ ]
);
pipeline = s.service.pipelines."metrics/self" or null;
reader = lib.head (s.service.telemetry.metrics.readers or [ { } ]);
stamped = lib.mapAttrsToList (key: value: {
inherit key value;
action = "upsert";
}) attrs;
in
job != null
&& job.static_configs == [ { targets = [ "127.0.0.1:${toString port}" ]; } ]
&&
(reader.pull.exporter.prometheus or null) == {
host = "127.0.0.1";
inherit port;
}
&& pipeline != null
&& pipeline.receivers == [ "prometheus/self" ]
&& (s.processors."resource/self".attributes or null) == stamped
&& pipeline.exporters != [ ]
&& lib.all (e: s.exporters ? ${e}) pipeline.exporters;
# The enables that switch owns. The hive tier's collector is not among them: it
# follows `deploy.hive-controller.enable`, and a services host without a hive
# runs none.
@ -283,6 +314,7 @@ in
inherit agentHarness;
inherit journaldSeverityParser;
inherit carriesJournaldSeverity;
inherit selfScrapes;
inherit swarmServiceEnables;
inherit runGroup;
inherit forceServiceConfigs;

16
nix/module-eval/nats-authelia.nix
View file

@ -20,6 +20,7 @@ let
})
hive
runGroup
selfScrapes
;
# No store identity, which the first case below relies on.
@ -157,12 +158,10 @@ let
h.services.hyperhive.swarm.nats.otel.telemetryPort
]
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) forwarderServices);
reader = lib.head s.service.telemetry.metrics.readers;
in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
&& !(lib.elem svc.port taken)
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == svc.port;
&& selfScrapes s svc.port { container = svc.name; };
}
]
) forwarderServices;
@ -269,12 +268,10 @@ let
h.services.hyperhive.swarm.nats.otel.telemetryPort
]
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
reader = lib.head s.service.telemetry.metrics.readers;
in
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
&& !(lib.elem svc.port taken)
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
&& reader.pull.exporter.prometheus.port == svc.port;
&& selfScrapes s svc.port { container = svc.name; };
}
]
) hostNetnsForwarders;
@ -740,6 +737,7 @@ let
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
&& lib.elem "oauth2client/swarm" s.service.extensions
&& s.service.pipelines.logs.receivers == [ "journald" ]
&& selfScrapes s h.services.hyperhive.swarm.nats.otel.telemetryPort { container = "nats"; }
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
@ -997,7 +995,7 @@ let
registered = lib.findFirst (
x: x.id == clientId
) null h.services.hyperhive.swarm.authelia.oidc.clients;
reader = lib.head s.service.telemetry.metrics.readers;
port = c.services.hyperhive.swarmJournalForwarder.telemetryPort;
taken = [
8888
8890
@ -1022,8 +1020,8 @@ let
&& registered.audience == [ clientId ]
&& !(lib.elem "--link-journal=host" container.extraFlags)
&& lib.any (lib.hasPrefix "--load-credential=publisher-bao-cert:") container.extraFlags
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
&& !(lib.elem reader.pull.exporter.prometheus.port taken)
&& selfScrapes s port { container = "authelia"; }
&& !(lib.elem port taken)
&& !autheliaColocated.containers.swarm-authelia.config.services.opentelemetry-collector.enable
&& !(
autheliaColocated.containers.swarm-authelia.config.systemd.services ? swarm-journal-forwarder-oidc