vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity
hyperhive/nix/module-eval/hive-otel.nix
atlas d30cb7d122 keep authelia's oidc client secrets in the store, not on disk 
The mint creates each client's secret in the swarm secret store and writes only the digest to disk. It mints only when the store answers and holds none. A new unit moves the plaintext files an earlier release left into the store and removes each once the store holds the same value. The bridge reads its secret from the store into /run before each start. The secret publisher unit and its path unit are gone.
2026-10-30 16:57:58 +01:00

770 lines
37 KiB
Nix
Raw Blame History

# `checks.module-eval-hive-otel` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
pkgs,
lib,
self,
nixosSystem,
shared,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
hiveWithController
runGroup
otelSettings
;
# This hive's own collector, which is a HOST service — unlike the swarm
# tier's, which lives in a container and is read through `otelSettings`.
# The IdP is here too, with its secret file named at the path the fetch
# unit used to write: the collector's wait for that file exists beside
# authelia's container.
hiveOtel = hiveWithController {
otel.clientSecretFile = "/var/lib/hive-otel-oidc/hive-h1.secret";
deploy.authelia.enable = true;
};
# A hive holding its own store leaf, placed by hand, and naming no secret
# file: the collector's secret comes from the store under that leaf.
hiveOtelLeaf = hiveWithController {
gateway.enable = false;
deploy.bao.clientCertFile = "/etc/pki/hive.pem";
deploy.bao.clientKeyFile = "/etc/pki/hive-key.pem";
};
# The IdP, the store and the collector on one host: the publisher that
# writes the secret and the unit that reads it, in one config.
hiveOtelAllLocal = hiveWithController {
deploy.bao.enable = true;
deploy.authelia.enable = true;
# The seed `allowedFile` starts as; the canary is listed too, to show it
# is not written twice.
deploy.hive-controller.squid.allowedDomains = [
"github.com"
"cache.nixos.org"
];
};
# Neither a leaf nor a secret file: the one shape the collector has no
# credential for.
hiveOtelBare = hiveWithController { };
hiveOtelFetch = machine: machine.systemd.services.swarm-bao-hive-otel-oidc;
hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;
hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings;
# The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for
# the swarm tier — the option is one string, what prometheus is handed is a
# job — but this collector is a host service, so the path to it differs.
hiveScrapeJob =
job:
lib.findFirst (
c: c.job_name == job
) null hiveOtelSettings.receivers.prometheus.config.scrape_configs;
# The store with and without a collector on the same host. `scrapeTargets`
# is only ever read by a local collector, so the metrics endpoint is a
# function of the pairing rather than of the store.
inherit (shared) baoWithCollector;
# The store standing beside a HIVE collector: the topology the port case
# below needs, two collectors in one netns with nothing but their
# self-telemetry ports keeping them apart.
inherit (shared) baoWithHiveOtel;
# The collector INSIDE the store's container — neither the host's nor the
# swarm tier's.
baoForwarderSettings =
machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
cases = [
{
# The tier in the middle. Its OTLP receiver takes both signals on one
# port, so without this pipeline an agent's push is answered 404 on
# `/v1/logs` — and a forwarder retrying into a 404 is indistinguish-
# able from one with nothing to send. `journald` joined the receiver
# list for this collector's own host journal (hive-c0re's crash-watch
# log line, see ./otel.nix) — both have to be present, in either
# order, so this stays a set check rather than a list-equality one.
# Compared against the metrics pipeline's exporters rather than a
# name, so the two signals cannot drift to different destinations.
name = "the hive collector forwards logs upstream, not only metrics";
ok =
(hiveOtelPipelines ? logs)
&& hiveOtelPipelines.metrics.exporters != [ ]
&&
lib.sort builtins.lessThan hiveOtelPipelines.logs.receivers == [
"journald"
"otlp"
]
&& hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters;
}
{
# `deltatocumulative` is metrics-only: naming it in a logs pipeline
# kills the collector at startup rather than doing nothing. The second
# clause is the control — the metrics pipeline still names it, so a
# pass means the two processor lists differ rather than that the
# processor left the module.
name = "the hive collector keeps the metrics-only processor out of its logs pipeline";
ok =
!(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors)
&& builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors;
}
{
# `otelcol validate` refuses a `service.extensions` entry with no
# definition, and that runs when the unit is built, not in this suite.
# `hiveOtel` authenticates upstream, so both extensions are listed;
# the length clause keeps the check from passing on an empty list.
name = "the hive collector defines every extension its service lists";
ok =
lib.length hiveOtelSettings.service.extensions == 2
&& lib.all (e: hiveOtelSettings.extensions ? ${e}) hiveOtelSettings.service.extensions;
}
{
# The counters that say telemetry is being LOST — refused, failed,
# queue depth — are served on loopback and reach no store unless
# something reads them. Read off the rendered job rather than the
# option: only the job is what prometheus actually requests.
name = "the hive collector scrapes its own telemetry endpoint";
ok =
let
j = hiveScrapeJob "collector";
in
j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ];
}
{
# A `prometheus` receiver no pipeline names collects nothing while
# rendering and starting perfectly, so the scrape above is inert
# without this. The path is newly reachable: until the hive tier had a
# target of its own, this receiver was never emitted on any hive.
name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
}
{
# The receiver, the pipeline entry and the nginx server it reads have to
# agree on one address, and none of them may exist without a gateway on
# the host. `hiveOtelLeaf` is the control: it opts out of the gateway, and the
# clause asserting that keeps the control from going vacuous.
name = "the hive collector reads the gateway's stub_status, and only where a gateway runs";
ok =
let
nginxConf = hiveOtel.services.nginx.appendHttpConfig;
leafSettings = hiveOtelLeaf.services.opentelemetry-collector.settings;
in
hiveOtel.services.hyperhive.gateway.enable
&& hiveOtelSettings.receivers.nginx.endpoint == "http://127.0.0.1:8085/stub_status"
&& builtins.elem "nginx" hiveOtelPipelines.metrics.receivers
&& lib.hasInfix "listen 127.0.0.1:8085;" nginxConf
&& lib.hasInfix "location = /stub_status {" nginxConf
&& !hiveOtelLeaf.services.hyperhive.gateway.enable
&& !(leafSettings.receivers ? nginx)
&& !(builtins.elem "nginx" leafSettings.service.pipelines.metrics.receivers);
}
{
# `metrics.address` is the spelling that looks right and is rejected by
# this collector version. The first clause is the control: without it a
# missing telemetry block would pass the port check vacuously.
name = "the hive collector binds its telemetry port through readers, not address";
ok =
let
m = hiveOtelSettings.service.telemetry.metrics;
in
!(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
}
{
# A bind that loses the race with the outgoing process is the whole
# failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
# without this the unit spends its five default attempts inside two
# seconds and lands in `start-limit-hit`, where it stops retrying. The
# third clause is the one that has to hold — `StartLimit*` are `[Unit]`
# settings that systemd ignores under `[Service]`, so a bound written
# into `serviceConfig` renders, deploys and does nothing. Asserted
# where nixpkgs puts it rather than where it was written, same as
# ./bao-grants.nix.
name = "the hive collector backs off a failed bind from [Unit], not [Service]";
ok =
let
u = hiveOtel.systemd.services.opentelemetry-collector;
in
u.serviceConfig.RestartSec or 0 > 0
&& toString u.unitConfig.StartLimitBurst == "12"
&& !(u.serviceConfig ? StartLimitBurst)
# The window has to outlast every attempt, or the burst is unreachable.
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
}
{
# Collectors share a network namespace whenever they are co-located,
# and this port appears in no config the port-collision assertion can
# read — so equal defaults mean the second to start dies at `bind()`.
# Pinned as a case rather than an assertion: enforcing it belongs with
# the other port checks, not here.
#
# THREE tiers, because the store's container grew a collector of its
# own and this case could not see it: an UNDECLARED port is not an
# absent one, it is the binary's 8888, which is the hive tier's. That
# is why `portOf` falls back to 8888 rather than throwing — a tier that
# declares nothing has to fail this case loudly, not evaluate away.
name = "no two collector tiers claim the same self-telemetry port";
ok =
let
portOf =
s:
(lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
hivePort = portOf hiveOtelSettings;
swarmPort = portOf (otelSettings baoWithCollector);
baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
in
hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
}
{
# The collector authenticates as the hive, so it reads its secret as
# the hive: the leaf `deploy.bao.clientCertFile` names, whose policy
# grants `swarm/hives/<hive>/*`. The last two clauses tie the unit's
# output to what the collector loads.
name = "the hive collector's secret is fetched from the store under the hive's own leaf";
ok =
let
u = hiveOtelFetch hiveOtelLeaf;
b = hiveOtelLeaf.services.hyperhive.deploy.bao;
file = hiveOtelLeaf.services.hyperhive.otel.clientSecretFile;
s = builtins.unsafeDiscardStringContext u.script;
in
hiveOtelLeaf.systemd.services ? swarm-bao-hive-otel-oidc
&& u.environment.BAO_CLIENT_CERT == b.clientCertFile
&& u.environment.BAO_CLIENT_KEY == b.clientKeyFile
&& lib.hasInfix "bao login -method=cert" s
&& lib.hasInfix "bao kv get -field=value secret/swarm/hives/h1/oidc/client" s
&& lib.hasInfix "atomic_write_secret 0400 \"\" ${file} " s
&& builtins.elem "swarm-client.secret:${file}" hiveOtelLeaf.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& builtins.elem "opentelemetry-collector.service" u.wantedBy
&& builtins.elem "opentelemetry-collector.service" u.before;
}
{
# The secret lives on tmpfs: an empty placeholder gives `LoadCredential`
# a source before the first fetch, and the copy the fetch used to leave
# on persistent disk is removed. The fixture naming that old path as
# its secret keeps it, and is warned that nothing refreshes it.
name = "the hive collector's fetched secret is an empty file on /run and the legacy file is removed";
ok =
let
run = "/run/hive-otel-oidc/hive-h1.secret";
legacy = "/var/lib/hive-otel-oidc/hive-h1.secret";
rules = m: m.systemd.tmpfiles.rules;
warned = m: lib.any (w: lib.hasInfix "swarm-bao-hive-otel-oidc.service used to write" w) m.warnings;
in
hiveOtelLeaf.services.hyperhive.otel.clientSecretFile == run
&& builtins.elem "d /run/hive-otel-oidc 0700 root root -" (rules hiveOtelLeaf)
&& builtins.elem "f ${run} 0400 root root -" (rules hiveOtelLeaf)
&& builtins.elem "r ${legacy}" (rules hiveOtelLeaf)
&& !(warned hiveOtelLeaf)
&& !(builtins.elem "r ${legacy}" (rules hiveOtel))
&& !(builtins.elem "f ${run} 0400 root root -" (rules hiveOtel))
&& warned hiveOtel;
}
{
# Without a store identity, a hand-named secret file is warned to place
# the leaf; with the leaf nothing is warned and the fetch unit exists;
# with neither, the assertion still refuses the build. `hiveOtelBare`
# and `hiveOtelLeaf` are the controls: a lookup that never matches
# would pass the negated arms.
name = "a hive host with no leaf and an operator secret file is warned to place the leaf";
ok =
let
warned = m: lib.any (lib.hasInfix "holds no store identity") m.warnings;
refused = m: lib.any (a: !a.assertion && lib.hasInfix "identity to present" a.message) m.assertions;
in
warned hiveOtel
&& lib.any (lib.hasInfix "/var/lib/swarm-bao-pki/hives/h1/") hiveOtel.warnings
&& !(warned hiveOtelLeaf)
&& hiveOtelLeaf.systemd.services ? swarm-bao-hive-otel-oidc
&& !(warned hiveOtelBare)
&& refused hiveOtelBare
&& !(refused hiveOtel)
&& !(refused hiveOtelLeaf);
}
{
# `LoadCredential` copies the file at the collector's start, so a value
# landed later reaches it only through a restart. Retries every 30s
# (./lib/store-retry.nix) while the store is down or the secret
# unpublished.
name = "the hive collector's fetch retries and restarts the collector when the file is newer";
ok =
let
u = hiveOtelFetch hiveOtelLeaf;
s = builtins.unsafeDiscardStringContext u.script;
in
u.serviceConfig.Restart == "on-failure"
&& u.serviceConfig.RestartSec == 30
&& toString u.unitConfig.StartLimitBurst == "2880"
&& lib.hasInfix "refresh_unit opentelemetry-collector.service /run/hive-otel-oidc/hive-h1.secret" s
&& lib.hasInfix "ActiveEnterTimestamp" s
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
}
{
# On a first boot the file does not exist until the fetch succeeds, so
# the collector fails `LoadCredential` into its start limit, i.e. the
# `failed` state. Only `reset-failed` clears that limit, so the fetch
# has to reset before it starts; `try-restart` acts on no failed unit.
# A never-active collector has an empty `ActiveEnterTimestamp`, read as
# 0, so the file always counts as newer. The collector only `wants` the
# fetch, so an unreachable store does not hold it back.
name = "the hive collector's fetch revives a collector that hit its start limit without the file";
ok =
let
s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script;
c = hiveOtelLeaf.systemd.services.opentelemetry-collector;
failedArm = lib.last (lib.splitString ''is-failed --quiet "$unit"; then'' s);
in
lib.hasInfix ''is-failed --quiet "$unit"; then'' s
&& lib.hasInfix ''reset-failed "$unit"'' failedArm
&& lib.hasInfix ''start --no-block "$unit"'' (
lib.last (lib.splitString ''reset-failed "$unit"'' failedArm)
)
&& lib.hasInfix "started_us=0" s
&& !(builtins.elem "swarm-bao-hive-otel-oidc.service" (c.requires or [ ]));
}
{
# The store is the one route, on the IdP's own host too: no default into
# authelia's tree and no unit waiting for a file to appear there.
name = "the hive collector reads no secret out of authelia's tree and waits on no copied file";
ok =
let
file = hiveOtelAllLocal.services.hyperhive.otel.clientSecretFile;
c = hiveOtelAllLocal.systemd.services.opentelemetry-collector;
in
!(hiveOtelAllLocal.systemd.services ? hive-otel-client-secret)
&& !(hiveOtelLeaf.systemd.services ? hive-otel-client-secret)
&& !(lib.hasInfix "authelia" file)
&& hiveOtelAllLocal.systemd.services ? swarm-bao-hive-otel-oidc
&& !(builtins.elem "hive-otel-client-secret.service" (c.requires or [ ]));
}
{
# The writer and the reader spell the path independently; this is the
# one place that sees both rendered.
name = "the hive collector reads the path the mint writes";
ok =
let
fetch = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelAllLocal).script;
publish = builtins.unsafeDiscardStringContext hiveOtelAllLocal.containers.swarm-authelia.config.systemd.services.authelia-swarm-secrets.script;
path = "secret/swarm/hives/h1/oidc/client";
in
lib.hasInfix "bao kv get -field=value ${path}" fetch
&& lib.hasInfix "mint_client_secret hive-h1 ${path}" publish;
}
{
# The store's units exist only on its host. The PKI unit is wanted, never
# required: a refused pki start fails a requirer's job with `dependency`.
name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
ok =
let
storeUnits = m: [
"swarm-bao-pki.service"
"container@${m.services.hyperhive.swarm.bao.machine}.service"
];
here = hiveOtelFetch hiveOtelAllLocal;
elsewhere = hiveOtelFetch hiveOtelLeaf;
in
lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
&& lib.elem "swarm-bao-pki.service" here.wants
&& !(lib.elem "swarm-bao-pki.service" here.requires)
&& lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
&& !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
storeUnits hiveOtelLeaf
));
}
{
# A secret delivered some other way is the operator's: no unit fetches
# into a file the collector does not load, or restarts it on one. Nothing
# restarts the collector when that file lands, so it waits for the file
# instead of failing `LoadCredential` into its start limit.
name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
ok =
let
waitsFor =
file: machine:
let
c = machine.systemd.services.opentelemetry-collector;
w = machine.systemd.services.hive-otel-client-secret;
in
builtins.elem "hive-otel-client-secret.service" c.requires
&& builtins.elem "hive-otel-client-secret.service" c.after
&& lib.hasInfix file w.script;
in
!(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
&& !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
&& builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& waitsFor "/var/lib/hive-otel-oidc/hive-h1.secret" hiveOtel
&& waitsFor "/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel;
}
{
# ../host-modules/glue-hive-otel-authelia.nix orders the wait after
# authelia's container. `hiveOtel` has the IdP; `baoWithHiveOtel` waits
# on the same kind of file without it, and must not name a container
# that is not there — `Requires=` on an absent unit fails the job.
name = "the hive collector's secret wait is ordered after authelia's container only where authelia runs here";
ok =
let
container = "container@${hiveOtel.services.hyperhive.swarm.authelia.machine}.service";
wait = machine: machine.systemd.services.hive-otel-client-secret;
in
builtins.elem container (wait hiveOtel).after
&& builtins.elem container (wait hiveOtel).requires
&& !(builtins.elem container (wait baoWithHiveOtel).after)
&& !(builtins.elem container (wait baoWithHiveOtel).requires);
}
{
# What the hive collector presents and where it presents it, both derived
# from the hive's name: pinned so the name's option path can move without
# the collector's identity changing.
name = "the hive collector authenticates as hive-<name> and pushes to the swarm collector under the name's path";
ok =
let
otel = hiveOtel.services.hyperhive.swarm.otel;
in
hiveOtelSettings.extensions."oauth2client/swarm".client_id == "hive-h1"
&& hiveOtelSettings.extensions."oauth2client/swarm".endpoint_params.audience == "hive-h1"
&& hiveOtelSettings.exporters."otlphttp/swarm".endpoint == "https://${otel.domain}/h1";
}
{
# Same reason as the swarm tier's case in ./swarm-otel-core.nix: the
# queue in front of the swarm exporter has to be on `file_storage` or a
# restart drops what the journald receiver already counted as consumed.
name = "the hive collector queues on disk and retries without a time limit";
ok =
let
e = hiveOtelSettings.exporters."otlphttp/swarm";
in
e.sending_queue.enabled
&& e.sending_queue.storage == "file_storage"
&& e.sending_queue.queue_size > 0
&& e.retry_on_failure.max_elapsed_time == "0s"
&& e.endpoint == "https://${hiveOtel.services.hyperhive.swarm.otel.domain}/h1"
&& hiveOtelSettings.extensions ? file_storage
&& builtins.elem "file_storage" hiveOtelSettings.service.extensions;
}
{
# Squid rides this fixture because its journal and metrics reach the
# swarm through this collector. The allow-list of journal units must stay
# exactly the four named: hive-c0re's (control: the list is not just
# squid's), the container units', squid's and its self-check's.
name = "the hive collector forwards the journal of squid, its self-check, hive-c0re and the container units and no other unit";
ok =
lib.sort builtins.lessThan hiveOtelSettings.receivers.journald.units == [
"container@*.service"
"hive-c0re.service"
"squid-selfcheck.service"
"squid.service"
];
}
{
name = "the hive collector scrapes the squid exporter on loopback";
ok =
let
j = hiveScrapeJob "squid";
in
j != null
&& j.static_configs == [ { targets = [ "127.0.0.1:9301" ]; } ]
&& lib.hasInfix "-listen 127.0.0.1:9301" hiveOtel.systemd.services.squid-exporter.serviceConfig.ExecStart;
}
{
# Control for the absence checks: the bridge listener is there, so the
# absences below are not a misspelled lookup.
name = "squid listens on the bridge IP only, without a second listener or ssl_bump";
ok =
let
conf = hiveOtel.services.squid.configText;
lines = lib.splitString "\n" conf;
in
builtins.elem "http_port ${hiveOtel.services.hyperhive.network.bridgeIp}:3128" lines
&& lib.length (lib.filter (l: lib.hasPrefix "http_port " l) lines) == 1
&& !(lib.hasInfix "ssl_bump" conf)
&& !(lib.hasInfix "ssl-bump" conf)
&& !(lib.hasInfix "localnet" conf);
}
{
# First match wins, so a destination deny after an allow is dead, and a
# credential check after one never challenges anyone. `allow unrestricted`
# is an allow: above the three destination denies an unrestricted user
# reaches loopback, link-local and the bridge through the proxy. The two
# IP-literal denies stay ahead of every allow: `dstdomain` on an IP
# literal does a reverse lookup.
name = "squid refuses loopback, link-local, bridge and private destinations and IP literals, then a missing credential, before any allow";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
allow = at "http_access allow allowed_dst";
base = at "http_access allow base_dst";
unrestricted = at "http_access allow unrestricted";
authed = at "http_access deny !authed";
# The generated per-agent `http_access allow` rules sit in this file.
agentRules = at "include ${hiveOtel.services.hyperhive.deploy.hive-controller.squid.agentAllowFile}";
firstAllow = lib.min agentRules (lib.min base (lib.min unrestricted allow));
in
allow != null
&& base != null
&& unrestricted != null
&& authed != null
&& agentRules != null
&& authed < agentRules
&& base < agentRules
&& agentRules < unrestricted
&& at "http_access deny to_localhost" != null
&& at "http_access deny to_linklocal" != null
&& at "http_access deny to_bridge" != null
&& at "http_access deny to_private" != null
&& at "http_access deny ip_v4_host" != null
&& at "http_access deny ip_v6_host" != null
&& lib.all (l: at l < firstAllow) [
"http_access deny to_localhost"
"http_access deny to_linklocal"
"http_access deny to_bridge"
"http_access deny to_private"
"http_access deny ip_v4_host"
"http_access deny ip_v6_host"
]
&& at "http_access deny to_localhost" < authed
&& at "http_access deny to_linklocal" < authed
&& at "http_access deny to_bridge" < authed
&& at "http_access deny to_private" < authed
&& authed < firstAllow
&& builtins.elem "cache deny all" lines
&& lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
}
{
# Control: the allow ACL line is found and holds the canary, so the
# absence of the swarm's service names is not a lookup of nothing.
name = "squid's default allow-list is the self-check canary alone and its base list is the Claude Code hosts, with no swarm service name";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
acl = lib.findFirst (l: lib.hasPrefix "acl allowed_dst dstdomain " l) "" (map lib.trim lines);
baseAcl = lib.findFirst (l: lib.hasPrefix "acl base_dst dstdomain " l) "" (map lib.trim lines);
in
acl
== "acl allowed_dst dstdomain ${hiveOtel.services.hyperhive.deploy.hive-controller.squid.selfcheck.canary}"
&& baseAcl == "acl base_dst dstdomain api.anthropic.com claude.ai platform.claude.com"
&& hiveOtel.services.hyperhive.deploy.hive-controller.squid.selfcheck.canary == "cache.nixos.org"
&& !(builtins.elem hiveOtel.services.hyperhive.swarm.forge.domain (lib.splitString " " acl));
}
{
# The ranges are what keeps an unrestricted user on the public internet.
# Control: the bridge deny and the ACL line are found, so a range missing
# below is not a lookup of nothing.
name = "squid's private-destination ACL names every private, shared, multicast and reserved range";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
acl = lib.findFirst (l: lib.hasPrefix "acl to_private dst " l) null lines;
ranges = lib.splitString " " (lib.removePrefix "acl to_private dst " acl);
in
acl != null
&& builtins.elem "acl to_bridge dst ${hiveOtel.services.hyperhive.network.networkCidr}" lines
&&
lib.sort lib.lessThan ranges == lib.sort lib.lessThan [
"10.0.0.0/8"
"172.16.0.0/12"
"192.168.0.0/16"
"100.64.0.0/10"
"0.0.0.0/8"
"224.0.0.0/4"
"240.0.0.0/4"
"fc00::/7"
"ff00::/8"
"::/128"
];
}
{
# An unrestricted user is let through before the unlisted-name deny and
# everyone else is held to the listed names after it. Control: both are
# there, so the ordering below is not a lookup of nothing.
name = "squid lets an unrestricted user through before the allow-list rules that bind everyone else";
ok =
let
conf = hiveOtel.services.squid.configText;
lines = lib.splitString "\n" conf;
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
list = hiveOtel.services.hyperhive.deploy.hive-controller.squid.unrestrictedFile;
in
at "http_access allow unrestricted" != null
&& at "http_access deny !allowed_dst" != null
&& at "http_access deny !authed" < at "http_access allow base_dst"
&& at "http_access allow base_dst" < at "http_access allow unrestricted"
&& at "http_access allow unrestricted" < at "http_access deny !allowed_dst"
&& at "http_access deny !allowed_dst" < at "http_access allow allowed_dst"
&& builtins.elem ''acl unrestricted proxy_auth "${list}"'' lines
&& list == "/var/lib/hive-squid/unrestricted";
}
{
# The list is read when squid parses its configuration, so a change to
# the file reloads squid. Control: the watched path is the list's own.
name = "squid is reloaded when its unrestricted-users list changes";
ok =
let
list = hiveOtel.services.hyperhive.deploy.hive-controller.squid.unrestrictedFile;
rules = hiveOtel.systemd.tmpfiles.rules;
path = hiveOtel.systemd.paths.squid-unrestricted-reload;
svc = hiveOtel.systemd.services.squid-unrestricted-reload.serviceConfig;
in
path.pathConfig.PathChanged == [
list
hiveOtel.services.hyperhive.deploy.hive-controller.squid.allowedFile
hiveOtel.services.hyperhive.deploy.hive-controller.squid.agentAllowFile
]
&& hiveOtel.systemd.services.squid-unrestricted-reload.startLimitIntervalSec == 0
&& builtins.elem "paths.target" path.wantedBy
&& lib.hasSuffix "systemctl try-reload-or-restart squid.service" svc.ExecStart
&& builtins.elem "f ${list} 0640 hive-core squid - -" rules
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_UNRESTRICTED_FILE == list;
}
{
# The swarm's allow-list reaches squid as a file of names, which starts
# as this hive's own `allowedDomains` (the canary left out, since the
# configuration names it), and as an include that holds the per-agent
# rules. Control: the seed holds the listed name, and the configuration
# no longer names it, so the absence of the canary and of the list from
# the seed and the configuration is not a lookup of nothing.
name = "squid reads the swarm's allow-list from a file seeded with the hive's own names";
ok =
let
sq = hiveOtelAllLocal.services.hyperhive.deploy.hive-controller.squid;
lines = lib.splitString "\n" hiveOtelAllLocal.services.squid.configText;
rules = hiveOtelAllLocal.systemd.tmpfiles.rules;
env = hiveOtelAllLocal.systemd.services.hive-c0re.environment;
in
builtins.elem ''acl allowed_dst dstdomain "${sq.allowedFile}"'' lines
&& builtins.elem "acl allowed_dst dstdomain ${sq.selfcheck.canary}" lines
&& !(lib.hasInfix "github.com" hiveOtelAllLocal.services.squid.configText)
&& sq.allowedFile == "/var/lib/hive-squid/allowed"
&& sq.agentAllowFile == "/var/lib/hive-squid/agent-allow.conf"
&& sq.allowedSeedFile.text == "github.com\n"
&& builtins.elem "C ${sq.allowedFile} 0640 hive-core squid - ${sq.allowedSeedFile}" rules
&& builtins.elem "f ${sq.agentAllowFile} 0640 hive-core squid - -" rules
&& env.HIVE_SQUID_ALLOWED_FILE == sq.allowedFile
&& env.HIVE_SQUID_AGENT_ALLOW_FILE == sq.agentAllowFile
# The unit's environment holds the seed as a derivation until the unit file
# is rendered, so it is compared as a path string.
&& toString env.HIVE_SQUID_ALLOWED_SEED_FILE == toString sq.allowedSeedFile
&&
env.HIVE_SQUID_ALWAYS_ALLOWED == "api.anthropic.com claude.ai platform.claude.com cache.nixos.org";
}
{
# The nixpkgs module parses the configuration at build time, which cannot
# open the include; the parse moves to a check on the same text, so a
# broken configuration still fails the host build.
name = "squid's configuration is still parsed at build time, without the include";
ok =
!hiveOtel.services.squid.validateConfig
&& lib.any (d: (d.name or "") == "squid-conf-check") hiveOtel.system.extraDependencies;
}
{
# The helper is the NCSA one, on the file hive-c0re renders. The file
# exists, empty, before squid starts (the helper exits on a missing one),
# and the self-check imports its credential without failing when there is
# none.
name = "squid authenticates with the NCSA helper on a users file that exists before it starts";
ok =
let
conf = hiveOtel.services.squid.configText;
lines = lib.splitString "\n" conf;
users = hiveOtel.services.hyperhive.deploy.hive-controller.squid.usersFile;
rules = hiveOtel.systemd.tmpfiles.rules;
sc = hiveOtel.systemd.services.squid-selfcheck.serviceConfig;
in
lib.any (
l: lib.hasInfix "/libexec/basic_ncsa_auth ${users}" l && lib.hasPrefix "auth_param basic program " l
) lines
&& builtins.elem "acl authed proxy_auth REQUIRED" lines
&& users == "/var/lib/hive-squid/users"
&& builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules
&& builtins.elem "f ${users} 0640 hive-core squid - -" rules
&&
sc.ImportCredential
== hiveOtel.services.hyperhive.deploy.hive-controller.squid.selfcheck.credentialName
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users;
}
{
name = "squid's port is open on the bridge interface and on no other";
ok =
let
fw = hiveOtel.networking.firewall;
in
builtins.elem 3128 fw.interfaces.${hiveOtel.services.hyperhive.network.bridgeName}.allowedTCPPorts
&& !(builtins.elem 3128 fw.allowedTCPPorts);
}
{
# The facts that tell an agent this host's squid checks its credential and
# applies its flag. Presence arm: a hive holding a store identity writes
# both, and the direct hosts carry the names the brief's destinations
# resolve under. Absence arm: a hive with no identity renders no users
# file, so it announces no proxy.
name = "hive-c0re hands agents the squid address and the hosts that skip it, only with a store identity";
ok =
let
e = hiveOtelAllLocal.systemd.services.hive-c0re.environment;
s = hiveOtelAllLocal.services.hyperhive;
hosts = lib.splitString "," e.HIVE_EGRESS_NO_PROXY;
in
e.HIVE_SQUID_PROXY == "${s.network.bridgeIp}:3128"
&& e.HIVE_SQUID_PROXY == s.deploy.hive-controller.squid.listen
&& lib.all (h: lib.elem h hosts) (
[
s.network.bridgeIp
s.swarm.domain
s.swarm.bao.domain
s.swarm.nats.domain
s.swarm.attic.domain
s.swarm.forge.domain
s.swarm.matrix.gatewayHost
s.swarm.victorialogs.domain
s.swarm.victoriametrics.domain
s.swarm.otel.domain
s.otel.collector.host
s.domain
]
++ s.gateway.localNames
)
&& !(lib.elem "example.com" hosts)
&& lib.length hosts == lib.length (lib.unique hosts)
&& !(hiveOtelBare.systemd.services.hive-c0re.environment ? HIVE_SQUID_PROXY)
&& !(hiveOtelBare.systemd.services.hive-c0re.environment ? HIVE_EGRESS_NO_PROXY);
}
{
# The store and the queue are in no list the gateway builds, so a hive whose
# store is elsewhere gets them from their own options. Control: this
# fixture runs no store, so the gateway does not name it.
name = "a hive without a local store still lists the store and the queue as direct hosts";
ok =
let
s = hiveOtelLeaf.services.hyperhive;
hosts = lib.splitString "," hiveOtelLeaf.systemd.services.hive-c0re.environment.HIVE_EGRESS_NO_PROXY;
in
!(lib.elem s.swarm.bao.domain s.gateway.localNames)
&& lib.elem s.swarm.bao.domain hosts
&& lib.elem s.swarm.nats.domain hosts;
}
{
# Squid's listener and the fact are one value.
name = "the squid fact is the address squid binds";
ok =
let
s = hiveOtelAllLocal.services.hyperhive;
in
lib.elem "http_port ${s.deploy.hive-controller.squid.listen}" (
lib.splitString "\n" hiveOtelAllLocal.services.squid.configText
);
}
];
in
runGroup "hive-otel" cases
View git blame Copy permalink