765 lines
36 KiB
Nix
765 lines
36 KiB
Nix
# `checks.module-eval-hive-otel` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
otelSettings
|
|
;
|
|
|
|
# This hive's own collector, which is a HOST service — unlike the swarm
|
|
# tier's, which lives in a container and is read through `otelSettings`.
|
|
# The IdP is here too, with its secret file named at the path the fetch
|
|
# unit used to write: the collector's wait for that file exists beside
|
|
# authelia's container.
|
|
hiveOtel = hiveWithController {
|
|
otel.clientSecretFile = "/var/lib/hive-otel-oidc/hive-h1.secret";
|
|
deploy.authelia.enable = true;
|
|
};
|
|
|
|
# A hive holding its own store leaf, placed by hand, and naming no secret
|
|
# file: the collector's secret comes from the store under that leaf.
|
|
hiveOtelLeaf = hiveWithController {
|
|
gateway.enable = false;
|
|
deploy.bao.clientCertFile = "/etc/pki/hive.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/hive-key.pem";
|
|
};
|
|
|
|
# The IdP, the store and the collector on one host: the publisher that
|
|
# writes the secret and the unit that reads it, in one config.
|
|
hiveOtelAllLocal = hiveWithController {
|
|
deploy.bao.enable = true;
|
|
deploy.authelia.enable = true;
|
|
# The seed `allowedFile` starts as; the canary is listed too, to show it
|
|
# is not written twice.
|
|
squid.allowedDomains = [
|
|
"github.com"
|
|
"cache.nixos.org"
|
|
];
|
|
};
|
|
|
|
# Neither a leaf nor a secret file: the one shape the collector has no
|
|
# credential for.
|
|
hiveOtelBare = hiveWithController { };
|
|
|
|
hiveOtelFetch = machine: machine.systemd.services.swarm-bao-hive-otel-oidc;
|
|
|
|
hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;
|
|
|
|
hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings;
|
|
|
|
# The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for
|
|
# the swarm tier — the option is one string, what prometheus is handed is a
|
|
# job — but this collector is a host service, so the path to it differs.
|
|
hiveScrapeJob =
|
|
job:
|
|
lib.findFirst (
|
|
c: c.job_name == job
|
|
) null hiveOtelSettings.receivers.prometheus.config.scrape_configs;
|
|
|
|
# The store with and without a collector on the same host. `scrapeTargets`
|
|
# is only ever read by a local collector, so the metrics endpoint is a
|
|
# function of the pairing rather than of the store.
|
|
inherit (shared) baoWithCollector;
|
|
|
|
# The store standing beside a HIVE collector: the topology the port case
|
|
# below needs, two collectors in one netns with nothing but their
|
|
# self-telemetry ports keeping them apart.
|
|
inherit (shared) baoWithHiveOtel;
|
|
|
|
# The collector INSIDE the store's container — neither the host's nor the
|
|
# swarm tier's.
|
|
baoForwarderSettings =
|
|
machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
|
|
cases = [
|
|
{
|
|
# The tier in the middle. Its OTLP receiver takes both signals on one
|
|
# port, so without this pipeline an agent's push is answered 404 on
|
|
# `/v1/logs` — and a forwarder retrying into a 404 is indistinguish-
|
|
# able from one with nothing to send. `journald` joined the receiver
|
|
# list for this collector's own host journal (hive-c0re's crash-watch
|
|
# log line, see ./otel.nix) — both have to be present, in either
|
|
# order, so this stays a set check rather than a list-equality one.
|
|
# Compared against the metrics pipeline's exporters rather than a
|
|
# name, so the two signals cannot drift to different destinations.
|
|
name = "the hive collector forwards logs upstream, not only metrics";
|
|
ok =
|
|
(hiveOtelPipelines ? logs)
|
|
&& hiveOtelPipelines.metrics.exporters != [ ]
|
|
&&
|
|
lib.sort builtins.lessThan hiveOtelPipelines.logs.receivers == [
|
|
"journald"
|
|
"otlp"
|
|
]
|
|
&& hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters;
|
|
}
|
|
{
|
|
# `deltatocumulative` is metrics-only: naming it in a logs pipeline
|
|
# kills the collector at startup rather than doing nothing. The second
|
|
# clause is the control — the metrics pipeline still names it, so a
|
|
# pass means the two processor lists differ rather than that the
|
|
# processor left the module.
|
|
name = "the hive collector keeps the metrics-only processor out of its logs pipeline";
|
|
ok =
|
|
!(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors)
|
|
&& builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors;
|
|
}
|
|
{
|
|
# `otelcol validate` refuses a `service.extensions` entry with no
|
|
# definition, and that runs when the unit is built, not in this suite.
|
|
# `hiveOtel` authenticates upstream, so both extensions are listed;
|
|
# the length clause keeps the check from passing on an empty list.
|
|
name = "the hive collector defines every extension its service lists";
|
|
ok =
|
|
lib.length hiveOtelSettings.service.extensions == 2
|
|
&& lib.all (e: hiveOtelSettings.extensions ? ${e}) hiveOtelSettings.service.extensions;
|
|
}
|
|
{
|
|
# The counters that say telemetry is being LOST — refused, failed,
|
|
# queue depth — are served on loopback and reach no store unless
|
|
# something reads them. Read off the rendered job rather than the
|
|
# option: only the job is what prometheus actually requests.
|
|
name = "the hive collector scrapes its own telemetry endpoint";
|
|
ok =
|
|
let
|
|
j = hiveScrapeJob "collector";
|
|
in
|
|
j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ];
|
|
}
|
|
{
|
|
# A `prometheus` receiver no pipeline names collects nothing while
|
|
# rendering and starting perfectly, so the scrape above is inert
|
|
# without this. The path is newly reachable: until the hive tier had a
|
|
# target of its own, this receiver was never emitted on any hive.
|
|
name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
|
|
ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
|
|
}
|
|
{
|
|
# The receiver, the pipeline entry and the nginx server it reads have to
|
|
# agree on one address, and none of them may exist without a gateway on
|
|
# the host. `hiveOtelLeaf` is the control: it opts out of the gateway, and the
|
|
# clause asserting that keeps the control from going vacuous.
|
|
name = "the hive collector reads the gateway's stub_status, and only where a gateway runs";
|
|
ok =
|
|
let
|
|
nginxConf = hiveOtel.services.nginx.appendHttpConfig;
|
|
leafSettings = hiveOtelLeaf.services.opentelemetry-collector.settings;
|
|
in
|
|
hiveOtel.services.hyperhive.gateway.enable
|
|
&& hiveOtelSettings.receivers.nginx.endpoint == "http://127.0.0.1:8085/stub_status"
|
|
&& builtins.elem "nginx" hiveOtelPipelines.metrics.receivers
|
|
&& lib.hasInfix "listen 127.0.0.1:8085;" nginxConf
|
|
&& lib.hasInfix "location = /stub_status {" nginxConf
|
|
&& !hiveOtelLeaf.services.hyperhive.gateway.enable
|
|
&& !(leafSettings.receivers ? nginx)
|
|
&& !(builtins.elem "nginx" leafSettings.service.pipelines.metrics.receivers);
|
|
}
|
|
{
|
|
# `metrics.address` is the spelling that looks right and is rejected by
|
|
# this collector version. The first clause is the control: without it a
|
|
# missing telemetry block would pass the port check vacuously.
|
|
name = "the hive collector binds its telemetry port through readers, not address";
|
|
ok =
|
|
let
|
|
m = hiveOtelSettings.service.telemetry.metrics;
|
|
in
|
|
!(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
|
|
}
|
|
{
|
|
# A bind that loses the race with the outgoing process is the whole
|
|
# failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
|
|
# without this the unit spends its five default attempts inside two
|
|
# seconds and lands in `start-limit-hit`, where it stops retrying. The
|
|
# third clause is the one that has to hold — `StartLimit*` are `[Unit]`
|
|
# settings that systemd ignores under `[Service]`, so a bound written
|
|
# into `serviceConfig` renders, deploys and does nothing. Asserted
|
|
# where nixpkgs puts it rather than where it was written, same as
|
|
# ./bao-grants.nix.
|
|
name = "the hive collector backs off a failed bind from [Unit], not [Service]";
|
|
ok =
|
|
let
|
|
u = hiveOtel.systemd.services.opentelemetry-collector;
|
|
in
|
|
u.serviceConfig.RestartSec or 0 > 0
|
|
&& toString u.unitConfig.StartLimitBurst == "12"
|
|
&& !(u.serviceConfig ? StartLimitBurst)
|
|
# The window has to outlast every attempt, or the burst is unreachable.
|
|
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
|
}
|
|
{
|
|
# Collectors share a network namespace whenever they are co-located,
|
|
# and this port appears in no config the port-collision assertion can
|
|
# read — so equal defaults mean the second to start dies at `bind()`.
|
|
# Pinned as a case rather than an assertion: enforcing it belongs with
|
|
# the other port checks, not here.
|
|
#
|
|
# THREE tiers, because the store's container grew a collector of its
|
|
# own and this case could not see it: an UNDECLARED port is not an
|
|
# absent one, it is the binary's 8888, which is the hive tier's. That
|
|
# is why `portOf` falls back to 8888 rather than throwing — a tier that
|
|
# declares nothing has to fail this case loudly, not evaluate away.
|
|
name = "no two collector tiers claim the same self-telemetry port";
|
|
ok =
|
|
let
|
|
portOf =
|
|
s:
|
|
(lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
|
|
hivePort = portOf hiveOtelSettings;
|
|
swarmPort = portOf (otelSettings baoWithCollector);
|
|
baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
|
|
in
|
|
hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
|
|
}
|
|
{
|
|
# The collector authenticates as the hive, so it reads its secret as
|
|
# the hive: the leaf `deploy.bao.clientCertFile` names, whose policy
|
|
# grants `swarm/hives/<hive>/*`. The last two clauses tie the unit's
|
|
# output to what the collector loads.
|
|
name = "the hive collector's secret is fetched from the store under the hive's own leaf";
|
|
ok =
|
|
let
|
|
u = hiveOtelFetch hiveOtelLeaf;
|
|
b = hiveOtelLeaf.services.hyperhive.deploy.bao;
|
|
file = hiveOtelLeaf.services.hyperhive.otel.clientSecretFile;
|
|
s = builtins.unsafeDiscardStringContext u.script;
|
|
in
|
|
hiveOtelLeaf.systemd.services ? swarm-bao-hive-otel-oidc
|
|
&& u.environment.BAO_CLIENT_CERT == b.clientCertFile
|
|
&& u.environment.BAO_CLIENT_KEY == b.clientKeyFile
|
|
&& lib.hasInfix "bao login -method=cert" s
|
|
&& lib.hasInfix "bao kv get -field=value secret/swarm/hives/h1/oidc/client" s
|
|
&& lib.hasInfix "atomic_write_secret 0400 \"\" ${file} " s
|
|
&& builtins.elem "swarm-client.secret:${file}" hiveOtelLeaf.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
|
&& builtins.elem "opentelemetry-collector.service" u.wantedBy
|
|
&& builtins.elem "opentelemetry-collector.service" u.before;
|
|
}
|
|
{
|
|
# The secret lives on tmpfs: an empty placeholder gives `LoadCredential`
|
|
# a source before the first fetch, and the copy the fetch used to leave
|
|
# on persistent disk is removed. The fixture naming that old path as
|
|
# its secret keeps it, and is warned that nothing refreshes it.
|
|
name = "the hive collector's fetched secret is an empty file on /run and the legacy file is removed";
|
|
ok =
|
|
let
|
|
run = "/run/hive-otel-oidc/hive-h1.secret";
|
|
legacy = "/var/lib/hive-otel-oidc/hive-h1.secret";
|
|
rules = m: m.systemd.tmpfiles.rules;
|
|
warned = m: lib.any (w: lib.hasInfix "swarm-bao-hive-otel-oidc.service used to write" w) m.warnings;
|
|
in
|
|
hiveOtelLeaf.services.hyperhive.otel.clientSecretFile == run
|
|
&& builtins.elem "d /run/hive-otel-oidc 0700 root root -" (rules hiveOtelLeaf)
|
|
&& builtins.elem "f ${run} 0400 root root -" (rules hiveOtelLeaf)
|
|
&& builtins.elem "r ${legacy}" (rules hiveOtelLeaf)
|
|
&& !(warned hiveOtelLeaf)
|
|
&& !(builtins.elem "r ${legacy}" (rules hiveOtel))
|
|
&& !(builtins.elem "f ${run} 0400 root root -" (rules hiveOtel))
|
|
&& warned hiveOtel;
|
|
}
|
|
{
|
|
# Without a store identity, a hand-named secret file is warned to place
|
|
# the leaf; with the leaf nothing is warned and the fetch unit exists;
|
|
# with neither, the assertion still refuses the build. `hiveOtelBare`
|
|
# and `hiveOtelLeaf` are the controls: a lookup that never matches
|
|
# would pass the negated arms.
|
|
name = "a hive host with no leaf and an operator secret file is warned to place the leaf";
|
|
ok =
|
|
let
|
|
warned = m: lib.any (lib.hasInfix "holds no store identity") m.warnings;
|
|
refused = m: lib.any (a: !a.assertion && lib.hasInfix "identity to present" a.message) m.assertions;
|
|
in
|
|
warned hiveOtel
|
|
&& lib.any (lib.hasInfix "/var/lib/swarm-bao-pki/hives/h1/") hiveOtel.warnings
|
|
&& !(warned hiveOtelLeaf)
|
|
&& hiveOtelLeaf.systemd.services ? swarm-bao-hive-otel-oidc
|
|
&& !(warned hiveOtelBare)
|
|
&& refused hiveOtelBare
|
|
&& !(refused hiveOtel)
|
|
&& !(refused hiveOtelLeaf);
|
|
}
|
|
{
|
|
# `LoadCredential` copies the file at the collector's start, so a value
|
|
# landed later reaches it only through a restart. Retries every 30s
|
|
# (./lib/store-retry.nix) while the store is down or the secret
|
|
# unpublished.
|
|
name = "the hive collector's fetch retries and restarts the collector when the file is newer";
|
|
ok =
|
|
let
|
|
u = hiveOtelFetch hiveOtelLeaf;
|
|
s = builtins.unsafeDiscardStringContext u.script;
|
|
in
|
|
u.serviceConfig.Restart == "on-failure"
|
|
&& u.serviceConfig.RestartSec == 30
|
|
&& toString u.unitConfig.StartLimitBurst == "2880"
|
|
&& lib.hasInfix "refresh_unit opentelemetry-collector.service /run/hive-otel-oidc/hive-h1.secret" s
|
|
&& lib.hasInfix "ActiveEnterTimestamp" s
|
|
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
|
|
}
|
|
{
|
|
# On a first boot the file does not exist until the fetch succeeds, so
|
|
# the collector fails `LoadCredential` into its start limit, i.e. the
|
|
# `failed` state. Only `reset-failed` clears that limit, so the fetch
|
|
# has to reset before it starts; `try-restart` acts on no failed unit.
|
|
# A never-active collector has an empty `ActiveEnterTimestamp`, read as
|
|
# 0, so the file always counts as newer. The collector only `wants` the
|
|
# fetch, so an unreachable store does not hold it back.
|
|
name = "the hive collector's fetch revives a collector that hit its start limit without the file";
|
|
ok =
|
|
let
|
|
s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script;
|
|
c = hiveOtelLeaf.systemd.services.opentelemetry-collector;
|
|
failedArm = lib.last (lib.splitString ''is-failed --quiet "$unit"; then'' s);
|
|
in
|
|
lib.hasInfix ''is-failed --quiet "$unit"; then'' s
|
|
&& lib.hasInfix ''reset-failed "$unit"'' failedArm
|
|
&& lib.hasInfix ''start --no-block "$unit"'' (
|
|
lib.last (lib.splitString ''reset-failed "$unit"'' failedArm)
|
|
)
|
|
&& lib.hasInfix "started_us=0" s
|
|
&& !(builtins.elem "swarm-bao-hive-otel-oidc.service" (c.requires or [ ]));
|
|
}
|
|
{
|
|
# The store is the one route, on the IdP's own host too: no default into
|
|
# authelia's tree and no unit waiting for a file to appear there.
|
|
name = "the hive collector reads no secret out of authelia's tree and waits on no copied file";
|
|
ok =
|
|
let
|
|
file = hiveOtelAllLocal.services.hyperhive.otel.clientSecretFile;
|
|
c = hiveOtelAllLocal.systemd.services.opentelemetry-collector;
|
|
in
|
|
!(hiveOtelAllLocal.systemd.services ? hive-otel-client-secret)
|
|
&& !(hiveOtelLeaf.systemd.services ? hive-otel-client-secret)
|
|
&& !(lib.hasInfix "authelia" file)
|
|
&& hiveOtelAllLocal.systemd.services ? swarm-bao-hive-otel-oidc
|
|
&& !(builtins.elem "hive-otel-client-secret.service" (c.requires or [ ]));
|
|
}
|
|
{
|
|
# The writer and the reader spell the path independently; this is the
|
|
# one place that sees both rendered.
|
|
name = "the hive collector reads the path the secret publisher writes";
|
|
ok =
|
|
let
|
|
fetch = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelAllLocal).script;
|
|
publish = builtins.unsafeDiscardStringContext hiveOtelAllLocal.containers.swarm-authelia.config.systemd.services.swarm-secret-publish.script;
|
|
path = "secret/swarm/hives/h1/oidc/client";
|
|
in
|
|
lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
|
|
}
|
|
{
|
|
# The store's units exist only on its host. The PKI unit is wanted, never
|
|
# required: a refused pki start fails a requirer's job with `dependency`.
|
|
name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
|
|
ok =
|
|
let
|
|
storeUnits = m: [
|
|
"swarm-bao-pki.service"
|
|
"container@${m.services.hyperhive.swarm.bao.machine}.service"
|
|
];
|
|
here = hiveOtelFetch hiveOtelAllLocal;
|
|
elsewhere = hiveOtelFetch hiveOtelLeaf;
|
|
in
|
|
lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
|
|
&& lib.elem "swarm-bao-pki.service" here.wants
|
|
&& !(lib.elem "swarm-bao-pki.service" here.requires)
|
|
&& lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
|
|
&& !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
|
|
storeUnits hiveOtelLeaf
|
|
));
|
|
}
|
|
{
|
|
# A secret delivered some other way is the operator's: no unit fetches
|
|
# into a file the collector does not load, or restarts it on one. Nothing
|
|
# restarts the collector when that file lands, so it waits for the file
|
|
# instead of failing `LoadCredential` into its start limit.
|
|
name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
|
|
ok =
|
|
let
|
|
waitsFor =
|
|
file: machine:
|
|
let
|
|
c = machine.systemd.services.opentelemetry-collector;
|
|
w = machine.systemd.services.hive-otel-client-secret;
|
|
in
|
|
builtins.elem "hive-otel-client-secret.service" c.requires
|
|
&& builtins.elem "hive-otel-client-secret.service" c.after
|
|
&& lib.hasInfix file w.script;
|
|
in
|
|
!(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
|
&& !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
|
&& builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
|
&& waitsFor "/var/lib/hive-otel-oidc/hive-h1.secret" hiveOtel
|
|
&& waitsFor "/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel;
|
|
}
|
|
{
|
|
# ../host-modules/glue-hive-otel-authelia.nix orders the wait after
|
|
# authelia's container. `hiveOtel` has the IdP; `baoWithHiveOtel` waits
|
|
# on the same kind of file without it, and must not name a container
|
|
# that is not there — `Requires=` on an absent unit fails the job.
|
|
name = "the hive collector's secret wait is ordered after authelia's container only where authelia runs here";
|
|
ok =
|
|
let
|
|
container = "container@${hiveOtel.services.hyperhive.swarm.authelia.machine}.service";
|
|
wait = machine: machine.systemd.services.hive-otel-client-secret;
|
|
in
|
|
builtins.elem container (wait hiveOtel).after
|
|
&& builtins.elem container (wait hiveOtel).requires
|
|
&& !(builtins.elem container (wait baoWithHiveOtel).after)
|
|
&& !(builtins.elem container (wait baoWithHiveOtel).requires);
|
|
}
|
|
{
|
|
# What the hive collector presents and where it presents it, both derived
|
|
# from the hive's name: pinned so the name's option path can move without
|
|
# the collector's identity changing.
|
|
name = "the hive collector authenticates as hive-<name> and pushes to the swarm collector under the name's path";
|
|
ok =
|
|
let
|
|
otel = hiveOtel.services.hyperhive.swarm.otel;
|
|
in
|
|
hiveOtelSettings.extensions."oauth2client/swarm".client_id == "hive-h1"
|
|
&& hiveOtelSettings.extensions."oauth2client/swarm".endpoint_params.audience == "hive-h1"
|
|
&& hiveOtelSettings.exporters."otlphttp/swarm".endpoint == "https://${otel.domain}/h1";
|
|
}
|
|
{
|
|
# Same reason as the swarm tier's case in ./swarm-otel-core.nix: the
|
|
# queue in front of the swarm exporter has to be on `file_storage` or a
|
|
# restart drops what the journald receiver already counted as consumed.
|
|
name = "the hive collector queues on disk and retries without a time limit";
|
|
ok =
|
|
let
|
|
e = hiveOtelSettings.exporters."otlphttp/swarm";
|
|
in
|
|
e.sending_queue.enabled
|
|
&& e.sending_queue.storage == "file_storage"
|
|
&& e.sending_queue.queue_size > 0
|
|
&& e.retry_on_failure.max_elapsed_time == "0s"
|
|
&& e.endpoint == "https://${hiveOtel.services.hyperhive.swarm.otel.domain}/h1"
|
|
&& hiveOtelSettings.extensions ? file_storage
|
|
&& builtins.elem "file_storage" hiveOtelSettings.service.extensions;
|
|
}
|
|
{
|
|
# Squid rides this fixture because its journal and metrics reach the
|
|
# swarm through this collector. The allow-list of journal units must stay
|
|
# exactly the three named: hive-c0re's (control: the list is not just
|
|
# squid's), squid's and its self-check's.
|
|
name = "the hive collector forwards the journal of squid, its self-check and hive-c0re and no other unit";
|
|
ok =
|
|
lib.sort builtins.lessThan hiveOtelSettings.receivers.journald.units == [
|
|
"hive-c0re.service"
|
|
"squid-selfcheck.service"
|
|
"squid.service"
|
|
];
|
|
}
|
|
{
|
|
name = "the hive collector scrapes the squid exporter on loopback";
|
|
ok =
|
|
let
|
|
j = hiveScrapeJob "squid";
|
|
in
|
|
j != null
|
|
&& j.static_configs == [ { targets = [ "127.0.0.1:9301" ]; } ]
|
|
&& lib.hasInfix "-listen 127.0.0.1:9301" hiveOtel.systemd.services.squid-exporter.serviceConfig.ExecStart;
|
|
}
|
|
{
|
|
# Control for the absence checks: the bridge listener is there, so the
|
|
# absences below are not a misspelled lookup.
|
|
name = "squid listens on the bridge IP only, without a second listener or ssl_bump";
|
|
ok =
|
|
let
|
|
conf = hiveOtel.services.squid.configText;
|
|
lines = lib.splitString "\n" conf;
|
|
in
|
|
builtins.elem "http_port ${hiveOtel.services.hyperhive.network.bridgeIp}:3128" lines
|
|
&& lib.length (lib.filter (l: lib.hasPrefix "http_port " l) lines) == 1
|
|
&& !(lib.hasInfix "ssl_bump" conf)
|
|
&& !(lib.hasInfix "ssl-bump" conf)
|
|
&& !(lib.hasInfix "localnet" conf);
|
|
}
|
|
{
|
|
# First match wins, so a destination deny after an allow is dead, and a
|
|
# credential check after one never challenges anyone. `allow unrestricted`
|
|
# is an allow: above the three destination denies an unrestricted user
|
|
# reaches loopback, link-local and the bridge through the proxy. The two
|
|
# IP-literal denies stay ahead of every allow: `dstdomain` on an IP
|
|
# literal does a reverse lookup.
|
|
name = "squid refuses loopback, link-local, bridge and private destinations and IP literals, then a missing credential, before any allow";
|
|
ok =
|
|
let
|
|
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
|
|
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
|
|
allow = at "http_access allow allowed_dst";
|
|
base = at "http_access allow base_dst";
|
|
unrestricted = at "http_access allow unrestricted";
|
|
authed = at "http_access deny !authed";
|
|
# The generated per-agent `http_access allow` rules sit in this file.
|
|
agentRules = at "include ${hiveOtel.services.hyperhive.squid.agentAllowFile}";
|
|
firstAllow = lib.min agentRules (lib.min base (lib.min unrestricted allow));
|
|
in
|
|
allow != null
|
|
&& base != null
|
|
&& unrestricted != null
|
|
&& authed != null
|
|
&& agentRules != null
|
|
&& authed < agentRules
|
|
&& base < agentRules
|
|
&& agentRules < unrestricted
|
|
&& at "http_access deny to_localhost" != null
|
|
&& at "http_access deny to_linklocal" != null
|
|
&& at "http_access deny to_bridge" != null
|
|
&& at "http_access deny to_private" != null
|
|
&& at "http_access deny ip_v4_host" != null
|
|
&& at "http_access deny ip_v6_host" != null
|
|
&& lib.all (l: at l < firstAllow) [
|
|
"http_access deny to_localhost"
|
|
"http_access deny to_linklocal"
|
|
"http_access deny to_bridge"
|
|
"http_access deny to_private"
|
|
"http_access deny ip_v4_host"
|
|
"http_access deny ip_v6_host"
|
|
]
|
|
&& at "http_access deny to_localhost" < authed
|
|
&& at "http_access deny to_linklocal" < authed
|
|
&& at "http_access deny to_bridge" < authed
|
|
&& at "http_access deny to_private" < authed
|
|
&& authed < firstAllow
|
|
&& builtins.elem "cache deny all" lines
|
|
&& lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
|
|
}
|
|
{
|
|
# Control: the allow ACL line is found and holds the canary, so the
|
|
# absence of the swarm's service names is not a lookup of nothing.
|
|
name = "squid's default allow-list is the self-check canary alone and its base list is the Claude Code hosts, with no swarm service name";
|
|
ok =
|
|
let
|
|
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
|
|
acl = lib.findFirst (l: lib.hasPrefix "acl allowed_dst dstdomain " l) "" (map lib.trim lines);
|
|
baseAcl = lib.findFirst (l: lib.hasPrefix "acl base_dst dstdomain " l) "" (map lib.trim lines);
|
|
in
|
|
acl == "acl allowed_dst dstdomain ${hiveOtel.services.hyperhive.squid.selfcheck.canary}"
|
|
&& baseAcl == "acl base_dst dstdomain api.anthropic.com claude.ai platform.claude.com"
|
|
&& hiveOtel.services.hyperhive.squid.selfcheck.canary == "cache.nixos.org"
|
|
&& !(builtins.elem hiveOtel.services.hyperhive.swarm.forge.domain (lib.splitString " " acl));
|
|
}
|
|
{
|
|
# The ranges are what keeps an unrestricted user on the public internet.
|
|
# Control: the bridge deny and the ACL line are found, so a range missing
|
|
# below is not a lookup of nothing.
|
|
name = "squid's private-destination ACL names every private, shared, multicast and reserved range";
|
|
ok =
|
|
let
|
|
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
|
|
acl = lib.findFirst (l: lib.hasPrefix "acl to_private dst " l) null lines;
|
|
ranges = lib.splitString " " (lib.removePrefix "acl to_private dst " acl);
|
|
in
|
|
acl != null
|
|
&& builtins.elem "acl to_bridge dst ${hiveOtel.services.hyperhive.network.networkCidr}" lines
|
|
&&
|
|
lib.sort lib.lessThan ranges == lib.sort lib.lessThan [
|
|
"10.0.0.0/8"
|
|
"172.16.0.0/12"
|
|
"192.168.0.0/16"
|
|
"100.64.0.0/10"
|
|
"0.0.0.0/8"
|
|
"224.0.0.0/4"
|
|
"240.0.0.0/4"
|
|
"fc00::/7"
|
|
"ff00::/8"
|
|
"::/128"
|
|
];
|
|
}
|
|
{
|
|
# An unrestricted user is let through before the unlisted-name deny and
|
|
# everyone else is held to the listed names after it. Control: both are
|
|
# there, so the ordering below is not a lookup of nothing.
|
|
name = "squid lets an unrestricted user through before the allow-list rules that bind everyone else";
|
|
ok =
|
|
let
|
|
conf = hiveOtel.services.squid.configText;
|
|
lines = lib.splitString "\n" conf;
|
|
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
|
|
list = hiveOtel.services.hyperhive.squid.unrestrictedFile;
|
|
in
|
|
at "http_access allow unrestricted" != null
|
|
&& at "http_access deny !allowed_dst" != null
|
|
&& at "http_access deny !authed" < at "http_access allow base_dst"
|
|
&& at "http_access allow base_dst" < at "http_access allow unrestricted"
|
|
&& at "http_access allow unrestricted" < at "http_access deny !allowed_dst"
|
|
&& at "http_access deny !allowed_dst" < at "http_access allow allowed_dst"
|
|
&& builtins.elem ''acl unrestricted proxy_auth "${list}"'' lines
|
|
&& list == "/var/lib/hive-squid/unrestricted";
|
|
}
|
|
{
|
|
# The list is read when squid parses its configuration, so a change to
|
|
# the file reloads squid. Control: the watched path is the list's own.
|
|
name = "squid is reloaded when its unrestricted-users list changes";
|
|
ok =
|
|
let
|
|
list = hiveOtel.services.hyperhive.squid.unrestrictedFile;
|
|
rules = hiveOtel.systemd.tmpfiles.rules;
|
|
path = hiveOtel.systemd.paths.squid-unrestricted-reload;
|
|
svc = hiveOtel.systemd.services.squid-unrestricted-reload.serviceConfig;
|
|
in
|
|
path.pathConfig.PathChanged == [
|
|
list
|
|
hiveOtel.services.hyperhive.squid.allowedFile
|
|
hiveOtel.services.hyperhive.squid.agentAllowFile
|
|
]
|
|
&& hiveOtel.systemd.services.squid-unrestricted-reload.startLimitIntervalSec == 0
|
|
&& builtins.elem "paths.target" path.wantedBy
|
|
&& lib.hasSuffix "systemctl try-reload-or-restart squid.service" svc.ExecStart
|
|
&& builtins.elem "f ${list} 0640 hive-core squid - -" rules
|
|
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_UNRESTRICTED_FILE == list;
|
|
}
|
|
{
|
|
# The swarm's allow-list reaches squid as a file of names, which starts
|
|
# as this hive's own `allowedDomains` (the canary left out, since the
|
|
# configuration names it), and as an include that holds the per-agent
|
|
# rules. Control: the seed holds the listed name, and the configuration
|
|
# no longer names it, so the absence of the canary and of the list from
|
|
# the seed and the configuration is not a lookup of nothing.
|
|
name = "squid reads the swarm's allow-list from a file seeded with the hive's own names";
|
|
ok =
|
|
let
|
|
sq = hiveOtelAllLocal.services.hyperhive.squid;
|
|
lines = lib.splitString "\n" hiveOtelAllLocal.services.squid.configText;
|
|
rules = hiveOtelAllLocal.systemd.tmpfiles.rules;
|
|
env = hiveOtelAllLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
builtins.elem ''acl allowed_dst dstdomain "${sq.allowedFile}"'' lines
|
|
&& builtins.elem "acl allowed_dst dstdomain ${sq.selfcheck.canary}" lines
|
|
&& !(lib.hasInfix "github.com" hiveOtelAllLocal.services.squid.configText)
|
|
&& sq.allowedFile == "/var/lib/hive-squid/allowed"
|
|
&& sq.agentAllowFile == "/var/lib/hive-squid/agent-allow.conf"
|
|
&& sq.allowedSeedFile.text == "github.com\n"
|
|
&& builtins.elem "C ${sq.allowedFile} 0640 hive-core squid - ${sq.allowedSeedFile}" rules
|
|
&& builtins.elem "f ${sq.agentAllowFile} 0640 hive-core squid - -" rules
|
|
&& env.HIVE_SQUID_ALLOWED_FILE == sq.allowedFile
|
|
&& env.HIVE_SQUID_AGENT_ALLOW_FILE == sq.agentAllowFile
|
|
# The unit's environment holds the seed as a derivation until the unit file
|
|
# is rendered, so it is compared as a path string.
|
|
&& toString env.HIVE_SQUID_ALLOWED_SEED_FILE == toString sq.allowedSeedFile
|
|
&&
|
|
env.HIVE_SQUID_ALWAYS_ALLOWED == "api.anthropic.com claude.ai platform.claude.com cache.nixos.org";
|
|
}
|
|
{
|
|
# The nixpkgs module parses the configuration at build time, which cannot
|
|
# open the include; the parse moves to a check on the same text, so a
|
|
# broken configuration still fails the host build.
|
|
name = "squid's configuration is still parsed at build time, without the include";
|
|
ok =
|
|
!hiveOtel.services.squid.validateConfig
|
|
&& lib.any (d: (d.name or "") == "squid-conf-check") hiveOtel.system.extraDependencies;
|
|
}
|
|
{
|
|
# The helper is the NCSA one, on the file hive-c0re renders. The file
|
|
# exists, empty, before squid starts (the helper exits on a missing one),
|
|
# and the self-check imports its credential without failing when there is
|
|
# none.
|
|
name = "squid authenticates with the NCSA helper on a users file that exists before it starts";
|
|
ok =
|
|
let
|
|
conf = hiveOtel.services.squid.configText;
|
|
lines = lib.splitString "\n" conf;
|
|
users = hiveOtel.services.hyperhive.squid.usersFile;
|
|
rules = hiveOtel.systemd.tmpfiles.rules;
|
|
sc = hiveOtel.systemd.services.squid-selfcheck.serviceConfig;
|
|
in
|
|
lib.any (
|
|
l: lib.hasInfix "/libexec/basic_ncsa_auth ${users}" l && lib.hasPrefix "auth_param basic program " l
|
|
) lines
|
|
&& builtins.elem "acl authed proxy_auth REQUIRED" lines
|
|
&& users == "/var/lib/hive-squid/users"
|
|
&& builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules
|
|
&& builtins.elem "f ${users} 0640 hive-core squid - -" rules
|
|
&& sc.ImportCredential == hiveOtel.services.hyperhive.squid.selfcheck.credentialName
|
|
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users;
|
|
}
|
|
{
|
|
name = "squid's port is open on the bridge interface and on no other";
|
|
ok =
|
|
let
|
|
fw = hiveOtel.networking.firewall;
|
|
in
|
|
builtins.elem 3128 fw.interfaces.${hiveOtel.services.hyperhive.network.bridgeName}.allowedTCPPorts
|
|
&& !(builtins.elem 3128 fw.allowedTCPPorts);
|
|
}
|
|
{
|
|
# The facts that tell an agent this host's squid checks its credential and
|
|
# applies its flag. Presence arm: a hive holding a store identity writes
|
|
# both, and the direct hosts carry the names the brief's destinations
|
|
# resolve under. Absence arm: a hive with no identity renders no users
|
|
# file, so it announces no proxy.
|
|
name = "hive-c0re hands agents the squid address and the hosts that skip it, only with a store identity";
|
|
ok =
|
|
let
|
|
e = hiveOtelAllLocal.systemd.services.hive-c0re.environment;
|
|
s = hiveOtelAllLocal.services.hyperhive;
|
|
hosts = lib.splitString "," e.HIVE_EGRESS_NO_PROXY;
|
|
in
|
|
e.HIVE_SQUID_PROXY == "${s.network.bridgeIp}:3128"
|
|
&& e.HIVE_SQUID_PROXY == s.squid.listen
|
|
&& lib.all (h: lib.elem h hosts) (
|
|
[
|
|
s.network.bridgeIp
|
|
s.swarm.domain
|
|
s.swarm.bao.domain
|
|
s.swarm.nats.domain
|
|
s.swarm.attic.domain
|
|
s.swarm.forge.domain
|
|
s.swarm.matrix.gatewayHost
|
|
s.swarm.victorialogs.domain
|
|
s.swarm.victoriametrics.domain
|
|
s.swarm.otel.domain
|
|
s.otel.collector.host
|
|
s.domain
|
|
]
|
|
++ s.gateway.localNames
|
|
)
|
|
&& !(lib.elem "example.com" hosts)
|
|
&& lib.length hosts == lib.length (lib.unique hosts)
|
|
&& !(hiveOtelBare.systemd.services.hive-c0re.environment ? HIVE_SQUID_PROXY)
|
|
&& !(hiveOtelBare.systemd.services.hive-c0re.environment ? HIVE_EGRESS_NO_PROXY);
|
|
}
|
|
{
|
|
# The store and the queue are in no list the gateway builds, so a hive whose
|
|
# store is elsewhere gets them from their own options. Control: this
|
|
# fixture runs no store, so the gateway does not name it.
|
|
name = "a hive without a local store still lists the store and the queue as direct hosts";
|
|
ok =
|
|
let
|
|
s = hiveOtelLeaf.services.hyperhive;
|
|
hosts = lib.splitString "," hiveOtelLeaf.systemd.services.hive-c0re.environment.HIVE_EGRESS_NO_PROXY;
|
|
in
|
|
!(lib.elem s.swarm.bao.domain s.gateway.localNames)
|
|
&& lib.elem s.swarm.bao.domain hosts
|
|
&& lib.elem s.swarm.nats.domain hosts;
|
|
}
|
|
{
|
|
# Squid's listener and the fact are one value.
|
|
name = "the squid fact is the address squid binds";
|
|
ok =
|
|
let
|
|
s = hiveOtelAllLocal.services.hyperhive;
|
|
in
|
|
lib.elem "http_port ${s.squid.listen}" (
|
|
lib.splitString "\n" hiveOtelAllLocal.services.squid.configText
|
|
);
|
|
}
|
|
];
|
|
in
|
|
runGroup "hive-otel" cases
|