Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/module-eval/builder.nix

507 lines
23 KiB
Nix

# `checks.module-eval-builder` — see ./lib.nix for the shared rationale (why
# this suite exists, naming convention, "evaluates not executes"). The swarm
# builder's container, its default and its queue and IdP registrations; its
# store grants are in ./bao-grants.nix beside every other principal's.
{
pkgs,
lib,
self,
nixosSystem,
shared,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
hiveWithController
runGroup
;
bare = shared.bareWithController;
# On the store's host, which mints the builder's store identity.
withBuilder = hive {
deploy.bao.enable = true;
deploy.builder.enable = true;
swarm.attic.domain = "cache.example.org";
swarm.attic.cache = "grid";
};
servicesHere = shared.swarmServicesWithController;
# Short-circuits on `assertion`, so only a failing one has its message read.
refusedWithoutIdentity =
m:
lib.any (a: !a.assertion && lib.hasInfix "deploy.builder.enable requires" a.message) m.assertions;
# The one fixture that fires the cache assertion: a domain with no cache name.
halfCache = hive {
deploy.bao.enable = true;
deploy.builder.enable = true;
swarm.attic.domain = "cache.example.org";
swarm.attic.cache = null;
};
refusedHalfCache =
m:
lib.any (
a:
!a.assertion && lib.hasInfix "swarm.attic.domain and services.hyperhive.swarm.attic.cache" a.message
) m.assertions;
# Only the option is read, so the host around it is never evaluated.
cacheNamed = name: (hive { swarm.attic.cache = name; }).services.hyperhive.swarm.attic.cache;
natsHere = shared.natsQueueOnly;
autheliaHere = hive {
deploy.bao.enable = true;
deploy.authelia.enable = true;
};
container = withBuilder.containers.swarm-builder;
inner = container.config;
svc = inner.systemd.services.swarm-builder;
env = svc.environment;
hasOnPath = name: s: lib.any (p: lib.getName p == name) s.path;
# A builder that opts out of pushing: both options null.
withBuilderNoCache = hive {
deploy.bao.enable = true;
deploy.builder.enable = true;
swarm.attic.domain = null;
swarm.attic.cache = null;
};
withoutCache = withBuilderNoCache.containers.swarm-builder.config.systemd.services.swarm-builder;
# A builder on a host that doesn't run the cache and names nothing.
builderDefaults = hive {
deploy.bao.enable = true;
deploy.builder.enable = true;
};
defaultsEnv =
builderDefaults.containers.swarm-builder.config.systemd.services.swarm-builder.environment;
# Hosting the shared services runs the swarm's own cache, which names itself.
atticHere = servicesHere;
atticDeploy = atticHere.services.hyperhive.deploy.attic;
atticSwarm = m: m.services.hyperhive.swarm.attic;
atticUnit = atticHere.containers.swarm-attic.config.systemd.services.swarm-attic;
atticSecret = atticHere.systemd.services.swarm-bao-attic-secret;
atticTokens = atticHere.systemd.services.swarm-bao-attic-tokens;
atticVhost = atticHere.services.hyperhive.gateway.vhosts."attic.t.local";
# The cache switched on while the operator overrides the options.
atticByHand = hiveWithController {
deploy.bao.enable = true;
deploy.attic.enable = true;
swarm.attic.domain = "cache.example.org";
swarm.attic.cache = "grid";
};
# An external cache on a host that hosts the shared services: the switch forced
# off, the two options named.
atticExternal = hiveWithController {
deploy.allSwarmServices = true;
deploy.attic.enable = lib.mkForce false;
swarm.attic.domain = "cache.example.org";
swarm.attic.cache = "grid";
};
atticWarnings =
m:
lib.length (
lib.filter (lib.hasInfix "services.hyperhive.deploy.attic.enable is on while") m.warnings
);
hasAtticHost =
m:
m.containers ? swarm-attic
|| m.systemd.services ? swarm-bao-attic-secret
|| m.systemd.services ? swarm-bao-attic-tokens;
# The check flake on the inputs the builder's unit names, called as
# `nix build` calls it. `hyperhive` is entered through the filtered source's
# own `flake.nix`.
builderCheck =
agentModule:
let
src = env.SWARM_BUILDER_HYPERHIVE_FLAKE;
hyperhive = (import "${src}/flake.nix").outputs (
self.inputs
// {
self = hyperhive // {
outPath = src;
inherit (self) inputs;
};
}
);
in
((import "${env.SWARM_BUILDER_CHECK_FLAKE}/flake.nix").outputs {
inherit hyperhive;
agent.nixosModules.default = agentModule;
}).packages.x86_64-linux.check;
cases = [
{
name = "the swarm builder's container exists only where its host enables it";
ok = !(bare.containers ? swarm-builder) && withBuilder.containers ? swarm-builder;
}
{
# The store's host mints the builder's leaf, so the switch needs no
# manual identity step: the pair is set and the module's identity
# assertion, which nothing else reads in this fixture, passes. `nats` is
# the control that the switch is on.
name = "hosting the shared services turns the builder on, with its store identity";
ok =
servicesHere.services.hyperhive.deploy.builder.enable
&& servicesHere.containers ? swarm-builder
&& servicesHere.services.hyperhive.deploy.bao.builderClientCertFile != null
&& servicesHere.services.hyperhive.deploy.bao.builderClientKeyFile != null
&& !(refusedWithoutIdentity servicesHere)
&& servicesHere.services.hyperhive.deploy.bao.enable
&& servicesHere.services.hyperhive.deploy.nats.enable;
}
{
# A config PR is unreviewed input to an evaluation: the container keeps
# its own netns, and reaches the host's nix daemon through its socket
# alone, with no daemon of its own racing the bind.
name = "the builder's container has its own netns and builds only through the host's daemon socket";
ok =
container.privateNetwork
&& inner.services.hyperhive.swarmContainer.privateNetwork
&& container.bindMounts."/nix/var/nix/daemon-socket".hostPath == "/nix/var/nix/daemon-socket"
&& !inner.systemd.services.nix-daemon.enable
&& !inner.systemd.sockets.nix-daemon.enable
&& inner.systemd.globalEnvironment.NIX_REMOTE == "daemon";
}
{
# The daemon holds its store leaf, handed in by nspawn without a copy on
# disk; the forge token is read per build and reaches only `git`.
name = "the builder loads only its store leaf, runs as a dynamic user and carries no token in its unit";
ok =
lib.elem "builder-bao-key" svc.serviceConfig.LoadCredential
&& env.BAO_CLIENT_KEY == "/run/credentials/swarm-builder.service/builder-bao-key"
&& env.SWARM_BUILDER_OIDC_CLIENT_ID == "swarm-builder"
&& lib.elem "--load-credential=builder-bao-key:/var/lib/swarm-bao-pki/builder-key.pem" container.extraFlags
&& svc.serviceConfig.DynamicUser
&& !(lib.any (lib.hasSuffix "TOKEN") (lib.attrNames env));
}
{
# The closure is pushed with `attic` from the unit, whose config file holds
# the push token: it goes in a runtime directory only the dynamic user can
# enter, and the token is in no variable. The host without a cache is the
# control: same unit, none of it, with `git` still on the path to show
# the lookup finds what is there.
name = "the builder pushes to the configured cache with attic from a private runtime directory, and pushes nowhere without one";
ok =
env.SWARM_BUILDER_ATTIC_DOMAIN == "cache.example.org"
&& env.SWARM_BUILDER_ATTIC_CACHE == "grid"
&& hasOnPath "attic" svc
&& svc.serviceConfig.RuntimeDirectory == "swarm-builder"
&& svc.serviceConfig.RuntimeDirectoryMode == "0700"
&& !(lib.any (lib.hasSuffix "TOKEN") (lib.attrNames env))
&& hasOnPath "git" withoutCache
&& !(hasOnPath "attic" withoutCache)
&& !(withoutCache.environment ? SWARM_BUILDER_ATTIC_DOMAIN)
&& !(withoutCache.environment ? SWARM_BUILDER_ATTIC_CACHE);
}
{
# A domain with no cache name would crash-loop the builder at start, so
# evaluation refuses it. `withBuilder`, with both set, is the control: the
# same assertion list holds the entry and it passes.
name = "the builder refuses a cache domain without a cache name";
ok =
refusedHalfCache halfCache
&& !(refusedHalfCache withBuilder)
&& lib.any (a: lib.hasInfix "swarm.attic.domain and" a.message) withBuilder.assertions;
}
{
# attic refuses a cache name outside its rule, which would fail every push.
# `grid` is the control that the lookup reads a name.
name = "the cache name must follow attic's rule";
ok =
cacheNamed "grid" == "grid"
&& cacheNamed "A-b_c+d" == "A-b_c+d"
&& lib.all (n: !(builtins.tryEval (cacheNamed n)).success) [
"grid.v2"
"-grid"
""
(lib.concatStrings (lib.genList (_: "a") 51))
];
}
{
# The minting unit goes inactive after each run, so a start that pulls it
# runs it again and mints a leaf that is missing; an active
# `RemainAfterExit` unit would satisfy the pull without running.
name = "the builder's container runs the leaf-minting unit before every start";
ok =
let
pki = withBuilder.systemd.services.swarm-bao-pki;
c = withBuilder.systemd.services."container@swarm-builder";
in
pki.serviceConfig.Type == "oneshot"
&& !(pki.serviceConfig.RemainAfterExit or false)
&& lib.elem "multi-user.target" pki.wantedBy
&& lib.elem "swarm-bao-pki.service" c.wants
&& lib.elem "swarm-bao-pki.service" c.after;
}
{
# The check builds on the sources this host was deployed with, so a green check means
# the same hyperhive and nixpkgs base.
name = "the builder builds on the hyperhive and nixpkgs this host was deployed with";
ok =
env.SWARM_BUILDER_HYPERHIVE_FLAKE == withBuilder.services.hyperhive.c0re.hyperhiveFlake
&& env.SWARM_BUILDER_NIXPKGS_FLAKE == withBuilder.services.hyperhive.c0re.nixpkgsFlake
&& lib.hasSuffix "-check-flake" env.SWARM_BUILDER_CHECK_FLAKE
&& env.SWARM_BUILDER_SYSTEM == pkgs.stdenv.hostPlatform.system
&& env.SWARM_BUILDER_CONFIG_ORG == "agent-configs"
&& env.SWARM_BUILDER_FORGE_URL == "https://${withBuilder.services.hyperhive.swarm.forge.domain}";
}
{
# The builder evaluates `builtins.mapAttrs (_: a: a.meta)` over this
# output and prints it as JSON. A derivation or a path in `meta` would
# make that evaluation build or copy something; a function would not
# print at all.
name = "the built-in archetypes carry a module and a meta of plain data, and reviewer can run commands and android-dev installs its toolchain and reverse-engineer turns rea on";
ok =
let
plain =
v:
if lib.isDerivation v || lib.isFunction v || builtins.isPath v then
false
else if lib.isAttrs v then
lib.all plain (lib.attrValues v)
else if lib.isList v then
lib.all plain v
else
true;
archetypes = self.hyperhiveArchetypes;
metas = builtins.mapAttrs (_: a: a.meta) archetypes;
reviewer = metas.reviewer.permissions;
in
lib.all (a: a ? module && a.meta.description != "") (lib.attrValues archetypes)
&& plain metas
&& lib.elem "messaging" reviewer
&& lib.elem "inbox" reviewer
&& lib.elem "execution" reviewer
&& !(lib.elem "meta" reviewer)
&&
lib.elem pkgs.android-tools
(archetypes.android-dev.module { inherit pkgs; }).environment.systemPackages
&& archetypes.reverse-engineer.module.services.hyperhive.agent.rea.enable;
}
{
# `docs.enable` reads `docs.source`, whose default builds from the
# `docs/` the `hyperhive` source has none of; an eval error there fails
# this whole group.
name = "the builder's check evaluates a config with the reference docs on";
ok =
lib.hasSuffix ".drv"
(builderCheck {
services.hyperhive.agent.user.name = "a1";
services.hyperhive.agent.docs.enable = true;
}).drvPath;
}
{
# The responder denies a client id it does not know, and a NATS denial
# arrives as a timeout.
name = "the queue's responder grants the builder's client its subject";
ok = lib.hasInfix "--builder-client swarm-builder" natsHere.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart;
}
{
name = "the builder's client is registered as a machine, and the publisher carries its secret";
ok =
let
c = lib.findFirst (
c: c.id == "swarm-builder"
) null autheliaHere.services.hyperhive.swarm.authelia.oidc.clients;
in
c != null
&& c.kind == "machine"
&& lib.hasInfix "secret/swarm/services/swarm-builder/oidc/client" (
autheliaHere.containers.swarm-authelia.config.systemd.services.swarm-secret-publish.script
);
}
{
# The switch is on wherever the shared services are and the host holds the
# cache's leaf.
# The builder on the same host pushes to it. `bare` is the control that no
# cache renders where nothing enables it.
name = "hosting the shared services runs the swarm's cache, and the builder pushes to it";
ok =
atticDeploy.enable
&& atticHere.containers ? swarm-attic
&& atticHere.services.hyperhive.deploy.bao.atticClientCertFile != null
&& atticHere.services.hyperhive.deploy.bao.atticClientKeyFile != null
&& (atticSwarm atticHere).domain == "attic.t.local"
&& (atticSwarm atticHere).cache == "swarm"
&&
atticHere.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_ATTIC_DOMAIN
== "attic.t.local"
&&
atticHere.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_ATTIC_CACHE
== "swarm"
&& !(hasAtticHost bare);
}
{
# The options default to the swarm's own cache on every host, so a
# builder host that runs no cache pushes to it. `atticHere` is the control
# that the cache host reads the same values.
name = "a builder host that doesn't run the cache pushes to the swarm's own by default";
ok =
!(hasAtticHost builderDefaults)
&& (atticSwarm builderDefaults).domain == "attic.t.local"
&& (atticSwarm builderDefaults).cache == "swarm"
&& (atticSwarm bare).domain == (atticSwarm atticHere).domain
&& (atticSwarm bare).cache == (atticSwarm atticHere).cache
&& defaultsEnv.SWARM_BUILDER_ATTIC_DOMAIN == "attic.t.local"
&& defaultsEnv.SWARM_BUILDER_ATTIC_CACHE == "swarm"
&& !(builderDefaults.systemd.services ? swarm-bao-attic-tokens);
}
{
# Names the operator gives win over the cache's own, and the gateway then
# serves the cache under the operator's name. `atticHere` is the control
# that the same lookups find the defaults.
name = "options named by hand win over the cache's defaults";
ok =
atticByHand.services.hyperhive.deploy.attic.enable
&& atticByHand.containers ? swarm-attic
&& (atticSwarm atticByHand).domain == "cache.example.org"
&& (atticSwarm atticByHand).cache == "grid"
&& atticByHand.services.hyperhive.gateway.vhosts ? "cache.example.org"
&& !(atticByHand.services.hyperhive.gateway.vhosts ? "attic.t.local")
&& atticHere.services.hyperhive.gateway.vhosts ? "attic.t.local";
}
{
# The cache running here under a name other than its own warns once, naming
# the fix. `atticHere` (default name) and `atticExternal` (cache off) are
# the controls that the same lookup finds no warning.
name = "the cache running under another name warns once, and the default and an external cache do not";
ok =
atticWarnings atticByHand == 1
&& lib.any (lib.hasInfix "lib.mkForce false") atticByHand.warnings
&& lib.any (lib.hasInfix "cache.example.org") atticByHand.warnings
&& atticWarnings atticHere == 0
&& atticWarnings atticExternal == 0;
}
{
# An operator's external cache on a host that hosts the shared services:
# with the switch forced off nothing of the cache is rendered, and the
# options keep what was named. `atticHere` is the control that the same
# lookups find the cache's pieces where it runs.
name = "with the cache switched off the options are the operator's and no cache runs";
ok =
!atticExternal.services.hyperhive.deploy.attic.enable
&& !(hasAtticHost atticExternal)
&& !(atticExternal.services.hyperhive.gateway.vhosts ? "attic.t.local")
&& (atticSwarm atticExternal).domain == "cache.example.org"
&& (atticSwarm atticExternal).cache == "grid"
&&
atticExternal.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_ATTIC_DOMAIN
== "cache.example.org"
&& hasAtticHost atticHere
&& atticHere.systemd.services ? swarm-bao-attic-secret
&& atticHere.systemd.services ? swarm-bao-attic-tokens;
}
{
# The hive collector's journal allow-list carries the cache's init units
# on a hive host with the cache and no swarm collector (`atticByHand`).
# Where the swarm collector runs too (`atticHere`) it reads the whole
# journal, so the units are not named twice; without the cache
# (`atticExternal`) they are absent. `hive-c0re.service` is the control
# that each list is the one the collector reads.
name = "the hive collector ships the cache's init units only where nothing else reads the host journal";
ok =
let
units = m: m.services.opentelemetry-collector.settings.receivers.journald.units;
initUnits = [
"swarm-bao-attic-secret.service"
"swarm-bao-attic-tokens.service"
];
has = m: lib.all (u: lib.elem u (units m)) initUnits;
hasNone = m: lib.all (u: !(lib.elem u (units m))) initUnits;
in
has atticByHand
&& hasNone atticHere
&& hasNone atticExternal
&& lib.all (m: lib.elem "hive-c0re.service" (units m)) [
atticByHand
atticHere
atticExternal
];
}
{
# Both host units log in as the cache's principal, not the hive's, and
# the container reaches the key through a read-only directory. The
# unit script holds the create calls only in ./lib/attic-init.nix, which a
# script test runs.
name = "the cache's host units use its own store leaf, and the container gets the key read-only";
ok =
atticSecret.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/attic.pem"
&& atticSecret.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/attic-key.pem"
&& atticTokens.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/attic.pem"
&& atticSecret.serviceConfig.Type == "oneshot"
&& lib.elem "container@swarm-attic.service" atticSecret.before
&& lib.elem "container@swarm-attic.service" atticTokens.after
&& lib.elem "swarm-bao-attic-policy.service" atticSecret.after
&& lib.elem "swarm-bao-attic-policy.service" atticTokens.after
&& atticHere.containers.swarm-attic.bindMounts."/run/swarm-attic-secrets".isReadOnly
&& !atticHere.containers.swarm-attic.privateNetwork
&& atticUnit.serviceConfig.EnvironmentFile == "/run/swarm-attic-secrets/server.env"
&& lib.hasInfix "--mode monolithic" atticUnit.serviceConfig.ExecStart
&& atticDeploy.retentionPeriod == "14 days";
}
{
# atticd's journal reaches the swarm collector through the container's own
# forwarder, under the cache's store leaf. The host with the switch forced
# off is the control that nothing of it renders there, and `atticHere`
# that the same lookups find it where the cache runs.
name = "the cache's container forwards its journal to the swarm collector as its own client, and only where the cache runs";
ok =
let
inner = atticHere.containers.swarm-attic;
f = inner.config.services.hyperhive.swarmJournalForwarder;
fetch = inner.config.systemd.services.swarm-journal-forwarder-oidc;
collector = inner.config.systemd.services.opentelemetry-collector;
in
f.enable
&& f.serviceName == "attic"
&& f.clientId == "swarm-attic-collector"
&& f.endpoint == "https://${atticHere.services.hyperhive.swarm.otel.domain}/swarm-attic"
&& f.store.secretPath == "secret/swarm/services/swarm-attic-collector/oidc/client"
&& f.store.clientCertFile == "/run/credentials/swarm-journal-forwarder-oidc.service/attic-bao-cert"
&& lib.elem "attic-bao-key" fetch.serviceConfig.LoadCredential
&& lib.elem "--load-credential=attic-bao-cert:/var/lib/swarm-bao-pki/attic.pem" inner.extraFlags
&& !(lib.elem "--link-journal=host" inner.extraFlags)
&& collector.enable
&&
atticHere.services.hyperhive.swarm.otel.serviceProducers.swarm-attic.clientId
== "swarm-attic-collector"
&& lib.hasInfix "services/swarm-attic-collector/oidc/client" atticHere.systemd.services.swarm-bao-attic-policy.script
&& !(atticExternal.containers ? swarm-attic)
&& atticExternal.services.hyperhive.swarm.otel.serviceProducers ? swarm-attic;
}
{
# attic checks its own tokens and a nix client cannot follow a login
# redirect, so the vhost carries no `auth_request`; the tokens ride in a
# header, so it is TLS only; and a store path is one long upload.
name = "the cache's vhost is TLS only, unauthenticated at the gateway and streams uploads, and its name is a swarm service name on every host";
ok =
let
names = m: m.services.hyperhive.swarm.serviceDomains;
in
atticVhost.forceSSL
&& atticVhost.locations."/".proxyPass == "http://127.0.0.1:8910"
&& !(lib.hasInfix "auth_request" atticVhost.locations."/".extraConfig)
&& lib.hasInfix "client_max_body_size 0" atticVhost.locations."/".extraConfig
&& lib.hasInfix "proxy_request_buffering off" atticVhost.locations."/".extraConfig
&& lib.elem "attic.t.local" (names atticHere)
&& names atticHere == names bare
&& names atticExternal == names bare
&& !(lib.elem "cache.example.org" (names atticExternal));
}
];
in
runGroup "builder" cases