1d3ab2fae5
the gateway reaches atticd over a unix socket a relay in the container forwards, and the host unit that creates the cache dials its public name.
1932 lines
81 KiB
Nix
1932 lines
81 KiB
Nix
# `checks.module-eval-swarm-services-switch` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
bridgePortsOrNone
|
|
swarmServiceEnables
|
|
;
|
|
|
|
# A dedicated services host with hives elsewhere: the swarm-services
|
|
# toggle and no hive, the recipe docs/swarm/services.md gives.
|
|
swarmServicesOnly = shared.servicesHere;
|
|
|
|
# A services host with hives elsewhere and the store elsewhere too: no hive
|
|
# and no store identity minted here.
|
|
swarmServicesOnlyNoStore = swarmServicesBaoElsewhere;
|
|
|
|
# A dedicated secret-store host: no hive, and the store's browser UI the
|
|
# only swarm name its gateway fronts.
|
|
storeOnlyNoHive = shared.baoOnly;
|
|
|
|
# The homeserver on a host without a hive.
|
|
matrixNoHive = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.matrix.enable = true;
|
|
};
|
|
|
|
# The same switch with every service it owns placed elsewhere, so the only
|
|
# thing that can enable the gateway/resolver/bridge here is that toggle's
|
|
# own module — each service module asserts them from its own `enable`.
|
|
swarmServicesSwitchAlone = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.allSwarmServices = true;
|
|
deploy.matrix.enable = false;
|
|
deploy.authelia.enable = false;
|
|
deploy.nats.enable = false;
|
|
deploy.swarm-otel.enable = false;
|
|
deploy.victoriametrics.enable = false;
|
|
deploy.grafana.enable = false;
|
|
deploy.victorialogs.enable = false;
|
|
deploy.bao.enable = false;
|
|
deploy.forgejo.enable = false;
|
|
};
|
|
|
|
# Ports asked for on such a host. The request is the whole condition: the
|
|
# firewall hole exists because an operator named a port, not because the
|
|
# hive is running, and an agent reaching a host service is a claim about
|
|
# the host's own listeners either way.
|
|
exposedPortsNoHive = hive {
|
|
deploy.hive-controller.enable = false;
|
|
network.exposeHostPorts = [ 5432 ];
|
|
};
|
|
|
|
# The swarm UI on the controller's host — the control the absence arm
|
|
# needs, since nothing else in this suite renders this vhost and an arm
|
|
# saying "it is not there" would hold just as well if it were never there.
|
|
# Package stubbed per this file's header: the vhost roots at it. Set through
|
|
# its old `deploy.swarm-ui.package` spelling, which must reach the vhost.
|
|
#
|
|
# Also sets both ignored UI switches to `false`: the vhost must render
|
|
# regardless, and each must evaluate and warn. Neither
|
|
# is read anywhere else, so they change nothing the other arms see.
|
|
swarmUiHere = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.swarm-ui.package = pkgs.emptyDirectory;
|
|
deploy.swarm-ui.enable = false;
|
|
swarm.ui.enable = false;
|
|
};
|
|
|
|
# The swarm's shared services hosted HERE without the all-local mode — a
|
|
# services box with hives elsewhere, the shape ./host-modules/
|
|
# swarm-required-services.nix documents the switch for.
|
|
swarmServicesHere = shared.swarmServicesWithController;
|
|
|
|
# Same, with the operator opting out of the classifier and the builder,
|
|
# which the switch defaults on and everything else it owns stays on.
|
|
swarmServicesOptedOut = hiveWithController {
|
|
deploy.allSwarmServices = true;
|
|
deploy.review-classifier.enable = false;
|
|
deploy.builder.enable = false;
|
|
};
|
|
|
|
# Same, with the claude-code build the agents run pinned on this host. Any
|
|
# package serves: only the path the containers are pointed at is read.
|
|
swarmServicesPinnedClaude = hiveWithController {
|
|
deploy.allSwarmServices = true;
|
|
deploy.hive-controller.claudeCodePackage = pkgs.hello;
|
|
};
|
|
|
|
# Same, with one of those services placed on another host. Every derivation
|
|
# in that module is `mkDefault` so this stays expressible.
|
|
swarmServicesBaoElsewhere = hive {
|
|
deploy.allSwarmServices = true;
|
|
deploy.bao.enable = false;
|
|
};
|
|
|
|
# The swarm-wide forge objects are the controller's (forge/objects.rs), so
|
|
# the mirror list and the org avatar go to its unit, not hive-c0re's.
|
|
aMirror = {
|
|
upstream = "https://example.invalid/tool";
|
|
dest = "mirrors/tool";
|
|
};
|
|
controllerWithMirrors = hiveWithController {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
deploy.forgejo.actionsFromSelf = false;
|
|
deploy.swarm-controller.configOrgAvatarPng = "/etc/marker-avatar.png";
|
|
};
|
|
mirrorsNoController = hive {
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
};
|
|
forgeOnly = shared.forgeHere;
|
|
forgeWithController = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
};
|
|
controllerWithSelfActions = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
deploy.forgejo.actionsFromSelf = true;
|
|
};
|
|
|
|
# The controller with every shared service on another host, and the same
|
|
# controller hosting them all: the swarm UI's links must not tell the two
|
|
# apart.
|
|
inherit (shared) controllerAlone;
|
|
controllerWithServices = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.allSwarmServices = true;
|
|
};
|
|
|
|
# Each shared service with a web UI, and the link it must have: the label
|
|
# and the swarm-level domain the URL is built from.
|
|
swarmServiceLinkDomains =
|
|
let
|
|
s = controllerAlone.services.hyperhive.swarm;
|
|
in
|
|
{
|
|
Authelia = s.authelia.domain;
|
|
Grafana = s.grafana.domain;
|
|
Metrics = s.victoriametrics.domain;
|
|
Logs = s.victorialogs.domain;
|
|
Matrix = s.matrix.gatewayHost;
|
|
Bao = s.bao.ui.domain;
|
|
Forge = s.forge.domain;
|
|
};
|
|
swarmServiceLinksOf =
|
|
cfg:
|
|
lib.filter (
|
|
l: swarmServiceLinkDomains ? ${l.label}
|
|
) cfg.services.hyperhive.deploy.swarm-controller.links;
|
|
|
|
# Every service container at once: the services-only host plus the CI
|
|
# runner. The CI runner, the review classifier, the notification router, the
|
|
# builder, the claude login minter, the metrics store, the homeserver,
|
|
# authelia, Grafana, the collector and the cache are the eleven containers on
|
|
# a netns of their own.
|
|
serviceContainersWithCi = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.allSwarmServices = true;
|
|
deploy.ci-runner.enable = true;
|
|
};
|
|
serviceContainers = serviceContainersWithCi.containers;
|
|
|
|
bootWait = import ./boot-wait.nix { inherit lib; };
|
|
|
|
# Every container hyperhive declares, by name: a generic walk over
|
|
# `containers` would also judge an operator's own.
|
|
hyperhiveContainers = [
|
|
"hive-forge"
|
|
"hive-matrix"
|
|
"swarm-attic"
|
|
"swarm-authelia"
|
|
"swarm-bao"
|
|
"swarm-builder"
|
|
"swarm-ci-runner"
|
|
"swarm-claude-login"
|
|
"swarm-grafana"
|
|
"swarm-nats"
|
|
"swarm-notify-router"
|
|
"swarm-otel"
|
|
"swarm-review-classifier"
|
|
"swarm-victorialogs"
|
|
"swarm-victoriametrics"
|
|
];
|
|
|
|
# A container's start limit, in seconds.
|
|
startLimitOf = name: bootWait.spanSeconds serviceContainers.${name}.timeoutStartSec;
|
|
|
|
# The boot-wait units of container `name` whose start job can last `limit`
|
|
# (default: the container's own start limit) or more.
|
|
reachingLimit =
|
|
name:
|
|
{
|
|
limit ? startLimitOf name,
|
|
}:
|
|
bootWait.overBudget {
|
|
guest = serviceContainers.${name}.config;
|
|
inherit limit;
|
|
};
|
|
|
|
# Boot-wait units that state no `TimeoutStartSec` and are let through, each
|
|
# group with the reason. A unit that states a budget is judged on it whatever
|
|
# this says. A new unit that states none fails the check until it states one
|
|
# or is added here with its reason. A unit listed here also adds 0 s to the
|
|
# chain of units ordered one after another that `bootWaitChainOffenders` sums.
|
|
unstatedAllowed = [
|
|
{
|
|
why = "NixOS's own units: they change local files, interfaces and rules, and wait on no peer";
|
|
units = [
|
|
"dhcpcd.service"
|
|
"firewall.service"
|
|
"linger-users.service"
|
|
"logrotate-checkconf.service"
|
|
"network-addresses-eth0.service"
|
|
"network-local-commands.service"
|
|
"nscd.service"
|
|
"resolvconf.service"
|
|
"suid-sgid-wrappers.service"
|
|
];
|
|
}
|
|
{
|
|
why = "hyperhive's local oneshots: they write the resolver and trust files, set up local state or initialise the store's own token, and wait on no peer";
|
|
units = [
|
|
"forgejo-gpg-init.service"
|
|
"forgejo-secrets.service"
|
|
"swarm-bao-token.service"
|
|
"swarm-bridge-dns.service"
|
|
"victoriametrics-state-migrate.service"
|
|
];
|
|
suffixes = [
|
|
"-ca-bundle.service"
|
|
"-ca-bundle-refresh.service"
|
|
];
|
|
}
|
|
{
|
|
why = "the service's own start: a daemon that does not come up in time is a container that is not useful, whatever its budget";
|
|
units = [
|
|
"authelia-swarm.service"
|
|
"forgejo.service"
|
|
"gitea-runner-swarm.service"
|
|
"grafana.service"
|
|
"nats.service"
|
|
"opentelemetry-collector.service"
|
|
"tuwunel.service"
|
|
"victorialogs.service"
|
|
"victoriametrics.service"
|
|
];
|
|
}
|
|
];
|
|
allowedUnstated =
|
|
key:
|
|
lib.any (
|
|
g: lib.elem key (g.units or [ ]) || lib.any (s: lib.hasSuffix s key) (g.suffixes or [ ])
|
|
) unstatedAllowed;
|
|
|
|
# `<container>: <unit> <seconds>s (<source>) >= <limit>s`, over every named
|
|
# container.
|
|
bootWaitOffenders = lib.concatMap (
|
|
name:
|
|
map (
|
|
o: "${name}: ${o.key} ${toString o.seconds}s (${o.source}) >= ${toString (startLimitOf name)}s"
|
|
) (lib.filter (o: o.source == "stated" || !allowedUnstated o.key) (reachingLimit name { }))
|
|
) hyperhiveContainers;
|
|
|
|
# Container `name`'s longest chain of boot-wait units ordered one after
|
|
# another, the `unstatedAllowed` ones adding 0 s.
|
|
chainOf =
|
|
name:
|
|
bootWait.longestChain {
|
|
guest = serviceContainers.${name}.config;
|
|
weightless = o: o.source != "stated" && allowedUnstated o.key;
|
|
};
|
|
|
|
# `<container>: <chain>` where the chain reaches the container's start limit
|
|
# with more than one unit in it (a single unit is `bootWaitOffenders`'), or
|
|
# `<container>: ordering cycle <unit> -> … -> <unit>`.
|
|
bootWaitChainOffenders = lib.concatMap (
|
|
name:
|
|
let
|
|
chain = chainOf name;
|
|
limit = startLimitOf name;
|
|
in
|
|
if chain.cycle != [ ] then
|
|
[
|
|
"${name}: ordering cycle ${lib.concatStringsSep " -> " (chain.cycle ++ [ (lib.head chain.cycle) ])}"
|
|
]
|
|
else if lib.length chain.units > 1 && (chain.seconds == null || chain.seconds >= limit) then
|
|
[ "${name}: ${bootWait.describeChain chain} >= ${toString limit}s" ]
|
|
else
|
|
[ ]
|
|
) hyperhiveContainers;
|
|
|
|
# The metrics store on a build older than its socket needs. Only its
|
|
# `assertions` are read, and the package is a stub carrying a version.
|
|
oldMetricsStore = hive {
|
|
deploy.victoriametrics.enable = true;
|
|
deploy.victoriametrics.package = pkgs.emptyDirectory // {
|
|
version = "1.152.0";
|
|
};
|
|
};
|
|
# Failing assertions first, then their messages: a message is only
|
|
# well-defined when its assertion fails (nixpkgs' fileSystems one throws
|
|
# otherwise).
|
|
failedMetricsStoreVersion =
|
|
host:
|
|
lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") (
|
|
map (a: a.message) (lib.filter (a: !a.assertion) host.assertions)
|
|
);
|
|
cases = [
|
|
{
|
|
# `lib.all` over an empty set holds vacuously, so the roster is counted
|
|
# before it is read: a helper that lost a member would otherwise turn
|
|
# this case green by measuring nothing.
|
|
name = "hosting the swarm's shared services turns on every service that switch owns";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables swarmServicesHere;
|
|
in
|
|
lib.length (lib.attrNames es) == 8 && lib.all lib.id (lib.attrValues es);
|
|
}
|
|
{
|
|
# The container is the control that the toggle reaches the module, not
|
|
# only the option; `nats` is the control that the switch is on at all.
|
|
name = "hosting the swarm's shared services turns on the review classifier";
|
|
ok =
|
|
swarmServicesHere.services.hyperhive.deploy.review-classifier.enable
|
|
&& swarmServicesHere.containers ? swarm-review-classifier
|
|
&& swarmServicesHere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The default yields to an operator who has spoken. The same fixture
|
|
# without the line is the control.
|
|
name = "an explicit false keeps the review classifier off where the shared services are hosted";
|
|
ok =
|
|
!swarmServicesOptedOut.services.hyperhive.deploy.review-classifier.enable
|
|
&& !(swarmServicesOptedOut.containers ? swarm-review-classifier)
|
|
&& swarmServicesOptedOut.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# Same as the classifier's case above, for the builder: the opt-out is
|
|
# the only difference from `swarmServicesHere`, which has the container.
|
|
name = "an explicit false keeps the swarm builder off where the shared services are hosted";
|
|
ok =
|
|
!swarmServicesOptedOut.services.hyperhive.deploy.builder.enable
|
|
&& !(swarmServicesOptedOut.containers ? swarm-builder)
|
|
&& swarmServicesHere.containers ? swarm-builder
|
|
&& swarmServicesOptedOut.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The classifier's module asserts a store identity that only the store's
|
|
# host mints, so with the store placed elsewhere the default stays off
|
|
# rather than failing that host's eval. `nats` is the control.
|
|
name = "the review classifier stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.review-classifier.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-review-classifier)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The builder's module asserts its own store identity, so the same
|
|
# holds for it. `nats` is the control.
|
|
name = "the swarm builder stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.builder.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-builder)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# Static, not `DynamicUser`: the user is declared in the container and
|
|
# the unit names it. `nats` is the control that the switch is on at all.
|
|
name = "hosting the swarm's shared services runs the claude login minter in its container as a static user";
|
|
ok =
|
|
let
|
|
c = swarmServicesHere.containers.swarm-claude-login.config;
|
|
sc = c.systemd.services.swarm-claude-login.serviceConfig;
|
|
in
|
|
swarmServicesHere.services.hyperhive.deploy.claude-login.enable
|
|
&& c.users.users.swarm-claude-login.isSystemUser
|
|
&& sc.User == "swarm-claude-login"
|
|
&& !(sc.DynamicUser or false)
|
|
&& swarmServicesHere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The minter's module asserts the store identity the store's host mints,
|
|
# as the classifier's does.
|
|
name = "the claude login minter stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.claude-login.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-claude-login)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# Same shape as the minter's: a static user in its own container. `nats`
|
|
# is the control that the switch is on at all.
|
|
name = "hosting the swarm's shared services runs the notification router in its container as a static user";
|
|
ok =
|
|
let
|
|
c = swarmServicesHere.containers.swarm-notify-router.config;
|
|
sc = c.systemd.services.swarm-notify-router.serviceConfig;
|
|
in
|
|
swarmServicesHere.services.hyperhive.deploy.notify-router.enable
|
|
&& c.users.users.swarm-notify-router.isSystemUser
|
|
&& sc.User == "swarm-notify-router"
|
|
&& !(sc.DynamicUser or false)
|
|
&& swarmServicesHere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# dhcpcd without IPv4LL, waiting for IPv4: `network-online` is a lease.
|
|
# The router sets no `nohook` itself and authelia does; `nohook` is the
|
|
# control that the lookup finds `extraConfig`. The queue shares the host
|
|
# netns and keeps the default.
|
|
name = "private-netns service containers wait for a DHCP lease and the forwarder fetch waits for it";
|
|
ok =
|
|
let
|
|
cfgOf = name: swarmServicesHere.containers.${name}.config;
|
|
leased =
|
|
name:
|
|
let
|
|
d = (cfgOf name).networking.dhcpcd;
|
|
in
|
|
lib.hasInfix "noipv4ll" d.extraConfig && d.wait == "ipv4";
|
|
fetch = (cfgOf "swarm-builder").systemd.services.swarm-journal-forwarder-oidc;
|
|
in
|
|
leased "swarm-notify-router"
|
|
&& leased "swarm-authelia"
|
|
&& lib.hasInfix "nohook resolv.conf" (cfgOf "swarm-authelia").networking.dhcpcd.extraConfig
|
|
&& !(lib.hasInfix "noipv4ll" (cfgOf "swarm-nats").networking.dhcpcd.extraConfig)
|
|
&& (cfgOf "swarm-nats").networking.dhcpcd.wait == "any"
|
|
&& lib.elem "network-online.target" fetch.after
|
|
&& lib.elem "network-online.target" fetch.wants;
|
|
}
|
|
{
|
|
name = "the notification router stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.notify-router.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-notify-router)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The cache's units assert the store identity the store's host mints, as
|
|
# the router's do; its options keep the same defaults.
|
|
name = "the swarm's cache stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.attic.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-attic)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.swarm.attic.domain == "attic.t.local"
|
|
&& swarmServicesHere.services.hyperhive.deploy.attic.enable
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The CLI is the one the agents run, not the container's own nixpkgs'.
|
|
# Unpinned is the control: it falls back to the container's own `claude-code`.
|
|
# serve.json is what hive-c0re hands every agent's `HYPERHIVE_CLAUDE_CODE_PATH` from.
|
|
name = "the agents, the claude login minter and the review classifier run the claude-code this host pins";
|
|
ok =
|
|
let
|
|
claudeOf =
|
|
host: container: unit: var:
|
|
host.containers.${container}.config.systemd.services.${unit}.environment.${var};
|
|
pinned = "${pkgs.hello}/bin/claude";
|
|
servedClaude =
|
|
host:
|
|
(builtins.fromJSON (
|
|
builtins.unsafeDiscardStringContext host.environment.etc."hyperhive/serve.json".text
|
|
)).claude_code_path;
|
|
in
|
|
servedClaude swarmServicesPinnedClaude == "${pkgs.hello}"
|
|
&& servedClaude swarmServicesHere == null
|
|
&&
|
|
claudeOf swarmServicesPinnedClaude "swarm-claude-login" "swarm-claude-login"
|
|
"SWARM_CLAUDE_LOGIN_CLAUDE" == pinned
|
|
&&
|
|
claudeOf swarmServicesPinnedClaude "swarm-review-classifier" "swarm-review-classifier"
|
|
"SWARM_REVIEW_CLASSIFIER_CLAUDE" == pinned
|
|
&& lib.hasInfix "claude-code" (
|
|
claudeOf swarmServicesHere "swarm-claude-login" "swarm-claude-login" "SWARM_CLAUDE_LOGIN_CLAUDE"
|
|
)
|
|
&& lib.hasInfix "claude-code" (
|
|
claudeOf swarmServicesHere "swarm-review-classifier" "swarm-review-classifier"
|
|
"SWARM_REVIEW_CLASSIFIER_CLAUDE"
|
|
);
|
|
}
|
|
{
|
|
# The switch fills in for an operator who has not spoken and yields to
|
|
# one who has — that is what keeps a shared service placeable on a host
|
|
# of its own. A plain assignment or `mkForce` would satisfy both cases
|
|
# above and break this one. `nats` is the control: without it the case
|
|
# also passes on a fixture where nothing came on at all.
|
|
name = "placing one shared service elsewhere survives the switch that would enable it";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables swarmServicesBaoElsewhere;
|
|
in
|
|
!es.bao && es.nats;
|
|
}
|
|
{
|
|
# The controller sits on the OTHER tier: `singleHostSwarm` places it
|
|
# (./host-modules/local-defaults.nix) and swarm-ui follows the
|
|
# controller. Pinned so that moving a service between tiers is a
|
|
# decision someone makes rather than a merge nobody reads.
|
|
name = "hosting the swarm's shared services does not make a hive the swarm's control plane";
|
|
ok =
|
|
!swarmServicesHere.services.hyperhive.deploy.swarm-controller.enable
|
|
&& !(swarmServicesHere.services.nginx.virtualHosts ? "t.local");
|
|
}
|
|
{
|
|
# Control for the arm above, and the one place this slice is not inert:
|
|
# the ports are opened because they were named, on a host that never
|
|
# turned the hive on. The bridge firewall is the host's own, so there is
|
|
# nothing here for the hive toggle to have been protecting.
|
|
name = "a named exposeHostPorts opens its bridge port on the host's own say-so";
|
|
ok = builtins.elem 5432 (bridgePortsOrNone exposedPortsNoHive);
|
|
}
|
|
{
|
|
# The default server name is the swarm domain, so the apex answers
|
|
# discovery from `gatewayHost` alone: this fixture runs no homeserver and
|
|
# no hive. The URLs carry no port, and `m.server` names 443 because the
|
|
# spec reads a bare host as 8448. The `/` arm is the presence control on
|
|
# the vhost; the pinned counterpart is in ./matrix-core.nix.
|
|
name = "the swarm apex serves both matrix discovery routes under the default server name";
|
|
ok =
|
|
let
|
|
l = swarmUiHere.services.nginx.virtualHosts."t.local".locations;
|
|
in
|
|
l ? "/"
|
|
&& lib.hasInfix ''{"m.homeserver":{"base_url":"https://chat.t.local"}}'' (
|
|
l."= /.well-known/matrix/client".extraConfig or ""
|
|
)
|
|
&& lib.hasInfix ''{"m.server":"chat.t.local:443"}'' (
|
|
l."= /.well-known/matrix/server".extraConfig or ""
|
|
)
|
|
&& !(lib.hasInfix "auth_request" l."= /.well-known/matrix/client".extraConfig)
|
|
&& !(lib.hasInfix "auth_request" l."= /.well-known/matrix/server".extraConfig);
|
|
}
|
|
{
|
|
name = "the old swarm UI package spelling sets the controller's UI package, which the vhost serves";
|
|
ok =
|
|
let
|
|
uiPackage = swarmUiHere.services.hyperhive.deploy.swarm-controller.ui.package;
|
|
in
|
|
uiPackage == pkgs.emptyDirectory
|
|
&& swarmUiHere.services.nginx.virtualHosts."t.local".locations."/".root == "${uiPackage}"
|
|
&& lib.any (
|
|
w:
|
|
lib.hasInfix "services.hyperhive.deploy.swarm-ui.package" w
|
|
&& lib.hasInfix "services.hyperhive.deploy.swarm-controller.ui.package" w
|
|
) swarmUiHere.warnings;
|
|
}
|
|
{
|
|
# Control: `swarmServicesHere` sets neither ignored switch.
|
|
name = "a definition of an ignored swarm UI switch evaluates and warns, naming the option and the controller";
|
|
ok =
|
|
let
|
|
warned =
|
|
m: path:
|
|
lib.any (
|
|
w:
|
|
lib.hasInfix "services.hyperhive.${path} no longer has any effect" w
|
|
&& lib.hasInfix "swarm-ui now runs wherever swarm-controller runs" w
|
|
) m.warnings;
|
|
paths = [
|
|
"deploy.swarm-ui.enable"
|
|
"swarm.ui.enable"
|
|
];
|
|
in
|
|
lib.all (warned swarmUiHere) paths && !(lib.any (warned swarmServicesHere) paths);
|
|
}
|
|
{
|
|
# The swarm-services toggle enables them explicitly, from its own
|
|
# module rather than from any of their defaults.
|
|
name = "the swarm-services toggle turns on the gateway, resolver and bridge by itself";
|
|
ok =
|
|
swarmServicesSwitchAlone.services.hyperhive.gateway.enable
|
|
&& swarmServicesSwitchAlone.services.hyperhive.gateway.dns.enable
|
|
&& swarmServicesSwitchAlone.services.hyperhive.network.enable;
|
|
}
|
|
{
|
|
# No hive, so no hive leaf, no hive identity and no hive-domain vhost:
|
|
# the default certificate is the swarm-services leaf, or a throwaway
|
|
# pair until it lands. A host fronting many swarm names waits for that
|
|
# leaf; a store-only host does not. The control is a hive host with
|
|
# the same services.
|
|
name = "a gateway host without a hive serves the swarm-services leaf as its default cert and has no hive-leaf unit";
|
|
ok =
|
|
let
|
|
# Every hive-leaf reference a unit could carry: the leaf files,
|
|
# the identity's file and subject. The store's own per-hive role
|
|
# unit is left out: it writes roles for the hives in the swarm
|
|
# directory, which live elsewhere.
|
|
hiveLeafRefs = [
|
|
"gateway.pem"
|
|
"gateway-key.pem"
|
|
"hive-leaf"
|
|
"swarm-bao-hive-leaf-"
|
|
];
|
|
noHiveLeafAnywhere =
|
|
m:
|
|
let
|
|
units = removeAttrs m.systemd.services [ "swarm-bao-hive-leaf-policy" ];
|
|
texts = lib.concatMap (u: [
|
|
(u.script or "")
|
|
(builtins.toJSON (u.environment or { }))
|
|
]) (lib.attrValues units);
|
|
in
|
|
!(lib.any (t: lib.any (r: lib.hasInfix r t) hiveLeafRefs) texts)
|
|
&& m.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
|
|
&& noHiveSurface m;
|
|
# No hive-domain or dashboard vhost, and no vhost on the host carrying the
|
|
# hive's locations: c0re's `/health/`, the agent routes, swagger,
|
|
# or the hive domain's matrix discovery and `/matrix/` redirect.
|
|
noHiveSurface =
|
|
m:
|
|
let
|
|
vh = m.services.nginx.virtualHosts;
|
|
in
|
|
!(vh ? "h1.t.local")
|
|
&& !(vh ? "h1.hives.t.local")
|
|
&& !(lib.any (
|
|
v:
|
|
lib.any (l: (v.locations or { }) ? ${l}) [
|
|
"/health/"
|
|
"/agent/"
|
|
"/api/docs/"
|
|
"/matrix/"
|
|
"= /.well-known/matrix/client"
|
|
"= /.well-known/matrix/server"
|
|
]
|
|
|| lib.hasInfix "agents.conf" (v.extraConfig or "")
|
|
) (lib.attrValues vh));
|
|
noHive =
|
|
m:
|
|
let
|
|
s = m.systemd.services;
|
|
in
|
|
!(s ? hive-tls-resign) && !(m.systemd.timers ? hive-tls-resign) && noHiveLeafAnywhere m;
|
|
waitsForLeaf =
|
|
m:
|
|
let
|
|
s = m.systemd.services;
|
|
in
|
|
lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.requiredBy
|
|
&& lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.before;
|
|
# The cert it installs on the default and swarm-services paths:
|
|
# the services leaf when present.
|
|
importsServicesLeaf =
|
|
m:
|
|
let
|
|
imp = m.systemd.services.hive-gateway-self-signed-cert;
|
|
in
|
|
lib.hasInfix "cert=/var/lib/hive-tls/swarm-services.pem" imp.script
|
|
&& lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/cert.pem'' imp.script
|
|
&& lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/swarm-services.pem'' imp.script;
|
|
# Without it: a throwaway pair where the gateway does not wait for
|
|
# the leaf, a failure naming the issuing unit where it does.
|
|
standsInThrowaway =
|
|
m:
|
|
let
|
|
imp = m.systemd.services.hive-gateway-self-signed-cert;
|
|
in
|
|
lib.hasInfix "openssl req -x509" imp.script
|
|
&& lib.hasInfix "cert=/var/lib/hive-gateway/tls/fallback.pem" imp.script
|
|
&& lib.any (p: lib.hasPrefix "openssl" (p.pname or (p.name or ""))) imp.path;
|
|
failsWithoutLeaf =
|
|
m:
|
|
let
|
|
s = m.systemd.services.hive-gateway-self-signed-cert.script;
|
|
in
|
|
lib.hasInfix "swarm-services-cert.service issues it" s
|
|
&& lib.hasInfix "exit 1" s
|
|
&& !(lib.hasInfix "fallback.pem" s);
|
|
store = storeOnlyNoHive;
|
|
hs = swarmServicesHere.systemd.services;
|
|
in
|
|
noHive swarmServicesOnly
|
|
&& noHive swarmServicesOnlyNoStore
|
|
&& noHive store
|
|
&& importsServicesLeaf swarmServicesOnly
|
|
&& importsServicesLeaf swarmServicesOnlyNoStore
|
|
&& importsServicesLeaf store
|
|
# Fronting many swarm names: the gateway waits for their leaf, and
|
|
# its import fails without it.
|
|
&& waitsForLeaf swarmServicesOnly
|
|
&& waitsForLeaf swarmServicesOnlyNoStore
|
|
&& failsWithoutLeaf swarmServicesOnly
|
|
&& failsWithoutLeaf swarmServicesOnlyNoStore
|
|
&& standsInThrowaway store
|
|
&& !(failsWithoutLeaf store)
|
|
# The store's UI alone: its nginx carries the passthrough to the
|
|
# store, so it starts on the fallback with the store sealed.
|
|
&&
|
|
store.services.hyperhive.deploy.hive-controller.tls.localServiceDomains
|
|
== [ store.services.hyperhive.swarm.bao.ui.domain ]
|
|
&& !(waitsForLeaf store)
|
|
# The homeserver without a hive: no hive-domain vhost at all, and the
|
|
# homeserver's own vhost still proxies `/_matrix/`.
|
|
&& noHive matrixNoHive
|
|
&& (
|
|
let
|
|
gh = matrixNoHive.services.hyperhive.swarm.matrix.gatewayHost;
|
|
vh = matrixNoHive.services.nginx.virtualHosts;
|
|
in
|
|
gh != null && vh ? ${gh} && lib.any (lib.hasPrefix "/_matrix") (lib.attrNames vh.${gh}.locations)
|
|
)
|
|
&& !(store.services.nginx.virtualHosts ? "h1.t.local")
|
|
&& lib.hasInfix "want_svc=1" store.systemd.services.swarm-services-cert.script
|
|
# Control: a hive host with the same services keeps the hive leaf,
|
|
# its unit and identity, the dashboard's vhost and the hive-domain one.
|
|
&& hs ? hive-tls-resign
|
|
&& lib.hasInfix "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" hs.hive-gateway-self-signed-cert.script
|
|
&& swarmServicesHere.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile != null
|
|
&& (
|
|
let
|
|
vh = swarmServicesHere.services.nginx.virtualHosts;
|
|
l = vh."h1.hives.t.local".locations;
|
|
in
|
|
l ? "/api/" && l ? "/health/" && vh."h1.t.local".locations ? "/"
|
|
)
|
|
&& !(noHiveLeafAnywhere swarmServicesHere);
|
|
}
|
|
{
|
|
# Each service is gated on its own deployment toggle alone, so a host
|
|
# that runs no hive still runs every shared service the switch turns
|
|
# on, plus the plumbing and the self-signed CA they are served through.
|
|
# Nothing starts the hive's own daemons. `hive-c0re` is checked by what
|
|
# starts it rather than by key: ../host-modules/hive-network.nix's bridge
|
|
# block writes to its environment wherever the bridge is up.
|
|
name = "a services-only host runs the swarm's shared services and no hive";
|
|
ok =
|
|
let
|
|
c = swarmServicesOnly.containers;
|
|
s = swarmServicesOnly.systemd.services;
|
|
h = swarmServicesOnly.services.hyperhive;
|
|
in
|
|
lib.all (m: c ? ${m}) [
|
|
"swarm-authelia"
|
|
"swarm-bao"
|
|
"swarm-grafana"
|
|
"swarm-victorialogs"
|
|
"swarm-victoriametrics"
|
|
"swarm-otel"
|
|
"swarm-nats"
|
|
"hive-matrix"
|
|
"hive-forge"
|
|
]
|
|
&& h.gateway.enable
|
|
&& h.gateway.dns.enable
|
|
&& h.network.enable
|
|
&& s ? hive-tls-ca
|
|
&& s ? swarm-bao-pki
|
|
&& (s.hive-c0re.wantedBy or [ ]) == [ ]
|
|
&& !(swarmServicesOnly.systemd.sockets ? hive-c0re)
|
|
&& !(swarmServicesOnly.systemd.sockets ? hive-priv)
|
|
&& !(s ? swarm-bao-queue-agent)
|
|
# The root check the platform containers carry runs through a link
|
|
# this host still declares, since no controller module does. Presence
|
|
# control: the unit and the rule are both found.
|
|
&& lib.any (
|
|
r: lib.hasPrefix "L+ /run/hive-rootfs-check " r
|
|
) swarmServicesOnly.systemd.tmpfiles.rules
|
|
&& lib.any (
|
|
p: lib.hasPrefix "-/run/hive-rootfs-check check-rootfs --warn-only " (toString p)
|
|
) s."container@swarm-bao".serviceConfig.ExecStartPre;
|
|
}
|
|
{
|
|
# On a controller that runs none of them, so the link cannot come from
|
|
# the service's own module.
|
|
name = "the swarm UI links every shared service with a web UI at its swarm domain";
|
|
ok =
|
|
lib.listToAttrs (map (l: lib.nameValuePair l.label l.url) (swarmServiceLinksOf controllerAlone))
|
|
== lib.mapAttrs (_: domain: "https://${domain}/") swarmServiceLinkDomains;
|
|
}
|
|
{
|
|
# Exact list equality also catches an entry rendered twice where the
|
|
# service runs on the controller's own host.
|
|
name = "the swarm UI's service links are the same whichever host runs the services";
|
|
ok =
|
|
lib.length (swarmServiceLinksOf controllerAlone)
|
|
== lib.length (lib.attrNames swarmServiceLinkDomains)
|
|
&& swarmServiceLinksOf controllerWithServices == swarmServiceLinksOf controllerAlone;
|
|
}
|
|
{
|
|
# The in-container option drives the container's firewall, and host
|
|
# `false` with container `true` would let the container's
|
|
# firewall.service rewrite the HOST ruleset. Counted first so a fixture
|
|
# that lost a container can't pass by comparing fewer of them.
|
|
name = "every service container restates its host-side privateNetwork";
|
|
ok =
|
|
lib.length (lib.attrNames serviceContainers) == 15
|
|
&& lib.all (
|
|
c: c.privateNetwork == (c.config.services.hyperhive.swarmContainer.privateNetwork or null)
|
|
) (lib.attrValues serviceContainers);
|
|
}
|
|
{
|
|
# Both values come from ../container-modules/swarm-container.nix.
|
|
# Four of the 15 service containers share the host netns (hive-forge, bao,
|
|
# the queue, the log store) and eleven have one of their own.
|
|
name = "a service container on the host netns runs no firewall or resolvconf of its own";
|
|
ok =
|
|
let
|
|
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
|
|
in
|
|
lib.length shared == 4
|
|
&& lib.all (
|
|
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
|
|
) shared;
|
|
}
|
|
{
|
|
# The `privateNetwork = true` and `writesOwnResolvConf = false` arms:
|
|
# NixOS enables both by default, so the module must leave them alone.
|
|
name = "a service container on a netns of its own keeps its firewall and resolvconf";
|
|
ok =
|
|
let
|
|
c = serviceContainers.swarm-ci-runner.config;
|
|
in
|
|
c.networking.firewall.enable && c.networking.resolvconf.enable;
|
|
}
|
|
{
|
|
# No interface but loopback, so the socket is the store's only listener
|
|
# the host can reach. Every location that forwards to the store goes
|
|
# through it; the internal auth subrequests are the only other upstreams.
|
|
# The read, push and scrape locations are named so an empty filter
|
|
# can't pass.
|
|
name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
c = host.containers.swarm-victoriametrics;
|
|
dir = "/run/swarm-victoriametrics";
|
|
sock = "${dir}/victoriametrics.sock";
|
|
viaSock = path: "http://unix:${sock}:${path}";
|
|
locs =
|
|
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations;
|
|
toStore = map (l: l.proxyPass) (
|
|
lib.filter (l: l.proxyPass != null && !(lib.hasInfix "internal;" l.extraConfig)) (
|
|
lib.attrValues locs
|
|
)
|
|
);
|
|
unit = c.config.systemd.services.victoriametrics;
|
|
nginxGid = host.ids.gids.nginx;
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == null
|
|
&& c.hostAddress == null
|
|
&& c.config.services.victoriametrics.listenAddress == "unix:${sock}"
|
|
&& locs."= /api/v1/query".proxyPass == viaSock "/api/v1/query"
|
|
&& locs."= /opentelemetry/api/v1/push".proxyPass == viaSock "/opentelemetry/api/v1/push"
|
|
&& locs."= /metrics".proxyPass == viaSock "/metrics"
|
|
&& lib.all (lib.hasPrefix (viaSock "")) toStore
|
|
&& c.bindMounts.${dir}.hostPath == dir
|
|
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
|
&& c.config.users.groups.gateway-nginx.gid == nginxGid
|
|
&& unit.serviceConfig.Group == "gateway-nginx"
|
|
&& !unit.serviceConfig.DynamicUser
|
|
&& unit.serviceConfig.UMask == "0007"
|
|
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
|
|
}
|
|
{
|
|
# On the bridge by DHCP, like the CI runner, and its socket the only
|
|
# listener the gateway dials. `address` unset is what keeps tuwunel off
|
|
# TCP. Every vhost location that forwards goes through the socket; the
|
|
# `/_matrix/` one is named so an empty filter can't pass.
|
|
name = "the homeserver runs on a netns of its own and the gateway reaches it over its socket";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
net = host.services.hyperhive.network;
|
|
c = host.containers.hive-matrix;
|
|
dir = "/run/hive-matrix";
|
|
sock = "${dir}/tuwunel.sock";
|
|
locs =
|
|
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.matrix.gatewayHost}.locations;
|
|
forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs));
|
|
tuwunel = c.config.services.matrix-tuwunel;
|
|
unit = c.config.systemd.services.tuwunel;
|
|
nginxGid = host.ids.gids.nginx;
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == net.bridgeName
|
|
&& c.hostAddress == null
|
|
&& c.localAddress == null
|
|
&& c.config.networking.interfaces.eth0.useDHCP
|
|
&& c.config.networking.nameservers == [ net.bridgeIp ]
|
|
&& c.config.networking.firewall.enable
|
|
&& c.config.networking.resolvconf.enable
|
|
&& (tuwunel.settings.global.address or null) == null
|
|
&& tuwunel.settings.global.unix_socket_path == sock
|
|
&& tuwunel.settings.global.unix_socket_perms == 660
|
|
&& locs."/_matrix/".proxyPass == "http://unix:${sock}:"
|
|
&& forwarded == [ "http://unix:${sock}:" ]
|
|
&& c.bindMounts.${dir}.hostPath == dir
|
|
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
|
&& c.config.users.groups.gateway-nginx.gid == nginxGid
|
|
&& tuwunel.group == "gateway-nginx"
|
|
&& unit.serviceConfig.Group == "gateway-nginx"
|
|
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
|
}
|
|
{
|
|
# On the bridge by DHCP, with all three listeners on sockets: nothing
|
|
# the bridge can dial. Every vhost location that forwards goes through
|
|
# one of the sockets, except the internal authz subrequest, which is
|
|
# `https://`. The main, bridge and metrics locations are named so an
|
|
# empty filter can't pass.
|
|
name = "authelia runs on a netns of its own and the gateway reaches all three listeners over sockets";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
net = host.services.hyperhive.network;
|
|
c = host.containers.swarm-authelia;
|
|
dir = "/run/swarm-authelia";
|
|
sock = n: "${dir}/${n}.sock";
|
|
locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.authelia.domain}.locations;
|
|
forwarded = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
|
|
map (l: l.proxyPass) (lib.attrValues locs)
|
|
);
|
|
inner = c.config;
|
|
settings = inner.services.authelia.instances.swarm.settings;
|
|
unit = inner.systemd.services.authelia-swarm;
|
|
bridge = inner.systemd.services.swarm-authelia-bridge;
|
|
nginxGid = host.ids.gids.nginx;
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == net.bridgeName
|
|
&& c.hostAddress == null
|
|
&& c.localAddress == null
|
|
&& inner.networking.interfaces.eth0.useDHCP
|
|
&& inner.networking.firewall.enable
|
|
&& settings.server.address == "unix://${sock "authelia"}?umask=0007"
|
|
&& settings.telemetry.metrics.address == "unix://${sock "metrics"}?umask=0007"
|
|
&& bridge.environment.SWARM_AUTHELIA_BRIDGE_SOCKET == sock "bridge"
|
|
&& !(bridge.environment ? SWARM_AUTHELIA_BRIDGE_BIND)
|
|
&& locs."/".proxyPass == "http://unix:${sock "authelia"}:/"
|
|
&& locs."/bridge/".proxyPass == "http://unix:${sock "bridge"}:/"
|
|
&& locs."= /metrics".proxyPass == "http://unix:${sock "metrics"}:/"
|
|
&& lib.all (lib.hasPrefix "http://unix:${dir}/") forwarded
|
|
&& c.bindMounts.${dir}.hostPath == dir
|
|
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
|
&& inner.users.groups.gateway-nginx.gid == nginxGid
|
|
&& unit.serviceConfig.Group == "gateway-nginx"
|
|
&& bridge.serviceConfig.Group == "gateway-nginx"
|
|
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
|
}
|
|
{
|
|
# On the bridge by DHCP, listening on a socket (protocol `socket` ignores the
|
|
# upstream-default `http_port`, which stays in the settings): the
|
|
# gateway's locations (UI, metrics, websocket upgrade) all forward to it.
|
|
# `forwarded != []` keeps an empty filter from passing.
|
|
name = "grafana runs on a netns of its own and the gateway reaches it over its socket";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
net = host.services.hyperhive.network;
|
|
c = host.containers.swarm-grafana;
|
|
dir = "/run/swarm-grafana";
|
|
sock = "${dir}/grafana.sock";
|
|
locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.grafana.domain}.locations;
|
|
forwarded = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
|
|
map (l: l.proxyPass) (lib.attrValues locs)
|
|
);
|
|
server = c.config.services.grafana.settings.server;
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == net.bridgeName
|
|
&& c.hostAddress == null
|
|
&& c.localAddress == null
|
|
&& c.config.networking.interfaces.eth0.useDHCP
|
|
&& c.config.networking.firewall.enable
|
|
&& server.protocol == "socket"
|
|
&& server.socket == sock
|
|
&& forwarded != [ ]
|
|
&& lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded
|
|
&& c.bindMounts.${dir}.hostPath == dir;
|
|
}
|
|
{
|
|
# atticd listens on the container's own loopback, a relay there forwards
|
|
# the socket nginx connects to, and the host unit that drives the
|
|
# TCP-only `attic` client dials the published name, not a loopback. The
|
|
# `cache create` clause and the trust-bundle clauses are controls: they
|
|
# find something in the same script and unit the absences are claimed of.
|
|
# `upstreams != []` keeps an empty vhost from passing.
|
|
name = "the cache runs on a netns of its own, the gateway reaches it over a socket and the host unit dials its public name";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
h = host.services.hyperhive;
|
|
c = host.containers.swarm-attic;
|
|
dir = "/run/swarm-attic";
|
|
sock = "${dir}/atticd.sock";
|
|
domain = h.swarm.attic.domain;
|
|
upstreams = map (l: l.proxyPass) (lib.attrValues h.gateway.vhosts.${domain}.locations);
|
|
relay = c.config.systemd.services.swarm-attic-relay;
|
|
tokens = host.systemd.services.swarm-bao-attic-tokens;
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == h.network.bridgeName
|
|
&& c.hostAddress == null
|
|
&& c.localAddress == null
|
|
&& c.config.networking.interfaces.eth0.useDHCP
|
|
&& c.config.networking.firewall.enable
|
|
&& upstreams != [ ]
|
|
&& lib.all (p: p == "http://unix:${sock}") upstreams
|
|
&& c.config.systemd.sockets.swarm-attic-relay.listenStreams == [ sock ]
|
|
&& lib.hasSuffix "systemd-socket-proxyd 127.0.0.1:${toString h.deploy.attic.port}" relay.serviceConfig.ExecStart
|
|
&& c.bindMounts.${dir}.hostPath == dir
|
|
&& lib.elem "d ${dir} 0770 root ${toString host.ids.gids.nginx} - -" host.systemd.tmpfiles.rules
|
|
&& lib.hasInfix "cache create" tokens.script
|
|
&& lib.hasInfix "https://${domain}" tokens.script
|
|
&& !(lib.hasInfix "127.0.0.1" tokens.script)
|
|
&& lib.elem "nginx.service" tokens.after
|
|
&& h.gateway.useSelfSigned
|
|
&& tokens.environment.SSL_CERT_FILE == "/run/swarm-bao-attic-tokens-ca/trust-bundle.pem"
|
|
&& lib.elem "swarm-bao-attic-tokens-ca-bundle.service" tokens.requires;
|
|
}
|
|
{
|
|
# On the bridge by DHCP, with every receiver on a socket: the hive's, the
|
|
# swarm tier's and each service forwarder's. Every gateway location that
|
|
# forwards goes through one of them, and each receiver's socket is the
|
|
# one its location names. The three kinds are named so an empty filter
|
|
# can't pass.
|
|
name = "the swarm collector runs on a netns of its own and the gateway reaches every receiver over a socket";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
net = host.services.hyperhive.network;
|
|
c = host.containers.swarm-otel;
|
|
otel = host.services.hyperhive.swarm.otel;
|
|
dir = "/run/swarm-otel";
|
|
sock = n: "${dir}/${n}.sock";
|
|
locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
|
|
forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs));
|
|
inner = c.config;
|
|
s = inner.services.opentelemetry-collector.settings;
|
|
unit = inner.systemd.services.opentelemetry-collector.serviceConfig;
|
|
nginxGid = host.ids.gids.nginx;
|
|
receivers = lib.filterAttrs (n: _: lib.hasPrefix "otlp/" n) s.receivers;
|
|
onSocket =
|
|
owner:
|
|
let
|
|
r = s.receivers."otlp/${owner}".protocols.http;
|
|
in
|
|
r.transport == "unix"
|
|
&& r.endpoint == sock owner
|
|
&& locs."/${owner}/".proxyPass == "http://unix:${sock owner}:/";
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == net.bridgeName
|
|
&& c.hostAddress == null
|
|
&& c.localAddress == null
|
|
&& inner.networking.interfaces.eth0.useDHCP
|
|
&& inner.networking.firewall.enable
|
|
&& onSocket "h1"
|
|
&& onSocket otel.producerName
|
|
&& onSocket "swarm-grafana"
|
|
&& lib.all (n: (receivers.${n}.protocols.http.transport or null) == "unix") (
|
|
lib.attrNames receivers
|
|
)
|
|
&&
|
|
lib.length (lib.attrNames receivers)
|
|
== lib.length (lib.attrNames host.services.hyperhive.swarm.hives)
|
|
+ 1
|
|
+ lib.length (lib.attrNames otel.serviceProducers)
|
|
&& forwarded != [ ]
|
|
&& lib.all (lib.hasPrefix "http://unix:${dir}/") forwarded
|
|
&& c.bindMounts.${dir}.hostPath == dir
|
|
&& builtins.elem "d ${dir} 2770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
|
&& inner.users.groups.gateway-nginx.gid == nginxGid
|
|
&& lib.elem "gateway-nginx" unit.SupplementaryGroups
|
|
&& unit.UMask == "0007";
|
|
}
|
|
{
|
|
# The routes the gateway forwards to are exactly the sockets the
|
|
# collector listens on, and none is a TCP address: a route left on
|
|
# loopback after the collector moved into its own netns returns a 502 to
|
|
# every producer while the collector runs. `verdict` is applied to
|
|
# broken copies of the routes too, so a check that cannot fail is caught.
|
|
name = "the gateway's collector routes are the collector's unix receivers and none is loopback";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
otel = host.services.hyperhive.swarm.otel;
|
|
s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings;
|
|
locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
|
|
sorted = xs: lib.sort lib.lessThan (lib.unique xs);
|
|
unixReceivers = lib.filter (r: (r.protocols.http.transport or null) == "unix") (
|
|
lib.attrValues s.receivers
|
|
);
|
|
receiverSocks = sorted (map (r: r.protocols.http.endpoint) unixReceivers);
|
|
socketOfRoute =
|
|
p:
|
|
if lib.hasPrefix "http://unix:" p then
|
|
lib.removeSuffix ":/" (lib.removePrefix "http://unix:" p)
|
|
else
|
|
p;
|
|
verdict =
|
|
routes:
|
|
let
|
|
passes = lib.filter (p: p != null) (map (l: l.proxyPass or null) (lib.attrValues routes));
|
|
in
|
|
sorted (map socketOfRoute passes) == receiverSocks
|
|
&& !(lib.any (p: lib.hasInfix "127.0.0.1" p || lib.hasInfix "localhost" p) passes);
|
|
in
|
|
verdict locs
|
|
&& lib.elem "/run/swarm-otel/h1.sock" receiverSocks
|
|
&& lib.elem "/run/swarm-otel/${otel.producerName}.sock" receiverSocks
|
|
&& lib.elem "/run/swarm-otel/swarm-grafana.sock" receiverSocks
|
|
&&
|
|
lib.length unixReceivers == lib.length (lib.attrNames host.services.hyperhive.swarm.hives)
|
|
+ 1
|
|
+ lib.length (lib.attrNames otel.serviceProducers)
|
|
# Each broken copy must fail: a TCP route, a missing route, a route to
|
|
# a socket no receiver has.
|
|
&& !verdict (locs // { "/h1/".proxyPass = "http://127.0.0.1:4319/"; })
|
|
&& !verdict (lib.removeAttrs locs [ "/h1/" ])
|
|
&& !verdict (locs // { "/x/".proxyPass = "http://unix:/run/swarm-otel/x.sock:/"; });
|
|
}
|
|
{
|
|
# After the container, restarted with it, and run as the gateway's user,
|
|
# so a socket nginx cannot open fails the unit. Every receiver socket is
|
|
# in the script.
|
|
name = "the collector's receiver sockets are connect-checked after the container starts";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
u = host.systemd.services.swarm-otel-sockets;
|
|
s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings;
|
|
socks = map (r: r.protocols.http.endpoint) (
|
|
lib.filter (r: (r.protocols.http.transport or null) == "unix") (lib.attrValues s.receivers)
|
|
);
|
|
in
|
|
socks != [ ]
|
|
&& lib.all (sock: lib.hasInfix sock u.script) socks
|
|
&& !(lib.hasInfix "/run/swarm-otel/absent.sock" u.script)
|
|
&& u.serviceConfig.Type == "oneshot"
|
|
&& u.serviceConfig.User == host.services.nginx.user
|
|
&& lib.elem "container@swarm-otel.service" u.after
|
|
&& lib.elem "container@swarm-otel.service" u.partOf
|
|
&& lib.elem "multi-user.target" u.wantedBy;
|
|
}
|
|
{
|
|
# `DynamicUser` forces `ProtectSystem=strict`, which leaves `/run`
|
|
# read-only unless the path is in `ReadWritePaths`. The controls: the
|
|
# unit really is DynamicUser, and a copy without the path, or with some
|
|
# other path, is read-only.
|
|
name = "the collector unit can write the directory its receiver sockets are created in";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
sc = host.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig;
|
|
s = host.containers.swarm-otel.config.services.opentelemetry-collector.settings;
|
|
socks = map (r: r.protocols.http.endpoint) (
|
|
lib.filter (r: (r.protocols.http.transport or null) == "unix") (lib.attrValues s.receivers)
|
|
);
|
|
writes =
|
|
c: sock:
|
|
let
|
|
strict = (c.DynamicUser or false) || (c.ProtectSystem or "no") == "strict";
|
|
dir = dirOf sock;
|
|
under = p: p == dir || lib.hasPrefix "${p}/" dir;
|
|
in
|
|
!strict || lib.any under (c.ReadWritePaths or [ ]);
|
|
in
|
|
socks != [ ]
|
|
&& sc.DynamicUser
|
|
&& lib.all (writes sc) socks
|
|
&& !(lib.any (writes (sc // { ReadWritePaths = [ ]; })) socks)
|
|
&& !(lib.any (writes (sc // { ReadWritePaths = [ "/var/lib/opentelemetry-collector" ]; })) socks);
|
|
}
|
|
{
|
|
# The readiness unit tells a route nginx has from its catch-all 404 by
|
|
# this header, so every route carries its own name in it, `always` so a
|
|
# 4xx or 5xx from the collector carries it too. A location with an
|
|
# `add_header` inherits none of the server-scope ones, so the security
|
|
# headers have to be repeated. The catch-all carries no route header.
|
|
name = "every collector route sets its own route header and keeps the security headers";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
otel = host.services.hyperhive.swarm.otel;
|
|
locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
|
|
sec = host.services.hyperhive.gateway.lib.securityHeaders;
|
|
proxied = lib.filter (l: (locs.${l}.proxyPass or null) != null) (lib.attrNames locs);
|
|
nameOf = l: lib.removeSuffix "/" (lib.removePrefix "/" l);
|
|
verdict =
|
|
ls:
|
|
lib.all (
|
|
l:
|
|
let
|
|
extra = ls.${l}.extraConfig or "";
|
|
in
|
|
lib.hasInfix "add_header X-Swarm-Otel-Route ${nameOf l} always;" extra
|
|
&& lib.hasInfix "add_header X-Frame-Options" extra
|
|
) proxied;
|
|
in
|
|
lib.length proxied > 2
|
|
&& lib.hasInfix "add_header X-Frame-Options" sec
|
|
&& !(lib.hasInfix "X-Swarm-Otel-Route" (locs."/".extraConfig or ""))
|
|
&& verdict locs
|
|
&& !verdict (
|
|
locs
|
|
// {
|
|
"/h1/" = locs."/h1/" // {
|
|
extraConfig = sec;
|
|
};
|
|
}
|
|
)
|
|
&& !verdict (
|
|
locs
|
|
// {
|
|
"/h1/" = locs."/h1/" // {
|
|
extraConfig = "add_header X-Swarm-Otel-Route h1 always;";
|
|
};
|
|
}
|
|
)
|
|
&& !verdict (
|
|
locs
|
|
// {
|
|
"/h1/" = locs."/h1/" // {
|
|
extraConfig = sec + "add_header X-Swarm-Otel-Route swarm always;";
|
|
};
|
|
}
|
|
)
|
|
&& !verdict (
|
|
locs
|
|
// {
|
|
"/h1/" = locs."/h1/" // {
|
|
extraConfig = sec + "add_header X-Swarm-Otel-Route h1;";
|
|
};
|
|
}
|
|
);
|
|
}
|
|
{
|
|
# The probe loop names each gateway route, and the routes are read from
|
|
# the vhost, not from the receivers the probe is built from. A script
|
|
# missing one route, treating 502 as a pass, or passing on status alone
|
|
# without the route header, is rejected.
|
|
name = "the readiness unit probes every collector route through the gateway";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
otel = host.services.hyperhive.swarm.otel;
|
|
locs = host.services.nginx.virtualHosts.${otel.domain}.locations;
|
|
routes = lib.sort lib.lessThan (
|
|
map (l: lib.removeSuffix "/" (lib.removePrefix "/" l)) (
|
|
lib.filter (l: (locs.${l}.proxyPass or null) != null) (lib.attrNames locs)
|
|
)
|
|
);
|
|
lines = script: map (lib.removePrefix " ") (lib.splitString "\n" script);
|
|
probed =
|
|
script:
|
|
let
|
|
loop = lib.findFirst (l: lib.hasPrefix "for route in " l) "" (lines script);
|
|
in
|
|
lib.sort lib.lessThan (
|
|
lib.filter (w: w != "") (
|
|
lib.splitString " " (lib.removeSuffix "; do" (lib.removePrefix "for route in " loop))
|
|
)
|
|
);
|
|
verdict =
|
|
script:
|
|
probed script == routes
|
|
&& lib.hasInfix "--resolve ${otel.domain}:443:127.0.0.1" script
|
|
&& lib.hasInfix "https://${otel.domain}/$route/" script
|
|
&& lib.hasInfix "case $code in 000 | 502 | 504) ;;" script
|
|
&& lib.hasInfix "%header{x-swarm-otel-route}" script
|
|
&& lib.hasInfix ''[ "$seen" = "$route" ] && continue 2'' script;
|
|
script = host.systemd.services.swarm-otel-sockets.script;
|
|
in
|
|
lib.length routes > 2
|
|
&& lib.elem "h1" routes
|
|
&& lib.elem "swarm-grafana" routes
|
|
&& verdict script
|
|
&& !verdict (lib.replaceStrings [ " swarm-grafana" ] [ "" ] script)
|
|
&& !verdict (lib.replaceStrings [ "000 | 502 | 504" ] [ "000 | 504" ] script)
|
|
&& !verdict (lib.replaceStrings [ "--resolve" ] [ "--noresolve" ] script)
|
|
&& !verdict (lib.replaceStrings [ "%header{x-swarm-otel-route}" ] [ "" ] script)
|
|
&& !verdict (lib.replaceStrings [ ''[ "$seen" = "$route" ] && '' ] [ "" ] script);
|
|
}
|
|
{
|
|
# The DynamicUser layout keeps the history under /var/lib/private; the
|
|
# static user's StateDirectory is the plain path. The move runs as its
|
|
# own unit ahead of the store.
|
|
name = "the metrics store's history moves from the DynamicUser path before the store starts";
|
|
ok =
|
|
let
|
|
cfg = serviceContainers.swarm-victoriametrics.config;
|
|
m = cfg.systemd.services.victoriametrics-state-migrate;
|
|
in
|
|
cfg.services.victoriametrics.stateDir == "victoriametrics"
|
|
&& builtins.elem "victoriametrics.service" m.before
|
|
&& builtins.elem "victoriametrics.service" m.requiredBy
|
|
&& lib.hasInfix "old=/var/lib/private/victoriametrics" m.script
|
|
&& lib.hasInfix "new=/var/lib/victoriametrics" m.script
|
|
&& lib.hasInfix ''mv "$old" "$new"'' m.script
|
|
&& lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script;
|
|
}
|
|
{
|
|
# The control is the default package on the all-services host.
|
|
name = "a metrics store package older than 1.153.0 fails evaluation";
|
|
ok =
|
|
let
|
|
old = failedMetricsStoreVersion oldMetricsStore;
|
|
in
|
|
lib.length old == 1
|
|
&& lib.hasInfix "1.152.0" (lib.head old)
|
|
&& failedMetricsStoreVersion serviceContainersWithCi == [ ];
|
|
}
|
|
{
|
|
name = "a service container that sets no stateVersion of its own is on 26.05";
|
|
ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";
|
|
}
|
|
{
|
|
name = "a service container that pins its own stateVersion keeps it";
|
|
ok = serviceContainers.hive-forge.config.system.stateVersion == "25.11";
|
|
}
|
|
{
|
|
# An operator's explicit `false` beats every `mkDefault` assertion,
|
|
# which is what keeps "asserted by whoever needs it" from being a
|
|
# setting the operator cannot turn off.
|
|
name = "an explicit gateway.enable = false wins over the modules asserting it";
|
|
ok = !(hiveWithController { gateway.enable = false; }).services.nginx.enable;
|
|
}
|
|
{
|
|
name = "declared forge mirrors reach the swarm-controller, not hive-c0re";
|
|
ok =
|
|
builtins.fromJSON controllerWithMirrors.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS
|
|
== [ aMirror ]
|
|
&& !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HYPERHIVE_FORGE_MIRRORS);
|
|
}
|
|
{
|
|
name = "forge mirrors on a host without the controller warn that nothing seeds them";
|
|
ok =
|
|
let
|
|
warns = cfg: lib.any (lib.hasInfix "declare them on the swarm-controller's host") cfg.warnings;
|
|
in
|
|
warns mirrorsNoController && !(warns controllerWithMirrors);
|
|
}
|
|
{
|
|
# The mirrors are what `self` resolves `actions/*` to, so the two come
|
|
# together: one per action `.forgejo/workflows/` names. The control is
|
|
# the same forge without the option.
|
|
name = "actionsFromSelf adds the actions mirrors and resolves actions from the forge";
|
|
ok =
|
|
let
|
|
mirrorsOf =
|
|
m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS;
|
|
actionsUrlOf =
|
|
m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null;
|
|
in
|
|
map (m: m.dest) (mirrorsOf controllerWithSelfActions) == [
|
|
"mirrors/tool"
|
|
"actions/checkout"
|
|
"actions/upload-artifact"
|
|
]
|
|
&& actionsUrlOf controllerWithSelfActions == "self"
|
|
&& mirrorsOf controllerWithMirrors == [ aMirror ]
|
|
&& actionsUrlOf controllerWithMirrors == null;
|
|
}
|
|
{
|
|
# The default follows the controller, the only thing that seeds the
|
|
# checkout mirror: a forge alone must not resolve actions from itself.
|
|
name = "actionsFromSelf defaults to whether this host runs the swarm-controller";
|
|
ok =
|
|
let
|
|
actionsUrlOf =
|
|
m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null;
|
|
mirrorsOf =
|
|
m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS;
|
|
in
|
|
!forgeOnly.services.hyperhive.deploy.forgejo.actionsFromSelf
|
|
&& actionsUrlOf forgeOnly == null
|
|
&& forgeOnly.warnings == [ ]
|
|
&& !(
|
|
(forgeOnly.systemd.services.swarm-controller.environment or { }) ? SWARM_CONTROLLER_FORGE_MIRRORS
|
|
)
|
|
&& !(
|
|
(mirrorsNoController.systemd.services.swarm-controller.environment or { })
|
|
? SWARM_CONTROLLER_FORGE_MIRRORS
|
|
)
|
|
&& forgeWithController.services.hyperhive.deploy.forgejo.actionsFromSelf
|
|
&& actionsUrlOf forgeWithController == "self"
|
|
&& lib.elem "actions/checkout" (map (m: m.dest) (mirrorsOf forgeWithController));
|
|
}
|
|
{
|
|
name = "the org avatar lands on the controller's unit, not hive-c0re's";
|
|
ok =
|
|
controllerWithMirrors.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_CONFIG_ORG_AVATAR_PNG
|
|
== "/etc/marker-avatar.png"
|
|
&& !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HIVE_ORG_AVATAR_PNG);
|
|
}
|
|
{
|
|
# Where the store runs here, each reader waits for its container and
|
|
# the PKI unit minting the leaf it logs in with, and wants the PKI unit only:
|
|
# a `Wants=` on the container would start it mid-switch on the old closure.
|
|
# It never requires the PKI unit: a refused pki start fails a requirer's
|
|
# job with `dependency`, which its `Restart=on-failure` never retries.
|
|
# Elsewhere neither unit exists. On the store's host without the
|
|
# readers, none of them is rendered as a stub. Rosters are counted: an
|
|
# empty one holds vacuously.
|
|
name = "a store reader waits for the store's container and PKI unit only where the store runs here, wants only the PKI unit, and requires neither";
|
|
ok =
|
|
let
|
|
machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine;
|
|
storeUnits = [
|
|
"swarm-bao-pki.service"
|
|
"container@${machine}.service"
|
|
];
|
|
readers = [
|
|
"swarm-bao-forge-oidc"
|
|
"swarm-bao-matrix-oidc"
|
|
"swarm-bao-grafana-oidc"
|
|
"swarm-bao-grafana-admin"
|
|
"swarm-bao-grafana-metrics-bearer"
|
|
"swarm-bao-grafana-secret-key"
|
|
"swarm-bao-grafana-controller-token"
|
|
"swarm-bao-nats-callout-xkey"
|
|
"swarm-bao-ci-runner"
|
|
"swarm-bao-otel-oidc"
|
|
];
|
|
here = serviceContainersWithCi.systemd.services;
|
|
elsewhere = swarmServicesBaoElsewhere.systemd.services;
|
|
readersHere = lib.filter (u: here ? ${u}) readers;
|
|
readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers;
|
|
waits =
|
|
u:
|
|
here ? ${u}
|
|
&& lib.elem "swarm-bao-pki.service" here.${u}.after
|
|
&& lib.elem "swarm-bao-pki.service" here.${u}.wants
|
|
&& !(lib.any (n: lib.elem n here.${u}.requires) storeUnits)
|
|
&& lib.elem "container@${machine}.service" here.${u}.after
|
|
&& !(lib.elem "container@${machine}.service" here.${u}.wants);
|
|
namesNoStoreUnit =
|
|
u:
|
|
let
|
|
s = elsewhere.${u};
|
|
in
|
|
!(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
|
|
in
|
|
lib.all waits readers
|
|
&& lib.length readersHere == 10
|
|
&& lib.elem "swarm-bao-grafana-admin" readersHere
|
|
&& lib.length readersElsewhere == 8
|
|
&& lib.all namesNoStoreUnit readersElsewhere
|
|
&& !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
|
|
}
|
|
{
|
|
# A swarm service container is `Type=notify`: its start job lasts until
|
|
# the guest's job queue is empty and is killed at `timeoutStartSec`. A
|
|
# unit in that queue whose start job can last that long takes the whole
|
|
# container down with it. A unit that states no budget has the manager's
|
|
# 90 s default, or none if it is a oneshot, and counts too unless
|
|
# `unstatedAllowed` names it. Judged per unit; the next case adds the units
|
|
# ordered one after another. The offenders are in the name, which is printed
|
|
# on failure only.
|
|
name = "no unit in a service container's boot wait has a start budget that reaches the container's start limit${
|
|
lib.optionalString (bootWaitOffenders != [ ]) ": ${lib.concatStringsSep "; " bootWaitOffenders}"
|
|
}";
|
|
ok = bootWaitOffenders == [ ];
|
|
}
|
|
{
|
|
# The same limit, judged on the longest chain of units ordered one after
|
|
# another (`after`/`before`), the budgets adding up. A chain of one unit is
|
|
# the case above; the offenders here have two or more.
|
|
name = "no chain of units in a service container's boot wait that start one after another adds up to the container's start limit${
|
|
lib.optionalString (
|
|
bootWaitChainOffenders != [ ]
|
|
) ": ${lib.concatStringsSep "; " bootWaitChainOffenders}"
|
|
}";
|
|
ok = bootWaitChainOffenders == [ ];
|
|
}
|
|
{
|
|
# The presence arms for the check above: it finds the chains the real
|
|
# containers have. hive-matrix's render runs before its sync (20 s + 20 s)
|
|
# and authelia's secret fetch before the bridge (30 s + 10 s).
|
|
name = "the boot-wait chain check finds the ordered units of the real containers and sums them";
|
|
ok =
|
|
let
|
|
units = name: map (u: u.key) (chainOf name).units;
|
|
in
|
|
units "hive-matrix" == [
|
|
"swarm-matrix-appservice-render.service"
|
|
"swarm-matrix-appservice-sync.service"
|
|
]
|
|
&& (chainOf "hive-matrix").seconds == 40
|
|
&&
|
|
units "swarm-authelia" == [
|
|
"authelia-swarm-secrets.service"
|
|
"swarm-authelia-bridge.service"
|
|
]
|
|
&& (chainOf "swarm-authelia").seconds == 40
|
|
&& (chainOf "hive-forge").cycle == [ ]
|
|
&& lib.all (n: (chainOf n).cycle == [ ]) hyperhiveContainers;
|
|
}
|
|
{
|
|
# The chain walk on hand-made guests. Budgets add along an ordering,
|
|
# written with `after` or with `before`; two units with no ordering run
|
|
# side by side and do not add; a unit that adds nothing (a target, a
|
|
# service that ends at the fork) passes the ordering on; an ordering to a
|
|
# unit outside the boot wait is not a step; the heaviest path of a diamond
|
|
# wins; one unbounded unit makes the chain unbounded; and a unit the
|
|
# caller calls weightless adds 0.
|
|
name = "the boot-wait chain walk adds budgets along after and before, and not across units that run side by side";
|
|
ok =
|
|
let
|
|
svc =
|
|
budget: extra:
|
|
{
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
TimeoutStartSec = budget;
|
|
};
|
|
}
|
|
// extra;
|
|
guestOf = services: {
|
|
systemd = {
|
|
inherit services;
|
|
targets.multi-user.wants = [ "gate.target" ];
|
|
targets.gate = { };
|
|
};
|
|
};
|
|
chain =
|
|
services: weightless:
|
|
bootWait.longestChain {
|
|
guest = guestOf services;
|
|
inherit weightless;
|
|
};
|
|
sum = services: (chain services (_: false)).seconds;
|
|
keys = services: map (u: u.key) (chain services (_: false)).units;
|
|
in
|
|
# Two units in sequence: over the limit of 60 s, and under it.
|
|
sum {
|
|
a = svc 40 { };
|
|
b = svc 40 { after = [ "a.service" ]; };
|
|
} == 80
|
|
&&
|
|
sum {
|
|
a = svc 20 { };
|
|
b = svc 30 { after = [ "a.service" ]; };
|
|
} == 50
|
|
&&
|
|
keys {
|
|
a = svc 40 { };
|
|
b = svc 40 { after = [ "a.service" ]; };
|
|
} == [
|
|
"a.service"
|
|
"b.service"
|
|
]
|
|
# `before` is the same ordering from the other side.
|
|
&&
|
|
sum {
|
|
a = svc 40 { before = [ "b.service" ]; };
|
|
b = svc 40 { };
|
|
} == 80
|
|
# Side by side: the larger, not the sum.
|
|
&&
|
|
sum {
|
|
a = svc 40 { };
|
|
b = svc 40 { };
|
|
} == 40
|
|
# Through a unit that adds nothing.
|
|
&&
|
|
sum {
|
|
a = svc 40 { before = [ "gate.target" ]; };
|
|
b = svc 40 { after = [ "gate.target" ]; };
|
|
} == 80
|
|
&&
|
|
sum {
|
|
a = svc 40 { before = [ "plain.service" ]; };
|
|
plain = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "b.service" ];
|
|
};
|
|
b = svc 40 { };
|
|
} == 80
|
|
# Outside the boot wait: not a step.
|
|
&&
|
|
sum {
|
|
a = svc 40 { };
|
|
b = svc 40 { after = [ "a.service" ]; };
|
|
offBoot = {
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
TimeoutStartSec = 900;
|
|
};
|
|
after = [ "b.service" ];
|
|
};
|
|
} == 80
|
|
&&
|
|
sum {
|
|
a = svc 40 { after = [ "offBoot.service" ]; };
|
|
offBoot = {
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
TimeoutStartSec = 900;
|
|
};
|
|
};
|
|
} == 40
|
|
# A diamond: the heavier side.
|
|
&&
|
|
keys {
|
|
a = svc 10 { };
|
|
b = svc 50 { after = [ "a.service" ]; };
|
|
c = svc 20 { after = [ "a.service" ]; };
|
|
d = svc 5 {
|
|
after = [
|
|
"b.service"
|
|
"c.service"
|
|
];
|
|
};
|
|
} == [
|
|
"a.service"
|
|
"b.service"
|
|
"d.service"
|
|
]
|
|
# The default (90 s) and an unbounded oneshot add like any other.
|
|
&&
|
|
sum {
|
|
a = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
preStart = "true";
|
|
};
|
|
b = svc 10 { after = [ "a.service" ]; };
|
|
} == 100
|
|
&&
|
|
sum {
|
|
a = svc 40 { };
|
|
b = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig.Type = "oneshot";
|
|
after = [ "a.service" ];
|
|
};
|
|
} == null
|
|
# A weightless unit adds 0 and the rest still add up.
|
|
&&
|
|
(chain {
|
|
a = svc 40 { };
|
|
b = svc 40 { after = [ "a.service" ]; };
|
|
c = svc 15 { after = [ "b.service" ]; };
|
|
} (o: o.key == "b.service")).seconds == 55
|
|
&& sum { } == 0;
|
|
}
|
|
{
|
|
# A cycle in the ordering is named, with the units that merely follow it
|
|
# left out. The control is the same units without the edge that closes it.
|
|
name = "the boot-wait chain walk names a cycle in the ordering";
|
|
ok =
|
|
let
|
|
svc =
|
|
extra:
|
|
{
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
TimeoutStartSec = 10;
|
|
};
|
|
}
|
|
// extra;
|
|
walk =
|
|
services:
|
|
bootWait.longestChain {
|
|
guest.systemd = { inherit services; };
|
|
};
|
|
cyclic = walk {
|
|
a = svc { after = [ "c.service" ]; };
|
|
b = svc { after = [ "a.service" ]; };
|
|
c = svc { after = [ "b.service" ]; };
|
|
tail = svc { after = [ "c.service" ]; };
|
|
head = svc { before = [ "a.service" ]; };
|
|
};
|
|
open = walk {
|
|
a = svc { };
|
|
b = svc { after = [ "a.service" ]; };
|
|
c = svc { after = [ "b.service" ]; };
|
|
tail = svc { after = [ "c.service" ]; };
|
|
head = svc { before = [ "a.service" ]; };
|
|
};
|
|
in
|
|
lib.sort lib.lessThan cyclic.cycle == [
|
|
"a.service"
|
|
"b.service"
|
|
"c.service"
|
|
]
|
|
&& cyclic.seconds == null
|
|
&& open.cycle == [ ]
|
|
&& open.seconds == 50
|
|
&&
|
|
(walk {
|
|
a = svc { after = [ "a.service" ]; };
|
|
}).cycle == [ "a.service" ];
|
|
}
|
|
{
|
|
# A name that no container's boot wait carries any more is a dead entry,
|
|
# and would let a unit of that name back in unseen. The presence arm is
|
|
# that each entry is found.
|
|
name = "every unit the boot-wait check lets through without a stated budget is in some container's boot wait";
|
|
ok =
|
|
let
|
|
unstated = lib.concatMap (
|
|
name: lib.filter (o: o.source != "stated") (reachingLimit name { })
|
|
) hyperhiveContainers;
|
|
keys = map (o: o.key) unstated;
|
|
found = key: lib.elem key keys;
|
|
foundSuffix = s: lib.any (k: lib.hasSuffix s k) keys;
|
|
in
|
|
lib.all (
|
|
g: lib.all found (g.units or [ ]) && lib.all foundSuffix (g.suffixes or [ ])
|
|
) unstatedAllowed;
|
|
}
|
|
{
|
|
# Names, not `lib.attrNames containers`: a walk over every container
|
|
# would also judge an operator's own. Every container the fixture
|
|
# declares is in the list, so a new one cannot dodge the check.
|
|
name = "the boot-wait check names every service container the fixture declares";
|
|
ok =
|
|
lib.sort lib.lessThan hyperhiveContainers
|
|
== lib.sort lib.lessThan (lib.attrNames serviceContainers);
|
|
}
|
|
{
|
|
# The presence arms for the check above: it reads real units and real
|
|
# limits. Each named unit states its budget (30 s, 30 s, 30 s, 30 s, 20 s, 20 s),
|
|
# so it is found at a limit equal to it and not one second above (the
|
|
# boundary), and the container limits are the ones the check reads.
|
|
name = "the boot-wait check finds the units that state a budget, at the boundary, and reads each container's own limit";
|
|
ok =
|
|
let
|
|
keys = name: limit: map (o: o.key) (reachingLimit name { inherit limit; });
|
|
mint = "forgejo-swarm-controller-account.service";
|
|
fetch = "swarm-journal-forwarder-oidc.service";
|
|
secrets = "authelia-swarm-secrets.service";
|
|
sync = "swarm-matrix-appservice-sync.service";
|
|
render = "swarm-matrix-appservice-render.service";
|
|
sso = "forgejo-sso-source.service";
|
|
in
|
|
lib.elem mint (keys "hive-forge" 30)
|
|
&& lib.elem sso (keys "hive-forge" 30)
|
|
&& !(lib.elem sso (keys "hive-forge" 31))
|
|
&& lib.elem fetch (keys "hive-forge" 30)
|
|
&& !(lib.elem mint (keys "hive-forge" 31))
|
|
&& !(lib.elem fetch (keys "hive-forge" 31))
|
|
&& lib.elem secrets (keys "swarm-authelia" 30)
|
|
&& !(lib.elem secrets (keys "swarm-authelia" 31))
|
|
&& lib.elem sync (keys "hive-matrix" 20)
|
|
&& lib.elem render (keys "hive-matrix" 20)
|
|
&& !(lib.elem sync (keys "hive-matrix" 21))
|
|
&& !(lib.elem render (keys "hive-matrix" 21))
|
|
&& startLimitOf "hive-forge" == 60
|
|
&& startLimitOf "swarm-bao" == 300;
|
|
}
|
|
{
|
|
# A unit held to a budget that fits the container's start retries with a
|
|
# growing delay, not at a fixed one, and the authelia bridge's secret
|
|
# fetch states a budget so that the check above judges it.
|
|
name = "the store-fetching units in a container's boot wait restart with a growing delay and the bridge's fetch states its budget";
|
|
ok =
|
|
let
|
|
grows =
|
|
u:
|
|
let
|
|
sc = u.serviceConfig;
|
|
in
|
|
sc.Restart == "on-failure" && sc.RestartSteps > 0 && sc.RestartMaxDelaySec > sc.RestartSec;
|
|
forge = serviceContainers.hive-forge.config.systemd.services;
|
|
authelia = serviceContainers.swarm-authelia.config.systemd.services;
|
|
in
|
|
grows forge.forgejo-swarm-controller-account
|
|
&& grows forge.forgejo-sso-source
|
|
&& forge.forgejo-sso-source.serviceConfig.TimeoutStartSec == 30
|
|
# Both registration calls fetch the issuer's discovery document and
|
|
# run under a client timeout.
|
|
&& lib.hasInfix "timeout --kill-after=3 20 forgejo admin auth add-oauth" forge.forgejo-sso-source.script
|
|
&& lib.hasInfix "timeout --kill-after=3 20 forgejo admin auth update-oauth" forge.forgejo-sso-source.script
|
|
&& grows authelia.authelia-swarm-secrets
|
|
&& authelia.swarm-authelia-bridge.serviceConfig.TimeoutStartSec == 10;
|
|
}
|
|
{
|
|
# The forwarder's collector is started `--no-block` by the fetch and is
|
|
# not in the boot wait, whatever its own budget; its fetch is. The
|
|
# presence arm is the fetch, found by the same walk.
|
|
name = "the boot-wait walk includes a unit wanted by multi-user.target and leaves out one started on demand";
|
|
ok =
|
|
let
|
|
booted = bootWait.bootSet serviceContainers.hive-forge.config.systemd;
|
|
in
|
|
lib.elem "swarm-journal-forwarder-oidc.service" booted
|
|
&& lib.elem "forgejo-swarm-controller-account.service" booted
|
|
&& !(lib.elem "opentelemetry-collector.service" booted);
|
|
}
|
|
{
|
|
# The walk and the comparison on a hand-made guest: each unit that must
|
|
# be found is, and each that must not be is not. Seconds are counted
|
|
# from every spelling a unit uses, infinity included.
|
|
name = "the boot-wait check flags a budget at or over the limit in the boot wait, pulled in any way, and nothing else";
|
|
ok =
|
|
let
|
|
oneshot = budget: {
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
TimeoutStartSec = budget;
|
|
};
|
|
};
|
|
guest = {
|
|
systemd = {
|
|
targets.multi-user.wants = [ "viaTarget.service" ];
|
|
services = {
|
|
wantedBy = oneshot "3min" // {
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
viaRequires = oneshot 120;
|
|
parent = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
requires = [ "viaRequires.service" ];
|
|
wants = [ "gate.target" ];
|
|
};
|
|
viaTarget = oneshot "1min 30s";
|
|
viaRequiredBy = oneshot 100 // {
|
|
requiredBy = [ "multi-user.target" ];
|
|
};
|
|
viaGate = oneshot 100 // {
|
|
wantedBy = [ "gate.target" ];
|
|
};
|
|
atLimit = oneshot "1min" // {
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
forever = oneshot "infinity" // {
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
belowLimit = oneshot 59 // {
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
notPulledIn = oneshot 900;
|
|
orderedOnly = oneshot 900 // {
|
|
after = [ "wantedBy.service" ];
|
|
};
|
|
forks = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
TimeoutStartSec = 900;
|
|
};
|
|
};
|
|
hook = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
preStart = "true";
|
|
serviceConfig.TimeoutStartSec = "1h";
|
|
};
|
|
unstated = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig.Type = "oneshot";
|
|
};
|
|
defaulted = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
preStart = "true";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
in
|
|
lib.sort lib.lessThan (
|
|
map (o: "${o.key}:${o.source}") (
|
|
bootWait.overBudget {
|
|
inherit guest;
|
|
limit = 60;
|
|
}
|
|
)
|
|
) == [
|
|
"atLimit.service:stated"
|
|
"defaulted.service:default"
|
|
"forever.service:stated"
|
|
"hook.service:stated"
|
|
"unstated.service:unbounded"
|
|
"viaGate.service:stated"
|
|
"viaRequiredBy.service:stated"
|
|
"viaRequires.service:stated"
|
|
"viaTarget.service:stated"
|
|
"wantedBy.service:stated"
|
|
]
|
|
# The default is the manager's: over a limit above it, or with the guest
|
|
# setting a shorter one, the unit is not reached, and the unbounded
|
|
# oneshot still is.
|
|
&& lib.elem "defaulted.service" (
|
|
map (o: o.key) (
|
|
bootWait.overBudget {
|
|
inherit guest;
|
|
limit = 90;
|
|
}
|
|
)
|
|
)
|
|
&& !(lib.elem "defaulted.service" (
|
|
map (o: o.key) (
|
|
bootWait.overBudget {
|
|
inherit guest;
|
|
limit = 91;
|
|
}
|
|
)
|
|
))
|
|
&& !(lib.elem "defaulted.service" (
|
|
map (o: o.key) (
|
|
bootWait.overBudget {
|
|
guest = guest // {
|
|
systemd = guest.systemd // {
|
|
settings.Manager.DefaultTimeoutStartSec = "45s";
|
|
};
|
|
};
|
|
limit = 60;
|
|
}
|
|
)
|
|
));
|
|
}
|
|
];
|
|
in
|
|
runGroup "swarm-services-switch" cases
|