481 lines
20 KiB
Nix
481 lines
20 KiB
Nix
# `checks.module-eval-swarm-services-switch` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
bridgePortsOrNone
|
|
swarmServiceEnables
|
|
;
|
|
|
|
# A dedicated services host with hives elsewhere: the swarm-services
|
|
# toggle and no hive, the recipe docs/swarm/services.md gives.
|
|
swarmServicesOnly = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.allSwarmServices = true;
|
|
};
|
|
|
|
# A services host with hives elsewhere and the store elsewhere too: no hive
|
|
# and no store identity minted here.
|
|
swarmServicesOnlyNoStore = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.allSwarmServices = true;
|
|
deploy.bao.enable = false;
|
|
};
|
|
|
|
# A dedicated secret-store host: no hive, and the store's browser UI the
|
|
# only swarm name its gateway fronts.
|
|
storeOnlyNoHive = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.bao.enable = true;
|
|
};
|
|
|
|
# The homeserver on a host without a hive.
|
|
matrixNoHive = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.matrix.enable = true;
|
|
};
|
|
|
|
# The same switch with every service it owns placed elsewhere, so the only
|
|
# thing that can enable the gateway/resolver/bridge here is that toggle's
|
|
# own module — each service module asserts them from its own `enable`.
|
|
swarmServicesSwitchAlone = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.allSwarmServices = true;
|
|
deploy.matrix.enable = false;
|
|
otel.enable = false;
|
|
deploy.authelia.enable = false;
|
|
deploy.nats.enable = false;
|
|
deploy.swarm-otel.enable = false;
|
|
deploy.victoriametrics.enable = false;
|
|
deploy.grafana.enable = false;
|
|
deploy.victorialogs.enable = false;
|
|
deploy.bao.enable = false;
|
|
deploy.forgejo.enable = false;
|
|
};
|
|
|
|
# Ports asked for on such a host. The request is the whole condition: the
|
|
# firewall hole exists because an operator named a port, not because the
|
|
# hive is running, and an agent reaching a host service is a claim about
|
|
# the host's own listeners either way.
|
|
exposedPortsNoHive = hive {
|
|
deploy.hive-controller.enable = false;
|
|
network.exposeHostPorts = [ 5432 ];
|
|
};
|
|
|
|
# The swarm UI where its own toggle is on — the control the absence arm
|
|
# needs, since nothing else in this suite renders this vhost and an arm
|
|
# saying "it is not there" would hold just as well if it were never there.
|
|
# Package stubbed per this file's header: the vhost roots at it.
|
|
swarmUiHere = hive {
|
|
deploy.swarm-ui.enable = true;
|
|
deploy.swarm-ui.package = pkgs.emptyDirectory;
|
|
};
|
|
|
|
# The swarm's shared services hosted HERE without the all-local mode — a
|
|
# services box with hives elsewhere, the shape ./host-modules/
|
|
# swarm-required-services.nix documents the switch for.
|
|
swarmServicesHere = hiveWithController { deploy.allSwarmServices = true; };
|
|
|
|
# Same, with one of those services placed on another host. Every derivation
|
|
# in that module is `mkDefault` so this stays expressible.
|
|
swarmServicesBaoElsewhere = hive {
|
|
deploy.allSwarmServices = true;
|
|
deploy.bao.enable = false;
|
|
};
|
|
|
|
# The swarm-wide forge objects are the controller's (forge/objects.rs), so
|
|
# the mirror list and the org avatar go to its unit, not hive-c0re's.
|
|
aMirror = {
|
|
upstream = "https://example.invalid/tool";
|
|
dest = "mirrors/tool";
|
|
};
|
|
controllerWithMirrors = hiveWithController {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
deploy.forgejo.actionsFromSelf = false;
|
|
c0re.orgAvatarPng = "/etc/marker-avatar.png";
|
|
};
|
|
mirrorsNoController = hive {
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
};
|
|
controllerWithSelfActions = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
deploy.forgejo.actionsFromSelf = true;
|
|
};
|
|
|
|
# The controller with every shared service on another host, and the same
|
|
# controller hosting them all: the swarm UI's links must not tell the two
|
|
# apart.
|
|
controllerAlone = hive { deploy.swarm-controller.enable = true; };
|
|
controllerWithServices = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.allSwarmServices = true;
|
|
};
|
|
|
|
# Each shared service with a web UI, and the link it must have: the label
|
|
# and the swarm-level domain the URL is built from.
|
|
swarmServiceLinkDomains =
|
|
let
|
|
s = controllerAlone.services.hyperhive.swarm;
|
|
in
|
|
{
|
|
Authelia = s.authelia.domain;
|
|
Grafana = s.grafana.domain;
|
|
Metrics = s.victoriametrics.domain;
|
|
Logs = s.victorialogs.domain;
|
|
Matrix = s.matrix.gatewayHost;
|
|
Bao = s.bao.ui.domain;
|
|
Forge = s.forge.domain;
|
|
};
|
|
swarmServiceLinksOf =
|
|
cfg:
|
|
lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links;
|
|
|
|
# Every service container at once: the services-only host plus the CI
|
|
# runner, the one container on a netns of its own.
|
|
serviceContainersWithCi = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.allSwarmServices = true;
|
|
deploy.ci-runner.enable = true;
|
|
};
|
|
serviceContainers = serviceContainersWithCi.containers;
|
|
cases = [
|
|
{
|
|
# `lib.all` over an empty set holds vacuously, so the roster is counted
|
|
# before it is read: a helper that lost a member would otherwise turn
|
|
# this case green by measuring nothing.
|
|
name = "hosting the swarm's shared services turns on every service that switch owns";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables swarmServicesHere;
|
|
in
|
|
lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es);
|
|
}
|
|
{
|
|
# The switch fills in for an operator who has not spoken and yields to
|
|
# one who has — that is what keeps a shared service placeable on a host
|
|
# of its own. A plain assignment or `mkForce` would satisfy both cases
|
|
# above and break this one. `nats` is the control: without it the case
|
|
# also passes on a fixture where nothing came on at all.
|
|
name = "placing one shared service elsewhere survives the switch that would enable it";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables swarmServicesBaoElsewhere;
|
|
in
|
|
!es.bao && es.nats;
|
|
}
|
|
{
|
|
# The controller sits on the OTHER tier: `singleHostSwarm` places it
|
|
# (./host-modules/local-defaults.nix) and swarm-ui follows the
|
|
# controller. Pinned so that moving a service between tiers is a
|
|
# decision someone makes rather than a merge nobody reads.
|
|
name = "hosting the swarm's shared services does not make a hive the swarm's control plane";
|
|
ok =
|
|
!swarmServicesHere.services.hyperhive.deploy.swarm-controller.enable
|
|
&& !swarmServicesHere.services.hyperhive.deploy.swarm-ui.enable;
|
|
}
|
|
{
|
|
# Control for the arm above, and the one place this slice is not inert:
|
|
# the ports are opened because they were named, on a host that never
|
|
# turned the hive on. The bridge firewall is the host's own, so there is
|
|
# nothing here for the hive toggle to have been protecting.
|
|
name = "a named exposeHostPorts opens its bridge port on the host's own say-so";
|
|
ok = builtins.elem 5432 (bridgePortsOrNone exposedPortsNoHive);
|
|
}
|
|
{
|
|
name = "the swarm UI claims the swarm apex where this host serves it";
|
|
ok = swarmUiHere.services.nginx.virtualHosts ? "t.local";
|
|
}
|
|
{
|
|
# The swarm-services toggle enables them explicitly, from its own
|
|
# module rather than from any of their defaults.
|
|
name = "the swarm-services toggle turns on the gateway, resolver and bridge by itself";
|
|
ok =
|
|
swarmServicesSwitchAlone.services.hyperhive.gateway.enable
|
|
&& swarmServicesSwitchAlone.services.hyperhive.gateway.dns.enable
|
|
&& swarmServicesSwitchAlone.services.hyperhive.network.enable;
|
|
}
|
|
{
|
|
# No hive, so no hive leaf, no hive identity and no hive-domain vhost:
|
|
# the default certificate is the swarm-services leaf, or a throwaway
|
|
# pair until it lands. A host fronting many swarm names waits for that
|
|
# leaf; a store-only host does not. The control is a hive host with
|
|
# the same services.
|
|
name = "a gateway host without a hive serves the swarm-services leaf as its default cert and has no hive-leaf unit";
|
|
ok =
|
|
let
|
|
# Every hive-leaf reference a unit could carry: the leaf files,
|
|
# the identity's file and subject. The store's own per-hive role
|
|
# unit is left out: it writes roles for the hives in the swarm
|
|
# directory, which live elsewhere.
|
|
hiveLeafRefs = [
|
|
"gateway.pem"
|
|
"gateway-key.pem"
|
|
"hive-leaf"
|
|
"swarm-bao-hive-leaf-"
|
|
];
|
|
noHiveLeafAnywhere =
|
|
m:
|
|
let
|
|
units = removeAttrs m.systemd.services [ "swarm-bao-hive-leaf-policy" ];
|
|
texts = lib.concatMap (u: [
|
|
(u.script or "")
|
|
(builtins.toJSON (u.environment or { }))
|
|
]) (lib.attrValues units);
|
|
in
|
|
!(lib.any (t: lib.any (r: lib.hasInfix r t) hiveLeafRefs) texts)
|
|
&& m.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
|
|
&& noHiveSurface m;
|
|
# No hive-domain vhost, and no vhost on the host carrying the
|
|
# hive's locations: c0re's `/health/`, the agent routes, swagger,
|
|
# or the hive domain's matrix discovery and `/matrix/` redirect.
|
|
noHiveSurface =
|
|
m:
|
|
let
|
|
vh = m.services.nginx.virtualHosts;
|
|
in
|
|
!(vh ? "h1.t.local")
|
|
&& !(lib.any (
|
|
v:
|
|
lib.any (l: (v.locations or { }) ? ${l}) [
|
|
"/health/"
|
|
"/agent/"
|
|
"/api/docs/"
|
|
"/matrix/"
|
|
"= /.well-known/matrix/client"
|
|
"= /.well-known/matrix/server"
|
|
]
|
|
|| lib.hasInfix "agents.conf" (v.extraConfig or "")
|
|
) (lib.attrValues vh));
|
|
noHive =
|
|
m:
|
|
let
|
|
s = m.systemd.services;
|
|
in
|
|
!(s ? hive-tls-resign) && !(m.systemd.timers ? hive-tls-resign) && noHiveLeafAnywhere m;
|
|
waitsForLeaf =
|
|
m:
|
|
let
|
|
s = m.systemd.services;
|
|
in
|
|
lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.requiredBy
|
|
&& lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.before;
|
|
# The cert it installs on the default and swarm-services paths:
|
|
# the services leaf when present, a throwaway pair otherwise.
|
|
importsServicesLeafOrFallback =
|
|
m:
|
|
let
|
|
imp = m.systemd.services.hive-gateway-self-signed-cert;
|
|
in
|
|
lib.hasInfix "cert=/var/lib/hive-tls/swarm-services.pem" imp.script
|
|
&& lib.hasInfix "openssl req -x509" imp.script
|
|
&& lib.hasInfix "cert=/var/lib/hive-gateway/tls/fallback.pem" imp.script
|
|
&& lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/cert.pem'' imp.script
|
|
&& lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/swarm-services.pem'' imp.script
|
|
&& lib.any (p: lib.hasPrefix "openssl" (p.pname or (p.name or ""))) imp.path;
|
|
store = storeOnlyNoHive;
|
|
hs = swarmServicesHere.systemd.services;
|
|
in
|
|
noHive swarmServicesOnly
|
|
&& noHive swarmServicesOnlyNoStore
|
|
&& noHive store
|
|
&& importsServicesLeafOrFallback swarmServicesOnly
|
|
&& importsServicesLeafOrFallback swarmServicesOnlyNoStore
|
|
&& importsServicesLeafOrFallback store
|
|
# Fronting many swarm names: the gateway waits for their leaf.
|
|
&& waitsForLeaf swarmServicesOnly
|
|
&& waitsForLeaf swarmServicesOnlyNoStore
|
|
# The store's UI alone: its nginx carries the passthrough to the
|
|
# store, so it starts on the fallback with the store sealed.
|
|
&&
|
|
store.services.hyperhive.swarm.localServiceDomains
|
|
== [ store.services.hyperhive.swarm.bao.ui.domain ]
|
|
&& !(waitsForLeaf store)
|
|
# The homeserver without a hive: no hive-domain vhost at all, and the
|
|
# homeserver's own vhost still proxies `/_matrix/`.
|
|
&& noHive matrixNoHive
|
|
&& (
|
|
let
|
|
gh = matrixNoHive.services.hyperhive.swarm.matrix.gatewayHost;
|
|
vh = matrixNoHive.services.nginx.virtualHosts;
|
|
in
|
|
gh != null && vh ? ${gh} && lib.any (lib.hasPrefix "/_matrix") (lib.attrNames vh.${gh}.locations)
|
|
)
|
|
&& !(store.services.nginx.virtualHosts ? "h1.t.local")
|
|
&& lib.hasInfix "want_svc=1" store.systemd.services.swarm-services-cert.script
|
|
# Control: a hive host with the same services keeps the hive leaf,
|
|
# its unit and identity, and the hive-domain vhost.
|
|
&& hs ? hive-tls-resign
|
|
&& lib.hasInfix "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" hs.hive-gateway-self-signed-cert.script
|
|
&& swarmServicesHere.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile != null
|
|
&& (
|
|
let
|
|
l = swarmServicesHere.services.nginx.virtualHosts."h1.t.local".locations;
|
|
in
|
|
l ? "/api/" && l ? "/health/" && l ? "= /.well-known/matrix/client"
|
|
)
|
|
&& !(noHiveLeafAnywhere swarmServicesHere);
|
|
}
|
|
{
|
|
# Each service is gated on its own deployment toggle alone, so a host
|
|
# that runs no hive still runs every shared service the switch turns
|
|
# on, plus the plumbing and the self-signed CA they are served through.
|
|
# Nothing starts the hive's own daemons. `hive-c0re` is checked by what
|
|
# starts it rather than by key: ../host-modules/hive-network.nix's bridge
|
|
# block writes to its environment wherever the bridge is up.
|
|
name = "a services-only host runs the swarm's shared services and no hive";
|
|
ok =
|
|
let
|
|
c = swarmServicesOnly.containers;
|
|
s = swarmServicesOnly.systemd.services;
|
|
h = swarmServicesOnly.services.hyperhive;
|
|
in
|
|
lib.all (m: c ? ${m}) [
|
|
"swarm-authelia"
|
|
"swarm-bao"
|
|
"swarm-grafana"
|
|
"swarm-victorialogs"
|
|
"swarm-victoriametrics"
|
|
"swarm-otel"
|
|
"swarm-nats"
|
|
"hive-matrix"
|
|
"hive-forge"
|
|
]
|
|
&& h.gateway.enable
|
|
&& h.gateway.dns.enable
|
|
&& h.network.enable
|
|
&& s ? hive-tls-ca
|
|
&& s ? swarm-bao-pki
|
|
&& (s.hive-c0re.wantedBy or [ ]) == [ ]
|
|
&& !(swarmServicesOnly.systemd.sockets ? hive-c0re)
|
|
&& !(swarmServicesOnly.systemd.sockets ? hive-priv)
|
|
&& !(s ? swarm-bao-queue-agent);
|
|
}
|
|
{
|
|
# On a controller that runs none of them, so the link cannot come from
|
|
# the service's own module.
|
|
name = "the swarm UI links every shared service with a web UI at its swarm domain";
|
|
ok =
|
|
lib.listToAttrs (map (l: lib.nameValuePair l.label l.url) (swarmServiceLinksOf controllerAlone))
|
|
== lib.mapAttrs (_: domain: "https://${domain}/") swarmServiceLinkDomains;
|
|
}
|
|
{
|
|
# Exact list equality also catches an entry rendered twice where the
|
|
# service runs on the controller's own host.
|
|
name = "the swarm UI's service links are the same whichever host runs the services";
|
|
ok =
|
|
lib.length (swarmServiceLinksOf controllerAlone)
|
|
== lib.length (lib.attrNames swarmServiceLinkDomains)
|
|
&& swarmServiceLinksOf controllerWithServices == swarmServiceLinksOf controllerAlone;
|
|
}
|
|
{
|
|
# The in-container option drives the container's firewall, and host
|
|
# `false` with container `true` would let the container's
|
|
# firewall.service rewrite the HOST ruleset. Counted first so a fixture
|
|
# that lost a container can't pass by comparing fewer of them.
|
|
name = "every service container restates its host-side privateNetwork";
|
|
ok =
|
|
lib.length (lib.attrNames serviceContainers) == 10
|
|
&& lib.all (
|
|
c: c.privateNetwork == (c.config.services.hyperhive.swarmContainer.privateNetwork or null)
|
|
) (lib.attrValues serviceContainers);
|
|
}
|
|
{
|
|
# Both values come from ../container-modules/swarm-container.nix.
|
|
name = "a service container on the host netns runs no firewall or resolvconf of its own";
|
|
ok =
|
|
let
|
|
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
|
|
in
|
|
lib.length shared == 9
|
|
&& lib.all (
|
|
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
|
|
) shared;
|
|
}
|
|
{
|
|
# The `privateNetwork = true` and `writesOwnResolvConf = false` arms:
|
|
# NixOS enables both by default, so the module must leave them alone.
|
|
name = "a service container on a netns of its own keeps its firewall and resolvconf";
|
|
ok =
|
|
let
|
|
c = serviceContainers.swarm-ci-runner.config;
|
|
in
|
|
c.networking.firewall.enable && c.networking.resolvconf.enable;
|
|
}
|
|
{
|
|
name = "a service container that sets no stateVersion of its own is on 26.05";
|
|
ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";
|
|
}
|
|
{
|
|
name = "a service container that pins its own stateVersion keeps it";
|
|
ok = serviceContainers.hive-forge.config.system.stateVersion == "25.11";
|
|
}
|
|
{
|
|
# An operator's explicit `false` beats every `mkDefault` assertion,
|
|
# which is what keeps "asserted by whoever needs it" from being a
|
|
# setting the operator cannot turn off.
|
|
name = "an explicit gateway.enable = false wins over the modules asserting it";
|
|
ok = !(hiveWithController { gateway.enable = false; }).services.nginx.enable;
|
|
}
|
|
{
|
|
name = "declared forge mirrors reach the swarm-controller, not hive-c0re";
|
|
ok =
|
|
builtins.fromJSON controllerWithMirrors.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS
|
|
== [ aMirror ]
|
|
&& !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HYPERHIVE_FORGE_MIRRORS);
|
|
}
|
|
{
|
|
name = "forge mirrors on a host without the controller warn that nothing seeds them";
|
|
ok =
|
|
let
|
|
warns = cfg: lib.any (lib.hasInfix "declare them on the swarm-controller's host") cfg.warnings;
|
|
in
|
|
warns mirrorsNoController && !(warns controllerWithMirrors);
|
|
}
|
|
{
|
|
# The mirror is what `self` resolves `actions/checkout` to, so the two
|
|
# come together. The control is the same forge without the option.
|
|
name = "actionsFromSelf adds the checkout mirror and resolves actions from the forge";
|
|
ok =
|
|
let
|
|
mirrorsOf =
|
|
m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS;
|
|
actionsUrlOf =
|
|
m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null;
|
|
in
|
|
lib.elem "actions/checkout" (map (m: m.dest) (mirrorsOf controllerWithSelfActions))
|
|
&& actionsUrlOf controllerWithSelfActions == "self"
|
|
&& mirrorsOf controllerWithMirrors == [ aMirror ]
|
|
&& actionsUrlOf controllerWithMirrors == null;
|
|
}
|
|
{
|
|
name = "the old c0re org-avatar option lands on the controller's unit";
|
|
ok =
|
|
controllerWithMirrors.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_CONFIG_ORG_AVATAR_PNG
|
|
== "/etc/marker-avatar.png"
|
|
&& !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HIVE_ORG_AVATAR_PNG);
|
|
}
|
|
];
|
|
in
|
|
runGroup "swarm-services-switch" cases
|