0f7b403803
A new swarm service, swarm-notify-router, in its own container on the host
that sets deploy.notify-router.enable (defaulted on by allSwarmServices where
the host has the router's store leaf, like the claude login minter).
- swarm-controller registers instance-wide hooks for `issues` +
`issue_assign` and `pull_request` + `pull_request_assign`, updates the
event list of a hook already registered with fewer (leaving its active flag as it is), and keeps each
verified body unchanged in the SWARM_EVENTS stream on
$SWARM.events.forge.<X-Forgejo-Event-Type>, deduped by X-Forgejo-Delivery.
A delivery the stream did not keep answers 503.
- swarm-notify-router drains its durable consumer on that stream, runs its
flows (source -> match -> todo sink, stored as nodes plus edges and checked
to be a tree) and queues each op on $SWARM.router.todo.<agent> in the
work-queue stream ROUTER_TODOS. Built-in flows: assigned, unassigned (same
key, rewritten) and closed (prefix-clears every holder's router todos). A
per-issue snapshot in its notify-router-state bucket holds the last
assignees and the todo holders; only agents in the agent-status bucket get
ops, and an assignee that was not one is left out of the snapshot so the
next assign event retries it. After the first event that must be retried,
the rest of the batch is nakked so redelivery keeps stream order.
- hive-agent creates its own durable consumer todo-<agent> and applies each
op to its todo store as subsystem `router`, then acks; a refused op stops
the rest of its batch the same way. The store's (subsystem, key) dedup
makes a redelivery harmless. New Todos::clear_prefix.
- swarm-nats-auth: a --notify-router-client arm; the controller may publish
$SWARM.events.forge.*; each agent gets three {agent} templates (extended
consumer create whose filter the server checks, MSG.NEXT, ACK).
- Credentials: a notify-router store leaf minted by glue-bao-tls, a
swarm-notify-router bao policy (read on its own queue client secret), an
authelia machine client, the secret publisher entry, a journal forwarder
route. No manual step.
hive-forge-notify is unchanged, so agents also still get forge's own
notifications.
Refs #5571
932 lines
41 KiB
Nix
932 lines
41 KiB
Nix
# `checks.module-eval-swarm-services-switch` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
bridgePortsOrNone
|
|
swarmServiceEnables
|
|
;
|
|
|
|
# A dedicated services host with hives elsewhere: the swarm-services
|
|
# toggle and no hive, the recipe docs/swarm/services.md gives.
|
|
swarmServicesOnly = shared.servicesHere;
|
|
|
|
# A services host with hives elsewhere and the store elsewhere too: no hive
|
|
# and no store identity minted here.
|
|
swarmServicesOnlyNoStore = swarmServicesBaoElsewhere;
|
|
|
|
# A dedicated secret-store host: no hive, and the store's browser UI the
|
|
# only swarm name its gateway fronts.
|
|
storeOnlyNoHive = shared.baoOnly;
|
|
|
|
# The homeserver on a host without a hive.
|
|
matrixNoHive = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.matrix.enable = true;
|
|
};
|
|
|
|
# The same switch with every service it owns placed elsewhere, so the only
|
|
# thing that can enable the gateway/resolver/bridge here is that toggle's
|
|
# own module — each service module asserts them from its own `enable`.
|
|
swarmServicesSwitchAlone = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.allSwarmServices = true;
|
|
deploy.matrix.enable = false;
|
|
deploy.authelia.enable = false;
|
|
deploy.nats.enable = false;
|
|
deploy.swarm-otel.enable = false;
|
|
deploy.victoriametrics.enable = false;
|
|
deploy.grafana.enable = false;
|
|
deploy.victorialogs.enable = false;
|
|
deploy.bao.enable = false;
|
|
deploy.forgejo.enable = false;
|
|
};
|
|
|
|
# Ports asked for on such a host. The request is the whole condition: the
|
|
# firewall hole exists because an operator named a port, not because the
|
|
# hive is running, and an agent reaching a host service is a claim about
|
|
# the host's own listeners either way.
|
|
exposedPortsNoHive = hive {
|
|
deploy.hive-controller.enable = false;
|
|
network.exposeHostPorts = [ 5432 ];
|
|
};
|
|
|
|
# The swarm UI on the controller's host — the control the absence arm
|
|
# needs, since nothing else in this suite renders this vhost and an arm
|
|
# saying "it is not there" would hold just as well if it were never there.
|
|
# Package stubbed per this file's header: the vhost roots at it. Set through
|
|
# its old `deploy.swarm-ui.package` spelling, which must reach the vhost.
|
|
#
|
|
# Also sets both ignored UI switches to `false`: the vhost must render
|
|
# regardless, and each must evaluate and warn. Neither
|
|
# is read anywhere else, so they change nothing the other arms see.
|
|
swarmUiHere = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.swarm-ui.package = pkgs.emptyDirectory;
|
|
deploy.swarm-ui.enable = false;
|
|
swarm.ui.enable = false;
|
|
};
|
|
|
|
# The swarm's shared services hosted HERE without the all-local mode — a
|
|
# services box with hives elsewhere, the shape ./host-modules/
|
|
# swarm-required-services.nix documents the switch for.
|
|
swarmServicesHere = shared.swarmServicesWithController;
|
|
|
|
# Same, with the operator opting out of the classifier and the builder,
|
|
# which the switch defaults on and everything else it owns stays on.
|
|
swarmServicesOptedOut = hiveWithController {
|
|
deploy.allSwarmServices = true;
|
|
deploy.review-classifier.enable = false;
|
|
deploy.builder.enable = false;
|
|
};
|
|
|
|
# Same, with the claude-code build the agents run pinned on this host. Any
|
|
# package serves: only the path the containers are pointed at is read.
|
|
swarmServicesPinnedClaude = hiveWithController {
|
|
deploy.allSwarmServices = true;
|
|
deploy.hive-controller.claudeCodePackage = pkgs.hello;
|
|
};
|
|
|
|
# Same, with one of those services placed on another host. Every derivation
|
|
# in that module is `mkDefault` so this stays expressible.
|
|
swarmServicesBaoElsewhere = hive {
|
|
deploy.allSwarmServices = true;
|
|
deploy.bao.enable = false;
|
|
};
|
|
|
|
# The swarm-wide forge objects are the controller's (forge/objects.rs), so
|
|
# the mirror list and the org avatar go to its unit, not hive-c0re's.
|
|
aMirror = {
|
|
upstream = "https://example.invalid/tool";
|
|
dest = "mirrors/tool";
|
|
};
|
|
controllerWithMirrors = hiveWithController {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
deploy.forgejo.actionsFromSelf = false;
|
|
deploy.swarm-controller.configOrgAvatarPng = "/etc/marker-avatar.png";
|
|
};
|
|
mirrorsNoController = hive {
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
};
|
|
forgeOnly = shared.forgeHere;
|
|
forgeWithController = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
};
|
|
controllerWithSelfActions = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.mirrors = [ aMirror ];
|
|
deploy.forgejo.actionsFromSelf = true;
|
|
};
|
|
|
|
# The controller with every shared service on another host, and the same
|
|
# controller hosting them all: the swarm UI's links must not tell the two
|
|
# apart.
|
|
inherit (shared) controllerAlone;
|
|
controllerWithServices = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.allSwarmServices = true;
|
|
};
|
|
|
|
# Each shared service with a web UI, and the link it must have: the label
|
|
# and the swarm-level domain the URL is built from.
|
|
swarmServiceLinkDomains =
|
|
let
|
|
s = controllerAlone.services.hyperhive.swarm;
|
|
in
|
|
{
|
|
Authelia = s.authelia.domain;
|
|
Grafana = s.grafana.domain;
|
|
Metrics = s.victoriametrics.domain;
|
|
Logs = s.victorialogs.domain;
|
|
Matrix = s.matrix.gatewayHost;
|
|
Bao = s.bao.ui.domain;
|
|
Forge = s.forge.domain;
|
|
};
|
|
swarmServiceLinksOf =
|
|
cfg:
|
|
lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links;
|
|
|
|
# Every service container at once: the services-only host plus the CI
|
|
# runner. The CI runner, the review classifier, the builder, the claude login
|
|
# minter, the metrics store, the homeserver and authelia are the seven containers on a
|
|
# netns of their own.
|
|
serviceContainersWithCi = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.allSwarmServices = true;
|
|
deploy.ci-runner.enable = true;
|
|
};
|
|
serviceContainers = serviceContainersWithCi.containers;
|
|
|
|
# The metrics store on a build older than its socket needs. Only its
|
|
# `assertions` are read, and the package is a stub carrying a version.
|
|
oldMetricsStore = hive {
|
|
deploy.victoriametrics.enable = true;
|
|
deploy.victoriametrics.package = pkgs.emptyDirectory // {
|
|
version = "1.152.0";
|
|
};
|
|
};
|
|
# Failing assertions first, then their messages: a message is only
|
|
# well-defined when its assertion fails (nixpkgs' fileSystems one throws
|
|
# otherwise).
|
|
failedMetricsStoreVersion =
|
|
host:
|
|
lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") (
|
|
map (a: a.message) (lib.filter (a: !a.assertion) host.assertions)
|
|
);
|
|
cases = [
|
|
{
|
|
# `lib.all` over an empty set holds vacuously, so the roster is counted
|
|
# before it is read: a helper that lost a member would otherwise turn
|
|
# this case green by measuring nothing.
|
|
name = "hosting the swarm's shared services turns on every service that switch owns";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables swarmServicesHere;
|
|
in
|
|
lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es);
|
|
}
|
|
{
|
|
# The container is the control that the toggle reaches the module, not
|
|
# only the option; `nats` is the control that the switch is on at all.
|
|
name = "hosting the swarm's shared services turns on the review classifier";
|
|
ok =
|
|
swarmServicesHere.services.hyperhive.deploy.review-classifier.enable
|
|
&& swarmServicesHere.containers ? swarm-review-classifier
|
|
&& swarmServicesHere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The default yields to an operator who has spoken. The same fixture
|
|
# without the line is the control.
|
|
name = "an explicit false keeps the review classifier off where the shared services are hosted";
|
|
ok =
|
|
!swarmServicesOptedOut.services.hyperhive.deploy.review-classifier.enable
|
|
&& !(swarmServicesOptedOut.containers ? swarm-review-classifier)
|
|
&& swarmServicesOptedOut.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# Same as the classifier's case above, for the builder: the opt-out is
|
|
# the only difference from `swarmServicesHere`, which has the container.
|
|
name = "an explicit false keeps the swarm builder off where the shared services are hosted";
|
|
ok =
|
|
!swarmServicesOptedOut.services.hyperhive.deploy.builder.enable
|
|
&& !(swarmServicesOptedOut.containers ? swarm-builder)
|
|
&& swarmServicesHere.containers ? swarm-builder
|
|
&& swarmServicesOptedOut.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The classifier's module asserts a store identity that only the store's
|
|
# host mints, so with the store placed elsewhere the default stays off
|
|
# rather than failing that host's eval. `nats` is the control.
|
|
name = "the review classifier stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.review-classifier.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-review-classifier)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The builder's module asserts its own store identity, so the same
|
|
# holds for it. `nats` is the control.
|
|
name = "the swarm builder stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.builder.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-builder)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# Static, not `DynamicUser`: the user is declared in the container and
|
|
# the unit names it. `nats` is the control that the switch is on at all.
|
|
name = "hosting the swarm's shared services runs the claude login minter in its container as a static user";
|
|
ok =
|
|
let
|
|
c = swarmServicesHere.containers.swarm-claude-login.config;
|
|
sc = c.systemd.services.swarm-claude-login.serviceConfig;
|
|
in
|
|
swarmServicesHere.services.hyperhive.deploy.claude-login.enable
|
|
&& c.users.users.swarm-claude-login.isSystemUser
|
|
&& sc.User == "swarm-claude-login"
|
|
&& !(sc.DynamicUser or false)
|
|
&& swarmServicesHere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The minter's module asserts the store identity the store's host mints,
|
|
# as the classifier's does.
|
|
name = "the claude login minter stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.claude-login.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-claude-login)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# Same shape as the minter's: a static user in its own container. `nats`
|
|
# is the control that the switch is on at all.
|
|
name = "hosting the swarm's shared services runs the notification router in its container as a static user";
|
|
ok =
|
|
let
|
|
c = swarmServicesHere.containers.swarm-notify-router.config;
|
|
sc = c.systemd.services.swarm-notify-router.serviceConfig;
|
|
in
|
|
swarmServicesHere.services.hyperhive.deploy.notify-router.enable
|
|
&& c.users.users.swarm-notify-router.isSystemUser
|
|
&& sc.User == "swarm-notify-router"
|
|
&& !(sc.DynamicUser or false)
|
|
&& swarmServicesHere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
name = "the notification router stays off where the shared services are hosted without the store";
|
|
ok =
|
|
!swarmServicesBaoElsewhere.services.hyperhive.deploy.notify-router.enable
|
|
&& !(swarmServicesBaoElsewhere.containers ? swarm-notify-router)
|
|
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# The CLI is the one the agents run, not the container's own nixpkgs'.
|
|
# Unpinned is the control: it falls back to the container's own `claude-code`.
|
|
# serve.json is what hive-c0re hands every agent's `HYPERHIVE_CLAUDE_CODE_PATH` from.
|
|
name = "the agents, the claude login minter and the review classifier run the claude-code this host pins";
|
|
ok =
|
|
let
|
|
claudeOf =
|
|
host: container: unit: var:
|
|
host.containers.${container}.config.systemd.services.${unit}.environment.${var};
|
|
pinned = "${pkgs.hello}/bin/claude";
|
|
servedClaude =
|
|
host:
|
|
(builtins.fromJSON (
|
|
builtins.unsafeDiscardStringContext host.environment.etc."hyperhive/serve.json".text
|
|
)).claude_code_path;
|
|
in
|
|
servedClaude swarmServicesPinnedClaude == "${pkgs.hello}"
|
|
&& servedClaude swarmServicesHere == null
|
|
&&
|
|
claudeOf swarmServicesPinnedClaude "swarm-claude-login" "swarm-claude-login"
|
|
"SWARM_CLAUDE_LOGIN_CLAUDE" == pinned
|
|
&&
|
|
claudeOf swarmServicesPinnedClaude "swarm-review-classifier" "swarm-review-classifier"
|
|
"SWARM_REVIEW_CLASSIFIER_CLAUDE" == pinned
|
|
&& lib.hasInfix "claude-code" (
|
|
claudeOf swarmServicesHere "swarm-claude-login" "swarm-claude-login" "SWARM_CLAUDE_LOGIN_CLAUDE"
|
|
)
|
|
&& lib.hasInfix "claude-code" (
|
|
claudeOf swarmServicesHere "swarm-review-classifier" "swarm-review-classifier"
|
|
"SWARM_REVIEW_CLASSIFIER_CLAUDE"
|
|
);
|
|
}
|
|
{
|
|
# The switch fills in for an operator who has not spoken and yields to
|
|
# one who has — that is what keeps a shared service placeable on a host
|
|
# of its own. A plain assignment or `mkForce` would satisfy both cases
|
|
# above and break this one. `nats` is the control: without it the case
|
|
# also passes on a fixture where nothing came on at all.
|
|
name = "placing one shared service elsewhere survives the switch that would enable it";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables swarmServicesBaoElsewhere;
|
|
in
|
|
!es.bao && es.nats;
|
|
}
|
|
{
|
|
# The controller sits on the OTHER tier: `singleHostSwarm` places it
|
|
# (./host-modules/local-defaults.nix) and swarm-ui follows the
|
|
# controller. Pinned so that moving a service between tiers is a
|
|
# decision someone makes rather than a merge nobody reads.
|
|
name = "hosting the swarm's shared services does not make a hive the swarm's control plane";
|
|
ok =
|
|
!swarmServicesHere.services.hyperhive.deploy.swarm-controller.enable
|
|
&& !(swarmServicesHere.services.nginx.virtualHosts ? "t.local");
|
|
}
|
|
{
|
|
# Control for the arm above, and the one place this slice is not inert:
|
|
# the ports are opened because they were named, on a host that never
|
|
# turned the hive on. The bridge firewall is the host's own, so there is
|
|
# nothing here for the hive toggle to have been protecting.
|
|
name = "a named exposeHostPorts opens its bridge port on the host's own say-so";
|
|
ok = builtins.elem 5432 (bridgePortsOrNone exposedPortsNoHive);
|
|
}
|
|
{
|
|
name = "the old swarm UI package spelling sets the controller's UI package, which the vhost serves";
|
|
ok =
|
|
let
|
|
uiPackage = swarmUiHere.services.hyperhive.deploy.swarm-controller.ui.package;
|
|
in
|
|
uiPackage == pkgs.emptyDirectory
|
|
&& swarmUiHere.services.nginx.virtualHosts."t.local".locations."/".root == "${uiPackage}"
|
|
&& lib.any (
|
|
w:
|
|
lib.hasInfix "services.hyperhive.deploy.swarm-ui.package" w
|
|
&& lib.hasInfix "services.hyperhive.deploy.swarm-controller.ui.package" w
|
|
) swarmUiHere.warnings;
|
|
}
|
|
{
|
|
# Control: `swarmServicesHere` sets neither ignored switch.
|
|
name = "a definition of an ignored swarm UI switch evaluates and warns, naming the option and the controller";
|
|
ok =
|
|
let
|
|
warned =
|
|
m: path:
|
|
lib.any (
|
|
w:
|
|
lib.hasInfix "services.hyperhive.${path} no longer has any effect" w
|
|
&& lib.hasInfix "swarm-ui now runs wherever swarm-controller runs" w
|
|
) m.warnings;
|
|
paths = [
|
|
"deploy.swarm-ui.enable"
|
|
"swarm.ui.enable"
|
|
];
|
|
in
|
|
lib.all (warned swarmUiHere) paths && !(lib.any (warned swarmServicesHere) paths);
|
|
}
|
|
{
|
|
# The swarm-services toggle enables them explicitly, from its own
|
|
# module rather than from any of their defaults.
|
|
name = "the swarm-services toggle turns on the gateway, resolver and bridge by itself";
|
|
ok =
|
|
swarmServicesSwitchAlone.services.hyperhive.gateway.enable
|
|
&& swarmServicesSwitchAlone.services.hyperhive.gateway.dns.enable
|
|
&& swarmServicesSwitchAlone.services.hyperhive.network.enable;
|
|
}
|
|
{
|
|
# No hive, so no hive leaf, no hive identity and no hive-domain vhost:
|
|
# the default certificate is the swarm-services leaf, or a throwaway
|
|
# pair until it lands. A host fronting many swarm names waits for that
|
|
# leaf; a store-only host does not. The control is a hive host with
|
|
# the same services.
|
|
name = "a gateway host without a hive serves the swarm-services leaf as its default cert and has no hive-leaf unit";
|
|
ok =
|
|
let
|
|
# Every hive-leaf reference a unit could carry: the leaf files,
|
|
# the identity's file and subject. The store's own per-hive role
|
|
# unit is left out: it writes roles for the hives in the swarm
|
|
# directory, which live elsewhere.
|
|
hiveLeafRefs = [
|
|
"gateway.pem"
|
|
"gateway-key.pem"
|
|
"hive-leaf"
|
|
"swarm-bao-hive-leaf-"
|
|
];
|
|
noHiveLeafAnywhere =
|
|
m:
|
|
let
|
|
units = removeAttrs m.systemd.services [ "swarm-bao-hive-leaf-policy" ];
|
|
texts = lib.concatMap (u: [
|
|
(u.script or "")
|
|
(builtins.toJSON (u.environment or { }))
|
|
]) (lib.attrValues units);
|
|
in
|
|
!(lib.any (t: lib.any (r: lib.hasInfix r t) hiveLeafRefs) texts)
|
|
&& m.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
|
|
&& noHiveSurface m;
|
|
# No hive-domain or dashboard vhost, and no vhost on the host carrying the
|
|
# hive's locations: c0re's `/health/`, the agent routes, swagger,
|
|
# or the hive domain's matrix discovery and `/matrix/` redirect.
|
|
noHiveSurface =
|
|
m:
|
|
let
|
|
vh = m.services.nginx.virtualHosts;
|
|
in
|
|
!(vh ? "h1.t.local")
|
|
&& !(vh ? "h1.hives.t.local")
|
|
&& !(lib.any (
|
|
v:
|
|
lib.any (l: (v.locations or { }) ? ${l}) [
|
|
"/health/"
|
|
"/agent/"
|
|
"/api/docs/"
|
|
"/matrix/"
|
|
"= /.well-known/matrix/client"
|
|
"= /.well-known/matrix/server"
|
|
]
|
|
|| lib.hasInfix "agents.conf" (v.extraConfig or "")
|
|
) (lib.attrValues vh));
|
|
noHive =
|
|
m:
|
|
let
|
|
s = m.systemd.services;
|
|
in
|
|
!(s ? hive-tls-resign) && !(m.systemd.timers ? hive-tls-resign) && noHiveLeafAnywhere m;
|
|
waitsForLeaf =
|
|
m:
|
|
let
|
|
s = m.systemd.services;
|
|
in
|
|
lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.requiredBy
|
|
&& lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.before;
|
|
# The cert it installs on the default and swarm-services paths:
|
|
# the services leaf when present.
|
|
importsServicesLeaf =
|
|
m:
|
|
let
|
|
imp = m.systemd.services.hive-gateway-self-signed-cert;
|
|
in
|
|
lib.hasInfix "cert=/var/lib/hive-tls/swarm-services.pem" imp.script
|
|
&& lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/cert.pem'' imp.script
|
|
&& lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/swarm-services.pem'' imp.script;
|
|
# Without it: a throwaway pair where the gateway does not wait for
|
|
# the leaf, a failure naming the issuing unit where it does.
|
|
standsInThrowaway =
|
|
m:
|
|
let
|
|
imp = m.systemd.services.hive-gateway-self-signed-cert;
|
|
in
|
|
lib.hasInfix "openssl req -x509" imp.script
|
|
&& lib.hasInfix "cert=/var/lib/hive-gateway/tls/fallback.pem" imp.script
|
|
&& lib.any (p: lib.hasPrefix "openssl" (p.pname or (p.name or ""))) imp.path;
|
|
failsWithoutLeaf =
|
|
m:
|
|
let
|
|
s = m.systemd.services.hive-gateway-self-signed-cert.script;
|
|
in
|
|
lib.hasInfix "swarm-services-cert.service issues it" s
|
|
&& lib.hasInfix "exit 1" s
|
|
&& !(lib.hasInfix "fallback.pem" s);
|
|
store = storeOnlyNoHive;
|
|
hs = swarmServicesHere.systemd.services;
|
|
in
|
|
noHive swarmServicesOnly
|
|
&& noHive swarmServicesOnlyNoStore
|
|
&& noHive store
|
|
&& importsServicesLeaf swarmServicesOnly
|
|
&& importsServicesLeaf swarmServicesOnlyNoStore
|
|
&& importsServicesLeaf store
|
|
# Fronting many swarm names: the gateway waits for their leaf, and
|
|
# its import fails without it.
|
|
&& waitsForLeaf swarmServicesOnly
|
|
&& waitsForLeaf swarmServicesOnlyNoStore
|
|
&& failsWithoutLeaf swarmServicesOnly
|
|
&& failsWithoutLeaf swarmServicesOnlyNoStore
|
|
&& standsInThrowaway store
|
|
&& !(failsWithoutLeaf store)
|
|
# The store's UI alone: its nginx carries the passthrough to the
|
|
# store, so it starts on the fallback with the store sealed.
|
|
&&
|
|
store.services.hyperhive.deploy.hive-controller.tls.localServiceDomains
|
|
== [ store.services.hyperhive.swarm.bao.ui.domain ]
|
|
&& !(waitsForLeaf store)
|
|
# The homeserver without a hive: no hive-domain vhost at all, and the
|
|
# homeserver's own vhost still proxies `/_matrix/`.
|
|
&& noHive matrixNoHive
|
|
&& (
|
|
let
|
|
gh = matrixNoHive.services.hyperhive.swarm.matrix.gatewayHost;
|
|
vh = matrixNoHive.services.nginx.virtualHosts;
|
|
in
|
|
gh != null && vh ? ${gh} && lib.any (lib.hasPrefix "/_matrix") (lib.attrNames vh.${gh}.locations)
|
|
)
|
|
&& !(store.services.nginx.virtualHosts ? "h1.t.local")
|
|
&& lib.hasInfix "want_svc=1" store.systemd.services.swarm-services-cert.script
|
|
# Control: a hive host with the same services keeps the hive leaf,
|
|
# its unit and identity, the dashboard's vhost and the hive-domain one.
|
|
&& hs ? hive-tls-resign
|
|
&& lib.hasInfix "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" hs.hive-gateway-self-signed-cert.script
|
|
&& swarmServicesHere.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile != null
|
|
&& (
|
|
let
|
|
vh = swarmServicesHere.services.nginx.virtualHosts;
|
|
l = vh."h1.hives.t.local".locations;
|
|
in
|
|
l ? "/api/" && l ? "/health/" && vh."h1.t.local".locations ? "= /.well-known/matrix/client"
|
|
)
|
|
&& !(noHiveLeafAnywhere swarmServicesHere);
|
|
}
|
|
{
|
|
# Each service is gated on its own deployment toggle alone, so a host
|
|
# that runs no hive still runs every shared service the switch turns
|
|
# on, plus the plumbing and the self-signed CA they are served through.
|
|
# Nothing starts the hive's own daemons. `hive-c0re` is checked by what
|
|
# starts it rather than by key: ../host-modules/hive-network.nix's bridge
|
|
# block writes to its environment wherever the bridge is up.
|
|
name = "a services-only host runs the swarm's shared services and no hive";
|
|
ok =
|
|
let
|
|
c = swarmServicesOnly.containers;
|
|
s = swarmServicesOnly.systemd.services;
|
|
h = swarmServicesOnly.services.hyperhive;
|
|
in
|
|
lib.all (m: c ? ${m}) [
|
|
"swarm-authelia"
|
|
"swarm-bao"
|
|
"swarm-grafana"
|
|
"swarm-victorialogs"
|
|
"swarm-victoriametrics"
|
|
"swarm-otel"
|
|
"swarm-nats"
|
|
"hive-matrix"
|
|
"hive-forge"
|
|
]
|
|
&& h.gateway.enable
|
|
&& h.gateway.dns.enable
|
|
&& h.network.enable
|
|
&& s ? hive-tls-ca
|
|
&& s ? swarm-bao-pki
|
|
&& (s.hive-c0re.wantedBy or [ ]) == [ ]
|
|
&& !(swarmServicesOnly.systemd.sockets ? hive-c0re)
|
|
&& !(swarmServicesOnly.systemd.sockets ? hive-priv)
|
|
&& !(s ? swarm-bao-queue-agent);
|
|
}
|
|
{
|
|
# On a controller that runs none of them, so the link cannot come from
|
|
# the service's own module.
|
|
name = "the swarm UI links every shared service with a web UI at its swarm domain";
|
|
ok =
|
|
lib.listToAttrs (map (l: lib.nameValuePair l.label l.url) (swarmServiceLinksOf controllerAlone))
|
|
== lib.mapAttrs (_: domain: "https://${domain}/") swarmServiceLinkDomains;
|
|
}
|
|
{
|
|
# Exact list equality also catches an entry rendered twice where the
|
|
# service runs on the controller's own host.
|
|
name = "the swarm UI's service links are the same whichever host runs the services";
|
|
ok =
|
|
lib.length (swarmServiceLinksOf controllerAlone)
|
|
== lib.length (lib.attrNames swarmServiceLinkDomains)
|
|
&& swarmServiceLinksOf controllerWithServices == swarmServiceLinksOf controllerAlone;
|
|
}
|
|
{
|
|
# The in-container option drives the container's firewall, and host
|
|
# `false` with container `true` would let the container's
|
|
# firewall.service rewrite the HOST ruleset. Counted first so a fixture
|
|
# that lost a container can't pass by comparing fewer of them.
|
|
name = "every service container restates its host-side privateNetwork";
|
|
ok =
|
|
lib.length (lib.attrNames serviceContainers) == 14
|
|
&& lib.all (
|
|
c: c.privateNetwork == (c.config.services.hyperhive.swarmContainer.privateNetwork or null)
|
|
) (lib.attrValues serviceContainers);
|
|
}
|
|
{
|
|
# Both values come from ../container-modules/swarm-container.nix.
|
|
name = "a service container on the host netns runs no firewall or resolvconf of its own";
|
|
ok =
|
|
let
|
|
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
|
|
in
|
|
lib.length shared == 6
|
|
&& lib.all (
|
|
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
|
|
) shared;
|
|
}
|
|
{
|
|
# The `privateNetwork = true` and `writesOwnResolvConf = false` arms:
|
|
# NixOS enables both by default, so the module must leave them alone.
|
|
name = "a service container on a netns of its own keeps its firewall and resolvconf";
|
|
ok =
|
|
let
|
|
c = serviceContainers.swarm-ci-runner.config;
|
|
in
|
|
c.networking.firewall.enable && c.networking.resolvconf.enable;
|
|
}
|
|
{
|
|
# No interface but loopback, so the socket is the store's only listener
|
|
# the host can reach. Every location that forwards to the store goes
|
|
# through it; the internal auth subrequests are the only other upstreams.
|
|
# The read, push and scrape locations are named so an empty filter
|
|
# can't pass.
|
|
name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
c = host.containers.swarm-victoriametrics;
|
|
dir = "/run/swarm-victoriametrics";
|
|
sock = "${dir}/victoriametrics.sock";
|
|
viaSock = path: "http://unix:${sock}:${path}";
|
|
locs =
|
|
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations;
|
|
toStore = map (l: l.proxyPass) (
|
|
lib.filter (l: l.proxyPass != null && !(lib.hasInfix "internal;" l.extraConfig)) (
|
|
lib.attrValues locs
|
|
)
|
|
);
|
|
unit = c.config.systemd.services.victoriametrics;
|
|
nginxGid = host.ids.gids.nginx;
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == null
|
|
&& c.hostAddress == null
|
|
&& c.config.services.victoriametrics.listenAddress == "unix:${sock}"
|
|
&& locs."= /api/v1/query".proxyPass == viaSock "/api/v1/query"
|
|
&& locs."= /opentelemetry/api/v1/push".proxyPass == viaSock "/opentelemetry/api/v1/push"
|
|
&& locs."= /metrics".proxyPass == viaSock "/metrics"
|
|
&& lib.all (lib.hasPrefix (viaSock "")) toStore
|
|
&& c.bindMounts.${dir}.hostPath == dir
|
|
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
|
&& c.config.users.groups.gateway-nginx.gid == nginxGid
|
|
&& unit.serviceConfig.Group == "gateway-nginx"
|
|
&& !unit.serviceConfig.DynamicUser
|
|
&& unit.serviceConfig.UMask == "0007"
|
|
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
|
|
}
|
|
{
|
|
# On the bridge by DHCP, like the CI runner, and its socket the only
|
|
# listener the gateway dials. `address` unset is what keeps tuwunel off
|
|
# TCP. Every vhost location that forwards goes through the socket; the
|
|
# `/_matrix/` one is named so an empty filter can't pass.
|
|
name = "the homeserver runs on a netns of its own and the gateway reaches it over its socket";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
net = host.services.hyperhive.network;
|
|
c = host.containers.hive-matrix;
|
|
dir = "/run/hive-matrix";
|
|
sock = "${dir}/tuwunel.sock";
|
|
locs =
|
|
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.matrix.gatewayHost}.locations;
|
|
forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs));
|
|
tuwunel = c.config.services.matrix-tuwunel;
|
|
unit = c.config.systemd.services.tuwunel;
|
|
nginxGid = host.ids.gids.nginx;
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == net.bridgeName
|
|
&& c.hostAddress == null
|
|
&& c.localAddress == null
|
|
&& c.config.networking.interfaces.eth0.useDHCP
|
|
&& c.config.networking.nameservers == [ net.bridgeIp ]
|
|
&& c.config.networking.firewall.enable
|
|
&& c.config.networking.resolvconf.enable
|
|
&& (tuwunel.settings.global.address or null) == null
|
|
&& tuwunel.settings.global.unix_socket_path == sock
|
|
&& tuwunel.settings.global.unix_socket_perms == 660
|
|
&& locs."/_matrix/".proxyPass == "http://unix:${sock}:"
|
|
&& forwarded == [ "http://unix:${sock}:" ]
|
|
&& c.bindMounts.${dir}.hostPath == dir
|
|
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
|
&& c.config.users.groups.gateway-nginx.gid == nginxGid
|
|
&& tuwunel.group == "gateway-nginx"
|
|
&& unit.serviceConfig.Group == "gateway-nginx"
|
|
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
|
}
|
|
{
|
|
# On the bridge by DHCP, with all three listeners on sockets: nothing
|
|
# the bridge can dial. Every vhost location that forwards goes through
|
|
# one of the sockets, except the internal authz subrequest, which is
|
|
# `https://`. The main, bridge and metrics locations are named so an
|
|
# empty filter can't pass.
|
|
name = "authelia runs on a netns of its own and the gateway reaches all three listeners over sockets";
|
|
ok =
|
|
let
|
|
host = serviceContainersWithCi;
|
|
net = host.services.hyperhive.network;
|
|
c = host.containers.swarm-authelia;
|
|
dir = "/run/swarm-authelia";
|
|
sock = n: "${dir}/${n}.sock";
|
|
locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.authelia.domain}.locations;
|
|
forwarded = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
|
|
map (l: l.proxyPass) (lib.attrValues locs)
|
|
);
|
|
inner = c.config;
|
|
settings = inner.services.authelia.instances.swarm.settings;
|
|
unit = inner.systemd.services.authelia-swarm;
|
|
bridge = inner.systemd.services.swarm-authelia-bridge;
|
|
nginxGid = host.ids.gids.nginx;
|
|
in
|
|
c.privateNetwork
|
|
&& c.hostBridge == net.bridgeName
|
|
&& c.hostAddress == null
|
|
&& c.localAddress == null
|
|
&& inner.networking.interfaces.eth0.useDHCP
|
|
&& inner.networking.firewall.enable
|
|
&& settings.server.address == "unix://${sock "authelia"}?umask=0007"
|
|
&& settings.telemetry.metrics.address == "unix://${sock "metrics"}?umask=0007"
|
|
&& bridge.environment.SWARM_AUTHELIA_BRIDGE_SOCKET == sock "bridge"
|
|
&& !(bridge.environment ? SWARM_AUTHELIA_BRIDGE_BIND)
|
|
&& locs."/".proxyPass == "http://unix:${sock "authelia"}:/"
|
|
&& locs."/bridge/".proxyPass == "http://unix:${sock "bridge"}:/"
|
|
&& locs."= /metrics".proxyPass == "http://unix:${sock "metrics"}:/"
|
|
&& lib.all (lib.hasPrefix "http://unix:${dir}/") forwarded
|
|
&& c.bindMounts.${dir}.hostPath == dir
|
|
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
|
&& inner.users.groups.gateway-nginx.gid == nginxGid
|
|
&& unit.serviceConfig.Group == "gateway-nginx"
|
|
&& bridge.serviceConfig.Group == "gateway-nginx"
|
|
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
|
}
|
|
{
|
|
# The DynamicUser layout keeps the history under /var/lib/private; the
|
|
# static user's StateDirectory is the plain path. The move runs as its
|
|
# own unit ahead of the store.
|
|
name = "the metrics store's history moves from the DynamicUser path before the store starts";
|
|
ok =
|
|
let
|
|
cfg = serviceContainers.swarm-victoriametrics.config;
|
|
m = cfg.systemd.services.victoriametrics-state-migrate;
|
|
in
|
|
cfg.services.victoriametrics.stateDir == "victoriametrics"
|
|
&& builtins.elem "victoriametrics.service" m.before
|
|
&& builtins.elem "victoriametrics.service" m.requiredBy
|
|
&& lib.hasInfix "old=/var/lib/private/victoriametrics" m.script
|
|
&& lib.hasInfix "new=/var/lib/victoriametrics" m.script
|
|
&& lib.hasInfix ''mv "$old" "$new"'' m.script
|
|
&& lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script;
|
|
}
|
|
{
|
|
# The control is the default package on the all-services host.
|
|
name = "a metrics store package older than 1.153.0 fails evaluation";
|
|
ok =
|
|
let
|
|
old = failedMetricsStoreVersion oldMetricsStore;
|
|
in
|
|
lib.length old == 1
|
|
&& lib.hasInfix "1.152.0" (lib.head old)
|
|
&& failedMetricsStoreVersion serviceContainersWithCi == [ ];
|
|
}
|
|
{
|
|
name = "a service container that sets no stateVersion of its own is on 26.05";
|
|
ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";
|
|
}
|
|
{
|
|
name = "a service container that pins its own stateVersion keeps it";
|
|
ok = serviceContainers.hive-forge.config.system.stateVersion == "25.11";
|
|
}
|
|
{
|
|
# An operator's explicit `false` beats every `mkDefault` assertion,
|
|
# which is what keeps "asserted by whoever needs it" from being a
|
|
# setting the operator cannot turn off.
|
|
name = "an explicit gateway.enable = false wins over the modules asserting it";
|
|
ok = !(hiveWithController { gateway.enable = false; }).services.nginx.enable;
|
|
}
|
|
{
|
|
name = "declared forge mirrors reach the swarm-controller, not hive-c0re";
|
|
ok =
|
|
builtins.fromJSON controllerWithMirrors.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS
|
|
== [ aMirror ]
|
|
&& !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HYPERHIVE_FORGE_MIRRORS);
|
|
}
|
|
{
|
|
name = "forge mirrors on a host without the controller warn that nothing seeds them";
|
|
ok =
|
|
let
|
|
warns = cfg: lib.any (lib.hasInfix "declare them on the swarm-controller's host") cfg.warnings;
|
|
in
|
|
warns mirrorsNoController && !(warns controllerWithMirrors);
|
|
}
|
|
{
|
|
# The mirrors are what `self` resolves `actions/*` to, so the two come
|
|
# together: one per action `.forgejo/workflows/` names. The control is
|
|
# the same forge without the option.
|
|
name = "actionsFromSelf adds the actions mirrors and resolves actions from the forge";
|
|
ok =
|
|
let
|
|
mirrorsOf =
|
|
m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS;
|
|
actionsUrlOf =
|
|
m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null;
|
|
in
|
|
map (m: m.dest) (mirrorsOf controllerWithSelfActions) == [
|
|
"mirrors/tool"
|
|
"actions/checkout"
|
|
"actions/upload-artifact"
|
|
]
|
|
&& actionsUrlOf controllerWithSelfActions == "self"
|
|
&& mirrorsOf controllerWithMirrors == [ aMirror ]
|
|
&& actionsUrlOf controllerWithMirrors == null;
|
|
}
|
|
{
|
|
# The default follows the controller, the only thing that seeds the
|
|
# checkout mirror: a forge alone must not resolve actions from itself.
|
|
name = "actionsFromSelf defaults to whether this host runs the swarm-controller";
|
|
ok =
|
|
let
|
|
actionsUrlOf =
|
|
m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null;
|
|
mirrorsOf =
|
|
m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS;
|
|
in
|
|
!forgeOnly.services.hyperhive.deploy.forgejo.actionsFromSelf
|
|
&& actionsUrlOf forgeOnly == null
|
|
&& forgeOnly.warnings == [ ]
|
|
&& !(
|
|
(forgeOnly.systemd.services.swarm-controller.environment or { }) ? SWARM_CONTROLLER_FORGE_MIRRORS
|
|
)
|
|
&& !(
|
|
(mirrorsNoController.systemd.services.swarm-controller.environment or { })
|
|
? SWARM_CONTROLLER_FORGE_MIRRORS
|
|
)
|
|
&& forgeWithController.services.hyperhive.deploy.forgejo.actionsFromSelf
|
|
&& actionsUrlOf forgeWithController == "self"
|
|
&& lib.elem "actions/checkout" (map (m: m.dest) (mirrorsOf forgeWithController));
|
|
}
|
|
{
|
|
name = "the org avatar lands on the controller's unit, not hive-c0re's";
|
|
ok =
|
|
controllerWithMirrors.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_CONFIG_ORG_AVATAR_PNG
|
|
== "/etc/marker-avatar.png"
|
|
&& !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HIVE_ORG_AVATAR_PNG);
|
|
}
|
|
{
|
|
# Where the store runs here, each reader waits for its container and
|
|
# the PKI unit minting the leaf it logs in with, and wants both. It
|
|
# never requires the PKI unit: a refused pki start fails a requirer's
|
|
# job with `dependency`, which its `Restart=on-failure` never retries.
|
|
# Elsewhere neither unit exists. On the store's host without the
|
|
# readers, none of them is rendered as a stub. Rosters are counted: an
|
|
# empty one holds vacuously.
|
|
name = "a store reader waits for and wants the store's container and PKI unit only where the store runs here, and requires neither";
|
|
ok =
|
|
let
|
|
machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine;
|
|
storeUnits = [
|
|
"swarm-bao-pki.service"
|
|
"container@${machine}.service"
|
|
];
|
|
readers = [
|
|
"swarm-bao-forge-oidc"
|
|
"swarm-bao-matrix-oidc"
|
|
"swarm-bao-hive-otel-oidc"
|
|
"swarm-bao-grafana-oidc"
|
|
"swarm-bao-grafana-admin"
|
|
"swarm-bao-grafana-metrics-bearer"
|
|
"swarm-bao-grafana-controller-token"
|
|
"swarm-bao-nats-callout-xkey"
|
|
"swarm-bao-ci-runner"
|
|
"swarm-bao-otel-oidc"
|
|
];
|
|
here = serviceContainersWithCi.systemd.services;
|
|
elsewhere = swarmServicesBaoElsewhere.systemd.services;
|
|
readersHere = lib.filter (u: here ? ${u}) readers;
|
|
readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers;
|
|
waits =
|
|
u:
|
|
here ? ${u}
|
|
&& lib.elem "swarm-bao-pki.service" here.${u}.after
|
|
&& lib.elem "swarm-bao-pki.service" here.${u}.wants
|
|
&& !(lib.any (n: lib.elem n here.${u}.requires) storeUnits)
|
|
&& lib.elem "container@${machine}.service" here.${u}.after
|
|
&& lib.elem "container@${machine}.service" here.${u}.wants;
|
|
namesNoStoreUnit =
|
|
u:
|
|
let
|
|
s = elsewhere.${u};
|
|
in
|
|
!(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
|
|
in
|
|
lib.all waits readers
|
|
&& lib.length readersHere == 10
|
|
&& lib.elem "swarm-bao-grafana-admin" readersHere
|
|
&& lib.length readersElsewhere == 7
|
|
&& lib.all namesNoStoreUnit readersElsewhere
|
|
&& !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
|
|
}
|
|
];
|
|
in
|
|
runGroup "swarm-services-switch" cases
|