swarm-ci-runner: run the forge CI runner as a swarm service
The CI runner was a hive deploy module: hive-ci.nix ran it beside the forge, and hive-c0re registered it through the forge's admin API with the `core` token, writing the registration token through hive-priv. It could only run on the forge's host, and it kept an admin-scoped forge token on a hive. Now the runner is a swarm service: - swarm-ci-runner.nix runs it in a `swarm-ci-runner` container on the host that sets deploy.ci-runner.enable, reaching the forge by name. It needs neither the forge nor a hive there. - swarm-controller reads the forge's runner registration token with its own admin token and writes it to swarm/services/ci-runner/ registration-token whenever it differs from what it last wrote. Its grant on that path is create/update only. - The runner's own leaf (CN swarm-ci-runner, role and policy of the same name) reads that one path. swarm-bao-ci-runner.service fetches it into the runner's EnvironmentFile and restarts the runner on a change. - A pre-start step removes .runner when the daemon's journal since its last start holds Forgejo's "unregistered runner" refusal (the daemon exits 1 when its startup Declare gets it), or when .runner names another forge address; upstream's pre-start step then registers again. Removed from the hive side: hive-ci.nix and deploy.forgejo.ci.*, forge/ci_runner.rs, PrivRequest::RegisterCiRunner and its handler and ReadWritePaths entry, InfraContainer::Ci with hivectl's --ci and the dashboard's hive-ci journal entry, HYPERHIVE_FORGE_CI_ENABLED, and read:admin,write:admin from CORE_TOKEN_SCOPES: nothing else in hive-c0re calls /api/v1/admin/*. The old ci options, under both swarm.forge.ci and deploy.forgejo.ci, now fail evaluation naming deploy.ci-runner. The forge's half of the old ci.enable, the actions/checkout mirror and DEFAULT_ACTIONS_URL = self, is deploy.forgejo.actionsFromSelf. Closes #4954
This commit is contained in:
atlas
mara
parent
3d43e9588c
commit
e2da5509ae
60 changed files with 1088 additions and 1201 deletions
|
|
@ -170,7 +170,7 @@ let
|
|||
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
|
||||
) baoGrantWithConsumers.systemd.services;
|
||||
|
||||
# The thirteen units that write a `swarm-*` grant, by name, for the discovery
|
||||
# The fourteen units that write a `swarm-*` grant, by name, for the discovery
|
||||
# control below.
|
||||
grantingUnitNames = [
|
||||
"swarm-bao-controller-policy"
|
||||
|
|
@ -186,6 +186,7 @@ let
|
|||
"swarm-bao-nats-auth-policy"
|
||||
"swarm-bao-operator-viewer-policy"
|
||||
"swarm-bao-hive-leaf-policy"
|
||||
"swarm-bao-ci-runner-policy"
|
||||
];
|
||||
|
||||
# Comment lines dropped first: both the HCL and the scripts explain
|
||||
|
|
@ -794,24 +795,24 @@ let
|
|||
}
|
||||
{
|
||||
# A store host without the granter's pair writes its grants some other
|
||||
# way, so none of the thirteen units may exist. Without this arm
|
||||
# way, so none of the fourteen units may exist. Without this arm
|
||||
# `lib.mkIf haveGranter` could be dropped from any of them and every other
|
||||
# case here would still pass.
|
||||
name = "without the granter's pair none of the thirteen granting units render";
|
||||
name = "without the granter's pair none of the fourteen granting units render";
|
||||
ok =
|
||||
let
|
||||
s = baoGranterOptOut.systemd.services;
|
||||
in
|
||||
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
|
||||
# The control: the same store with the pair renders all thirteen.
|
||||
# The control: the same store with the pair renders all fourteen.
|
||||
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
|
||||
}
|
||||
{
|
||||
# 🩸 What replaced the silent skip. With no bootstrap token the thirteen still
|
||||
# 🩸 What replaced the silent skip. With no bootstrap token the fourteen still
|
||||
# render, and a refused granter fails them with the step that fixes it.
|
||||
# A store host that never named a token is told to name one, since the
|
||||
# unit that sets the granter up renders only where it has.
|
||||
name = "a store host without a bootstrap token renders the thirteen, each failing loudly with the one-time step";
|
||||
name = "a store host without a bootstrap token renders the fourteen, each failing loudly with the one-time step";
|
||||
ok =
|
||||
let
|
||||
s = baoGranterNoToken.systemd.services;
|
||||
|
|
@ -1254,6 +1255,62 @@ let
|
|||
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
|
||||
&& !(lib.hasInfix "secret/metadata/swarm/hives" s);
|
||||
}
|
||||
{
|
||||
# `forge::runner_token` writes the runner's registration token and keeps
|
||||
# the last value in memory, so its stanza has no `read`. Pinned as the
|
||||
# whole stanza, so an added capability fails, and spelled to the leaf:
|
||||
# `services/*` would reach every swarm service's OIDC client secret.
|
||||
name = "the controller writes the CI runner's registration token and cannot read it back";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/services/ci-runner/registration-token\" {\n capabilities = [\"create\", \"update\"]\n}" s
|
||||
&& !(lib.hasInfix "secret/data/swarm/services/*" s);
|
||||
}
|
||||
{
|
||||
# The runner's role is the token's only reader. Its policy is the one
|
||||
# leaf, read-only, under the runner's own subject.
|
||||
name = "the CI runner's grant is its registration token and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-ci-runner-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/services/ci-runner/registration-token\" {" s
|
||||
&& lib.hasInfix "capabilities = [\"read\"]" s
|
||||
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 1
|
||||
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
|
||||
&& !(lib.hasInfix "secret/data/swarm/agents" s)
|
||||
&& !(lib.hasInfix "secret/data/swarm/hives" s)
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-ci-runner" s
|
||||
&& lib.hasInfix "allowed_common_names=swarm-ci-runner" s
|
||||
&& lib.hasInfix "token_policies=swarm-ci-runner" s;
|
||||
}
|
||||
{
|
||||
# Read access to the token is a grant on its path, so the policy units
|
||||
# that name it are the whole list of who reaches it: the controller,
|
||||
# which writes it, and the runner. The publisher's `services/*` write
|
||||
# also covers it and holds no read.
|
||||
name = "no granting unit but the controller's and the CI runner's names the runner's token";
|
||||
ok =
|
||||
let
|
||||
naming = lib.filter (
|
||||
u: lib.hasInfix "ci-runner/registration-token" baoGrantHere.systemd.services.${u}.script
|
||||
) grantingUnitNames;
|
||||
in
|
||||
lib.sort lib.lessThan naming == [
|
||||
"swarm-bao-ci-runner-policy"
|
||||
"swarm-bao-controller-policy"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "the PKI unit signs the CI runner's leaf under its own subject";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
|
||||
in
|
||||
lib.hasInfix "/ci-runner.pem ]" s && lib.hasInfix "swarm-ci-runner \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# Revocation, and the reason it is a stanza of its own: `delete` on the
|
||||
# `data/` path soft-deletes the newest version and leaves earlier ones
|
||||
|
|
@ -1386,8 +1443,8 @@ let
|
|||
}
|
||||
{
|
||||
# What makes the case above mean something: discovery by the granter's
|
||||
# certificate reaches all thirteen units, and each yields calls.
|
||||
name = "the granter-policy check sees all thirteen granting units, and parses calls from each";
|
||||
# certificate reaches all fourteen units, and each yields calls.
|
||||
name = "the granter-policy check sees all fourteen granting units, and parses calls from each";
|
||||
ok =
|
||||
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
|
||||
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)
|
||||
|
|
|
|||
|
|
@ -47,10 +47,10 @@ let
|
|||
# only hive without one is a hive told to have none.
|
||||
noAgentQueue = hiveWithController { deploy.hive-controller.queue.agentNatsUrl = null; };
|
||||
|
||||
# On the forge's host: the runner is refused anywhere else.
|
||||
# On the store's host, which mints the runner's store identity.
|
||||
withCi = hive {
|
||||
deploy.forgejo.enable = true;
|
||||
deploy.forgejo.ci.enable = true;
|
||||
deploy.bao.enable = true;
|
||||
deploy.ci-runner.enable = true;
|
||||
};
|
||||
|
||||
# A priority collision is a property of the *option*, not
|
||||
|
|
@ -63,7 +63,7 @@ let
|
|||
# stack (measured — this is not a hypothetical).
|
||||
forceCiServiceConfigs =
|
||||
let
|
||||
svcs = withCi.containers.hive-ci.config.systemd.services;
|
||||
svcs = withCi.containers.swarm-ci-runner.config.systemd.services;
|
||||
vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs);
|
||||
in
|
||||
builtins.foldl' (acc: v: builtins.seq v acc) true vals;
|
||||
|
|
@ -245,7 +245,7 @@ let
|
|||
# `gitea-actions-runner.nix` adds a plain `serviceConfig.*`
|
||||
# definition that collides with one of ours, and nix refuses to
|
||||
# merge two plain definitions at *host* eval. No other check
|
||||
# instantiates a host with `containers.hive-ci` actually enabled, so
|
||||
# instantiates a host with `containers.swarm-ci-runner` actually enabled, so
|
||||
# the collision only surfaces on operator deploy, not in CI.
|
||||
name = "the CI container's unit definitions merge without a priority collision";
|
||||
ok = forceCiServiceConfigs;
|
||||
|
|
|
|||
|
|
@ -46,11 +46,21 @@ let
|
|||
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
|
||||
};
|
||||
|
||||
ciNoForge = hive { deploy.forgejo.ci.enable = true; };
|
||||
ciWithForge = hive {
|
||||
deploy.forgejo.enable = true;
|
||||
deploy.forgejo.ci.enable = true;
|
||||
# The runner alone: no hive, no forge, no store, its store identity placed
|
||||
# by hand.
|
||||
ciAlone = hive {
|
||||
deploy.hive-controller.enable = false;
|
||||
deploy.ci-runner.enable = true;
|
||||
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
||||
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
||||
};
|
||||
ciNoIdentity = hive {
|
||||
deploy.hive-controller.enable = false;
|
||||
deploy.ci-runner.enable = true;
|
||||
};
|
||||
|
||||
# A host config that still names the runner under the forge.
|
||||
ciUnderForge = hive { deploy.forgejo.ci.enable = true; };
|
||||
|
||||
runsForge = m: m.services.hyperhive.deploy.forgejo.enable && m.containers ? hive-forge;
|
||||
|
||||
|
|
@ -122,13 +132,28 @@ let
|
|||
== "https://forge.t.local/user/oauth2/authelia/callback";
|
||||
}
|
||||
{
|
||||
# The runner reaches the forge through this host's gateway and is
|
||||
# registered through the local container. The second arm is the
|
||||
# control: a refusal that fires everywhere is not a check.
|
||||
name = "CI is refused on a host that does not run the forge";
|
||||
# The runner reaches the forge by name and reads its token with its own
|
||||
# certificate, so nothing about the host it runs on is asked.
|
||||
name = "the CI runner runs on a host with neither the forge nor a hive";
|
||||
ok =
|
||||
refusedOver ciNoForge "deploy.forgejo.enable on the same host"
|
||||
&& !(refusedOver ciWithForge "deploy.forgejo.enable on the same host");
|
||||
ciAlone.containers ? swarm-ci-runner
|
||||
&& !(ciAlone.containers ? hive-forge)
|
||||
&& !ciAlone.services.hyperhive.deploy.hive-controller.enable
|
||||
&& !(refusedOver ciAlone "ci-runner")
|
||||
&&
|
||||
ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT
|
||||
== "/etc/pki/bao-ci-runner.pem";
|
||||
}
|
||||
{
|
||||
# Without the identity the token never arrives; the control is the
|
||||
# placed pair above.
|
||||
name = "the CI runner is refused without its store identity";
|
||||
ok = refusedOver ciNoIdentity "deploy.bao.ciRunnerClientCertFile";
|
||||
}
|
||||
{
|
||||
# A removed option evaluates to an assertion naming the replacement.
|
||||
name = "a CI option under deploy.forgejo names its replacement under deploy.ci-runner";
|
||||
ok = refusedOver ciUnderForge "services.hyperhive.deploy.ci-runner.enable instead";
|
||||
}
|
||||
{
|
||||
# Nothing writes the delivery path away from the forge, and a
|
||||
|
|
|
|||
|
|
@ -116,6 +116,12 @@ let
|
|||
deploy.forgejo.enable = true;
|
||||
deploy.forgejo.mirrors = [ aMirror ];
|
||||
};
|
||||
controllerWithSelfActions = hive {
|
||||
deploy.swarm-controller.enable = true;
|
||||
deploy.forgejo.enable = true;
|
||||
deploy.forgejo.mirrors = [ aMirror ];
|
||||
deploy.forgejo.actionsFromSelf = true;
|
||||
};
|
||||
|
||||
# The controller with every shared service on another host, and the same
|
||||
# controller hosting them all: the swarm UI's links must not tell the two
|
||||
|
|
@ -150,7 +156,7 @@ let
|
|||
serviceContainersWithCi = hive {
|
||||
deploy.hive-controller.enable = false;
|
||||
deploy.allSwarmServices = true;
|
||||
deploy.forgejo.ci.enable = true;
|
||||
deploy.ci-runner.enable = true;
|
||||
};
|
||||
serviceContainers = serviceContainersWithCi.containers;
|
||||
cases = [
|
||||
|
|
@ -412,7 +418,7 @@ let
|
|||
name = "a service container on a netns of its own keeps its firewall and resolvconf";
|
||||
ok =
|
||||
let
|
||||
c = serviceContainers.hive-ci.config;
|
||||
c = serviceContainers.swarm-ci-runner.config;
|
||||
in
|
||||
c.networking.firewall.enable && c.networking.resolvconf.enable;
|
||||
}
|
||||
|
|
@ -446,6 +452,22 @@ let
|
|||
in
|
||||
warns mirrorsNoController && !(warns controllerWithMirrors);
|
||||
}
|
||||
{
|
||||
# The mirror is what `self` resolves `actions/checkout` to, so the two
|
||||
# come together. The control is the same forge without the option.
|
||||
name = "actionsFromSelf adds the checkout mirror and resolves actions from the forge";
|
||||
ok =
|
||||
let
|
||||
mirrorsOf =
|
||||
m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS;
|
||||
actionsUrlOf =
|
||||
m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null;
|
||||
in
|
||||
lib.elem "actions/checkout" (map (m: m.dest) (mirrorsOf controllerWithSelfActions))
|
||||
&& actionsUrlOf controllerWithSelfActions == "self"
|
||||
&& mirrorsOf controllerWithMirrors == [ aMirror ]
|
||||
&& actionsUrlOf controllerWithMirrors == null;
|
||||
}
|
||||
{
|
||||
name = "the old c0re org-avatar option lands on the controller's unit";
|
||||
ok =
|
||||
|
|
|
|||
Loading…
Reference in a new issue