09ad7386b2
swarm-bao-forge-oidc fetched the forge's OIDC client secret from bao and wrote it to /var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/<id>.secret, on the container's persistent root, where it outlived a bao rotation and was in every backup of that tree. Now the fetch writes it to /run/hive-forge-oidc/<id>.secret on the host (tmpfs, root 0400). The directory is bound read-only into the container at the same path, and forgejo-sso-source loads it through LoadCredential. A tmpfiles rule creates the directory at boot, so the bind source always exists, and removes the old persistent copy. After a reboot the file is absent until the fetch succeeds, and forgejo-sso-source fails to start until then; the fetch starts it again once it writes the file. Logins keep working meanwhile: forgejo holds the secret in its login-source row. deploy.forgejo.sso.clientSecretFile (and its old swarm.forge.sso name) is removed: it only chose where in the container the fetched file landed. The fetch no longer needs the container's forgejo uid, so it no longer fails on a fresh host before the container has created its state dir. Part of #4998.
443 lines
18 KiB
Nix
443 lines
18 KiB
Nix
# `checks.module-eval-nats-tls` — see ./lib.nix for the shared rationale (why
|
|
# this suite exists, naming convention, "evaluates not executes").
|
|
#
|
|
# The queue's name, its bao-issued leaf, and the clients that dial it.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
bridgePorts
|
|
;
|
|
|
|
natsName = "nats.t.local";
|
|
natsUrl = "tls://${natsName}:4222";
|
|
|
|
# Every service on one host. The queue, the store and every in-tree client of the queue
|
|
# are all here, so the scan below reads each of them.
|
|
inherit (shared) allLocal;
|
|
|
|
# The same host on the mesh.
|
|
allLocalMesh = hive {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.wireguard.enable = true;
|
|
deploy.wireguard.address = "10.100.0.1/24";
|
|
};
|
|
|
|
# The queue on a host whose store is elsewhere: no local policy unit.
|
|
queueNoStore = hive {
|
|
deploy.nats.enable = true;
|
|
};
|
|
|
|
# The store's host with the hive's own reader pointed somewhere else.
|
|
natsOnStoreHostReaderMoved = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.nats.enable = true;
|
|
deploy.nats.autoGenerateCallout = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
};
|
|
|
|
# A queue host holding only the hive's own store reader, placed by hand: no
|
|
# store here, and no queue leaf named.
|
|
queueWithHiveReaderOnly = hive {
|
|
deploy.nats.enable = true;
|
|
deploy.nats.autoGenerateCallout = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
};
|
|
|
|
# A hive that is not the queue's host, with nothing about the queue's
|
|
# address set by hand: what every hive but one in a multi-host swarm looks
|
|
# like. The controller is on too, since it may run away from the queue.
|
|
#
|
|
# The controller's store identity is what a hive away from the store already
|
|
# has to be handed, and it is not an address; without it this hive would
|
|
# fail assertions that have nothing to do with the queue.
|
|
remoteSecrets = {
|
|
deploy.swarm-controller.baoClientCertFile = "/var/lib/swarm-controller/bao-client.pem";
|
|
deploy.swarm-controller.baoClientKeyFile = "/var/lib/swarm-controller/bao-client-key.pem";
|
|
};
|
|
remote = hiveWithController (
|
|
lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; }
|
|
);
|
|
|
|
# The same hive given its own store leaf by hand, which is what turns
|
|
# publishing on away from the IdP's host: hive-c0re reads the status secret
|
|
# from the store under it. The queue-credential reader's pair comes with it,
|
|
# because a hive that reads the store is refused at eval without one.
|
|
hiveLeaf = {
|
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
deploy.bao.queueAgentClientCertFile = "/etc/pki/h1-queue-agent.pem";
|
|
deploy.bao.queueAgentClientKeyFile = "/etc/pki/h1-queue-agent-key.pem";
|
|
};
|
|
remotePublishing = hiveWithController (lib.recursiveUpdate remoteSecrets hiveLeaf);
|
|
|
|
# The same leaf with the queue's URL taken away.
|
|
remoteLeafNoUrl = hiveWithController (
|
|
lib.recursiveUpdate remoteSecrets (
|
|
lib.recursiveUpdate hiveLeaf { deploy.hive-controller.statusPublish.natsUrl = null; }
|
|
)
|
|
);
|
|
|
|
failedAssertions = m: lib.filter (a: !a.assertion) m.assertions;
|
|
|
|
policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script;
|
|
leafUnit = allLocal.systemd.services.swarm-bao-nats-tls;
|
|
natsContainer = allLocal.containers.swarm-nats.config;
|
|
natsTls = natsContainer.services.nats.settings.tls;
|
|
|
|
# Every `(nats|tls)://…` in a string.
|
|
urlsIn =
|
|
s: map builtins.head (builtins.filter builtins.isList (builtins.split "((nats|tls)://[^ '\"]+)" s));
|
|
|
|
# Every in-tree queue client, found rather than listed. The Rust client reads
|
|
# its address from `<PREFIX>_NATS_URL` (`swarm_queue_client::QueueConfig::
|
|
# from_env`), so any unit on the host or in a container that is handed one
|
|
# carries a variable of that shape. The responder takes a flag instead.
|
|
# Keyed by where each came from, so the control below can name them.
|
|
clientUrls =
|
|
machine:
|
|
let
|
|
fromUnits =
|
|
where: services:
|
|
lib.concatLists (
|
|
lib.mapAttrsToList (
|
|
unit: s:
|
|
lib.mapAttrsToList (var: v: {
|
|
name = "${where}/${unit}/${var}";
|
|
value = v;
|
|
}) (lib.filterAttrs (var: v: lib.hasSuffix "_NATS_URL" var && v != null) (s.environment or { }))
|
|
) services
|
|
);
|
|
containerUnits = lib.concatLists (
|
|
lib.mapAttrsToList (c: cc: fromUnits c cc.config.systemd.services) machine.containers
|
|
);
|
|
# The responder runs beside the queue only, so a hive without one has
|
|
# none to read.
|
|
responder =
|
|
map
|
|
(u: {
|
|
name = "swarm-nats/swarm-nats-auth/--nats-url";
|
|
value = u;
|
|
})
|
|
(
|
|
lib.optionals (machine.containers ? swarm-nats) (
|
|
urlsIn machine.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart
|
|
)
|
|
);
|
|
in
|
|
lib.listToAttrs (fromUnits "host" machine.systemd.services ++ containerUnits ++ responder);
|
|
|
|
scanned = clientUrls allLocal;
|
|
|
|
cases = [
|
|
{
|
|
# Control first: a scan that found nothing would pass the next case
|
|
# vacuously. These are the four clients in the tree today.
|
|
name = "the client scan finds hive-c0re, the agents, the controller and the responder";
|
|
ok = lib.all (k: scanned ? ${k}) [
|
|
"host/hive-c0re/HIVE_C0RE_NATS_URL"
|
|
"host/hive-c0re/HIVE_AGENT_NATS_URL"
|
|
"host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
|
"swarm-nats/swarm-nats-auth/--nats-url"
|
|
];
|
|
}
|
|
{
|
|
# The server requires TLS and its leaf carries the name alone, so a
|
|
# `nats://` URL or an address is a client that cannot connect. Every one
|
|
# found, not the four above: a client added later is held to it too.
|
|
name = "every in-tree queue client dials tls://<the queue's name>:4222";
|
|
ok = lib.all (u: u == natsUrl) (lib.attrValues scanned);
|
|
}
|
|
{
|
|
# The option defaults the scan reads through, so a client that stops
|
|
# reading them does not also escape the property above.
|
|
name = "the queue URL options default to the name on the queue's host";
|
|
ok =
|
|
let
|
|
d = allLocal.services.hyperhive.deploy;
|
|
in
|
|
d.hive-controller.statusPublish.natsUrl == natsUrl
|
|
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
|
&& allLocal.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
|
}
|
|
{
|
|
name = "the queue's name is served by this host's resolver, at the bridge address";
|
|
ok =
|
|
lib.elem natsName allLocal.services.hyperhive.gateway.localNames
|
|
&& lib.elem "/${natsName}/${allLocal.services.hyperhive.network.bridgeIp}" allLocal.services.dnsmasq.settings.address;
|
|
}
|
|
{
|
|
name = "the queue's pki role issues for its name alone";
|
|
ok = lib.all (arg: lib.hasInfix arg policyScript) [
|
|
"roles/swarm-nats \\"
|
|
"allowed_domains=${lib.escapeShellArg natsName} \\"
|
|
"allow_bare_domains=true \\"
|
|
"allow_subdomains=false \\"
|
|
"allow_glob_domains=false \\"
|
|
"allow_localhost=false \\"
|
|
"allow_any_name=false \\"
|
|
"allow_ip_sans=false \\"
|
|
"server_flag=true \\"
|
|
"client_flag=false \\"
|
|
];
|
|
}
|
|
{
|
|
# Every `path` the policy names, not a search for the one expected: a
|
|
# second grant added later fails here.
|
|
name = "the queue's policy grants pki/issue/swarm-nats and nothing else";
|
|
ok =
|
|
let
|
|
paths = map builtins.head (
|
|
builtins.filter builtins.isList (builtins.split "path \"([^\"]*)\"" policyScript)
|
|
);
|
|
in
|
|
paths == [ "pki/issue/swarm-nats" ]
|
|
&& lib.hasInfix ''capabilities = ["update"]'' policyScript
|
|
&& lib.hasInfix "allowed_common_names=swarm-nats" policyScript
|
|
&& lib.hasInfix "token_policies=swarm-nats" policyScript;
|
|
}
|
|
{
|
|
# Mint to consume: the leaf the unit writes is the one the server reads,
|
|
# and the login leaf glue-bao-tls signs is the one the unit presents.
|
|
name = "the server serves the leaf the host unit issues, and the unit logs in as swarm-nats";
|
|
ok =
|
|
natsTls.cert_file == "/var/lib/swarm-nats-tls/cert.pem"
|
|
&& natsTls.key_file == "/run/credentials/nats.service/tls-key"
|
|
&& lib.elem "tls-key:/var/lib/swarm-nats-tls/key.pem" natsContainer.systemd.services.nats.serviceConfig.LoadCredential
|
|
&& allLocal.containers.swarm-nats.bindMounts ? "/var/lib/swarm-nats-tls"
|
|
&& lib.hasInfix "d=/var/lib/swarm-nats-tls" leafUnit.script
|
|
&& lib.hasInfix "pki/issue/swarm-nats" leafUnit.script
|
|
&& leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats.pem"
|
|
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script
|
|
&& lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script;
|
|
}
|
|
{
|
|
# The hive's own reader is not the queue's identity, and no path is
|
|
# derived from it: a queue host without the store that holds only the
|
|
# hive's reader gets the leaf unit that fails naming the options, like
|
|
# one holding nothing.
|
|
name = "a queue host without the store is refused when it holds only the hive's reader";
|
|
ok =
|
|
let
|
|
u = queueWithHiveReaderOnly.systemd.services.swarm-bao-nats-tls;
|
|
in
|
|
queueWithHiveReaderOnly.services.hyperhive.deploy.nats.baoClientCertFile == null
|
|
&& !(u.environment ? BAO_CLIENT_CERT)
|
|
&& lib.hasInfix "services.hyperhive.deploy.nats.baoClient{Cert,Key}File" u.script;
|
|
}
|
|
{
|
|
# The default is the file the minting unit writes, wherever the hive's
|
|
# own reader is pointed.
|
|
name = "on the store's host the queue's leaf is the one minted there, not one beside the hive's reader";
|
|
ok =
|
|
let
|
|
n = natsOnStoreHostReaderMoved.services.hyperhive.deploy.nats;
|
|
mint = natsOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script;
|
|
in
|
|
n.baoClientCertFile == "/var/lib/swarm-bao-pki/nats.pem"
|
|
&& n.baoClientKeyFile == "/var/lib/swarm-bao-pki/nats-key.pem"
|
|
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ] ||" mint
|
|
&& lib.hasInfix "/var/lib/swarm-bao-pki nats " mint;
|
|
}
|
|
{
|
|
name = "the server requires TLS: no allow_non_tls";
|
|
ok = !(natsContainer.services.nats.settings ? allow_non_tls);
|
|
}
|
|
{
|
|
name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
|
|
ok =
|
|
lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts
|
|
&& !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
|
|
&& lib.elem 4222 (bridgePorts allLocalMesh);
|
|
}
|
|
{
|
|
name = "4222 is not opened on wg-hive when this host is not on the mesh";
|
|
ok =
|
|
!(lib.elem 4222
|
|
(allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts
|
|
)
|
|
&& !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts);
|
|
}
|
|
{
|
|
# Ordering, never a requirement: a policy unit that failed still counts
|
|
# as done, and the leaf unit's own retries carry it past that.
|
|
name = "the leaf unit is ordered after its policy unit, with no requires";
|
|
ok =
|
|
let
|
|
p = "swarm-bao-nats-tls-policy.service";
|
|
in
|
|
lib.elem p leafUnit.after && lib.elem p leafUnit.wants && !(lib.elem p (leafUnit.requires or [ ]));
|
|
}
|
|
{
|
|
name = "a queue host whose store is elsewhere orders its leaf unit after no policy unit";
|
|
ok =
|
|
let
|
|
u = queueNoStore.systemd.services.swarm-bao-nats-tls;
|
|
in
|
|
!(lib.elem "swarm-bao-nats-tls-policy.service" u.after)
|
|
&& !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants);
|
|
}
|
|
{
|
|
# A oneshot has no start timeout by default, so an attempt that never
|
|
# ends never fails into a retry, and the daily timer's start is a no-op.
|
|
name = "the leaf unit bounds its attempt";
|
|
ok =
|
|
!(lib.elem (toString (leafUnit.serviceConfig.TimeoutStartSec or "infinity")) [
|
|
"infinity"
|
|
"0"
|
|
]);
|
|
}
|
|
{
|
|
# Every hive dials the queue by the same name, so a hive away from it
|
|
# needs no URL of its own. Control and property in one: the scan must
|
|
# reach the controller and the agents' address here, and every URL it
|
|
# finds, like the options it reads through, is the name.
|
|
name = "a hive that is not the queue's host dials tls://<the queue's name>:4222 with nothing set";
|
|
ok =
|
|
let
|
|
s = clientUrls remote;
|
|
d = remote.services.hyperhive.deploy;
|
|
in
|
|
!d.nats.enable
|
|
&& s ? "host/hive-c0re/HIVE_AGENT_NATS_URL"
|
|
&& s ? "host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
|
&& lib.all (u: u == natsUrl) (lib.attrValues s)
|
|
&& d.hive-controller.statusPublish.natsUrl == natsUrl
|
|
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
|
&& remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
|
}
|
|
{
|
|
# The URL is set and the hive holds no store leaf, which is every such
|
|
# hive until an operator places one: publishing is off, not
|
|
# misconfigured.
|
|
name = "that hive evaluates without an assertion failure";
|
|
ok = failedAssertions remote == [ ];
|
|
}
|
|
{
|
|
# Off means off: hive-c0re is handed no status coordinates it could not
|
|
# fetch a secret for.
|
|
name = "without a store leaf, that hive's hive-c0re is given no status coordinates";
|
|
ok = !(remote.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL);
|
|
}
|
|
{
|
|
# The leaf alone turns publishing on, at the default URL, with the
|
|
# secret read from the store rather than named as a file.
|
|
name = "given its store leaf, that hive publishes to the queue's name as hive-<name>";
|
|
ok =
|
|
let
|
|
s = remotePublishing.systemd.services.hive-c0re;
|
|
in
|
|
failedAssertions remotePublishing == [ ]
|
|
&& s.environment.HIVE_C0RE_NATS_URL == natsUrl
|
|
&& s.environment.HIVE_C0RE_OIDC_CLIENT_ID == "hive-h1"
|
|
&& !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE);
|
|
}
|
|
{
|
|
# The host that runs the queue, the IdP and the store gets the same
|
|
# coordinates from the leaf ../host-modules/glue-bao-tls.nix mints there,
|
|
# and no file.
|
|
name = "the all-local hive publishes from the store too";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
e.HIVE_C0RE_NATS_URL == natsUrl
|
|
&& e.HIVE_C0RE_OIDC_CLIENT_ID == "hive-h1"
|
|
&& !(e ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE);
|
|
}
|
|
{
|
|
# A null URL turns publishing off.
|
|
name = "a hive with a store leaf and no queue URL is given no status coordinates";
|
|
ok =
|
|
failedAssertions remoteLeafNoUrl == [ ]
|
|
&& !(remoteLeafNoUrl.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL);
|
|
}
|
|
{
|
|
# Only `/metrics`, from the exporter, behind the collector's subrequest:
|
|
# no `error_page` (a scraper would parse a login page as metrics), the
|
|
# monitoring endpoint never proxied, every other path refused. In
|
|
# self-signed mode the vhost names the queue's own leaf, as copied for
|
|
# nginx, not the swarm-services one.
|
|
name = "the queue's name serves only /metrics, behind the collector's auth subrequest, on the queue's own leaf";
|
|
ok =
|
|
let
|
|
h = allLocal.services.hyperhive;
|
|
v = allLocal.services.nginx.virtualHosts.${natsName};
|
|
locs = v.locations;
|
|
checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig);
|
|
authz = locs."= /__nats_metrics_authz";
|
|
in
|
|
lib.attrNames locs == [
|
|
"/"
|
|
"= /__nats_metrics_authz"
|
|
"= /metrics"
|
|
]
|
|
&& locs."/".return == "403"
|
|
&& locs."/".proxyPass == null
|
|
&& locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.nats.metricsPort}/metrics"
|
|
&& checksOf locs."= /metrics" == [ "auth_request /__nats_metrics_authz;" ]
|
|
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
|
|
&& authz.proxyPass == "https://auth.t.local/api/authz/auth-request"
|
|
&& lib.hasInfix "internal;" authz.extraConfig
|
|
&& !(lib.any (
|
|
l:
|
|
lib.hasInfix ":${toString h.swarm.nats.monitorPort}" (
|
|
if l.proxyPass == null then "" else l.proxyPass
|
|
)
|
|
) (lib.attrValues locs))
|
|
&& v.forceSSL
|
|
&& v.sslCertificate == "/var/lib/swarm-nats-gateway-tls/cert.pem"
|
|
&& v.sslCertificateKey == "/var/lib/swarm-nats-gateway-tls/key.pem";
|
|
}
|
|
{
|
|
# A host whose only swarm service is the queue: its gateway serves the
|
|
# queue's name with the queue's leaf, fronts no swarm-services name, and
|
|
# so neither waits for nor requires the swarm-services leaf. The copy
|
|
# unit is only wanted by nginx, never required, and a rotated queue leaf
|
|
# re-runs it.
|
|
name = "a host running only the queue renders its gateway without requiring the swarm-services leaf";
|
|
ok =
|
|
let
|
|
h = queueNoStore.services.hyperhive;
|
|
s = queueNoStore.systemd.services;
|
|
copy = s.swarm-nats-gateway-cert;
|
|
in
|
|
h.deploy.nats.enable
|
|
&& !h.deploy.hive-controller.enable
|
|
&& h.gateway.enable
|
|
&& h.gateway.useSelfSigned
|
|
&&
|
|
queueNoStore.services.nginx.virtualHosts.${natsName}.sslCertificate
|
|
== "/var/lib/swarm-nats-gateway-tls/cert.pem"
|
|
&& h.swarm.localServiceDomains == [ ]
|
|
&& !(lib.elem natsName h.swarm.serviceDomains)
|
|
&& !(lib.elem "hive-gateway-self-signed-cert.service" (s.swarm-services-cert.requiredBy or [ ]))
|
|
&& !(lib.hasInfix "serves swarm names only with it" s.hive-gateway-self-signed-cert.script)
|
|
&& lib.elem "nginx.service" copy.wantedBy
|
|
&& lib.elem "nginx.service" copy.before
|
|
&& !(lib.elem "nginx.service" (copy.requiredBy or [ ]))
|
|
&& lib.hasInfix "throwaway self-signed" copy.script
|
|
&& lib.hasInfix "chgrp nginx" copy.script
|
|
&& lib.hasInfix "systemctl start swarm-nats-gateway-cert.service" s.swarm-bao-nats-tls.script;
|
|
}
|
|
];
|
|
in
|
|
runGroup "nats-tls" cases
|