agents fetch the hive CA from bao at boot instead of the meta flake embedding it
This commit is contained in:
atlas
mara
parent
0831b85b73
commit
d547b521f6
12 changed files with 329 additions and 246 deletions
|
|
@ -43,6 +43,7 @@ in
|
|||
./frontend.nix
|
||||
./github.nix
|
||||
./github-token.nix
|
||||
./hive-ca.nix
|
||||
./hive-facts.nix
|
||||
./logs.nix
|
||||
./matrix.nix
|
||||
|
|
|
|||
193
nix/agent-modules/hive-ca.nix
Normal file
193
nix/agent-modules/hive-ca.nix
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
# The swarm's root certificate in this agent's trust, fetched from the store
|
||||
# at boot by the agent itself.
|
||||
#
|
||||
# The root is the store's services `pki` mount's CA. It does not exist when the
|
||||
# agent is built, so the meta flake names no CA file: this unit reads it from
|
||||
# the store with the agent's own certificate (./bao.nix keeps it current) and
|
||||
# writes two files, both public:
|
||||
#
|
||||
# - `bundle`: the system CAs plus the swarm root. `SSL_CERT_FILE` replaces the
|
||||
# trust store of Rust (rustls platform verifier), Go and OpenSSL clients, so
|
||||
# it carries both halves.
|
||||
# - `root`: the swarm root alone, for `NODE_EXTRA_CA_CERTS`, which adds to
|
||||
# Node's store (Claude Code).
|
||||
#
|
||||
# A process reads its trust when it starts, so a rotated root reaches the
|
||||
# consumers below at their next start. They `Requires=` this unit, which keeps
|
||||
# one from running with a store that lacks the root.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.hyperhive.agent.bao;
|
||||
agentCfg = config.services.hyperhive.agent;
|
||||
|
||||
agentName = agentCfg.user.name;
|
||||
|
||||
# The server CA id ./bao.nix loads.
|
||||
serverCaCredential = "hive-agent-bao-server-ca";
|
||||
|
||||
unitName = "hive-agent-trust-bundle";
|
||||
runtimeDir = unitName;
|
||||
bundleFile = "/run/${runtimeDir}/bundle.pem";
|
||||
rootFile = "/run/${runtimeDir}/root.pem";
|
||||
|
||||
systemBundle = "/etc/ssl/certs/ca-certificates.crt";
|
||||
|
||||
configured = cfg.addr != null;
|
||||
|
||||
storeRetry = import ../host-modules/lib/store-retry.nix { };
|
||||
|
||||
# Units that dial a swarm service over TLS. The first group is defined
|
||||
# whenever the agent module is; the rest exist under the same condition as
|
||||
# their definitions, because ordering on a unit that is not defined would
|
||||
# generate an inert one.
|
||||
consumers = [
|
||||
"hive-agent"
|
||||
"hive-bash-daemon"
|
||||
"hive-subagent-daemon"
|
||||
"hive-mcp-http"
|
||||
"hive-forge-notify"
|
||||
]
|
||||
++ lib.optional (agentCfg.matrixAccounts != { }) "hive-matrix-daemon"
|
||||
++ lib.optional (agentCfg.github.enable) "hive-github-notify"
|
||||
++ lib.optional (agentCfg.icon != null && agentCfg.forge.url != null) "forge-avatar-sync";
|
||||
|
||||
trustEnv = {
|
||||
SSL_CERT_FILE = bundleFile;
|
||||
NODE_EXTRA_CA_CERTS = rootFile;
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.agent.bao.servicesPkiMount = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "pki";
|
||||
example = "pki";
|
||||
description = ''
|
||||
Mount path of the store's services `pki` engine, whose root is the one
|
||||
anchor every swarm service certificate chains to. Set by the generated
|
||||
meta flake from the host's
|
||||
`services.hyperhive.deploy.bao.servicesPkiMountPath`.
|
||||
`${unitName}.service` reads `<mount>/ca/pem` from it.
|
||||
'';
|
||||
};
|
||||
|
||||
config = lib.mkIf configured {
|
||||
# Every process, including a bash task's children, resolves trust through
|
||||
# these, as it resolves `HIVE_FORGE_TOKEN_FILE`.
|
||||
systemd.globalEnvironment = trustEnv;
|
||||
environment.variables = trustEnv;
|
||||
|
||||
systemd.services = {
|
||||
${unitName} = {
|
||||
description = "fetch the swarm root certificate from the secret store";
|
||||
after = [
|
||||
"network.target"
|
||||
"hive-agent-bao-identity.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.openssl
|
||||
];
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = false;
|
||||
TimeoutStartSec = 60;
|
||||
User = agentName;
|
||||
Group = agentName;
|
||||
RuntimeDirectory = runtimeDir;
|
||||
# Public certificates, read by every consumer's user.
|
||||
RuntimeDirectoryMode = "0755";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
UMask = "0022";
|
||||
LoadCredential = [ serverCaCredential ];
|
||||
};
|
||||
# Overrides the global value: this unit runs before the bundle exists,
|
||||
# and its own TLS is checked against the credential below or the
|
||||
# system bundle.
|
||||
environment = {
|
||||
SSL_CERT_FILE = lib.mkForce systemBundle;
|
||||
BAO_ADDR = cfg.addr;
|
||||
BAO_CLIENT_CERT = cfg.clientCertFile;
|
||||
BAO_CLIENT_KEY = cfg.clientKeyFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
|
||||
if [ ! -s "$f" ]; then
|
||||
echo "this agent has no store identity at $f, so it cannot fetch the swarm root certificate." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
cacert=
|
||||
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
||||
cacert="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
|
||||
fi
|
||||
|
||||
dir=/run/${runtimeDir}
|
||||
err=$(mktemp -p "$dir" err.XXXXXX)
|
||||
raw=$(mktemp -p "$dir" raw.XXXXXX)
|
||||
root=$(mktemp -p "$dir" root.XXXXXX)
|
||||
bundle=$(mktemp -p "$dir" bundle.XXXXXX)
|
||||
trap 'rm -f "$err" "$raw" "$root" "$bundle"' EXIT
|
||||
|
||||
# The listener demands a client certificate and the endpoint takes no
|
||||
# token, so no login precedes this.
|
||||
if ! curl --fail --silent --show-error \
|
||||
--connect-timeout 10 --max-time 30 \
|
||||
''${cacert:+--cacert "$cacert"} \
|
||||
--cert "$BAO_CLIENT_CERT" --key "$BAO_CLIENT_KEY" \
|
||||
--output "$raw" \
|
||||
${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"; then
|
||||
echo "could not read the store's ${cfg.servicesPkiMount} root at $BAO_ADDR." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Re-encoded rather than copied: it fails on a body that is not a
|
||||
# certificate and ends the file in the newline the concatenation
|
||||
# below relies on.
|
||||
if ! openssl x509 -in "$raw" -out "$root" 2>"$err"; then
|
||||
echo "the store's ${cfg.servicesPkiMount} root does not parse as a certificate." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat ${systemBundle} "$root" >"$bundle"
|
||||
|
||||
# Neither file moves unless it changed, so a reader never sees a half
|
||||
# write and the timer's rerun is silent.
|
||||
if cmp -s "$bundle" ${bundleFile} && cmp -s "$root" ${rootFile}; then
|
||||
echo "the store's ${cfg.servicesPkiMount} root is unchanged"
|
||||
exit 0
|
||||
fi
|
||||
chmod 0644 "$root" "$bundle"
|
||||
mv -f "$root" ${rootFile}
|
||||
mv -f "$bundle" ${bundleFile}
|
||||
echo "wrote the swarm root certificate from the store's ${cfg.servicesPkiMount} mount"
|
||||
'';
|
||||
};
|
||||
}
|
||||
// lib.genAttrs consumers (_: {
|
||||
after = [ "${unitName}.service" ];
|
||||
requires = [ "${unitName}.service" ];
|
||||
});
|
||||
|
||||
systemd.timers.${unitName} = {
|
||||
description = "re-read the swarm root certificate from the secret store";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnUnitInactiveSec = "1h";
|
||||
RandomizedDelaySec = "1min";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -2,13 +2,10 @@
|
|||
#
|
||||
# ONE PAIRING PER FILE — hive-c0re and the swarm controller ← the trust bundle
|
||||
# ./hive-tls.nix assembles. Deleting this leaves daemons that trust only the
|
||||
# platform roots: hive-c0re's lifecycle cannot hand agents the bundle, and the
|
||||
# swarm-queue clients die at `invalid peer certificate: UnknownIssuer` against
|
||||
# an authelia signed by the swarm CA.
|
||||
# platform roots: the swarm-queue clients die at
|
||||
# `invalid peer certificate: UnknownIssuer` against an authelia signed by the
|
||||
# swarm CA.
|
||||
#
|
||||
# `HIVE_TLS_CA_PATH` signals the hive-c0re lifecycle that a trust bundle exists:
|
||||
# it bind-mounts the file (read-only, public certs ONLY — never a key) into each
|
||||
# agent container, and the meta flake wires the per-agent trust bundle.
|
||||
# `*_OIDC_CA_FILE` names the same anchor for the swarm-queue clients that mint a
|
||||
# token from authelia over TLS. `swarm-queue-client` builds a bare
|
||||
# `reqwest::Client`, so it trusts only the platform roots without it. Where the
|
||||
|
|
@ -43,7 +40,6 @@ in
|
|||
lib.mkMerge [
|
||||
(lib.mkIf deployCfg.hive-controller.enable {
|
||||
systemd.services.hive-c0re.environment = {
|
||||
HIVE_TLS_CA_PATH = trustBundle;
|
||||
HIVE_C0RE_OIDC_CA_FILE = trustBundle;
|
||||
};
|
||||
})
|
||||
|
|
|
|||
|
|
@ -278,6 +278,9 @@ in
|
|||
# environment wholesale would inherit the hive's certificate paths with it,
|
||||
# and those name a credential no agent may present.
|
||||
HIVE_AGENT_BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
# The mount whose root an agent fetches to trust the swarm's services; the
|
||||
# same value ../hive-tls.nix reads the root from.
|
||||
HIVE_AGENT_BAO_SERVICES_PKI_MOUNT = baoDeploy.servicesPkiMountPath;
|
||||
# `%d`, not the paths themselves: the key is `0600` root-owned and this
|
||||
# daemon runs as hive-core, so it never gets read access to the original.
|
||||
# See the LoadCredential in ./default.nix.
|
||||
|
|
|
|||
|
|
@ -47,6 +47,38 @@ let
|
|||
in
|
||||
let
|
||||
cases = [
|
||||
{
|
||||
# ../agent-modules/hive-ca.nix. Presence arm first (the unit, the
|
||||
# timer, the trust variables, and every consumer ordered after the
|
||||
# unit and requiring it), then the absence arm for an agent with no
|
||||
# store, which has no root to fetch.
|
||||
name = "an agent with a store address fetches the swarm root and its consumers wait for it";
|
||||
ok =
|
||||
let
|
||||
svc = agentForgeBao.systemd.services;
|
||||
unit = "hive-agent-trust-bundle.service";
|
||||
bundle = "/run/hive-agent-trust-bundle/bundle.pem";
|
||||
consumers = [
|
||||
"hive-agent"
|
||||
"hive-bash-daemon"
|
||||
"hive-subagent-daemon"
|
||||
"hive-mcp-http"
|
||||
"hive-forge-notify"
|
||||
"forge-avatar-sync"
|
||||
];
|
||||
in
|
||||
svc ? hive-agent-trust-bundle
|
||||
&& agentForgeBao.systemd.timers ? hive-agent-trust-bundle
|
||||
&& lib.hasInfix "/v1/pki/ca/pem" svc.hive-agent-trust-bundle.script
|
||||
&& agentForgeBao.systemd.globalEnvironment.SSL_CERT_FILE == bundle
|
||||
&&
|
||||
agentForgeBao.systemd.globalEnvironment.NODE_EXTRA_CA_CERTS
|
||||
== "/run/hive-agent-trust-bundle/root.pem"
|
||||
&& lib.all (c: lib.elem unit svc.${c}.after && lib.elem unit svc.${c}.requires) consumers
|
||||
&& !(agentForgeNoBao.systemd.services ? hive-agent-trust-bundle)
|
||||
&& !(agentForgeNoBao.systemd.globalEnvironment ? SSL_CERT_FILE)
|
||||
&& !(lib.elem unit (agentForgeNoBao.systemd.services.hive-agent.after or [ ]));
|
||||
}
|
||||
{
|
||||
# The whole switch is the store address, as for the queue credential.
|
||||
name = "an agent with a store address fetches its forge token";
|
||||
|
|
|
|||
|
|
@ -317,6 +317,12 @@ let
|
|||
in
|
||||
e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR;
|
||||
}
|
||||
{
|
||||
name = "an agent is told the mount its swarm root is read from, only where it is told the store";
|
||||
ok =
|
||||
allLocal.systemd.services.hive-c0re.environment.HIVE_AGENT_BAO_SERVICES_PKI_MOUNT == "pki"
|
||||
&& !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_SERVICES_PKI_MOUNT);
|
||||
}
|
||||
{
|
||||
# A hive with no certificate of its own can collect no agent's identity,
|
||||
# so forwarding an address would name a store nothing in the container
|
||||
|
|
|
|||
|
|
@ -693,10 +693,7 @@ let
|
|||
let
|
||||
bundle = "/var/lib/hive-tls/trust-bundle.pem";
|
||||
envOf = m: unit: m.systemd.services.${unit}.environment or { };
|
||||
hiveVars = [
|
||||
"HIVE_TLS_CA_PATH"
|
||||
"HIVE_C0RE_OIDC_CA_FILE"
|
||||
];
|
||||
hiveVars = [ "HIVE_C0RE_OIDC_CA_FILE" ];
|
||||
anyOf =
|
||||
m: unit: vars:
|
||||
lib.any (v: (envOf m unit) ? ${v}) vars;
|
||||
|
|
|
|||
Loading…
Reference in a new issue