Watch
0
0
Fork
You've already forked hyperhive
0

agent ssh unit on every agent with a store, one ssh_config, controller reads the ca key

This commit is contained in:
atlas 2026-11-16 18:33:03 +01:00 • committed by mara
commit 624d5f24f8
4 changed files with 99 additions and 165 deletions

View file

@ -1,14 +1,20 @@
# Outbound SSH for this agent: a certificate for a key it generates itself,
# signed by the swarm's SSH CA, and a client config for the hosts it may reach.
# signed by the swarm's SSH CA, and a client config that offers it.
#
# Nothing is configured unless `services.hyperhive.agent.ssh.enable` is set. The
# agent has no standing SSH credential: `${unitName}.service` generates an
# ed25519 key in its own `/run` directory (tmpfs, never on disk), logs in to the
# swarm secret store with the identity ./bao.nix keeps current, and has the
# public half signed by the store's SSH engine
# (`swarm_secret_client::policy::SSH_MOUNT`). The role it signs through issues a
# user certificate with this agent's name as its only principal, valid for an
# hour; a timer signs again well before that runs out.
# On every agent that has a store address (the same switch as ./bao.nix: with no
# store there is nothing to sign with). The agent has no standing SSH credential:
# `${unitName}.service` generates an ed25519 key in its own `/run` directory
# (tmpfs, never on disk), logs in to the swarm secret store with the identity
# ./bao.nix keeps current, and has the public half signed by the store's SSH
# engine (`swarm_secret_client::policy::SSH_MOUNT`). The role it signs through
# issues a user certificate with this agent's name as its only principal, valid
# for an hour; a timer signs again well before that runs out.
#
# Whether the store signs is the operator's per-agent switch in the swarm UI
# (the `ssh` permission, `swarm-controller`'s `agent_ssh`). With it off the store
# refuses, the unit logs one line and ends successfully, and the next tick asks
# again; with it on the agent has a certificate within one tick. The unit
# needs no change to be switched.
#
# A target machine trusts the CA, not this key: ../target-modules/agent-ssh.nix.
{
@ -18,7 +24,6 @@
...
}:
let
cfg = config.services.hyperhive.agent.ssh;
baoCfg = config.services.hyperhive.agent.bao;
# The unix user is named for the agent, so this is also the principal the
@ -41,113 +46,28 @@ let
stagingFile = "/run/${runtimeDir}/cert.new";
errFile = "/run/${runtimeDir}/bao.err";
# In the state dir, which is bound from the host, so a pin survives a restart.
knownHostsFile = "/agents/${agentName}/state/ssh_known_hosts";
signPath = "${ca.mountPath}/sign/${ca.roleName}";
storeRetry = import ../host-modules/lib/store-retry.nix { };
# `[host]:port` is how a known_hosts line names a host on a non-default port.
knownHostName = h: if h.port == 22 then h.hostName else "[${h.hostName}]:${toString h.port}";
hostBlock = name: h: ''
Host ${name}
HostName ${h.hostName}
User ${h.user}
Port ${toString h.port}
IdentityFile ${keyFile}
CertificateFile ${certFile}
IdentitiesOnly yes
StrictHostKeyChecking yes
'';
hostType = lib.types.submodule {
options = {
hostName = lib.mkOption {
type = lib.types.strMatching "[A-Za-z0-9._:-]+";
example = "vm1.pc2.example.com";
description = ''
Name or address the agent connects to. The host key below is pinned to
exactly this spelling.
'';
};
user = lib.mkOption {
type = lib.types.strMatching "[A-Za-z0-9._-]+";
example = "test";
description = ''
Account to log in as on the target. The target's
{option}`principals` for that account must list this agent.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 22;
description = "TCP port of the target's sshd.";
};
hostKey = lib.mkOption {
type = lib.types.strMatching "(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp(256|384|521)) [A-Za-z0-9+/]+=*( [A-Za-z0-9@._-]+)?";
example = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA…";
description = ''
The target's SSH host public key, one line as in
{file}`/etc/ssh/ssh_host_ed25519_key.pub` on the target. It is pinned:
the agent refuses a target that presents any other key, and there is
no trust on first use.
'';
};
};
};
in
{
options.services.hyperhive.agent.ssh = {
enable = lib.mkEnableOption ''
outbound SSH for this agent with short-lived certificates from the swarm
secret store's SSH CA. Needs {option}`services.hyperhive.agent.bao.addr`
config = lib.mkIf (baoCfg.addr != null) {
# Every host: the certificate, and the pins in the state dir, which outlives
# the container root. `accept-new` records a host's key at first contact and
# refuses a changed one after that.
programs.ssh.extraConfig = ''
Host *
IdentityFile ${keyFile}
CertificateFile ${certFile}
IdentitiesOnly yes
StrictHostKeyChecking accept-new
UserKnownHostsFile ${knownHostsFile}
'';
hosts = lib.mkOption {
type = lib.types.attrsOf hostType;
default = { };
example = lib.literalExpression ''
{
testvm = {
hostName = "10.0.0.7";
user = "test";
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA…";
};
}
'';
description = ''
Hosts this agent may SSH to, by the name it types: `ssh testvm`. Each
entry becomes a `Host` block in the system {file}`ssh_config` that
offers this agent's certificate and nothing else, and a pinned line in
the system {file}`ssh_known_hosts`.
This does not open the network path: with the hive's egress set to
reject, direct TCP out of the container is refused whatever is
configured here.
'';
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = baoCfg.addr != null;
message = "services.hyperhive.agent.ssh.enable needs services.hyperhive.agent.bao.addr: the certificate is signed by the swarm secret store, and this agent has no store address";
}
{
assertion = lib.all (n: builtins.match "[A-Za-z0-9._-]+" n != null) (lib.attrNames cfg.hosts);
message = "services.hyperhive.agent.ssh.hosts names are written into ssh_config and may hold only letters, digits, `.`, `_` and `-`";
}
];
programs.ssh.extraConfig = lib.concatStrings (lib.mapAttrsToList hostBlock cfg.hosts);
programs.ssh.knownHosts = lib.mapAttrs' (
name: h:
lib.nameValuePair "hive-agent-ssh-${name}" {
hostNames = [ (knownHostName h) ];
publicKey = h.hostKey;
}
) cfg.hosts;
systemd.services.${unitName} = {
description = "sign this agent's SSH key with the swarm's SSH CA";
after = [
@ -233,6 +153,13 @@ in
# name, and it refuses any other.
rm -f "$err" "$staged"
if ! bao write -field=signed_key ${lib.escapeShellArg signPath} public_key=@"$pub" >"$staged" 2>"$err"; then
# A 403 is the operator's switch being off for this agent: not a fault,
# so no failed unit. Anything else (a sealed or unreachable store) is
# one, and retried.
if [[ "$(<"$err")" == *"Code: 403"* ]]; then
echo "SSH certificates are not enabled for this agent (the swarm store refused to sign); asking again next tick."
exit 0
fi
echo "the swarm secret store would not sign this agent's SSH key at ${signPath}:" >&2
if [ -s "$err" ]; then cat "$err" >&2; fi
exit 1

View file

@ -410,6 +410,11 @@ let
#
# The agent PKI grant is its only path on that mount: it can ask the one role
# for a certificate, not write that role or reach the issuer.
#
# The one SSH grant is a `read` of the SSH CA, which answers with its public
# key: the swarm UI shows it for a target machine to trust. The controller
# cannot sign, and the signing policy it writes (`hive-agent-ssh`) is
# attached to an agent's role by it only while the operator has switched SSH on.
# The identity grants name each agent's login entity after the agent
# (`agent_entity.rs`) and disable it when the agent is destroyed
@ -628,6 +633,10 @@ let
path "${agentPkiMountPath}/issue/${agentPkiRoleName}" {
capabilities = ["update"]
}
path "${sshAgentsMountPath}/config/ca" {
capabilities = ["read"]
}
'';
# The identity authelia's mint creates its OIDC client secrets in the store

View file

@ -7,7 +7,7 @@
# cover ../agent-modules/egress-proxy.nix, the agent's proxy credential and the
# environment built from it.
# They also cover ../agent-modules/ssh.nix: the agent's SSH certificate unit and
# client config, on the same store address.
# client config, which every agent with a store address gets.
{
pkgs,
lib,
@ -34,30 +34,12 @@ let
# A forge and a store: the fetch exists and every reader points at it.
agentForgeBao = agentWith {
services.hyperhive.agent.bao.addr = baoAddr;
services.hyperhive.agent.ssh = {
enable = true;
hosts.testvm = {
hostName = "10.0.0.7";
user = "test";
hostKey = hostKeyA;
};
hosts.altvm = {
hostName = "vm2.t.local";
user = "ops";
port = 2222;
hostKey = hostKeyB;
};
};
services.hyperhive.agent.forge.url = forgeUrl;
# A second `EnvironmentFile=` on the harness, which the proxy file must join.
services.hyperhive.agent.backendEnvironmentFile = "/var/lib/t/backend.env";
services.hyperhive.agent.icon = pkgs.emptyFile;
};
# Outbound SSH on the same agent: two hosts, one on a non-default port.
hostKeyA = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOHfYeWemIBqSc5Ac51ecuhi0G9LTN+57DFRq6mFKKk0";
hostKeyB = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMY1xboa8d2ncPDhTSPKsq5WMyYfWwIqLcCOnkHLO3ad";
# A forge and no store: the absence arm, and what makes the cases above able
# to fail.
agentForgeNoBao = agentWith {
@ -65,12 +47,6 @@ let
services.hyperhive.agent.icon = pkgs.emptyFile;
};
# SSH asked for with no store: the certificate has nowhere to be signed.
agentSshNoBao = agentWith {
services.hyperhive.agent.ssh.enable = true;
services.hyperhive.agent.icon = pkgs.emptyFile;
};
sshUnit = machine: machine.systemd.services.hive-agent-ssh-cert;
sshCa = import ../host-modules/lib/ssh-agent-ca.nix { };
@ -551,15 +527,18 @@ let
&& lib.hasInfix "url=\"http://${name}:$secret@$proxy\"" s;
}
{
# ../agent-modules/ssh.nix. Presence arm first, then the agent that
# did not ask for SSH, which gets neither the unit nor a host block.
name = "an agent that enables ssh gets the certificate unit and its timer, and one that does not gets neither";
# ../agent-modules/ssh.nix. There is no agent-config switch: the store
# address is the whole gate, and the operator's switch is the UI's.
name = "an agent with a store address gets the ssh certificate unit and its timer with no setting, and one without gets neither";
ok =
agentForgeBao.systemd.services ? hive-agent-ssh-cert
&& agentForgeBao.systemd.timers ? hive-agent-ssh-cert
&& !(agentForgeNoBao.systemd.services ? hive-agent-ssh-cert)
&& !(agentForgeNoBao.systemd.timers ? hive-agent-ssh-cert)
&& !(lib.hasInfix "Host testvm" agentForgeNoBao.environment.etc."ssh/ssh_config".text);
&&
lib.hasInfix "IdentityFile /run/hive-agent-ssh-cert/key"
agentForgeBao.environment.etc."ssh/ssh_config".text
&& !(lib.hasInfix "hive-agent-ssh-cert" agentForgeNoBao.environment.etc."ssh/ssh_config".text);
}
{
# The key and the certificate stay in the unit's own tmpfs directory,
@ -598,39 +577,39 @@ let
every * 2 < sshCa.certTtlMinutes && every > 0;
}
{
# `ssh testvm` has to work with no other flag: the host block names the
# target, the user, the port, the one identity and its certificate, and
# refuses an unknown host key.
name = "each ssh host gets a Host block offering only the agent's certificate, and a pinned known_hosts line";
# `ssh user@host` works for any host the target trusts: one `Host *` block
# offers the agent's certificate and only it, and keeps its pins in the
# state dir. The host key is recorded at first contact: the explainer in
# the swarm UI says so.
name = "the ssh config offers only the agent's certificate to every host and records host keys in the state dir";
ok =
let
conf = agentForgeBao.environment.etc."ssh/ssh_config".text;
known = agentForgeBao.environment.etc."ssh/ssh_known_hosts".text;
block = lib.concatStringsSep "\n" [
"Host testvm"
" HostName 10.0.0.7"
" User test"
" Port 22"
"Host *"
" IdentityFile /run/hive-agent-ssh-cert/key"
" CertificateFile /run/hive-agent-ssh-cert/key-cert.pub"
" IdentitiesOnly yes"
" StrictHostKeyChecking yes"
" StrictHostKeyChecking accept-new"
" UserKnownHostsFile /agents/a1/state/ssh_known_hosts"
];
in
lib.hasInfix block conf
&& lib.hasInfix "Host altvm\n HostName vm2.t.local\n User ops\n Port 2222\n" conf
&& lib.hasInfix "10.0.0.7 ${hostKeyA}" known
# A port other than 22 is spelled the way known_hosts keys it.
&& lib.hasInfix "[vm2.t.local]:2222 ${hostKeyB}" known
&& !(lib.hasInfix "\nvm2.t.local " "\n${known}");
&& !(lib.hasInfix "StrictHostKeyChecking no" conf)
&& !(lib.hasInfix "hive-agent-ssh-cert" agentForgeNoBao.environment.etc."ssh/ssh_config".text);
}
{
name = "enabling ssh on an agent with no store address is refused, naming the option";
# A refused signing is the switch being off, not a fault: one line and a
# clean exit. Any other failure still fails the unit so it retries.
name = "the ssh certificate unit ends cleanly on a 403 and fails on any other signing error";
ok =
let
refuses = m: lib.any (a: !a.assertion && lib.hasInfix "agent.bao.addr" a.message) m.assertions;
s = (sshUnit agentForgeBao).script;
at = lib.splitString "Code: 403" s;
in
refuses agentSshNoBao && !(refuses agentForgeBao);
lib.length at == 2
&& lib.hasInfix "exit 0" (lib.head (lib.splitString "exit 1" (lib.elemAt at 1)))
&& lib.hasInfix "exit 1" (lib.elemAt at 1);
}
];
in

View file

@ -2879,28 +2879,46 @@ let
&& lib.length roleWriters == 1;
}
{
# The agents' `update` on the sign path comes from the shared agent
# policy (swarm-secret-client), not from a unit here. Of the policies
# these units write, only the granter's names the mount: the controller,
# which writes the shared one, holds nothing on it.
name = "of the policies the host writes, only the granter's has a grant on the agent SSH mount";
# The agents' `update` on the sign path is the `hive-agent-ssh` policy
# (swarm-secret-client), written by the controller at run time, not a unit
# here. Of the policies these units write, the granter's makes the mount,
# and the controller's only reads the CA's public key: it cannot sign,
# write the role or reach any other path on the mount.
name = "of the policies the host writes, only the granter's writes the agent SSH mount and the controller's only reads its CA";
ok =
let
cg = grantsIn baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
onMount = lib.filter (g: lib.hasPrefix "ssh-agents" g.path);
in
onMount cg == [ ]
onMount cg == [
{
path = "ssh-agents/config/ca";
caps = [ "read" ];
}
]
&& lib.all (p: grantFor cg p == null) [
"ssh-agents/sign/swarm-agent"
"ssh-agents/roles/swarm-agent"
"ssh-agents/config/ca/x"
"sys/mounts/ssh-agents"
]
# The control: the granter's grants do name the mount.
&& onMount granterGrants != [ ]
&& lib.all (u: onMount (grantsIn baoGrantHere.systemd.services.${u}.script) == [ ]) (
lib.filter (u: u != "swarm-bao-agent-ssh-policy") grantingUnitNames
lib.filter (
u:
!(lib.elem u [
"swarm-bao-agent-ssh-policy"
"swarm-bao-controller-policy"
])
) grantingUnitNames
);
}
{
# The shared agent policy (swarm-secret-client) names the mount and the
# The SSH signing policy (swarm-secret-client) names the mount and the
# role; the unit above and the agent module spell the same two strings
# from ../host-modules/lib/ssh-agent-ca.nix.
name = "the SSH mount and role the shared agent policy names are the ones the swarm writes";
name = "the SSH mount and role the signing policy names are the ones the swarm writes";
ok =
let
ca = import ../host-modules/lib/ssh-agent-ca.nix { };
@ -2908,6 +2926,7 @@ let
in
lib.hasInfix ''pub const SSH_MOUNT: &str = "${ca.mountPath}";'' policy
&& lib.hasInfix ''pub const SSH_ROLE: &str = "${ca.roleName}";'' policy
&& lib.hasInfix ''pub const SSH_POLICY: &str = "hive-agent-ssh";'' policy
&& lib.hasInfix "bao write ssh-agents/roles/${ca.roleName} -" baoGrantHere.systemd.services.swarm-bao-agent-ssh-policy.script;
}