diff --git a/nix/agent-modules/ssh.nix b/nix/agent-modules/ssh.nix index 991863499..8f451bd29 100644 --- a/nix/agent-modules/ssh.nix +++ b/nix/agent-modules/ssh.nix @@ -1,14 +1,20 @@ # Outbound SSH for this agent: a certificate for a key it generates itself, -# signed by the swarm's SSH CA, and a client config for the hosts it may reach. +# signed by the swarm's SSH CA, and a client config that offers it. # -# Nothing is configured unless `services.hyperhive.agent.ssh.enable` is set. The -# agent has no standing SSH credential: `${unitName}.service` generates an -# ed25519 key in its own `/run` directory (tmpfs, never on disk), logs in to the -# swarm secret store with the identity ./bao.nix keeps current, and has the -# public half signed by the store's SSH engine -# (`swarm_secret_client::policy::SSH_MOUNT`). The role it signs through issues a -# user certificate with this agent's name as its only principal, valid for an -# hour; a timer signs again well before that runs out. +# On every agent that has a store address (the same switch as ./bao.nix: with no +# store there is nothing to sign with). The agent has no standing SSH credential: +# `${unitName}.service` generates an ed25519 key in its own `/run` directory +# (tmpfs, never on disk), logs in to the swarm secret store with the identity +# ./bao.nix keeps current, and has the public half signed by the store's SSH +# engine (`swarm_secret_client::policy::SSH_MOUNT`). The role it signs through +# issues a user certificate with this agent's name as its only principal, valid +# for an hour; a timer signs again well before that runs out. +# +# Whether the store signs is the operator's per-agent switch in the swarm UI +# (the `ssh` permission, `swarm-controller`'s `agent_ssh`). With it off the store +# refuses, the unit logs one line and ends successfully, and the next tick asks +# again; with it on the agent has a certificate within one tick. The unit +# needs no change to be switched. # # A target machine trusts the CA, not this key: ../target-modules/agent-ssh.nix. { @@ -18,7 +24,6 @@ ... }: let - cfg = config.services.hyperhive.agent.ssh; baoCfg = config.services.hyperhive.agent.bao; # The unix user is named for the agent, so this is also the principal the @@ -41,113 +46,28 @@ let stagingFile = "/run/${runtimeDir}/cert.new"; errFile = "/run/${runtimeDir}/bao.err"; + # In the state dir, which is bound from the host, so a pin survives a restart. + knownHostsFile = "/agents/${agentName}/state/ssh_known_hosts"; + signPath = "${ca.mountPath}/sign/${ca.roleName}"; storeRetry = import ../host-modules/lib/store-retry.nix { }; - # `[host]:port` is how a known_hosts line names a host on a non-default port. - knownHostName = h: if h.port == 22 then h.hostName else "[${h.hostName}]:${toString h.port}"; - - hostBlock = name: h: '' - Host ${name} - HostName ${h.hostName} - User ${h.user} - Port ${toString h.port} - IdentityFile ${keyFile} - CertificateFile ${certFile} - IdentitiesOnly yes - StrictHostKeyChecking yes - ''; - - hostType = lib.types.submodule { - options = { - hostName = lib.mkOption { - type = lib.types.strMatching "[A-Za-z0-9._:-]+"; - example = "vm1.pc2.example.com"; - description = '' - Name or address the agent connects to. The host key below is pinned to - exactly this spelling. - ''; - }; - user = lib.mkOption { - type = lib.types.strMatching "[A-Za-z0-9._-]+"; - example = "test"; - description = '' - Account to log in as on the target. The target's - {option}`principals` for that account must list this agent. - ''; - }; - port = lib.mkOption { - type = lib.types.port; - default = 22; - description = "TCP port of the target's sshd."; - }; - hostKey = lib.mkOption { - type = lib.types.strMatching "(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp(256|384|521)) [A-Za-z0-9+/]+=*( [A-Za-z0-9@._-]+)?"; - example = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA…"; - description = '' - The target's SSH host public key, one line as in - {file}`/etc/ssh/ssh_host_ed25519_key.pub` on the target. It is pinned: - the agent refuses a target that presents any other key, and there is - no trust on first use. - ''; - }; - }; - }; in { - options.services.hyperhive.agent.ssh = { - enable = lib.mkEnableOption '' - outbound SSH for this agent with short-lived certificates from the swarm - secret store's SSH CA. Needs {option}`services.hyperhive.agent.bao.addr` + config = lib.mkIf (baoCfg.addr != null) { + # Every host: the certificate, and the pins in the state dir, which outlives + # the container root. `accept-new` records a host's key at first contact and + # refuses a changed one after that. + programs.ssh.extraConfig = '' + Host * + IdentityFile ${keyFile} + CertificateFile ${certFile} + IdentitiesOnly yes + StrictHostKeyChecking accept-new + UserKnownHostsFile ${knownHostsFile} ''; - hosts = lib.mkOption { - type = lib.types.attrsOf hostType; - default = { }; - example = lib.literalExpression '' - { - testvm = { - hostName = "10.0.0.7"; - user = "test"; - hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA…"; - }; - } - ''; - description = '' - Hosts this agent may SSH to, by the name it types: `ssh testvm`. Each - entry becomes a `Host` block in the system {file}`ssh_config` that - offers this agent's certificate and nothing else, and a pinned line in - the system {file}`ssh_known_hosts`. - - This does not open the network path: with the hive's egress set to - reject, direct TCP out of the container is refused whatever is - configured here. - ''; - }; - }; - - config = lib.mkIf cfg.enable { - assertions = [ - { - assertion = baoCfg.addr != null; - message = "services.hyperhive.agent.ssh.enable needs services.hyperhive.agent.bao.addr: the certificate is signed by the swarm secret store, and this agent has no store address"; - } - { - assertion = lib.all (n: builtins.match "[A-Za-z0-9._-]+" n != null) (lib.attrNames cfg.hosts); - message = "services.hyperhive.agent.ssh.hosts names are written into ssh_config and may hold only letters, digits, `.`, `_` and `-`"; - } - ]; - - programs.ssh.extraConfig = lib.concatStrings (lib.mapAttrsToList hostBlock cfg.hosts); - programs.ssh.knownHosts = lib.mapAttrs' ( - name: h: - lib.nameValuePair "hive-agent-ssh-${name}" { - hostNames = [ (knownHostName h) ]; - publicKey = h.hostKey; - } - ) cfg.hosts; - systemd.services.${unitName} = { description = "sign this agent's SSH key with the swarm's SSH CA"; after = [ @@ -233,6 +153,13 @@ in # name, and it refuses any other. rm -f "$err" "$staged" if ! bao write -field=signed_key ${lib.escapeShellArg signPath} public_key=@"$pub" >"$staged" 2>"$err"; then + # A 403 is the operator's switch being off for this agent: not a fault, + # so no failed unit. Anything else (a sealed or unreachable store) is + # one, and retried. + if [[ "$(<"$err")" == *"Code: 403"* ]]; then + echo "SSH certificates are not enabled for this agent (the swarm store refused to sign); asking again next tick." + exit 0 + fi echo "the swarm secret store would not sign this agent's SSH key at ${signPath}:" >&2 if [ -s "$err" ]; then cat "$err" >&2; fi exit 1 diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index df22ab6c0..bb08c4a93 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -410,6 +410,11 @@ let # # The agent PKI grant is its only path on that mount: it can ask the one role # for a certificate, not write that role or reach the issuer. + # + # The one SSH grant is a `read` of the SSH CA, which answers with its public + # key: the swarm UI shows it for a target machine to trust. The controller + # cannot sign, and the signing policy it writes (`hive-agent-ssh`) is + # attached to an agent's role by it only while the operator has switched SSH on. # The identity grants name each agent's login entity after the agent # (`agent_entity.rs`) and disable it when the agent is destroyed @@ -628,6 +633,10 @@ let path "${agentPkiMountPath}/issue/${agentPkiRoleName}" { capabilities = ["update"] } + + path "${sshAgentsMountPath}/config/ca" { + capabilities = ["read"] + } ''; # The identity authelia's mint creates its OIDC client secrets in the store diff --git a/nix/module-eval/agent-forge-bao.nix b/nix/module-eval/agent-forge-bao.nix index 23e100533..7e51d3e8b 100644 --- a/nix/module-eval/agent-forge-bao.nix +++ b/nix/module-eval/agent-forge-bao.nix @@ -7,7 +7,7 @@ # cover ../agent-modules/egress-proxy.nix, the agent's proxy credential and the # environment built from it. # They also cover ../agent-modules/ssh.nix: the agent's SSH certificate unit and -# client config, on the same store address. +# client config, which every agent with a store address gets. { pkgs, lib, @@ -34,30 +34,12 @@ let # A forge and a store: the fetch exists and every reader points at it. agentForgeBao = agentWith { services.hyperhive.agent.bao.addr = baoAddr; - services.hyperhive.agent.ssh = { - enable = true; - hosts.testvm = { - hostName = "10.0.0.7"; - user = "test"; - hostKey = hostKeyA; - }; - hosts.altvm = { - hostName = "vm2.t.local"; - user = "ops"; - port = 2222; - hostKey = hostKeyB; - }; - }; services.hyperhive.agent.forge.url = forgeUrl; # A second `EnvironmentFile=` on the harness, which the proxy file must join. services.hyperhive.agent.backendEnvironmentFile = "/var/lib/t/backend.env"; services.hyperhive.agent.icon = pkgs.emptyFile; }; - # Outbound SSH on the same agent: two hosts, one on a non-default port. - hostKeyA = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOHfYeWemIBqSc5Ac51ecuhi0G9LTN+57DFRq6mFKKk0"; - hostKeyB = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMY1xboa8d2ncPDhTSPKsq5WMyYfWwIqLcCOnkHLO3ad"; - # A forge and no store: the absence arm, and what makes the cases above able # to fail. agentForgeNoBao = agentWith { @@ -65,12 +47,6 @@ let services.hyperhive.agent.icon = pkgs.emptyFile; }; - # SSH asked for with no store: the certificate has nowhere to be signed. - agentSshNoBao = agentWith { - services.hyperhive.agent.ssh.enable = true; - services.hyperhive.agent.icon = pkgs.emptyFile; - }; - sshUnit = machine: machine.systemd.services.hive-agent-ssh-cert; sshCa = import ../host-modules/lib/ssh-agent-ca.nix { }; @@ -551,15 +527,18 @@ let && lib.hasInfix "url=\"http://${name}:$secret@$proxy\"" s; } { - # ../agent-modules/ssh.nix. Presence arm first, then the agent that - # did not ask for SSH, which gets neither the unit nor a host block. - name = "an agent that enables ssh gets the certificate unit and its timer, and one that does not gets neither"; + # ../agent-modules/ssh.nix. There is no agent-config switch: the store + # address is the whole gate, and the operator's switch is the UI's. + name = "an agent with a store address gets the ssh certificate unit and its timer with no setting, and one without gets neither"; ok = agentForgeBao.systemd.services ? hive-agent-ssh-cert && agentForgeBao.systemd.timers ? hive-agent-ssh-cert && !(agentForgeNoBao.systemd.services ? hive-agent-ssh-cert) && !(agentForgeNoBao.systemd.timers ? hive-agent-ssh-cert) - && !(lib.hasInfix "Host testvm" agentForgeNoBao.environment.etc."ssh/ssh_config".text); + && + lib.hasInfix "IdentityFile /run/hive-agent-ssh-cert/key" + agentForgeBao.environment.etc."ssh/ssh_config".text + && !(lib.hasInfix "hive-agent-ssh-cert" agentForgeNoBao.environment.etc."ssh/ssh_config".text); } { # The key and the certificate stay in the unit's own tmpfs directory, @@ -598,39 +577,39 @@ let every * 2 < sshCa.certTtlMinutes && every > 0; } { - # `ssh testvm` has to work with no other flag: the host block names the - # target, the user, the port, the one identity and its certificate, and - # refuses an unknown host key. - name = "each ssh host gets a Host block offering only the agent's certificate, and a pinned known_hosts line"; + # `ssh user@host` works for any host the target trusts: one `Host *` block + # offers the agent's certificate and only it, and keeps its pins in the + # state dir. The host key is recorded at first contact: the explainer in + # the swarm UI says so. + name = "the ssh config offers only the agent's certificate to every host and records host keys in the state dir"; ok = let conf = agentForgeBao.environment.etc."ssh/ssh_config".text; - known = agentForgeBao.environment.etc."ssh/ssh_known_hosts".text; block = lib.concatStringsSep "\n" [ - "Host testvm" - " HostName 10.0.0.7" - " User test" - " Port 22" + "Host *" " IdentityFile /run/hive-agent-ssh-cert/key" " CertificateFile /run/hive-agent-ssh-cert/key-cert.pub" " IdentitiesOnly yes" - " StrictHostKeyChecking yes" + " StrictHostKeyChecking accept-new" + " UserKnownHostsFile /agents/a1/state/ssh_known_hosts" ]; in lib.hasInfix block conf - && lib.hasInfix "Host altvm\n HostName vm2.t.local\n User ops\n Port 2222\n" conf - && lib.hasInfix "10.0.0.7 ${hostKeyA}" known - # A port other than 22 is spelled the way known_hosts keys it. - && lib.hasInfix "[vm2.t.local]:2222 ${hostKeyB}" known - && !(lib.hasInfix "\nvm2.t.local " "\n${known}"); + && !(lib.hasInfix "StrictHostKeyChecking no" conf) + && !(lib.hasInfix "hive-agent-ssh-cert" agentForgeNoBao.environment.etc."ssh/ssh_config".text); } { - name = "enabling ssh on an agent with no store address is refused, naming the option"; + # A refused signing is the switch being off, not a fault: one line and a + # clean exit. Any other failure still fails the unit so it retries. + name = "the ssh certificate unit ends cleanly on a 403 and fails on any other signing error"; ok = let - refuses = m: lib.any (a: !a.assertion && lib.hasInfix "agent.bao.addr" a.message) m.assertions; + s = (sshUnit agentForgeBao).script; + at = lib.splitString "Code: 403" s; in - refuses agentSshNoBao && !(refuses agentForgeBao); + lib.length at == 2 + && lib.hasInfix "exit 0" (lib.head (lib.splitString "exit 1" (lib.elemAt at 1))) + && lib.hasInfix "exit 1" (lib.elemAt at 1); } ]; in diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index f672ecb9d..bc48af733 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -2879,28 +2879,46 @@ let && lib.length roleWriters == 1; } { - # The agents' `update` on the sign path comes from the shared agent - # policy (swarm-secret-client), not from a unit here. Of the policies - # these units write, only the granter's names the mount: the controller, - # which writes the shared one, holds nothing on it. - name = "of the policies the host writes, only the granter's has a grant on the agent SSH mount"; + # The agents' `update` on the sign path is the `hive-agent-ssh` policy + # (swarm-secret-client), written by the controller at run time, not a unit + # here. Of the policies these units write, the granter's makes the mount, + # and the controller's only reads the CA's public key: it cannot sign, + # write the role or reach any other path on the mount. + name = "of the policies the host writes, only the granter's writes the agent SSH mount and the controller's only reads its CA"; ok = let cg = grantsIn baoGrantHere.systemd.services.swarm-bao-controller-policy.script; onMount = lib.filter (g: lib.hasPrefix "ssh-agents" g.path); in - onMount cg == [ ] + onMount cg == [ + { + path = "ssh-agents/config/ca"; + caps = [ "read" ]; + } + ] + && lib.all (p: grantFor cg p == null) [ + "ssh-agents/sign/swarm-agent" + "ssh-agents/roles/swarm-agent" + "ssh-agents/config/ca/x" + "sys/mounts/ssh-agents" + ] # The control: the granter's grants do name the mount. && onMount granterGrants != [ ] && lib.all (u: onMount (grantsIn baoGrantHere.systemd.services.${u}.script) == [ ]) ( - lib.filter (u: u != "swarm-bao-agent-ssh-policy") grantingUnitNames + lib.filter ( + u: + !(lib.elem u [ + "swarm-bao-agent-ssh-policy" + "swarm-bao-controller-policy" + ]) + ) grantingUnitNames ); } { - # The shared agent policy (swarm-secret-client) names the mount and the + # The SSH signing policy (swarm-secret-client) names the mount and the # role; the unit above and the agent module spell the same two strings # from ../host-modules/lib/ssh-agent-ca.nix. - name = "the SSH mount and role the shared agent policy names are the ones the swarm writes"; + name = "the SSH mount and role the signing policy names are the ones the swarm writes"; ok = let ca = import ../host-modules/lib/ssh-agent-ca.nix { }; @@ -2908,6 +2926,7 @@ let in lib.hasInfix ''pub const SSH_MOUNT: &str = "${ca.mountPath}";'' policy && lib.hasInfix ''pub const SSH_ROLE: &str = "${ca.roleName}";'' policy + && lib.hasInfix ''pub const SSH_POLICY: &str = "hive-agent-ssh";'' policy && lib.hasInfix "bao write ssh-agents/roles/${ca.roleName} -" baoGrantHere.systemd.services.swarm-bao-agent-ssh-policy.script; }