diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 938ddedd..94d31f4b 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -89,10 +89,11 @@ The same unit mints the client secrets, straight into the store. It reads only when the store answers and holds none does it run `authelia crypto hash generate`, create the leaf and write the digest. Authelia keeps only the digest on disk, in `oidc-clients/.digest`, and starts from it while the store is down. A store that -can't answer a read never mints, because a second secret would rotate the first. A client with -a secret in the store but no digest, or a plaintext file but no digest, stops the unit -with exit status 3 and an `err` line naming the client; restore the digest, or delete the -leaf to mint again. `swarm-authelia-bridge` fetches its own secret from +can't answer a read never mints, because a second secret would rotate the first. A client with a +secret but no digest, in the store or in a leftover plaintext file, gets the digest of +that secret and keeps the secret; `authelia crypto hash generate` computes the digest, +with the password typed into its prompt through a pty, never in a command line. +`swarm-authelia-bridge` fetches its own secret from `swarm/services/swarm-authelia-bridge/oidc/client` into `/run/swarm-authelia-bridge/client-secret` before each start, and a store that doesn't answer keeps the bridge down while it retries. A container without the publisher's @@ -102,7 +103,8 @@ identity keeps the plaintext in `oidc-clients/.secret`, as before. release left. For each, it stores the file's value when the store holds none, reads it back, and removes the file only when the store holds the same value; it logs one line per client containing `removed the file`. A store that holds a different value keeps both and -logs an `err` line, and a store that doesn't answer keeps the file and retries. Once no +logs an `err` line, and the unit stays failed without retrying, because only an operator can +say which value is right; a store that doesn't answer keeps the file and retries. Once no file remains the unit does nothing. A file for a client that's no longer registered stays in place, with a log line. diff --git a/nix/host-modules/lib/oidc-client-secrets.nix b/nix/host-modules/lib/oidc-client-secrets.nix index 37824a25..a3af762e 100644 --- a/nix/host-modules/lib/oidc-client-secrets.nix +++ b/nix/host-modules/lib/oidc-client-secrets.nix @@ -3,24 +3,27 @@ # digest of each, in `$clients/.digest`, and starts from it whether or not # the store answers. # -# `mint_client_secret` is the only writer of a new secret. It creates the -# store's leaf (`-cas=0`, so an existing value is never replaced) before it -# writes the digest, so a digest on disk always has a secret behind it. It mints -# only after a read that says the store holds none; a store that cannot be read -# never mints, because a second secret would rotate the first. +# `mint_client_secret` keeps every client with a digest. A client without one +# gets it from the secret that exists: the plaintext file an earlier release +# left, else the store's value. Only a client with none of the three is minted, +# and then the store's leaf is created (`-cas=0`, so an existing value is never +# replaced) before the digest is written, so a digest on disk always has a +# secret behind it. A store that cannot be read never mints, because a second +# secret would rotate the first. # # `migrate_client_secret` moves the plaintext file an earlier release left in # `$clients/.secret` into the store and removes it once a read-back shows # the store holds the same value. It never overwrites a store value and never # removes a file the store does not match. # -# Return statuses of both: 0 done, 1 retry later (the store could not answer), -# 3 an operator has to look (logged at `err`; retrying changes nothing). +# Return statuses: 0 done, 1 retry later (the store could not answer, or a +# digest could not be computed), 3 (migration only) the file and the store hold +# different secrets, and only an operator can say which is right. # # Pure function — NOT a NixOS module. Call it after `./store-seeded-secret.nix` # and `./atomic-write-secret.nix`, with `store_init` and `store_login` run and -# `$clients` set to the digest directory. Needs `authelia`, `bao`, `coreutils` -# and `sed` on the caller's `path`. +# `$clients` set to the digest directory. Needs `authelia`, `bao`, `script` +# (util-linux), `coreutils`, `grep` and `sed` on the caller's `path`. { }: '' # . Sets CLIENT_SECRET_VALUE. 0: the store holds a value. 1: it holds @@ -40,25 +43,50 @@ return 2 } - # . Sets MINTED when it created a secret. + # . Prints authelia's digest of it, the form `mint` would have + # written. The CLI takes a password only on a terminal prompt or in argv, + # and argv is world-readable in /proc, so the prompt is answered through a + # pty from stdin. The pty echoes the password into the pipe, and only the + # `Digest:` line leaves it. + client_secret_digest() { + printf '%s\n' "$1" \ + | timeout 60 script -qec "authelia crypto hash generate pbkdf2 --variant sha512 --no-confirm" /dev/null \ + | tr -d '\r' | sed -n 's/^Digest: *//p' + } + + # . Writes the digest of an + # existing secret; the secret itself is untouched. + client_secret_derive_digest() { + local id="$1" secret="$2" source="$3" digest + digest="$(client_secret_digest "$secret")" || digest="" + if [ -z "$digest" ]; then + echo "<3>oidc client $id: could not compute a digest from the secret in $source" >&2 + return 1 + fi + atomic_write_secret 0600 "" "$clients/$id.digest" "$digest" || return 1 + MINTED=1 + echo "oidc client $id: wrote the digest from the secret in $source; the secret is unchanged" + } + + # . Sets MINTED when it wrote a digest. mint_client_secret() { local id="$1" leaf="$2" dig="$clients/$1.digest" legacy="$clients/$1.secret" local state=0 out secret digest - if [ -e "$legacy" ]; then + if [ -s "$legacy" ]; then if [ -s "$dig" ]; then return 0 fi - echo "<3>oidc client $id: $legacy exists but its digest does not; not minting a second secret. Restore the digest, or remove the file and the store's leaf $leaf to mint again." >&2 - return 3 + client_secret_derive_digest "$id" "$(< "$legacy")" "$legacy" + return fi client_secret_read "$leaf" || state=$? if [ "$state" = 0 ]; then - if [ ! -s "$dig" ]; then - echo "<3>oidc client $id: the store holds a secret at $leaf but $dig is missing; not minting a second secret. Restore the digest, or remove the leaf to mint again." >&2 - return 3 + if [ -s "$dig" ]; then + chmod 0600 "$dig" + return 0 fi - chmod 0600 "$dig" - return 0 + client_secret_derive_digest "$id" "$CLIENT_SECRET_VALUE" "the store at $leaf" + return fi if [ "$state" = 2 ]; then if [ -s "$dig" ]; then @@ -83,8 +111,9 @@ fi client_secret_read "$leaf" || return 1 if [ "$CLIENT_SECRET_VALUE" != "$secret" ]; then - echo "<3>oidc client $id: $leaf holds a different value than the one minted; not writing a digest" >&2 - return 3 + echo "oidc client $id: $leaf holds a value another run created; using it" + client_secret_derive_digest "$id" "$CLIENT_SECRET_VALUE" "the store at $leaf" + return fi atomic_write_secret 0600 "" "$dig" "$digest" || return 1 MINTED=1 diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index fa90795e..9fa57198 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -522,9 +522,8 @@ let # The plaintext goes straight into the store and only the digest is written # to disk. A digest on disk is enough for authelia to start while the store is # down; a client with no digest cannot be rendered, so that run fails and - # retries (./lib/store-retry.nix). A status of 3 is a state a retry does not - # change, and `RestartPreventExitStatus` leaves the unit failed for an - # operator. + # retries (./lib/store-retry.nix). A client whose digest is missing gets it + # from the secret that exists, so nothing is minted again. storeMintScript = '' ${oidcClientSecrets} ${lib.concatMapStrings (c: '' @@ -1246,6 +1245,7 @@ in deployCfg.bao.package pkgs.gnugrep pkgs.systemd + pkgs.util-linux ]; # A store that cannot answer, with a key missing on disk, is a # failed run that retries (./lib/store-retry.nix). @@ -1266,9 +1266,6 @@ in // { LoadCredential = lib.attrNames storeFiles; TimeoutStartSec = 120; - # A secret that has no digest, or a digest that has no secret, - # is for an operator; minting again would rotate it. - RestartPreventExitStatus = 3; } ); environment = lib.optionalAttrs publishes ( diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix index c40142f1..01884f72 100644 --- a/nix/module-eval/secret-publisher.nix +++ b/nix/module-eval/secret-publisher.nix @@ -488,7 +488,8 @@ let && !(lib.hasInfix "swarmctl" migrate.script) && !(lib.hasInfix "mint_client_secret swarmctl" mint.script) && migrate.serviceConfig.RestartPreventExitStatus == 3 - && mint.serviceConfig.RestartPreventExitStatus == 3 + # The mint answers authelia's password prompt through a pty. + && lib.any (p: lib.hasPrefix "util-linux" (p.name or "")) mint.path && migrate.serviceConfig.Restart == "on-failure" && migrate.serviceConfig.Type == "oneshot"; } diff --git a/nix/script-tests/oidc-client-secrets.nix b/nix/script-tests/oidc-client-secrets.nix index c0ae6aaf..ac438220 100644 --- a/nix/script-tests/oidc-client-secrets.nix +++ b/nix/script-tests/oidc-client-secrets.nix @@ -55,12 +55,33 @@ let esac ''; - # Prints what `authelia crypto hash generate pbkdf2 --random` prints, or - # nothing when `FAKE_AUTHELIA_EMPTY` is set. Every call is logged. + # With `--random`, prints what `authelia crypto hash generate pbkdf2 --random` + # prints. Without it, answers the password prompt from stdin the way the CLI + # does on a terminal: echo, then the digest of what was typed. Prints nothing + # when `FAKE_AUTHELIA_EMPTY` is set, and fails without a digest when + # `FAKE_AUTHELIA_NODIGEST` is set. Every call is logged. fakeAuthelia = pkgs.writeShellScriptBin "authelia" '' echo "$*" >> "$FAKE_AUTHELIA_LOG" [ -z "''${FAKE_AUTHELIA_EMPTY-}" ] || exit 0 - printf 'Random Password: minted-secret\n\nDigest: $pbkdf2-sha512$fake\n' + case " $* " in + *" --random "*) + printf 'Random Password: minted-secret\n\nDigest: $pbkdf2-sha512$fake\n' + ;; + *) + [ -z "''${FAKE_AUTHELIA_NODIGEST-}" ] || exit 1 + IFS= read -r typed + printf 'Enter Password: %s\r\nDigest: digest-of-%s\r\n' "$typed" "$typed" + ;; + esac + ''; + + # Runs the command it is given, as util-linux's `script -qec /dev/null` + # does, without needing a pty in the build sandbox. The real pty is checked + # against the real authelia by hand. + fakeScript = pkgs.writeShellScriptBin "script" '' + echo "script $1 $3" >> "$FAKE_AUTHELIA_LOG" + [ "$1" = -qec ] || exit 99 + exec ${pkgs.bash}/bin/bash -c "$2" ''; in pkgs.runCommand "hyperhive-script-test-oidc-client-secrets" @@ -75,6 +96,7 @@ pkgs.runCommand "hyperhive-script-test-oidc-client-secrets" paths = [ fakeBao fakeAuthelia + fakeScript ]; }; nativeBuildInputs = [ diff --git a/nix/script-tests/oidc-client-secrets.sh b/nix/script-tests/oidc-client-secrets.sh index 3edee189..4a0a128f 100644 --- a/nix/script-tests/oidc-client-secrets.sh +++ b/nix/script-tests/oidc-client-secrets.sh @@ -24,7 +24,7 @@ setup() { mkdir "$dir/kv" "$dir/clients" export FAKE_BAO_KV=$dir/kv FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_LOGIN_RC=0 export FAKE_AUTHELIA_LOG=$dir/authelia.log - unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_AUTHELIA_EMPTY + unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_AUTHELIA_EMPTY FAKE_AUTHELIA_NODIGEST : >"$FAKE_BAO_LOG" : >"$FAKE_AUTHELIA_LOG" clients=$dir/clients @@ -76,8 +76,9 @@ digest_is() { fi } +# Whether a new secret was minted, as opposed to a digest computed. ran_authelia() { - [ -s "$FAKE_AUTHELIA_LOG" ] + grep -q -e "--random" "$FAKE_AUTHELIA_LOG" } store_called() { @@ -158,14 +159,18 @@ expect 1 digest_is "" if ran_authelia; then fail "minted although the read was refused"; fi -setup "mint: no digest but a secret in the store is for an operator, and nothing is minted" +setup "mint: no digest but a secret in the store gets the digest of that secret, and the secret is unchanged" printf 'old-secret' >"$stored" run mint_client_secret -expect 3 +expect 0 held old-secret -digest_is "" +digest_is digest-of-old-secret if ran_authelia; then fail "minted over the store's secret"; fi -if ! logged "<3>oidc client $id"; then fail "no err-priority line"; fi +if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote to a leaf that held a value"; fi +if [ "$minted" != 1 ]; then fail "did not report the new digest"; fi +if [ "$(stat -c %a "$digest")" != 600 ]; then fail "digest mode is $(stat -c %a "$digest")"; fi +if ! logged "oidc client $id: wrote the digest from the secret in the store"; then fail "no info line"; fi +no_secret_in_argv setup "mint: a plaintext file with its digest is left for the migration" printf 'old-secret\n' >"$legacy" @@ -176,14 +181,25 @@ digest_is old-digest if ran_authelia; then fail "minted although a file held the secret"; fi if store_called; then fail "called the store"; fi -setup "mint: a plaintext file with no digest is for an operator, and nothing is minted" +setup "mint: a plaintext file with no digest gets the digest of the file's secret, and the file and the store are untouched" printf 'old-secret\n' >"$legacy" run mint_client_secret -expect 3 -digest_is "" +expect 0 +digest_is digest-of-old-secret held "" +if [ "$(cat "$legacy")" != old-secret ]; then fail "changed the file"; fi if ran_authelia; then fail "minted over the file's secret"; fi -if ! logged "<3>oidc client $id"; then fail "no err-priority line"; fi +if store_called; then fail "called the store"; fi +if ! logged "oidc client $id: wrote the digest from the secret in $legacy"; then fail "no info line"; fi +no_secret_in_argv + +setup "mint: a secret whose digest cannot be computed is a failure and writes no digest" +printf 'old-secret' >"$stored" +export FAKE_AUTHELIA_NODIGEST=1 +run mint_client_secret +expect 1 +digest_is "" +held old-secret setup "mint: an authelia that prints nothing writes nothing" export FAKE_AUTHELIA_EMPTY=1 @@ -199,12 +215,12 @@ expect 1 held "" digest_is "" -setup "mint: losing the race to another run writes no digest and keeps the winner's secret" +setup "mint: losing the race to another run takes the winner's secret and writes its digest" export FAKE_BAO_RACE=winner-secret run mint_client_secret -expect 3 +expect 0 held winner-secret -digest_is "" +digest_is digest-of-winner-secret if [ "$(id -u)" != 0 ]; then setup "mint: a digest that cannot be written is a failure, not a mint" diff --git a/swarm-nats-auth/src/oidc_secret.rs b/swarm-nats-auth/src/oidc_secret.rs index 6580ea38..c31c7270 100644 --- a/swarm-nats-auth/src/oidc_secret.rs +++ b/swarm-nats-auth/src/oidc_secret.rs @@ -25,8 +25,9 @@ use crate::introspect::{Answer, Caller}; use crate::keystore::KeyStore; /// The path under the store's KV mount holding `client_id`'s secret. -/// `swarm-secret-publisher.nix` writes it, and `natsAuthReaders` in -/// `swarm-bao.nix` grants this responder's role the read. +/// Authelia's mint (`authelia-swarm-secrets`) writes it, at the path +/// `swarm-secret-publisher.nix` lists, and `natsAuthReaders` in `swarm-bao.nix` +/// grants this responder's role the read. /// /// # Errors /// When `client_id` is not a single path segment. diff --git a/swarm-secret-client/src/builder.rs b/swarm-secret-client/src/builder.rs index 2e3444c2..1151f7d7 100644 --- a/swarm-secret-client/src/builder.rs +++ b/swarm-secret-client/src/builder.rs @@ -52,7 +52,7 @@ pub struct AtticPushToken { } /// The path holding the queue client secret of the builder's OIDC client -/// `client_id`, where `swarm-secret-publisher.nix` puts every swarm service's. +/// `client_id`, where authelia's mint puts every swarm service's. /// /// # Errors /// [`Error::PathSegment`] when `client_id` is not a single path segment. diff --git a/swarm-secret-client/src/ci_metrics.rs b/swarm-secret-client/src/ci_metrics.rs index 37ecfbae..f2d095aa 100644 --- a/swarm-secret-client/src/ci_metrics.rs +++ b/swarm-secret-client/src/ci_metrics.rs @@ -27,7 +27,7 @@ pub fn forge_token_path() -> Result { } /// The path holding the secret of the poller's OIDC client `client_id`, where -/// `swarm-secret-publisher.nix` puts every swarm service's. +/// authelia's mint puts every swarm service's. /// /// # Errors /// [`Error::PathSegment`] when `client_id` is not a single path segment. diff --git a/swarm-secret-client/src/claude_login.rs b/swarm-secret-client/src/claude_login.rs index 3cfcda4b..f5477454 100644 --- a/swarm-secret-client/src/claude_login.rs +++ b/swarm-secret-client/src/claude_login.rs @@ -46,7 +46,7 @@ impl std::fmt::Debug for Credentials { } /// The path holding the queue client secret of the minter's OIDC client -/// `client_id`, where `swarm-secret-publisher.nix` puts every swarm service's. +/// `client_id`, where authelia's mint puts every swarm service's. /// /// # Errors /// [`Error::PathSegment`] when `client_id` is not a single path segment. diff --git a/swarm-secret-client/src/queue.rs b/swarm-secret-client/src/queue.rs index 0c32251d..70e78744 100644 --- a/swarm-secret-client/src/queue.rs +++ b/swarm-secret-client/src/queue.rs @@ -44,8 +44,9 @@ pub fn agent_queue_path(agent: &str) -> Result { /// Under [`Kind::Controller`] because no hive's policy reads that kind, and /// this client may read every hive's status. /// -/// The nix half is `swarm-secret-publisher.nix`, which writes it, and -/// `swarm-bao.nix`'s `controllerPolicyText`, which grants the read. +/// The nix half is `swarm-secret-publisher.nix`, which lists the path +/// authelia's mint writes it at, and `swarm-bao.nix`'s `controllerPolicyText`, +/// which grants the read. /// /// # Errors /// Never in practice: the name segment is a constant. The `Result` is @@ -70,7 +71,8 @@ pub struct ControllerCredential { /// Under the hive's own prefix, which the hive's policy reads /// ([`crate::policy::render_shared_hives`]). /// -/// The nix half is `swarm-secret-publisher.nix`, which writes it. +/// The nix half is `swarm-secret-publisher.nix`, which lists the path authelia's +/// mint writes it at. /// /// # Errors /// [`Error::PathSegment`] when `hive` contains anything but `[A-Za-z0-9_-]`. diff --git a/swarm-secret-client/src/review_classifier.rs b/swarm-secret-client/src/review_classifier.rs index 7d573e2a..7c6a0b54 100644 --- a/swarm-secret-client/src/review_classifier.rs +++ b/swarm-secret-client/src/review_classifier.rs @@ -60,7 +60,7 @@ pub fn forge_token_path() -> Result { } /// The path holding the queue client secret of the classifier's OIDC client -/// `client_id`, where `swarm-secret-publisher.nix` puts every swarm service's. +/// `client_id`, where authelia's mint puts every swarm service's. /// /// # Errors /// [`Error::PathSegment`] when `client_id` is not a single path segment.