hyperhive/nix/script-tests/agent-bao-fetch.sh

# Cases for ./agent-bao-fetch.nix. Each case gets a fresh runtime directory and
# credentials directory, runs one unit's script under the unit's own `UMask=`
# with a clean environment, and asserts on the exit code, the output, the
# files left behind and the `bao` calls made.
set -uo pipefail

failures=0
count=0

fail() {
  echo "FAILED: $case: $*" >&2
  failures=$((failures + 1))
}

# The umask cases are only meaningful if a 0400 file cannot be reopened for
# writing, which is not so for root.
probe=$TMPDIR/umask-probe
: >"$probe"
chmod 0400 "$probe"
if (: >"$probe") 2>/dev/null; then
  echo "a 0400 file is writable by this builder, so the UMask=0377 cases would prove nothing" >&2
  exit 1
fi

# setup FORGE <description>
setup() {
  prefix=$1
  case="$prefix: $2"
  local unit_var=${prefix}_UNIT script_var=${prefix}_SCRIPT path_var=${prefix}_PATH addr_var=${prefix}_BAO_ADDR
  unit=${!unit_var}
  unit_path=${!path_var}
  bao_addr=${!addr_var}
  dir=$(mktemp -d)
  rt=$dir/rt
  creds=$dir/creds
  log=$dir/bao.log
  mkdir -m 0700 "$rt"
  mkdir "$creds"
  # Per-case stubs that shadow the unit's own tools; empty unless a case adds one.
  mkdir "$dir/bin"
  printf cert >"$creds/hive-agent-bao-cert"
  printf key >"$creds/hive-agent-bao-key"
  # The identity pair the unit presents: the delivered one for the fetch, the
  # live copy it keeps for the identity unit.
  if [ "$prefix" = IDENT ]; then
    client_cert=$rt/current/cert client_key=$rt/current/key
  else
    client_cert=$creds/hive-agent-bao-cert client_key=$creds/hive-agent-bao-key
  fi
  : >"$log"
  # The proxy fetch reads the egress facts the hive writes into a
  # container, moved under the build directory like /run.
  mkdir "$dir/etc"
  printf 'HIVE_DASHBOARD_PORT=7000\nHIVE_SQUID_PROXY=10.42.0.1:3128\nHIVE_EGRESS_NO_PROXY=10.42.0.1,t.hive,bao.t.local\n' >"$dir/etc/hyperhive-hive-facts"
  umask_value=0377
  if [ "$prefix" = PROXY ]; then umask_value=0177; fi
  if ! sed -e "s|/run/$unit/|$rt/|g" -e "s|/etc/hyperhive-|$dir/etc/hyperhive-|g" "${!script_var}" >"$dir/script"; then
    echo "cannot read the rendered script for $unit" >&2
    exit 1
  fi
  chmod +x "$dir/script"
  if grep -q '/run/' "$dir/script"; then fail "the script still names /run after the rewrite"; fi
  if ! grep -qF "$rt/bao.err" "$dir/script"; then fail "the rewrite did not reach the script's error file"; fi

  login_rc=0
  login_out=s.fake-token
  login_err=
  kv_rc=0
  kv_out=the-secret
  kv_err=
  refuse_cert=
}

go() {
  count=$((count + 1))
  (
    umask "$umask_value"
    cd "$dir" || exit 99
    exec env -i \
      PATH="$dir/bin:$FAKE_BAO_BIN:$unit_path" \
      BAO_ADDR="$bao_addr" \
      BAO_CLIENT_CERT="$client_cert" \
      BAO_CLIENT_KEY="$client_key" \
      CREDENTIALS_DIRECTORY="$creds" \
      FAKE_BAO_LOG="$log" \
      FAKE_BAO_LOGIN_RC="$login_rc" \
      FAKE_BAO_LOGIN_OUT="$login_out" \
      FAKE_BAO_LOGIN_ERR="$login_err" \
      FAKE_BAO_KV_RC="$kv_rc" \
      FAKE_BAO_KV_OUT="$kv_out" \
      FAKE_BAO_KV_ERR="$kv_err" \
      FAKE_BAO_REFUSE_CERT="$refuse_cert" \
      "$dir/script"
  ) >"$dir/out" 2>"$dir/err"
  rc=$?
}

expect_rc() {
  if [ "$rc" != "$1" ]; then fail "exit $rc, expected $1; stderr: $(<"$dir/err")"; fi
}

expect_err() {
  if ! grep -qF -- "$1" "$dir/err"; then fail "stderr lacks '$1'; stderr: $(<"$dir/err")"; fi
}

expect_no_err() {
  if grep -qF -- "$1" "$dir/err"; then fail "stderr has '$1'; stderr: $(<"$dir/err")"; fi
}

expect_out() {
  if ! grep -qF -- "$1" "$dir/out"; then fail "stdout lacks '$1'; stdout: $(<"$dir/out")"; fi
}

# expect_calls <line>... — the exact `bao` argument lines, in order.
expect_calls() {
  local want
  want=$(printf '%s\n' "$@")
  if [ "$(<"$log")" != "${want%$'\n'}" ]; then fail "bao calls were '$(<"$log")', expected '$*'"; fi
}

expect_file() {
  if [ ! -e "$1" ]; then
    fail "$1 missing"
    return
  fi
  if [ "$(<"$1")" != "$2" ]; then fail "$1 holds '$(<"$1")', expected '$2'"; fi
}

expect_no_file() {
  if [ -e "$1" ]; then fail "$1 left behind"; fi
}

secret_name=token
path=secret/swarm/agents/a1/forge-token
missing_msg="no forge token at $path yet"
login_call="login -method=cert -token-only"
kv_call="kv get -field=value $path"

setup FORGE "no store identity delivered"
: >"$creds/hive-agent-bao-cert"
go
expect_rc 0
expect_err "has no store identity"
expect_calls

setup FORGE "a credential that cannot be read"
chmod 0000 "$creds/hive-agent-bao-key"
go
expect_rc 1
expect_err "cannot read $creds/hive-agent-bao-key"
expect_calls

setup FORGE "bao's stderr file cannot be created"
chmod 0500 "$rt"
go
chmod 0700 "$rt"
expect_rc 1
expect_err "could not create $rt/bao.err, so bao never ran"
expect_calls

setup FORGE "the store refuses the login with an HTTP 4xx"
login_rc=2
login_err=$'Error authenticating: Error making API request.\n\nURL: PUT '"$bao_addr"$'/v1/auth/cert/login\nCode: 403. Errors:\n\n* permission denied\n'
go
expect_rc 1
expect_err "refused this agent's certificate login with HTTP 403:"
expect_err "* permission denied"
expect_calls "$login_call"

setup FORGE "the store fails the login with another HTTP status"
login_rc=2
login_err=$'Error authenticating: Error making API request.\n\nCode: 500. Errors:\n\n* internal error\n'
go
expect_rc 1
expect_err "failed this agent's certificate login with HTTP 500:"
expect_err "* internal error"
expect_calls "$login_call"

setup FORGE "the store refuses the certificate with a TLS alert"
login_rc=2
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": remote error: tls: unknown certificate authority"
go
expect_rc 1
expect_err "refused this agent's certificate in the TLS handshake:"
expect_err "remote error: tls: unknown certificate authority"
expect_calls "$login_call"

setup FORGE "the store does not answer"
login_rc=2
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host"
go
expect_rc 1
expect_err "got no answer from the swarm secret store at $bao_addr"
expect_err "no such host"
expect_calls "$login_call"

# The unit keeps its runtime directory between runs.
setup FORGE "nothing minted at the agent's path yet"
printf old >"$rt/$secret_name"
chmod 0400 "$rt/$secret_name"
kv_rc=2
kv_err="No value found at secret/data/swarm/agents/a1"
go
expect_rc 0
expect_err "$missing_msg"
expect_err "No value found"
expect_calls "$login_call" "$kv_call"
expect_file "$rt/$secret_name" old

setup FORGE "a first fetch writes the secret 0400"
go
expect_rc 0
expect_out "fetched this agent's"
expect_no_err "Permission denied"
expect_calls "$login_call" "$kv_call"
expect_file "$rt/$secret_name" the-secret
if [ "$(stat -c %a "$rt/$secret_name")" != 400 ]; then fail "$secret_name is mode $(stat -c %a "$rt/$secret_name"), expected 400"; fi
expect_no_file "$rt/bao.err"
expect_no_file "$rt/token.new"

# `UMask=0377` makes every file the script creates 0400, the login's own
# `bao.err` included, and a redirect into a 0400 file fails before bao starts.
setup FORGE "0400 files already in place do not block the fetch"
printf stale >"$rt/bao.err"
chmod 0400 "$rt/bao.err"
printf stale >"$rt/token.new"
chmod 0400 "$rt/token.new"
go
expect_rc 0
expect_out "fetched this agent's"
expect_no_err "Permission denied"
expect_no_err "refused"
expect_calls "$login_call" "$kv_call"
expect_file "$rt/$secret_name" the-secret

setup FORGE "an unchanged token is left in place"
printf the-secret >"$rt/token"
chmod 0400 "$rt/token"
before=$(stat -c %i "$rt/token")
go
expect_rc 0
expect_out "is unchanged"
expect_file "$rt/token" the-secret
if [ "$(stat -c %i "$rt/token")" != "$before" ]; then fail "the unchanged token was replaced"; fi
expect_no_file "$rt/token.new"

setup FORGE "a rotated token replaces the old one"
printf old >"$rt/token"
chmod 0400 "$rt/token"
go
expect_rc 0
expect_out "fetched this agent's forge token"
expect_file "$rt/token" the-secret
expect_no_file "$rt/token.new"

ident_path=secret/swarm/agents/a1/bao-mtls
ident_kv_call="kv get -format=json $ident_path"
leaf() { printf '{"data":{"data":{"cert":"%s","key":"%s"}}}' "$1" "$2"; }

expect_pair() {
  expect_file "$rt/current/cert" "$1"
  expect_file "$rt/current/key" "$2"
}

expect_pairs() {
  local d n=0
  for d in "$rt"/pair.*; do
    if [ -d "$d" ]; then n=$((n + 1)); fi
  done
  if [ "$n" != "$1" ]; then fail "$n pair directories left, expected $1"; fi
}

# live_pair <name> <cert> <key> — a pair an earlier run made live.
live_pair() {
  mkdir -m 0700 "$rt/pair.$1"
  printf '%s' "$2" >"$rt/pair.$1/cert"
  printf '%s' "$3" >"$rt/pair.$1/key"
  chmod 0400 "$rt/pair.$1/cert" "$rt/pair.$1/key"
  ln -sfn "pair.$1" "$rt/current"
}

# stop_at <n> — the n-th `ln` or `mv` the script runs fails, which leaves
# things as a run killed just before that step would.
stop_at() {
  local tool real
  mkdir "$dir/steps"
  for tool in ln mv; do
    real=$(PATH=$unit_path command -v "$tool")
    cat >"$dir/bin/$tool" <<STUB
#!$BASH
mktemp -p "$dir/steps" >/dev/null
steps=("$dir/steps"/*)
if [ "\${#steps[@]}" -ge "$1" ]; then exit 1; fi
exec "$real" "\$@"
STUB
    chmod +x "$dir/bin/$tool"
  done
}

setup IDENT "no store identity delivered"
: >"$creds/hive-agent-bao-cert"
go
expect_rc 1
expect_err "has no store identity"
expect_calls
if [ -e "$rt/current" ]; then fail "a live pair was made from nothing"; fi

setup IDENT "a first run makes the delivered pair live"
kv_out=$(leaf cert key)
go
expect_rc 0
expect_out "is unchanged"
expect_calls "$login_call" "$ident_kv_call"
if [ ! -L "$rt/current" ]; then fail "$rt/current is not a symlink"; fi
expect_pair cert key
if [ "$(stat -L -c %a "$rt/current/key")" != 400 ]; then fail "key is mode $(stat -L -c %a "$rt/current/key"), expected 400"; fi
expect_pairs 1
expect_no_file "$rt/bao-mtls.json"

setup IDENT "the live pair outlives the delivered one"
live_pair old cert2 key2
kv_out=$(leaf cert2 key2)
go
expect_rc 0
expect_out "is unchanged"
expect_pair cert2 key2
expect_pairs 1

setup IDENT "a re-issued leaf goes live once a login with it succeeds"
kv_out=$(leaf cert2 key2)
go
expect_rc 0
expect_out "replaced this agent's store certificate"
expect_calls "$login_call" "$ident_kv_call" "$login_call"
expect_pair cert2 key2
# The replaced pair stays for a reader that resolved the link before the swap.
expect_pairs 2
go
expect_rc 0
expect_out "is unchanged"
expect_pairs 1

setup IDENT "a re-issued leaf the store refuses does not go live"
kv_out=$(leaf cert2 key2)
refuse_cert=cert2
go
expect_rc 1
expect_err "refused the certificate at $ident_path"
expect_pair cert key
expect_pairs 1

setup IDENT "an object without a key keeps the live pair"
kv_out='{"data":{"data":{"cert":"cert2"}}}'
go
expect_rc 1
expect_err "holds no certificate and key"
expect_pair cert key
expect_pairs 1

setup IDENT "what a stopped run left behind is removed"
live_pair old cert key
mkdir -m 0700 "$rt/pair.stale"
printf cert2 >"$rt/pair.stale/cert"
ln -s pair.stale "$rt/current.new"
kv_out=$(leaf cert key)
go
expect_rc 0
expect_pair cert key
expect_pairs 1
if [ -L "$rt/current.new" ]; then fail "current.new left behind"; fi

# The swap is atomic: wherever a run stops, the link names one whole pair,
# and the next run finishes the job.
for n in 1 2 3; do
  setup IDENT "a swap stopped at its step $n leaves one whole pair live"
  live_pair old cert key
  kv_out=$(leaf cert2 key2)
  stop_at "$n"
  go
  got="$(<"$rt/current/cert") $(<"$rt/current/key")"
  case "$got" in
    "cert key" | "cert2 key2") ;;
    *) fail "the live pair is '$got', one file of each" ;;
  esac
  rm -f "$dir/bin/ln" "$dir/bin/mv"
  go
  expect_rc 0
  expect_pair cert2 key2
done

# The proxy fetch writes the agent's proxy variables, credential included, to
# one file the units load as an `EnvironmentFile=`. Nothing here may print the
# credential, so every case checks the output for it.
proxy_url=http://a1:the-secret@10.42.0.1:3128
proxy_no=localhost,127.0.0.1,::1,10.42.0.1,t.hive,bao.t.local
proxy_path=secret/swarm/agents/a1/proxy-credential
proxy_kv_call="kv get -field=value $proxy_path"
proxy_env() {
  printf 'HTTPS_PROXY=%s\nHTTP_PROXY=%s\nhttps_proxy=%s\nhttp_proxy=%s\nNODE_USE_ENV_PROXY=1\nNO_PROXY=%s\nno_proxy=%s\n' \
    "$proxy_url" "$proxy_url" "$proxy_url" "$proxy_url" "$1" "$1"
}

expect_secret_unprinted() {
  if grep -qF -e the-secret -e s.fake-token "$dir/out" "$dir/err"; then
    fail "a secret reached the unit's output: $(cat "$dir/out" "$dir/err")"
  fi
}

setup PROXY "no store identity delivered"
: >"$creds/hive-agent-bao-cert"
go
expect_rc 0
expect_err "has no store identity"
expect_calls
expect_no_file "$rt/proxy.env"

setup PROXY "a first fetch writes the variables 0600, and prints no secret"
go
expect_rc 0
expect_out "wrote this agent's proxy settings from $proxy_path"
expect_calls "$login_call" "$proxy_kv_call"
expect_file "$rt/proxy.env" "$(proxy_env "$proxy_no")"
if [ "$(stat -c %a "$rt/proxy.env")" != 600 ]; then fail "proxy.env is mode $(stat -c %a "$rt/proxy.env"), expected 600"; fi
expect_no_file "$rt/proxy.env.new"
expect_no_file "$rt/secret.new"
expect_no_file "$rt/bao.err"
expect_secret_unprinted

setup PROXY "no credential minted yet writes nothing"
kv_rc=2
kv_err="No value found at secret/data/swarm/agents/a1"
go
expect_rc 0
expect_err "no egress-proxy credential at $proxy_path yet"
expect_calls "$login_call" "$proxy_kv_call"
expect_no_file "$rt/proxy.env"
expect_no_file "$rt/proxy.env.new"
expect_no_file "$rt/secret.new"

setup PROXY "no credential minted yet leaves the file in place"
proxy_env old.example >"$rt/proxy.env"
chmod 0600 "$rt/proxy.env"
kv_rc=2
go
expect_rc 0
expect_file "$rt/proxy.env" "$(proxy_env old.example)"

setup PROXY "a store that refuses the login leaves the file in place"
proxy_env old.example >"$rt/proxy.env"
login_rc=2
login_err=$'Error authenticating: Error making API request.\n\nCode: 503. Errors:\n\n* Vault is sealed\n'
go
expect_rc 1
expect_err "could not log in to the swarm secret store"
expect_err "Vault is sealed"
expect_calls "$login_call"
expect_file "$rt/proxy.env" "$(proxy_env old.example)"

setup PROXY "a store that does not answer leaves the file in place"
proxy_env old.example >"$rt/proxy.env"
login_rc=2
login_err="Error authenticating: Put \"$bao_addr/v1/auth/cert/login\": dial tcp: lookup bao.t.local: no such host"
go
expect_rc 1
expect_file "$rt/proxy.env" "$(proxy_env old.example)"

setup PROXY "0600 files already in place do not block the fetch"
printf stale >"$rt/proxy.env.new"
printf stale >"$rt/secret.new"
printf stale >"$rt/bao.err"
go
expect_rc 0
expect_no_err "Permission denied"
expect_file "$rt/proxy.env" "$(proxy_env "$proxy_no")"

setup PROXY "an unchanged credential leaves the file alone"
proxy_env "$proxy_no" >"$rt/proxy.env"
before=$(stat -c %i "$rt/proxy.env")
go
expect_rc 0
expect_out "are unchanged"
if [ "$(stat -c %i "$rt/proxy.env")" != "$before" ]; then fail "the unchanged file was replaced"; fi
expect_no_file "$rt/proxy.env.new"

setup PROXY "a changed credential replaces the file whole"
proxy_env old.example >"$rt/proxy.env"
go
expect_rc 0
expect_file "$rt/proxy.env" "$(proxy_env "$proxy_no")"

setup PROXY "a value that is not a base64url secret is refused and the file kept"
proxy_env old.example >"$rt/proxy.env"
kv_out=$'abc\nNO_PROXY=*'
go
expect_rc 1
expect_err "is not a base64url secret"
expect_file "$rt/proxy.env" "$(proxy_env old.example)"
if grep -qF abc "$dir/out" "$dir/err"; then fail "the refused value was printed"; fi

setup PROXY "an empty value is refused"
kv_out=
go
expect_rc 1
expect_err "is not a base64url secret"
expect_no_file "$rt/proxy.env"

# A hive that predates the facts, or whose squid does not announce itself, gets
# no proxy: the store is not asked and nothing is written.
no_fact() {
  expect_rc 0
  expect_err "this hive's facts name no squid proxy"
  expect_calls
  expect_no_file "$rt/proxy.env"
  expect_secret_unprinted
}

setup PROXY "a hive that writes no egress facts means no proxy"
printf 'HIVE_DASHBOARD_PORT=7000\nHYPERHIVE_HIVE_DOMAIN=t.hive\n' >"$dir/etc/hyperhive-hive-facts"
go
no_fact

setup PROXY "no facts file means no proxy"
rm "$dir/etc/hyperhive-hive-facts"
go
no_fact

setup PROXY "a proxy fact without the direct hosts means no proxy"
printf 'HIVE_SQUID_PROXY=10.42.0.1:3128\n' >"$dir/etc/hyperhive-hive-facts"
go
no_fact

setup PROXY "direct hosts without a proxy fact mean no proxy"
printf 'HIVE_EGRESS_NO_PROXY=10.42.0.1,t.hive\n' >"$dir/etc/hyperhive-hive-facts"
go
no_fact

for bad in 'not-an-address' 'http://10.42.0.1:3128' '10.42.0.1' 'proxy.t.local:3128'; do
  setup PROXY "a proxy fact that is not an address and a port is refused: $bad"
  printf 'HIVE_SQUID_PROXY=%s\nHIVE_EGRESS_NO_PROXY=10.42.0.1\n' "$bad" >"$dir/etc/hyperhive-hive-facts"
  go
  no_fact
done

setup PROXY "direct hosts that are not host names are refused"
printf 'HIVE_SQUID_PROXY=10.42.0.1:3128\nHIVE_EGRESS_NO_PROXY=a b,c\n' >"$dir/etc/hyperhive-hive-facts"
go
no_fact

setup PROXY "no facts leave the file in place"
proxy_env old.example >"$rt/proxy.env"
rm "$dir/etc/hyperhive-hive-facts"
go
expect_rc 0
expect_file "$rt/proxy.env" "$(proxy_env old.example)"

setup PROXY "the direct hosts are used as the hive wrote them, after the loopback names"
printf 'HIVE_SQUID_PROXY=10.9.8.7:3129\nHIVE_EGRESS_NO_PROXY=10.9.8.7,x.t.hive,y.t.hive\n' >"$dir/etc/hyperhive-hive-facts"
go
expect_rc 0
expect_file "$rt/proxy.env" "$(proxy_url=http://a1:the-secret@10.9.8.7:3129 proxy_env localhost,127.0.0.1,::1,10.9.8.7,x.t.hive,y.t.hive)"

if [ "$failures" -ne 0 ]; then
  echo "script-test-agent-bao-fetch: $failures failed assertions over $count runs" >&2
  exit 1
fi
echo "script-test-agent-bao-fetch: $count runs, all assertions hold"