send agent traffic through squid once its hive announces it
This commit is contained in:
atlas
parent
86e9f49fe6
commit
b551eb02ff
3 changed files with 885 additions and 27 deletions
|
|
@ -71,6 +71,7 @@ in
|
|||
./claude-settings.nix
|
||||
./dashboard-links.nix
|
||||
./docs.nix
|
||||
./egress-proxy.nix
|
||||
./forge.nix
|
||||
./forge-accounts.nix
|
||||
./forge-token.nix
|
||||
|
|
|
|||
240
nix/agent-modules/egress-proxy.nix
Normal file
240
nix/agent-modules/egress-proxy.nix
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
# This agent's own egress-proxy credential, fetched from the swarm secret store
|
||||
# by the agent itself, and the proxy environment built from it.
|
||||
#
|
||||
# `swarm-controller` mints the credential at
|
||||
# `swarm/agents/<agent>/proxy-credential`
|
||||
# (`swarm_secret_client::proxy::agent_credential_path`); the hive's squid checks
|
||||
# it against the hash `hive-c0re` renders. This unit logs in with the
|
||||
# certificate ./bao.nix keeps current, reads the agent's own path, and writes
|
||||
# `proxy.env` for the units below to load as an `EnvironmentFile=`:
|
||||
#
|
||||
# - `HTTPS_PROXY` and `HTTP_PROXY`, upper and lower case, with the credential in
|
||||
# the URL's userinfo, which Claude Code, Node, curl, git and reqwest send as
|
||||
# `Proxy-Authorization`;
|
||||
# - `NODE_USE_ENV_PROXY=1`, without which Node's global `fetch` ignores them;
|
||||
# - `NO_PROXY` and `no_proxy`.
|
||||
#
|
||||
# The file is written only when the credential AND the hive's facts are both
|
||||
# there. `HIVE_SQUID_PROXY` (the proxy's address) and `HIVE_EGRESS_NO_PROXY` (the
|
||||
# hosts that skip it) are written by a hive whose squid checks credentials and
|
||||
# applies each agent's flag; on any other hive the agent has no proxy variable
|
||||
# and reaches the internet directly. The units load the file with a leading `-`,
|
||||
# so an agent without it is unchanged. A failed read keeps the file already in
|
||||
# place, and units read it when they start, so an agent takes the proxy at its
|
||||
# next start.
|
||||
#
|
||||
# `NO_PROXY` is the loopback names and the hive's list as it is: squid refuses
|
||||
# every destination that resolves to the bridge, and the hive's resolver answers
|
||||
# the bridge address for the swarm's services. Node has no CIDR support, so the
|
||||
# list holds the bridge as an address.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.hyperhive.agent.bao;
|
||||
agentCfg = config.services.hyperhive.agent;
|
||||
|
||||
# The name `swarm-controller` minted the credential under — see ./queue-identity.nix.
|
||||
agentName = agentCfg.user.name;
|
||||
|
||||
# The server CA id ./bao.nix loads.
|
||||
serverCaCredential = "hive-agent-bao-server-ca";
|
||||
|
||||
unitName = "hive-agent-proxy-credential";
|
||||
|
||||
# The nix half of `swarm_secret_client::proxy::agent_credential_path` plus
|
||||
# `path::MOUNT`.
|
||||
credentialPath = "secret/swarm/agents/${agentName}/proxy-credential";
|
||||
|
||||
runtimeDir = unitName;
|
||||
envFile = "/run/${runtimeDir}/proxy.env";
|
||||
# Beside the file, so the rename that replaces it stays in one directory.
|
||||
stagingFile = "/run/${runtimeDir}/proxy.env.new";
|
||||
secretFile = "/run/${runtimeDir}/secret.new";
|
||||
# bao's stderr, in the unit's own `0700` directory rather than `/tmp`.
|
||||
errFile = "/run/${runtimeDir}/bao.err";
|
||||
|
||||
# Written by `hive-priv` into every container (./hive-facts.nix). The two
|
||||
# egress facts are read here and nowhere else: the generator there exports
|
||||
# only the keys it names, so neither reaches a unit's environment.
|
||||
factsFile = "/etc/hyperhive-hive-facts";
|
||||
|
||||
loopback = [
|
||||
"localhost"
|
||||
"127.0.0.1"
|
||||
"::1"
|
||||
];
|
||||
|
||||
# The store's address is the whole switch, as in ./bao.nix and
|
||||
# ./forge-token.nix.
|
||||
configured = cfg.addr != null;
|
||||
|
||||
storeRetry = import ../host-modules/lib/store-retry.nix { };
|
||||
|
||||
# Units that make outbound calls: the harness and the claude it starts, the
|
||||
# bash tool and the subagents it runs, MCP servers, and the pollers. They
|
||||
# exist under the same conditions as their definitions, because a unit that
|
||||
# is not defined would be generated inert.
|
||||
consumers = [
|
||||
"hive-agent"
|
||||
"hive-bash-daemon"
|
||||
"hive-subagent-daemon"
|
||||
"hive-mcp-http"
|
||||
"hive-forge-notify"
|
||||
]
|
||||
++ lib.optional (agentCfg.matrixAccounts != { }) "hive-matrix-daemon"
|
||||
++ lib.optional agentCfg.github.enable "hive-github-notify"
|
||||
++ lib.optional (agentCfg.icon != null && agentCfg.forge.url != null) "forge-avatar-sync";
|
||||
in
|
||||
{
|
||||
config = lib.mkIf configured {
|
||||
systemd.services = {
|
||||
${unitName} = {
|
||||
description = "fetch this agent's egress-proxy credential from the secret store";
|
||||
after = [
|
||||
"network.target"
|
||||
# Ordering only: that unit reports a store this agent cannot reach as
|
||||
# itself.
|
||||
"hive-agent-bao-identity.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
pkgs.openbao
|
||||
pkgs.coreutils
|
||||
pkgs.diffutils
|
||||
];
|
||||
# ../host-modules/lib/store-retry.nix.
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
# Not `RemainAfterExit`: the timer below has to be able to start this
|
||||
# unit again. `RuntimeDirectoryPreserve` keeps the file between runs.
|
||||
RemainAfterExit = false;
|
||||
TimeoutStartSec = 30;
|
||||
User = agentName;
|
||||
Group = agentName;
|
||||
RuntimeDirectory = runtimeDir;
|
||||
RuntimeDirectoryMode = "0700";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
UMask = "0177";
|
||||
LoadCredential = [ serverCaCredential ];
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = cfg.addr;
|
||||
BAO_CLIENT_CERT = cfg.clientCertFile;
|
||||
BAO_CLIENT_KEY = cfg.clientKeyFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
# No identity delivered: ./bao.nix's check reports that.
|
||||
for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
|
||||
if [ ! -s "$f" ]; then
|
||||
echo "this agent has no store identity, so it cannot fetch its egress-proxy credential." >&2
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -r "$f" ]; then
|
||||
echo "cannot read $f, so this agent cannot present its store identity." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# The hive says its squid checks each agent's credential and applies
|
||||
# its flag by writing both facts. Without them (a hive that predates
|
||||
# them) there is no proxy to name, and the store is not asked.
|
||||
proxy= direct=
|
||||
if [ -r ${factsFile} ]; then
|
||||
while IFS='=' read -r key value || [ -n "$key" ]; do
|
||||
case "$key" in
|
||||
HIVE_SQUID_PROXY) proxy=$value ;;
|
||||
HIVE_EGRESS_NO_PROXY) direct=$value ;;
|
||||
esac
|
||||
done < ${factsFile}
|
||||
fi
|
||||
if [[ ! "$proxy" =~ ^[0-9]+(\.[0-9]+){3}:[0-9]+$ || ! "$direct" =~ ^[A-Za-z0-9.,_-]+$ ]]; then
|
||||
echo "this hive's facts name no squid proxy, so this agent stays direct." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
||||
export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
|
||||
fi
|
||||
|
||||
err=${lib.escapeShellArg errFile}
|
||||
trap 'rm -f "$err" ${lib.escapeShellArg stagingFile} ${lib.escapeShellArg secretFile}' EXIT
|
||||
|
||||
# 🩸 Every exit below that is not a rename leaves ${envFile} as it
|
||||
# was: a store that is briefly unreachable, or has no credential for
|
||||
# this agent yet, must not take a working proxy away, and must not
|
||||
# invent one.
|
||||
rm -f "$err"
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to the swarm secret store at $BAO_ADDR; the proxy settings in place, if any, are kept:" >&2
|
||||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
# ⚠️ Written by redirect, never echoed: the field is the secret.
|
||||
rm -f "$err" ${lib.escapeShellArg secretFile}
|
||||
if ! bao kv get -field=value ${lib.escapeShellArg credentialPath} > ${lib.escapeShellArg secretFile} 2>"$err"; then
|
||||
echo "no egress-proxy credential at ${credentialPath} yet; this agent sends no proxy variables until there is one." >&2
|
||||
if [ -s "$err" ]; then cat "$err" >&2; fi
|
||||
exit 0
|
||||
fi
|
||||
secret="$(<${lib.escapeShellArg secretFile})"
|
||||
|
||||
# The controller mints base64url, which needs no escaping in a URL's
|
||||
# userinfo or an env file. Anything else is not that secret.
|
||||
if [[ ! "$secret" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
||||
echo "the value at ${credentialPath} is not a base64url secret, so no proxy variables are written." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
no_proxy=${lib.escapeShellArg (lib.concatStringsSep "," loopback)},$direct
|
||||
url="http://${agentName}:$secret@$proxy"
|
||||
|
||||
rm -f ${lib.escapeShellArg stagingFile}
|
||||
{
|
||||
printf 'HTTPS_PROXY=%s\n' "$url"
|
||||
printf 'HTTP_PROXY=%s\n' "$url"
|
||||
printf 'https_proxy=%s\n' "$url"
|
||||
printf 'http_proxy=%s\n' "$url"
|
||||
printf 'NODE_USE_ENV_PROXY=1\n'
|
||||
printf 'NO_PROXY=%s\n' "$no_proxy"
|
||||
printf 'no_proxy=%s\n' "$no_proxy"
|
||||
} > ${lib.escapeShellArg stagingFile}
|
||||
|
||||
if cmp -s ${lib.escapeShellArg stagingFile} ${lib.escapeShellArg envFile}; then
|
||||
echo "this agent's proxy settings are unchanged."
|
||||
exit 0
|
||||
fi
|
||||
mv -f ${lib.escapeShellArg stagingFile} ${lib.escapeShellArg envFile}
|
||||
echo "wrote this agent's proxy settings from ${credentialPath}."
|
||||
'';
|
||||
};
|
||||
}
|
||||
// lib.genAttrs consumers (_: {
|
||||
# `wants`, not `requires`: a store outage must not stop the harness, which
|
||||
# then starts direct. `after` waits for one attempt.
|
||||
wants = [ "${unitName}.service" ];
|
||||
after = [ "${unitName}.service" ];
|
||||
# A list, which merges with the entries other modules give a unit.
|
||||
serviceConfig.EnvironmentFile = [ "-${envFile}" ];
|
||||
});
|
||||
|
||||
# The controller re-checks every agent's credential every five minutes; the
|
||||
# fetch picks one up within about ten more.
|
||||
systemd.timers.${unitName} = {
|
||||
description = "re-fetch this agent's egress-proxy credential from the secret store";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnUnitInactiveSec = "10min";
|
||||
RandomizedDelaySec = "1min";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -12,6 +12,8 @@
|
|||
# `OTEL_RESOURCE_ATTRIBUTES` by ./otel.nix (no extra labels when absent).
|
||||
# - `HIVE_AGENT_NATS_URL`, where the harness reaches the swarm queue (it fails
|
||||
# at boot when absent).
|
||||
# - `HIVE_SQUID_PROXY` and `HIVE_EGRESS_NO_PROXY`: the squid address and the hosts
|
||||
# that skip it, read only by ./egress-proxy.nix; the generator drops them.
|
||||
#
|
||||
# Any of them may be absent: a hive-c0re older than the agent's build writes
|
||||
# fewer. Two readers hand the lines on:
|
||||
|
|
|
|||
Loading…
Reference in a new issue