453 lines
17 KiB
Nix
453 lines
17 KiB
Nix
# The swarm's binary cache: an attic server in a `swarm-attic` container on the
|
||
# host that sets `deploy.attic.enable`, which `deploy.allSwarmServices` defaults
|
||
# on where the host has the cache's store identity. The swarm builder pushes
|
||
# green builds to it (docs/swarm/builder.md).
|
||
#
|
||
# Two host units run under the cache's own store leaf, the shape
|
||
# ./swarm-grafana.nix gives its admin password. One creates the key atticd signs
|
||
# tokens with, if the store holds none, and hands it to the container. The other
|
||
# creates the cache, the push and pull tokens, and the facts hives pull with
|
||
# (its address, name and public key), each only while absent
|
||
# (./lib/attic-init.nix). Neither needs a manual step.
|
||
#
|
||
# The container shares the host's network namespace, as ./swarm-victorialogs.nix
|
||
# does, and atticd listens on loopback only: the gateway's vhost is the way in,
|
||
# and attic's own tokens are the authentication. Storage is local disk under the
|
||
# container's state.
|
||
#
|
||
# The two `swarm.attic` options every host reads (./swarm-attic-options.nix)
|
||
# default to this cache's names on every host, whether or not it runs here.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
cfg = hyperhiveCfg.deploy.attic;
|
||
atticCfg = hyperhiveCfg.swarm.attic;
|
||
baoCfg = hyperhiveCfg.swarm.bao;
|
||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
||
networkCfg = hyperhiveCfg.network;
|
||
gatewayCfg = hyperhiveCfg.gateway;
|
||
|
||
machine = "swarm-attic";
|
||
unit = "swarm-attic";
|
||
|
||
# Where the container keeps its database and the cache's files.
|
||
stateName = "swarm-attic";
|
||
stateDir = "/var/lib/${stateName}";
|
||
|
||
# The host directory that carries the signing key into the container. The
|
||
# key's file is the only thing in it.
|
||
secretDir = "/run/swarm-attic-secrets";
|
||
envFile = "${secretDir}/server.env";
|
||
|
||
# Where the host units reach atticd: the shared loopback, not the gateway.
|
||
endpoint = "http://127.0.0.1:${toString cfg.port}";
|
||
|
||
configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" {
|
||
listen = "127.0.0.1:${toString cfg.port}";
|
||
# What clients are told to use for the cache's API.
|
||
api-endpoint = "https://${atticCfg.domain}/";
|
||
database.url = "sqlite://${stateDir}/server.db?mode=rwc";
|
||
storage = {
|
||
type = "local";
|
||
path = "${stateDir}/storage";
|
||
};
|
||
# attic's defaults, which its config requires spelled out.
|
||
chunking = {
|
||
nar-size-threshold = 65536;
|
||
min-size = 16384;
|
||
avg-size = 65536;
|
||
max-size = 262144;
|
||
};
|
||
garbage-collection = {
|
||
interval = "12 hours";
|
||
default-retention-period = cfg.retentionPeriod;
|
||
};
|
||
};
|
||
|
||
atticInit = import ./lib/attic-init.nix { inherit lib; } {
|
||
jwtPath = "secret/swarm/services/attic/jwt-secret";
|
||
pullPath = "secret/swarm/services/attic/pull-token";
|
||
pushPath = "secret/swarm/services/attic/push-token";
|
||
urlPath = "secret/swarm/services/attic/url";
|
||
cachePath = "secret/swarm/services/attic/cache";
|
||
publicKeyPath = "secret/swarm/services/attic/public-key";
|
||
url = "https://${atticCfg.domain}";
|
||
cache = atticCfg.cache;
|
||
inherit
|
||
endpoint
|
||
configFile
|
||
envFile
|
||
machine
|
||
unit
|
||
;
|
||
};
|
||
|
||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||
storeRetry = import ./lib/store-retry.nix { };
|
||
|
||
haveClientIdentity = baoDeploy.atticClientCertFile != null && baoDeploy.atticClientKeyFile != null;
|
||
|
||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||
# collector, the route `serviceProducers.swarm-attic` in
|
||
# ./swarm-otel-service.nix declares.
|
||
forwarderClientId = atticCfg.otel.clientId;
|
||
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${machine}";
|
||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||
# In the host netns: distinct from every other collector's there.
|
||
forwarderTelemetryPort = 8903;
|
||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||
# The cache's store identity, handed into the container as system credentials
|
||
# for the forwarder's fetch unit. The host units keep reading the same files.
|
||
forwarderStoreFiles = lib.filterAttrs (_: source: source != null) {
|
||
attic-bao-cert = baoDeploy.atticClientCertFile;
|
||
attic-bao-key = baoDeploy.atticClientKeyFile;
|
||
attic-bao-server-ca = baoDeploy.serverCaFile;
|
||
};
|
||
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
||
# one word.
|
||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||
|
||
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
||
# self-signed mode their leaves chain to the hive CA.
|
||
caTrust = import ./lib/hive-ca-trust.nix {
|
||
inherit lib gatewayCfg;
|
||
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
|
||
};
|
||
|
||
# Both host units log in as the cache's principal.
|
||
storeEnvironment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = baoDeploy.atticClientCertFile;
|
||
BAO_CLIENT_KEY = baoDeploy.atticClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
in
|
||
{
|
||
options.services.hyperhive.deploy.attic = {
|
||
package = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.attic-server;
|
||
defaultText = lib.literalExpression "pkgs.attic-server";
|
||
description = "The attic server package: `atticd`, and `atticadm` for the tokens.";
|
||
};
|
||
|
||
clientPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.attic-client;
|
||
defaultText = lib.literalExpression "pkgs.attic-client";
|
||
description = "The attic client package, which creates the cache.";
|
||
};
|
||
|
||
port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 8910;
|
||
description = ''
|
||
Loopback port atticd listens on, which the gateway proxies to.
|
||
|
||
⚠️ Every swarm container shares the host's network namespace, so this
|
||
is a swarm-wide claim: two modules picking the same number collide at
|
||
runtime with no bind error. No repo-wide check catches it.
|
||
'';
|
||
};
|
||
|
||
retentionPeriod = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "14 days";
|
||
example = "30 days";
|
||
description = ''
|
||
How long a store path stays in the cache after it was added or last
|
||
fetched. A garbage collection every twelve hours removes what is older;
|
||
swarm-controller rebuilds a closure the cache no longer holds and
|
||
the builder pushes it again.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf cfg.enable {
|
||
assertions = [
|
||
{
|
||
assertion = haveClientIdentity;
|
||
message = ''
|
||
services.hyperhive.deploy.attic.enable requires the cache's
|
||
swarm-secret-store client identity: set both
|
||
|
||
services.hyperhive.deploy.bao.atticClientCertFile
|
||
services.hyperhive.deploy.bao.atticClientKeyFile
|
||
|
||
The cache creates its signing key and tokens in the store with it.
|
||
|
||
Both default to the pair glue-bao-tls.nix mints when
|
||
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
||
store). Otherwise copy attic.pem and attic-key.pem from the store
|
||
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
||
'';
|
||
}
|
||
{
|
||
assertion = atticCfg.domain != null && atticCfg.cache != null;
|
||
message = ''
|
||
services.hyperhive.deploy.attic.enable needs
|
||
services.hyperhive.swarm.attic.domain and
|
||
services.hyperhive.swarm.attic.cache: it serves the cache at that name
|
||
and creates it by that name. They default when unset, so one of them
|
||
is set to null.
|
||
'';
|
||
}
|
||
{
|
||
assertion = lib.all oneWord (lib.attrValues forwarderStoreFiles);
|
||
message = ''
|
||
The cache's store identity paths must not contain whitespace:
|
||
nspawn's --load-credential receives them through nixos-container's
|
||
unquoted extra flags.
|
||
'';
|
||
}
|
||
{
|
||
assertion =
|
||
(hyperhiveCfg.swarm.otel.serviceProducers.${machine}.clientId or null) == forwarderClientId;
|
||
message = ''
|
||
services.hyperhive.swarm.otel.serviceProducers has no entry
|
||
"${machine}" with clientId "${forwarderClientId}", so the swarm
|
||
collector has no receiver for the cache's journal forwarder.
|
||
Declare it in swarm-otel-service.nix.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# nspawn fails to start the container when a credential source is
|
||
# missing, so where this host mints the leaf the container waits for it. The
|
||
# CA ordering keeps the trust bundle's bind source there first.
|
||
systemd.services."container@${machine}" = lib.mkMerge [
|
||
caTrust.containerOrdering
|
||
{
|
||
after = [ "swarm-bao-pki.service" ];
|
||
wants = [ "swarm-bao-pki.service" ];
|
||
}
|
||
];
|
||
|
||
services.hyperhive.gateway.localNames = [ atticCfg.domain ];
|
||
|
||
# The init units' journal reaches the log store through the hive collector.
|
||
# A host that also runs the swarm collector ships its whole journal already
|
||
# (./swarm-otel.nix), so naming them there would send each line twice.
|
||
services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
|
||
"swarm-bao-attic-secret.service"
|
||
"swarm-bao-attic-tokens.service"
|
||
];
|
||
|
||
# This host serves the vhost, and the host's own resolver finds the name.
|
||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||
|
||
# `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks
|
||
# its own tokens, and a nix client cannot follow a login redirect. A store
|
||
# path is uploaded as one request that can run for minutes, so the body is
|
||
# neither size-limited nor buffered.
|
||
services.hyperhive.gateway.vhosts."${atticCfg.domain}" = {
|
||
forceSSL = true;
|
||
locations."/" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||
extraConfig = ''
|
||
client_max_body_size 0;
|
||
proxy_request_buffering off;
|
||
proxy_read_timeout 1h;
|
||
proxy_send_timeout 1h;
|
||
'';
|
||
};
|
||
};
|
||
|
||
# The key's directory is the container's read-only bind source, so it has
|
||
# to exist before the container starts even when the store does not answer.
|
||
# The delivery unit creates it too, which covers a switch.
|
||
systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ];
|
||
|
||
# The signing key, created in the store if it holds none and written where
|
||
# the container reads it. Before the container and wanted by it: a store
|
||
# that comes up later fails this run, atticd restarts until the file is
|
||
# there, and the retry that lands it restarts atticd.
|
||
systemd.services.swarm-bao-attic-secret = {
|
||
description = "create the swarm cache's signing key in the secret store if absent and hand it to the container";
|
||
# Ordered after the store where it runs here:
|
||
# ./glue-bao-readers-store-order.nix and ./glue-bao-readers-policy-order.nix.
|
||
before = [ "container@${machine}.service" ];
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"container@${machine}.service"
|
||
];
|
||
path = [
|
||
baoDeploy.package
|
||
pkgs.coreutils
|
||
pkgs.openssl
|
||
pkgs.systemd
|
||
];
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-bao-attic-secret";
|
||
# Generating a 4096-bit key takes seconds.
|
||
TimeoutStartSec = 120;
|
||
};
|
||
environment = storeEnvironment;
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
${atomicWriteSecret}
|
||
${refreshConsumer}
|
||
|
||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||
|
||
${atticInit.prelude}
|
||
${atticInit.ensureJwt}
|
||
'';
|
||
};
|
||
|
||
# The cache, its two tokens and its public facts, once atticd answers. The
|
||
# push token is the path the swarm builder reads; the pull token and the
|
||
# facts are for the hives, which the controller copies them to.
|
||
systemd.services.swarm-bao-attic-tokens = {
|
||
description = "create the swarm cache, and its tokens and public facts in the secret store, if absent";
|
||
after = [
|
||
"container@${machine}.service"
|
||
"swarm-bao-attic-secret.service"
|
||
];
|
||
wants = [ "container@${machine}.service" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
path = [
|
||
baoDeploy.package
|
||
cfg.package
|
||
cfg.clientPackage
|
||
pkgs.coreutils
|
||
pkgs.curl
|
||
pkgs.gnused
|
||
];
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-bao-attic-tokens";
|
||
# Up to a minute waiting for atticd.
|
||
TimeoutStartSec = 180;
|
||
};
|
||
environment = storeEnvironment;
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
${atticInit.prelude}
|
||
${atticInit.provision}
|
||
'';
|
||
};
|
||
|
||
containers.${machine} = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# No `--link-journal=host`: the journal stays in the guest, where the
|
||
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
||
extraFlags = lib.mapAttrsToList (
|
||
id: source: "--load-credential=${id}:${source}"
|
||
) forwarderStoreFiles;
|
||
privateNetwork = false;
|
||
bindMounts = {
|
||
${secretDir} = {
|
||
hostPath = secretDir;
|
||
isReadOnly = true;
|
||
};
|
||
}
|
||
// caTrust.bindMount;
|
||
|
||
config =
|
||
{ ... }:
|
||
{
|
||
imports = [
|
||
../container-modules/swarm-container.nix
|
||
../container-modules/swarm-journal-forwarder.nix
|
||
(import ../container-modules/swarm-container-resolver.nix {
|
||
inherit (networkCfg) bridgeIp;
|
||
# The forwarder and its secret fetch dial the swarm collector,
|
||
# authelia and the store by name.
|
||
dnsConsumers = [
|
||
"opentelemetry-collector.service"
|
||
"${forwarderUnit}.service"
|
||
];
|
||
})
|
||
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
||
# collector's and authelia's names.
|
||
(caTrust.trustBundle {
|
||
inherit pkgs;
|
||
name = machine;
|
||
consumers = [ "opentelemetry-collector" ];
|
||
})
|
||
];
|
||
|
||
services.hyperhive.swarmContainer.privateNetwork = false;
|
||
|
||
# Authenticated as the forwarder's own OIDC client, with the cache's
|
||
# store identity: one identity for the whole container. It is a
|
||
# system credential, so the fetch unit loads it.
|
||
services.hyperhive.swarmJournalForwarder = {
|
||
enable = true;
|
||
serviceName = "attic";
|
||
endpoint = forwarderEndpoint;
|
||
clientId = forwarderClientId;
|
||
tokenUrl = "${toString autheliaCfg.url}/api/oidc/token";
|
||
telemetryPort = forwarderTelemetryPort;
|
||
store = {
|
||
package = baoDeploy.package;
|
||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
||
clientCertFile = forwarderCredential "attic-bao-cert";
|
||
clientKeyFile = forwarderCredential "attic-bao-key";
|
||
caFile =
|
||
if forwarderStoreFiles ? attic-bao-server-ca then
|
||
forwarderCredential "attic-bao-server-ca"
|
||
else
|
||
null;
|
||
};
|
||
};
|
||
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
|
||
|
||
systemd.services.${unit} = {
|
||
description = "attic binary cache server";
|
||
wantedBy = [ "multi-user.target" ];
|
||
serviceConfig = {
|
||
ExecStart = "${cfg.package}/bin/atticd -f ${configFile} --mode monolithic";
|
||
# The signing key, in an environment file the host unit writes.
|
||
EnvironmentFile = envFile;
|
||
StateDirectory = stateName;
|
||
DynamicUser = true;
|
||
Restart = "on-failure";
|
||
RestartSec = 10;
|
||
SyslogIdentifier = unit;
|
||
UMask = "0077";
|
||
CapabilityBoundingSet = [ "" ];
|
||
NoNewPrivileges = true;
|
||
PrivateDevices = true;
|
||
PrivateTmp = true;
|
||
ProtectSystem = "strict";
|
||
ProtectHome = true;
|
||
ProtectKernelTunables = true;
|
||
ProtectKernelModules = true;
|
||
ProtectControlGroups = true;
|
||
RestrictAddressFamilies = [
|
||
"AF_INET"
|
||
"AF_INET6"
|
||
"AF_UNIX"
|
||
];
|
||
RestrictNamespaces = true;
|
||
RestrictRealtime = true;
|
||
RestrictSUIDSGID = true;
|
||
LockPersonality = true;
|
||
SystemCallArchitectures = "native";
|
||
};
|
||
};
|
||
};
|
||
};
|
||
};
|
||
}
|