# The swarm's binary cache: an attic server in a `swarm-attic` container on the # host that sets `deploy.attic.enable`, which `deploy.allSwarmServices` defaults # on where the host has the cache's store identity. The swarm builder pushes # green builds to it (docs/swarm/builder.md). # # Two host units run under the cache's own store leaf, the shape # ./swarm-grafana.nix gives its admin password. One creates the key atticd signs # tokens with, if the store holds none, and hands it to the container. The other # creates the cache and the push and pull tokens, each only while absent # (./lib/attic-init.nix). Neither needs a manual step. # # The container shares the host's network namespace, as ./swarm-victorialogs.nix # does, and atticd listens on loopback only: the gateway's vhost is the way in, # and attic's own tokens are the authentication. Storage is local disk under the # container's state. # # The two `swarm.attic` options every host reads (./swarm-attic-options.nix) get # their values from here, at `mkDefault`: naming them yourself wins, and with # the switch off nothing is set. { pkgs, lib, config, ... }: let hyperhiveCfg = config.services.hyperhive; cfg = hyperhiveCfg.deploy.attic; atticCfg = hyperhiveCfg.swarm.attic; baoCfg = hyperhiveCfg.swarm.bao; baoDeploy = hyperhiveCfg.deploy.bao; machine = "swarm-attic"; unit = "swarm-attic"; # Where the container keeps its database and the cache's files. stateName = "swarm-attic"; stateDir = "/var/lib/${stateName}"; # The host directory that carries the signing key into the container. The # key's file is the only thing in it. secretDir = "/run/swarm-attic-secrets"; envFile = "${secretDir}/server.env"; # Where the host units reach atticd: the shared loopback, not the gateway. endpoint = "http://127.0.0.1:${toString cfg.port}"; configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" { listen = "127.0.0.1:${toString cfg.port}"; # What clients are told to use for the cache's API. api-endpoint = "https://${atticCfg.domain}/"; database.url = "sqlite://${stateDir}/server.db?mode=rwc"; storage = { type = "local"; path = "${stateDir}/storage"; }; # attic's defaults, which its config requires spelled out. chunking = { nar-size-threshold = 65536; min-size = 16384; avg-size = 65536; max-size = 262144; }; garbage-collection = { interval = "12 hours"; default-retention-period = cfg.retentionPeriod; }; }; atticInit = import ./lib/attic-init.nix { inherit lib; } { jwtPath = "secret/swarm/services/attic/jwt-secret"; pullPath = "secret/swarm/services/attic/pull-token"; pushPath = "secret/swarm/services/attic/push-token"; cache = atticCfg.cache; inherit endpoint configFile envFile machine unit ; }; atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; storeRetry = import ./lib/store-retry.nix { }; haveClientIdentity = baoDeploy.atticClientCertFile != null && baoDeploy.atticClientKeyFile != null; # Both host units log in as the cache's principal. storeEnvironment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; BAO_CLIENT_CERT = baoDeploy.atticClientCertFile; BAO_CLIENT_KEY = baoDeploy.atticClientKeyFile; } // lib.optionalAttrs (baoDeploy.serverCaFile != null) { BAO_CACERT = baoDeploy.serverCaFile; }; in { options.services.hyperhive.deploy.attic = { package = lib.mkOption { type = lib.types.package; default = pkgs.attic-server; defaultText = lib.literalExpression "pkgs.attic-server"; description = "The attic server package: `atticd`, and `atticadm` for the tokens."; }; clientPackage = lib.mkOption { type = lib.types.package; default = pkgs.attic-client; defaultText = lib.literalExpression "pkgs.attic-client"; description = "The attic client package, which creates the cache."; }; port = lib.mkOption { type = lib.types.port; default = 8910; description = '' Loopback port atticd listens on, which the gateway proxies to. ⚠️ Every swarm container shares the host's network namespace, so this is a swarm-wide claim: two modules picking the same number collide at runtime with no bind error. No repo-wide check catches it. ''; }; retentionPeriod = lib.mkOption { type = lib.types.str; default = "14 days"; example = "30 days"; description = '' How long a store path stays in the cache after it was added or last fetched. A garbage collection every twelve hours removes what is older; a build the cache no longer holds is rebuilt on the next miss. ''; }; }; config = lib.mkIf cfg.enable { services.hyperhive.swarm.attic = { domain = lib.mkDefault atticCfg.ownDomain; cache = lib.mkDefault "swarm"; }; assertions = [ { assertion = haveClientIdentity; message = '' services.hyperhive.deploy.attic.enable requires the cache's swarm-secret-store client identity: set both services.hyperhive.deploy.bao.atticClientCertFile services.hyperhive.deploy.bao.atticClientKeyFile The cache creates its signing key and tokens in the store with it. Both default to the pair glue-bao-tls.nix mints when services.hyperhive.deploy.bao.enable is true (the host that runs the store). Otherwise copy attic.pem and attic-key.pem from the store host's /var/lib/swarm-bao-pki/ and name them in those two options. ''; } { assertion = atticCfg.domain != null && atticCfg.cache != null; message = '' services.hyperhive.deploy.attic.enable needs services.hyperhive.swarm.attic.domain and services.hyperhive.swarm.attic.cache: it serves the cache at that name and creates it by that name. They default when unset, so one of them is set to null. ''; } ]; services.hyperhive.gateway.localNames = [ atticCfg.domain ]; # This host serves the vhost, and the host's own resolver finds the name. services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; # `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks # its own tokens, and a nix client cannot follow a login redirect. A store # path is uploaded as one request that can run for minutes, so the body is # neither size-limited nor buffered. services.hyperhive.gateway.vhosts."${atticCfg.domain}" = { forceSSL = true; locations."/" = { proxyPass = "http://127.0.0.1:${toString cfg.port}"; extraConfig = '' client_max_body_size 0; proxy_request_buffering off; proxy_read_timeout 1h; proxy_send_timeout 1h; ''; }; }; # The key's directory is the container's read-only bind source, so it has # to exist before the container starts even when the store does not answer. # The delivery unit creates it too, which covers a switch. systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ]; # The signing key, created in the store if it holds none and written where # the container reads it. Before the container and wanted by it: a store # that comes up later fails this run, atticd restarts until the file is # there, and the retry that lands it restarts atticd. systemd.services.swarm-bao-attic-secret = { description = "create the swarm cache's signing key in the secret store if absent and hand it to the container"; # Ordered after the store where it runs here: # ./glue-bao-readers-store-order.nix and ./glue-bao-readers-policy-order.nix. before = [ "container@${machine}.service" ]; wantedBy = [ "multi-user.target" "container@${machine}.service" ]; path = [ baoDeploy.package pkgs.coreutils pkgs.openssl pkgs.systemd ]; inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = "swarm-bao-attic-secret"; # Generating a 4096-bit key takes seconds. TimeoutStartSec = 120; }; environment = storeEnvironment; script = '' set -euo pipefail ${atomicWriteSecret} ${refreshConsumer} install -d -m 0700 ${lib.escapeShellArg secretDir} ${atticInit.prelude} ${atticInit.ensureJwt} ''; }; # The cache and its two tokens, once atticd answers. The push token is the # path the swarm builder reads; the pull token is for the hives. systemd.services.swarm-bao-attic-tokens = { description = "create the swarm cache, and its push and pull tokens in the secret store, if absent"; after = [ "container@${machine}.service" "swarm-bao-attic-secret.service" ]; wants = [ "container@${machine}.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoDeploy.package cfg.package cfg.clientPackage pkgs.coreutils pkgs.curl ]; inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = "swarm-bao-attic-tokens"; # Up to a minute waiting for atticd. TimeoutStartSec = 180; }; environment = storeEnvironment; script = '' set -euo pipefail ${atticInit.prelude} ${atticInit.provision} ''; }; containers.${machine} = { autoStart = true; ephemeral = false; privateNetwork = false; bindMounts.${secretDir} = { hostPath = secretDir; isReadOnly = true; }; config = { ... }: { imports = [ ../container-modules/swarm-container.nix ]; services.hyperhive.swarmContainer.privateNetwork = false; systemd.services.${unit} = { description = "attic binary cache server"; wantedBy = [ "multi-user.target" ]; serviceConfig = { ExecStart = "${cfg.package}/bin/atticd -f ${configFile} --mode monolithic"; # The signing key, in an environment file the host unit writes. EnvironmentFile = envFile; StateDirectory = stateName; DynamicUser = true; Restart = "on-failure"; RestartSec = 10; SyslogIdentifier = unit; UMask = "0077"; CapabilityBoundingSet = [ "" ]; NoNewPrivileges = true; PrivateDevices = true; PrivateTmp = true; ProtectSystem = "strict"; ProtectHome = true; ProtectKernelTunables = true; ProtectKernelModules = true; ProtectControlGroups = true; RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; LockPersonality = true; SystemCallArchitectures = "native"; }; }; }; }; }; }