332 lines
11 KiB
Nix
332 lines
11 KiB
Nix
# The swarm's binary cache: an attic server in a `swarm-attic` container on the
|
||
# host that sets `deploy.attic.enable`, which `deploy.allSwarmServices` defaults
|
||
# on where the host has the cache's store identity. The swarm builder pushes
|
||
# green builds to it (docs/swarm/builder.md).
|
||
#
|
||
# Two host units run under the cache's own store leaf, the shape
|
||
# ./swarm-grafana.nix gives its admin password. One creates the key atticd signs
|
||
# tokens with, if the store holds none, and hands it to the container. The other
|
||
# creates the cache and the push and pull tokens, each only while absent
|
||
# (./lib/attic-init.nix). Neither needs a manual step.
|
||
#
|
||
# The container shares the host's network namespace, as ./swarm-victorialogs.nix
|
||
# does, and atticd listens on loopback only: the gateway's vhost is the way in,
|
||
# and attic's own tokens are the authentication. Storage is local disk under the
|
||
# container's state.
|
||
#
|
||
# The two `swarm.attic` options every host reads (./swarm-attic-options.nix) get
|
||
# their values from here, at `mkDefault`: naming them yourself wins, and with
|
||
# the switch off nothing is set.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
cfg = hyperhiveCfg.deploy.attic;
|
||
atticCfg = hyperhiveCfg.swarm.attic;
|
||
baoCfg = hyperhiveCfg.swarm.bao;
|
||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||
|
||
machine = "swarm-attic";
|
||
unit = "swarm-attic";
|
||
|
||
# Where the container keeps its database and the cache's files.
|
||
stateName = "swarm-attic";
|
||
stateDir = "/var/lib/${stateName}";
|
||
|
||
# The host directory that carries the signing key into the container. The
|
||
# key's file is the only thing in it.
|
||
secretDir = "/run/swarm-attic-secrets";
|
||
envFile = "${secretDir}/server.env";
|
||
|
||
# Where the host units reach atticd: the shared loopback, not the gateway.
|
||
endpoint = "http://127.0.0.1:${toString cfg.port}";
|
||
|
||
configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" {
|
||
listen = "127.0.0.1:${toString cfg.port}";
|
||
# What clients are told to use for the cache's API.
|
||
api-endpoint = "https://${atticCfg.domain}/";
|
||
database.url = "sqlite://${stateDir}/server.db?mode=rwc";
|
||
storage = {
|
||
type = "local";
|
||
path = "${stateDir}/storage";
|
||
};
|
||
# attic's defaults, which its config requires spelled out.
|
||
chunking = {
|
||
nar-size-threshold = 65536;
|
||
min-size = 16384;
|
||
avg-size = 65536;
|
||
max-size = 262144;
|
||
};
|
||
garbage-collection = {
|
||
interval = "12 hours";
|
||
default-retention-period = cfg.retentionPeriod;
|
||
};
|
||
};
|
||
|
||
atticInit = import ./lib/attic-init.nix { inherit lib; } {
|
||
jwtPath = "secret/swarm/services/attic/jwt-secret";
|
||
pullPath = "secret/swarm/services/attic/pull-token";
|
||
pushPath = "secret/swarm/services/attic/push-token";
|
||
cache = atticCfg.cache;
|
||
inherit
|
||
endpoint
|
||
configFile
|
||
envFile
|
||
machine
|
||
unit
|
||
;
|
||
};
|
||
|
||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||
storeRetry = import ./lib/store-retry.nix { };
|
||
|
||
haveClientIdentity = baoDeploy.atticClientCertFile != null && baoDeploy.atticClientKeyFile != null;
|
||
|
||
# Both host units log in as the cache's principal.
|
||
storeEnvironment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = baoDeploy.atticClientCertFile;
|
||
BAO_CLIENT_KEY = baoDeploy.atticClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
in
|
||
{
|
||
options.services.hyperhive.deploy.attic = {
|
||
package = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.attic-server;
|
||
defaultText = lib.literalExpression "pkgs.attic-server";
|
||
description = "The attic server package: `atticd`, and `atticadm` for the tokens.";
|
||
};
|
||
|
||
clientPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.attic-client;
|
||
defaultText = lib.literalExpression "pkgs.attic-client";
|
||
description = "The attic client package, which creates the cache.";
|
||
};
|
||
|
||
port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 8910;
|
||
description = ''
|
||
Loopback port atticd listens on, which the gateway proxies to.
|
||
|
||
⚠️ Every swarm container shares the host's network namespace, so this
|
||
is a swarm-wide claim: two modules picking the same number collide at
|
||
runtime with no bind error. No repo-wide check catches it.
|
||
'';
|
||
};
|
||
|
||
retentionPeriod = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "14 days";
|
||
example = "30 days";
|
||
description = ''
|
||
How long a store path stays in the cache after it was added or last
|
||
fetched. A garbage collection every twelve hours removes what is older;
|
||
a build the cache no longer holds is rebuilt on the next miss.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf cfg.enable {
|
||
services.hyperhive.swarm.attic = {
|
||
domain = lib.mkDefault atticCfg.ownDomain;
|
||
cache = lib.mkDefault "swarm";
|
||
};
|
||
|
||
assertions = [
|
||
{
|
||
assertion = haveClientIdentity;
|
||
message = ''
|
||
services.hyperhive.deploy.attic.enable requires the cache's
|
||
swarm-secret-store client identity: set both
|
||
|
||
services.hyperhive.deploy.bao.atticClientCertFile
|
||
services.hyperhive.deploy.bao.atticClientKeyFile
|
||
|
||
The cache creates its signing key and tokens in the store with it.
|
||
|
||
Both default to the pair glue-bao-tls.nix mints when
|
||
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
||
store). Otherwise copy attic.pem and attic-key.pem from the store
|
||
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
||
'';
|
||
}
|
||
{
|
||
assertion = atticCfg.domain != null && atticCfg.cache != null;
|
||
message = ''
|
||
services.hyperhive.deploy.attic.enable needs
|
||
services.hyperhive.swarm.attic.domain and
|
||
services.hyperhive.swarm.attic.cache: it serves the cache at that name
|
||
and creates it by that name. They default when unset, so one of them
|
||
is set to null.
|
||
'';
|
||
}
|
||
];
|
||
|
||
services.hyperhive.gateway.localNames = [ atticCfg.domain ];
|
||
|
||
# This host serves the vhost, and the host's own resolver finds the name.
|
||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||
|
||
# `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks
|
||
# its own tokens, and a nix client cannot follow a login redirect. A store
|
||
# path is uploaded as one request that can run for minutes, so the body is
|
||
# neither size-limited nor buffered.
|
||
services.hyperhive.gateway.vhosts."${atticCfg.domain}" = {
|
||
forceSSL = true;
|
||
locations."/" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||
extraConfig = ''
|
||
client_max_body_size 0;
|
||
proxy_request_buffering off;
|
||
proxy_read_timeout 1h;
|
||
proxy_send_timeout 1h;
|
||
'';
|
||
};
|
||
};
|
||
|
||
# The key's directory is the container's read-only bind source, so it has
|
||
# to exist before the container starts even when the store does not answer.
|
||
# The delivery unit creates it too, which covers a switch.
|
||
systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ];
|
||
|
||
# The signing key, created in the store if it holds none and written where
|
||
# the container reads it. Before the container and wanted by it: a store
|
||
# that comes up later fails this run, atticd restarts until the file is
|
||
# there, and the retry that lands it restarts atticd.
|
||
systemd.services.swarm-bao-attic-secret = {
|
||
description = "create the swarm cache's signing key in the secret store if absent and hand it to the container";
|
||
# Ordered after the store where it runs here:
|
||
# ./glue-bao-readers-store-order.nix and ./glue-bao-readers-policy-order.nix.
|
||
before = [ "container@${machine}.service" ];
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"container@${machine}.service"
|
||
];
|
||
path = [
|
||
baoDeploy.package
|
||
pkgs.coreutils
|
||
pkgs.openssl
|
||
pkgs.systemd
|
||
];
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-bao-attic-secret";
|
||
# Generating a 4096-bit key takes seconds.
|
||
TimeoutStartSec = 120;
|
||
};
|
||
environment = storeEnvironment;
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
${atomicWriteSecret}
|
||
${refreshConsumer}
|
||
|
||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||
|
||
${atticInit.prelude}
|
||
${atticInit.ensureJwt}
|
||
'';
|
||
};
|
||
|
||
# The cache and its two tokens, once atticd answers. The push token is the
|
||
# path the swarm builder reads; the pull token is for the hives.
|
||
systemd.services.swarm-bao-attic-tokens = {
|
||
description = "create the swarm cache, and its push and pull tokens in the secret store, if absent";
|
||
after = [
|
||
"container@${machine}.service"
|
||
"swarm-bao-attic-secret.service"
|
||
];
|
||
wants = [ "container@${machine}.service" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
path = [
|
||
baoDeploy.package
|
||
cfg.package
|
||
cfg.clientPackage
|
||
pkgs.coreutils
|
||
pkgs.curl
|
||
];
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-bao-attic-tokens";
|
||
# Up to a minute waiting for atticd.
|
||
TimeoutStartSec = 180;
|
||
};
|
||
environment = storeEnvironment;
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
${atticInit.prelude}
|
||
${atticInit.provision}
|
||
'';
|
||
};
|
||
|
||
containers.${machine} = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
privateNetwork = false;
|
||
bindMounts.${secretDir} = {
|
||
hostPath = secretDir;
|
||
isReadOnly = true;
|
||
};
|
||
|
||
config =
|
||
{ ... }:
|
||
{
|
||
imports = [ ../container-modules/swarm-container.nix ];
|
||
|
||
services.hyperhive.swarmContainer.privateNetwork = false;
|
||
|
||
systemd.services.${unit} = {
|
||
description = "attic binary cache server";
|
||
wantedBy = [ "multi-user.target" ];
|
||
serviceConfig = {
|
||
ExecStart = "${cfg.package}/bin/atticd -f ${configFile} --mode monolithic";
|
||
# The signing key, in an environment file the host unit writes.
|
||
EnvironmentFile = envFile;
|
||
StateDirectory = stateName;
|
||
DynamicUser = true;
|
||
Restart = "on-failure";
|
||
RestartSec = 10;
|
||
SyslogIdentifier = unit;
|
||
UMask = "0077";
|
||
CapabilityBoundingSet = [ "" ];
|
||
NoNewPrivileges = true;
|
||
PrivateDevices = true;
|
||
PrivateTmp = true;
|
||
ProtectSystem = "strict";
|
||
ProtectHome = true;
|
||
ProtectKernelTunables = true;
|
||
ProtectKernelModules = true;
|
||
ProtectControlGroups = true;
|
||
RestrictAddressFamilies = [
|
||
"AF_INET"
|
||
"AF_INET6"
|
||
"AF_UNIX"
|
||
];
|
||
RestrictNamespaces = true;
|
||
RestrictRealtime = true;
|
||
RestrictSUIDSGID = true;
|
||
LockPersonality = true;
|
||
SystemCallArchitectures = "native";
|
||
};
|
||
};
|
||
};
|
||
};
|
||
};
|
||
}
|