run attic as a swarm service with deploy.attic.enable
This commit is contained in:
atlas
parent
f59ed01353
commit
458a046b8d
1 changed files with 1242 additions and 66 deletions
332
nix/host-modules/swarm-attic.nix
Normal file
332
nix/host-modules/swarm-attic.nix
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
# The swarm's binary cache: an attic server in a `swarm-attic` container on the
|
||||
# host that sets `deploy.attic.enable`, which `deploy.allSwarmServices` defaults
|
||||
# on where the host has the cache's store identity. The swarm builder pushes
|
||||
# green builds to it (docs/swarm/builder.md).
|
||||
#
|
||||
# Two host units run under the cache's own store leaf, the shape
|
||||
# ./swarm-grafana.nix gives its admin password. One creates the key atticd signs
|
||||
# tokens with, if the store holds none, and hands it to the container. The other
|
||||
# creates the cache and the push and pull tokens, each only while absent
|
||||
# (./lib/attic-init.nix). Neither needs a manual step.
|
||||
#
|
||||
# The container shares the host's network namespace, as ./swarm-victorialogs.nix
|
||||
# does, and atticd listens on loopback only: the gateway's vhost is the way in,
|
||||
# and attic's own tokens are the authentication. Storage is local disk under the
|
||||
# container's state.
|
||||
#
|
||||
# The two `swarm.attic` options every host reads (./swarm-attic-options.nix) get
|
||||
# their values from here, at `mkDefault`: naming them yourself wins, and with
|
||||
# the switch off nothing is set.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
cfg = hyperhiveCfg.deploy.attic;
|
||||
atticCfg = hyperhiveCfg.swarm.attic;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||||
|
||||
machine = "swarm-attic";
|
||||
unit = "swarm-attic";
|
||||
|
||||
# Where the container keeps its database and the cache's files.
|
||||
stateName = "swarm-attic";
|
||||
stateDir = "/var/lib/${stateName}";
|
||||
|
||||
# The host directory that carries the signing key into the container. The
|
||||
# key's file is the only thing in it.
|
||||
secretDir = "/run/swarm-attic-secrets";
|
||||
envFile = "${secretDir}/server.env";
|
||||
|
||||
# Where the host units reach atticd: the shared loopback, not the gateway.
|
||||
endpoint = "http://127.0.0.1:${toString cfg.port}";
|
||||
|
||||
configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" {
|
||||
listen = "127.0.0.1:${toString cfg.port}";
|
||||
# What clients are told to use for the cache's API.
|
||||
api-endpoint = "https://${atticCfg.domain}/";
|
||||
database.url = "sqlite://${stateDir}/server.db?mode=rwc";
|
||||
storage = {
|
||||
type = "local";
|
||||
path = "${stateDir}/storage";
|
||||
};
|
||||
# attic's defaults, which its config requires spelled out.
|
||||
chunking = {
|
||||
nar-size-threshold = 65536;
|
||||
min-size = 16384;
|
||||
avg-size = 65536;
|
||||
max-size = 262144;
|
||||
};
|
||||
garbage-collection = {
|
||||
interval = "12 hours";
|
||||
default-retention-period = cfg.retentionPeriod;
|
||||
};
|
||||
};
|
||||
|
||||
atticInit = import ./lib/attic-init.nix { inherit lib; } {
|
||||
jwtPath = "secret/swarm/services/attic/jwt-secret";
|
||||
pullPath = "secret/swarm/services/attic/pull-token";
|
||||
pushPath = "secret/swarm/services/attic/push-token";
|
||||
cache = atticCfg.cache;
|
||||
inherit
|
||||
endpoint
|
||||
configFile
|
||||
envFile
|
||||
machine
|
||||
unit
|
||||
;
|
||||
};
|
||||
|
||||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||||
storeRetry = import ./lib/store-retry.nix { };
|
||||
|
||||
haveClientIdentity = baoDeploy.atticClientCertFile != null && baoDeploy.atticClientKeyFile != null;
|
||||
|
||||
# Both host units log in as the cache's principal.
|
||||
storeEnvironment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = baoDeploy.atticClientCertFile;
|
||||
BAO_CLIENT_KEY = baoDeploy.atticClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.deploy.attic = {
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = pkgs.attic-server;
|
||||
defaultText = lib.literalExpression "pkgs.attic-server";
|
||||
description = "The attic server package: `atticd`, and `atticadm` for the tokens.";
|
||||
};
|
||||
|
||||
clientPackage = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = pkgs.attic-client;
|
||||
defaultText = lib.literalExpression "pkgs.attic-client";
|
||||
description = "The attic client package, which creates the cache.";
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8910;
|
||||
description = ''
|
||||
Loopback port atticd listens on, which the gateway proxies to.
|
||||
|
||||
⚠️ Every swarm container shares the host's network namespace, so this
|
||||
is a swarm-wide claim: two modules picking the same number collide at
|
||||
runtime with no bind error. No repo-wide check catches it.
|
||||
'';
|
||||
};
|
||||
|
||||
retentionPeriod = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "14 days";
|
||||
example = "30 days";
|
||||
description = ''
|
||||
How long a store path stays in the cache after it was added or last
|
||||
fetched. A garbage collection every twelve hours removes what is older;
|
||||
a build the cache no longer holds is rebuilt on the next miss.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.hyperhive.swarm.attic = {
|
||||
domain = lib.mkDefault atticCfg.ownDomain;
|
||||
cache = lib.mkDefault "swarm";
|
||||
};
|
||||
|
||||
assertions = [
|
||||
{
|
||||
assertion = haveClientIdentity;
|
||||
message = ''
|
||||
services.hyperhive.deploy.attic.enable requires the cache's
|
||||
swarm-secret-store client identity: set both
|
||||
|
||||
services.hyperhive.deploy.bao.atticClientCertFile
|
||||
services.hyperhive.deploy.bao.atticClientKeyFile
|
||||
|
||||
The cache creates its signing key and tokens in the store with it.
|
||||
|
||||
Both default to the pair glue-bao-tls.nix mints when
|
||||
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
||||
store). Otherwise copy attic.pem and attic-key.pem from the store
|
||||
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = atticCfg.domain != null && atticCfg.cache != null;
|
||||
message = ''
|
||||
services.hyperhive.deploy.attic.enable needs
|
||||
services.hyperhive.swarm.attic.domain and
|
||||
services.hyperhive.swarm.attic.cache: it serves the cache at that name
|
||||
and creates it by that name. They default when unset, so one of them
|
||||
is set to null.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
services.hyperhive.gateway.localNames = [ atticCfg.domain ];
|
||||
|
||||
# This host serves the vhost, and the host's own resolver finds the name.
|
||||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||||
|
||||
# `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks
|
||||
# its own tokens, and a nix client cannot follow a login redirect. A store
|
||||
# path is uploaded as one request that can run for minutes, so the body is
|
||||
# neither size-limited nor buffered.
|
||||
services.hyperhive.gateway.vhosts."${atticCfg.domain}" = {
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||||
extraConfig = ''
|
||||
client_max_body_size 0;
|
||||
proxy_request_buffering off;
|
||||
proxy_read_timeout 1h;
|
||||
proxy_send_timeout 1h;
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# The key's directory is the container's read-only bind source, so it has
|
||||
# to exist before the container starts even when the store does not answer.
|
||||
# The delivery unit creates it too, which covers a switch.
|
||||
systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ];
|
||||
|
||||
# The signing key, created in the store if it holds none and written where
|
||||
# the container reads it. Before the container and wanted by it: a store
|
||||
# that comes up later fails this run, atticd restarts until the file is
|
||||
# there, and the retry that lands it restarts atticd.
|
||||
systemd.services.swarm-bao-attic-secret = {
|
||||
description = "create the swarm cache's signing key in the secret store if absent and hand it to the container";
|
||||
# Ordered after the store where it runs here:
|
||||
# ./glue-bao-readers-store-order.nix and ./glue-bao-readers-policy-order.nix.
|
||||
before = [ "container@${machine}.service" ];
|
||||
wantedBy = [
|
||||
"multi-user.target"
|
||||
"container@${machine}.service"
|
||||
];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
pkgs.openssl
|
||||
pkgs.systemd
|
||||
];
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
SyslogIdentifier = "swarm-bao-attic-secret";
|
||||
# Generating a 4096-bit key takes seconds.
|
||||
TimeoutStartSec = 120;
|
||||
};
|
||||
environment = storeEnvironment;
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${atomicWriteSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||||
|
||||
${atticInit.prelude}
|
||||
${atticInit.ensureJwt}
|
||||
'';
|
||||
};
|
||||
|
||||
# The cache and its two tokens, once atticd answers. The push token is the
|
||||
# path the swarm builder reads; the pull token is for the hives.
|
||||
systemd.services.swarm-bao-attic-tokens = {
|
||||
description = "create the swarm cache, and its push and pull tokens in the secret store, if absent";
|
||||
after = [
|
||||
"container@${machine}.service"
|
||||
"swarm-bao-attic-secret.service"
|
||||
];
|
||||
wants = [ "container@${machine}.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
cfg.package
|
||||
cfg.clientPackage
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
];
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
SyslogIdentifier = "swarm-bao-attic-tokens";
|
||||
# Up to a minute waiting for atticd.
|
||||
TimeoutStartSec = 180;
|
||||
};
|
||||
environment = storeEnvironment;
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${atticInit.prelude}
|
||||
${atticInit.provision}
|
||||
'';
|
||||
};
|
||||
|
||||
containers.${machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
privateNetwork = false;
|
||||
bindMounts.${secretDir} = {
|
||||
hostPath = secretDir;
|
||||
isReadOnly = true;
|
||||
};
|
||||
|
||||
config =
|
||||
{ ... }:
|
||||
{
|
||||
imports = [ ../container-modules/swarm-container.nix ];
|
||||
|
||||
services.hyperhive.swarmContainer.privateNetwork = false;
|
||||
|
||||
systemd.services.${unit} = {
|
||||
description = "attic binary cache server";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${cfg.package}/bin/atticd -f ${configFile} --mode monolithic";
|
||||
# The signing key, in an environment file the host unit writes.
|
||||
EnvironmentFile = envFile;
|
||||
StateDirectory = stateName;
|
||||
DynamicUser = true;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 10;
|
||||
SyslogIdentifier = unit;
|
||||
UMask = "0077";
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectControlGroups = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
"AF_UNIX"
|
||||
];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue