swarm-archive: external s3 backend (options, prune path, docs, module-eval cases, vm fake s3)
- swarm-archive.nix, lib/archive-server.nix: deploy.archive.backend and deploy.archive.s3.* options; the s3 server and the prune read the key from the store into rclone's environment; the prune lists and addresses repositories through rclone instead of a local directory - swarm-bao.nix: the archive reads swarm/services/archive/s3/* - docs: archive.md "External S3" section, two credentials-reference rows - module-eval: builder.nix cases for local and s3, the missing-setting assertion; bao-grants.nix gains the s3 read grant - vm test: second node with a fake rclone serve s3 whose key is generated at test time
This commit is contained in:
atlas
mara
parent
ebbf4fa37b
commit
86c5dac89d
1 changed files with 605 additions and 156 deletions
|
|
@ -46,6 +46,8 @@ let
|
|||
deploy.builder.enable = true;
|
||||
swarm.attic.domain = "cache.example.org";
|
||||
swarm.attic.cache = null;
|
||||
deploy.archive.enable = true;
|
||||
deploy.archive.backend = "s3";
|
||||
};
|
||||
refusedHalfCache =
|
||||
m:
|
||||
|
|
@ -109,6 +111,13 @@ let
|
|||
deploy.attic.enable = lib.mkForce false;
|
||||
swarm.attic.domain = "cache.example.org";
|
||||
swarm.attic.cache = "grid";
|
||||
deploy.archive.backend = "s3";
|
||||
deploy.archive.s3 = {
|
||||
endpoint = "https://s3.example.org";
|
||||
region = "eu-central-1";
|
||||
bucket = "bkt";
|
||||
prefix = "archive";
|
||||
};
|
||||
};
|
||||
atticWarnings =
|
||||
m:
|
||||
|
|
@ -600,6 +609,102 @@ let
|
|||
&& !(bare.systemd.services ? swarm-archive)
|
||||
&& !(bare.systemd.services ? swarm-archive-users);
|
||||
}
|
||||
{
|
||||
# The default is the directory backend, and its unit is the one that ran
|
||||
# before the S3 option: the literal command line is the control.
|
||||
name = "the archive's default backend is the local directory and renders the unchanged server unit";
|
||||
ok =
|
||||
let
|
||||
a = atticHere.systemd.services.swarm-archive;
|
||||
prune = atticHere.systemd.services.swarm-archive-prune;
|
||||
cfg = atticHere.services.hyperhive.deploy.archive;
|
||||
in
|
||||
cfg.backend == "local"
|
||||
&&
|
||||
a.serviceConfig.ExecStart
|
||||
== "${cfg.package}/bin/rclone serve restic --addr 127.0.0.1:8911 --append-only --private-repos --htpasswd /run/swarm-archive-auth/htpasswd /var/lib/swarm-archive/repos"
|
||||
&& a.environment.RCLONE_CONFIG == "/dev/null"
|
||||
&& !(lib.any (n: lib.hasPrefix "RCLONE_S3" n) (lib.attrNames a.environment))
|
||||
&& a.serviceConfig.ReadWritePaths == [ "/var/lib/swarm-archive/repos" ]
|
||||
&& !(a.serviceConfig ? LoadCredential)
|
||||
&& prune.serviceConfig.ReadWritePaths == [ "/var/lib/swarm-archive/repos" ]
|
||||
&& lib.hasInfix "/var/lib/swarm-archive/repos" prune.script
|
||||
&& !(lib.hasInfix "rclone" prune.script)
|
||||
&& lib.elem "d /var/lib/swarm-archive/repos 0750 swarm-archive swarm-archive -" atticHere.systemd.tmpfiles.rules;
|
||||
}
|
||||
{
|
||||
# The key is read as the server starts and lives in rclone's environment:
|
||||
# the unit carries the connection settings and the store identity, never
|
||||
# the key, and the prune reaches the bucket through restic's rclone
|
||||
# backend. The local case above is the control for each absence.
|
||||
name = "the archive on an external S3 serves and prunes the bucket, reads its key from the store at start and keeps it out of the unit";
|
||||
ok =
|
||||
let
|
||||
a = atticExternal.systemd.services.swarm-archive;
|
||||
prune = atticExternal.systemd.services.swarm-archive-prune;
|
||||
cfg = atticExternal.services.hyperhive.deploy.archive;
|
||||
server = import ../host-modules/lib/archive-server.nix { inherit lib; } {
|
||||
rclone = pkgs.rclone;
|
||||
restic = pkgs.restic;
|
||||
addr = "127.0.0.1:8911";
|
||||
authFile = "/run/swarm-archive-auth/htpasswd";
|
||||
authGroup = "swarm-archive";
|
||||
retention = {
|
||||
keepWithin = "3d";
|
||||
keepLast = 3;
|
||||
};
|
||||
s3 = cfg.s3;
|
||||
};
|
||||
unitText = builtins.toJSON [
|
||||
a.environment
|
||||
a.serviceConfig
|
||||
prune.environment
|
||||
prune.serviceConfig
|
||||
];
|
||||
in
|
||||
cfg.backend == "s3"
|
||||
&& lib.hasInfix "swarm-archive-serve" a.serviceConfig.ExecStart
|
||||
&& a.environment.RCLONE_S3_ENDPOINT == "https://s3.example.org"
|
||||
&& a.environment.RCLONE_S3_REGION == "eu-central-1"
|
||||
&& a.environment.RCLONE_S3_PROVIDER == "Other"
|
||||
&& a.environment.RCLONE_S3_NO_CHECK_BUCKET == "true"
|
||||
&& a.environment.RCLONE_S3_ENV_AUTH == "false"
|
||||
&& a.environment.RCLONE_CONFIG == "/dev/null"
|
||||
&& a.environment.BAO_CLIENT_CERT == "%d/bao-cert"
|
||||
&& lib.elem "bao-cert:/var/lib/swarm-bao-pki/archive.pem" a.serviceConfig.LoadCredential
|
||||
&& !(a.serviceConfig ? ReadWritePaths)
|
||||
&& !(prune.serviceConfig ? ReadWritePaths)
|
||||
&& prune.environment.RCLONE_S3_ENDPOINT == "https://s3.example.org"
|
||||
&& !(lib.elem "d /var/lib/swarm-archive/repos 0750 swarm-archive swarm-archive -" atticExternal.systemd.tmpfiles.rules)
|
||||
&& lib.hasInfix "RCLONE_S3_ENDPOINT" unitText
|
||||
&& !(lib.hasInfix "ACCESS_KEY" unitText)
|
||||
&& lib.hasInfix "--append-only" server.serverCommand
|
||||
&& lib.hasInfix "--private-repos" server.serverCommand
|
||||
&& lib.hasSuffix " :s3:bkt/archive" server.serverCommand
|
||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/archive/s3/access-key-id" server.s3ServerScript
|
||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/archive/s3/secret-access-key" server.s3ServerScript
|
||||
&& lib.hasInfix "\nexec /nix/store/" server.s3ServerScript
|
||||
&& lib.hasInfix "/bin/rclone serve restic" server.s3ServerScript
|
||||
&& lib.hasInfix "rclone lsf --dirs-only :s3:bkt/archive/" server.pruneScript
|
||||
&& lib.hasInfix "repo=rclone::s3:bkt/archive/\"$agent\"" server.pruneScript
|
||||
&& lib.hasInfix "RCLONE_S3_SECRET_ACCESS_KEY" server.pruneScript
|
||||
&& !(lib.hasInfix "/var/lib/swarm-archive" server.pruneScript);
|
||||
}
|
||||
{
|
||||
# `halfCache` hosts the archive on the S3 backend with neither endpoint nor
|
||||
# bucket; `atticExternal` names both and is the control that the
|
||||
# assertion fires for the missing settings alone.
|
||||
name = "the archive on S3 without an endpoint and a bucket fails evaluation, and with both it does not";
|
||||
ok =
|
||||
let
|
||||
refused =
|
||||
m:
|
||||
lib.any (
|
||||
a: !a.assertion && lib.hasInfix "deploy.archive.backend = \"s3\" requires" a.message
|
||||
) m.assertions;
|
||||
in
|
||||
refused halfCache && !(refused atticExternal);
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "builder" cases
|
||||
|
|
|
|||
Loading…
Reference in a new issue