push green meta builds to the attic cache and record cached

nix/host-modules/default.nix

@@ -62,6 +62,7 @@
     ./glue-swarm-otel-oidc-client.nix
     ./glue-victorialogs-otel-oidc-client.nix
     ./glue-victoriametrics-otel-oidc-client.nix
+    ./swarm-attic-options.nix
     ./swarm-authelia-service.nix
     ./swarm-authelia.nix
     ./swarm-bao-service.nix

nix/host-modules/swarm-attic-options.nix

@@ -0,0 +1,37 @@
+# The binary cache the swarm pushes agent closures to, as every host sees it:
+# which attic server, and which cache on it. The swarm runs no attic of its
+# own yet; the instance is the operator's, outside the swarm.
+{ lib, ... }:
+{
+  options.services.hyperhive.swarm.attic = {
+    domain = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "cache.example.org";
+      description = ''
+        Name of the attic server, with no scheme: the swarm builder pushes to
+        `https://<domain>`, and trusts the certificate the system roots do, so
+        a Let's Encrypt certificate works and none is pinned. Set together with
+        {option}`services.hyperhive.swarm.attic.cache`.
+
+        Null pushes nothing: the swarm builder does not push, and the closure
+        record says it is not cached.
+
+        The push token is not an option. The operator writes it to
+        `secret/swarm/services/attic/push-token` in the swarm secret store
+        (field `value`), and the builder reads it on every push.
+      '';
+    };
+
+    cache = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "grid";
+      description = ''
+        Name of the cache on {option}`services.hyperhive.swarm.attic.domain`
+        that the swarm builder pushes closures to. The push token must be
+        allowed to push to it.
+      '';
+    };
+  };
+}

nix/host-modules/swarm-bao.nix

@@ -692,6 +692,12 @@ let
   # to fetch config repos.
   builderForgeTokenLeaf = "swarm/services/builder/forge/token";
 
+  # The token the swarm builder pushes closures to the binary cache with, the nix
+  # half of `swarm_secret_client::builder::attic_push_token_path`. The operator
+  # writes it by hand; the builder reads it on every push, and nothing else in
+  # the swarm reaches it.
+  atticPushTokenLeaf = "swarm/services/attic/push-token";
+
   # The CI run poller's forge token, the nix half of
   # `swarm_secret_client::ci_metrics::forge_token_path`. The controller mints it
   # and reads it back before deciding whether to re-mint; the poller reads it
@@ -1146,14 +1152,16 @@ let
     }
   ];
 
-  # The swarm builder: its forge token, read once per build, and its own queue
-  # client secret, read at start. Both by the daemon inside its container.
+  # The swarm builder: its forge token, read once per build, the cache push token,
+  # read once per push, and its own queue client secret, read at start. All by the
+  # daemon inside its container.
   builderReaders = [
     {
       name = "swarm-builder";
       cn = baoDeploy.builderCommonName;
       policyText =
         readStanza "${credentialMountPath}/data/${builderForgeTokenLeaf}"
+        + readStanza "${credentialMountPath}/data/${atticPushTokenLeaf}"
         + readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.builder.clientId}/oidc/client";
     }
   ];
@@ -2081,8 +2089,8 @@ in
       description = ''
         Subject the store's `swarm-builder` cert-auth role accepts: the
         identity the swarm builder presents from inside its container. Its
-        grant is two paths, read only: `swarm/services/builder/forge/token`
-        and its own queue client secret.
+        grant is three paths, read only: `swarm/services/builder/forge/token`,
+        `swarm/services/attic/push-token` and its own queue client secret.
 
         ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
       '';

nix/host-modules/swarm-builder.nix

@@ -17,8 +17,8 @@
 #
 # The container holds one credential, its own store leaf, which nspawn hands in
 # as a system credential with no copy on disk. The daemon reads everything else
-# from the store with it: its queue client secret at start and its forge token
-# on every build.
+# from the store with it: its queue client secret at start, its forge token
+# on every build and, where a cache is set, its cache push token on every push.
 {
   pkgs,
   lib,
@@ -33,6 +33,7 @@ let
   natsCfg = hyperhiveCfg.swarm.nats;
   autheliaCfg = hyperhiveCfg.swarm.authelia;
   forgeCfg = hyperhiveCfg.swarm.forge;
+  atticCfg = hyperhiveCfg.swarm.attic;
   gatewayCfg = hyperhiveCfg.gateway;
   networkCfg = hyperhiveCfg.network;
   tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
@@ -89,6 +90,9 @@ let
   hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
   nixpkgsFlake = hyperhiveCfg.c0re.nixpkgsFlake;
 
+  # Either half alone reaches the daemon too, which refuses to start on it.
+  pushesToCache = atticCfg.domain != null || atticCfg.cache != null;
+
   privateNetwork = true;
 in
 {
@@ -230,8 +234,9 @@ in
             wantedBy = [ "multi-user.target" ];
             wants = [ "network-online.target" ];
             after = [ "network-online.target" ];
-            # nix fetches the config repo with `git`.
-            path = [ pkgs.git ];
+            # nix fetches the config repo with `git`, and a green build is pushed
+            # with `attic` where a cache is set.
+            path = [ pkgs.git ] ++ lib.optional pushesToCache pkgs.attic-client;
             environment = {
               BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
               BAO_CLIENT_CERT = credential "builder-bao-cert";
@@ -252,6 +257,12 @@ in
               SWARM_BUILDER_FORGE_URL = "https://${forgeCfg.domain}";
               SWARM_BUILDER_CONFIG_ORG = "agent-configs";
             }
+            // lib.optionalAttrs (atticCfg.domain != null) {
+              SWARM_BUILDER_ATTIC_DOMAIN = atticCfg.domain;
+            }
+            // lib.optionalAttrs (atticCfg.cache != null) {
+              SWARM_BUILDER_ATTIC_CACHE = atticCfg.cache;
+            }
             // lib.optionalAttrs (storeFiles ? builder-bao-server-ca) {
               BAO_CACERT = credential "builder-bao-server-ca";
             }
@@ -264,6 +275,10 @@ in
               ExecStart = "${cfg.package}/bin/swarm-builder";
               DynamicUser = true;
               CacheDirectory = unit;
+              # /run/swarm-builder, mode 0700 and owned by the dynamic user: the
+              # push writes its token-bearing `attic` config under it.
+              RuntimeDirectory = unit;
+              RuntimeDirectoryMode = "0700";
               # Bare ids: the system credentials nspawn handed in.
               LoadCredential = lib.attrNames storeFiles;
               Restart = "on-failure";

nix/module-eval/bao-grants.nix

@@ -1743,17 +1743,18 @@ let
         ];
     }
     {
-      # The builder reads its forge token and its own queue client secret,
-      # each read-only and spelled to the leaf: `services/*` would reach every
-      # swarm service's secret.
-      name = "the swarm builder's grant is its forge token and its queue client secret, and nothing else";
+      # The builder reads its forge token, the cache push token and its own queue
+      # client secret, each read-only and spelled to the leaf: `services/*` would
+      # reach every swarm service's secret.
+      name = "the swarm builder's grant is its forge token, the cache push token and its queue client secret, and nothing else";
       ok =
         let
           s = baoGrantHere.systemd.services.swarm-bao-builder-policy.script;
         in
         lib.hasInfix "path \"secret/data/swarm/services/builder/forge/token\" {\n  capabilities = [\"read\"]\n}" s
+        && lib.hasInfix "path \"secret/data/swarm/services/attic/push-token\" {\n  capabilities = [\"read\"]\n}" s
         && lib.hasInfix "path \"secret/data/swarm/services/swarm-builder/oidc/client\" {\n  capabilities = [\"read\"]\n}" s
-        && lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 2
+        && lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 3
         && !(lib.hasInfix "secret/data/swarm/services/*" s)
         && lib.hasInfix "auth/cert/certs/swarm-builder" s
         && lib.hasInfix "allowed_common_names=swarm-builder" s

nix/module-eval/builder.nix

@@ -29,6 +29,8 @@ let
   withBuilder = hive {
     deploy.bao.enable = true;
     deploy.builder.enable = true;
+    swarm.attic.domain = "cache.example.org";
+    swarm.attic.cache = "grid";
   };
 
   servicesHere = shared.swarmServicesWithController;
@@ -48,6 +50,8 @@ let
   inner = container.config;
   svc = inner.systemd.services.swarm-builder;
   env = svc.environment;
+  hasOnPath = name: s: lib.any (p: lib.getName p == name) s.path;
+  withoutCache = servicesHere.containers.swarm-builder.config.systemd.services.swarm-builder;
 
   # The check flake on the inputs the builder's unit names, called as
   # `nix build` calls it. `hyperhive` is entered through the filtered source's
@@ -116,6 +120,25 @@ let
         && svc.serviceConfig.DynamicUser
         && !(lib.any (lib.hasSuffix "TOKEN") (lib.attrNames env));
     }
+    {
+      # The closure is pushed with `attic` from the unit, whose config file holds
+      # the push token: it goes in a runtime directory only the dynamic user can
+      # enter, and the token is in no variable. The host without a cache is the
+      # control: same unit, none of it, with `git` still on the path to show
+      # the lookup finds what is there.
+      name = "the builder pushes to the configured cache with attic from a private runtime directory, and pushes nowhere without one";
+      ok =
+        env.SWARM_BUILDER_ATTIC_DOMAIN == "cache.example.org"
+        && env.SWARM_BUILDER_ATTIC_CACHE == "grid"
+        && hasOnPath "attic-client" svc
+        && svc.serviceConfig.RuntimeDirectory == "swarm-builder"
+        && svc.serviceConfig.RuntimeDirectoryMode == "0700"
+        && !(lib.any (lib.hasInfix "TOKEN") (lib.attrNames env))
+        && hasOnPath "git" withoutCache
+        && !(hasOnPath "attic-client" withoutCache)
+        && !(withoutCache.environment ? SWARM_BUILDER_ATTIC_DOMAIN)
+        && !(withoutCache.environment ? SWARM_BUILDER_ATTIC_CACHE);
+    }
     {
       # The minting unit goes inactive after each run, so a start that pulls it
       # runs it again and mints a leaf that is missing; an active