run the root check warn-only on hyperhive's own containers, per unit, not on the container@ template
This commit is contained in:
parent
c191975aab
commit
24afc89ab3
18 changed files with 707 additions and 172 deletions
|
|
@ -1,44 +0,0 @@
|
|||
# The pre-start check of a container's root, for every `container@` instance.
|
||||
#
|
||||
# nixpkgs' start script runs as host root against the container's own root,
|
||||
# which is persistent and writable by the container, and its `mkdir`, `chmod`,
|
||||
# `touch` and `cp` follow links the container planted there. A drop-in on the
|
||||
# `container@` template runs `hive-priv check-rootfs %i` as an `ExecStartPre=`
|
||||
# before the script (`hive-priv/src/prestart.rs` has the rules). It covers the
|
||||
# agents' imperative instances and the declarative ones (hive-forge,
|
||||
# hive-matrix, the swarm services), each of which has a unit file of its own:
|
||||
# systemd reads the template's drop-in directory for every instance.
|
||||
#
|
||||
# Order: systemd parses a drop-in after the unit file, so this line follows the
|
||||
# unit file's own `ExecStartPre=` (nixpkgs' `preStart`, which touches no path
|
||||
# under the root), and `ExecStart=` runs only after every `ExecStartPre=`
|
||||
# succeeded.
|
||||
#
|
||||
# The drop-in is a unit-file package rather than `systemd.services."container@"`
|
||||
# because that attribute renders the template's own unit file, which a
|
||||
# declarative instance's unit file replaces.
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
dropin = pkgs.writeTextFile {
|
||||
name = "hyperhive-container-rootfs-check";
|
||||
destination = "/lib/systemd/system/container@.service.d/10-hyperhive-rootfs-check.conf";
|
||||
text = ''
|
||||
[Service]
|
||||
ExecStartPre=${config.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs %i
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
config = lib.mkIf config.boot.enableContainers {
|
||||
systemd.packages = [ dropin ];
|
||||
|
||||
# The per-agent drop-ins this check replaced rooted a hive-priv build per
|
||||
# agent here. Boot is the one time no such drop-in is loaded.
|
||||
systemd.tmpfiles.rules = [ "R! /nix/var/nix/gcroots/hyperhive-check-rootfs - - - - -" ];
|
||||
};
|
||||
}
|
||||
|
|
@ -22,7 +22,6 @@
|
|||
./hive-matrix-service.nix
|
||||
./hive-matrix.nix
|
||||
./hive-network.nix
|
||||
./container-rootfs-check.nix
|
||||
./hive-priv.nix
|
||||
./hive-priv-cache.nix
|
||||
./hive-squid.nix
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ let
|
|||
deployCfg.bao.forgePublisherClientCertFile != null
|
||||
&& deployCfg.bao.forgePublisherClientKeyFile != null;
|
||||
|
||||
rootfsCheck = import ../lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
||||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||||
|
|
@ -471,6 +473,7 @@ in
|
|||
# ../swarm-builder.nix's does.
|
||||
systemd.services."container@hive-forge" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly "hive-forge")
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ let
|
|||
# the client is pointed at the server name instead.
|
||||
hyperhiveDomain = config.services.hyperhive.domain;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# Same runtime→build-time bridge hive-forge and swarm-ci-runner cross:
|
||||
# binds the hive trust bundle (which folds in the store's root) into the
|
||||
# container and orders the container after `hive-tls-ca.service`. The
|
||||
|
|
@ -1096,6 +1098,7 @@ in
|
|||
systemd.services."container@hive-matrix" = lib.mkMerge [
|
||||
{ after = [ "dnsmasq.service" ]; }
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly "hive-matrix")
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
32
nix/host-modules/lib/container-rootfs-check.nix
Normal file
32
nix/host-modules/lib/container-rootfs-check.nix
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
# The pre-start check of a container's root, for one of hyperhive's own
|
||||
# declarative containers.
|
||||
#
|
||||
# nixpkgs' start script runs as host root against the container's own root,
|
||||
# which is persistent and writable by the container, and its `mkdir`, `chmod`,
|
||||
# `touch` and `cp` follow links the container planted there. `hive-priv
|
||||
# check-rootfs` looks at the paths the script touches (rules in
|
||||
# `hive-priv/src/prestart.rs`). With `--warn-only` it logs each violation at
|
||||
# warning level and lets the start go on; the agent containers run it strict
|
||||
# from their per-agent drop-in (hive-priv's `WriteResourceLimits`).
|
||||
#
|
||||
# Pure function, not a NixOS module. A container's module calls it with its
|
||||
# own name and folds the result into its unit:
|
||||
#
|
||||
# rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
# # systemd.services."container@${machine}" = lib.mkMerge [ (rootfsCheck.warnOnly machine) … ];
|
||||
#
|
||||
# The name list is the callers', never `config.containers`: that also holds
|
||||
# containers the operator runs that hyperhive doesn't own, and those get
|
||||
# nothing. nixpkgs renders each declarative container as its own
|
||||
# `container@<name>` unit (`nixos-containers.nix`, the `mapAttrsToList` that
|
||||
# builds `container@${name}`), so a setting on that unit reaches that
|
||||
# container only.
|
||||
{ config }:
|
||||
{
|
||||
# Attributes for `systemd.services."container@<name>"`.
|
||||
warnOnly = name: {
|
||||
serviceConfig.ExecStartPre = [
|
||||
"${config.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}"
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -114,6 +114,8 @@ let
|
|||
# one word.
|
||||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
||||
# self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
|
|
@ -226,6 +228,7 @@ in
|
|||
# CA ordering keeps the trust bundle's bind source there first.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -183,6 +183,8 @@ let
|
|||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||||
storeSeededSecret = import ./lib/store-seeded-secret.nix { };
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
inherit lib gatewayCfg;
|
||||
tlsCfg = deployCfg.hive-controller.tls;
|
||||
|
|
@ -970,6 +972,7 @@ in
|
|||
# not be up.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
(lib.mkIf publishes {
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -1689,6 +1689,8 @@ let
|
|||
forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}";
|
||||
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# Outbound TLS for that collector: it dials the swarm collector's gateway
|
||||
# name and authelia's token endpoint, both leaves under the store's runtime
|
||||
# pki root, which cannot be baked into a derivation. Same helper, same two
|
||||
|
|
@ -4091,6 +4093,7 @@ in
|
|||
# cannot start is a store that is down.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
{ after = [ "systemd-tmpfiles-resetup.service" ]; }
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ let
|
|||
# containers' forwarders all the same.
|
||||
forwarderTelemetryPort = 8892;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# `git` reaches the forge; in self-signed mode their leaves chain to the hive
|
||||
# CA.
|
||||
|
|
@ -165,6 +167,7 @@ in
|
|||
# the unit does not exist.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ let
|
|||
# below compares the two, so both come from here.
|
||||
forgeUrl = "https://${forgeCfg.domain}";
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The forge's gateway serves a leaf from the store's `pki` mount in
|
||||
# self-signed mode.
|
||||
# The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
|
||||
|
|
@ -329,6 +331,7 @@ in
|
|||
# minting unit in and waits for it, as ./swarm-builder.nix's does.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ let
|
|||
# containers' forwarders all the same.
|
||||
forwarderTelemetryPort = 8893;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# in self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
|
@ -146,6 +148,7 @@ in
|
|||
# missing, so where this host mints the leaf the container waits for it.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{ after = [ "swarm-bao-pki.service" ]; }
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ let
|
|||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = deployCfg.bao;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
inherit lib gatewayCfg;
|
||||
tlsCfg = deployCfg.hive-controller.tls;
|
||||
|
|
@ -686,6 +688,7 @@ in
|
|||
# ./swarm-builder.nix's does.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ let
|
|||
|
||||
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
||||
gatewayCfg = config.services.hyperhive.gateway;
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
# The responder introspects authelia over https BY NAME. Its HTTP client is
|
||||
# reqwest/rustls, and `rustls-platform-verifier` resolves roots through
|
||||
|
|
@ -961,7 +963,10 @@ in
|
|||
# Order the container after the host CA generator, so the bind source
|
||||
# exists before nspawn sets the mount up. Without it a late CA fails the
|
||||
# container start outright rather than degrading.
|
||||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
];
|
||||
|
||||
# Deliver the responder's store identity into the container before it
|
||||
# starts. A copy rather than a `bindMounts` entry: nixos-container refuses
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ let
|
|||
# containers' forwarders all the same.
|
||||
forwarderTelemetryPort = 8902;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# in self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
|
@ -134,6 +136,7 @@ in
|
|||
# missing, so where this host mints the leaf the container waits for it.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{ after = [ "swarm-bao-pki.service" ]; }
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -173,6 +173,8 @@ let
|
|||
# that ever names the socket.
|
||||
hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The swarm's authelia is reached by its gateway name, whose leaf is
|
||||
# issued by the swarm services sub-CA — so this container needs the same
|
||||
# runtime CA trust every other consumer of a swarm-service name needs.
|
||||
|
|
@ -706,7 +708,10 @@ in
|
|||
# The CA bind source is written at runtime by a host unit, so the
|
||||
# container has to start after it — otherwise nspawn sets up a mount
|
||||
# over a file that does not exist yet.
|
||||
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
];
|
||||
|
||||
# A collector that is up but not listening on a socket the gateway names
|
||||
# answers every producer with a 502 and fails no unit. This one fails when
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@ let
|
|||
# The standard OTLP/HTTP port, on the container's loopback.
|
||||
forwarderOtlpPort = 4318;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# in self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
|
@ -172,6 +174,7 @@ in
|
|||
# the unit does not exist.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -99,6 +99,8 @@ let
|
|||
# one word.
|
||||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
||||
# self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
|
|
@ -290,6 +292,7 @@ in
|
|||
# ordering keeps the trust bundle's bind source there first.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ let
|
|||
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
|
||||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
in
|
||||
{
|
||||
# What the store IS from any hive's point of view, the name it answers on,
|
||||
|
|
@ -728,6 +729,8 @@ in
|
|||
systemd.timers.${forwarderFetchUnit} =
|
||||
storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store";
|
||||
|
||||
systemd.services."container@${cfg.machine}" = rootfsCheck.warnOnly cfg.machine;
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
|
|||
Loading…
Reference in a new issue