nix: share the service-container settings through one in-container module
The ten hand-rolled `containers.<name>` blocks each repeat the same
in-container lines: `system.stateVersion`, a firewall turned off because
the container shares the host netns, and resolvconf forced off because
something in the container writes /etc/resolv.conf itself.
`nix/host-modules/swarm-container.nix` now owns those lines. It is
imported inside the container's own config and exposes
`services.hyperhive.swarmContainer.{privateNetwork,writesOwnResolvConf}`
for the host module to set. `stateVersion` is a `mkDefault`, so the two
containers on another value can keep theirs. `--link-journal=host` stays
per module, and so do the host-side attrs (autoStart, ephemeral,
privateNetwork, bindMounts).
swarm-victoriametrics is converted as the first user. Its container
toplevel drvPath is unchanged. A module-eval case now forces that
container's config, which nothing in the suite read before.
Refs #3773
This commit is contained in:
parent
544a8dd228
commit
024067f3f8
3 changed files with 74 additions and 14 deletions
52
nix/host-modules/swarm-container.nix
Normal file
52
nix/host-modules/swarm-container.nix
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
# What every hyperhive service nixos-container sets for itself, imported
|
||||||
|
# inside the container's own `config`.
|
||||||
|
#
|
||||||
|
# The host module that declares `containers.<name>` sets the options below.
|
||||||
|
# They restate host-side facts the container cannot read on its own: its
|
||||||
|
# evaluation is nested inside `containers.<name>`, and reading the host side
|
||||||
|
# back from in here recurses.
|
||||||
|
{ config, lib, ... }:
|
||||||
|
let
|
||||||
|
cfg = config.services.hyperhive.swarmContainer;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
options.services.hyperhive.swarmContainer = {
|
||||||
|
privateNetwork = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
description = ''
|
||||||
|
Must equal the host-side `containers.<name>.privateNetwork`. When
|
||||||
|
`false` the container shares the host netns, so its own
|
||||||
|
firewall.service would rewrite the HOST ruleset at every boot; the
|
||||||
|
container's firewall is turned off and the host firewall owns all
|
||||||
|
filtering.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
writesOwnResolvConf = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = true;
|
||||||
|
description = ''
|
||||||
|
Something in this container writes `/etc/resolv.conf` itself (the
|
||||||
|
`swarm-container-resolver.nix` unit, or a static file), so
|
||||||
|
resolvconf is forced off. Left on, host-tracking would regenerate the
|
||||||
|
file empty, since the host's copy doesn't cross the boundary after
|
||||||
|
start.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkMerge [
|
||||||
|
{
|
||||||
|
# A container that sets its own keeps it. Changing this value changes
|
||||||
|
# it for every container that doesn't, and that is a state migration
|
||||||
|
# for each of them.
|
||||||
|
system.stateVersion = lib.mkDefault "26.05";
|
||||||
|
}
|
||||||
|
(lib.mkIf (!cfg.privateNetwork) {
|
||||||
|
networking.firewall.enable = false;
|
||||||
|
})
|
||||||
|
(lib.mkIf cfg.writesOwnResolvConf {
|
||||||
|
networking.resolvconf.enable = lib.mkForce false;
|
||||||
|
})
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
@ -27,6 +27,10 @@ let
|
||||||
# the required-domain assertion in hive-network.nix should be what an
|
# the required-domain assertion in hive-network.nix should be what an
|
||||||
# operator sees, not a coercion error from here.
|
# operator sees, not a coercion error from here.
|
||||||
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
||||||
|
|
||||||
|
# Shared host netns, like every sibling swarm container: the gateway
|
||||||
|
# reaches this at 127.0.0.1:<port>.
|
||||||
|
privateNetwork = false;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
# What stays here is what the store IS from any hive's point of view: the
|
# What stays here is what the store IS from any hive's point of view: the
|
||||||
|
|
@ -190,31 +194,20 @@ in
|
||||||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||||||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||||||
extraFlags = [ "--link-journal=host" ];
|
extraFlags = [ "--link-journal=host" ];
|
||||||
# Shared host netns, like every sibling swarm container: the gateway
|
inherit privateNetwork;
|
||||||
# reaches this at 127.0.0.1:<port>.
|
|
||||||
privateNetwork = false;
|
|
||||||
|
|
||||||
config =
|
config =
|
||||||
{ ... }:
|
{ ... }:
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
|
./swarm-container.nix
|
||||||
(import ./swarm-container-resolver.nix {
|
(import ./swarm-container-resolver.nix {
|
||||||
inherit (networkCfg) bridgeIp;
|
inherit (networkCfg) bridgeIp;
|
||||||
dnsConsumers = [ "victoriametrics.service" ];
|
dnsConsumers = [ "victoriametrics.service" ];
|
||||||
})
|
})
|
||||||
];
|
];
|
||||||
|
|
||||||
system.stateVersion = "26.05";
|
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||||||
|
|
||||||
# This container shares the host netns, so its own firewall.service
|
|
||||||
# would rewrite the HOST ruleset at every boot. The host firewall
|
|
||||||
# owns all filtering.
|
|
||||||
networking.firewall.enable = false;
|
|
||||||
# resolvconf stays off because the resolver unit imported above
|
|
||||||
# owns /etc/resolv.conf. Leaving it on would let host-tracking
|
|
||||||
# regenerate the file empty, since the host's copy doesn't cross
|
|
||||||
# the boundary after start.
|
|
||||||
networking.resolvconf.enable = lib.mkForce false;
|
|
||||||
|
|
||||||
services.victoriametrics = {
|
services.victoriametrics = {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
|
|
||||||
|
|
@ -185,6 +185,21 @@ let
|
||||||
&& !(swarmServicesOnly.systemd.sockets ? hive-priv)
|
&& !(swarmServicesOnly.systemd.sockets ? hive-priv)
|
||||||
&& !(s ? swarm-bao-queue-agent);
|
&& !(s ? swarm-bao-queue-agent);
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# Both values come from ../host-modules/swarm-container.nix. Read
|
||||||
|
# through the metrics store: nothing else in this suite evaluates that
|
||||||
|
# container's config.
|
||||||
|
name = "a service container on the host netns runs no firewall or resolvconf of its own";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
c = swarmServicesOnly.containers.swarm-victoriametrics.config;
|
||||||
|
in
|
||||||
|
!c.networking.firewall.enable && !c.networking.resolvconf.enable;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "a service container that sets no stateVersion of its own is on 26.05";
|
||||||
|
ok = swarmServicesOnly.containers.swarm-victoriametrics.config.system.stateVersion == "26.05";
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# An operator's explicit `false` beats every `mkDefault` assertion,
|
# An operator's explicit `false` beats every `mkDefault` assertion,
|
||||||
# which is what keeps "asserted by whoever needs it" from being a
|
# which is what keeps "asserted by whoever needs it" from being a
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue