vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

move the small per-service host options to deploy.<service>

Browse source
This commit is contained in:
atlas 2026-10-29 01:48:34 +01:00 • committed by mara
commit 525de62b06
26 changed files with 407 additions and 251 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/integrations/matrix.md
View file

@ -6,7 +6,7 @@ client at `chat.<swarm-domain>/` (the `gatewayHost` vhost). A swarm runs one
homeserver, on one host. Two namespaces configure it:
- `services.hyperhive.swarm.matrix.*` — what the homeserver **is**, as
every hive sees it: `serverName`, `gatewayHost`, `allowEncryption`.
every hive sees it: `serverName`, `gatewayHost`.
- `services.hyperhive.deploy.matrix.*` — what the host running it decides:
`enable`, `gui.package`, `trustedServers`,
`maxRequestSize`, `sso.clientSecretFile`.
@ -293,7 +293,7 @@ Initial rollout settings:
tuwunel refuse to start unless
`yes_i_am_very_very_sure_…_open_registration_…` is also set.
- `allow_encryption` — server-side E2EE switch, sourced from
`services.hyperhive.swarm.matrix.allowEncryption` (**default `false`**, opt-in).
`services.hyperhive.deploy.matrix.allowEncryption` (**default `false`**, opt-in).
Off by default because on the swarm's own homeserver the operator
already controls the transport; turn it on for encrypted rooms on
external / federated homeservers or to keep contents opaque to the

10
docs/networking/gateway.md
View file

@ -151,12 +151,10 @@ Only the swarm's forge host runs it (`services.hyperhive.deploy.forgejo.enable`,
### Network and port configuration
```nix
services.hyperhive.swarm.forge = {
httpPort = 3000; # default — outside hyperhive's 7000 / 8100-8999
sshPort = 2222; # default — git-over-SSH, off the host's openssh on 22
};
# Which ports the forge answers on is swarm-wide; whether THIS host opens
# them in its firewall is a deployment decision, so it lives under deploy.*.
# The SSH port is swarm-wide: every hive dials it. The HTTP port and whether
# THIS host opens them in its firewall are this host's deployment decisions.
services.hyperhive.swarm.forge.sshPort = 2222; # default — git-over-SSH, off the host's openssh on 22
services.hyperhive.deploy.forgejo.httpPort = 3000; # default — outside hyperhive's 7000 / 8100-8999
services.hyperhive.deploy.forgejo.openFirewall = false; # default
```

2
docs/swarm/builder.md
View file

@ -32,7 +32,7 @@ Where it runs and how it's enabled: [`services.md`](services.md#builder).
3. The builder runs `nix build` of `nix/builder/check-flake`'s `check` output
with every input overridden:
- `hyperhive` and `nixpkgs`: the sources the builder's own host was
deployed with (`c0re.hyperhiveFlake`, `c0re.nixpkgsFlake`). A hive's meta
deployed with (`c0re.hyperhiveFlake`, `deploy.builder.nixpkgsFlake`). A hive's meta
flake pins `hyperhive` by forge URL and rev instead.
- `agent`: `git+https://<forge>/agent-configs/<agent>?ref=refs/pull/<pr>/head&rev=<head>`.
A build has one hour.

2
docs/swarm/services.md
View file

@ -310,7 +310,7 @@ swarm.
| `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. |
| `deploy.victorialogs.grafanaAuthPackage` | The `swarm-metrics-auth` build that runs as `swarm-logs-grafana-auth`. Set on its own: `deploy.victoriametrics.authPackage` doesn't reach it. |
| `swarm.victorialogs.domain` | Only to rename it. |
| `swarm.victorialogs.port` | Only if something else on the services host already claims `9428`. |
| `deploy.victorialogs.port` | Only if something else on the services host already claims `9428`. |
It binds loopback and takes no credential of its own: the gateway vhost is
the only way in, and the collector is the only intended writer.

46
nix/host-modules/deploy.nix
View file

@ -259,6 +259,52 @@ in
[ "services" "hyperhive" "deploy" "hive-controller" "otel" "extraResourceAttributes" ]
)
# Per-service settings of the host that runs the service: they sit with the
# service's other deployment options, not under the swarm facts every host shares.
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "controller" "queue" "natsUrl" ]
[ "services" "hyperhive" "deploy" "swarm-controller" "queue" "natsUrl" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "controller" "queue" "tokenEndpoint" ]
[ "services" "hyperhive" "deploy" "swarm-controller" "queue" "tokenEndpoint" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "bao" "otel" "telemetryPort" ]
[ "services" "hyperhive" "deploy" "bao" "otel" "telemetryPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "nats" "monitorPort" ]
[ "services" "hyperhive" "deploy" "nats" "monitorPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "nats" "metricsPort" ]
[ "services" "hyperhive" "deploy" "nats" "metricsPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "nats" "otel" "telemetryPort" ]
[ "services" "hyperhive" "deploy" "nats" "otel" "telemetryPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "victorialogs" "port" ]
[ "services" "hyperhive" "deploy" "victorialogs" "port" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "matrix" "allowEncryption" ]
[ "services" "hyperhive" "deploy" "matrix" "allowEncryption" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "c0re" "nixpkgsFlake" ]
[ "services" "hyperhive" "deploy" "builder" "nixpkgsFlake" ]
)
# The homeserver's port and its firewall switch, removed rather than
# renamed: tuwunel listens on a unix socket only the gateway reaches, so
# there is no host port to choose or open.

18
nix/host-modules/hive-c0re/options.nix
View file

@ -1,6 +1,6 @@
# Option declarations for the c0re daemon. `services.hyperhive.c0re.*`
# keeps the two flake-source options the swarm builder also reads
# (`hyperhiveFlake`, `nixpkgsFlake`); every other knob and the
# keeps the flake-source option the swarm builder also reads
# (`hyperhiveFlake`); every other knob and the
# package/asset options the flake's `nixosModules.default` wires to its
# own outputs (they carry no in-module defaults; see ../../../flake.nix)
# live under `services.hyperhive.deploy.hive-controller`, alongside the
@ -8,7 +8,6 @@
# `servedFrontend` option lives in ./theme.nix with the stylix wiring
# that computes it.
{
pkgs,
lib,
...
}:
@ -34,19 +33,6 @@
filtered source.
'';
};
nixpkgsFlake = lib.mkOption {
type = lib.types.str;
default = "path:${pkgs.path}";
defaultText = lib.literalMD "`\"path:\${pkgs.path}\"`";
description = ''
Store-path URL of the nixpkgs the swarm builder's check flake
builds on. The swarm meta flake does not use it: its `nixpkgs`
follows `hyperhive/nixpkgs`.
Defaults to `"path:''${pkgs.path}"`, the nixpkgs the host NixOS
module was evaluated with.
'';
};
};
# The hive-controller's deployment options.

25
nix/host-modules/hive-forge/default.nix
View file

@ -179,6 +179,17 @@ in
# where its host-local secrets sit. Same rule as ../swarm-victorialogs.nix,
# and the renames are in ../deploy.nix with the rest.
options.services.hyperhive.deploy.forgejo = {
httpPort = lib.mkOption {
type = lib.types.port;
default = 3000;
description = ''
TCP port the forge serves HTTP on. Default 3000 sits outside
hyperhive's claimed ranges (dashboard 7000, every agent in
8100..8999 via FNV-1a hash). Change this if you already have
another forgejo bound to 3000.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.forgejo;
@ -304,7 +315,7 @@ in
"${cfg.domain}" = {
locations = {
"/" = {
proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/";
proxyPass = "http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/";
proxyWebsockets = true;
extraConfig = ''
proxy_buffering off;
@ -325,7 +336,7 @@ in
# `publishedScrapeTargets.forgejo` in ../swarm-otel-service.nix;
# the two must agree, since that string is also the token's audience.
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/metrics";
proxyPass = "http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/metrics";
extraConfig = ''
auth_request /__forge_metrics_authz;
'';
@ -453,8 +464,8 @@ in
assertion =
!(lib.elem forwarderTelemetryPort [
config.services.hyperhive.otel.telemetryPort
config.services.hyperhive.swarm.bao.otel.telemetryPort
config.services.hyperhive.swarm.nats.otel.telemetryPort
config.services.hyperhive.deploy.bao.otel.telemetryPort
config.services.hyperhive.deploy.nats.otel.telemetryPort
]);
message = ''
The forge's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
@ -608,7 +619,7 @@ in
server = {
DOMAIN = cfg.domain;
ROOT_URL = effectiveRootUrl;
HTTP_PORT = cfg.httpPort;
HTTP_PORT = deployCfg.forgejo.httpPort;
START_SSH_SERVER = true;
SSH_PORT = cfg.sshPort;
SSH_LISTEN_PORT = cfg.sshPort;
@ -1135,7 +1146,7 @@ in
curl --silent --show-error --connect-timeout 5 --max-time 15 \
-H @<(printf 'Authorization: token %s\n' "$token") \
-X "$method" -o "$body" -w '%{http_code}' \
"http://127.0.0.1:${toString cfg.httpPort}/api/v1$path"
"http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/api/v1$path"
}
# Every other token this unit minted: one whose write to the
@ -1300,7 +1311,7 @@ in
networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
allowedTCPPorts = [
cfg.httpPort
deployCfg.forgejo.httpPort
cfg.sshPort
];
};

11
nix/host-modules/hive-forge/service.nix
View file

@ -30,17 +30,6 @@ in
# only one host runs it: `deploy.forgejo.enable`, in ../deploy.nix. What
# follows is what every hive needs to reach it, wherever it runs.
options.services.hyperhive.swarm.forge = {
httpPort = lib.mkOption {
type = lib.types.port;
default = 3000;
description = ''
TCP port the forge serves HTTP on. Default 3000 sits outside
hyperhive's claimed ranges (dashboard 7000, every agent in
8100..8999 via FNV-1a hash). Change this if you already have
another forgejo bound to 3000.
'';
};
sshPort = lib.mkOption {
type = lib.types.port;
default = 2222;

19
nix/host-modules/hive-matrix-service.nix
View file

@ -90,25 +90,6 @@ in
'';
};
allowEncryption = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Server-side switch for matrix end-to-end encryption — sets
tuwunel's `allow_encryption`. Off by default: on the hive-internal
homeserver the operator already controls the transport, so server
E2EE adds key-management overhead (cross-signing, device
verification, undecryptable-message recovery) without a clear
threat-model win for the common single-hive case. Turn on when
agents join encrypted rooms on external / federated homeservers,
or when the operator wants message contents opaque to the
homeserver admin. Independent of the agent matrix client, which
always supports decryption so it can read encrypted rooms it is
invited to regardless of this flag; this option only governs
whether THIS homeserver permits room encryption.
'';
};
adminSync.ignoreUsers = lib.mkOption {
type = lib.types.listOf (lib.types.strMatching "@[^:[:space:]]+:[^[:space:]]+");
default = [ ];

21
nix/host-modules/hive-matrix.nix
View file

@ -290,6 +290,25 @@ in
# how large a request it accepts, and where its host-local secrets sit. Same rule as
# ./swarm-victorialogs.nix; `enable` lives in ./deploy.nix.
options.services.hyperhive.deploy.matrix = {
allowEncryption = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Server-side switch for matrix end-to-end encryption — sets
tuwunel's `allow_encryption`. Off by default: on the hive-internal
homeserver the operator already controls the transport, so server
E2EE adds key-management overhead (cross-signing, device
verification, undecryptable-message recovery) without a clear
threat-model win for the common single-hive case. Turn on when
agents join encrypted rooms on external / federated homeservers,
or when the operator wants message contents opaque to the
homeserver admin. Independent of the agent matrix client, which
always supports decryption so it can read encrypted rooms it is
invited to regardless of this flag; this option only governs
whether THIS homeserver permits room encryption.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.matrix-tuwunel;
@ -933,7 +952,7 @@ in
// {
# Server-side E2EE is opt-in (default off); the agent matrix
# client always supports decryption regardless.
allow_encryption = cfg.allowEncryption;
allow_encryption = deployCfg.matrix.allowEncryption;
# Tuwunel's default suffix is " 💕" — suppress it so agent
# display names are clean (just the agent name, no emoji).
new_user_displayname_suffix = "";

21
nix/host-modules/swarm-bao-service.nix
View file

@ -126,26 +126,5 @@ in
Two spellings present as a healthy-looking 401.
'';
};
otel.telemetryPort = lib.mkOption {
type = lib.types.port;
default = 8890;
description = ''
Port the collector inside the store's container serves its **own**
metrics on — queue depth, refused and dropped samples, exporter
failures. How you find out that telemetry is being lost, so it is
worth keeping rather than switching off.
⚠️ **Deliberately not 8888.** 8888 is the collector
binary's built-in default, which the hive tier
({option}`services.hyperhive.otel.telemetryPort`) already binds. This
container runs with `privateNetwork = false`, so both share the
host's network namespace whenever they are co-located — and unlike
the OTLP ports this one appears nowhere in either config when it is
left undeclared, so nothing that compares configured ports can see
the clash. The second collector to start simply dies with
`bind: address already in use`.
'';
};
};
}

27
nix/host-modules/swarm-bao.nix
View file

@ -1604,7 +1604,7 @@ let
baoDeploy.uiPort
(baoDeploy.uiPort + 1)
baoDeploy.uiProxyPort
cfg.otel.telemetryPort
baoDeploy.otel.telemetryPort
];
# Paths a browser session has no business sending to the store: they unseal,
@ -2772,6 +2772,27 @@ in
'';
};
otel.telemetryPort = lib.mkOption {
type = lib.types.port;
default = 8890;
description = ''
Port the collector inside the store's container serves its **own**
metrics on — queue depth, refused and dropped samples, exporter
failures. How you find out that telemetry is being lost, so it is
worth keeping rather than switching off.
⚠️ **Deliberately not 8888.** 8888 is the collector
binary's built-in default, which the hive tier
({option}`services.hyperhive.otel.telemetryPort`) already binds. This
container runs with `privateNetwork = false`, so both share the
host's network namespace whenever they are co-located — and unlike
the OTLP ports this one appears nowhere in either config when it is
left undeclared, so nothing that compares configured ports can see
the clash. The second collector to start simply dies with
`bind: address already in use`.
'';
};
metricsPort = lib.mkOption {
type = lib.types.port;
default = 8202;
@ -2879,7 +2900,7 @@ in
services.hyperhive.swarm.bao.port and +1, deploy.bao.metricsPort and
+1, deploy.bao.uiPort and +1 (openbao binds each listener's port + 1
as its cluster address), deploy.bao.uiProxyPort, and
swarm.bao.otel.telemetryPort. Pick free ports.
deploy.bao.otel.telemetryPort. Pick free ports.
'';
}
{
@ -4603,7 +4624,7 @@ in
# holds: this container shares the host's netns.
(import ../collector-self-scrape.nix {
inherit lib;
port = cfg.otel.telemetryPort;
port = baoDeploy.otel.telemetryPort;
exporter = "otlphttp";
attributes.container = collectorServiceName;
})

16
nix/host-modules/swarm-builder.nix
View file

@ -90,7 +90,7 @@ let
# Strings, not paths: both are already store paths.
hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
nixpkgsFlake = hyperhiveCfg.c0re.nixpkgsFlake;
nixpkgsFlake = hyperhiveCfg.deploy.builder.nixpkgsFlake;
# The assertion below keeps the two options together.
pushesToCache = atticCfg.domain != null;
@ -110,6 +110,20 @@ in
'';
};
options.services.hyperhive.deploy.builder.nixpkgsFlake = lib.mkOption {
type = lib.types.str;
default = "path:${pkgs.path}";
defaultText = lib.literalMD "`\"path:\${pkgs.path}\"`";
description = ''
Store-path URL of the nixpkgs the swarm builder's check flake
builds on. The swarm meta flake does not use it: its `nixpkgs`
follows `hyperhive/nixpkgs`.
Defaults to `"path:''${pkgs.path}"`, the nixpkgs the host NixOS
module was evaluated with.
'';
};
options.services.hyperhive.deploy.builder.package = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-builder";

80
nix/host-modules/swarm-controller.nix
View file

@ -193,8 +193,8 @@ let
# didn't need to; the assertions cover the split case so nobody gets a
# daemon that silently never connects.
queueEnv = {
SWARM_CONTROLLER_NATS_URL = cfg.queue.natsUrl;
SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = cfg.queue.tokenEndpoint;
SWARM_CONTROLLER_NATS_URL = deployCfg.swarm-controller.queue.natsUrl;
SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = deployCfg.swarm-controller.queue.tokenEndpoint;
SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
};
@ -391,39 +391,6 @@ in
'';
};
queue = {
natsUrl = lib.mkOption {
type = lib.types.str;
default = "tls://${natsCfg.domain}:${toString natsCfg.port}";
defaultText = lib.literalExpression ''"tls://''${swarm.nats.domain}:''${swarm.nats.port}"'';
example = "tls://nats.example.com:4222";
description = ''
Where the controller reaches the swarm queue.
Defaults to the queue's name, which is the same URL on every
host: the queue's own host resolves it locally, and any other
through the operator's DNS.
Empty means unset, which the assertion below refuses — a
controller with no queue is not a lighter controller.
'';
};
tokenEndpoint = lib.mkOption {
type = lib.types.str;
default = "${autheliaCfg.url}/api/oidc/token";
defaultText = lib.literalExpression ''"''${swarm.authelia.url}/api/oidc/token"'';
description = ''
The OIDC token endpoint the controller mints its own access
token from.
Derived from `swarm.authelia.url`, which is the half of the
authelia module that exists on every host — so this is already
correct for a remote provider as long as that URL is set.
'';
};
};
links = lib.mkOption {
type = lib.types.listOf (
lib.types.submodule {
@ -483,6 +450,39 @@ in
# the latter is hive-c0re's namespace, where `tls.*` lives. A definition on
# the wrong one lands on a live, unrelated option rather than failing.
options.services.hyperhive.deploy.swarm-controller = {
queue = {
natsUrl = lib.mkOption {
type = lib.types.str;
default = "tls://${natsCfg.domain}:${toString natsCfg.port}";
defaultText = lib.literalExpression ''"tls://''${swarm.nats.domain}:''${swarm.nats.port}"'';
example = "tls://nats.example.com:4222";
description = ''
Where the controller reaches the swarm queue.
Defaults to the queue's name, which is the same URL on every
host: the queue's own host resolves it locally, and any other
through the operator's DNS.
Empty means unset, which the assertion below refuses — a
controller with no queue is not a lighter controller.
'';
};
tokenEndpoint = lib.mkOption {
type = lib.types.str;
default = "${autheliaCfg.url}/api/oidc/token";
defaultText = lib.literalExpression ''"''${swarm.authelia.url}/api/oidc/token"'';
description = ''
The OIDC token endpoint the controller mints its own access
token from.
Derived from `swarm.authelia.url`, which is the half of the
authelia module that exists on every host — so this is already
correct for a remote provider as long as that URL is set.
'';
};
};
package = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
@ -558,7 +558,7 @@ in
No new credential to configure: the bearer token presented to
the bridge is minted from THIS daemon's own existing queue OIDC
identity — its endpoints stay in `swarm.controller.queue`, its secret is
identity — its endpoints stay in `deploy.swarm-controller.queue`, its secret is
read from the swarm's secret store — "one identity per principal"
already covers it. `null` means no agent-identity support:
`CreateIdentity` jobs fail with a clear "no auth bridge
@ -696,18 +696,18 @@ in
# because on an all-local deployment nothing was wrong.
assertions = [
{
assertion = cfg.queue.natsUrl != "";
assertion = deployCfg.swarm-controller.queue.natsUrl != "";
message = ''
services.hyperhive.swarm.controller.queue.natsUrl is unset.
services.hyperhive.deploy.swarm-controller.queue.natsUrl is unset.
It defaults to the queue's name; something in this configuration
set it to "". Remove that setting, or set the URL.
'';
}
{
assertion = cfg.queue.tokenEndpoint != "";
assertion = deployCfg.swarm-controller.queue.tokenEndpoint != "";
message = ''
services.hyperhive.swarm.controller.queue.tokenEndpoint is unset.
services.hyperhive.deploy.swarm-controller.queue.tokenEndpoint is unset.
It defaults from services.hyperhive.swarm.authelia.url, so this
only happens when it has been cleared by hand. The controller

60
nix/host-modules/swarm-nats-service.nix
View file

@ -60,69 +60,13 @@ in
`wg-hive` when this host is on the mesh, for other hives. Never
host-wide. TLS only: a client that does not speak it is refused.
Unlike {option}`monitorPort` and {option}`metricsPort`, the
Unlike {option}`services.hyperhive.deploy.nats.monitorPort` and
{option}`services.hyperhive.deploy.nats.metricsPort`, the
address is not what bounds who may use this port: the queue's
`auth_callout` refuses every client it cannot identify.
'';
};
monitorPort = lib.mkOption {
type = lib.types.port;
default = 8222;
description = ''
Port NATS serves its **monitoring** endpoint on, bound to
loopback.
Not a metrics endpoint: the server has no Prometheus format of its
own. This serves `/varz`, `/connz`, `/routez` as JSON, and the
exporter below is what translates it — which is why enabling the
exporter without this produces a process that starts cleanly and
scrapes nothing.
⚠️ Loopback, and the exporter is the only intended reader. The
endpoint is unauthenticated and `/connz` names every connected
client, so the address it binds is the whole access control. Do
not widen it, and do not put it behind a gateway vhost expecting
that to add one.
'';
};
metricsPort = lib.mkOption {
type = lib.types.port;
default = 7777;
description = ''
Port the Prometheus exporter serves NATS's metrics on, bound to
loopback. The gateway's `= /metrics` location on
{option}`services.hyperhive.swarm.nats.domain` proxies it, behind
the swarm collector's authelia check.
⚠️ This and {option}`monitorPort` are two more claims on a port
space every swarm container shares — they run in this container but
`privateNetwork = false`, so a collision with any other hyperhive
service is a runtime coin toss over which process gets the port,
with nothing in any log saying so. Both defaults are upstream's
own (`nats-server` 8222, `prometheus-nats-exporter` 7777) and
neither is claimed elsewhere in this repo, checked when they were
added.
'';
};
otel.telemetryPort = lib.mkOption {
type = lib.types.port;
default = 8891;
description = ''
Loopback port the journal forwarder inside the queue's container
serves its own metrics on (refused and dropped records, exporter
failures).
⚠️ Not 8888 or 8890: the hive collector and the store's forwarder
bind those ({option}`services.hyperhive.otel.telemetryPort`,
{option}`services.hyperhive.swarm.bao.otel.telemetryPort`), and this
container shares the host netns with them when co-located. The second
collector to bind a port dies with `bind: address already in use`.
'';
};
clientId = lib.mkOption {
type = lib.types.str;
default = "swarm-nats";

80
nix/host-modules/swarm-nats.nix
View file

@ -364,6 +364,64 @@ in
# ./deploy.nix, which also carries the renames. The callout's keys are no
# option at all: they live in the secret store (docs/swarm/credentials.md).
options.services.hyperhive.deploy.nats = {
monitorPort = lib.mkOption {
type = lib.types.port;
default = 8222;
description = ''
Port NATS serves its **monitoring** endpoint on, bound to
loopback.
Not a metrics endpoint: the server has no Prometheus format of its
own. This serves `/varz`, `/connz`, `/routez` as JSON, and the
exporter below is what translates it — which is why enabling the
exporter without this produces a process that starts cleanly and
scrapes nothing.
⚠️ Loopback, and the exporter is the only intended reader. The
endpoint is unauthenticated and `/connz` names every connected
client, so the address it binds is the whole access control. Do
not widen it, and do not put it behind a gateway vhost expecting
that to add one.
'';
};
metricsPort = lib.mkOption {
type = lib.types.port;
default = 7777;
description = ''
Port the Prometheus exporter serves NATS's metrics on, bound to
loopback. The gateway's `= /metrics` location on
{option}`services.hyperhive.swarm.nats.domain` proxies it, behind
the swarm collector's authelia check.
⚠️ This and {option}`services.hyperhive.deploy.nats.monitorPort` are two
more claims on a port
space every swarm container shares — they run in this container but
`privateNetwork = false`, so a collision with any other hyperhive
service is a runtime coin toss over which process gets the port,
with nothing in any log saying so. Both defaults are upstream's
own (`nats-server` 8222, `prometheus-nats-exporter` 7777) and
neither is claimed elsewhere in this repo, checked when they were
added.
'';
};
otel.telemetryPort = lib.mkOption {
type = lib.types.port;
default = 8891;
description = ''
Loopback port the journal forwarder inside the queue's container
serves its own metrics on (refused and dropped records, exporter
failures).
⚠️ Not 8888 or 8890: the hive collector and the store's forwarder
bind those ({option}`services.hyperhive.otel.telemetryPort`,
{option}`services.hyperhive.deploy.bao.otel.telemetryPort`), and this
container shares the host netns with them when co-located. The second
collector to bind a port dies with `bind: address already in use`.
'';
};
authPackage = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth";
@ -485,7 +543,7 @@ in
# Authelia's rules for it admit the collector's client and deny
# every other subject (`metricsRules` in ./swarm-authelia.nix).
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics";
proxyPass = "http://127.0.0.1:${toString deployCfg.nats.metricsPort}/metrics";
extraConfig = ''
auth_request /__nats_metrics_authz;
'';
@ -597,17 +655,17 @@ in
let
ports = [
cfg.port
cfg.monitorPort
cfg.metricsPort
cfg.otel.telemetryPort
deployCfg.nats.monitorPort
deployCfg.nats.metricsPort
deployCfg.nats.otel.telemetryPort
config.services.hyperhive.otel.telemetryPort
config.services.hyperhive.swarm.bao.otel.telemetryPort
deployCfg.bao.otel.telemetryPort
];
in
lib.length (lib.unique ports) == lib.length ports;
message = ''
services.hyperhive.swarm.nats.otel.telemetryPort
(${toString cfg.otel.telemetryPort}) collides with another port in
services.hyperhive.deploy.nats.otel.telemetryPort
(${toString deployCfg.nats.otel.telemetryPort}) collides with another port in
the host netns: the queue's own ports, or the telemetry port of the
hive collector or the store's forwarder. Pick
a free port.
@ -687,7 +745,7 @@ in
endpoint = forwarderEndpoint;
inherit (cfg) clientId;
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
telemetryPort = cfg.otel.telemetryPort;
telemetryPort = deployCfg.nats.otel.telemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
@ -749,7 +807,7 @@ in
#
# ⚠️ Loopback: unauthenticated, and `/connz` lists every
# connected client.
http = "127.0.0.1:${toString cfg.monitorPort}";
http = "127.0.0.1:${toString deployCfg.nats.monitorPort}";
# TLS required on the client port: a server with a `tls`
# block advertises `tls_required` and refuses a client that
@ -782,8 +840,8 @@ in
services.prometheus.exporters.nats = {
enable = true;
listenAddress = "127.0.0.1";
port = cfg.metricsPort;
url = "http://127.0.0.1:${toString cfg.monitorPort}";
port = deployCfg.nats.metricsPort;
url = "http://127.0.0.1:${toString deployCfg.nats.monitorPort}";
# ⚠️ NOT optional, despite the name. Upstream renders
# `-addr … -port … ${extraFlags} ${url}` and defaults

14
nix/host-modules/swarm-otel-service.nix
View file

@ -50,20 +50,6 @@ in
'';
};
telemetryPort = lib.mkOption {
type = lib.types.port;
default = 8889;
description = ''
Port this collector serves its **own** metrics on — queue depth,
refused and dropped samples, exporter failures. How you find out
that telemetry is being lost, so it is worth keeping rather than
switching off.
Bound on the container's own loopback, so no other collector
shares it; only this collector scrapes it.
'';
};
serviceProducers = lib.mkOption {
internal = true;
type = lib.types.attrsOf (

19
nix/host-modules/swarm-otel.nix
View file

@ -333,6 +333,20 @@ in
# operator's choice for this deployment, so both hang off the deployment.
# `enable` already lives in ./deploy.nix, which also carries the rename.
options.services.hyperhive.deploy.swarm-otel = {
telemetryPort = lib.mkOption {
type = lib.types.port;
default = 8889;
description = ''
Port this collector serves its **own** metrics on — queue depth,
refused and dropped samples, exporter failures. How you find out
that telemetry is being lost, so it is worth keeping rather than
switching off.
Bound on the container's own loopback, so no other collector
shares it; only this collector scrapes it.
'';
};
upstream = {
endpoint = lib.mkOption {
type = lib.types.str;
@ -484,7 +498,8 @@ in
# alive, these counters say whether it is dropping what it receives.
# Loopback is the container's own: this collector scrapes `telemetryPort`
# inside its netns, and nothing else reads it.
services.hyperhive.swarm.otel.scrapeTargets.collector = "127.0.0.1:${toString cfg.telemetryPort}";
services.hyperhive.swarm.otel.scrapeTargets.collector =
"127.0.0.1:${toString deployCfg.swarm-otel.telemetryPort}";
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
# every sibling swarm service: TLS terminates here, then plain http over
@ -1320,7 +1335,7 @@ in
{
pull.exporter.prometheus = {
host = "127.0.0.1";
port = cfg.telemetryPort;
port = deployCfg.swarm-otel.telemetryPort;
};
}
];

16
nix/host-modules/swarm-victorialogs-service.nix
View file

@ -62,21 +62,5 @@ in
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 9428;
description = ''
Loopback port the store listens on. Upstream's default, kept
because there is no reason to move it and a familiar number is
one less thing an operator has to look up.
⚠️ Every swarm container shares the host's network namespace, so
this is a swarm-wide claim rather than a per-container one — two
modules picking the same number collide at runtime with no bind
error and nothing in any log. No repo-wide
check catches it.
'';
};
};
}

36
nix/host-modules/swarm-victorialogs.nix
View file

@ -109,9 +109,9 @@ let
};
in
{
# What the store IS from any hive's point of view, the name it answers on and
# the port, is `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
# `enable`, `package` and `retentionPeriod` are decisions of the host that
# What the store IS from any hive's point of view, the name it answers on, is
# `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
# `enable`, `package`, `port` and `retentionPeriod` are decisions of the host that
# runs it and live under `deploy.*`.
# Retention is a property of the store this host runs, not something the
@ -126,6 +126,22 @@ in
description = "VictoriaLogs package to run.";
};
options.services.hyperhive.deploy.victorialogs.port = lib.mkOption {
type = lib.types.port;
default = 9428;
description = ''
Loopback port the store listens on. Upstream's default, kept
because there is no reason to move it and a familiar number is
one less thing an operator has to look up.
⚠️ Every swarm container shares the host's network namespace, so
this is a swarm-wide claim rather than a per-container one — two
modules picking the same number collide at runtime with no bind
error and nothing in any log. No repo-wide
check catches it.
'';
};
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
type = lib.types.str;
default = "30d";
@ -274,8 +290,8 @@ in
assertion =
!(lib.elem forwarderTelemetryPort [
hyperhiveCfg.otel.telemetryPort
hyperhiveCfg.swarm.bao.otel.telemetryPort
hyperhiveCfg.swarm.nats.otel.telemetryPort
hyperhiveCfg.deploy.bao.otel.telemetryPort
hyperhiveCfg.deploy.nats.otel.telemetryPort
]);
message = ''
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
@ -334,7 +350,7 @@ in
forceSSL = true;
locations = {
"/" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/";
extraConfig = swarmAuthRequest;
};
@ -353,7 +369,7 @@ in
# note on `swarmAuthRequest` above), so naming it here is required
# rather than redundant.
"= /insert/opentelemetry/v1/logs" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}/insert/opentelemetry/v1/logs";
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs";
extraConfig = ''
auth_request /__hive_authelia;
'';
@ -387,7 +403,7 @@ in
"^~ /select/logsql/" = {
# No URI part, so the request path and its query string reach
# the store as the caller sent them.
proxyPass = "http://127.0.0.1:${toString cfg.port}";
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}";
extraConfig = ''
auth_request /__swarm_logs_auth;
'';
@ -404,7 +420,7 @@ in
# authelia's rules for this URL admit the collector's client and
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/metrics";
extraConfig = ''
auth_request /__hive_authelia;
'';
@ -653,7 +669,7 @@ in
# Nothing authenticates it either — upstream offers one basic-auth
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
# module sets neither. The bind address is the boundary.
listenAddress = "127.0.0.1:${toString cfg.port}";
listenAddress = "127.0.0.1:${toString deployCfg.victorialogs.port}";
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
};

Some files were not shown because too many files have changed in this diff Show more