move the small per-service host options to deploy.<service>
This commit is contained in:
parent
400fdd7f0e
commit
525de62b06
26 changed files with 407 additions and 251 deletions
|
|
@ -6,7 +6,7 @@ client at `chat.<swarm-domain>/` (the `gatewayHost` vhost). A swarm runs one
|
||||||
homeserver, on one host. Two namespaces configure it:
|
homeserver, on one host. Two namespaces configure it:
|
||||||
|
|
||||||
- `services.hyperhive.swarm.matrix.*` — what the homeserver **is**, as
|
- `services.hyperhive.swarm.matrix.*` — what the homeserver **is**, as
|
||||||
every hive sees it: `serverName`, `gatewayHost`, `allowEncryption`.
|
every hive sees it: `serverName`, `gatewayHost`.
|
||||||
- `services.hyperhive.deploy.matrix.*` — what the host running it decides:
|
- `services.hyperhive.deploy.matrix.*` — what the host running it decides:
|
||||||
`enable`, `gui.package`, `trustedServers`,
|
`enable`, `gui.package`, `trustedServers`,
|
||||||
`maxRequestSize`, `sso.clientSecretFile`.
|
`maxRequestSize`, `sso.clientSecretFile`.
|
||||||
|
|
@ -293,7 +293,7 @@ Initial rollout settings:
|
||||||
tuwunel refuse to start unless
|
tuwunel refuse to start unless
|
||||||
`yes_i_am_very_very_sure_…_open_registration_…` is also set.
|
`yes_i_am_very_very_sure_…_open_registration_…` is also set.
|
||||||
- `allow_encryption` — server-side E2EE switch, sourced from
|
- `allow_encryption` — server-side E2EE switch, sourced from
|
||||||
`services.hyperhive.swarm.matrix.allowEncryption` (**default `false`**, opt-in).
|
`services.hyperhive.deploy.matrix.allowEncryption` (**default `false`**, opt-in).
|
||||||
Off by default because on the swarm's own homeserver the operator
|
Off by default because on the swarm's own homeserver the operator
|
||||||
already controls the transport; turn it on for encrypted rooms on
|
already controls the transport; turn it on for encrypted rooms on
|
||||||
external / federated homeservers or to keep contents opaque to the
|
external / federated homeservers or to keep contents opaque to the
|
||||||
|
|
|
||||||
|
|
@ -151,12 +151,10 @@ Only the swarm's forge host runs it (`services.hyperhive.deploy.forgejo.enable`,
|
||||||
### Network and port configuration
|
### Network and port configuration
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
services.hyperhive.swarm.forge = {
|
# The SSH port is swarm-wide: every hive dials it. The HTTP port and whether
|
||||||
httpPort = 3000; # default — outside hyperhive's 7000 / 8100-8999
|
# THIS host opens them in its firewall are this host's deployment decisions.
|
||||||
sshPort = 2222; # default — git-over-SSH, off the host's openssh on 22
|
services.hyperhive.swarm.forge.sshPort = 2222; # default — git-over-SSH, off the host's openssh on 22
|
||||||
};
|
services.hyperhive.deploy.forgejo.httpPort = 3000; # default — outside hyperhive's 7000 / 8100-8999
|
||||||
# Which ports the forge answers on is swarm-wide; whether THIS host opens
|
|
||||||
# them in its firewall is a deployment decision, so it lives under deploy.*.
|
|
||||||
services.hyperhive.deploy.forgejo.openFirewall = false; # default
|
services.hyperhive.deploy.forgejo.openFirewall = false; # default
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ Where it runs and how it's enabled: [`services.md`](services.md#builder).
|
||||||
3. The builder runs `nix build` of `nix/builder/check-flake`'s `check` output
|
3. The builder runs `nix build` of `nix/builder/check-flake`'s `check` output
|
||||||
with every input overridden:
|
with every input overridden:
|
||||||
- `hyperhive` and `nixpkgs`: the sources the builder's own host was
|
- `hyperhive` and `nixpkgs`: the sources the builder's own host was
|
||||||
deployed with (`c0re.hyperhiveFlake`, `c0re.nixpkgsFlake`). A hive's meta
|
deployed with (`c0re.hyperhiveFlake`, `deploy.builder.nixpkgsFlake`). A hive's meta
|
||||||
flake pins `hyperhive` by forge URL and rev instead.
|
flake pins `hyperhive` by forge URL and rev instead.
|
||||||
- `agent`: `git+https://<forge>/agent-configs/<agent>?ref=refs/pull/<pr>/head&rev=<head>`.
|
- `agent`: `git+https://<forge>/agent-configs/<agent>?ref=refs/pull/<pr>/head&rev=<head>`.
|
||||||
A build has one hour.
|
A build has one hour.
|
||||||
|
|
|
||||||
|
|
@ -310,7 +310,7 @@ swarm.
|
||||||
| `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. |
|
| `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. |
|
||||||
| `deploy.victorialogs.grafanaAuthPackage` | The `swarm-metrics-auth` build that runs as `swarm-logs-grafana-auth`. Set on its own: `deploy.victoriametrics.authPackage` doesn't reach it. |
|
| `deploy.victorialogs.grafanaAuthPackage` | The `swarm-metrics-auth` build that runs as `swarm-logs-grafana-auth`. Set on its own: `deploy.victoriametrics.authPackage` doesn't reach it. |
|
||||||
| `swarm.victorialogs.domain` | Only to rename it. |
|
| `swarm.victorialogs.domain` | Only to rename it. |
|
||||||
| `swarm.victorialogs.port` | Only if something else on the services host already claims `9428`. |
|
| `deploy.victorialogs.port` | Only if something else on the services host already claims `9428`. |
|
||||||
|
|
||||||
It binds loopback and takes no credential of its own: the gateway vhost is
|
It binds loopback and takes no credential of its own: the gateway vhost is
|
||||||
the only way in, and the collector is the only intended writer.
|
the only way in, and the collector is the only intended writer.
|
||||||
|
|
|
||||||
|
|
@ -259,6 +259,52 @@ in
|
||||||
[ "services" "hyperhive" "deploy" "hive-controller" "otel" "extraResourceAttributes" ]
|
[ "services" "hyperhive" "deploy" "hive-controller" "otel" "extraResourceAttributes" ]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Per-service settings of the host that runs the service: they sit with the
|
||||||
|
# service's other deployment options, not under the swarm facts every host shares.
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "controller" "queue" "natsUrl" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "swarm-controller" "queue" "natsUrl" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "controller" "queue" "tokenEndpoint" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "swarm-controller" "queue" "tokenEndpoint" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "bao" "otel" "telemetryPort" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "bao" "otel" "telemetryPort" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "nats" "monitorPort" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "nats" "monitorPort" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "nats" "metricsPort" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "nats" "metricsPort" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "nats" "otel" "telemetryPort" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "nats" "otel" "telemetryPort" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "victorialogs" "port" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "victorialogs" "port" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "swarm" "matrix" "allowEncryption" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "matrix" "allowEncryption" ]
|
||||||
|
)
|
||||||
|
(lib.mkRenamedOptionModule
|
||||||
|
[ "services" "hyperhive" "c0re" "nixpkgsFlake" ]
|
||||||
|
[ "services" "hyperhive" "deploy" "builder" "nixpkgsFlake" ]
|
||||||
|
)
|
||||||
# The homeserver's port and its firewall switch, removed rather than
|
# The homeserver's port and its firewall switch, removed rather than
|
||||||
# renamed: tuwunel listens on a unix socket only the gateway reaches, so
|
# renamed: tuwunel listens on a unix socket only the gateway reaches, so
|
||||||
# there is no host port to choose or open.
|
# there is no host port to choose or open.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Option declarations for the c0re daemon. `services.hyperhive.c0re.*`
|
# Option declarations for the c0re daemon. `services.hyperhive.c0re.*`
|
||||||
# keeps the two flake-source options the swarm builder also reads
|
# keeps the flake-source option the swarm builder also reads
|
||||||
# (`hyperhiveFlake`, `nixpkgsFlake`); every other knob and the
|
# (`hyperhiveFlake`); every other knob and the
|
||||||
# package/asset options the flake's `nixosModules.default` wires to its
|
# package/asset options the flake's `nixosModules.default` wires to its
|
||||||
# own outputs (they carry no in-module defaults; see ../../../flake.nix)
|
# own outputs (they carry no in-module defaults; see ../../../flake.nix)
|
||||||
# live under `services.hyperhive.deploy.hive-controller`, alongside the
|
# live under `services.hyperhive.deploy.hive-controller`, alongside the
|
||||||
|
|
@ -8,7 +8,6 @@
|
||||||
# `servedFrontend` option lives in ./theme.nix with the stylix wiring
|
# `servedFrontend` option lives in ./theme.nix with the stylix wiring
|
||||||
# that computes it.
|
# that computes it.
|
||||||
{
|
{
|
||||||
pkgs,
|
|
||||||
lib,
|
lib,
|
||||||
...
|
...
|
||||||
}:
|
}:
|
||||||
|
|
@ -34,19 +33,6 @@
|
||||||
filtered source.
|
filtered source.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
nixpkgsFlake = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = "path:${pkgs.path}";
|
|
||||||
defaultText = lib.literalMD "`\"path:\${pkgs.path}\"`";
|
|
||||||
description = ''
|
|
||||||
Store-path URL of the nixpkgs the swarm builder's check flake
|
|
||||||
builds on. The swarm meta flake does not use it: its `nixpkgs`
|
|
||||||
follows `hyperhive/nixpkgs`.
|
|
||||||
|
|
||||||
Defaults to `"path:''${pkgs.path}"`, the nixpkgs the host NixOS
|
|
||||||
module was evaluated with.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# The hive-controller's deployment options.
|
# The hive-controller's deployment options.
|
||||||
|
|
|
||||||
|
|
@ -179,6 +179,17 @@ in
|
||||||
# where its host-local secrets sit. Same rule as ../swarm-victorialogs.nix,
|
# where its host-local secrets sit. Same rule as ../swarm-victorialogs.nix,
|
||||||
# and the renames are in ../deploy.nix with the rest.
|
# and the renames are in ../deploy.nix with the rest.
|
||||||
options.services.hyperhive.deploy.forgejo = {
|
options.services.hyperhive.deploy.forgejo = {
|
||||||
|
httpPort = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 3000;
|
||||||
|
description = ''
|
||||||
|
TCP port the forge serves HTTP on. Default 3000 sits outside
|
||||||
|
hyperhive's claimed ranges (dashboard 7000, every agent in
|
||||||
|
8100..8999 via FNV-1a hash). Change this if you already have
|
||||||
|
another forgejo bound to 3000.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
package = lib.mkOption {
|
package = lib.mkOption {
|
||||||
type = lib.types.package;
|
type = lib.types.package;
|
||||||
default = pkgs.forgejo;
|
default = pkgs.forgejo;
|
||||||
|
|
@ -304,7 +315,7 @@ in
|
||||||
"${cfg.domain}" = {
|
"${cfg.domain}" = {
|
||||||
locations = {
|
locations = {
|
||||||
"/" = {
|
"/" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/";
|
proxyPass = "http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/";
|
||||||
proxyWebsockets = true;
|
proxyWebsockets = true;
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
|
|
@ -325,7 +336,7 @@ in
|
||||||
# `publishedScrapeTargets.forgejo` in ../swarm-otel-service.nix;
|
# `publishedScrapeTargets.forgejo` in ../swarm-otel-service.nix;
|
||||||
# the two must agree, since that string is also the token's audience.
|
# the two must agree, since that string is also the token's audience.
|
||||||
"= /metrics" = {
|
"= /metrics" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString cfg.httpPort}/metrics";
|
proxyPass = "http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/metrics";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
auth_request /__forge_metrics_authz;
|
auth_request /__forge_metrics_authz;
|
||||||
'';
|
'';
|
||||||
|
|
@ -453,8 +464,8 @@ in
|
||||||
assertion =
|
assertion =
|
||||||
!(lib.elem forwarderTelemetryPort [
|
!(lib.elem forwarderTelemetryPort [
|
||||||
config.services.hyperhive.otel.telemetryPort
|
config.services.hyperhive.otel.telemetryPort
|
||||||
config.services.hyperhive.swarm.bao.otel.telemetryPort
|
config.services.hyperhive.deploy.bao.otel.telemetryPort
|
||||||
config.services.hyperhive.swarm.nats.otel.telemetryPort
|
config.services.hyperhive.deploy.nats.otel.telemetryPort
|
||||||
]);
|
]);
|
||||||
message = ''
|
message = ''
|
||||||
The forge's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
The forge's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
||||||
|
|
@ -608,7 +619,7 @@ in
|
||||||
server = {
|
server = {
|
||||||
DOMAIN = cfg.domain;
|
DOMAIN = cfg.domain;
|
||||||
ROOT_URL = effectiveRootUrl;
|
ROOT_URL = effectiveRootUrl;
|
||||||
HTTP_PORT = cfg.httpPort;
|
HTTP_PORT = deployCfg.forgejo.httpPort;
|
||||||
START_SSH_SERVER = true;
|
START_SSH_SERVER = true;
|
||||||
SSH_PORT = cfg.sshPort;
|
SSH_PORT = cfg.sshPort;
|
||||||
SSH_LISTEN_PORT = cfg.sshPort;
|
SSH_LISTEN_PORT = cfg.sshPort;
|
||||||
|
|
@ -1135,7 +1146,7 @@ in
|
||||||
curl --silent --show-error --connect-timeout 5 --max-time 15 \
|
curl --silent --show-error --connect-timeout 5 --max-time 15 \
|
||||||
-H @<(printf 'Authorization: token %s\n' "$token") \
|
-H @<(printf 'Authorization: token %s\n' "$token") \
|
||||||
-X "$method" -o "$body" -w '%{http_code}' \
|
-X "$method" -o "$body" -w '%{http_code}' \
|
||||||
"http://127.0.0.1:${toString cfg.httpPort}/api/v1$path"
|
"http://127.0.0.1:${toString deployCfg.forgejo.httpPort}/api/v1$path"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Every other token this unit minted: one whose write to the
|
# Every other token this unit minted: one whose write to the
|
||||||
|
|
@ -1300,7 +1311,7 @@ in
|
||||||
|
|
||||||
networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
|
networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
|
||||||
allowedTCPPorts = [
|
allowedTCPPorts = [
|
||||||
cfg.httpPort
|
deployCfg.forgejo.httpPort
|
||||||
cfg.sshPort
|
cfg.sshPort
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -30,17 +30,6 @@ in
|
||||||
# only one host runs it: `deploy.forgejo.enable`, in ../deploy.nix. What
|
# only one host runs it: `deploy.forgejo.enable`, in ../deploy.nix. What
|
||||||
# follows is what every hive needs to reach it, wherever it runs.
|
# follows is what every hive needs to reach it, wherever it runs.
|
||||||
options.services.hyperhive.swarm.forge = {
|
options.services.hyperhive.swarm.forge = {
|
||||||
httpPort = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 3000;
|
|
||||||
description = ''
|
|
||||||
TCP port the forge serves HTTP on. Default 3000 sits outside
|
|
||||||
hyperhive's claimed ranges (dashboard 7000, every agent in
|
|
||||||
8100..8999 via FNV-1a hash). Change this if you already have
|
|
||||||
another forgejo bound to 3000.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
sshPort = lib.mkOption {
|
sshPort = lib.mkOption {
|
||||||
type = lib.types.port;
|
type = lib.types.port;
|
||||||
default = 2222;
|
default = 2222;
|
||||||
|
|
|
||||||
|
|
@ -90,25 +90,6 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
allowEncryption = lib.mkOption {
|
|
||||||
type = lib.types.bool;
|
|
||||||
default = false;
|
|
||||||
description = ''
|
|
||||||
Server-side switch for matrix end-to-end encryption — sets
|
|
||||||
tuwunel's `allow_encryption`. Off by default: on the hive-internal
|
|
||||||
homeserver the operator already controls the transport, so server
|
|
||||||
E2EE adds key-management overhead (cross-signing, device
|
|
||||||
verification, undecryptable-message recovery) without a clear
|
|
||||||
threat-model win for the common single-hive case. Turn on when
|
|
||||||
agents join encrypted rooms on external / federated homeservers,
|
|
||||||
or when the operator wants message contents opaque to the
|
|
||||||
homeserver admin. Independent of the agent matrix client, which
|
|
||||||
always supports decryption so it can read encrypted rooms it is
|
|
||||||
invited to regardless of this flag; this option only governs
|
|
||||||
whether THIS homeserver permits room encryption.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
adminSync.ignoreUsers = lib.mkOption {
|
adminSync.ignoreUsers = lib.mkOption {
|
||||||
type = lib.types.listOf (lib.types.strMatching "@[^:[:space:]]+:[^[:space:]]+");
|
type = lib.types.listOf (lib.types.strMatching "@[^:[:space:]]+:[^[:space:]]+");
|
||||||
default = [ ];
|
default = [ ];
|
||||||
|
|
|
||||||
|
|
@ -290,6 +290,25 @@ in
|
||||||
# how large a request it accepts, and where its host-local secrets sit. Same rule as
|
# how large a request it accepts, and where its host-local secrets sit. Same rule as
|
||||||
# ./swarm-victorialogs.nix; `enable` lives in ./deploy.nix.
|
# ./swarm-victorialogs.nix; `enable` lives in ./deploy.nix.
|
||||||
options.services.hyperhive.deploy.matrix = {
|
options.services.hyperhive.deploy.matrix = {
|
||||||
|
allowEncryption = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = ''
|
||||||
|
Server-side switch for matrix end-to-end encryption — sets
|
||||||
|
tuwunel's `allow_encryption`. Off by default: on the hive-internal
|
||||||
|
homeserver the operator already controls the transport, so server
|
||||||
|
E2EE adds key-management overhead (cross-signing, device
|
||||||
|
verification, undecryptable-message recovery) without a clear
|
||||||
|
threat-model win for the common single-hive case. Turn on when
|
||||||
|
agents join encrypted rooms on external / federated homeservers,
|
||||||
|
or when the operator wants message contents opaque to the
|
||||||
|
homeserver admin. Independent of the agent matrix client, which
|
||||||
|
always supports decryption so it can read encrypted rooms it is
|
||||||
|
invited to regardless of this flag; this option only governs
|
||||||
|
whether THIS homeserver permits room encryption.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
package = lib.mkOption {
|
package = lib.mkOption {
|
||||||
type = lib.types.package;
|
type = lib.types.package;
|
||||||
default = pkgs.matrix-tuwunel;
|
default = pkgs.matrix-tuwunel;
|
||||||
|
|
@ -933,7 +952,7 @@ in
|
||||||
// {
|
// {
|
||||||
# Server-side E2EE is opt-in (default off); the agent matrix
|
# Server-side E2EE is opt-in (default off); the agent matrix
|
||||||
# client always supports decryption regardless.
|
# client always supports decryption regardless.
|
||||||
allow_encryption = cfg.allowEncryption;
|
allow_encryption = deployCfg.matrix.allowEncryption;
|
||||||
# Tuwunel's default suffix is " 💕" — suppress it so agent
|
# Tuwunel's default suffix is " 💕" — suppress it so agent
|
||||||
# display names are clean (just the agent name, no emoji).
|
# display names are clean (just the agent name, no emoji).
|
||||||
new_user_displayname_suffix = "";
|
new_user_displayname_suffix = "";
|
||||||
|
|
|
||||||
|
|
@ -126,26 +126,5 @@ in
|
||||||
Two spellings present as a healthy-looking 401.
|
Two spellings present as a healthy-looking 401.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
otel.telemetryPort = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 8890;
|
|
||||||
description = ''
|
|
||||||
Port the collector inside the store's container serves its **own**
|
|
||||||
metrics on — queue depth, refused and dropped samples, exporter
|
|
||||||
failures. How you find out that telemetry is being lost, so it is
|
|
||||||
worth keeping rather than switching off.
|
|
||||||
|
|
||||||
⚠️ **Deliberately not 8888.** 8888 is the collector
|
|
||||||
binary's built-in default, which the hive tier
|
|
||||||
({option}`services.hyperhive.otel.telemetryPort`) already binds. This
|
|
||||||
container runs with `privateNetwork = false`, so both share the
|
|
||||||
host's network namespace whenever they are co-located — and unlike
|
|
||||||
the OTLP ports this one appears nowhere in either config when it is
|
|
||||||
left undeclared, so nothing that compares configured ports can see
|
|
||||||
the clash. The second collector to start simply dies with
|
|
||||||
`bind: address already in use`.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1604,7 +1604,7 @@ let
|
||||||
baoDeploy.uiPort
|
baoDeploy.uiPort
|
||||||
(baoDeploy.uiPort + 1)
|
(baoDeploy.uiPort + 1)
|
||||||
baoDeploy.uiProxyPort
|
baoDeploy.uiProxyPort
|
||||||
cfg.otel.telemetryPort
|
baoDeploy.otel.telemetryPort
|
||||||
];
|
];
|
||||||
|
|
||||||
# Paths a browser session has no business sending to the store: they unseal,
|
# Paths a browser session has no business sending to the store: they unseal,
|
||||||
|
|
@ -2772,6 +2772,27 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
otel.telemetryPort = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 8890;
|
||||||
|
description = ''
|
||||||
|
Port the collector inside the store's container serves its **own**
|
||||||
|
metrics on — queue depth, refused and dropped samples, exporter
|
||||||
|
failures. How you find out that telemetry is being lost, so it is
|
||||||
|
worth keeping rather than switching off.
|
||||||
|
|
||||||
|
⚠️ **Deliberately not 8888.** 8888 is the collector
|
||||||
|
binary's built-in default, which the hive tier
|
||||||
|
({option}`services.hyperhive.otel.telemetryPort`) already binds. This
|
||||||
|
container runs with `privateNetwork = false`, so both share the
|
||||||
|
host's network namespace whenever they are co-located — and unlike
|
||||||
|
the OTLP ports this one appears nowhere in either config when it is
|
||||||
|
left undeclared, so nothing that compares configured ports can see
|
||||||
|
the clash. The second collector to start simply dies with
|
||||||
|
`bind: address already in use`.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
metricsPort = lib.mkOption {
|
metricsPort = lib.mkOption {
|
||||||
type = lib.types.port;
|
type = lib.types.port;
|
||||||
default = 8202;
|
default = 8202;
|
||||||
|
|
@ -2879,7 +2900,7 @@ in
|
||||||
services.hyperhive.swarm.bao.port and +1, deploy.bao.metricsPort and
|
services.hyperhive.swarm.bao.port and +1, deploy.bao.metricsPort and
|
||||||
+1, deploy.bao.uiPort and +1 (openbao binds each listener's port + 1
|
+1, deploy.bao.uiPort and +1 (openbao binds each listener's port + 1
|
||||||
as its cluster address), deploy.bao.uiProxyPort, and
|
as its cluster address), deploy.bao.uiProxyPort, and
|
||||||
swarm.bao.otel.telemetryPort. Pick free ports.
|
deploy.bao.otel.telemetryPort. Pick free ports.
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
|
|
@ -4603,7 +4624,7 @@ in
|
||||||
# holds: this container shares the host's netns.
|
# holds: this container shares the host's netns.
|
||||||
(import ../collector-self-scrape.nix {
|
(import ../collector-self-scrape.nix {
|
||||||
inherit lib;
|
inherit lib;
|
||||||
port = cfg.otel.telemetryPort;
|
port = baoDeploy.otel.telemetryPort;
|
||||||
exporter = "otlphttp";
|
exporter = "otlphttp";
|
||||||
attributes.container = collectorServiceName;
|
attributes.container = collectorServiceName;
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -90,7 +90,7 @@ let
|
||||||
|
|
||||||
# Strings, not paths: both are already store paths.
|
# Strings, not paths: both are already store paths.
|
||||||
hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
|
hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
|
||||||
nixpkgsFlake = hyperhiveCfg.c0re.nixpkgsFlake;
|
nixpkgsFlake = hyperhiveCfg.deploy.builder.nixpkgsFlake;
|
||||||
|
|
||||||
# The assertion below keeps the two options together.
|
# The assertion below keeps the two options together.
|
||||||
pushesToCache = atticCfg.domain != null;
|
pushesToCache = atticCfg.domain != null;
|
||||||
|
|
@ -110,6 +110,20 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
options.services.hyperhive.deploy.builder.nixpkgsFlake = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "path:${pkgs.path}";
|
||||||
|
defaultText = lib.literalMD "`\"path:\${pkgs.path}\"`";
|
||||||
|
description = ''
|
||||||
|
Store-path URL of the nixpkgs the swarm builder's check flake
|
||||||
|
builds on. The swarm meta flake does not use it: its `nixpkgs`
|
||||||
|
follows `hyperhive/nixpkgs`.
|
||||||
|
|
||||||
|
Defaults to `"path:''${pkgs.path}"`, the nixpkgs the host NixOS
|
||||||
|
module was evaluated with.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
options.services.hyperhive.deploy.builder.package = lib.mkOption {
|
options.services.hyperhive.deploy.builder.package = lib.mkOption {
|
||||||
type = lib.types.package;
|
type = lib.types.package;
|
||||||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-builder";
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-builder";
|
||||||
|
|
|
||||||
|
|
@ -193,8 +193,8 @@ let
|
||||||
# didn't need to; the assertions cover the split case so nobody gets a
|
# didn't need to; the assertions cover the split case so nobody gets a
|
||||||
# daemon that silently never connects.
|
# daemon that silently never connects.
|
||||||
queueEnv = {
|
queueEnv = {
|
||||||
SWARM_CONTROLLER_NATS_URL = cfg.queue.natsUrl;
|
SWARM_CONTROLLER_NATS_URL = deployCfg.swarm-controller.queue.natsUrl;
|
||||||
SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = cfg.queue.tokenEndpoint;
|
SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = deployCfg.swarm-controller.queue.tokenEndpoint;
|
||||||
SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
|
SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -391,39 +391,6 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
queue = {
|
|
||||||
natsUrl = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
|
||||||
defaultText = lib.literalExpression ''"tls://''${swarm.nats.domain}:''${swarm.nats.port}"'';
|
|
||||||
example = "tls://nats.example.com:4222";
|
|
||||||
description = ''
|
|
||||||
Where the controller reaches the swarm queue.
|
|
||||||
|
|
||||||
Defaults to the queue's name, which is the same URL on every
|
|
||||||
host: the queue's own host resolves it locally, and any other
|
|
||||||
through the operator's DNS.
|
|
||||||
|
|
||||||
Empty means unset, which the assertion below refuses — a
|
|
||||||
controller with no queue is not a lighter controller.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
tokenEndpoint = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = "${autheliaCfg.url}/api/oidc/token";
|
|
||||||
defaultText = lib.literalExpression ''"''${swarm.authelia.url}/api/oidc/token"'';
|
|
||||||
description = ''
|
|
||||||
The OIDC token endpoint the controller mints its own access
|
|
||||||
token from.
|
|
||||||
|
|
||||||
Derived from `swarm.authelia.url`, which is the half of the
|
|
||||||
authelia module that exists on every host — so this is already
|
|
||||||
correct for a remote provider as long as that URL is set.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
links = lib.mkOption {
|
links = lib.mkOption {
|
||||||
type = lib.types.listOf (
|
type = lib.types.listOf (
|
||||||
lib.types.submodule {
|
lib.types.submodule {
|
||||||
|
|
@ -483,6 +450,39 @@ in
|
||||||
# the latter is hive-c0re's namespace, where `tls.*` lives. A definition on
|
# the latter is hive-c0re's namespace, where `tls.*` lives. A definition on
|
||||||
# the wrong one lands on a live, unrelated option rather than failing.
|
# the wrong one lands on a live, unrelated option rather than failing.
|
||||||
options.services.hyperhive.deploy.swarm-controller = {
|
options.services.hyperhive.deploy.swarm-controller = {
|
||||||
|
queue = {
|
||||||
|
natsUrl = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
||||||
|
defaultText = lib.literalExpression ''"tls://''${swarm.nats.domain}:''${swarm.nats.port}"'';
|
||||||
|
example = "tls://nats.example.com:4222";
|
||||||
|
description = ''
|
||||||
|
Where the controller reaches the swarm queue.
|
||||||
|
|
||||||
|
Defaults to the queue's name, which is the same URL on every
|
||||||
|
host: the queue's own host resolves it locally, and any other
|
||||||
|
through the operator's DNS.
|
||||||
|
|
||||||
|
Empty means unset, which the assertion below refuses — a
|
||||||
|
controller with no queue is not a lighter controller.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
tokenEndpoint = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "${autheliaCfg.url}/api/oidc/token";
|
||||||
|
defaultText = lib.literalExpression ''"''${swarm.authelia.url}/api/oidc/token"'';
|
||||||
|
description = ''
|
||||||
|
The OIDC token endpoint the controller mints its own access
|
||||||
|
token from.
|
||||||
|
|
||||||
|
Derived from `swarm.authelia.url`, which is the half of the
|
||||||
|
authelia module that exists on every host — so this is already
|
||||||
|
correct for a remote provider as long as that URL is set.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
package = lib.mkOption {
|
package = lib.mkOption {
|
||||||
type = lib.types.package;
|
type = lib.types.package;
|
||||||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
|
||||||
|
|
@ -558,7 +558,7 @@ in
|
||||||
|
|
||||||
No new credential to configure: the bearer token presented to
|
No new credential to configure: the bearer token presented to
|
||||||
the bridge is minted from THIS daemon's own existing queue OIDC
|
the bridge is minted from THIS daemon's own existing queue OIDC
|
||||||
identity — its endpoints stay in `swarm.controller.queue`, its secret is
|
identity — its endpoints stay in `deploy.swarm-controller.queue`, its secret is
|
||||||
read from the swarm's secret store — "one identity per principal"
|
read from the swarm's secret store — "one identity per principal"
|
||||||
already covers it. `null` means no agent-identity support:
|
already covers it. `null` means no agent-identity support:
|
||||||
`CreateIdentity` jobs fail with a clear "no auth bridge
|
`CreateIdentity` jobs fail with a clear "no auth bridge
|
||||||
|
|
@ -696,18 +696,18 @@ in
|
||||||
# because on an all-local deployment nothing was wrong.
|
# because on an all-local deployment nothing was wrong.
|
||||||
assertions = [
|
assertions = [
|
||||||
{
|
{
|
||||||
assertion = cfg.queue.natsUrl != "";
|
assertion = deployCfg.swarm-controller.queue.natsUrl != "";
|
||||||
message = ''
|
message = ''
|
||||||
services.hyperhive.swarm.controller.queue.natsUrl is unset.
|
services.hyperhive.deploy.swarm-controller.queue.natsUrl is unset.
|
||||||
|
|
||||||
It defaults to the queue's name; something in this configuration
|
It defaults to the queue's name; something in this configuration
|
||||||
set it to "". Remove that setting, or set the URL.
|
set it to "". Remove that setting, or set the URL.
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
assertion = cfg.queue.tokenEndpoint != "";
|
assertion = deployCfg.swarm-controller.queue.tokenEndpoint != "";
|
||||||
message = ''
|
message = ''
|
||||||
services.hyperhive.swarm.controller.queue.tokenEndpoint is unset.
|
services.hyperhive.deploy.swarm-controller.queue.tokenEndpoint is unset.
|
||||||
|
|
||||||
It defaults from services.hyperhive.swarm.authelia.url, so this
|
It defaults from services.hyperhive.swarm.authelia.url, so this
|
||||||
only happens when it has been cleared by hand. The controller
|
only happens when it has been cleared by hand. The controller
|
||||||
|
|
|
||||||
|
|
@ -60,69 +60,13 @@ in
|
||||||
`wg-hive` when this host is on the mesh, for other hives. Never
|
`wg-hive` when this host is on the mesh, for other hives. Never
|
||||||
host-wide. TLS only: a client that does not speak it is refused.
|
host-wide. TLS only: a client that does not speak it is refused.
|
||||||
|
|
||||||
Unlike {option}`monitorPort` and {option}`metricsPort`, the
|
Unlike {option}`services.hyperhive.deploy.nats.monitorPort` and
|
||||||
|
{option}`services.hyperhive.deploy.nats.metricsPort`, the
|
||||||
address is not what bounds who may use this port: the queue's
|
address is not what bounds who may use this port: the queue's
|
||||||
`auth_callout` refuses every client it cannot identify.
|
`auth_callout` refuses every client it cannot identify.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
monitorPort = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 8222;
|
|
||||||
description = ''
|
|
||||||
Port NATS serves its **monitoring** endpoint on, bound to
|
|
||||||
loopback.
|
|
||||||
|
|
||||||
Not a metrics endpoint: the server has no Prometheus format of its
|
|
||||||
own. This serves `/varz`, `/connz`, `/routez` as JSON, and the
|
|
||||||
exporter below is what translates it — which is why enabling the
|
|
||||||
exporter without this produces a process that starts cleanly and
|
|
||||||
scrapes nothing.
|
|
||||||
|
|
||||||
⚠️ Loopback, and the exporter is the only intended reader. The
|
|
||||||
endpoint is unauthenticated and `/connz` names every connected
|
|
||||||
client, so the address it binds is the whole access control. Do
|
|
||||||
not widen it, and do not put it behind a gateway vhost expecting
|
|
||||||
that to add one.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
metricsPort = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 7777;
|
|
||||||
description = ''
|
|
||||||
Port the Prometheus exporter serves NATS's metrics on, bound to
|
|
||||||
loopback. The gateway's `= /metrics` location on
|
|
||||||
{option}`services.hyperhive.swarm.nats.domain` proxies it, behind
|
|
||||||
the swarm collector's authelia check.
|
|
||||||
|
|
||||||
⚠️ This and {option}`monitorPort` are two more claims on a port
|
|
||||||
space every swarm container shares — they run in this container but
|
|
||||||
`privateNetwork = false`, so a collision with any other hyperhive
|
|
||||||
service is a runtime coin toss over which process gets the port,
|
|
||||||
with nothing in any log saying so. Both defaults are upstream's
|
|
||||||
own (`nats-server` 8222, `prometheus-nats-exporter` 7777) and
|
|
||||||
neither is claimed elsewhere in this repo, checked when they were
|
|
||||||
added.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
otel.telemetryPort = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 8891;
|
|
||||||
description = ''
|
|
||||||
Loopback port the journal forwarder inside the queue's container
|
|
||||||
serves its own metrics on (refused and dropped records, exporter
|
|
||||||
failures).
|
|
||||||
|
|
||||||
⚠️ Not 8888 or 8890: the hive collector and the store's forwarder
|
|
||||||
bind those ({option}`services.hyperhive.otel.telemetryPort`,
|
|
||||||
{option}`services.hyperhive.swarm.bao.otel.telemetryPort`), and this
|
|
||||||
container shares the host netns with them when co-located. The second
|
|
||||||
collector to bind a port dies with `bind: address already in use`.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
clientId = lib.mkOption {
|
clientId = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
default = "swarm-nats";
|
default = "swarm-nats";
|
||||||
|
|
|
||||||
|
|
@ -364,6 +364,64 @@ in
|
||||||
# ./deploy.nix, which also carries the renames. The callout's keys are no
|
# ./deploy.nix, which also carries the renames. The callout's keys are no
|
||||||
# option at all: they live in the secret store (docs/swarm/credentials.md).
|
# option at all: they live in the secret store (docs/swarm/credentials.md).
|
||||||
options.services.hyperhive.deploy.nats = {
|
options.services.hyperhive.deploy.nats = {
|
||||||
|
monitorPort = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 8222;
|
||||||
|
description = ''
|
||||||
|
Port NATS serves its **monitoring** endpoint on, bound to
|
||||||
|
loopback.
|
||||||
|
|
||||||
|
Not a metrics endpoint: the server has no Prometheus format of its
|
||||||
|
own. This serves `/varz`, `/connz`, `/routez` as JSON, and the
|
||||||
|
exporter below is what translates it — which is why enabling the
|
||||||
|
exporter without this produces a process that starts cleanly and
|
||||||
|
scrapes nothing.
|
||||||
|
|
||||||
|
⚠️ Loopback, and the exporter is the only intended reader. The
|
||||||
|
endpoint is unauthenticated and `/connz` names every connected
|
||||||
|
client, so the address it binds is the whole access control. Do
|
||||||
|
not widen it, and do not put it behind a gateway vhost expecting
|
||||||
|
that to add one.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
metricsPort = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 7777;
|
||||||
|
description = ''
|
||||||
|
Port the Prometheus exporter serves NATS's metrics on, bound to
|
||||||
|
loopback. The gateway's `= /metrics` location on
|
||||||
|
{option}`services.hyperhive.swarm.nats.domain` proxies it, behind
|
||||||
|
the swarm collector's authelia check.
|
||||||
|
|
||||||
|
⚠️ This and {option}`services.hyperhive.deploy.nats.monitorPort` are two
|
||||||
|
more claims on a port
|
||||||
|
space every swarm container shares — they run in this container but
|
||||||
|
`privateNetwork = false`, so a collision with any other hyperhive
|
||||||
|
service is a runtime coin toss over which process gets the port,
|
||||||
|
with nothing in any log saying so. Both defaults are upstream's
|
||||||
|
own (`nats-server` 8222, `prometheus-nats-exporter` 7777) and
|
||||||
|
neither is claimed elsewhere in this repo, checked when they were
|
||||||
|
added.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
otel.telemetryPort = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 8891;
|
||||||
|
description = ''
|
||||||
|
Loopback port the journal forwarder inside the queue's container
|
||||||
|
serves its own metrics on (refused and dropped records, exporter
|
||||||
|
failures).
|
||||||
|
|
||||||
|
⚠️ Not 8888 or 8890: the hive collector and the store's forwarder
|
||||||
|
bind those ({option}`services.hyperhive.otel.telemetryPort`,
|
||||||
|
{option}`services.hyperhive.deploy.bao.otel.telemetryPort`), and this
|
||||||
|
container shares the host netns with them when co-located. The second
|
||||||
|
collector to bind a port dies with `bind: address already in use`.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
authPackage = lib.mkOption {
|
authPackage = lib.mkOption {
|
||||||
type = lib.types.package;
|
type = lib.types.package;
|
||||||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth";
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth";
|
||||||
|
|
@ -485,7 +543,7 @@ in
|
||||||
# Authelia's rules for it admit the collector's client and deny
|
# Authelia's rules for it admit the collector's client and deny
|
||||||
# every other subject (`metricsRules` in ./swarm-authelia.nix).
|
# every other subject (`metricsRules` in ./swarm-authelia.nix).
|
||||||
"= /metrics" = {
|
"= /metrics" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics";
|
proxyPass = "http://127.0.0.1:${toString deployCfg.nats.metricsPort}/metrics";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
auth_request /__nats_metrics_authz;
|
auth_request /__nats_metrics_authz;
|
||||||
'';
|
'';
|
||||||
|
|
@ -597,17 +655,17 @@ in
|
||||||
let
|
let
|
||||||
ports = [
|
ports = [
|
||||||
cfg.port
|
cfg.port
|
||||||
cfg.monitorPort
|
deployCfg.nats.monitorPort
|
||||||
cfg.metricsPort
|
deployCfg.nats.metricsPort
|
||||||
cfg.otel.telemetryPort
|
deployCfg.nats.otel.telemetryPort
|
||||||
config.services.hyperhive.otel.telemetryPort
|
config.services.hyperhive.otel.telemetryPort
|
||||||
config.services.hyperhive.swarm.bao.otel.telemetryPort
|
deployCfg.bao.otel.telemetryPort
|
||||||
];
|
];
|
||||||
in
|
in
|
||||||
lib.length (lib.unique ports) == lib.length ports;
|
lib.length (lib.unique ports) == lib.length ports;
|
||||||
message = ''
|
message = ''
|
||||||
services.hyperhive.swarm.nats.otel.telemetryPort
|
services.hyperhive.deploy.nats.otel.telemetryPort
|
||||||
(${toString cfg.otel.telemetryPort}) collides with another port in
|
(${toString deployCfg.nats.otel.telemetryPort}) collides with another port in
|
||||||
the host netns: the queue's own ports, or the telemetry port of the
|
the host netns: the queue's own ports, or the telemetry port of the
|
||||||
hive collector or the store's forwarder. Pick
|
hive collector or the store's forwarder. Pick
|
||||||
a free port.
|
a free port.
|
||||||
|
|
@ -687,7 +745,7 @@ in
|
||||||
endpoint = forwarderEndpoint;
|
endpoint = forwarderEndpoint;
|
||||||
inherit (cfg) clientId;
|
inherit (cfg) clientId;
|
||||||
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
|
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
|
||||||
telemetryPort = cfg.otel.telemetryPort;
|
telemetryPort = deployCfg.nats.otel.telemetryPort;
|
||||||
store = {
|
store = {
|
||||||
package = baoDeploy.package;
|
package = baoDeploy.package;
|
||||||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||||
|
|
@ -749,7 +807,7 @@ in
|
||||||
#
|
#
|
||||||
# ⚠️ Loopback: unauthenticated, and `/connz` lists every
|
# ⚠️ Loopback: unauthenticated, and `/connz` lists every
|
||||||
# connected client.
|
# connected client.
|
||||||
http = "127.0.0.1:${toString cfg.monitorPort}";
|
http = "127.0.0.1:${toString deployCfg.nats.monitorPort}";
|
||||||
|
|
||||||
# TLS required on the client port: a server with a `tls`
|
# TLS required on the client port: a server with a `tls`
|
||||||
# block advertises `tls_required` and refuses a client that
|
# block advertises `tls_required` and refuses a client that
|
||||||
|
|
@ -782,8 +840,8 @@ in
|
||||||
services.prometheus.exporters.nats = {
|
services.prometheus.exporters.nats = {
|
||||||
enable = true;
|
enable = true;
|
||||||
listenAddress = "127.0.0.1";
|
listenAddress = "127.0.0.1";
|
||||||
port = cfg.metricsPort;
|
port = deployCfg.nats.metricsPort;
|
||||||
url = "http://127.0.0.1:${toString cfg.monitorPort}";
|
url = "http://127.0.0.1:${toString deployCfg.nats.monitorPort}";
|
||||||
|
|
||||||
# ⚠️ NOT optional, despite the name. Upstream renders
|
# ⚠️ NOT optional, despite the name. Upstream renders
|
||||||
# `-addr … -port … ${extraFlags} ${url}` and defaults
|
# `-addr … -port … ${extraFlags} ${url}` and defaults
|
||||||
|
|
|
||||||
|
|
@ -50,20 +50,6 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
telemetryPort = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 8889;
|
|
||||||
description = ''
|
|
||||||
Port this collector serves its **own** metrics on — queue depth,
|
|
||||||
refused and dropped samples, exporter failures. How you find out
|
|
||||||
that telemetry is being lost, so it is worth keeping rather than
|
|
||||||
switching off.
|
|
||||||
|
|
||||||
Bound on the container's own loopback, so no other collector
|
|
||||||
shares it; only this collector scrapes it.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
serviceProducers = lib.mkOption {
|
serviceProducers = lib.mkOption {
|
||||||
internal = true;
|
internal = true;
|
||||||
type = lib.types.attrsOf (
|
type = lib.types.attrsOf (
|
||||||
|
|
|
||||||
|
|
@ -333,6 +333,20 @@ in
|
||||||
# operator's choice for this deployment, so both hang off the deployment.
|
# operator's choice for this deployment, so both hang off the deployment.
|
||||||
# `enable` already lives in ./deploy.nix, which also carries the rename.
|
# `enable` already lives in ./deploy.nix, which also carries the rename.
|
||||||
options.services.hyperhive.deploy.swarm-otel = {
|
options.services.hyperhive.deploy.swarm-otel = {
|
||||||
|
telemetryPort = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 8889;
|
||||||
|
description = ''
|
||||||
|
Port this collector serves its **own** metrics on — queue depth,
|
||||||
|
refused and dropped samples, exporter failures. How you find out
|
||||||
|
that telemetry is being lost, so it is worth keeping rather than
|
||||||
|
switching off.
|
||||||
|
|
||||||
|
Bound on the container's own loopback, so no other collector
|
||||||
|
shares it; only this collector scrapes it.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
upstream = {
|
upstream = {
|
||||||
endpoint = lib.mkOption {
|
endpoint = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
|
|
@ -484,7 +498,8 @@ in
|
||||||
# alive, these counters say whether it is dropping what it receives.
|
# alive, these counters say whether it is dropping what it receives.
|
||||||
# Loopback is the container's own: this collector scrapes `telemetryPort`
|
# Loopback is the container's own: this collector scrapes `telemetryPort`
|
||||||
# inside its netns, and nothing else reads it.
|
# inside its netns, and nothing else reads it.
|
||||||
services.hyperhive.swarm.otel.scrapeTargets.collector = "127.0.0.1:${toString cfg.telemetryPort}";
|
services.hyperhive.swarm.otel.scrapeTargets.collector =
|
||||||
|
"127.0.0.1:${toString deployCfg.swarm-otel.telemetryPort}";
|
||||||
|
|
||||||
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
|
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
|
||||||
# every sibling swarm service: TLS terminates here, then plain http over
|
# every sibling swarm service: TLS terminates here, then plain http over
|
||||||
|
|
@ -1320,7 +1335,7 @@ in
|
||||||
{
|
{
|
||||||
pull.exporter.prometheus = {
|
pull.exporter.prometheus = {
|
||||||
host = "127.0.0.1";
|
host = "127.0.0.1";
|
||||||
port = cfg.telemetryPort;
|
port = deployCfg.swarm-otel.telemetryPort;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -62,21 +62,5 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
port = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 9428;
|
|
||||||
description = ''
|
|
||||||
Loopback port the store listens on. Upstream's default, kept
|
|
||||||
because there is no reason to move it and a familiar number is
|
|
||||||
one less thing an operator has to look up.
|
|
||||||
|
|
||||||
⚠️ Every swarm container shares the host's network namespace, so
|
|
||||||
this is a swarm-wide claim rather than a per-container one — two
|
|
||||||
modules picking the same number collide at runtime with no bind
|
|
||||||
error and nothing in any log. No repo-wide
|
|
||||||
check catches it.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -109,9 +109,9 @@ let
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
# What the store IS from any hive's point of view, the name it answers on and
|
# What the store IS from any hive's point of view, the name it answers on, is
|
||||||
# the port, is `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
|
# `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
|
||||||
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
# `enable`, `package`, `port` and `retentionPeriod` are decisions of the host that
|
||||||
# runs it and live under `deploy.*`.
|
# runs it and live under `deploy.*`.
|
||||||
|
|
||||||
# Retention is a property of the store this host runs, not something the
|
# Retention is a property of the store this host runs, not something the
|
||||||
|
|
@ -126,6 +126,22 @@ in
|
||||||
description = "VictoriaLogs package to run.";
|
description = "VictoriaLogs package to run.";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
options.services.hyperhive.deploy.victorialogs.port = lib.mkOption {
|
||||||
|
type = lib.types.port;
|
||||||
|
default = 9428;
|
||||||
|
description = ''
|
||||||
|
Loopback port the store listens on. Upstream's default, kept
|
||||||
|
because there is no reason to move it and a familiar number is
|
||||||
|
one less thing an operator has to look up.
|
||||||
|
|
||||||
|
⚠️ Every swarm container shares the host's network namespace, so
|
||||||
|
this is a swarm-wide claim rather than a per-container one — two
|
||||||
|
modules picking the same number collide at runtime with no bind
|
||||||
|
error and nothing in any log. No repo-wide
|
||||||
|
check catches it.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
|
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
default = "30d";
|
default = "30d";
|
||||||
|
|
@ -274,8 +290,8 @@ in
|
||||||
assertion =
|
assertion =
|
||||||
!(lib.elem forwarderTelemetryPort [
|
!(lib.elem forwarderTelemetryPort [
|
||||||
hyperhiveCfg.otel.telemetryPort
|
hyperhiveCfg.otel.telemetryPort
|
||||||
hyperhiveCfg.swarm.bao.otel.telemetryPort
|
hyperhiveCfg.deploy.bao.otel.telemetryPort
|
||||||
hyperhiveCfg.swarm.nats.otel.telemetryPort
|
hyperhiveCfg.deploy.nats.otel.telemetryPort
|
||||||
]);
|
]);
|
||||||
message = ''
|
message = ''
|
||||||
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
||||||
|
|
@ -334,7 +350,7 @@ in
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
locations = {
|
locations = {
|
||||||
"/" = {
|
"/" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
|
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/";
|
||||||
extraConfig = swarmAuthRequest;
|
extraConfig = swarmAuthRequest;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -353,7 +369,7 @@ in
|
||||||
# note on `swarmAuthRequest` above), so naming it here is required
|
# note on `swarmAuthRequest` above), so naming it here is required
|
||||||
# rather than redundant.
|
# rather than redundant.
|
||||||
"= /insert/opentelemetry/v1/logs" = {
|
"= /insert/opentelemetry/v1/logs" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString cfg.port}/insert/opentelemetry/v1/logs";
|
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
auth_request /__hive_authelia;
|
auth_request /__hive_authelia;
|
||||||
'';
|
'';
|
||||||
|
|
@ -387,7 +403,7 @@ in
|
||||||
"^~ /select/logsql/" = {
|
"^~ /select/logsql/" = {
|
||||||
# No URI part, so the request path and its query string reach
|
# No URI part, so the request path and its query string reach
|
||||||
# the store as the caller sent them.
|
# the store as the caller sent them.
|
||||||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
auth_request /__swarm_logs_auth;
|
auth_request /__swarm_logs_auth;
|
||||||
'';
|
'';
|
||||||
|
|
@ -404,7 +420,7 @@ in
|
||||||
# authelia's rules for this URL admit the collector's client and
|
# authelia's rules for this URL admit the collector's client and
|
||||||
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
|
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
|
||||||
"= /metrics" = {
|
"= /metrics" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
|
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/metrics";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
auth_request /__hive_authelia;
|
auth_request /__hive_authelia;
|
||||||
'';
|
'';
|
||||||
|
|
@ -653,7 +669,7 @@ in
|
||||||
# Nothing authenticates it either — upstream offers one basic-auth
|
# Nothing authenticates it either — upstream offers one basic-auth
|
||||||
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
|
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
|
||||||
# module sets neither. The bind address is the boundary.
|
# module sets neither. The bind address is the boundary.
|
||||||
listenAddress = "127.0.0.1:${toString cfg.port}";
|
listenAddress = "127.0.0.1:${toString deployCfg.victorialogs.port}";
|
||||||
|
|
||||||
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
|
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue