Compare commits
12 changed files with 305 additions and 122 deletions
|
|
@ -200,6 +200,7 @@ in
|
||||||
module-eval-agent-plugins = import ./module-eval/agent-plugins.nix {
|
module-eval-agent-plugins = import ./module-eval/agent-plugins.nix {
|
||||||
inherit pkgs self nixosSystem;
|
inherit pkgs self nixosSystem;
|
||||||
inherit (pkgs) lib;
|
inherit (pkgs) lib;
|
||||||
|
shared = moduleEvalShared;
|
||||||
};
|
};
|
||||||
module-eval-agent-runtime = import ./module-eval/agent-runtime.nix {
|
module-eval-agent-runtime = import ./module-eval/agent-runtime.nix {
|
||||||
inherit pkgs self nixosSystem;
|
inherit pkgs self nixosSystem;
|
||||||
|
|
|
||||||
|
|
@ -113,6 +113,16 @@ let
|
||||||
&& c.RuntimeDirectoryPreserve == "yes"
|
&& c.RuntimeDirectoryPreserve == "yes"
|
||||||
&& c.UMask == "0377";
|
&& c.UMask == "0377";
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# Rename in only on a change: a reader never sees half a token, and the
|
||||||
|
# avatar watcher does not fire on every timer tick.
|
||||||
|
name = "the fetch swaps the token in by rename, only when it changed";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
s = (fetchUnit agentForgeBao).script;
|
||||||
|
in
|
||||||
|
lib.hasInfix "cmp -s" s && lib.hasInfix "mv -f" s;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# Every unit and the bash-task runner find the token through this. A
|
# Every unit and the bash-task runner find the token through this. A
|
||||||
# path, never the value.
|
# path, never the value.
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
lib,
|
lib,
|
||||||
self,
|
self,
|
||||||
nixosSystem,
|
nixosSystem,
|
||||||
|
shared,
|
||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
inherit
|
inherit
|
||||||
|
|
@ -23,9 +24,16 @@ let
|
||||||
|
|
||||||
# `claudePlugins`'s additive-merge shape: a plain per-agent
|
# `claudePlugins`'s additive-merge shape: a plain per-agent
|
||||||
# definition must ADD to the module's own base-set definition rather than
|
# definition must ADD to the module's own base-set definition rather than
|
||||||
# replacing it.
|
# replacing it, while `lib.mkForce` must still replace the whole list
|
||||||
|
# outright — the two arms below plus the unset default (read directly off
|
||||||
|
# `bare`-shaped `agent { }`, no fixture of its own needed) are the three
|
||||||
|
# cases that shape has to hold.
|
||||||
|
agentPluginsDefault = shared.agentBare;
|
||||||
|
|
||||||
agentPluginsAdded = agent { claudePlugins = [ "foo@bar" ]; };
|
agentPluginsAdded = agent { claudePlugins = [ "foo@bar" ]; };
|
||||||
|
|
||||||
|
agentPluginsForced = agent { claudePlugins = lib.mkForce [ "foo@bar" ]; };
|
||||||
|
|
||||||
# The de-dup arm: an agent that names a base-set entry explicitly must not
|
# The de-dup arm: an agent that names a base-set entry explicitly must not
|
||||||
# get it installed twice.
|
# get it installed twice.
|
||||||
agentPluginsDuplicate = agent { claudePlugins = [ "base@hyperhive" ]; };
|
agentPluginsDuplicate = agent { claudePlugins = [ "base@hyperhive" ]; };
|
||||||
|
|
@ -33,7 +41,17 @@ let
|
||||||
agentPlugins = machine: machine.services.hyperhive.agent.claudePlugins;
|
agentPlugins = machine: machine.services.hyperhive.agent.claudePlugins;
|
||||||
cases = [
|
cases = [
|
||||||
{
|
{
|
||||||
# A plain per-agent definition ADDS to the base set (list-typed
|
# Case 1 of the additive-merge shape: an agent that declares
|
||||||
|
# nothing gets exactly the module's base set, no more and no less.
|
||||||
|
name = "an agent with no claudePlugins definition gets exactly the base set";
|
||||||
|
ok =
|
||||||
|
agentPlugins agentPluginsDefault == [
|
||||||
|
"skill-creator@claude-plugins-official"
|
||||||
|
"base@hyperhive"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
# Case 2: a plain per-agent definition ADDS to the base set (list-typed
|
||||||
# options at equal priority concatenate) rather than replacing it —
|
# options at equal priority concatenate) rather than replacing it —
|
||||||
# the property the operator ruling asked for instead of `mkDefault`.
|
# the property the operator ruling asked for instead of `mkDefault`.
|
||||||
# Sorted before comparing: concatenation order between two same-
|
# Sorted before comparing: concatenation order between two same-
|
||||||
|
|
@ -47,6 +65,12 @@ let
|
||||||
"skill-creator@claude-plugins-official"
|
"skill-creator@claude-plugins-official"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# Case 3: `lib.mkForce` is still the escape hatch — an operator who
|
||||||
|
# wants the base set gone, not extended, can still say so outright.
|
||||||
|
name = "an agent's mkForce claudePlugins replaces the base set outright";
|
||||||
|
ok = agentPlugins agentPluginsForced == [ "foo@bar" ];
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# De-dup arm: an agent that names a base-set entry itself must not get
|
# De-dup arm: an agent that names a base-set entry itself must not get
|
||||||
# it installed twice — `lib.unique` at the JSON-render site, not the
|
# it installed twice — `lib.unique` at the JSON-render site, not the
|
||||||
|
|
|
||||||
|
|
@ -125,6 +125,18 @@ let
|
||||||
(subagent opencodeBao)
|
(subagent opencodeBao)
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
name = "an opencode agent's subagent daemon gets the agent's store identity";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
u = subagent opencodeBao;
|
||||||
|
in
|
||||||
|
u.serviceConfig.LoadCredential == [ "hive-agent-bao-server-ca" ]
|
||||||
|
&& u.environment.HIVE_AGENT_NAME == "a1"
|
||||||
|
&& u.environment.BAO_ADDR == "https://bao.t.local:8200"
|
||||||
|
&& u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
||||||
|
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key";
|
||||||
|
}
|
||||||
{
|
{
|
||||||
name = "an opencode agent with no store hands its subagent daemon no store identity";
|
name = "an opencode agent with no store hands its subagent daemon no store identity";
|
||||||
ok =
|
ok =
|
||||||
|
|
|
||||||
|
|
@ -46,25 +46,15 @@ let
|
||||||
# files this host will never have.
|
# files this host will never have.
|
||||||
controllerNoStore = shared.controllerAlone;
|
controllerNoStore = shared.controllerAlone;
|
||||||
|
|
||||||
# Two independent matrix settings no default could supply, each read into
|
# A deployment that pinned `gatewayHost` away from its own swarm-domain
|
||||||
# its own controller variable.
|
# default (`hive-matrix-service.nix`'s own doc: "To use the per-hive name
|
||||||
#
|
|
||||||
# `gatewayHost` pinned away from its own swarm-domain default
|
|
||||||
# (`hive-matrix-service.nix`'s own doc: "To use the per-hive name
|
|
||||||
# ... instead, pin it here").
|
# ... instead, pin it here").
|
||||||
# Only this fixture can tell "the controller reads gatewayHost" apart from
|
# Only this fixture can tell "the controller reads gatewayHost" apart from
|
||||||
# "the controller re-derives gatewayHost's default itself" — both render
|
# "the controller re-derives gatewayHost's default itself" — both render
|
||||||
# the same URL on every other fixture in this file.
|
# the same URL on every other fixture in this file.
|
||||||
#
|
controllerPinnedGateway = hive {
|
||||||
# An ignore list with two entries, so the case reading it also pins the
|
|
||||||
# separator the daemon splits on.
|
|
||||||
controllerPinnedMatrix = hive {
|
|
||||||
deploy.swarm-controller.enable = true;
|
deploy.swarm-controller.enable = true;
|
||||||
swarm.matrix.gatewayHost = "matrix.hive.t.local";
|
swarm.matrix.gatewayHost = "matrix.hive.t.local";
|
||||||
swarm.matrix.adminSync.ignoreUsers = [
|
|
||||||
"@keep-a:t.local"
|
|
||||||
"@keep-b:t.local"
|
|
||||||
];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# No matrix vhost for this swarm at all, so there is no homeserver URL
|
# No matrix vhost for this swarm at all, so there is no homeserver URL
|
||||||
|
|
@ -74,6 +64,16 @@ let
|
||||||
swarm.matrix.gatewayHost = null;
|
swarm.matrix.gatewayHost = null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# An ignore list no default could supply, with two entries so the case below
|
||||||
|
# also pins the separator the daemon splits on.
|
||||||
|
controllerIgnoresAdmins = hive {
|
||||||
|
deploy.swarm-controller.enable = true;
|
||||||
|
swarm.matrix.adminSync.ignoreUsers = [
|
||||||
|
"@keep-a:t.local"
|
||||||
|
"@keep-b:t.local"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
# The two authorities told apart. A deployment that self-signs both ends
|
# The two authorities told apart. A deployment that self-signs both ends
|
||||||
# points `clientCaFile` and `serverCaFile` at one file, so on the fixture
|
# points `clientCaFile` and `serverCaFile` at one file, so on the fixture
|
||||||
# above the CA a hive is issued from and the CA the store is verified by are
|
# above the CA a hive is issued from and the CA the store is verified by are
|
||||||
|
|
@ -133,7 +133,7 @@ let
|
||||||
# the former passes.
|
# the former passes.
|
||||||
name = "the controller's homeserver follows a pinned gatewayHost, not its own chat.<domain> default";
|
name = "the controller's homeserver follows a pinned gatewayHost, not its own chat.<domain> default";
|
||||||
ok =
|
ok =
|
||||||
controllerPinnedMatrix.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_MATRIX_HOMESERVER_URL
|
controllerPinnedGateway.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_MATRIX_HOMESERVER_URL
|
||||||
or null == "https://matrix.hive.t.local";
|
or null == "https://matrix.hive.t.local";
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
|
|
@ -165,7 +165,7 @@ let
|
||||||
# an account the sync would revoke.
|
# an account the sync would revoke.
|
||||||
name = "the controller is told the operator admin sync's ignore list";
|
name = "the controller is told the operator admin sync's ignore list";
|
||||||
ok =
|
ok =
|
||||||
controllerPinnedMatrix.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS
|
controllerIgnoresAdmins.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS
|
||||||
or null == "@keep-a:t.local @keep-b:t.local";
|
or null == "@keep-a:t.local @keep-b:t.local";
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
|
|
@ -264,6 +264,19 @@ let
|
||||||
&& !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential
|
&& !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential
|
||||||
&& m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile;
|
&& m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile;
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# Same two-CA fixture, for the same reason: the wrapper verifies the
|
||||||
|
# STORE, so it takes `serverCaFile`. On a self-signing deployment both
|
||||||
|
# options name one file and either would pass; here the client CA in that
|
||||||
|
# slot is a case this arm fails.
|
||||||
|
#
|
||||||
|
# ⚠️ The package itself stays off `PATH` — `wrapProgram` renames the real
|
||||||
|
# binary, so an unwrapped `bao` is unreachable rather than merely
|
||||||
|
# discouraged. Operator's instruction, and the last assertion is what
|
||||||
|
# keeps a later "install the package too" from quietly undoing it.
|
||||||
|
name = "the host gets a wrapped bao CLI";
|
||||||
|
ok = baoWrapper != null;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
name = "the wrapped bao CLI carries this store's address";
|
name = "the wrapped bao CLI carries this store's address";
|
||||||
ok = lib.hasInfix "--set-default BAO_ADDR" baoWrapperCmd;
|
ok = lib.hasInfix "--set-default BAO_ADDR" baoWrapperCmd;
|
||||||
|
|
|
||||||
|
|
@ -108,6 +108,13 @@ let
|
||||||
&& lib.hasInfix "exit 1" loginBranch
|
&& lib.hasInfix "exit 1" loginBranch
|
||||||
&& lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script));
|
&& lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script));
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# The gate that decides this reader exists is the certificate rather
|
||||||
|
# than anything about agents: containers are created
|
||||||
|
# at runtime, so there is no static "this hive runs agents" fact to ask.
|
||||||
|
name = "a hive that names a client identity reads its agent queue credential";
|
||||||
|
ok = baoRemoteReader.systemd.services ? swarm-bao-queue-agent;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# The reader's own grant covers `swarm/hives/<this hive>/*`; a path
|
# The reader's own grant covers `swarm/hives/<this hive>/*`; a path
|
||||||
# outside it answers 403, not "no such key". This is the path
|
# outside it answers 403, not "no such key". This is the path
|
||||||
|
|
|
||||||
|
|
@ -144,6 +144,10 @@ let
|
||||||
in
|
in
|
||||||
lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es);
|
lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es);
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
name = "a hive that has not opted into all-local runs no swarm controller";
|
||||||
|
ok = !bare.services.hyperhive.deploy.swarm-controller.enable;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
name = "the all-local mode turns the swarm controller on";
|
name = "the all-local mode turns the swarm controller on";
|
||||||
ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
|
ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
|
||||||
|
|
|
||||||
|
|
@ -21,74 +21,104 @@ let
|
||||||
runGroup
|
runGroup
|
||||||
;
|
;
|
||||||
|
|
||||||
# One host per guard. Each guard reports every offender in one message
|
# The hive-name guards, with the collector explicitly OFF. That is the whole
|
||||||
# (`../host-modules/lib/name-guards.nix`), so one host carrying every
|
|
||||||
# colliding name still tells each name's arm apart: a name its arm misses
|
|
||||||
# is absent from the offenders line.
|
|
||||||
#
|
|
||||||
# Every fixture runs with the collector explicitly OFF. That is the whole
|
|
||||||
# property: the guards live where `swarm.hives` is declared, so they run in a
|
# property: the guards live where `swarm.hives` is declared, so they run in a
|
||||||
# deployment that has a secret store and no collector, rather than being
|
# deployment that has a secret store and no collector, rather than being
|
||||||
# assertions inside swarm-otel's own `mkIf`.
|
# assertions inside swarm-otel's own `mkIf`.
|
||||||
|
|
||||||
# Every cert-auth subject an operator can override, set to a name with no
|
|
||||||
# reserved fragment and taken by a hive. `controllerCommonName`'s default
|
|
||||||
# (`swarm-controller`) contains `swarm` and is caught by the substring guard
|
|
||||||
# whatever the cert-auth arm does, so the default could not tell the two
|
|
||||||
# apart.
|
|
||||||
#
|
#
|
||||||
# The fixed subjects are a LIST, and a list with one consulted element and
|
# ⚠️ `controllerCommonName` is overridden to a name containing NO reserved
|
||||||
# one dead one looks identical from the first element's case: each element
|
# fragment. Its default (`swarm-controller`) contains `swarm` and is caught
|
||||||
# needs its own name in the offenders line. matrix-ctl's grant names its
|
# by the substring guard whatever the cert-auth arm does — so a fixture using
|
||||||
# leaves, not a whole prefix, so a hive that inherited a dead entry there
|
# the default could not tell the two apart, and the arm under test would pass
|
||||||
# would silently gain the ability to overwrite the swarm's matrix credential.
|
# on the neighbour's work.
|
||||||
#
|
hiveNamedAfterCertSubject = hive {
|
||||||
# 🩸 The two per-hive subjects have a different shape: those roles are
|
|
||||||
# written PER HIVE, so the subject a hive must not be is
|
|
||||||
# `<prefix>-<some hive's name>` rather than the prefix itself. Reserving only
|
|
||||||
# the prefix would leave the composed spelling free, and a hive taking it
|
|
||||||
# would present a leaf the other hive's role accepts — a hive reading another
|
|
||||||
# hive's queue credential, the exact widening the split exists to avoid. The
|
|
||||||
# collision is with the OTHER hive's role, hence `other` beside `qr-other`
|
|
||||||
# and `hl-other`.
|
|
||||||
hiveNamedAfterEverySubject = hive {
|
|
||||||
deploy.swarm-otel.enable = false;
|
deploy.swarm-otel.enable = false;
|
||||||
deploy.bao = {
|
deploy.bao.controllerCommonName = "ctl";
|
||||||
controllerCommonName = "ctl";
|
swarm.hives.ctl.domain = "ctl.t.local";
|
||||||
secretPublisherCommonName = "pubctl";
|
|
||||||
matrixCtlCommonName = "mintctl";
|
|
||||||
grafanaOidcCommonName = "gfctl";
|
|
||||||
otelOidcCommonName = "otctl";
|
|
||||||
forwarderOidcCommonName = "fwctl";
|
|
||||||
forgePublisherCommonName = "fpctl";
|
|
||||||
granterCommonName = "grctl";
|
|
||||||
queueAgentCommonNamePrefix = "qr";
|
|
||||||
hiveLeafCommonNamePrefix = "hl";
|
|
||||||
};
|
|
||||||
swarm.hives = {
|
|
||||||
ctl.domain = "ctl.t.local";
|
|
||||||
pubctl.domain = "p.t.local";
|
|
||||||
mintctl.domain = "m.t.local";
|
|
||||||
gfctl.domain = "g.t.local";
|
|
||||||
otctl.domain = "o.t.local";
|
|
||||||
fwctl.domain = "f.t.local";
|
|
||||||
fpctl.domain = "fp.t.local";
|
|
||||||
grctl.domain = "gr.t.local";
|
|
||||||
other.domain = "other.t.local";
|
|
||||||
qr-other.domain = "q.t.local";
|
|
||||||
hl-other.domain = "hl.t.local";
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# The control for every refusal below: same shape, a roster nothing objects
|
# The control for both arms below: same shape, a roster nothing objects to.
|
||||||
# to. It carries both sides of the identifier rule's edges — 63 characters,
|
|
||||||
# and digits and hyphens in every position (the rule has no
|
|
||||||
# leading-character restriction).
|
|
||||||
hiveNamesAllLegal = hive {
|
hiveNamesAllLegal = hive {
|
||||||
deploy.swarm-otel.enable = false;
|
deploy.swarm-otel.enable = false;
|
||||||
deploy.bao.controllerCommonName = "ctl";
|
deploy.bao.controllerCommonName = "ctl";
|
||||||
swarm.hives.${lib.concatStrings (lib.replicate 63 "a")}.domain = "l.t.local";
|
};
|
||||||
swarm.hives."0-h-9".domain = "h.t.local";
|
|
||||||
|
# The reserved subjects are a LIST, and a list with one consulted element and
|
||||||
|
# one dead one looks identical from the first element's case. This fixture
|
||||||
|
# collides with the SECOND, leaving the controller's at its default.
|
||||||
|
hiveNamedAfterPublisherSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.secretPublisherCommonName = "pubctl";
|
||||||
|
swarm.hives.pubctl.domain = "p.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
# The THIRD element of the same list, colliding on its own so neither of the
|
||||||
|
# two above can carry it. matrix-ctl's grant names its leaves, not a whole
|
||||||
|
# prefix, which is exactly why a dead entry here would be easy to miss: a
|
||||||
|
# hive that inherited it would not obviously break anything, it would
|
||||||
|
# silently gain the ability to overwrite the swarm's matrix credential.
|
||||||
|
hiveNamedAfterMatrixCtlSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.matrixCtlCommonName = "mintctl";
|
||||||
|
swarm.hives.mintctl.domain = "m.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
# The two OIDC-secret readers' subjects, fixed strings like the three above.
|
||||||
|
hiveNamedAfterGrafanaOidcSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.grafanaOidcCommonName = "gfctl";
|
||||||
|
swarm.hives.gfctl.domain = "g.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
hiveNamedAfterOtelOidcSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.otelOidcCommonName = "otctl";
|
||||||
|
swarm.hives.otctl.domain = "o.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
# The store forwarder's OIDC reader, the fifth fixed subject.
|
||||||
|
hiveNamedAfterForwarderOidcSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.forwarderOidcCommonName = "fwctl";
|
||||||
|
swarm.hives.fwctl.domain = "f.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
# The forge publisher's, the one writer of the controller's forge token.
|
||||||
|
hiveNamedAfterForgePublisherSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.forgePublisherCommonName = "fpctl";
|
||||||
|
swarm.hives.fpctl.domain = "fp.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
# The granter's, the one subject whose role may write every `swarm-*` grant.
|
||||||
|
hiveNamedAfterGranterSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.granterCommonName = "grctl";
|
||||||
|
swarm.hives.grctl.domain = "gr.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
# 🩸 A different shape from every fixture above: the queue-credential roles
|
||||||
|
# are written PER HIVE, so the subject a hive must not
|
||||||
|
# be is `<prefix>-<some hive's name>` rather than the prefix itself. Reserving
|
||||||
|
# only the prefix would leave the composed spelling free, and a hive taking it
|
||||||
|
# would present a leaf the other hive's role accepts — which is a hive reading
|
||||||
|
# another hive's queue credential, the exact widening the split exists to
|
||||||
|
# avoid.
|
||||||
|
#
|
||||||
|
# Two hives here, not one: the collision is with the OTHER hive's role.
|
||||||
|
hiveNamedAfterPerHiveReaderSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.queueAgentCommonNamePrefix = "qr";
|
||||||
|
swarm.hives.other.domain = "o.t.local";
|
||||||
|
swarm.hives.qr-other.domain = "q.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
# The per-hive gateway-leaf subject, composed the same way.
|
||||||
|
hiveNamedAfterHiveLeafSubject = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
deploy.bao.hiveLeafCommonNamePrefix = "hl";
|
||||||
|
swarm.hives.other.domain = "o.t.local";
|
||||||
|
swarm.hives.hl-other.domain = "hl.t.local";
|
||||||
};
|
};
|
||||||
|
|
||||||
hiveNameWithComposedWord = hive {
|
hiveNameWithComposedWord = hive {
|
||||||
|
|
@ -98,36 +128,40 @@ let
|
||||||
|
|
||||||
# The key characters are checked by the same module, so these run with the
|
# The key characters are checked by the same module, so these run with the
|
||||||
# collector off too. `a/b` adds a certificate-subject component; `../x`
|
# collector off too. `a/b` adds a certificate-subject component; `../x`
|
||||||
# leaves a directory named after the hive; 64 characters is one over the cap.
|
# leaves a directory named after the hive.
|
||||||
hiveNamesNotIdents = hive {
|
hiveNameWithSlash = hive {
|
||||||
deploy.swarm-otel.enable = false;
|
deploy.swarm-otel.enable = false;
|
||||||
swarm.hives = {
|
swarm.hives."a/b".domain = "a.t.local";
|
||||||
"a/b".domain = "a.t.local";
|
|
||||||
"../x".domain = "x.t.local";
|
|
||||||
${lib.concatStrings (lib.replicate 64 "a")}.domain = "l.t.local";
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# The first line of a guard's message: the problem and its offenders. The
|
hiveNameWithParentDir = hive {
|
||||||
# lines after it list the forbidden values, which name a subject whether or
|
deploy.swarm-otel.enable = false;
|
||||||
# not any hive took it.
|
swarm.hives."../x".domain = "x.t.local";
|
||||||
offendersLine = a: lib.head (lib.splitString "\n" a.message);
|
};
|
||||||
|
|
||||||
|
# Both sides of the length cap: 63 characters passes, 64 does not.
|
||||||
|
hiveNameAtLengthCap = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
swarm.hives.${lib.concatStrings (lib.replicate 63 "a")}.domain = "l.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
hiveNameOverLengthCap = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
swarm.hives.${lib.concatStrings (lib.replicate 64 "a")}.domain = "l.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
|
# Digits and hyphens in every position: the rule has no leading-character
|
||||||
|
# restriction.
|
||||||
|
hiveNameWithDigitsAndHyphens = hive {
|
||||||
|
deploy.swarm-otel.enable = false;
|
||||||
|
swarm.hives."0-h-9".domain = "h.t.local";
|
||||||
|
};
|
||||||
|
|
||||||
# Markers from `lib/name-guards.nix`'s `problem` strings. Matching the
|
# Markers from `lib/name-guards.nix`'s `problem` strings. Matching the
|
||||||
# problem rather than the `why` prose keeps the messages rewordable.
|
# problem rather than the `why` prose keeps the messages rewordable.
|
||||||
equalityGuardFired =
|
equalityGuardFired =
|
||||||
h: lib.any (a: !a.assertion && lib.hasInfix "has reserved name(s)" a.message) h.assertions;
|
h: lib.any (a: !a.assertion && lib.hasInfix "has reserved name(s)" a.message) h.assertions;
|
||||||
|
|
||||||
# True when the equality guard names `name` as an offender.
|
|
||||||
equalityGuardRefuses =
|
|
||||||
h: name:
|
|
||||||
lib.any (
|
|
||||||
a:
|
|
||||||
!a.assertion
|
|
||||||
&& lib.hasInfix "has reserved name(s)" (offendersLine a)
|
|
||||||
&& lib.hasInfix "'${name}'" (offendersLine a)
|
|
||||||
) h.assertions;
|
|
||||||
|
|
||||||
fragmentGuardFired =
|
fragmentGuardFired =
|
||||||
h:
|
h:
|
||||||
lib.any (
|
lib.any (
|
||||||
|
|
@ -147,35 +181,44 @@ let
|
||||||
cases = [
|
cases = [
|
||||||
{
|
{
|
||||||
name = "a hive key containing a slash is refused, naming the key";
|
name = "a hive key containing a slash is refused, naming the key";
|
||||||
ok = identGuardRefuses hiveNamesNotIdents "a/b";
|
ok = identGuardRefuses hiveNameWithSlash "a/b";
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "a hive key that climbs out of its directory is refused, naming the key";
|
name = "a hive key that climbs out of its directory is refused, naming the key";
|
||||||
ok = identGuardRefuses hiveNamesNotIdents "../x";
|
ok = identGuardRefuses hiveNameWithParentDir "../x";
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "a hive key over 63 characters is refused";
|
name = "a hive key over 63 characters is refused";
|
||||||
ok = identGuardRefuses hiveNamesNotIdents (lib.concatStrings (lib.replicate 64 "a"));
|
ok = identGuardFired hiveNameOverLengthCap;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# Controls: without them an identifier guard that refuses everything
|
# Controls: without them an identifier guard that refuses everything
|
||||||
# passes the three cases above.
|
# passes the three cases above.
|
||||||
name = "63-character and digit-and-hyphen hive keys pass the identifier guard";
|
name = "63-character and digit-and-hyphen hive keys pass the identifier guard";
|
||||||
ok = !(identGuardFired hiveNamesAllLegal);
|
ok =
|
||||||
|
!(identGuardFired hiveNameAtLengthCap)
|
||||||
|
&& !(identGuardFired hiveNameWithDigitsAndHyphens)
|
||||||
|
&& !(identGuardFired hiveNamesAllLegal);
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# `ctl` is in no deny list — it is reserved *because it is the subject a
|
# `ctl` is in no deny list — it is reserved *because it is the subject a
|
||||||
# cert-auth role accepts*, which is a value an operator sets, so a
|
# cert-auth role accepts*, which is a value an operator sets, so a
|
||||||
# literal deny entry could never have covered it.
|
# literal deny entry could never have covered it.
|
||||||
name = "a hive named after a cert-auth subject is refused, with the collector off";
|
name = "a hive named after a cert-auth subject is refused, with the collector off";
|
||||||
ok = equalityGuardRefuses hiveNamedAfterEverySubject "ctl";
|
ok =
|
||||||
|
equalityGuardFired hiveNamedAfterCertSubject
|
||||||
|
&& lib.any (a: !a.assertion && lib.hasInfix "'ctl'" a.message) hiveNamedAfterCertSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# Every cert-auth subject is reserved, not just the first one in the
|
# Every cert-auth subject is reserved, not just the first one in the
|
||||||
# list. Without this case the second element could be dead and the case
|
# list. Without this case the second element could be dead and the case
|
||||||
# above would still pass.
|
# above would still pass.
|
||||||
name = "a hive named after the secret publisher's subject is refused too";
|
name = "a hive named after the secret publisher's subject is refused too";
|
||||||
ok = equalityGuardRefuses hiveNamedAfterEverySubject "pubctl";
|
ok =
|
||||||
|
equalityGuardFired hiveNamedAfterPublisherSubject
|
||||||
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'pubctl'" a.message
|
||||||
|
) hiveNamedAfterPublisherSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# And the third, for the reason the second one's comment gives one list
|
# And the third, for the reason the second one's comment gives one list
|
||||||
|
|
@ -183,7 +226,11 @@ let
|
||||||
# has to register itself, and nothing but a case per element notices when
|
# has to register itself, and nothing but a case per element notices when
|
||||||
# one forgets.
|
# one forgets.
|
||||||
name = "a hive named after matrix-ctl's subject is refused too";
|
name = "a hive named after matrix-ctl's subject is refused too";
|
||||||
ok = equalityGuardRefuses hiveNamedAfterEverySubject "mintctl";
|
ok =
|
||||||
|
equalityGuardFired hiveNamedAfterMatrixCtlSubject
|
||||||
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'mintctl'" a.message
|
||||||
|
) hiveNamedAfterMatrixCtlSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# The fourth and fifth, for the reason the case above gives: `certAuthCns`
|
# The fourth and fifth, for the reason the case above gives: `certAuthCns`
|
||||||
|
|
@ -193,25 +240,43 @@ let
|
||||||
# client secrets.
|
# client secrets.
|
||||||
name = "a hive named after either OIDC-secret reader's subject is refused too";
|
name = "a hive named after either OIDC-secret reader's subject is refused too";
|
||||||
ok =
|
ok =
|
||||||
equalityGuardRefuses hiveNamedAfterEverySubject "gfctl"
|
equalityGuardFired hiveNamedAfterGrafanaOidcSubject
|
||||||
&& equalityGuardRefuses hiveNamedAfterEverySubject "otctl";
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'gfctl'" a.message
|
||||||
|
) hiveNamedAfterGrafanaOidcSubject.assertions
|
||||||
|
&& equalityGuardFired hiveNamedAfterOtelOidcSubject
|
||||||
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'otctl'" a.message
|
||||||
|
) hiveNamedAfterOtelOidcSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# And the store forwarder's, for the same reason one element later.
|
# And the store forwarder's, for the same reason one element later.
|
||||||
name = "a hive named after the store forwarder's OIDC-reader subject is refused too";
|
name = "a hive named after the store forwarder's OIDC-reader subject is refused too";
|
||||||
ok = equalityGuardRefuses hiveNamedAfterEverySubject "fwctl";
|
ok =
|
||||||
|
equalityGuardFired hiveNamedAfterForwarderOidcSubject
|
||||||
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'fwctl'" a.message
|
||||||
|
) hiveNamedAfterForwarderOidcSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# And the forge publisher's: a hive holding a leaf it accepts could
|
# And the forge publisher's: a hive holding a leaf it accepts could
|
||||||
# replace the token the controller provisions every forge object with.
|
# replace the token the controller provisions every forge object with.
|
||||||
name = "a hive named after the forge publisher's subject is refused too";
|
name = "a hive named after the forge publisher's subject is refused too";
|
||||||
ok = equalityGuardRefuses hiveNamedAfterEverySubject "fpctl";
|
ok =
|
||||||
|
equalityGuardFired hiveNamedAfterForgePublisherSubject
|
||||||
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'fpctl'" a.message
|
||||||
|
) hiveNamedAfterForgePublisherSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# And the granter's, whose role is root-equivalent: a hive holding a leaf
|
# And the granter's, whose role is root-equivalent: a hive holding a leaf
|
||||||
# it accepts could grant itself anything.
|
# it accepts could grant itself anything.
|
||||||
name = "a hive named after the bao granter's subject is refused too";
|
name = "a hive named after the bao granter's subject is refused too";
|
||||||
ok = equalityGuardRefuses hiveNamedAfterEverySubject "grctl";
|
ok =
|
||||||
|
equalityGuardFired hiveNamedAfterGranterSubject
|
||||||
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'grctl'" a.message
|
||||||
|
) hiveNamedAfterGranterSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# 🩸 The per-hive half, and the one a prefix-only reservation would miss:
|
# 🩸 The per-hive half, and the one a prefix-only reservation would miss:
|
||||||
|
|
@ -220,14 +285,22 @@ let
|
||||||
# written for hive `other` — a leaf that reads a credential belonging to a
|
# written for hive `other` — a leaf that reads a credential belonging to a
|
||||||
# hive that is not it.
|
# hive that is not it.
|
||||||
name = "a hive named after another hive's per-hive reader subject is refused";
|
name = "a hive named after another hive's per-hive reader subject is refused";
|
||||||
ok = equalityGuardRefuses hiveNamedAfterEverySubject "qr-other";
|
ok =
|
||||||
|
equalityGuardFired hiveNamedAfterPerHiveReaderSubject
|
||||||
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'qr-other'" a.message
|
||||||
|
) hiveNamedAfterPerHiveReaderSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# A hive taking this spelling would present a leaf hive `other`'s
|
# A hive taking this spelling would present a leaf hive `other`'s
|
||||||
# gateway-leaf role accepts, and could ask the store for that hive's
|
# gateway-leaf role accepts, and could ask the store for that hive's
|
||||||
# certificate. The control is the legal roster below.
|
# certificate. The control is the legal roster below.
|
||||||
name = "a hive named after another hive's gateway-leaf subject is refused";
|
name = "a hive named after another hive's gateway-leaf subject is refused";
|
||||||
ok = equalityGuardRefuses hiveNamedAfterEverySubject "hl-other";
|
ok =
|
||||||
|
equalityGuardFired hiveNamedAfterHiveLeafSubject
|
||||||
|
&& lib.any (
|
||||||
|
a: !a.assertion && lib.hasInfix "'hl-other'" a.message
|
||||||
|
) hiveNamedAfterHiveLeafSubject.assertions;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# Without this the case above proves nothing: an arm that fires for every
|
# Without this the case above proves nothing: an arm that fires for every
|
||||||
|
|
@ -247,7 +320,7 @@ let
|
||||||
# pass while testing the arrangement they exist to rule out.
|
# pass while testing the arrangement they exist to rule out.
|
||||||
name = "the guard fixtures really do have the collector disabled";
|
name = "the guard fixtures really do have the collector disabled";
|
||||||
ok =
|
ok =
|
||||||
!hiveNamedAfterEverySubject.services.hyperhive.deploy.swarm-otel.enable
|
!hiveNamedAfterCertSubject.services.hyperhive.deploy.swarm-otel.enable
|
||||||
&& !hiveNamesAllLegal.services.hyperhive.deploy.swarm-otel.enable
|
&& !hiveNamesAllLegal.services.hyperhive.deploy.swarm-otel.enable
|
||||||
&& !hiveNameWithComposedWord.services.hyperhive.deploy.swarm-otel.enable;
|
&& !hiveNameWithComposedWord.services.hyperhive.deploy.swarm-otel.enable;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -49,7 +49,13 @@ let
|
||||||
deploy.nats.enable = true;
|
deploy.nats.enable = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
inherit (shared) natsOnStoreHostReaderMoved;
|
# The store's host with the hive's own reader pointed somewhere else.
|
||||||
|
natsOnStoreHostReaderMoved = hive {
|
||||||
|
deploy.bao.enable = true;
|
||||||
|
deploy.nats.enable = true;
|
||||||
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
||||||
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
||||||
|
};
|
||||||
|
|
||||||
# A queue host holding only the hive's own store reader, placed by hand: no
|
# A queue host holding only the hive's own store reader, placed by hand: no
|
||||||
# store here, and no responder leaf named.
|
# store here, and no responder leaf named.
|
||||||
|
|
@ -479,6 +485,16 @@ let
|
||||||
&& lib.hasInfix "--agent-token-publish-subject '$$SWARM.agent-state.{agent}'" exec
|
&& lib.hasInfix "--agent-token-publish-subject '$$SWARM.agent-state.{agent}'" exec
|
||||||
&& lib.hasInfix "--store-cert-role swarm-nats-auth" exec;
|
&& lib.hasInfix "--store-cert-role swarm-nats-auth" exec;
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# Its own arm, like the two above: the icon write is a separate feature
|
||||||
|
# and dropping it must not hide behind the streams still being granted.
|
||||||
|
name = "the responder grants a verified agent its own icon key";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
exec = (responderOf natsQueueOnly).serviceConfig.ExecStart;
|
||||||
|
in
|
||||||
|
lib.hasInfix "--agent-token-publish-subject '$$KV.agent-icons.{agent}'" exec;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# The whole per-agent grant, compared as a list rather than by infix: a
|
# The whole per-agent grant, compared as a list rather than by infix: a
|
||||||
# wider subject added beside these (`$$JS.API.>`, another stream's name)
|
# wider subject added beside these (`$$JS.API.>`, another stream's name)
|
||||||
|
|
@ -706,6 +722,21 @@ let
|
||||||
autheliaColocated.services.hyperhive.swarm.authelia.url
|
autheliaColocated.services.hyperhive.swarm.authelia.url
|
||||||
== autheliaNotColocated.services.hyperhive.swarm.authelia.url;
|
== autheliaNotColocated.services.hyperhive.swarm.authelia.url;
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# …and the name it resolves through is a DOMAIN, not a host this config
|
||||||
|
# picked. `127.0.0.1` is what a co-location-derived default renders on
|
||||||
|
# the IdP host, the bridge address is what the queue's defaults render,
|
||||||
|
# and neither is an address a client may be handed: the domain is
|
||||||
|
# allowed to resolve differently in different places, which is the
|
||||||
|
# whole property.
|
||||||
|
name = "the swarm IdP address names no host address";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
url = autheliaNotColocated.services.hyperhive.swarm.authelia.url;
|
||||||
|
bridgeIp = autheliaNotColocated.services.hyperhive.network.bridgeIp;
|
||||||
|
in
|
||||||
|
!(lib.hasInfix "127.0.0.1" url) && !(lib.hasInfix "localhost" url) && !(lib.hasInfix bridgeIp url);
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# The queue container's own journal forwarder, on the store's host and
|
# The queue container's own journal forwarder, on the store's host and
|
||||||
# on a queue host with no store and no swarm collector: it exports to
|
# on a queue host with no store and no swarm collector: it exports to
|
||||||
|
|
@ -742,6 +773,12 @@ let
|
||||||
natsWithStoreIdentity
|
natsWithStoreIdentity
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# The read grant the fetch relies on: the responder's role, the identity
|
||||||
|
# the forwarder logs in with, reads the queue client's secret.
|
||||||
|
name = "the queue container's store identity may read the forwarder's client secret";
|
||||||
|
ok = lib.hasInfix "path \"secret/data/swarm/services/swarm-nats/oidc/client\" {" natsOnStoreHost.systemd.services.swarm-bao-nats-auth-policy.script;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# The far end: a receiver, authenticator, pipeline entry and gateway
|
# The far end: a receiver, authenticator, pipeline entry and gateway
|
||||||
# route of its own, checking the queue client as audience. Rendered on
|
# route of its own, checking the queue client as audience. Rendered on
|
||||||
|
|
|
||||||
|
|
@ -44,7 +44,14 @@ let
|
||||||
deploy.nats.enable = true;
|
deploy.nats.enable = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
inherit (shared) natsOnStoreHostReaderMoved;
|
# The store's host with the hive's own reader pointed somewhere else.
|
||||||
|
natsOnStoreHostReaderMoved = hive {
|
||||||
|
deploy.bao.enable = true;
|
||||||
|
deploy.nats.enable = true;
|
||||||
|
deploy.nats.autoGenerateCallout = true;
|
||||||
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
||||||
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
||||||
|
};
|
||||||
|
|
||||||
# A queue host holding only the hive's own store reader, placed by hand: no
|
# A queue host holding only the hive's own store reader, placed by hand: no
|
||||||
# store here, and no queue leaf named.
|
# store here, and no queue leaf named.
|
||||||
|
|
|
||||||
|
|
@ -61,15 +61,6 @@ in
|
||||||
deploy.nats.authPackage = pkgs.emptyDirectory;
|
deploy.nats.authPackage = pkgs.emptyDirectory;
|
||||||
};
|
};
|
||||||
|
|
||||||
# The queue on the store's host with the hive's own reader pointed
|
|
||||||
# somewhere else.
|
|
||||||
natsOnStoreHostReaderMoved = hive {
|
|
||||||
deploy.bao.enable = true;
|
|
||||||
deploy.nats.enable = true;
|
|
||||||
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
||||||
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
||||||
};
|
|
||||||
|
|
||||||
controllerAlone = hive { deploy.swarm-controller.enable = true; };
|
controllerAlone = hive { deploy.swarm-controller.enable = true; };
|
||||||
|
|
||||||
matrixNoBaoIdentity = hiveWithController { deploy.matrix.enable = true; };
|
matrixNoBaoIdentity = hiveWithController { deploy.matrix.enable = true; };
|
||||||
|
|
|
||||||
|
|
@ -335,6 +335,10 @@ let
|
||||||
name = "a named exposeHostPorts opens its bridge port on the host's own say-so";
|
name = "a named exposeHostPorts opens its bridge port on the host's own say-so";
|
||||||
ok = builtins.elem 5432 (bridgePortsOrNone exposedPortsNoHive);
|
ok = builtins.elem 5432 (bridgePortsOrNone exposedPortsNoHive);
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
name = "the swarm UI claims the swarm apex wherever this host runs the swarm controller";
|
||||||
|
ok = swarmUiHere.services.nginx.virtualHosts ? "t.local";
|
||||||
|
}
|
||||||
{
|
{
|
||||||
name = "the old swarm UI package spelling sets the controller's UI package, which the vhost serves";
|
name = "the old swarm UI package spelling sets the controller's UI package, which the vhost serves";
|
||||||
ok =
|
ok =
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue