# The swarm's log store: one VictoriaLogs for the whole swarm, in a
# `swarm-victorialogs` nixos-container beside the metrics store it mirrors.
#
# Why a store at all rather than reading journals directly: an agent can
# verify that a unit was *launched* and never that it is *working*. The
# container journals are not reachable from an agent, so a diagnosis stops at
# the first component that is broken — which is precisely the component whose
# own instrument is least likely to be legible. Collecting logs centrally
# makes the question answerable without host access.
#
# It is the collector that feeds this, not the services directly: one ingest
# point per swarm, same shape as the metrics path.
#
# Gateway vhost is authenticated, unlike the metrics store's: an operator
# reaches this at `https://${cfg.domain}/` gated by the same `auth_request`
# check against authelia that swarm-ui's own vhost uses (`swarmAuthRequest`
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
# auth subrequest with 400, which `auth_request` cannot interpret as anything
# but a broken check). The store's own listener stays loopback-only and
# unauthenticated, and the vhost is now the ONLY way in from outside: the
# collector pushes through it too, at its own `= /insert/...` location, because
# a swarm has one log store and the collector need not share a host with it.
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
# pusher handed its `error_page 401 =302` follows the redirect and POSTs at a
# login page, which answers 200. See the location itself.
#
# The query API has a machine route of the same shape at `^~ /select/logsql/`,
# for the same reason, and with no filtering of what an authenticated caller
# may read — see that location.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.victorialogs;
  deployCfg = config.services.hyperhive.deploy;
  networkCfg = config.services.hyperhive.network;
  hyperhiveCfg = config.services.hyperhive;
  gatewayCfg = hyperhiveCfg.gateway;

  # Copy of `swarm-ui.nix`'s own `swarmAuthRequest` — not shared code because
  # this vhost needs exactly the two locations that reference it (`/` and the
  # internal auth-request target below) and pulling in swarm-ui.nix for one
  # string would couple this module to swarm-ui existing at all, which it
  # need not. Same three lines, same reasoning as that file's own comment.
  swarmAuthRequest = ''
    auth_request /__hive_authelia;
    auth_request_set $target_url $scheme://$http_host$request_uri;
    error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
  '';

  # The agents' `auth_request` responder (`swarm-logs-auth`). Its socket's
  # directory holds that socket alone and is traverse-only for others, the
  # shape `swarm-controller.nix` gives its own.
  logsAuthUnit = "swarm-logs-auth";
  logsAuthSocket = "/run/${logsAuthUnit}/auth.sock";
  logsAuthCredential = id: "/run/credentials/${logsAuthUnit}.service/${id}";
  # Credential id → host path of the responder's store identity.
  logsAuthStoreFiles = {
    "bao-client.pem" = deployCfg.victorialogs.authBaoClientCertFile;
    "bao-client-key.pem" = deployCfg.victorialogs.authBaoClientKeyFile;
  }
  // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
    "bao-ca.pem" = deployCfg.bao.serverCaFile;
  };
  baoCfg = hyperhiveCfg.swarm.bao;

  # Grafana's bearer, checked by a second `swarm-metrics-auth` instance. It
  # reads the store with the same leaf as `swarm-logs-auth`, which is granted
  # the one extra path below. The same leaf as `grafanaBearerStorePath` in
  # ./swarm-victoriametrics.nix and `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
  grafanaAuthUnit = "swarm-logs-grafana-auth";
  grafanaAuthSocket = "/run/${grafanaAuthUnit}/auth.sock";
  grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
  grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";

  # Shared host netns, like every sibling swarm container: Grafana reaches
  # this at 127.0.0.1:<port>.
  privateNetwork = false;

  # The container's journal forwarder pushes to its own receiver on the swarm
  # collector, the route `serviceProducers.swarm-victorialogs` in
  # ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in
  # ./swarm-bao.nix.
  forwarderClientId = cfg.otel.clientId;
  gatewayPortSuffix = lib.optionalString (
    gatewayCfg.httpsPort != 443
  ) ":${toString gatewayCfg.httpsPort}";
  forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
  forwarderUnit = "swarm-journal-forwarder-oidc";
  # In the host netns: distinct from every other collector's there, which the
  # assertion below checks.
  forwarderTelemetryPort = 8897;
  forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
  # The responder's store identity, handed into the container as system
  # credentials for the forwarder's fetch unit. The host units keep reading
  # the same files.
  forwarderStoreFiles = lib.filterAttrs (_: source: source != null) logsAuthStoreFiles;
  # nixos-container expands `extraFlags` unquoted, so a source path must be
  # one word.
  oneWord = path: builtins.match ".*[[:space:]].*" path == null;

  # The forwarder reaches the swarm collector and authelia by name over TLS; in
  # self-signed mode their leaves chain to the hive CA.
  caTrust = import ./lib/hive-ca-trust.nix {
    inherit lib gatewayCfg;
    tlsCfg = deployCfg.hive-controller.tls;
  };
in
{
  # What the store IS from any hive's point of view, the name it answers on and
  # the port, is `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
  # `enable`, `package` and `retentionPeriod` are decisions of the host that
  # runs it and live under `deploy.*`.

  # Retention is a property of the store this host runs, not something the
  # swarm has to agree on: it is read only where the container is defined,
  # and a hive that is a *client* of the log store never consults it. That
  # makes it a `deploy.*` value by the same rule as the seal on the secret
  # store — options on the auto-deployed service itself.
  options.services.hyperhive.deploy.victorialogs.package = lib.mkOption {
    type = lib.types.package;
    default = pkgs.victorialogs;
    defaultText = lib.literalExpression "pkgs.victorialogs";
    description = "VictoriaLogs package to run.";
  };

  options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
    type = lib.types.str;
    default = "30d";
    example = "90d";
    description = ''
      How long log data is kept.

      Deliberately far shorter than the metrics store's retention: logs
      are orders of magnitude larger per unit of time, and their value
      decays much faster. A log line answers "what happened during that
      incident"; a metric answers "is this worse than last quarter".
    '';
  };

  options.services.hyperhive.deploy.victorialogs.authPackage = lib.mkOption {
    type = lib.types.package;
    defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-logs-auth";
    description = ''
      The `swarm-logs-auth` package: the gateway's `auth_request` responder
      that checks an agent's `swarm-agent.<agent>.<secret>` bearer on
      `/select/logsql/` against the secret store.
    '';
  };

  options.services.hyperhive.deploy.victorialogs.grafanaAuthPackage = lib.mkOption {
    type = lib.types.package;
    defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth";
    description = ''
      The `swarm-metrics-auth` package, run as `swarm-logs-grafana-auth`: the
      gateway's `auth_request` responder that checks the bearer Grafana's
      logs datasource presents on `/select/logsql/` against the secret store.

      ⚠️ Not folded into `authPackage` above: that one is `swarm-logs-auth`,
      a different binary. Same split as
      {option}`services.hyperhive.deploy.nats.authPackage`.
    '';
  };

  options.services.hyperhive.deploy.victorialogs.authBaoClientCertFile = lib.mkOption {
    type = lib.types.nullOr lib.types.str;
    default = null;
    example = "/var/lib/swarm-bao-pki/logs-auth.pem";
    description = ''
      Client certificate `swarm-logs-auth` presents to the swarm's secret store
      to read agents' queue secrets. Its subject must be
      {option}`services.hyperhive.deploy.bao.logsAuthCommonName`.

      Required when `deploy.victorialogs.enable` is set; unset fails at eval.
      ./glue-logs-auth-bao-identity.nix points it at the leaf
      ./glue-bao-tls.nix mints on a host that runs the store; on any other
      log-store host it is a file the operator places.

      A path, never a value.
    '';
  };

  options.services.hyperhive.deploy.victorialogs.authBaoClientKeyFile = lib.mkOption {
    type = lib.types.nullOr lib.types.str;
    default = null;
    example = "/var/lib/swarm-bao-pki/logs-auth-key.pem";
    description = ''
      Private key for {option}`services.hyperhive.deploy.victorialogs.authBaoClientCertFile`.
      A path, never a value.
    '';
  };

  options.services.hyperhive.deploy.victorialogs.authVerdictTtl = lib.mkOption {
    type = lib.types.ints.unsigned;
    default = 60;
    description = ''
      Seconds `swarm-logs-auth` grants an agent bearer it has verified again
      without reading the secret store. A denial is never remembered.

      This is also how long a re-minted or deleted queue secret keeps working
      at the log store's query route after its last successful check. `0`
      checks the store on every query.
    '';
  };

  options.services.hyperhive.deploy.victorialogs.authRate = lib.mkOption {
    type = lib.types.str;
    default = "50r/s";
    example = "30r/m";
    description = ''
      nginx `limit_req` rate, per client address, of `/select/logsql/`
      requests carrying a `swarm-agent.` bearer, which nginx checks with
      `swarm-logs-auth`. Each such request that the responder has not granted
      within {option}`services.hyperhive.deploy.victorialogs.authVerdictTtl`
      costs a secret store login and read, whether the bearer is right or not.

      Counts every agent query, granted or not, and is keyed by client address,
      so every agent on one host shares one budget. The default covers about a
      hundred agents on a host cold-starting at once, each costing one store
      login and read for its first query. Requests over the rate and its burst
      are answered 403.
    '';
  };

  options.services.hyperhive.deploy.victorialogs.authBurst = lib.mkOption {
    type = lib.types.ints.positive;
    default = 500;
    description = ''
      Requests a client address may make above
      {option}`services.hyperhive.deploy.victorialogs.authRate` before nginx
      answers 403, served without delay.
    '';
  };

  config = lib.mkIf deployCfg.victorialogs.enable {
    assertions = [
      {
        assertion =
          deployCfg.victorialogs.authBaoClientCertFile != null
          && deployCfg.victorialogs.authBaoClientKeyFile != null;
        message = ''
          services.hyperhive.deploy.victorialogs.authBaoClientCertFile and
          authBaoClientKeyFile must both be set.

          swarm-logs-auth reads agents' queue secrets from the swarm's secret
          store with this certificate. They default to the leaf the store
          mints where services.hyperhive.deploy.bao.enable is set on this
          host; elsewhere, issue a leaf whose CN is
          services.hyperhive.deploy.bao.logsAuthCommonName and name it in
          both options.
        '';
      }
      {
        assertion = lib.all oneWord (lib.attrValues forwarderStoreFiles);
        message = ''
          The log store responder's store identity paths must not contain
          whitespace: nspawn's --load-credential receives them through
          nixos-container's unquoted extra flags.
        '';
      }
      {
        assertion =
          (hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
        message = ''
          services.hyperhive.swarm.otel.serviceProducers has no entry
          "${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
          collector has no receiver for the log store's journal forwarder.
          Declare it in swarm-otel-service.nix.
        '';
      }
      {
        assertion =
          !(lib.elem forwarderTelemetryPort [
            hyperhiveCfg.otel.telemetryPort
            hyperhiveCfg.swarm.otel.telemetryPort
            hyperhiveCfg.swarm.bao.otel.telemetryPort
            hyperhiveCfg.swarm.nats.otel.telemetryPort
          ]);
        message = ''
          The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
          collides with another collector's in the host netns: the hive
          collector's, the swarm collector's, the store forwarder's or the
          queue forwarder's.
        '';
      }
    ];

    # nspawn fails to start the container when a credential source is
    # missing, so where this host mints the leaf the container pulls the
    # minting unit in and waits for it, as ./swarm-builder.nix's does. The CA
    # ordering keeps the trust bundle's bind source there first.
    systemd.services."container@${cfg.machine}" = lib.mkMerge [
      caTrust.containerOrdering
      {
        after = [ "swarm-bao-pki.service" ];
        wants = [ "swarm-bao-pki.service" ];
      }
    ];

    # Which check a `/select/logsql/` request gets. `auth_request` takes a
    # fixed URI, so it names `/__swarm_logs_auth`, which rewrites to the
    # target this map picks from the caller's `Authorization`.
    services.nginx.appendHttpConfig = ''
      map $http_authorization $swarm_logs_auth_target {
        "~^Bearer swarm-agent\." /__swarm_logs_agent_auth;
        "~^Bearer [0-9a-f]{64}$" /__swarm_logs_grafana_auth;
        default /__hive_authelia;
      }
      limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate};
    '';

    # The gateway name, inside `deployCfg.victorialogs.enable` — same
    # "only the host that runs the service may claim the name" guard every
    # sibling swarm-service module uses (`swarm-grafana.nix`,
    # `swarm-victoriametrics.nix`).
    services.hyperhive.gateway.localNames = [ cfg.domain ];

    # This host serves the vhost, and the container behind it resolves
    # through the hive's dnsmasq.
    services.hyperhive.gateway.enable = lib.mkDefault true;
    services.hyperhive.gateway.dns.enable = lib.mkDefault true;

    # Authenticated front door onto the loopback-only store — see the
    # file-top comment for why this is safe to add without touching the
    # store's own (still unauthenticated, still loopback) listener at all.
    # `forceSSL`: same asymmetry `swarm-ui.nix` documents —
    # authelia answers a plain-http auth subrequest with 400, which
    # `auth_request` cannot read as anything but a broken check, so this
    # vhost needs `forceSSL` rather than the `addSSL` every unauthenticated
    # sibling vhost uses.
    services.hyperhive.gateway.vhosts."${cfg.domain}" = {
      forceSSL = true;
      locations = {
        "/" = {
          proxyPass = "http://127.0.0.1:${toString cfg.port}/";
          extraConfig = swarmAuthRequest;
        };

        # The swarm collector's ingest route. `=` so it outranks the `/`
        # prefix above — without a more specific location it would ride that
        # catch-all, and that is the whole hazard here.
        #
        # ⚠️ Deliberately NOT `swarmAuthRequest`. That block ends in
        # `error_page 401 =302`, which is right for a browser and wrong for a
        # pusher: handed a redirect it follows the redirect and POSTs its
        # batch at a login page, which answers 200. Ingest then reports
        # healthy while storing nothing. A machine route lets the 401 reach
        # the client unchanged.
        #
        # `auth_request` does not inherit across sibling locations (see the
        # note on `swarmAuthRequest` above), so naming it here is required
        # rather than redundant.
        "= /insert/opentelemetry/v1/logs" = {
          proxyPass = "http://127.0.0.1:${toString cfg.port}/insert/opentelemetry/v1/logs";
          extraConfig = ''
            auth_request /__hive_authelia;
          '';
        };

        # The read side of that same split: the route a caller holding a
        # bearer token queries, as opposed to the `/` above which is the
        # operator's browser. Bare `auth_request` for the ingest route's
        # exact reason — `error_page 401 =302` hands an unauthenticated
        # machine caller authelia's login page as a **200 with an HTML
        # body**, so a client that reads the status code records a
        # successful query that returned no rows. 401 is the only answer
        # here that a caller cannot mistake for an empty result.
        #
        # `^~` so it outranks the `/` catch-all and stays ahead of any
        # regex location a later change adds. The prefix is the whole
        # LogsQL query API and nothing else — `query`, `tail`, `hits`,
        # `facets`, the `stats_query*` and the `field_*`/`stream_*`
        # routes — while the browser UI sits beside it under
        # `/select/vmui`. Both read off the pinned build, not the docs.
        #
        # The query is forwarded unmodified — no scoping parameter is
        # injected, so any authenticated caller reads the whole swarm's
        # logs. That is the rule in force, not an omission: read
        # permissions are a later thing, and this location is where one
        # attaches when it exists.
        #
        # Three checks: an agent's own `swarm-agent.` bearer goes to
        # `swarm-logs-auth`, Grafana's 64-hex bearer to `swarm-metrics-auth`,
        # anything else to authelia (`appendHttpConfig` above picks).
        "^~ /select/logsql/" = {
          # No URI part, so the request path and its query string reach
          # the store as the caller sent them.
          proxyPass = "http://127.0.0.1:${toString cfg.port}";
          extraConfig = ''
            auth_request /__swarm_logs_auth;
          '';
        };

        # The store's own health, which it publishes as prometheus metrics
        # on the listener it serves queries on. The swarm collector requests
        # exactly this URL, declared as `publishedScrapeTargets.victorialogs`
        # in ./swarm-otel-service.nix; the two must agree, since that string
        # is also the token's audience.
        #
        # Bare `auth_request` to authelia, as on the ingest route above, and
        # never `/__swarm_logs_auth`, which admits an agent's bearer:
        # authelia's rules for this URL admit the collector's client and
        # deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
        "= /metrics" = {
          proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
          extraConfig = ''
            auth_request /__hive_authelia;
          '';
        };

        # `rewrite … last` re-matches locations, so the subrequest lands in
        # one of the two internal locations named by the map.
        "= /__swarm_logs_auth" = {
          extraConfig = ''
            internal;
            rewrite ^ $swarm_logs_auth_target last;
          '';
        };

        # The `Authorization` header rides the subrequest unchanged.
        #
        # Any well-formed agent bearer the responder has not recently granted
        # costs a secret store login and read, so the rate per client address
        # is capped here, ahead of the responder. 403 when over: the only
        # statuses `auth_request` passes through are 401 and 403, and
        # `swarm-logs` retries a 401.
        "= /__swarm_logs_agent_auth" = {
          proxyPass = "http://unix:${logsAuthSocket}:/";
          extraConfig = ''
            internal;
            limit_req zone=swarm_logs_agent_auth burst=${toString deployCfg.victorialogs.authBurst} nodelay;
            limit_req_status 403;
            proxy_pass_request_body off;
            proxy_set_header Content-Length "";
          '';
        };

        # Grafana's bearer, 64 hex digits (`openssl rand -hex 32`), checked by
        # `swarm-metrics-auth`. The responder reads the store at most once per
        # its cache TTL whatever the bearers, so no rate limit sits in front.
        "= /__swarm_logs_grafana_auth" = {
          proxyPass = "http://unix:${grafanaAuthSocket}:/";
          extraConfig = ''
            internal;
            proxy_pass_request_body off;
            proxy_set_header Content-Length "";
          '';
        };

        # The subrequest itself — same target, same header set, same
        # reasoning as `swarm-ui.nix`'s own copy (measured against the
        # pinned authelia binary, not copied from an example).
        "= /__hive_authelia" = {
          proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
          # nixpkgs appends its OWN `Host $host` after extraConfig,
          # which would override verifiedProxyTo's — see the comment
          # on verifiedProxyTo in hive-gateway/vhost-lib.nix.
          recommendedProxySettings = false;
          extraConfig = ''
            internal;
            ${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
            proxy_pass_request_body off;
            proxy_set_header Content-Length "";
            proxy_set_header X-Original-Method $request_method;
            proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-Forwarded-Host $http_host;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
          '';
        };
      };
    };

    # The agents' check on `/select/logsql/`. On the host rather than in the
    # store's container: it needs nothing of the store, only nginx's reach to
    # its socket and its own store identity.
    users.users.${logsAuthUnit} = {
      isSystemUser = true;
      group = logsAuthUnit;
      description = "swarm log store auth_request responder";
    };
    users.groups.${logsAuthUnit} = { };

    systemd.services.${logsAuthUnit} = {
      description = "swarm log store auth_request responder for agent credentials";
      wantedBy = [ "multi-user.target" ];
      # Where the store's PKI is minted on this host, the leaf is one of its
      # files. Ordering only: elsewhere the unit does not exist.
      after = [
        "network.target"
        "swarm-bao-pki.service"
      ];
      serviceConfig = {
        ExecStart = lib.concatStringsSep " " [
          "${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth"
          "--socket ${logsAuthSocket}"
          # The role ./swarm-bao.nix writes on the store's `cert` mount.
          "--store-cert-role swarm-logs-auth"
          "--verdict-ttl ${toString deployCfg.victorialogs.authVerdictTtl}"
        ];
        # Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
        LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
        # ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
        # systemd puts a dynamic user's runtime directory under
        # `/run/private/` (0700 root:root), which nginx cannot traverse.
        User = logsAuthUnit;
        Group = logsAuthUnit;
        RuntimeDirectory = logsAuthUnit;
        # Traverse-only for others, so nginx reaches the socket path.
        RuntimeDirectoryMode = "0751";
        # The path never vanishes from under a running nginx; the responder
        # unlinks a stale socket as it starts.
        RuntimeDirectoryPreserve = "yes";
        Restart = "on-failure";
        RestartSec = "5s";
        SyslogIdentifier = logsAuthUnit;
        ProtectSystem = "strict";
        ProtectHome = true;
        PrivateTmp = "disconnected";
        PrivateDevices = true;
        NoNewPrivileges = true;
        RestrictSUIDSGID = true;
        RemoveIPC = true;
      };
      environment = {
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
        BAO_CLIENT_CERT = logsAuthCredential "bao-client.pem";
        BAO_CLIENT_KEY = logsAuthCredential "bao-client-key.pem";
      }
      // lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
        BAO_CACERT = logsAuthCredential "bao-ca.pem";
      };
    };

    # Grafana's check on `/select/logsql/`: the metrics store's responder
    # binary, pointed at the bearer Grafana's datasources share.
    users.users.${grafanaAuthUnit} = {
      isSystemUser = true;
      group = grafanaAuthUnit;
      description = "swarm log store auth_request responder for Grafana's bearer";
    };
    users.groups.${grafanaAuthUnit} = { };

    systemd.services.${grafanaAuthUnit} = {
      description = "swarm log store auth_request responder for Grafana's bearer";
      wantedBy = [ "multi-user.target" ];
      after = [
        "network.target"
        "swarm-bao-pki.service"
      ];
      serviceConfig = {
        ExecStart = lib.concatStringsSep " " [
          "${deployCfg.victorialogs.grafanaAuthPackage}/bin/swarm-metrics-auth"
          "--socket ${grafanaAuthSocket}"
          "--store-cert-role swarm-logs-auth"
          "--store-path ${grafanaBearerStorePath}"
        ];
        LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
        # ⚠️ A static user, never `DynamicUser`: see the `swarm-logs-auth` unit above.
        User = grafanaAuthUnit;
        Group = grafanaAuthUnit;
        RuntimeDirectory = grafanaAuthUnit;
        RuntimeDirectoryMode = "0751";
        RuntimeDirectoryPreserve = "yes";
        Restart = "on-failure";
        RestartSec = "5s";
        SyslogIdentifier = grafanaAuthUnit;
        ProtectSystem = "strict";
        ProtectHome = true;
        PrivateTmp = "disconnected";
        PrivateDevices = true;
        NoNewPrivileges = true;
        RestrictSUIDSGID = true;
        RemoveIPC = true;
      };
      environment = {
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
        BAO_CLIENT_CERT = grafanaAuthCredential "bao-client.pem";
        BAO_CLIENT_KEY = grafanaAuthCredential "bao-client-key.pem";
      }
      // lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
        BAO_CACERT = grafanaAuthCredential "bao-ca.pem";
      };
    };

    containers.${cfg.machine} = {
      autoStart = true;
      ephemeral = false;
      # No `--link-journal=host`: the journal stays in the guest, where the
      # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
      extraFlags = lib.mapAttrsToList (
        id: source: "--load-credential=${id}:${source}"
      ) forwarderStoreFiles;
      inherit privateNetwork;
      bindMounts = caTrust.bindMount;

      config =
        { ... }:
        {
          imports = [
            ../container-modules/swarm-container.nix
            ../container-modules/swarm-journal-forwarder.nix
            (import ../container-modules/swarm-container-resolver.nix {
              inherit (networkCfg) bridgeIp;
              # The forwarder and its secret fetch dial the swarm collector,
              # authelia and the store by name too.
              dnsConsumers = [
                "victorialogs.service"
                "opentelemetry-collector.service"
                "${forwarderUnit}.service"
              ];
            })
            # Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
            # collector's and authelia's names.
            (caTrust.trustBundle {
              inherit pkgs;
              name = cfg.machine;
              consumers = [ "opentelemetry-collector" ];
            })
          ];

          services.hyperhive.swarmContainer = { inherit privateNetwork; };

          # Authenticated as the forwarder's own OIDC client, with the
          # responder's store identity: one identity for the whole container.
          # It is a system credential, so the fetch unit loads it.
          services.hyperhive.swarmJournalForwarder = {
            enable = true;
            serviceName = "victorialogs";
            endpoint = forwarderEndpoint;
            clientId = forwarderClientId;
            tokenUrl = "${toString hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
            telemetryPort = forwarderTelemetryPort;
            store = {
              package = deployCfg.bao.package;
              addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
              secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
              clientCertFile = forwarderCredential "bao-client.pem";
              clientKeyFile = forwarderCredential "bao-client-key.pem";
              caFile = if forwarderStoreFiles ? "bao-ca.pem" then forwarderCredential "bao-ca.pem" else null;
            };
          };
          systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;

          services.victorialogs = {
            enable = true;
            package = deployCfg.victorialogs.package;

            # ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
            # interface, and this endpoint accepts writes as well as reads.
            # Nothing authenticates it either — upstream offers one basic-auth
            # pair (`-httpAuth.username` / `-httpAuth.password`) and this
            # module sets neither. The bind address is the boundary.
            listenAddress = "127.0.0.1:${toString cfg.port}";

            extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
          };
        };
    };
  };

  # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured
  # against the pinned build rather than read from docs. The path differs
  # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter
  # configured by analogy with the metrics one is silently wrong.
  #
  # ⚠️ And the status code cannot tell you which is which: a wrong path, a
  # right path with a wrong body, and a nonsense path ALL answer 400. Only
  # the server log distinguishes them — the real route complains about the
  # encoding ("json encoding isn't supported ... use protobuf"), while
  # anything else logs "unsupported path requested". Verified with a
  # nonsense-path control, because a probe where every arm returns the same
  # code is not evidence.
  #
  # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
  # why `listenAddress` above is loopback and why nothing reaches it except
  # through the gateway, where the ingest route is authenticated. (This module
  # declares a vhost.)
}