# The swarm's metrics UI: one Grafana for the whole swarm, in a
# `swarm-grafana` nixos-container next to the VictoriaMetrics store it reads.
#
# Two containers rather than one, on the operator's call: Grafana can be
# restarted, reconfigured or broken without taking the TSDB down with it.
# They are a pair, not a unit.
#
# Dashboards are provisioned from files in this repo, deliberately not created
# in the UI: the store path is immutable, so Grafana is told not to accept UI
# edits it would lose on the next rebuild.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.grafana;
  deployCfg = config.services.hyperhive.deploy;
  networkCfg = config.services.hyperhive.network;
  hyperhiveCfg = config.services.hyperhive;
  gatewayCfg = hyperhiveCfg.gateway;
  baoCfg = hyperhiveCfg.swarm.bao;
  baoDeploy = deployCfg.bao;
  swarmDomain = hyperhiveCfg.swarm.domain;

  caTrust = import ./lib/hive-ca-trust.nix {
    inherit lib gatewayCfg;
    tlsCfg = deployCfg.hive-controller.tls;
  };

  # Spelled ONCE. The provisioned datasource declares it and every panel in the
  # provisioned dashboards references it; a second literal would be free to
  # drift, and the failure is a dashboard that renders with empty panels rather
  # than an error.
  datasourceUid = "swarm-victoriametrics";

  # The logs store's own uid. Separate from the metrics one and spelled once for
  # the same reason: a dashboard panel names it, and a second literal would be
  # free to drift into a panel that renders empty rather than erroring.
  logsDatasourceUid = "swarm-victorialogs";

  # The shipped dashboards carry `@datasourceUid@` / `@logsDatasourceUid@` where
  # a real deployment needs the uids above. They are substituted here rather
  # than committed with the literals so the single binding stays single.
  renderDashboard =
    name:
    pkgs.writeText name (
      builtins.replaceStrings
        [ "@datasourceUid@" "@logsDatasourceUid@" ]
        [ datasourceUid logsDatasourceUid ]
        (builtins.readFile (./swarm-grafana/dashboards + "/${name}"))
    );

  # Grafana's file provider wants a DIRECTORY to scan, so the rendered files are
  # collected into one.
  dashboardDir = pkgs.linkFarm "hyperhive-grafana-dashboards" (
    map
      (name: {
        inherit name;
        path = renderDashboard name;
      })
      [
        "agents.json"
        "authelia.json"
        "bao.json"
        "claude-usage.json"
        "forge.json"
        "logstore.json"
        "metricstore.json"
        "queue.json"
      ]
  );

  # Total on a null swarm domain for the same reason every sibling module is:
  # the required-domain assertion in hive-network.nix should be what an
  # operator sees, not a coercion error from here.
  domainBase = if swarmDomain == null then "invalid" else swarmDomain;

  # Is SSO configured for this SWARM. Swarm-wide by construction — `swarm.*` is
  # identical on every host — and the option's own description is what makes
  # this the right question to ask: a null URL means "no SSO configured".
  #
  # 🩸 A subject of an assertion below, NOT a gate. Dropping the OIDC block when
  # this is false looks conservative and is the outage: `disable_login_form` is
  # unconditional a few hundred lines down, so a Grafana with no OIDC settings
  # is a Grafana with no login of any kind, arrived at silently. SSO is a
  # requirement of running this service, so an unconfigured swarm fails to
  # build and says which option to set.
  ssoConfigured = hyperhiveCfg.swarm.authelia.url != null;

  # A reader of the store is defined by holding a certificate the store
  # accepts, never by standing next to it — the rule
  # ./glue-matrix-bao-token.nix states in full.
  #
  # Also asserted rather than gating anything, and that is the ruling this
  # module's second delivery route was deleted under: the store exists so that
  # ONE secret is placed out of band per host — this certificate — and every
  # other secret is read with it. A Grafana host without it has not been given
  # its identity yet, which is a thing to say out loud rather than to route
  # around by reaching into authelia's tree whenever it happens to be local.
  haveClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null;

  autheliaUrl = toString hyperhiveCfg.swarm.authelia.url;

  # Where the plaintext lands inside the container. Under /var/lib rather
  # than /run: Grafana may start before the delivery unit on a later boot,
  # and a secret that evaporates on reboot turns a working login into an
  # intermittent one.
  secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret";

  # The same file seen from the host, which is where the delivery unit writes
  # it. Spelled once: the unit that writes it and the config that names it are
  # a few hundred lines apart, and a Grafana reading a path nothing writes is
  # a login that fails with nothing in any log about the file.
  hostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${secretPath}";
  hostSecretDir = builtins.dirOf hostSecretPath;

  # Where the publisher on authelia's host leaves this client's secret. The
  # `services` segment is `swarm-secret-client`'s `path::Kind::Service` — a
  # swarm service's client is registered once for the whole swarm, so its
  # secret is one value and not one per hive. Both ends compose this from the
  # same swarm-wide client id, so a rename cannot leave one of them behind.
  #
  # ⚠️ The prefix is not decoration: it is what this host's read grant covers,
  # so a path outside it answers 403 rather than a miss, however correct it
  # looks.
  storeSecretPath = "secret/swarm/services/${cfg.oidc.clientId}/oidc/client";

  # Grafana's own datasource-encryption key. Generated in-container (see the
  # unit below) because nothing outside the container ever reads it — unlike
  # the OIDC secret above, whose other reader is authelia's container.
  secretKeyPath = "/var/lib/grafana-secret/secret_key";

  socketPath = "${deployCfg.grafana.socketDir}/grafana.sock";

  # Both static NixOS ids, and both checked rather than assumed: nginx has
  # `ids.gids.nginx = 60`, Grafana has `ids.uids.grafana = 196` — but there is
  # deliberately no `ids.gids.grafana`, which is why the OIDC-secret unit
  # below lands its file root-group at 0400 instead of reaching for one.
  nginxGid = config.ids.gids.nginx;
  grafanaUid = config.ids.uids.grafana;

in
{
  # `enable` moved to `services.hyperhive.deploy.grafana.enable` — see
  # ./deploy.nix. Whether this host runs the swarm's Grafana is a
  # deployment decision, and `swarm.*` has to be identical on every host.
  # What stays here is what the service IS to every hive: the name it is
  # served under, the port its `/metrics` is re-served on, and the OIDC
  # client it is registered as. What the host running it decides — which
  # build it runs, where its datasources point, which plugins are in its
  # store path, and the socket directory it shares with nginx — is below,
  # under `deploy.grafana`.
  options.services.hyperhive.swarm.grafana = {
    machine = lib.mkOption {
      type = lib.types.str;
      readOnly = true;
      default = "swarm-grafana";
      description = ''
        Container name. Read-only: the name appears in host paths and in
        `machinectl`, so it is a fact other modules may read rather than a
        knob.
      '';
    };

    domain = lib.mkOption {
      type = lib.types.str;
      default = "grafana.${domainBase}";
      defaultText = lib.literalExpression ''"grafana.''${services.hyperhive.swarm.domain}"'';
      description = ''
        Name the gateway serves this on. A sibling of the swarm's other
        service names, so the swarm-services sub-CA can issue for it — see
        `hive-tls.nix` for why a service name being a sibling rather than a
        child decides which CA may sign it.

        ⚠️ Changing this changes the OAuth redirect URI, which authelia
        matches exactly. Both sides move together because both derive from
        this option; an operator who pins one by hand breaks the login.
      '';
    };

    metricsPort = lib.mkOption {
      type = lib.types.port;
      default = 9095;
      description = ''
        Loopback port on which the gateway's nginx re-serves Grafana's
        `/metrics`, and nothing else, so the swarm's collector can scrape it.

        ⚠️ **This is nginx's port, not Grafana's.** Grafana still claims none —
        see {option}`services.hyperhive.deploy.grafana.socketDir` for why that
        matters. A prometheus scrape target is a `host:port`, and it cannot
        address a unix socket; rather than undo the socket decision, the one
        endpoint a scraper needs gets a listener of its own.

        Bound to loopback and unauthenticated, which is the same posture every
        other entry in
        {option}`services.hyperhive.swarm.otel.scrapeTargets` has: those
        targets are trusted by *proximity* rather than by credential.
        Deliberately **not** the published `grafana.<domain>` vhost, which
        would put an authorization decision in front of a scrape.

        The number itself is arbitrary and free today;
        `state/eval-port-collisions.sh` is what keeps it that way, since a
        second claim on a port in this shared namespace produces no bind error
        and nothing in any log.
      '';
    };

    oidc = {
      clientId = lib.mkOption {
        type = lib.types.str;
        default = "swarm-grafana";
        description = ''
          The authelia OIDC client id. Names the application rather than
          the protocol, per the convention in
          {option}`services.hyperhive.swarm.authelia.oidc.clients`.
        '';
      };

      redirectUri = lib.mkOption {
        type = lib.types.str;
        readOnly = true;
        default = "https://${cfg.domain}/login/generic_oauth";
        defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.grafana.domain}/login/generic_oauth"'';
        description = ''
          OAuth callback authelia sends the browser back to, and the URI it
          matches **exactly**.

          Read-only, like {option}`services.hyperhive.swarm.grafana.machine`
          and for the same reason: Grafana derives it from its own
          `root_url` (`<root_url>/login/generic_oauth`), so it is a fact
          other modules may read rather than a knob. The glue that registers
          this client wherever authelia runs reads it from here instead of
          restating the format — a second spelling of it is a silently
          rejected login.
        '';
      };

      role = lib.mkOption {
        type = lib.types.enum [
          "Viewer"
          "Editor"
          "Admin"
        ];
        default = "Admin";
        example = "Editor";
        description = ''
          Grafana org role every SSO user is assigned.

          `Admin` by default, and that is a considered default rather than
          a permissive one: the login form is disabled whenever SSO is
          configured, so this is the *only* way anyone reaches Grafana —
          a `Viewer` default would produce a swarm nobody can administer.
          Passing authelia already means being an operator of this swarm;
          its user store is the small, `swarmctl`-managed one.

          Lower it if a swarm ever grows read-only operators, which is a
          one-line change here.
        '';
      };
    };

  };

  # What stays above is what the service IS to every hive. What the host
  # running it decides is here: which build it runs, where its datasources
  # point, which plugins sit in its store path, and the directory it shares a
  # socket with nginx through. `enable` already lives in ./deploy.nix, which
  # also carries the renames.
  #
  # ⚠️ Both datasource URLs are wiring and still move. A URL's scope is the
  # scope of what it ADDRESSES, not the fact that it is a URL: both stores
  # bind loopback, so these can only ever mean "on this host".
  options.services.hyperhive.deploy.grafana = {
    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.grafana;
      defaultText = lib.literalExpression "pkgs.grafana";
      description = "Grafana package to run.";
    };

    socketDir = lib.mkOption {
      type = lib.types.str;
      default = "/run/swarm-grafana";
      description = ''
        Directory holding the unix socket Grafana serves on, shared between
        the host (where nginx runs) and the container (where Grafana runs).

        ⚠️ **Grafana takes no TCP port at all, and that is the point.** Every
        swarm service container shares the host's network namespace, so a
        port is a swarm-wide resource that two modules can silently both
        claim — which is exactly what happened: Grafana defaulted to
        upstream's 3000, so does the forge, and `grafana.<swarm-domain>`
        served the forge with no bind error and nothing in any log.

        A socket has a path, and a path collision is a build-time conflict
        rather than a runtime coin toss.
      '';
    };

    datasourceUrl = lib.mkOption {
      type = lib.types.str;
      default = "http://127.0.0.1:${toString hyperhiveCfg.swarm.victoriametrics.port}";
      defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victoriametrics.port}"'';
      description = ''
        Where the provisioned datasource points. Defaults to the metrics
        store on this host, which is the only place it can be: that store
        binds loopback, so a Grafana somewhere else could not reach it
        anyway. Set explicitly if a deployment fronts VictoriaMetrics with
        something that does listen wider.
      '';
    };

    logsDatasourceUrl = lib.mkOption {
      type = lib.types.str;
      default = "http://127.0.0.1:${toString hyperhiveCfg.swarm.victorialogs.port}";
      defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victorialogs.port}"'';
      description = ''
        Where the provisioned logs datasource points. Same reasoning as
        {option}`services.hyperhive.deploy.grafana.datasourceUrl`: the store
        binds loopback, so a Grafana elsewhere could not reach it anyway.

        Provisioned unconditionally, like the metrics datasource — the store
        being off is a deployment choice rather than a reason to withhold the
        connection, and an operator whose logs live elsewhere sets this.
      '';
    };

    plugins = lib.mkOption {
      type = lib.types.listOf lib.types.package;
      default = [
        pkgs.grafanaPlugins.grafana-metricsdrilldown-app
        # Reading the swarm's logs needs the datasource that speaks LogsQL.
        # Nothing is paired with it to browse them: Explore is that browser.
        pkgs.grafanaPlugins.victoriametrics-logs-datasource
      ];
      defaultText = lib.literalExpression ''
        [
          pkgs.grafanaPlugins.grafana-metricsdrilldown-app
          pkgs.grafanaPlugins.victoriametrics-logs-datasource
        ]'';
      example = lib.literalExpression "[ pkgs.grafanaPlugins.grafana-piechart-panel ]";
      description = ''
        Grafana plugins to install, as packages. Declarative rather than
        installed through the UI, which is the only shape that works here:
        plugin management is **server-admin** scoped, and on an SSO hive
        nobody holds that role — `users.auto_assign_org_role` grants an
        *org* role, and the built-in local admin that does hold server
        admin cannot log in because the login form is disabled whenever
        SSO is configured.

        That is a deliberate pair of decisions rather than an oversight,
        and this option is the way through it: plugins live in the store
        and in git, so they survive a container rebuild and a state reset,
        and the container needs no runtime egress to grafana.com.

        ⚠️ This list is the ONLY way a plugin reaches this Grafana.
        Setting it at all disables upstream's `preinstall` mechanism (see
        the `declarativePlugins` comment below), which is what normally
        fetches the Drilldown apps on first start — so a plugin left off
        this list is absent, not merely un-pinned.

        Metrics Drilldown is on by default because this swarm's Grafana
        exists to read VictoriaMetrics, and the queryless metrics browser
        is the part of that people actually reach for.

        Its Logs / Traces / Profiles siblings are deliberately NOT here,
        and for two different reasons. Traces and Profiles front Tempo
        and Pyroscope, which this swarm does not run — a UI for a backend
        that does not answer. Logs Drilldown is the subtler one: this
        swarm *does* run a log store, so that argument no longer applies
        to it, but the app is Loki-only and no configuration here changes
        that. Its volume views call `/loki/api/v1/index/volume`, which
        VictoriaLogs does not implement and answers "unsupported path
        requested". Logs are browsed through Explore instead.
      '';
    };
  };

  config = lib.mkIf (hyperhiveCfg.enable && deployCfg.grafana.enable) {
    # The gateway name and the quick-link, both inside `deploy.grafana` — that
    # guard is the load-bearing part. Every hive in a swarm may know this UI
    # exists, but only the host that RUNS it may claim the name; a client
    # hive declaring the vhost would answer for a service it does not have.
    services.hyperhive.gateway.localNames = [ cfg.domain ];

    # The secret oneshots as well as grafana itself: each runs before it and
    # fails in ways grafana then reports only as a login that does not work.
    #
    # Unconditional, because every unit named here now renders in every
    # deployment. This list used to be assembled with `lib.optional` per
    # delivery route, which was the right shape while there were two — a unit
    # name that never renders is a journald scrape target matching nothing,
    # which reads as a quiet unit rather than an absent one.
    services.hyperhive.swarm.otel.journaldUnits = [
      "grafana"
      "swarm-grafana-secret-key"
      "swarm-bao-grafana-oidc"
    ];

    services.hyperhive.swarm.controller.links = [
      {
        label = "Grafana";
        icon = "📊";
        url = "https://${cfg.domain}/";
      }
    ];

    # Registering the client is NOT here any more: it has to happen on the
    # host that runs authelia, and this whole block is gated on the host that
    # runs Grafana. ./glue-grafana-oidc-client.nix is where it moved to.

    # Both arms are what used to be a silent gate, and both fire only where
    # Grafana runs — this whole block is under `deploy.grafana.enable`. The
    # binding itself is asserted rather than a copy of its formula, the way
    # ./swarm-nats.nix's own `autheliaUrl` arm does it: two spellings of one
    # boolean is two places for a future edit to land in only one.
    assertions = [
      {
        # SSO is not optional for this service, and the reason is a hundred
        # lines below in `auth.disable_login_form = true`: Grafana ships an
        # `admin`/`admin` account on a public vhost, so the password box is
        # off whatever the topology. Emitting no OIDC block when the swarm
        # names no IdP therefore produces a container with no way in at all —
        # a state no log names, since nothing failed. Failing the build and
        # naming the option is the only report that reaches anyone.
        assertion = ssoConfigured;
        message = ''
          services.hyperhive.deploy.grafana.enable requires
          services.hyperhive.swarm.authelia.url — Grafana's only login is SSO,
          because its local login form is disabled unconditionally (it ships
          an admin/admin account and its vhost is on the public gateway).

          It defaults to this host's own instance only when this host runs
          authelia. A hive that federates with a swarm sets it explicitly to
          wherever that provider lives. Grafana exchanges its authorization
          code at `''${url}/api/oidc/token` from inside its container, so a
          null URL leaves no endpoint to name.
        '';
      }
      {
        # The other half of one login: the OIDC block names a `$__file{}` that
        # `swarm-bao-grafana-oidc.service` below writes, and that unit reads
        # the store with this host's client certificate. No certificate, no
        # secret, and the same no-way-in Grafana as the arm above.
        assertion = haveClientIdentity;
        message = ''
          services.hyperhive.deploy.grafana.enable requires this host to hold a
          swarm-secret-store client identity: set both

            services.hyperhive.deploy.bao.clientCertFile
            services.hyperhive.deploy.bao.clientKeyFile

          Grafana's OIDC client secret is minted by authelia and read out of
          the store, on every host that runs Grafana — including the host that
          runs authelia. That is one delivery route rather than two, and it is
          what the store is for: this certificate is the single credential
          placed out of band, and every other secret comes from the store with
          it.

          On a hive that runs the store, glue-bao-tls.nix supplies both as
          defaults and there is nothing to do. Elsewhere the leaf is issued
          from that CA out of band and named here — see docs/swarm/secrets.md.
        '';
      }
    ];

    # Websockets: Grafana Live streams panel updates over one, and without
    # the upgrade headers dashboards load and then never refresh — which
    # reads as stale data rather than as a proxy fault.
    services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
      listen = gatewayCfg.lib.listen;
      extraConfig = gatewayCfg.lib.securityHeaders;
      locations."/" = {
        proxyPass = "http://unix:${socketPath}:/";
        proxyWebsockets = true;
        extraConfig = ''
          # Grafana builds its OAuth redirect from the ORIGINAL request.
          # Without these every request looks like it arrived at 127.0.0.1
          # over plain http, and the redirect sent to authelia names a
          # host the browser cannot reach.
          proxy_set_header X-Forwarded-Proto $scheme;
          proxy_set_header X-Forwarded-Host $host;
          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        '';
      };
    };

    # The scrape listener: Grafana's `/metrics` and nothing else, on loopback.
    #
    # No `serverName` and no TLS on purpose — this listener exists for exactly
    # one client on the same host, so there is no name to match and nothing to
    # verify. It is the only vhost here that is not reachable from outside the
    # machine, and that is what lets it skip the authorization the published
    # one carries.
    #
    # ⚠️ `= /metrics` is an EXACT match, not a prefix: it is the difference
    # between exposing one endpoint and re-serving the whole of Grafana on an
    # unauthenticated port. Widening it would be a silent authorization bypass,
    # since nothing else about this block would have to change.
    services.nginx.virtualHosts."grafana-metrics" = {
      listen = [
        {
          addr = "127.0.0.1";
          port = cfg.metricsPort;
        }
      ];
      locations."= /metrics".proxyPass = "http://unix:${socketPath}:/metrics";
    };

    # Declared here rather than in the collector's module, per that option's
    # rule: an entry exists where the service that named it runs.
    services.hyperhive.swarm.otel.scrapeTargets =
      lib.mkIf config.services.hyperhive.deploy.swarm-otel.enable
        {
          grafana = "127.0.0.1:${toString cfg.metricsPort}";
        };

    # Order the container after the host CA service so the bind source below
    # exists before nspawn sets the mount up.
    systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;

    # The socket directory, created host-side before the container starts.
    # nixos-container refuses to start when a bind source is missing, so this
    # rule is a prerequisite of the mount rather than a tidiness measure.
    #
    # Owned by Grafana's uid so it can create the socket, group nginx so the
    # gateway can traverse; `0750` keeps everything else out. Numeric ids
    # because the host has no `grafana` account — Grafana lives in the
    # container, and only the number crosses that boundary.
    systemd.tmpfiles.rules = [
      "d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -"
    ];

    # THE delivery unit — one route, in every deployment. The secret authelia
    # minted arrives out of the swarm secret store, which the publisher on
    # authelia's host wrote it into, and that is true whether authelia is a
    # network away or in the container next door.
    #
    # 🩸 A second unit here used to copy the plaintext out of authelia's host
    # tree wherever the two were co-located, with the OIDC block gated on
    # either route being available. The ruling that deleted it: the store
    # exists so a host holds ONE out-of-band secret — its client certificate —
    # and reads everything else with it, so skipping the store when the
    # producer is local buys a round trip and costs a second delivery path, a
    # second way for the file to be wrong, and a gate whose false arm was a
    # Grafana with no login. Recorded in docs/swarm/secrets.md.
    #
    # Shaped after ./glue-queue-agent-credential.nix, because it is a store
    # reader and this is what the store's readers do: a cert login that fails
    # LOUDLY, since every state it fails on is one a retry fixes, then a read
    # that degrades QUIETLY, since no retry turns "no value there" into a value.
    #
    # ⚠️ An ABSENT secret is "not published yet", not a fault of this host's.
    # The publisher runs on the authelia host and authelia mints on its first
    # boot, so early in a swarm's life there is legitimately nothing to read.
    # Nothing here writes a stand-in: there is no such thing as a locally valid
    # OIDC client secret, and a placeholder would turn a Grafana that cannot
    # log anyone in into one whose token exchange is refused.
    #
    # ⇒ So the absence surfaces at Grafana rather than here, and this unit's job
    # is to leave the journal line that says which of the three it was: the
    # secret not arriving is a Grafana nobody can log into, and a named cause is
    # the only thing separating that from a mystery.
    systemd.services.swarm-bao-grafana-oidc = {
      description = "fetch Grafana's OIDC client secret from the swarm secret store";
      # Every one of these names a unit that exists only where the store runs.
      # `Requires=` on an absent unit fails the job outright, so the ordering
      # is conditional even though the read is not: off-host there is nothing
      # local to wait for, and the timeout below bounds the attempt instead.
      after = lib.optionals baoDeploy.enable [
        "swarm-bao-pki.service"
        "container@${baoCfg.machine}.service"
      ];
      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
      before = [ "container@${cfg.machine}.service" ];
      wantedBy = [
        "multi-user.target"
        "container@${cfg.machine}.service"
      ];
      path = [
        baoDeploy.package
        pkgs.coreutils
      ];
      # Sized for the race this loses, not for an unseal: `swarm-bao` comes up
      # seconds before this unit asks, and the cert-auth role it logs in
      # against is written seconds after, so a few short attempts cover it. An
      # hours-long window would be a bet on a sealed store, and the degrade
      # below is already correct for that.
      #
      # `StartLimit*` are `[Unit]` settings, so they go here and not in
      # `serviceConfig` — systemd ignores them under `[Service]`. The window
      # has to exceed `RestartSec × burst`.
      startLimitBurst = 4;
      startLimitIntervalSec = 300;
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        SyslogIdentifier = "swarm-bao-grafana-oidc";
        # What actually bounds the read below. Stated here rather than left to
        # systemd's default, so the number a boot waits on is in the file that
        # waits.
        TimeoutStartSec = 30;
        Restart = "on-failure";
        RestartSec = 15;
      };
      environment = {
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
        BAO_CLIENT_CERT = baoDeploy.clientCertFile;
        BAO_CLIENT_KEY = baoDeploy.clientKeyFile;
      }
      # Absent means the system trust store, which is what a deployment with a
      # real CA wants and what a self-signed one must not be left with.
      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
        BAO_CACERT = baoDeploy.serverCaFile;
      };
      script = ''
        set -euo pipefail

        # `bao`'s own message is the only thing separating a missing value from
        # a refused identity from an unreachable host. This unit's degraded
        # mode is correct for all three, so it reports which one rather than
        # asserting all three in a sentence of ours.
        err="$(mktemp)"
        trap 'rm -f "$err"' EXIT

        # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
        # variables above only decide which certificate the TLS handshake
        # presents; without a token `bao` asks its token helper instead, and
        # that is a `sh` this unit's `path` does not carry. `-token-only`
        # answers on stdout and skips the helper on both sides.
        if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
          echo "could not log in to swarm-bao with this host's certificate; leaving Grafana's OIDC client secret as it is." >&2
          if [ -s "$err" ]; then
            cat "$err" >&2
          else
            echo "bao failed without writing a diagnostic." >&2
          fi
          exit 1
        fi
        export BAO_TOKEN

        if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
          echo "swarm-bao did not return ${storeSecretPath}; Grafana has no OIDC client secret yet." >&2
          if [ -s "$err" ]; then
            cat "$err" >&2
          else
            echo "bao failed without writing a diagnostic." >&2
          fi
          exit 0
        fi

        if [ -z "$secret" ]; then
          echo "swarm-bao returned an empty ${storeSecretPath}; leaving the file as it is." >&2
          exit 0
        fi

        # Written with a shell builtin, never handed to a program: `printf` is
        # bash's own, so the plaintext never becomes an argument in /proc the
        # way `install <<<"$secret"` or an `echo` from `path` would.
        #
        # Owned by Grafana's own uid, unlike the matrix sibling which lands
        # root-owned: tuwunel's secret is read by `LoadCredential` as root
        # before the sandbox exists, whereas Grafana expands `$__file{}`
        # itself, as itself, while parsing its config. This container sets no
        # `privateUsers`, so the host uid is the container uid, and both sides
        # take it from the same static NixOS id.
        #
        # Group is root, not grafana, and that is forced rather than chosen:
        # `ids.uids.grafana` is a static id but there is no `ids.gids.grafana`
        # — the group's gid is allocated at activation inside the container, so
        # the host cannot know it at eval time. Harmless here because 0400
        # grants the group nothing; if this mode ever widens, the gid has to be
        # discovered at runtime rather than assumed.
        install -d -m 0755 ${lib.escapeShellArg hostSecretDir}
        umask 077
        printf '%s\n' "$secret" > ${lib.escapeShellArg hostSecretPath}
        chown ${toString config.ids.uids.grafana}:0 ${lib.escapeShellArg hostSecretPath}
        chmod 0400 ${lib.escapeShellArg hostSecretPath}
      '';
    };

    containers.${cfg.machine} = {
      autoStart = true;
      ephemeral = false;
      # Journal files on the host, not inside the container: nixpkgs hardcodes
      # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
      extraFlags = [ "--link-journal=host" ];
      # Shared host netns, like every sibling swarm container: the gateway
      # reaches this at 127.0.0.1:<port>.
      privateNetwork = false;

      # The socket directory, shared with the host so nginx can reach in.
      #
      # ⚠️ It has to be a bind mount rather than a path both sides happen to
      # know: a container's `/run` is an nspawn tmpfs, so it is NOT visible
      # from the host at `/var/lib/nixos-containers/<name>/run`. The host
      # creates the directory (tmpfiles rule below, which also fixes
      # ownership) and nspawn mounts it in — the same shape the per-agent
      # `/run/hive-agent/<name>/web.sock` already uses.
      bindMounts = {
        ${deployCfg.grafana.socketDir} = {
          hostPath = deployCfg.grafana.socketDir;
          isReadOnly = false;
        };
      }
      // caTrust.bindMount;

      config =
        { ... }:
        {
          imports = [
            (import ./swarm-container-resolver.nix {
              inherit (networkCfg) bridgeIp;
              dnsConsumers = [ "grafana.service" ];
            })
            # Grafana is Go, and `SSL_CERT_FILE` *replaces* the default store
            # rather than adding to it — so it needs the system CAs and the
            # hive CA concatenated, not the CA alone. Without it the browser
            # half of the login succeeds and the server-to-server token
            # exchange fails `x509: unknown authority`.
            (caTrust.trustBundle {
              inherit pkgs;
              name = cfg.machine;
              consumers = [ "grafana" ];
            })
          ];

          system.stateVersion = "26.05";

          # This container shares the host netns, so its own firewall.service
          # would rewrite the HOST ruleset at every boot. The host firewall
          # owns all filtering.
          networking.firewall.enable = false;
          # resolvconf stays off because the resolver unit imported above
          # owns /etc/resolv.conf. Leaving it on would let host-tracking
          # regenerate the file empty, since the host's copy doesn't cross
          # the boundary after start.
          networking.resolvconf.enable = lib.mkForce false;

          # Grafana's `secret_key` has **no default in nixpkgs** and an
          # assertion refuses the build without one — which is how the first
          # version of this module broke a deploy. It signs the datasource
          # secrets in Grafana's own database.
          #
          # Generated in-container on first boot, like authelia's own keys and
          # for the same reason: nothing outside this container ever reads it,
          # which makes in-container generation right rather than merely
          # easier. (The OIDC client secret is delivered host-side precisely
          # because it has a second reader.)
          #
          # ⚠️ Generated ONCE and kept — the `-s` guard is load-bearing.
          # Rotating this key does not re-encrypt what it already encrypted, so
          # a fresh key on every boot would leave Grafana unable to decrypt its
          # own stored datasource secrets. Under /var/lib, never /run.
          systemd.services.swarm-grafana-secret-key = {
            description = "generate Grafana's datasource encryption key on first boot";
            wantedBy = [ "grafana.service" ];
            before = [ "grafana.service" ];
            serviceConfig = {
              Type = "oneshot";
              RemainAfterExit = true;
              # Same User/Group/StateDirectory as grafana itself, so systemd
              # creates the directory owned by the account that has to read
              # the file — no chown, no mode juggling.
              User = "grafana";
              Group = "grafana";
              StateDirectory = "grafana-secret";
              StateDirectoryMode = "0700";
              SyslogIdentifier = "swarm-grafana-secret-key";
            };
            path = [
              pkgs.openssl
              pkgs.coreutils
            ];
            script = ''
              set -euo pipefail
              key=${lib.escapeShellArg secretKeyPath}
              if [ ! -s "$key" ]; then
                umask 077
                openssl rand -hex 32 > "$key"
                echo "generated Grafana's datasource encryption key"
              fi
            '';
          };

          # The group Grafana chgrps its socket to. Declared here because a
          # container has its own user database — the host knowing gid 60 as
          # `nginx` means nothing in here, only the number crosses.
          #
          # The name is local and arbitrary; the gid is the contract.
          users.groups.gateway-nginx.gid = nginxGid;
          users.users.grafana.extraGroups = [ "gateway-nginx" ];

          services.grafana = {
            enable = true;
            package = deployCfg.grafana.package;

            # Passed through unconditionally, empty default included.
            # Upstream distinguishes `null` from `[ ]`, and both differences
            # favour always handing it a list:
            #
            #   - `null` points the plugin path at grafana's mutable
            #     `<dataDir>/plugins`; any list points it at a store path.
            #     Switching on the day someone adds their first plugin would
            #     bury that change inside an unrelated one.
            #   - upstream defaults its plugin update-check to
            #     `declarativePlugins == null`, so a list also stops the
            #     container phoning grafana.com. That is the no-runtime-egress
            #     property this option exists for — it should not arrive only
            #     once a plugin happens to be listed.
            #
            #   - 🩸 and the one this comment used to miss: it also sets
            #     upstream's `plugins.preinstall_disabled` (nixpkgs'
            #     grafana.nix defaults it to `declarativePlugins != null`,
            #     saying preinstall "causes issues with declarativePlugins").
            #     Preinstall is what fetches the Drilldown apps on first
            #     start, so claiming this directory silently removes them.
            #     That is why the option's default is not empty.
            #
            # The claim that used to sit here — "nothing is taken over on a
            # hive with no plugins, manual installation is impossible anyway"
            # — reasoned only about plugins a PERSON installs. Upstream
            # installs some itself, and those are the ones that went missing.
            declarativePlugins = deployCfg.grafana.plugins;

            settings = {
              server = {
                # Already upstream's default (measured), but pinned rather
                # than inherited: the gateway is the only intended client,
                # and this being loopback is what keeps the UI from being
                # published on whatever else the host is reachable on.
                # A unix socket, not a port. `socket_gid` is nginx's, so the
                # gateway can connect; `0660` keeps everyone else out.
                #
                # ⚠️ Grafana chowns the socket to `socket_gid`, and a
                # non-root process may only chgrp to a group it is a MEMBER
                # of — hence the group declared for `grafana` in this
                # container's own user database below. Without that
                # membership Grafana starts, fails the chown, and the socket
                # is unreachable by nginx with nothing obviously wrong.
                protocol = "socket";
                socket = socketPath;
                socket_gid = nginxGid;
                socket_mode = "0660";
                domain = cfg.domain;
                # Grafana builds its own OAuth redirect from this. Left at
                # upstream's `%(protocol)s://%(domain)s:%(http_port)s/` it
                # would name `http://<domain>:3000/`, which authelia has
                # never heard of.
                root_url = "https://${cfg.domain}/";
              };

              analytics = {
                reporting_enabled = false;
                check_for_updates = false;
              };

              # Stated rather than inherited. This is upstream's default, but
              # a scrape target now depends on it, and a default that changes
              # silently takes the target with it — the endpoint disappears,
              # nginx keeps answering, and the scrape fails with a 404 far
              # from anything that mentions Grafana.
              metrics.enabled = true;

              users.auto_assign_org_role = cfg.oidc.role;

              # ⚠️ `$__file{}`, and required rather than optional: nixpkgs
              # dropped the default and asserts on null, so without this the
              # whole host fails to build. The unit below generates it.
              security.secret_key = "$__file{${secretKeyPath}}";

              # No local password path, ever. This is not tidiness: Grafana
              # ships an `admin`/`admin` account and this vhost is on the
              # public gateway, so SSO is the only way in — not merely the
              # preferred one.
              #
              # Not conditional on anything, and the block below is not
              # either, which is what keeps the two from ever disagreeing.
              # Whether a password box sits on a public login page cannot
              # depend on which host happens to run authelia — and it used to,
              # in a module where the OIDC block was gated and this line was
              # not.
              auth.disable_login_form = true;

              # Emitted in every deployment, because SSO is a requirement of
              # running this service rather than a feature of some topologies:
              # the assertions above refuse to build a Grafana that lacks an
              # IdP to point at or the identity to fetch its secret with. A
              # `lib.optionalAttrs` here would be the silent half of that pair,
              # dropping the only login this container has whenever a value
              # went null.
              "auth.generic_oauth" = {
                enabled = true;
                name = "HyperHive";
                client_id = cfg.oidc.clientId;
                # ⚠️ `$__file{}`, never the secret itself — anything else
                # here is world-readable in the nix store. Note the module's
                # own leak assertion does NOT cover this key (it checks
                # `database.password`, `security.admin_password` and
                # datasource `secureJsonData`), so nothing but this comment
                # stands between a literal and the store.
                #
                # `swarm-bao-grafana-oidc.service` above is what writes this
                # path, on every host that runs Grafana.
                client_secret = "$__file{${secretPath}}";
                scopes = "openid profile email groups";
                auth_url = "${autheliaUrl}/api/oidc/authorization";
                token_url = "${autheliaUrl}/api/oidc/token";
                api_url = "${autheliaUrl}/api/oidc/userinfo";
                use_pkce = true;
              };
            };

            provision.datasources.settings = {
              apiVersion = 1;
              datasources = [
                {
                  name = "VictoriaMetrics";
                  type = "prometheus";
                  uid = datasourceUid;
                  url = deployCfg.grafana.datasourceUrl;
                  access = "proxy";
                  isDefault = true;
                }
                {
                  name = "VictoriaLogs";
                  # The plugin's own id, read from its `plugin.json` rather
                  # than guessed. A `type` Grafana does not recognise is
                  # provisioned without complaint and every query against it
                  # fails at use, far from anything naming this file.
                  type = "victoriametrics-logs-datasource";
                  uid = logsDatasourceUid;
                  url = deployCfg.grafana.logsDatasourceUrl;
                  access = "proxy";
                  # Exactly one datasource may claim this, and metrics has it:
                  # two defaults is a coin toss over which one an untyped panel
                  # gets.
                  isDefault = false;
                }
              ];
            };

            provision.dashboards.settings = {
              apiVersion = 1;
              providers = [
                {
                  name = "hyperhive";
                  type = "file";
                  # Read-only in the UI: the store path is immutable, so an edit
                  # saved here would be silently discarded on the next rebuild.
                  # Better to refuse the edit than to lose it.
                  allowUiUpdates = false;
                  options.path = dashboardDir;
                }
              ];
            };
          };
        };
    };
  };
}