# The swarm's backup archive: `rclone serve restic` on the host that sets
# `deploy.archive.enable`, behind the gateway at `archive.<swarm domain>`.
# Every hive's backup unit (./hive-archive-backup.nix) writes one restic
# repository per agent to it; nothing else uses it.
#
# - `--append-only`: a client cannot delete or overwrite a snapshot, a pack or
#   the index, so a compromised hive cannot erase history. Dropping history is
#   the prune unit's job, on this host, by retention.
# - `--private-repos`: a login reaches `/<its own name>/` only. There is one
#   login per agent, created by swarm-controller (../../swarm-controller/src/archive_access.rs)
#   when it places the agent, replaced when it moves and removed when it is
#   destroyed. An agent's hive can write that agent's repository and no other.
#
# Everything secret lives in the swarm secret store and is read when needed:
#
# - `swarm-archive-users` (root, every two minutes) renders the htpasswd file
#   from the users leaves into `/run`, which is tmpfs. rclone re-reads the file
#   when it changes. The leaves hold password hashes only.
#   The unit also creates `swarm/services/archive/ready`, which tells
#   swarm-controller an archive exists: it grants no credentials before that.
# - `swarm-archive-prune` (daily) reads each agent's restic password through
#   `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
#   therefore decrypt every backup, which is the cost of pruning where clients
#   cannot delete.
#
# Both units log in as the archive's own store principal (a leaf of the store's
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
# no secret crosses a mount, and the backend is one string
# (`reposDir`, in ./lib/archive-server.nix) that names where the repositories
# are.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;
  cfg = hyperhiveCfg.deploy.archive;
  archiveCfg = hyperhiveCfg.swarm.archive;
  baoCfg = hyperhiveCfg.swarm.bao;
  baoDeploy = hyperhiveCfg.deploy.bao;

  swarmDomain = hyperhiveCfg.swarm.domain;
  # Total on a null swarm domain, as the sibling options modules are: the
  # required-domain assertion in hive-network.nix is what an operator sees.
  domainBase = if swarmDomain == null then "invalid" else swarmDomain;

  user = "swarm-archive";
  authDir = "/run/swarm-archive-auth";
  authFile = "${authDir}/htpasswd";
  reposDir = "${cfg.dataDir}/repos";

  server = import ./lib/archive-server.nix { inherit lib; } {
    inherit (cfg) retention;
    inherit authFile reposDir;
    rclone = cfg.package;
    restic = cfg.resticPackage;
    addr = "127.0.0.1:${toString cfg.port}";
    authGroup = user;
  };

  storeRetry = import ./lib/store-retry.nix { };

  haveClientIdentity =
    baoDeploy.archiveClientCertFile != null && baoDeploy.archiveClientKeyFile != null;

  storeEnvironment = {
    BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
  }
  // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
    BAO_CACERT = baoDeploy.serverCaFile;
  };
in
{
  options.services.hyperhive.swarm.archive.ownDomain = lib.mkOption {
    type = lib.types.str;
    readOnly = true;
    default = "archive.${domainBase}";
    defaultText = lib.literalExpression ''"archive.''${services.hyperhive.swarm.domain}"'';
    description = ''
      Read-only: the name the swarm's backup archive is served under and every
      hive's backup unit dials. A sibling of the swarm's other service names, so
      the swarm-services sub-CA can issue for it, and in `swarm.serviceDomains`
      on every host whether or not one runs the archive.
    '';
  };

  options.services.hyperhive.deploy.archive = {
    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.rclone;
      defaultText = lib.literalExpression "pkgs.rclone";
      description = "The rclone package: `rclone serve restic`.";
    };

    resticPackage = lib.mkOption {
      type = lib.types.package;
      default = pkgs.restic;
      defaultText = lib.literalExpression "pkgs.restic";
      description = "The restic package the prune unit runs.";
    };

    port = lib.mkOption {
      type = lib.types.port;
      default = 8911;
      description = ''
        Port rclone listens on, on the host's loopback. The gateway's vhost
        proxies to it; every login is checked by rclone itself.
      '';
    };

    dataDir = lib.mkOption {
      type = lib.types.path;
      default = "/var/lib/swarm-archive";
      description = ''
        Where the repositories live, under `repos/<agent>/`. Holds restic
        ciphertext: every backup of every agent, so size it for that.
      '';
    };

    retention = {
      keepWithin = lib.mkOption {
        type = lib.types.nullOr lib.types.str;
        default = "3d";
        example = "14d";
        description = ''
          Every snapshot younger than this is kept (restic's `--keep-within`,
          written `1y2m3d4h`). It bounds how fast history can be evicted: a
          hive that appends many snapshots cannot push the last few days out.
          `null` leaves the flag off.
        '';
      };

      keepLast = lib.mkOption {
        type = lib.types.ints.positive;
        default = 3;
        description = ''
          The newest this many snapshots of each agent are kept, however old
          (restic's `--keep-last`). Together with
          {option}`services.hyperhive.deploy.archive.retention.keepWithin` the
          prune keeps a snapshot either rule keeps.
        '';
      };
    };

    pruneSchedule = lib.mkOption {
      type = lib.types.str;
      default = "daily";
      example = "*-*-* 04:00:00";
      description = "systemd calendar expression for the prune: the retention rules are applied then.";
    };
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = haveClientIdentity;
        message = ''
          services.hyperhive.deploy.archive.enable requires the archive's
          swarm-secret-store client identity: set both

            services.hyperhive.deploy.bao.archiveClientCertFile
            services.hyperhive.deploy.bao.archiveClientKeyFile

          The archive reads its users and the repositories' passwords in the
          store with it.

          Both default to the pair glue-bao-tls.nix mints when
          services.hyperhive.deploy.bao.enable is true (the host that runs the
          store). Otherwise copy archive.pem and archive-key.pem from the store
          host's /var/lib/swarm-bao-pki/ and name them in those two options.
        '';
      }
    ];

    users.users.${user} = {
      isSystemUser = true;
      group = user;
      description = "swarm backup archive";
    };
    users.groups.${user} = { };

    # This host serves the vhost, and the host's own resolver finds the name.
    services.hyperhive.gateway.localNames = [ archiveCfg.ownDomain ];
    services.hyperhive.gateway.enable = lib.mkDefault true;
    services.hyperhive.gateway.dns.enable = lib.mkDefault true;

    # `forceSSL`: the logins ride in a header. No `auth_request`: rclone checks
    # them, and restic cannot follow a login redirect. The vhost keeps the
    # gateway's default `public`: `public = false` needs the gateway's
    # `privateListenAddresses` set, and every request is authenticated either
    # way. A backup is one request that can run for minutes, so the body is
    # neither size-limited nor buffered.
    services.hyperhive.gateway.vhosts."${archiveCfg.ownDomain}" = {
      forceSSL = true;
      locations."/" = {
        proxyPass = "http://127.0.0.1:${toString cfg.port}";
        extraConfig = ''
          client_max_body_size 0;
          proxy_request_buffering off;
          proxy_read_timeout 1h;
          proxy_send_timeout 1h;
        '';
      };
    };

    services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
      "swarm-archive.service"
      "swarm-archive-users.service"
      "swarm-archive-prune.service"
    ];

    systemd.tmpfiles.rules = [
      "d ${cfg.dataDir} 0750 ${user} ${user} -"
      "d ${reposDir} 0750 ${user} ${user} -"
      "d ${authDir} 0750 root ${user} -"
    ];

    systemd.services.swarm-archive = {
      description = "swarm backup archive (rclone serve restic, append-only, one repository per agent)";
      wantedBy = [ "multi-user.target" ];
      wants = [ "swarm-archive-users.service" ];
      after = [
        "network.target"
        "swarm-archive-users.service"
        "systemd-tmpfiles-setup.service"
      ];
      # rclone writes no config, and reads none.
      environment.RCLONE_CONFIG = "/dev/null";
      serviceConfig = {
        ExecStart = server.serverCommand;
        User = user;
        Group = user;
        Restart = "on-failure";
        RestartSec = 5;
        SyslogIdentifier = "swarm-archive";
        NoNewPrivileges = true;
        ProtectSystem = "strict";
        ProtectHome = true;
        PrivateTmp = true;
        PrivateDevices = true;
        ProtectKernelTunables = true;
        ProtectControlGroups = true;
        RestrictAddressFamilies = [
          "AF_INET"
          "AF_INET6"
        ];
        ReadWritePaths = [ reposDir ];
        ReadOnlyPaths = [ authDir ];
      };
    };

    # The users file, from the store. A failed run leaves the file as it is;
    # the timer and the retry bring a store that comes back up to date.
    systemd.services.swarm-archive-users = {
      description = "render the archive's users file from the swarm secret store";
      wantedBy = [ "multi-user.target" ];
      wants = [ "network-online.target" ];
      after = [
        "network-online.target"
        "systemd-tmpfiles-setup.service"
      ];
      path = [
        baoDeploy.package
        pkgs.coreutils
        pkgs.gnugrep
        pkgs.jq
        pkgs.systemd
      ];
      environment = storeEnvironment // {
        BAO_CLIENT_CERT = baoDeploy.archiveClientCertFile;
        BAO_CLIENT_KEY = baoDeploy.archiveClientKeyFile;
      };
      inherit (storeRetry) startLimitBurst startLimitIntervalSec;
      serviceConfig = storeRetry.serviceConfig // {
        Type = "oneshot";
        UMask = "0027";
        SyslogIdentifier = "swarm-archive-users";
        TimeoutStartSec = "5min";
      };
      script = server.usersScript;
    };

    # A new grant reaches the server within the controller's pass (five
    # minutes) plus this interval.
    systemd.timers.swarm-archive-users = {
      description = "re-render the archive's users file";
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnUnitInactiveSec = "2min";
        RandomizedDelaySec = "20s";
      };
    };

    systemd.services.swarm-archive-prune = {
      description = "apply the archive's retention to every agent's repository";
      after = [ "network-online.target" ];
      wants = [ "network-online.target" ];
      path = [
        baoDeploy.package
        cfg.resticPackage
        pkgs.coreutils
        pkgs.gnugrep
        pkgs.jq
      ];
      # The unit runs as the repositories' owner, so the store identity comes
      # in as credentials, which only it can read.
      environment = {
        BAO_ADDR = storeEnvironment.BAO_ADDR;
        BAO_CLIENT_CERT = "%d/bao-cert";
        BAO_CLIENT_KEY = "%d/bao-key";
        RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
      }
      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
        BAO_CACERT = "%d/bao-ca";
      };
      serviceConfig = {
        Type = "oneshot";
        User = user;
        Group = user;
        SyslogIdentifier = "swarm-archive-prune";
        CacheDirectory = "swarm-archive-prune";
        LoadCredential = [
          "bao-cert:${baoDeploy.archiveClientCertFile}"
          "bao-key:${baoDeploy.archiveClientKeyFile}"
        ]
        ++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
        NoNewPrivileges = true;
        ProtectSystem = "strict";
        ProtectHome = true;
        PrivateTmp = true;
        PrivateDevices = true;
        ReadWritePaths = [ reposDir ];
        TimeoutStartSec = "6h";
      };
      script = server.pruneScript;
    };

    systemd.timers.swarm-archive-prune = {
      description = "apply the archive's retention";
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnCalendar = cfg.pruneSchedule;
        RandomizedDelaySec = "30min";
        Persistent = true;
      };
    };
  };
}