# The hive's own nftables tables, `inet hive_egress` and `bridge hive_egress`,
# loaded by a systemd oneshot. They are independent of the NixOS firewall:
# `networking.firewall` renders only the rules of the backend in use
# (`iptables` unless `networking.nftables.enable` is set), so a rule that has
# to hold on every host lives here, in tables no firewall backend writes. Only
# `networking.nftables.flushRuleset` removes them; the unit re-applies them then.
#
# The `inet` table holds two chains. `input`: traffic from the agent bridge to host
# loopback is dropped. The kernel does the same by default (a packet for
# 127.0.0.0/8 arriving on a non-loopback interface is a martian while
# `route_localnet` is 0); this rule is the second layer, and the assertion
# below keeps the first one on. `forward` (`egress.forward = "log"`): traffic
# the host routes from the agent bridge to anywhere else is logged and
# counted, never blocked.
#
# The `bridge` table checks the source address of every frame a port of the
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
# traffic between two ports is bridged: the `inet` hooks see it as coming from
# the bridge device, and only a bridge hook knows the port it came in on
# (`iifname` is the veth). Which MAC a port may use and which IPv4 address a MAC
# may use are not known until a container runs, so they are set contents.
# `hive-priv egress-load` puts the current ones into the same transaction as the
# tables; `egress-pin` (per new port from udev, and from a timer for the ports
# still unbound) and dnsmasq's `dhcp-script` (per lease) keep them current.
# `hive-priv`'s `egress.rs` documents how.
{
  lib,
  config,
  pkgs,
  ...
}:
let
  cfg = config.services.hyperhive.network;
  nft = lib.getExe pkgs.nftables;
  br = cfg.bridgeName;

  # The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress.
  meshInterface = "wg-hive";

  # The sub-commands of hive-priv that fill the bridge table's sets. The path is
  # the one every platform container's unit uses (hive-priv.nix points it at the
  # current build).
  hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
  pinCommand = args: lib.trim "${hivePriv} egress-pin ${args}";
  loadCommand = "${hivePriv} egress-load ${ruleset}";
  pinPath = [
    pkgs.nftables
    pkgs.iproute2
    pkgs.util-linux
  ];
  pinEnvironment.HIVE_EGRESS_BRIDGE = br;
  leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
    export PATH=${lib.makeBinPath [ pkgs.nftables ]}
    exec ${hivePriv} egress-lease "$@"
  '';

  # Logging is per source address, so one busy agent cannot use up the budget
  # of another: a burst of 20 lines, then one a second, and an address is
  # forgotten after a minute without a new flow. The rule accepts after it
  # logs. A flow the rule cannot log falls through to the unmetered rule below
  # it: the address was over its rate, or the set is full (`update` then fails
  # and the rule stops matching), which a container spoofing source addresses
  # can bring about. Only that case reaches the unmetered rule, so a flow
  # gets one line at most. The unmetered rule has one budget for all
  # addresses, 5 lines a second after a burst of 50, about 100 MB of journal a
  # day when it never rests.
  logRate = "1/second burst 20 packets";
  unmeteredRate = "5/second burst 50 packets";
  fromBridge = ''iifname "${br}" oifname != "${br}"'';
  logRule =
    family: addr:
    ''${fromBridge} ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "hive-egress-would-drop " accept'';

  # Priority `filter - 10` runs before the NixOS firewall's forward handling,
  # which sits at `filter` on both backends (`FORWARD` of the iptables backend,
  # the `forward` chain of `nixos-fw`). An `accept` here ends this chain only,
  # so the chain neither allows what those refuse nor refuses anything itself.
  # Traffic to the host's own addresses (squid, dnsmasq, the gateway) is
  # delivered locally and never reaches the forward hook.
  forwardChain = ''
    set log_v4 { type ipv4_addr; flags dynamic; timeout 1m; size 4096; }
    set log_v6 { type ipv6_addr; flags dynamic; timeout 1m; size 4096; }

    chain forward {
      type filter hook forward priority filter - 10; policy accept;
      iifname "${br}" oifname "${meshInterface}" accept
      ${fromBridge} counter
      ${logRule "v4" "ip"}
      ${logRule "v6" "ip6"}
      ${fromBridge} ct state new limit rate ${unmeteredRate} log prefix "hive-egress-would-drop-unmetered "
      ${fromBridge} accept
    }
  '';

  # The bridge table. A frame of a port goes through `prerouting` in this
  # order: the port must be bound to the frame's source MAC, then the frame's
  # protocol decides which source address it may carry. Everything else ends in
  # `spoof`, which logs and then drops or, with `antiSpoof = "log"`, accepts.
  #
  # - `port_mac` and `port_chaddr` hold the bound (port, MAC) pairs, the second
  #   as the 48 bits that sit at the `chaddr` field of a DHCP message. A port
  #   with no pair sends nothing, DHCP included.
  # - `lease_v4` holds the leased (MAC, IPv4) pairs; each times out with its
  #   lease.
  # - IPv6 has no leases: a source is link-local, or `::` in the packets that
  #   duplicate address detection and multicast listener reports send before an
  #   address exists. The bridge carries no other IPv6 traffic.
  # - DHCP: any message to port 67 must carry the sender's own MAC as `chaddr`,
  #   so a port cannot lease, renew or release another's address; a client with
  #   no address yet sends from 0.0.0.0 to the broadcast address.
  # - ARP: the sender's MAC is the port's, and its address 0.0.0.0 (a probe) or
  #   leased to that MAC.
  # - A VLAN tag is refused: a tagged frame would carry its packet past the
  #   protocol checks. The bridge uses no VLANs.
  #
  # Logging is per ingress port, in the style of the forward chain: a burst of
  # 20 lines, then one a second, and a port is forgotten after a minute. A
  # packet its port cannot log (over its rate, or the set is full) falls to the
  # unmetered line, which has one budget for all ports. It logs
  # packets, not flows: the bridge hook has no connection tracking.
  spoofVerdict = if cfg.egress.antiSpoof == "drop" then "drop" else "accept";
  bridgeTable = ''
    table bridge hive_egress {
      set port_mac { type ifname . ether_addr; }
      set port_chaddr { typeof iifname . @th,288,48; }
      set lease_v4 { type ether_addr . ipv4_addr; flags timeout; size 65536; }
      set spoof_ports { type ifname; flags dynamic; timeout 1m; size 4096; }

      chain spoof {
        update @spoof_ports { iifname limit rate ${logRate} } counter log prefix "hive-egress-spoof " ${spoofVerdict}
        limit rate ${unmeteredRate} counter log prefix "hive-egress-spoof-unmetered " ${spoofVerdict}
        counter ${spoofVerdict}
      }

      chain l3_v4 {
        udp dport 67 iifname . @th,288,48 != @port_chaddr goto spoof
        ip saddr 0.0.0.0 ip daddr 255.255.255.255 udp sport 68 udp dport 67 accept
        ether saddr . ip saddr @lease_v4 accept
        goto spoof
      }

      chain l3_v6 {
        ip6 saddr fe80::/10 accept
        ip6 saddr :: icmpv6 type { nd-neighbor-solicit, mld2-listener-report } accept
        goto spoof
      }

      chain l3_arp {
        iifname . arp saddr ether != @port_mac goto spoof
        arp saddr ip 0.0.0.0 accept
        arp saddr ether . arp saddr ip @lease_v4 accept
        goto spoof
      }

      chain prerouting {
        type filter hook prerouting priority -300; policy accept;
        meta ibrname != "${br}" accept
        iifname . ether saddr != @port_mac goto spoof
        ether type { 8021q, 8021ad } goto spoof
        ether type ip goto l3_v4
        ether type ip6 goto l3_v6
        ether type arp goto l3_arp
      }
    }
  '';

  # One transaction: `add` + `delete` drop an old copy of each table and the
  # definitions recreate them, so the file loads the same on first start, on a
  # reload and after something else flushed the ruleset.
  rulesetText = ''
    add table inet hive_egress
    delete table inet hive_egress
    add table bridge hive_egress
    delete table bridge hive_egress

    table inet hive_egress {
      chain input {
        type filter hook input priority filter - 10; policy accept;
        iifname "${br}" ip daddr 127.0.0.0/8 counter drop
        iifname "${br}" ip6 daddr ::1 counter drop
      }
      ${lib.optionalString (cfg.egress.forward == "log") forwardChain}
    }

    ${bridgeTable}
  '';

  routeLocalnetKeys = [
    "net.ipv4.conf.all.route_localnet"
    "net.ipv4.conf.default.route_localnet"
    "net.ipv4.conf.${cfg.bridgeName}.route_localnet"
  ];
  isOn =
    v:
    builtins.elem v [
      1
      "1"
      true
    ];
  ruleset = pkgs.writeText "hive-egress.nft" cfg.egress.ruleset;
in
{
  options.services.hyperhive.network.egress.forward = lib.mkOption {
    type = lib.types.enum [
      "accept"
      "log"
      "drop"
    ];
    default = "log";
    description = ''
      What the `inet hive_egress` table does with traffic the host forwards
      from the agent bridge to anywhere but the bridge and the mesh.

      - `accept`: nothing; the table has no forward chain.
      - `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
        rate limited per source address, and counted. A flow the per-source limit
        cannot log gets a line with the prefix `hive-egress-would-drop-unmetered `
        from a limit shared by all addresses. Nothing is blocked. The source
        address maps to an agent through the hive's DHCP leases.
      - `drop`: not available yet; the evaluation refuses it.
    '';
  };

  options.services.hyperhive.network.egress.antiSpoof = lib.mkOption {
    type = lib.types.enum [
      "drop"
      "log"
    ];
    default = "drop";
    description = ''
      What the `bridge hive_egress` table does with a frame whose source is not
      the sending container's own. Every port of the agent bridge is checked:
      the frame's MAC must be the one bound to the port when its container
      started, and its IPv4 address the one leased to that MAC. A port with no
      bound MAC sends nothing. Allowed besides the leased address: DHCP from
      `0.0.0.0` whose `chaddr` is the port's MAC, ARP probes, and IPv6 link-local
      sources (and `::` in neighbour discovery and listener reports).

      - `drop`: the frame is dropped and logged with the prefix
        `hive-egress-spoof `, rate limited per port. A packet the per-port limit
        cannot log gets a line with the prefix `hive-egress-spoof-unmetered `
        from a limit shared by all ports. A container sending from its leased
        address is never affected, so nothing legitimate breaks.
      - `log`: the same lines and counters, and the frame goes through. For
        finding out why a legitimate container is refused on a live hive.
    '';
  };

  options.services.hyperhive.network.egress.leaseScript = lib.mkOption {
    type = lib.types.path;
    internal = true;
    readOnly = true;
    default = leaseScript;
    defaultText = lib.literalMD "the script that loads a lease into the bridge table";
    description = ''
      The program dnsmasq runs on each lease change (its `dhcp-script`).
    '';
  };

  options.services.hyperhive.network.egress.ruleset = lib.mkOption {
    type = lib.types.lines;
    internal = true;
    readOnly = true;
    default = rulesetText;
    defaultText = lib.literalMD "the `inet hive_egress` and `bridge hive_egress` table definitions";
    description = ''
      The nft file the `hive-egress` unit loads, as text.
    '';
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = cfg.egress.forward != "drop";
        message = ''
          services.hyperhive.network.egress.forward = "drop" is not available yet:
          the interception that lets an agent's web traffic through squid has not
          landed, so a drop would cut every direct connection with no replacement.
          Use "log" or "accept".
        '';
      }
      {
        assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys);
        message = ''
          boot.kernel.sysctl must not turn on route_localnet (${lib.concatStringsSep ", " routeLocalnetKeys}).
          With it on, the kernel routes a packet from the agent bridge to 127.0.0.0/8
          instead of dropping it as a martian, which leaves the hive_egress
          table as the only barrier between an agent and the host's loopback services.
        '';
      }
    ];

    systemd.services.hive-egress = {
      description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)";
      wantedBy = [ "multi-user.target" ];
      wants = [ "network-pre.target" ];
      before = [ "network-pre.target" ];
      # `nftables.service` runs `flush ruleset` where `networking.nftables.flushRuleset`
      # is on: start after it, follow its restarts and re-apply on its reload.
      # The tmpfiles units link `hivePriv`.
      after = [
        "nftables.service"
        "systemd-tmpfiles-setup.service"
        "systemd-tmpfiles-resetup.service"
      ];
      partOf = [ "nftables.service" ];
      unitConfig.ReloadPropagatedFrom = [ "nftables.service" ];
      reloadIfChanged = true;
      path = pinPath;
      environment = pinEnvironment;
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        # The tables load with the bindings of the running containers' ports and
        # the leases of dnsmasq's file in the same transaction, so a reload never
        # leaves the bridge with empty sets.
        ExecStart = loadCommand;
        ExecReload = loadCommand;
        ExecStop = [
          "-${nft} delete table inet hive_egress"
          "-${nft} delete table bridge hive_egress"
        ];
      };
    };

    # Binds one port of the bridge once its veth exists: the run waits for the
    # container to register with systemd-machined.
    systemd.services."hive-egress-pin@" = {
      description = "bind agent bridge port %i to its container's MAC";
      wants = [ "hive-egress.service" ];
      after = [ "hive-egress.service" ];
      path = pinPath;
      environment = pinEnvironment;
      serviceConfig = {
        Type = "oneshot";
        ExecStart = pinCommand "%i";
      };
    };

    # Looks again at the ports still unbound. A port a rival MAC kept unbound
    # sends nothing, so nothing else would tell the hive it can be bound now.
    # Ports with a binding cost a listing of the bridge and no container is read.
    systemd.services.hive-egress-pin = {
      description = "bind the agent bridge ports that have no binding yet";
      wants = [ "hive-egress.service" ];
      after = [ "hive-egress.service" ];
      path = pinPath;
      environment = pinEnvironment;
      serviceConfig = {
        Type = "oneshot";
        ExecStart = pinCommand "";
      };
    };
    systemd.timers.hive-egress-pin = {
      description = "bind the agent bridge ports that have no binding yet";
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnBootSec = "1min";
        OnUnitActiveSec = "30s";
        AccuracySec = "5s";
      };
    };

    # nspawn names the host end of a container's veth `vb-<machine>` when the
    # container joins a bridge.
    services.udev.extraRules = ''
      SUBSYSTEM=="net", ACTION=="add", KERNEL=="vb-*", TAG+="systemd", ENV{SYSTEMD_WANTS}+="hive-egress-pin@%k.service"
    '';
  };
}