infra/hosts/monitoring/prometheus.nix

{
  config,
  pkgs,
  lib,
  ...
}:

{
  networking.extraHosts = ''
    2001:678:560:23:3036:24ff:fecf:965b mqtt.club.berlin.ccc.de
    2001:678:560:23:be24:11ff:fe37:a49a forgejo-runner.club.berlin.ccc.de
  '';

  services.prometheus = {
    enable = true;
    retentionTime = "14d";
    listenAddress = "[::1]";
    scrapeConfigs = [
      {
        job_name = "synapse";
        scrape_interval = "15s";
        scheme = "https";
        metrics_path = "/_synapse/metrics";
        static_configs = [ { targets = [ "matrix.berlin.ccc.de:443" ]; } ];
      }
      {
        job_name = "postgres";
        scrape_interval = "15s";
        static_configs = [
          {
            targets = [ "sql.berlin.ccc.de:${toString config.services.prometheus.exporters.postgres.port}" ];
          }
        ];
      }
      {
        job_name = "node";
        scrape_interval = "15s";
        static_configs = [
          {
            targets = [
              "matrix.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
              "md.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
              "sql.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
              "monitoring.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
              "www.berlin.ccc.de:${toString config.services.prometheus.exporters.node.port}"
              "mqtt.club.berlin.ccc.de:9100"
              "forgejo-runner.club.berlin.ccc.de:9100"
            ];
          }
        ];
      }
      {
        job_name = "nginx";
        scrape_interval = "15s";
        static_configs = [
          {
            targets = [
              "matrix.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
              "md.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
              "monitoring.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
              "www.berlin.ccc.de:${toString config.services.prometheus.exporters.nginx.port}"
            ];
          }
        ];
      }
      {
        # https://github.com/prometheus-pve/prometheus-pve-exporter
        job_name = "pve";
        scrape_interval = "15s";
        static_configs = [
          {
            targets = [ "ellsberg.berlin.ccc.de" ];
          }
        ];
        metrics_path = "/pve";
        params = {
          module = [ "default" ];
          cluster = [ "1" ];
          node = [ "1" ];
        };
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:${toString config.services.prometheus.exporters.pve.port}";
          }
        ];
      }
    ];
    ruleFiles = [
      # https://github.com/element-hq/synapse/tree/master/contrib/prometheus
      (pkgs.writeText "prom-synapse-rules.yaml" ''
        groups:
        - name: synapse
          rules:
          - record: 'synapse_federation_client_sent'
            labels:
              type: "EDU"
            expr: 'synapse_federation_client_sent_edus_total + 0'
          - record: 'synapse_federation_client_sent'
            labels:
              type: "PDU"
            expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
          - record: 'synapse_federation_client_sent'
            labels:
              type: "Query"
            expr: 'sum(synapse_federation_client_sent_queries) by (job)'
          - record: 'synapse_federation_server_received'
            labels:
              type: "EDU"
            expr: 'synapse_federation_server_received_edus_total + 0'
          - record: 'synapse_federation_server_received'
            labels:
              type: "PDU"
            expr: 'synapse_federation_server_received_pdus_total + 0'
          - record: 'synapse_federation_server_received'
            labels:
              type: "Query"
            expr: 'sum(synapse_federation_server_received_queries) by (job)'
          - record: 'synapse_federation_transaction_queue_pending'
            labels:
              type: "EDU"
            expr: 'synapse_federation_transaction_queue_pending_edus + 0'
          - record: 'synapse_federation_transaction_queue_pending'
            labels:
              type: "PDU"
            expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
      '')
    ];
    exporters.pve = {
      enable = true;
      listenAddress = "[::1]";
      environmentFile = config.age.secrets.pve-exporter.path;
      #configFile = "";
    };
  };
}