{
  "title": "hyperhive · binary cache",
  "uid": "hyperhive-attic",
  "description": "The swarm's binary cache: what the builder pushes, what each hive can pull, and the refusals.",
  "editable": true,
  "refresh": "1m",
  "schemaVersion": 39,
  "tags": ["hyperhive", "swarm", "attic", "cache"],
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "timezone": "utc",
  "version": 1,
  "templating": {
    "list": [
      {
        "name": "hive",
        "label": "Hive",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "@datasourceUid@"
        },
        "query": "label_values({__name__=\"hyperhive.agent.closure_pullable\"}, hive)",
        "refresh": 2,
        "includeAll": true,
        "allValue": ".*",
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "sort": 1
      },
      {
        "name": "agent",
        "label": "Agent",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "@datasourceUid@"
        },
        "query": "label_values({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\"}, agent)",
        "refresh": 2,
        "includeAll": true,
        "allValue": ".*",
        "multi": false,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "sort": 1
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "Pullable closures",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "panels": []
    },
    {
      "id": 2,
      "type": "stat",
      "title": "Agents pullable",
      "description": "Agents whose latest closure this hive can pull and verify. Source: closure_pullable gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 0,
        "y": 1
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "sum(max by (hive, agent) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m]))) or vector(0)",
          "instant": true,
          "legendFormat": "pullable"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "fixed",
            "fixedColor": "green"
          },
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "textMode": "value",
        "justifyMode": "auto",
        "reduceOptions": {
          "calcs": ["lastNotNull"],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 3,
      "type": "stat",
      "title": "Agents not pullable",
      "description": "Agents whose latest closure this hive cannot pull. Source: closure_pullable gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 6,
        "y": 1
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "count(max by (hive, agent) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m])) == 0) or vector(0)",
          "instant": true,
          "legendFormat": "not pullable"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "fixed",
            "fixedColor": "red"
          },
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "textMode": "value",
        "justifyMode": "auto",
        "reduceOptions": {
          "calcs": ["lastNotNull"],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 4,
      "type": "stat",
      "title": "Closures pushed (range)",
      "description": "Closures the builder pushed to the cache in the range. Source: swarm-builder logs.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 12,
        "y": 1
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure pushed to the cache\" | stats count_uniq(__CURSOR) as n",
          "queryType": "stats",
          "legendFormat": "pushed"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "fixed",
            "fixedColor": "green"
          },
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "textMode": "value",
        "justifyMode": "auto",
        "reduceOptions": {
          "calcs": ["lastNotNull"],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 5,
      "type": "stat",
      "title": "Push failures (range)",
      "description": "Closures the builder failed to push in the range. Source: swarm-builder logs.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 18,
        "y": 1
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\" | stats count_uniq(__CURSOR) as n",
          "queryType": "stats",
          "legendFormat": "failed"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "fixed",
            "fixedColor": "red"
          },
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "textMode": "value",
        "justifyMode": "auto",
        "reduceOptions": {
          "calcs": ["lastNotNull"],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 6,
      "type": "table",
      "title": "Pull verdict per agent",
      "description": "Per agent: whether the latest closure is pullable, and why not. Source: closure_pullable gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 5
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent, reason) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m]))",
          "instant": true,
          "format": "table"
        }
      ],
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true
            },
            "renameByName": {
              "agent": "Agent",
              "reason": "Reason",
              "Value": "Pullable"
            }
          }
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Pullable"
            },
            "properties": [
              {
                "id": "mappings",
                "value": [
                  {
                    "type": "value",
                    "options": {
                      "0": {
                        "text": "no",
                        "color": "red",
                        "index": 0
                      },
                      "1": {
                        "text": "yes",
                        "color": "green",
                        "index": 1
                      }
                    }
                  }
                ]
              },
              {
                "id": "custom.cellOptions",
                "value": {
                  "type": "color-text"
                }
              }
            ]
          }
        ]
      },
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "displayName": "Pullable",
            "desc": false
          }
        ]
      }
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "Pullable agents over time",
      "description": "Agents with a pullable closure against agents without. Source: closure_pullable gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 5
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "sum(max by (hive, agent) ({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"})) or vector(0)",
          "legendFormat": "pullable"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "count(max by (hive, agent) ({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}) == 0) or vector(0)",
          "legendFormat": "not pullable"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 10,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 10,
      "type": "row",
      "title": "Builder pushes",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 14
      },
      "panels": []
    },
    {
      "id": 11,
      "type": "timeseries",
      "title": "Pushes over time",
      "description": "Closures pushed and not pushed to the cache, per interval. Source: swarm-builder logs.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 15
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure pushed to the cache\" | stats count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "pushed"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\" | stats count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "not pushed"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "lineWidth": 1,
            "fillOpacity": 60,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 12,
      "type": "timeseries",
      "title": "Agent meta builds",
      "description": "Agent builds from the meta flake by outcome, per interval. Source: swarm-builder logs.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 15
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"agent meta build built\" | stats count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "built"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" (\"agent meta build failed\" OR \"agent meta build not run\" OR \"meta build refused\") | stats count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "failed or refused"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "lineWidth": 1,
            "fillOpacity": 60,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 13,
      "type": "logs",
      "title": "Recent push failures",
      "description": "The builder's push-failure lines with the reason, newest first. Source: swarm-builder logs.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\"",
          "queryType": "instant"
        }
      ],
      "options": {
        "showTime": true,
        "wrapLogMessage": true,
        "prettifyLogMessage": false,
        "enableLogDetails": true,
        "sortOrder": "Descending",
        "dedupStrategy": "none"
      }
    },
    {
      "id": 20,
      "type": "row",
      "title": "Hive pulls",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 32
      },
      "panels": []
    },
    {
      "id": 21,
      "type": "stat",
      "title": "Closures substituted (range)",
      "description": "Closures hive-c0re pulled from the cache for a prebuild or create. Source: hive-c0re logs.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "h": 4,
        "w": 24,
        "x": 0,
        "y": 33
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" (\"prebuild: substituted\" OR \"create: substituted\") | stats count_uniq(__CURSOR) as n",
          "queryType": "stats",
          "legendFormat": "substituted"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "fixed",
            "fixedColor": "green"
          },
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "textMode": "value",
        "justifyMode": "auto",
        "reduceOptions": {
          "calcs": ["lastNotNull"],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 23,
      "type": "timeseries",
      "title": "Substitutes and refusals",
      "description": "Closures pulled and deploy steps refused, per interval. Source: hive-c0re logs.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 37
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" (\"prebuild: substituted\" OR \"create: substituted\") | stats count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "substituted"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" \"nothing was changed\" (\"the swarm has recorded no build\" OR \"closure cannot be pulled\") | stats count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "refused"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "lineWidth": 1,
            "fillOpacity": 60,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 24,
      "type": "timeseries",
      "title": "Probe verdict changes by reason",
      "description": "Times the pull probe's verdict for an agent changed, by reason. Source: hive-c0re logs.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 45
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_AGENT:~\"$agent\" \"attic: can this hive pull\" F_REASON:* | stats by (F_REASON) count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "{{F_REASON}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "lineWidth": 1,
            "fillOpacity": 60,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    }
  ]
}