{
 "type": "excalidraw",
 "version": 2,
 "source": "https://excalidraw.com",
 "elements": [
  {
   "id": "q5-agent-evals-badge",
   "type": "rectangle",
   "x": 0,
   "y": 0,
   "width": 1500,
   "height": 300,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffc078",
   "fillStyle": "solid",
   "strokeWidth": 4,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 3
   },
   "seed": 740223520,
   "version": 1,
   "versionNonce": 607991153,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-badge-t"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "q5-agent-evals-badge-t",
   "type": "text",
   "x": 20,
   "y": 25,
   "width": 1460,
   "height": 250,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 2033636663,
   "version": 1,
   "versionNonce": 516841822,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q5 Evals",
   "originalText": "Q5 Evals",
   "fontSize": 200,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-badge",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-title",
   "type": "text",
   "x": 0,
   "y": 330,
   "width": 1400,
   "height": 40,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 265236939,
   "version": 1,
   "versionNonce": 1400082662,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q5 Evaluating the agent: replay past incidents in a sandbox",
   "originalText": "Q5 Evaluating the agent: replay past incidents in a sandbox",
   "fontSize": 32,
   "fontFamily": 5,
   "textAlign": "left",
   "verticalAlign": "middle",
   "containerId": null,
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-inc",
   "type": "rectangle",
   "x": 0,
   "y": 550,
   "width": 180,
   "height": 72,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#f3f0ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 3
   },
   "seed": 2072265938,
   "version": 1,
   "versionNonce": 455736474,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-inc-t"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e0"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "q5-agent-evals-inc-t",
   "type": "text",
   "x": 10,
   "y": 566.0,
   "width": 160,
   "height": 40,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 244051093,
   "version": 1,
   "versionNonce": 1067275002,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Labeled incidents\n(alert + true root cause)",
   "originalText": "Labeled incidents\n(alert + true root cause)",
   "fontSize": 16,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-inc",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-snap",
   "type": "rectangle",
   "x": 330,
   "y": 700,
   "width": 180,
   "height": 72,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#f3f0ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 3
   },
   "seed": 821653593,
   "version": 1,
   "versionNonce": 1189349777,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-snap-t"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e1"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "q5-agent-evals-snap-t",
   "type": "text",
   "x": 340,
   "y": 716.0,
   "width": 160,
   "height": 40,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 181242850,
   "version": 1,
   "versionNonce": 419812754,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Telemetry snapshots\n(frozen at incident time)",
   "originalText": "Telemetry snapshots\n(frozen at incident time)",
   "fontSize": 16,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-snap",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-sim",
   "type": "rectangle",
   "x": 330,
   "y": 550,
   "width": 180,
   "height": 72,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ebfbee",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 3
   },
   "seed": 1942080813,
   "version": 1,
   "versionNonce": 119682499,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-sim-t"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e1"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e2"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "q5-agent-evals-sim-t",
   "type": "text",
   "x": 340,
   "y": 566.0,
   "width": 160,
   "height": 40,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 272167094,
   "version": 1,
   "versionNonce": 1903737355,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Simulated tools\n(serve snapshot data)",
   "originalText": "Simulated tools\n(serve snapshot data)",
   "fontSize": 16,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-sim",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-run",
   "type": "rectangle",
   "x": 660,
   "y": 550,
   "width": 180,
   "height": 72,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#e6fcf5",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 3
   },
   "seed": 1398499908,
   "version": 1,
   "versionNonce": 856430244,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-run-t"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e0"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e2"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e3"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "q5-agent-evals-run-t",
   "type": "text",
   "x": 670,
   "y": 566.0,
   "width": 160,
   "height": 40,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 1190502837,
   "version": 1,
   "versionNonce": 1942810372,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Eval runner\n(N incidents x M agent versions)",
   "originalText": "Eval runner\n(N incidents x M agent versions)",
   "fontSize": 16,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-run",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-judge",
   "type": "rectangle",
   "x": 990,
   "y": 550,
   "width": 180,
   "height": 72,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ebfbee",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 3
   },
   "seed": 2053150340,
   "version": 1,
   "versionNonce": 1063673567,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-judge-t"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e3"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e4"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "q5-agent-evals-judge-t",
   "type": "text",
   "x": 1000,
   "y": 556.0,
   "width": 160,
   "height": 60,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 1114940396,
   "version": 1,
   "versionNonce": 870111104,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Scorer\n(exact match + LLM judge\n+ human review)",
   "originalText": "Scorer\n(exact match + LLM judge\n+ human review)",
   "fontSize": 16,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-judge",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-dash",
   "type": "ellipse",
   "x": 1320,
   "y": 550,
   "width": 180,
   "height": 72,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#e7f5ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 2
   },
   "seed": 1922119102,
   "version": 1,
   "versionNonce": 588987925,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-dash-t"
    },
    {
     "type": "arrow",
     "id": "q5-agent-evals-e4"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "q5-agent-evals-dash-t",
   "type": "text",
   "x": 1330,
   "y": 566.0,
   "width": 160,
   "height": 40,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 1789442529,
   "version": 1,
   "versionNonce": 522362321,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Results\n(accuracy, cost, latency)",
   "originalText": "Results\n(accuracy, cost, latency)",
   "fontSize": 16,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-dash",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-e0",
   "type": "arrow",
   "x": 180.0,
   "y": 586.0,
   "width": 480.0,
   "height": 0.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 2
   },
   "seed": 1685192165,
   "version": 1,
   "versionNonce": 1898882740,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     480.0,
     0.0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": {
    "elementId": "q5-agent-evals-inc",
    "focus": 0,
    "gap": 4
   },
   "endBinding": {
    "elementId": "q5-agent-evals-run",
    "focus": 0,
    "gap": 4
   },
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "q5-agent-evals-e1",
   "type": "arrow",
   "x": 420.0,
   "y": 700.0,
   "width": 0.0,
   "height": 78.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 2
   },
   "seed": 1357122901,
   "version": 1,
   "versionNonce": 311583110,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     0.0,
     -78.0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": {
    "elementId": "q5-agent-evals-snap",
    "focus": 0,
    "gap": 4
   },
   "endBinding": {
    "elementId": "q5-agent-evals-sim",
    "focus": 0,
    "gap": 4
   },
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "q5-agent-evals-e2",
   "type": "arrow",
   "x": 660.0,
   "y": 586.0,
   "width": 150.0,
   "height": 0.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 2
   },
   "seed": 1033535610,
   "version": 1,
   "versionNonce": 1839700616,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-e2-t"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     -150.0,
     0.0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": {
    "elementId": "q5-agent-evals-run",
    "focus": 0,
    "gap": 4
   },
   "endBinding": {
    "elementId": "q5-agent-evals-sim",
    "focus": 0,
    "gap": 4
   },
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "q5-agent-evals-e2-t",
   "type": "text",
   "x": 545.0,
   "y": 576.0,
   "width": 80,
   "height": 20,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 314051310,
   "version": 1,
   "versionNonce": 913495727,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "tool calls",
   "originalText": "tool calls",
   "fontSize": 14,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-e2",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-e3",
   "type": "arrow",
   "x": 840.0,
   "y": 586.0,
   "width": 150.0,
   "height": 0.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 2
   },
   "seed": 1300430509,
   "version": 1,
   "versionNonce": 525488742,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-e3-t"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     150.0,
     0.0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": {
    "elementId": "q5-agent-evals-run",
    "focus": 0,
    "gap": 4
   },
   "endBinding": {
    "elementId": "q5-agent-evals-judge",
    "focus": 0,
    "gap": 4
   },
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "q5-agent-evals-e3-t",
   "type": "text",
   "x": 871.0,
   "y": 576.0,
   "width": 88,
   "height": 20,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 663343194,
   "version": 1,
   "versionNonce": 1572745252,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "transcripts",
   "originalText": "transcripts",
   "fontSize": 14,
   "fontFamily": 5,
   "textAlign": "center",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-e3",
   "lineHeight": 1.25,
   "autoResize": true
  },
  {
   "id": "q5-agent-evals-e4",
   "type": "arrow",
   "x": 1170.0,
   "y": 586.0,
   "width": 150.0,
   "height": 0.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": {
    "type": 2
   },
   "seed": 614090117,
   "version": 1,
   "versionNonce": 1087089873,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     150.0,
     0.0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": {
    "elementId": "q5-agent-evals-judge",
    "focus": 0,
    "gap": 4
   },
   "endBinding": {
    "elementId": "q5-agent-evals-dash",
    "focus": 0,
    "gap": 4
   },
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "q5-agent-evals-notes",
   "type": "rectangle",
   "x": 0,
   "y": 850,
   "width": 420,
   "height": 176,
   "angle": 0,
   "strokeColor": "#e67700",
   "backgroundColor": "#ffec99",
   "fillStyle": "solid",
   "strokeWidth": 1,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 589504031,
   "version": 1,
   "versionNonce": 2008910112,
   "isDeleted": false,
   "boundElements": [
    {
     "type": "text",
     "id": "q5-agent-evals-notes-t"
    }
   ],
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "q5-agent-evals-notes-t",
   "type": "text",
   "x": 12,
   "y": 860,
   "width": 396,
   "height": 156,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "roundness": null,
   "seed": 943122534,
   "version": 1,
   "versionNonce": 404265717,
   "isDeleted": false,
   "boundElements": [],
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Trade-offs to raise\n\u2022 Freeze data so runs are reproducible\n\u2022 Score root cause, evidence and steps\n\u2022 Track cost and latency, not just accuracy\n\u2022 Gate releases on no regression\n\u2022 Grow the set from real misses",
   "originalText": "Trade-offs to raise\n\u2022 Freeze data so runs are reproducible\n\u2022 Score root cause, evidence and steps\n\u2022 Track cost and latency, not just accuracy\n\u2022 Gate releases on no regression\n\u2022 Grow the set from real misses",
   "fontSize": 16,
   "fontFamily": 5,
   "textAlign": "left",
   "verticalAlign": "middle",
   "containerId": "q5-agent-evals-notes",
   "lineHeight": 1.25,
   "autoResize": true
  }
 ],
 "appState": {
  "viewBackgroundColor": "#ffffff",
  "gridSize": 20
 },
 "files": {}
}