<?xml version="1.0" encoding="UTF-8"?>
<mxfile host="eli.ai" agent="eli.ai diagram exporter" version="1">
  <diagram id="evaluation-framework-layers-0156cf52" name="Evaluation framework — layers">
    <mxGraphModel dx="1600" dy="1200" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1600" pageHeight="1200" math="0" shadow="0">
      <root>
        <mxCell id="0"/>
        <object id="1" label="Evaluation framework — layers" schemaVersion="1" diagramId="evaluation-framework-layers-0156cf52" canonicalAppearanceKey="tsx:src/app/(docs)/docs/concepts/evaluation-framework/page.tsx:layersChart#1" sourceSha256="0156cf52621e40fc47164e9f754fa644d27bc03009839e65f2e45add9c2ddc2c" metadataPayloadSha256="7a6f02964cc85981d79555735820a230f63ee012fe28d4d1ef7f5c5e7b85db9b" generationKey="3a5fb50301e3e7905edce18872604f2898efd83d8ea0e89c2de0b9db7136ad05" concepts="[&quot;Evaluation framework&quot;,&quot;Layers&quot;]" keywords="[&quot;eval runner&quot;,&quot;answer generation&quot;,&quot;RAGAS answer metrics&quot;,&quot;error classification&quot;,&quot;APPLIES the config (not snapshot-only)&quot;,&quot;understand · fuse · rerank · assemble&quot;,&quot;retrieval metrics per item&quot;,&quot;faithfulness · relevancy · context · noise&quot;,&quot;abstention + CRAG truthfulness&quot;,&quot;one TX: append feedback + create draft&quot;,&quot;steward review + separate activation&quot;,&quot;precision/recall&quot;,&quot;/evals/feedback/promote&quot;,&quot;retrievePipeline&quot;,&quot;perfect 1 · acceptable 0.5 · missing 0 · incorrect −1&quot;,&quot;retrieval_miss · hallucination · citation_error · stale · …&quot;,&quot;POST&quot;,&quot;recall@k · precision@k · nDCG@10 · MRR · hit@k&quot;]" application="{&quot;name&quot;:&quot;eli-ai&quot;,&quot;version&quot;:&quot;0.1.0&quot;}" packages="{&quot;mermaid&quot;:&quot;11.16.0&quot;,&quot;mermaidCli&quot;:&quot;11.16.0&quot;,&quot;playwright&quot;:&quot;1.62.1&quot;,&quot;playwrightCore&quot;:&quot;1.62.1&quot;,&quot;puppeteer&quot;:&quot;25.6.0&quot;,&quot;puppeteerCore&quot;:&quot;25.6.0&quot;}" environment="{&quot;architecture&quot;:&quot;x64&quot;,&quot;containerImage&quot;:&quot;playwright:v1.62.1-noble@sha256:c091b21d9fae78c76e85cd4356431e9b018402f172a214fc7d7a5e9a7e29d8ac+node:26.3.1-bookworm-slim@sha256:209558c3e5e40e8adff68959d3c2ba7cd73723e716d89196878a0c362fa1e109&quot;,&quot;node&quot;:&quot;v26.3.1&quot;,&quot;platform&quot;:&quot;linux&quot;,&quot;yarn&quot;:&quot;4.17.1&quot;}" renderer="{&quot;browser&quot;:{&quot;executableSha256&quot;:&quot;0b20b130e7edd9dd51873be867761295fe0cfad490c2b9a64f95bd3cfc08fa71&quot;,&quot;name&quot;:&quot;chromium&quot;,&quot;revision&quot;:&quot;1234&quot;,&quot;version&quot;:&quot;151.0.7922.34&quot;},&quot;font&quot;:{&quot;family&quot;:&quot;Eli Diagram Inter Variable&quot;,&quot;fileSha256&quot;:&quot;3100e775e8616cd2611beecfa23a4263d7037586789b43f035236a2e6fbd4c62&quot;,&quot;licenseSha256&quot;:&quot;3b0a5fca3d17942cde889069889dedbbbd075e9b599968c82a95f4d944e9b345&quot;},&quot;mermaidConfigSha256&quot;:&quot;68c10966fe84406ee626034d58bfabd555df9f65f691204b7c46db24038da101&quot;,&quot;themeCssSha256&quot;:&quot;c80287a78d80ad63d27bd5ca348b2ef9a7e2f44da289e436be6484ea28a1b033&quot;}" adapterVersions="{&quot;diagramGenerator&quot;:&quot;2&quot;,&quot;drawioFlowchart&quot;:&quot;1&quot;,&quot;drawioGantt&quot;:&quot;1&quot;,&quot;drawioSequence&quot;:&quot;1&quot;,&quot;drawioState&quot;:&quot;1&quot;}" mermaidType="flowchart" direction="TB" generatedAt="2026-08-12T23:38:39.196Z" mermaidSource="flowchart TB&#10;  GS[(&quot;golden set&lt;br/&gt;+ concept anchors + qrels&quot;)] --&gt; Runner[&quot;eval runner&lt;br/&gt;APPLIES the config (not snapshot-only)&quot;]&#10;  Runner --&gt; Pipe[&quot;retrievePipeline&lt;br/&gt;(understand · fuse · rerank · assemble)&quot;]&#10;  Pipe --&gt; Ret[&quot;retrieval metrics per item&lt;br/&gt;recall@k · precision@k · nDCG@10 · MRR · hit@k&quot;]&#10;  Pipe --&gt; Gen[&quot;answer generation&quot;]&#10;  Gen --&gt; Rag[&quot;RAGAS answer metrics&lt;br/&gt;faithfulness · relevancy · context precision/recall · noise&quot;]&#10;  Gen --&gt; Abs[&quot;abstention + CRAG truthfulness&lt;br/&gt;perfect 1 · acceptable 0.5 · missing 0 · incorrect −1&quot;]&#10;  Ret --&gt; Class[&quot;error classification&lt;br/&gt;retrieval_miss · hallucination · citation_error · stale · …&quot;]&#10;  Rag --&gt; Class&#10;  Abs --&gt; Class&#10;  Class --&gt; Sig[&quot;paired-bootstrap + permutation&lt;br/&gt;two-gate significance (CI ∌ 0 AND p&lt;0.05)&quot;]&#10;  Class -.-&gt; Review[&quot;human review&lt;br/&gt;corrected assertions + lineage&quot;]&#10;  Review --&gt; Promote[&quot;POST /evals/feedback/promote&lt;br/&gt;one TX: append feedback + create draft&quot;]&#10;  Promote --&gt; Steward[&quot;steward review + separate activation&quot;]&#10;  Steward -.-&gt; GS">
          <mxCell parent="0"/>
        </object>
        <object id="n:GS" label="golden set+ concept anchors + qrels" mermaidId="GS" mermaidShape="cylinder">
          <mxCell parent="1" vertex="1" style="backgroundOutline=1;boundedLbl=1;fillColor=#ECECFF;fontColor=#333333;html=1;shape=cylinder3;size=15;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="416.914063" y="8.000005" width="213.171875" height="109.277411" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Runner" label="eval runnerAPPLIES the config (not snapshot-only)" mermaidId="Runner" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="314" y="167.27742" width="256" height="100" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Pipe" label="retrievePipeline(understand · fuse · rerank · assemble)" mermaidId="Pipe" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="314" y="317.27742" width="256" height="100" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Ret" label="retrieval metrics per itemrecall@k · precision@k · nDCG@10 · MRR · hit@k" mermaidId="Ret" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="8" y="581.27742" width="256" height="100" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Gen" label="answer generation" mermaidId="Gen" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="496.6875" y="467.27742" width="196.625" height="52" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Rag" label="RAGAS answer metricsfaithfulness · relevancy · context precision/recall · noise" mermaidId="Rag" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="314" y="569.27742" width="256" height="124" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Abs" label="abstention + CRAG truthfulnessperfect 1 · acceptable 0.5 · missing 0 · incorrect −1" mermaidId="Abs" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="620" y="569.27742" width="256" height="124" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Class" label="error classificationretrieval_miss · hallucination · citation_error · stale · …" mermaidId="Class" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="314" y="743.27742" width="256" height="124" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Sig" label="paired-bootstrap + permutationtwo-gate significance (CI ∌ 0 AND p&lt;0.05)" mermaidId="Sig" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="161" y="917.27742" width="256" height="124" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Review" label="human reviewcorrected assertions + lineage" mermaidId="Review" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="620" y="929.27742" width="256" height="100" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Promote" label="POST /evals/feedback/promoteone TX: append feedback + create draft" mermaidId="Promote" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="620" y="1091.27742" width="256" height="124" as="geometry"/>
          </mxCell>
        </object>
        <object id="n:Steward" label="steward review + separate activation" mermaidId="Steward" mermaidShape="rectangle">
          <mxCell parent="1" vertex="1" style="fillColor=#ECECFF;fontColor=#333333;html=1;rounded=0;strokeColor=#9370DB;whiteSpace=wrap;">
            <mxGeometry x="701.5" y="1265.27742" width="256" height="76" as="geometry"/>
          </mxCell>
        </object>
        <object id="e:L_GS_Runner_0" label="" mermaidId="L_GS_Runner_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:GS" target="n:Runner" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="467.584292" y="114.93474" as="sourcePoint"/>
              <mxPoint x="442" y="167.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="442" y="142.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Runner_Pipe_0" label="" mermaidId="L_Runner_Pipe_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Runner" target="n:Pipe" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="442" y="267.27742" as="sourcePoint"/>
              <mxPoint x="442" y="317.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="442" y="292.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Pipe_Ret_0" label="" mermaidId="L_Pipe_Ret_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Pipe" target="n:Ret" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="314" y="398.649969" as="sourcePoint"/>
              <mxPoint x="136" y="581.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="136" y="442.27742"/>
                <mxPoint x="136" y="493.27742"/>
                <mxPoint x="136" y="544.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Pipe_Gen_0" label="" mermaidId="L_Pipe_Gen_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Pipe" target="n:Gen" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="544" y="417.27742" as="sourcePoint"/>
              <mxPoint x="595" y="467.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="595" y="442.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Gen_Rag_0" label="" mermaidId="L_Gen_Rag_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Gen" target="n:Rag" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="517" y="519.27742" as="sourcePoint"/>
              <mxPoint x="442" y="569.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="442" y="544.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Gen_Abs_0" label="" mermaidId="L_Gen_Abs_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Gen" target="n:Abs" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="673" y="519.27742" as="sourcePoint"/>
              <mxPoint x="748" y="569.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="748" y="544.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Ret_Class_0" label="" mermaidId="L_Ret_Class_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Ret" target="n:Class" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="136" y="681.27742" as="sourcePoint"/>
              <mxPoint x="314" y="768.885263" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="136" y="718.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Rag_Class_0" label="" mermaidId="L_Rag_Class_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Rag" target="n:Class" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="442" y="693.27742" as="sourcePoint"/>
              <mxPoint x="442" y="743.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="442" y="718.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Abs_Class_0" label="" mermaidId="L_Abs_Class_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Abs" target="n:Class" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="748" y="693.27742" as="sourcePoint"/>
              <mxPoint x="570" y="768.885263" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="748" y="718.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Class_Sig_0" label="" mermaidId="L_Class_Sig_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Class" target="n:Sig" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="332.965517" y="867.27742" as="sourcePoint"/>
              <mxPoint x="289" y="917.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="289" y="892.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Class_Review_0" label="" mermaidId="L_Class_Review_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Class" target="n:Review" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="570" y="841.669577" as="sourcePoint"/>
              <mxPoint x="748" y="929.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="748" y="892.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Review_Promote_0" label="" mermaidId="L_Review_Promote_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Review" target="n:Promote" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="748" y="1029.27742" as="sourcePoint"/>
              <mxPoint x="748" y="1091.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="748" y="1066.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Promote_Steward_0" label="" mermaidId="L_Promote_Steward_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Promote" target="n:Steward" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="748" y="1215.27742" as="sourcePoint"/>
              <mxPoint x="780.34127" y="1265.27742" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="748" y="1240.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
        <object id="e:L_Steward_GS_0" label="" mermaidId="L_Steward_GS_0" mermaidKind="relationship">
          <mxCell parent="1" source="n:Steward" target="n:GS" edge="1" style="edgeStyle=orthogonalEdgeStyle;endArrow=block;endFill=1;html=1;jettySize=auto;orthogonalLoop=1;rounded=0;strokeColor=#333333;">
            <mxGeometry relative="1" as="geometry">
              <mxPoint x="878.65873" y="1265.27742" as="sourcePoint"/>
              <mxPoint x="630.085938" y="84.544172" as="targetPoint"/>
              <Array as="points">
                <mxPoint x="911" y="1240.27742"/>
                <mxPoint x="911" y="1153.27742"/>
                <mxPoint x="911" y="1066.27742"/>
                <mxPoint x="911" y="979.27742"/>
                <mxPoint x="911" y="892.27742"/>
                <mxPoint x="911" y="805.27742"/>
                <mxPoint x="911" y="718.27742"/>
                <mxPoint x="911" y="631.27742"/>
                <mxPoint x="911" y="544.27742"/>
                <mxPoint x="911" y="493.27742"/>
                <mxPoint x="911" y="442.27742"/>
                <mxPoint x="911" y="367.27742"/>
                <mxPoint x="911" y="292.27742"/>
                <mxPoint x="911" y="217.27742"/>
                <mxPoint x="911" y="142.27742"/>
              </Array>
            </mxGeometry>
          </mxCell>
        </object>
      </root>
    </mxGraphModel>
  </diagram>
</mxfile>
