{
  "ontology": "evalglass",
  "version": "0.1.0",
  "statusCaveat": "Pre-alpha (v0.1.0, untagged, not yet on PyPI). These describe the current site/docs shapes of the typed contracts; structure may change before a 0.1 tag. This is a caveated catalog for orientation, NOT a stability or runtime-compatibility guarantee.",
  "source": "https://evalglass.com/examples",
  "contracts": [
    {
      "name": "Scorecard",
      "summary": "The bounded, readable claim of a run: one verdict plus the evidence behind it.",
      "fields": {
        "verdict": "VerdictPayload: verdict (informational|pass|fail|blocked), ci_should_fail, gate lists, reasons.",
        "metrics": "Per-metric aggregated value, included_count, and status_counts — plus, on the epistemic core, a per-metric Estimate (point + Interval + n_effective) and tier/authority chips (experimental; an interval is informational and never implies a pass).",
        "clusters": "Scorecard.clusters (next): a run’s failing and non-scored items grouped by their shared Diagnostic.code, per metric; recompute-verified (a fabricated cluster fails closed), and a non-scored item is grouped by cause, never 0.0. Additive field, rendered in report.md and report.html.",
        "authority": "Per-metric can_gate / level / reasons resolved from host-owned records.",
        "baseline_state": "comparable | not_comparable | missing_baseline | comparison_not_requested.",
        "diagnostics": "Structured, machine-readable reasons for non-perfect states."
      },
      "sourceDoc": "https://evalglass.com/docs/reference/scorecard-json",
      "example": "https://evalglass.com/examples/scorecard.json"
    },
    {
      "name": "Estimate",
      "summary": "An honest measurement on the epistemic core (experimental): a point value plus its uncertainty. The unit a DecisionPolicy reads.",
      "fields": {
        "point": "The point estimate (e.g. a proportion or mean).",
        "interval": "A Wilson/Student-t confidence Interval; an order-statistics path returns an honest no-interval rather than a fabricated one.",
        "n_effective": "The effective sample size behind the estimate — a number with no n and no interval is not evidence."
      },
      "sourceDoc": "https://evalglass.com/docs/reference/runtime/evaluation-core/estimate",
      "example": "https://evalglass.com/examples/scorecard.json"
    },
    {
      "name": "DecisionPolicy",
      "summary": "The rule that turns an Estimate into a gate decision (experimental). Gates on the lower confidence bound, not the point.",
      "fields": {
        "decision_statistic": "lower_confidence_bound (default) | point | upper_confidence_bound. On the default, a perfect 3/3 fails the gate (its lower bound is far below 1.0); a named point-smoke policy (decision_statistic: point) is the explicit escape hatch.",
        "direction": "higher_is_better | lower_is_better — which bound of the interval the gate reads.",
        "authority": "Capability-typed: a fake or uncalibrated judge produces evidence, not authority, so it can never move a gate."
      },
      "sourceDoc": "https://evalglass.com/docs/reference/runtime/evaluation-core/decision-policy",
      "example": "https://evalglass.com/examples/scorecard.json"
    },
    {
      "name": "RunRecord",
      "summary": "The complete reproducible record: the embedded Scorecard plus every per-example Score and provenance.",
      "fields": {
        "run_id": "Identifier for the run.",
        "scorecard": "The embedded Scorecard (the same bounded claim).",
        "scores": "Every per-example Score: status, validity, value, evaluator_version, example_id/unit_id.",
        "provenance": "Fingerprints across framework, metric_spec, evaluator, dataset, example, evidence, config, policy, authority, baseline.",
        "comparable": "Whether this run is comparable to a baseline (drives regression claims)."
      },
      "sourceDoc": "https://evalglass.com/docs/reference/runrecord",
      "example": "https://evalglass.com/examples/runrecord.json"
    },
    {
      "name": "Config (evalglass.yaml)",
      "summary": "Host-owned configuration. Authority-safe by default: nothing gates until the host approves it.",
      "fields": {
        "datasets": "Paths + status (proposed until validated).",
        "metrics": "name, lens, evaluator, metric_status (informational until set to gating), threshold + approval.",
        "traces": "Exported OTel/OpenInference/local JSON paths (the default; no SDK, no network). An opt-in, experimental live lane is reachable via the connect --live verb (env-var-name credentials; a live pull is proposed and cannot gate). A route may set unit: trajectory|session|step (next) to grade the whole agent run — informational, cannot gate; an empty trajectory is non_evaluable, never 0.0.",
        "judges": "LLM-as-judge metrics; require calibration before they can gate. Optional command judge (subprocess, next) and OpenAI-compatible lane (experimental) exist as opt-in lanes.",
        "baselines": "Promoted explicitly; ordinary runs never auto-promote.",
        "data_policy": "unknown | permitted | redacted | forbidden egress per source. Defaults to unknown (fail-closed): egress is refused until the host sets permitted."
      },
      "sourceDoc": "https://evalglass.com/docs/config",
      "example": "https://evalglass.com/examples/evalglass.yaml"
    }
  ]
}