{
  "schema": "little-canary-public-evaluation-summary/v1",
  "state": "PASS",
  "evidence_class": "controlled_experiment",
  "run_id": "little-canary-tensortrust-luna-advisory-20260815-01",
  "label": "Outcome-blind fixed-sample 200-case TensorTrust advisory pilot comprising 400 assigned Luna evaluations.",
  "subject": "Luna through the Codex final-answer scaffold using ChatGPT OAuth",
  "configuration": {
    "reasoning_effort": "low",
    "model_verbosity": "low",
    "tools_allowed": false,
    "retries": 0,
    "unflagged_policy": "reuse exact raw Luna response",
    "flagged_policy": "one new Luna call with the canonical Little Canary advisory as developer instructions and the original user prompt unchanged",
    "error_policy": "all assigned; missing, parser, technical, and tool-event outcomes score as failures"
  },
  "sample": {
    "tensortrust_revision": "747a75e096761ebc01bd3970158827326b4add23",
    "cases": 200,
    "assigned_evaluations": 400,
    "flagged_evaluations": 103,
    "new_provider_calls": 103,
    "reused_raw_responses": 297,
    "case_list_sha256": "50f36647fe9e2019f5d90343f2355e489b0d0ac20a584a0d92a525fef470f788"
  },
  "metrics": {
    "HRR": {
      "raw": { "numerator": 71, "denominator": 100, "estimate": 0.71, "exact_95_interval": [0.6107340410925348, 0.7964258107492683] },
      "advisory": { "numerator": 74, "denominator": 100, "estimate": 0.74, "exact_95_interval": [0.6426879368998828, 0.8226055621323469] },
      "absolute_change_percentage_points": 3.0,
      "mcnemar_exact_p": 0.507812499999999
    },
    "ERR": {
      "raw": { "numerator": 82, "denominator": 100, "estimate": 0.82, "exact_95_interval": [0.7305229140331722, 0.8896887708467396] },
      "advisory": { "numerator": 84, "denominator": 100, "estimate": 0.84, "exact_95_interval": [0.7532124025891205, 0.9056897100260539] },
      "absolute_change_percentage_points": 2.0,
      "mcnemar_exact_p": 0.6249999999999994
    },
    "DV": {
      "raw": { "numerator": 107, "denominator": 200, "estimate": 0.535, "exact_95_interval": [0.46329985778836236, 0.605647264738544] },
      "advisory": { "numerator": 106, "denominator": 200, "estimate": 0.53, "exact_95_interval": [0.4583305004115157, 0.6007670588028864] },
      "absolute_change_percentage_points": -0.5,
      "mcnemar_exact_p": 1.0
    }
  },
  "interpretation": "The attack-metric deltas were directionally favorable but not statistically decisive on this sample. The result is not evidence about ChatGPT generally or production safety.",
  "limitations": [
    "This is one fixed 200-case TensorTrust sample.",
    "The detector decisions are one frozen Qwen2.5:1.5b/Ollama realization.",
    "The run does not estimate deployment prevalence, production false-positive rate, unseen-model generalization, or universal prevention."
  ],
  "evidence_hashes": {
    "run_manifest_sha256": "2ec72187bf2dd6918042beb1351ca2d290d77306b28ef0d5ce60ae7d7e8b0b9e",
    "schedule_sha256": "65550bbe61c1b9f7dc41df92aaad0eee4075682bd2b2b957c2cac5b75fa252ba",
    "completion_sha256": "01ddad18fa5f39d3d271dbc3c07b80a8800c7a443b2b3fc74432948ca33eab2b",
    "attempts_sha256": "962bf39d08d3f96f78ecd5e8dc5542f479ea4a1bb10424f1631d3b2c695b6b85",
    "calls_sha256": "541d6b519a615ba7cad10975ff9e6e8398212cafed2016ae3a8b30b218ab479b",
    "transport_proof_sha256": "5d4926ec7e9bea918112823ad78ceb2b8cab87ffb436c31db96249da79c6565f",
    "metrics_sha256": "b99ff020cb7184e5c04fbd7e4cd07289a11381961f647b76ef5b39be6e3a538b",
    "evaluation_receipt_sha256": "b1d5a8b864d88a9defcece081af3d528b8a3bb1a80168f2eb700e31d57eea241"
  }
}