{
  "study": "Industrial small-model desk study",
  "date": "2026-11-19",
  "operator": "GPT-6.2 Astra in ChatGPT, at the request of Hyperion Consulting's founder",
  "evidenceArchiveSha256": "f4e8210a86a5927fa1c7319b0bf02c0f76919e2bd683b92abcc1915bf78cb8e0",
  "evidenceArchiveNote": "SHA-256 of the evidence ZIP as supplied to Hyperion. The archive itself is not published here.",
  "evidenceManifestSha256": "c74b60b45c49fe0738ddba285d70a5b448221df56f19821acd64a6a512afcc25",
  "evidenceManifestNote": "SHA-256 of the SHA256SUMS file inside the archive, which lists 149 files.",
  "runtime": {
    "llamaCppFork": "OVH fork of llama.cpp",
    "revision": "55c22dca0c987f247c21b427c3b3a09e1a28187f",
    "threads": 4,
    "gpu": false,
    "guest": "AMD EPYC 9V74 guest; 5 visible vCPUs, cgroup quota 4; 16 GiB cgroup memory; CPU-only"
  },
  "protocols": {
    "local-raw-1-token": "Raw completion, one output token, grammar limited to the option letters (sensor questions) or Yes/No (return to origin), 2,048-token context, temperature 0, seed 42.",
    "local-chat-256": "Chat completion with the model's own template, up to 256 output tokens, 2,048-token context, temperature 0, seed 42. Reasoning switched off for Olala and Qwen3.5-9B.",
    "local-typed-32": "Templated raw completion with a grammar for the answer type, up to 32 output tokens, temperature 0, seed 42.",
    "local-documents-1536": "Chat completion, up to 1,536 output tokens, graded by the evaluating AI.",
    "local-chat-256-terminal-answer": "Secondary reading of the local-chat-256 replies: the scorer read the final answer of each completed reply.",
    "inspection": "Image classification, 4,096-token context, up to 12 output tokens, grammar limited to two labels."
  },
  "scoring": "One saved answer per case, model and protocol. A strict pass requires the submitted answer to equal the frozen expected answer; routes are checked move by move; a tool request passes only with exactly one call to the approved function with the right identifier.",
  "visionProjectors": {
    "Ministral 3B": {
      "url": "https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512-GGUF/resolve/eb599d408350ea2bb60452cb86be7c7b2fc28227/Ministral-3-3B-Instruct-2512-BF16-mmproj.gguf",
      "sha256": "70ca26e41ecea9da997e2887462ad68b848fbf64da011ff4107bbe7353519ee0",
      "bytes": 841501856
    },
    "Ministral 8B": {
      "url": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512-GGUF/resolve/0102285ad796bd99af90f58de616092e5630e970/Ministral-3-8B-Instruct-2512-BF16-mmproj.gguf",
      "sha256": "e799380f596d152ff4026a72d409ecc89c96cd437676804bc3f2ab3bd6b486ec",
      "bytes": 858283168
    },
    "Qwen3.5-9B": {
      "url": "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5/mmproj-BF16.gguf",
      "bytes": 921705024,
      "sha256": "853698ce7aa6c7ba732478bad280240969ddf7b0fcbf93900046f63903a83383",
      "upstream": "https://huggingface.co/Qwen/Qwen3.5-9B/tree/c202236235762e1c871ad0ccb60c8ee5ba337b9a"
    }
  },
  "models": {
    "Olala": {
      "url": "https://huggingface.co/OVHaiLLM/Olala-v1.0-mini-7A1B-GGUF/resolve/be995448a499b29e07c66ef64b20fa5a941b316d/Olala-v1.0-mini-7A1B-q8_0-safe2.gguf",
      "sha256": "0bb4560fa6800b41d16a15507ad1843ca17c0849ce0cc989bca337dcf55136d7",
      "bytes": 6876038592
    },
    "Ministral 3B": {
      "url": "https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512-GGUF/resolve/eb599d408350ea2bb60452cb86be7c7b2fc28227/Ministral-3-3B-Instruct-2512-Q8_0.gguf",
      "sha256": "8c2b72eb5861304fcfd5e82f1eddd6efa4115737f4239fd216a028a8852413ef",
      "bytes": 3652204704
    },
    "Ministral 8B": {
      "url": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512-GGUF/resolve/0102285ad796bd99af90f58de616092e5630e970/Ministral-3-8B-Instruct-2512-Q8_0.gguf",
      "sha256": "bd0ca58473b64618df4d08c1410cc25c910ca84e647833ebaf099e0fc0523b45",
      "bytes": 9029392800
    },
    "Qwen3.5-9B": {
      "url": "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5/Qwen3.5-9B-Q8_0.gguf",
      "bytes": 9527502048,
      "sha256": "809626574d0cb43d4becfa56169980da2bb448f2299270f7be443cb89d0a6ae4",
      "upstream": "https://huggingface.co/Qwen/Qwen3.5-9B/tree/c202236235762e1c871ad0ccb60c8ee5ba337b9a"
    }
  },
  "datasets": {
    "FailureSensorIQ": "https://huggingface.co/datasets/ibm-research/FailureSensorIQ/tree/5f9a736201916597345285bb6e712e3b8f4f0cfe",
    "BIG-Bench Hard Navigate": "https://github.com/suzgunmirac/BIG-Bench-Hard/blob/9ee07bd481feebf959a6b59d61ea57bdcf30964d/bbh/navigate.json",
    "VisA": "https://registry.opendata.aws/visa/"
  },
  "counts": [
    {
      "profile": "native",
      "protocol": "local-raw-1-token",
      "task": "Sensor selection",
      "counts": {
        "Olala": [41, 100],
        "Ministral 3B": [35, 100],
        "Ministral 8B": [45, 100],
        "Qwen3.5-9B": [52, 100]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "native",
      "protocol": "local-raw-1-token",
      "task": "Return to origin",
      "counts": {
        "Olala": [126, 250],
        "Ministral 3B": [105, 250],
        "Ministral 8B": [105, 250],
        "Qwen3.5-9B": [165, 250]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "native",
      "protocol": "local-chat-256",
      "task": "Coordinate transforms",
      "counts": {
        "Olala": [0, 12],
        "Ministral 3B": [0, 12],
        "Ministral 8B": [0, 12],
        "Qwen3.5-9B": [0, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 12,
        "Ministral 3B": 9,
        "Ministral 8B": 5,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "native",
      "protocol": "local-chat-256",
      "task": "Shortest legal route",
      "counts": {
        "Olala": [0, 12],
        "Ministral 3B": [0, 12],
        "Ministral 8B": [0, 12],
        "Qwen3.5-9B": [0, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 12,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "native",
      "protocol": "local-chat-256",
      "task": "Latched alarm",
      "counts": {
        "Olala": [0, 12],
        "Ministral 3B": [4, 12],
        "Ministral 8B": [0, 12],
        "Qwen3.5-9B": [9, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 12,
        "Ministral 3B": 0,
        "Ministral 8B": 1,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "native",
      "protocol": "local-chat-256",
      "task": "Production arithmetic",
      "counts": {
        "Olala": [0, 12],
        "Ministral 3B": [0, 12],
        "Ministral 8B": [0, 12],
        "Qwen3.5-9B": [3, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 5,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "native",
      "protocol": "local-chat-256",
      "task": "Current manual decision",
      "counts": {
        "Olala": [3, 12],
        "Ministral 3B": [0, 12],
        "Ministral 8B": [0, 12],
        "Qwen3.5-9B": [11, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "native",
      "protocol": "local-chat-256",
      "task": "Read-only tool call",
      "counts": {
        "Olala": [2, 8],
        "Ministral 3B": [8, 8],
        "Ministral 8B": [8, 8],
        "Qwen3.5-9B": [8, 8]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "type-constrained",
      "protocol": "local-typed-32",
      "task": "Coordinate transforms",
      "counts": {
        "Olala": [0, 12],
        "Ministral 3B": [1, 12],
        "Ministral 8B": [0, 12],
        "Qwen3.5-9B": [0, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "type-constrained",
      "protocol": "local-typed-32",
      "task": "Shortest legal route",
      "counts": {
        "Olala": [1, 12],
        "Ministral 3B": [0, 12],
        "Ministral 8B": [0, 12],
        "Qwen3.5-9B": [0, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "type-constrained",
      "protocol": "local-typed-32",
      "task": "Latched alarm",
      "counts": {
        "Olala": [6, 12],
        "Ministral 3B": [6, 12],
        "Ministral 8B": [7, 12],
        "Qwen3.5-9B": [9, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "type-constrained",
      "protocol": "local-typed-32",
      "task": "Production arithmetic",
      "counts": {
        "Olala": [0, 12],
        "Ministral 3B": [0, 12],
        "Ministral 8B": [0, 12],
        "Qwen3.5-9B": [3, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "type-constrained",
      "protocol": "local-typed-32",
      "task": "Current manual decision",
      "counts": {
        "Olala": [8, 12],
        "Ministral 3B": [5, 12],
        "Ministral 8B": [7, 12],
        "Qwen3.5-9B": [11, 12]
      },
      "cutOffAtOutputCap": {
        "Olala": 0,
        "Ministral 3B": 0,
        "Ministral 8B": 0,
        "Qwen3.5-9B": 0
      }
    },
    {
      "profile": "native documents",
      "protocol": "local-documents-1536",
      "task": "Translation fidelity",
      "counts": {
        "Olala": [5, 8],
        "Ministral 3B": [6, 8],
        "Ministral 8B": [8, 8],
        "Qwen3.5-9B": [6, 8]
      }
    },
    {
      "profile": "native documents",
      "protocol": "local-documents-1536",
      "task": "Handover fidelity",
      "counts": {
        "Olala": [3, 8],
        "Ministral 3B": [1, 8],
        "Ministral 8B": [5, 8],
        "Qwen3.5-9B": [7, 8]
      }
    },
    {
      "profile": "native, terminal answer extracted",
      "protocol": "local-chat-256-terminal-answer",
      "task": "Recovered coordinate pair",
      "counts": {
        "Olala": [0, 12],
        "Ministral 3B": [3, 12],
        "Ministral 8B": [6, 12],
        "Qwen3.5-9B": [0, 12]
      }
    },
    {
      "profile": "native, terminal answer extracted",
      "protocol": "local-chat-256-terminal-answer",
      "task": "Recovered alarm disposition",
      "counts": {
        "Olala": [0, 12],
        "Ministral 3B": [8, 12],
        "Ministral 8B": [10, 12],
        "Qwen3.5-9B": [9, 12]
      }
    },
    {
      "profile": "native, terminal answer extracted",
      "protocol": "local-chat-256-terminal-answer",
      "task": "Recovered manual disposition",
      "counts": {
        "Olala": [9, 12],
        "Ministral 3B": [6, 12],
        "Ministral 8B": [6, 12],
        "Qwen3.5-9B": [11, 12]
      }
    },
    {
      "profile": "type-constrained",
      "protocol": "local-typed-32",
      "task": "ACTIVE identified",
      "counts": {
        "Olala": [0, 6],
        "Ministral 3B": [0, 6],
        "Ministral 8B": [6, 6],
        "Qwen3.5-9B": [4, 6]
      }
    },
    {
      "profile": "type-constrained",
      "protocol": "local-typed-32",
      "task": "HOLD identified",
      "counts": {
        "Olala": [0, 3],
        "Ministral 3B": [2, 3],
        "Ministral 8B": [0, 3],
        "Qwen3.5-9B": [3, 3]
      }
    }
  ],
  "classOutcomes": [
    {
      "profile": "native",
      "model": "Olala",
      "task": "alarm",
      "required answer": "ACTIVE",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Olala",
      "task": "alarm",
      "required answer": "CLEAR",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Olala",
      "task": "manual",
      "required answer": "HOLD",
      "cases": 3,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Olala",
      "task": "manual",
      "required answer": "PASS",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Olala",
      "task": "manual",
      "required answer": "UNKNOWN",
      "cases": 3,
      "correct": 3,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 3B",
      "task": "alarm",
      "required answer": "ACTIVE",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 4
    },
    {
      "profile": "native",
      "model": "Ministral 3B",
      "task": "alarm",
      "required answer": "CLEAR",
      "cases": 6,
      "correct": 4,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 3B",
      "task": "manual",
      "required answer": "HOLD",
      "cases": 3,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 3B",
      "task": "manual",
      "required answer": "PASS",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 3B",
      "task": "manual",
      "required answer": "UNKNOWN",
      "cases": 3,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 8B",
      "task": "alarm",
      "required answer": "ACTIVE",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 8B",
      "task": "alarm",
      "required answer": "CLEAR",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 8B",
      "task": "manual",
      "required answer": "HOLD",
      "cases": 3,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 8B",
      "task": "manual",
      "required answer": "PASS",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Ministral 8B",
      "task": "manual",
      "required answer": "UNKNOWN",
      "cases": 3,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Qwen3.5-9B",
      "task": "alarm",
      "required answer": "ACTIVE",
      "cases": 6,
      "correct": 4,
      "incorrect CLEAR or PASS": 2
    },
    {
      "profile": "native",
      "model": "Qwen3.5-9B",
      "task": "alarm",
      "required answer": "CLEAR",
      "cases": 6,
      "correct": 5,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Qwen3.5-9B",
      "task": "manual",
      "required answer": "HOLD",
      "cases": 3,
      "correct": 3,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Qwen3.5-9B",
      "task": "manual",
      "required answer": "PASS",
      "cases": 6,
      "correct": 5,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "native",
      "model": "Qwen3.5-9B",
      "task": "manual",
      "required answer": "UNKNOWN",
      "cases": 3,
      "correct": 3,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Olala",
      "task": "alarm",
      "required answer": "ACTIVE",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 6
    },
    {
      "profile": "type-constrained",
      "model": "Olala",
      "task": "alarm",
      "required answer": "CLEAR",
      "cases": 6,
      "correct": 6,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Olala",
      "task": "manual",
      "required answer": "HOLD",
      "cases": 3,
      "correct": 0,
      "incorrect CLEAR or PASS": 3
    },
    {
      "profile": "type-constrained",
      "model": "Olala",
      "task": "manual",
      "required answer": "PASS",
      "cases": 6,
      "correct": 5,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Olala",
      "task": "manual",
      "required answer": "UNKNOWN",
      "cases": 3,
      "correct": 3,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 3B",
      "task": "alarm",
      "required answer": "ACTIVE",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 6
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 3B",
      "task": "alarm",
      "required answer": "CLEAR",
      "cases": 6,
      "correct": 6,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 3B",
      "task": "manual",
      "required answer": "HOLD",
      "cases": 3,
      "correct": 2,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 3B",
      "task": "manual",
      "required answer": "PASS",
      "cases": 6,
      "correct": 0,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 3B",
      "task": "manual",
      "required answer": "UNKNOWN",
      "cases": 3,
      "correct": 3,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 8B",
      "task": "alarm",
      "required answer": "ACTIVE",
      "cases": 6,
      "correct": 6,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 8B",
      "task": "alarm",
      "required answer": "CLEAR",
      "cases": 6,
      "correct": 1,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 8B",
      "task": "manual",
      "required answer": "HOLD",
      "cases": 3,
      "correct": 0,
      "incorrect CLEAR or PASS": 3
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 8B",
      "task": "manual",
      "required answer": "PASS",
      "cases": 6,
      "correct": 6,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Ministral 8B",
      "task": "manual",
      "required answer": "UNKNOWN",
      "cases": 3,
      "correct": 1,
      "incorrect CLEAR or PASS": 2
    },
    {
      "profile": "type-constrained",
      "model": "Qwen3.5-9B",
      "task": "alarm",
      "required answer": "ACTIVE",
      "cases": 6,
      "correct": 4,
      "incorrect CLEAR or PASS": 2
    },
    {
      "profile": "type-constrained",
      "model": "Qwen3.5-9B",
      "task": "alarm",
      "required answer": "CLEAR",
      "cases": 6,
      "correct": 5,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Qwen3.5-9B",
      "task": "manual",
      "required answer": "HOLD",
      "cases": 3,
      "correct": 3,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Qwen3.5-9B",
      "task": "manual",
      "required answer": "PASS",
      "cases": 6,
      "correct": 5,
      "incorrect CLEAR or PASS": 0
    },
    {
      "profile": "type-constrained",
      "model": "Qwen3.5-9B",
      "task": "manual",
      "required answer": "UNKNOWN",
      "cases": 3,
      "correct": 3,
      "incorrect CLEAR or PASS": 0
    }
  ],
  "inspection": [
    {
      "model": "Ministral 3B",
      "category": "all",
      "photos": 32,
      "correct": 20,
      "defective": 16,
      "detected": 8,
      "missed": 8,
      "normal": 16,
      "cleared": 12,
      "false alarms": 4
    },
    {
      "model": "Ministral 3B",
      "category": "candle",
      "photos": 8,
      "correct": 4,
      "defective": 4,
      "detected": 4,
      "missed": 0,
      "normal": 4,
      "cleared": 0,
      "false alarms": 4
    },
    {
      "model": "Ministral 3B",
      "category": "capsules",
      "photos": 8,
      "correct": 6,
      "defective": 4,
      "detected": 2,
      "missed": 2,
      "normal": 4,
      "cleared": 4,
      "false alarms": 0
    },
    {
      "model": "Ministral 3B",
      "category": "pcb1",
      "photos": 8,
      "correct": 4,
      "defective": 4,
      "detected": 0,
      "missed": 4,
      "normal": 4,
      "cleared": 4,
      "false alarms": 0
    },
    {
      "model": "Ministral 3B",
      "category": "pcb4",
      "photos": 8,
      "correct": 6,
      "defective": 4,
      "detected": 2,
      "missed": 2,
      "normal": 4,
      "cleared": 4,
      "false alarms": 0
    },
    {
      "model": "Ministral 8B",
      "category": "all",
      "photos": 32,
      "correct": 20,
      "defective": 16,
      "detected": 9,
      "missed": 7,
      "normal": 16,
      "cleared": 11,
      "false alarms": 5
    },
    {
      "model": "Ministral 8B",
      "category": "candle",
      "photos": 8,
      "correct": 5,
      "defective": 4,
      "detected": 4,
      "missed": 0,
      "normal": 4,
      "cleared": 1,
      "false alarms": 3
    },
    {
      "model": "Ministral 8B",
      "category": "capsules",
      "photos": 8,
      "correct": 6,
      "defective": 4,
      "detected": 2,
      "missed": 2,
      "normal": 4,
      "cleared": 4,
      "false alarms": 0
    },
    {
      "model": "Ministral 8B",
      "category": "pcb1",
      "photos": 8,
      "correct": 4,
      "defective": 4,
      "detected": 0,
      "missed": 4,
      "normal": 4,
      "cleared": 4,
      "false alarms": 0
    },
    {
      "model": "Ministral 8B",
      "category": "pcb4",
      "photos": 8,
      "correct": 5,
      "defective": 4,
      "detected": 3,
      "missed": 1,
      "normal": 4,
      "cleared": 2,
      "false alarms": 2
    },
    {
      "model": "Qwen3.5-9B",
      "category": "all",
      "photos": 32,
      "correct": 28,
      "defective": 16,
      "detected": 13,
      "missed": 3,
      "normal": 16,
      "cleared": 15,
      "false alarms": 1
    },
    {
      "model": "Qwen3.5-9B",
      "category": "candle",
      "photos": 8,
      "correct": 8,
      "defective": 4,
      "detected": 4,
      "missed": 0,
      "normal": 4,
      "cleared": 4,
      "false alarms": 0
    },
    {
      "model": "Qwen3.5-9B",
      "category": "capsules",
      "photos": 8,
      "correct": 8,
      "defective": 4,
      "detected": 4,
      "missed": 0,
      "normal": 4,
      "cleared": 4,
      "false alarms": 0
    },
    {
      "model": "Qwen3.5-9B",
      "category": "pcb1",
      "photos": 8,
      "correct": 5,
      "defective": 4,
      "detected": 1,
      "missed": 3,
      "normal": 4,
      "cleared": 4,
      "false alarms": 0
    },
    {
      "model": "Qwen3.5-9B",
      "category": "pcb4",
      "photos": 8,
      "correct": 7,
      "defective": 4,
      "detected": 4,
      "missed": 0,
      "normal": 4,
      "cleared": 3,
      "false alarms": 1
    }
  ],
  "olalaSpatialProfiles": [
    {
      "profile": "local direct 256",
      "task": "frames",
      "n": 12,
      "correct": 0,
      "length capped": 12,
      "median output tokens": 256.0,
      "median seconds": 16.3
    },
    {
      "profile": "local direct 256",
      "task": "routes",
      "n": 12,
      "correct": 0,
      "length capped": 12,
      "median output tokens": 256.0,
      "median seconds": 16.2
    },
    {
      "profile": "hosted provider default 2048",
      "task": "frames",
      "n": 12,
      "correct": 8,
      "length capped": 4,
      "median output tokens": 1728.5,
      "median seconds": 5.5
    },
    {
      "profile": "hosted provider default 2048",
      "task": "routes",
      "n": 12,
      "correct": 9,
      "length capped": 3,
      "median output tokens": 1486.0,
      "median seconds": 5.1
    },
    {
      "profile": "local medium 12000",
      "task": "frames",
      "n": 12,
      "correct": 10,
      "length capped": 0,
      "median output tokens": 3593.5,
      "median seconds": 215.7
    },
    {
      "profile": "local medium 12000",
      "task": "routes",
      "n": 12,
      "correct": 10,
      "length capped": 2,
      "median output tokens": 1795.5,
      "median seconds": 101.5
    }
  ],
  "throughput": [
    {
      "model": "Olala",
      "input tokens": 512,
      "generated tokens": 0,
      "threads": 4,
      "repetitions": 3,
      "mean tokens/s": 105.146998,
      "SD tokens/s": 7.60093,
      "model bytes (runtime)": 6870036224
    },
    {
      "model": "Olala",
      "input tokens": 0,
      "generated tokens": 128,
      "threads": 4,
      "repetitions": 3,
      "mean tokens/s": 17.046031,
      "SD tokens/s": 0.15769,
      "model bytes (runtime)": 6870036224
    },
    {
      "model": "Ministral 3B",
      "input tokens": 512,
      "generated tokens": 0,
      "threads": 4,
      "repetitions": 3,
      "mean tokens/s": 46.186913,
      "SD tokens/s": 4.033263,
      "model bytes (runtime)": 3643797504
    },
    {
      "model": "Ministral 3B",
      "input tokens": 0,
      "generated tokens": 128,
      "threads": 4,
      "repetitions": 3,
      "mean tokens/s": 8.862521,
      "SD tokens/s": 0.190172,
      "model bytes (runtime)": 3643797504
    },
    {
      "model": "Ministral 8B",
      "input tokens": 512,
      "generated tokens": 0,
      "threads": 4,
      "repetitions": 3,
      "mean tokens/s": 23.047832,
      "SD tokens/s": 1.258566,
      "model bytes (runtime)": 9020981248
    },
    {
      "model": "Ministral 8B",
      "input tokens": 0,
      "generated tokens": 128,
      "threads": 4,
      "repetitions": 3,
      "mean tokens/s": 3.835593,
      "SD tokens/s": 0.032049,
      "model bytes (runtime)": 9020981248
    },
    {
      "model": "Qwen3.5-9B",
      "input tokens": 512,
      "generated tokens": 0,
      "threads": 4,
      "repetitions": 3,
      "mean tokens/s": 18.111771,
      "SD tokens/s": 1.566119,
      "model bytes (runtime)": 9516533760
    },
    {
      "model": "Qwen3.5-9B",
      "input tokens": 0,
      "generated tokens": 128,
      "threads": 4,
      "repetitions": 3,
      "mean tokens/s": 3.57243,
      "SD tokens/s": 0.038317,
      "model bytes (runtime)": 9516533760
    }
  ],
  "latency": [
    {
      "model": "Olala",
      "task": "sensor",
      "requests": 100,
      "median seconds": 1.097,
      "p95 seconds": 1.628
    },
    {
      "model": "Olala",
      "task": "navigate",
      "requests": 250,
      "median seconds": 0.703,
      "p95 seconds": 1.018
    },
    {
      "model": "Olala",
      "task": "frames",
      "requests": 12,
      "median seconds": 16.305,
      "p95 seconds": 24.14
    },
    {
      "model": "Olala",
      "task": "routes",
      "requests": 12,
      "median seconds": 16.165,
      "p95 seconds": 16.792
    },
    {
      "model": "Olala",
      "task": "alarm",
      "requests": 12,
      "median seconds": 16.064,
      "p95 seconds": 18.635
    },
    {
      "model": "Olala",
      "task": "quantities",
      "requests": 12,
      "median seconds": 13.135,
      "p95 seconds": 16.327
    },
    {
      "model": "Olala",
      "task": "manual",
      "requests": 12,
      "median seconds": 1.563,
      "p95 seconds": 11.106
    },
    {
      "model": "Olala",
      "task": "tools",
      "requests": 8,
      "median seconds": 4.958,
      "p95 seconds": 5.122
    },
    {
      "model": "Ministral 3B",
      "task": "sensor",
      "requests": 100,
      "median seconds": 1.562,
      "p95 seconds": 2.062
    },
    {
      "model": "Ministral 3B",
      "task": "navigate",
      "requests": 250,
      "median seconds": 0.755,
      "p95 seconds": 1.242
    },
    {
      "model": "Ministral 3B",
      "task": "frames",
      "requests": 12,
      "median seconds": 31.872,
      "p95 seconds": 50.808
    },
    {
      "model": "Ministral 3B",
      "task": "routes",
      "requests": 12,
      "median seconds": 2.238,
      "p95 seconds": 4.936
    },
    {
      "model": "Ministral 3B",
      "task": "alarm",
      "requests": 12,
      "median seconds": 2.002,
      "p95 seconds": 30.688
    },
    {
      "model": "Ministral 3B",
      "task": "quantities",
      "requests": 12,
      "median seconds": 1.753,
      "p95 seconds": 16.053
    },
    {
      "model": "Ministral 3B",
      "task": "manual",
      "requests": 12,
      "median seconds": 3.923,
      "p95 seconds": 7.572
    },
    {
      "model": "Ministral 3B",
      "task": "tools",
      "requests": 8,
      "median seconds": 2.578,
      "p95 seconds": 6.752
    },
    {
      "model": "Ministral 8B",
      "task": "sensor",
      "requests": 100,
      "median seconds": 2.54,
      "p95 seconds": 3.545
    },
    {
      "model": "Ministral 8B",
      "task": "navigate",
      "requests": 250,
      "median seconds": 1.547,
      "p95 seconds": 2.593
    },
    {
      "model": "Ministral 8B",
      "task": "frames",
      "requests": 12,
      "median seconds": 67.717,
      "p95 seconds": 85.27
    },
    {
      "model": "Ministral 8B",
      "task": "routes",
      "requests": 12,
      "median seconds": 5.89,
      "p95 seconds": 8.433
    },
    {
      "model": "Ministral 8B",
      "task": "alarm",
      "requests": 12,
      "median seconds": 52.763,
      "p95 seconds": 71.254
    },
    {
      "model": "Ministral 8B",
      "task": "quantities",
      "requests": 12,
      "median seconds": 3.383,
      "p95 seconds": 29.801
    },
    {
      "model": "Ministral 8B",
      "task": "manual",
      "requests": 12,
      "median seconds": 15.539,
      "p95 seconds": 33.892
    },
    {
      "model": "Ministral 8B",
      "task": "tools",
      "requests": 8,
      "median seconds": 5.689,
      "p95 seconds": 15.146
    },
    {
      "model": "Qwen3.5-9B",
      "task": "sensor",
      "requests": 100,
      "median seconds": 5.3,
      "p95 seconds": 6.375
    },
    {
      "model": "Qwen3.5-9B",
      "task": "navigate",
      "requests": 250,
      "median seconds": 3.142,
      "p95 seconds": 4.447
    },
    {
      "model": "Qwen3.5-9B",
      "task": "frames",
      "requests": 12,
      "median seconds": 6.026,
      "p95 seconds": 6.575
    },
    {
      "model": "Qwen3.5-9B",
      "task": "routes",
      "requests": 12,
      "median seconds": 7.11,
      "p95 seconds": 8.123
    },
    {
      "model": "Qwen3.5-9B",
      "task": "alarm",
      "requests": 12,
      "median seconds": 5.332,
      "p95 seconds": 6.345
    },
    {
      "model": "Qwen3.5-9B",
      "task": "quantities",
      "requests": 12,
      "median seconds": 4.126,
      "p95 seconds": 5.715
    },
    {
      "model": "Qwen3.5-9B",
      "task": "manual",
      "requests": 12,
      "median seconds": 4.949,
      "p95 seconds": 5.788
    },
    {
      "model": "Qwen3.5-9B",
      "task": "tools",
      "requests": 8,
      "median seconds": 10.109,
      "p95 seconds": 31.721
    }
  ],
  "sensorPairedVsOlala": [
    {
      "profile": "native",
      "peer vs Olala": "Ministral 3B",
      "task": "sensor",
      "paired items": 100,
      "only peer correct": 12,
      "only Olala correct": 18,
      "difference pp": -6.0,
      "approx 95% low, sensor only": -16.7,
      "approx 95% high, sensor only": 4.7
    },
    {
      "profile": "native",
      "peer vs Olala": "Ministral 8B",
      "task": "sensor",
      "paired items": 100,
      "only peer correct": 17,
      "only Olala correct": 13,
      "difference pp": 4.0,
      "approx 95% low, sensor only": -6.9,
      "approx 95% high, sensor only": 14.9
    },
    {
      "profile": "native",
      "peer vs Olala": "Qwen3.5-9B",
      "task": "sensor",
      "paired items": 100,
      "only peer correct": 18,
      "only Olala correct": 7,
      "difference pp": 11.0,
      "approx 95% low, sensor only": 1.3,
      "approx 95% high, sensor only": 20.7
    }
  ],
  "sensorVsAlwaysA": [
    {
      "model": "Olala",
      "items": 100,
      "only model correct": 36,
      "only A correct": 27,
      "difference pp": 9.0,
      "approx 95% lower": -7.0,
      "approx 95% upper": 25.0
    },
    {
      "model": "Ministral 3B",
      "items": 100,
      "only model correct": 33,
      "only A correct": 30,
      "difference pp": 3.0,
      "approx 95% lower": -13.1,
      "approx 95% upper": 19.1
    },
    {
      "model": "Ministral 8B",
      "items": 100,
      "only model correct": 38,
      "only A correct": 25,
      "difference pp": 13.0,
      "approx 95% lower": -3.0,
      "approx 95% upper": 29.0
    },
    {
      "model": "Qwen3.5-9B",
      "items": 100,
      "only model correct": 41,
      "only A correct": 21,
      "difference pp": 20.0,
      "approx 95% lower": 4.5,
      "approx 95% upper": 35.5
    }
  ],
  "navigateByRequiredAnswer": [
    {
      "model": "Olala",
      "required answer": "Yes",
      "cases": 105,
      "correct": 61
    },
    {
      "model": "Olala",
      "required answer": "No",
      "cases": 145,
      "correct": 65
    },
    {
      "model": "Ministral 3B",
      "required answer": "Yes",
      "cases": 105,
      "correct": 105
    },
    {
      "model": "Ministral 3B",
      "required answer": "No",
      "cases": 145,
      "correct": 0
    },
    {
      "model": "Ministral 8B",
      "required answer": "Yes",
      "cases": 105,
      "correct": 105
    },
    {
      "model": "Ministral 8B",
      "required answer": "No",
      "cases": 145,
      "correct": 0
    },
    {
      "model": "Qwen3.5-9B",
      "required answer": "Yes",
      "cases": 105,
      "correct": 32
    },
    {
      "model": "Qwen3.5-9B",
      "required answer": "No",
      "cases": 145,
      "correct": 133
    }
  ],
  "hostedSensorByOptionOrder": [
    {
      "model": "Olala-v1.0-mini-7A1B",
      "cyclic shift": 0,
      "cases": 20,
      "correct": 8,
      "length capped": 0,
      "median reported reasoning tokens": 519.0,
      "reasoning tokens present": 20
    },
    {
      "model": "Olala-v1.0-mini-7A1B",
      "cyclic shift": 1,
      "cases": 20,
      "correct": 8,
      "length capped": 1,
      "median reported reasoning tokens": 535.0,
      "reasoning tokens present": 20
    },
    {
      "model": "Qwen3.5-9B",
      "cyclic shift": 0,
      "cases": 20,
      "correct": 5,
      "length capped": 2,
      "median reported reasoning tokens": 5229.5,
      "reasoning tokens present": 20
    },
    {
      "model": "Qwen3.5-9B",
      "cyclic shift": 1,
      "cases": 20,
      "correct": 4,
      "length capped": 3,
      "median reported reasoning tokens": 6610.0,
      "reasoning tokens present": 20
    }
  ],
  "hostedSensorConsistency": [
    {
      "model": "Olala-v1.0-mini-7A1B",
      "distinct cases": 20,
      "correct in both orders": 4,
      "same valid semantic choice in both orders": 8
    },
    {
      "model": "Qwen3.5-9B",
      "distinct cases": 20,
      "correct in both orders": 3,
      "same valid semantic choice in both orders": 6
    }
  ]
}
