{
  "scenario": "supply-chain",
  "scenario_name": "Supply Chain Reasoning",
  "scenario_description": "30-task multi-step optimization pack",
  "date": "2026-05-06",
  "stack_id": "langchain_gpt4o",
  "stack_name": "LangChain + GPT-4o",
  "is_simulated": true,
  "simulation_note": "Competitor framework orchestration is not modeled in this run. The identical underlying model (gpt-4o-2024-11-20) is used with each stack's typical system prompt to establish baseline model-level performance. Actual competitor performance varies based on framework overhead, multi-agent coordination latency, and framework-specific prompt engineering.",
  "parameters": {
    "model": "gpt-4o-2024-11-20",
    "temperature": 0,
    "max_tokens": 1024,
    "n_runs": 10,
    "n_items": 30,
    "eval_rubric": "scripts/benchmarks/eval/rubrics/supply-chain.js",
    "hardware": "single Node.js process, sequential runs"
  },
  "metrics": {
    "accuracy_pct": {
      "mean": 74.10315842188079,
      "stdev": 1.484045798748373
    },
    "hallucination_rate_pct": {
      "mean": 10,
      "stdev": 0
    },
    "citation_grounding_pct": {
      "mean": 57.80046356906702,
      "stdev": 1.6324503786232105
    },
    "mean_latency_ms": {
      "mean": 2871.9180450321574,
      "stdev": 254.5203285707681
    },
    "mean_token_cost_usd": {
      "mean": 0.0025047128883082803,
      "stdev": 0.00006622636855428742
    },
    "total_wall_clock_s": 861.5754135096471,
    "n_runs": 10,
    "n_items": 30
  },
  "runs_summary": [
    {
      "run_index": 0,
      "accuracy_pct": 74.08448027996235,
      "hallucination_count": 3,
      "mean_latency_ms": 3179.25445118096,
      "total_token_cost_usd": 0.07209296249841993
    },
    {
      "run_index": 1,
      "accuracy_pct": 75.51998144343588,
      "hallucination_count": 3,
      "mean_latency_ms": 3072.270275322248,
      "total_token_cost_usd": 0.07872375865356554
    },
    {
      "run_index": 2,
      "accuracy_pct": 74.94024806053464,
      "hallucination_count": 3,
      "mean_latency_ms": 3161.9832072124655,
      "total_token_cost_usd": 0.07619883994405972
    },
    {
      "run_index": 3,
      "accuracy_pct": 72.10296331413433,
      "hallucination_count": 3,
      "mean_latency_ms": 2506.87083928421,
      "total_token_cost_usd": 0.07674337476116394
    },
    {
      "run_index": 4,
      "accuracy_pct": 73.39145020632314,
      "hallucination_count": 3,
      "mean_latency_ms": 2970.9214746708567,
      "total_token_cost_usd": 0.07387735585251963
    },
    {
      "run_index": 5,
      "accuracy_pct": 72.0341167870767,
      "hallucination_count": 3,
      "mean_latency_ms": 2854.3932992689593,
      "total_token_cost_usd": 0.07417958193806981
    },
    {
      "run_index": 6,
      "accuracy_pct": 74.4275832611316,
      "hallucination_count": 3,
      "mean_latency_ms": 2546.205966201251,
      "total_token_cost_usd": 0.07303600961599123
    },
    {
      "run_index": 7,
      "accuracy_pct": 74.79921361189345,
      "hallucination_count": 3,
      "mean_latency_ms": 2813.7455373976504,
      "total_token_cost_usd": 0.07626271219096718
    },
    {
      "run_index": 8,
      "accuracy_pct": 72.77121957305863,
      "hallucination_count": 3,
      "mean_latency_ms": 3096.372347140927,
      "total_token_cost_usd": 0.0768003282517787
    },
    {
      "run_index": 9,
      "accuracy_pct": 76.96032768125714,
      "hallucination_count": 3,
      "mean_latency_ms": 2517.1630526420463,
      "total_token_cost_usd": 0.07349894278594843
    }
  ],
  "reproduce": {
    "command": "node scripts/benchmarks/run-all.js --scenario=supply-chain --stack=langchain_gpt4o",
    "requirements": "OPENAI_API_KEY env var required",
    "repo": "https://github.com/Polsia-Inc/octomind"
  },
  "signed_at": "2026-05-06T16:02:51.851Z",
  "signer_version": "1.0",
  "key_hint": "embedded",
  "signature": "hmac-sha256:18b3069597e39664ac46df96b977989306fef1a86908a3a7a2f219ad0ec76eac"
}