{
  "source_commit": "59a200c6d575d595120f1cb70fea53cef0632f6b",
  "kind": "historical trace reanalysis",
  "datasets": [
    {
      "dataset": "gpt-4o-airline",
      "attempts": 200,
      "tasks": 50,
      "trials_per_task": 4,
      "successful_attempts": 84,
      "success_rate": 0.42,
      "pass4": 0.2,
      "all_observed_success": 10,
      "at_least_one_success": 36,
      "assistant_messages": 2454,
      "failed_assistant_messages": 1625,
      "messages_per_attempt": 12.27,
      "messages_per_success": 29.214285714285715,
      "failed_message_share": 0.6621841890790546,
      "agent_total_cost_present": 0
    },
    {
      "dataset": "gpt-4o-retail",
      "attempts": 460,
      "tasks": 115,
      "trials_per_task": 4,
      "successful_attempts": 278,
      "success_rate": 0.6043478260869565,
      "pass4": 0.3826086956521739,
      "all_observed_success": 44,
      "at_least_one_success": 93,
      "assistant_messages": 6564,
      "failed_assistant_messages": 2778,
      "messages_per_attempt": 14.269565217391305,
      "messages_per_success": 23.611510791366907,
      "failed_message_share": 0.42321755027422303,
      "agent_total_cost_present": 0
    },
    {
      "dataset": "sonnet-35-new-airline",
      "attempts": 400,
      "tasks": 50,
      "trials_per_task": 8,
      "successful_attempts": 184,
      "success_rate": 0.46,
      "pass4": 0.22428571428571428,
      "all_observed_success": 8,
      "at_least_one_success": 40,
      "assistant_messages": 4674,
      "failed_assistant_messages": 2938,
      "messages_per_attempt": 11.685,
      "messages_per_success": 25.402173913043477,
      "failed_message_share": 0.6285836542575952,
      "agent_total_cost_present": 0
    },
    {
      "dataset": "sonnet-35-new-retail",
      "attempts": 920,
      "tasks": 115,
      "trials_per_task": 8,
      "successful_attempts": 637,
      "success_rate": 0.6923913043478261,
      "pass4": 0.4622360248447205,
      "all_observed_success": 41,
      "at_least_one_success": 108,
      "assistant_messages": 12624,
      "failed_assistant_messages": 4109,
      "messages_per_attempt": 13.721739130434782,
      "messages_per_success": 19.81789638932496,
      "failed_message_share": 0.3254911280101394,
      "agent_total_cost_present": 0
    }
  ],
  "files": [
    {
      "url": "https://raw.githubusercontent.com/sierra-research/tau-bench/59a200c6d575d595120f1cb70fea53cef0632f6b/historical_trajectories/gpt-4o-airline.json",
      "sha256": "e9e6c0297660c537f83d4fd9c476ce7a9a86ecd2784874b7bfc13be598e37bfa",
      "bytes": 4114038
    },
    {
      "url": "https://raw.githubusercontent.com/sierra-research/tau-bench/59a200c6d575d595120f1cb70fea53cef0632f6b/historical_trajectories/gpt-4o-retail.json",
      "sha256": "df01707894836168ff0ec9616b0bf08f66c7e5afcf313e5fe4f7a2f5c2ec938b",
      "bytes": 10813408
    },
    {
      "url": "https://raw.githubusercontent.com/sierra-research/tau-bench/59a200c6d575d595120f1cb70fea53cef0632f6b/historical_trajectories/sonnet-35-new-airline.json",
      "sha256": "fe62fcd514b855b36f156dd4c3c7748597b392b006aff739b53337a9f3ba94d1",
      "bytes": 10888682
    },
    {
      "url": "https://raw.githubusercontent.com/sierra-research/tau-bench/59a200c6d575d595120f1cb70fea53cef0632f6b/historical_trajectories/sonnet-35-new-retail.json",
      "sha256": "0df526398e9d2720c32d340815cffb04fe8c4f8a61b1f4f84bf3bb558f760131",
      "bytes": 26544890
    }
  ]
}