{
  "$format": "thethinkloop.benchmark/v1",
  "version": "1.0.0",
  "created": "2026-10-08",
  "license": "CC BY 4.0",
  "provenance": "Written and labeled by TheThinkLoop for this benchmark. Every message is synthetic: no customer data. Labels were assigned by a single annotator following the labeling guide; inter-annotator agreement has not been measured yet.",
  "id": "engineering-escalation",
  "title": "Engineering escalation",
  "templateSlug": "engineering-escalation",
  "description": "Support tickets labeled by whether they need engineering or can be resolved by support.",
  "question": {
    "type": "predicate",
    "instructions": "Does this support issue need to go to engineering?"
  },
  "labels": [
    "Yes",
    "No"
  ],
  "labelingGuide": "Yes: resolving it needs a code change, an infrastructure fix, or engineering-only access such as logs or database restores. No: how-to questions, account and billing changes, feature requests, user errors support can fix (permissions, reconnecting an integration), sales or security-review requests, and reports too vague to act on without a clarifying question.",
  "cases": [
    {
      "id": "eng-001",
      "input": "The /api/v2/orders endpoint has returned a 500 for every request since 09:40 UTC. Request IDs attached: req_91a, req_91b.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "eng-002",
      "input": "Webhook deliveries to our endpoint stopped entirely after your release last night. Our endpoint is up; we tested it with curl.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "eng-003",
      "input": "When two users edit the same document at the same time, one person's changes disappear after saving. We reproduced it three times.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "eng-004",
      "input": "The mobile app crashes immediately on launch on Android 15 since the last update.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "eng-005",
      "input": "Our scheduled report has run twice every night this week, emailing customers duplicate summaries.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "eng-006",
      "input": "Dates in exported CSVs are shifted by one day for users in time zones west of UTC.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "eng-007",
      "input": "We deleted a project by accident 3 days ago. Is there any way to restore it from a backup?",
      "expected": "Yes",
      "difficulty": "hard",
      "note": "Restoring from backup needs engineering access in most organisations. If support has a restore tool, this becomes No."
    },
    {
      "id": "eng-008",
      "input": "How do I invite a guest user who can only view one project?",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "eng-009",
      "input": "Please change the email address on our account from a@example.com to b@example.com.",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "eng-010",
      "input": "Can you add support for exporting to Excel directly? CSV is a bit awkward for our finance team.",
      "expected": "No",
      "difficulty": "typical",
      "note": "A feature request goes to product, not an engineering escalation."
    },
    {
      "id": "eng-011",
      "input": "Where can I find our API key?",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "eng-012",
      "input": "I'd like to downgrade to the Starter plan next month.",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "eng-013",
      "input": "The page loads slowly when I'm on hotel Wi-Fi. Is there an offline mode?",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "eng-014",
      "input": "I forgot my password and the reset email went to spam. Found it now, all good!",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "eng-015",
      "input": "What's your uptime SLA for the Business plan?",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "eng-016",
      "input": "It's broken.",
      "expected": "No",
      "difficulty": "hard",
      "note": "Too vague to escalate. Support should ask what is broken."
    },
    {
      "id": "eng-017",
      "input": "I get a 403 Forbidden when opening the Admin settings page.",
      "expected": "No",
      "difficulty": "hard",
      "note": "Usually a role permission. Support can check the user's role before anyone escalates."
    },
    {
      "id": "eng-018",
      "input": "Your API docs say the limit parameter maxes out at 500, but anything above 100 returns a validation error.",
      "expected": "Yes",
      "difficulty": "hard",
      "note": "Documentation and behavior disagree; engineering decides which is wrong."
    },
    {
      "id": "eng-019",
      "input": "The Salesforce integration has been failing since we changed our Salesforce password. Error: INVALID_SESSION_ID.",
      "expected": "No",
      "difficulty": "hard",
      "note": "Has an error code, but the fix is reconnecting the integration with the new credentials."
    },
    {
      "id": "eng-020",
      "input": "Feature request: bulk delete is too slow. Deleting 10,000 records takes 40 minutes and sometimes times out halfway, leaving half the records deleted.",
      "expected": "Yes",
      "difficulty": "hard",
      "note": "Framed as a feature request; partial deletes after a timeout are a defect."
    },
    {
      "id": "eng-021",
      "input": "Hey team, quick one: the 'Forgot password' link on your login page goes to a 404 page.",
      "expected": "Yes",
      "difficulty": "hard",
      "note": "Casual tone, real bug."
    },
    {
      "id": "eng-022",
      "input": "Is the app down? I can't load anything.",
      "expected": "No",
      "difficulty": "hard",
      "note": "A single vague report. Support checks the status page and the customer's connection first."
    },
    {
      "id": "eng-023",
      "input": "We're seeing intermittent 504 errors on about 2% of requests to the reporting API, starting around when you announced the database migration.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "eng-024",
      "input": "Can someone from your engineering team join a call to discuss your security architecture for our vendor review?",
      "expected": "No",
      "difficulty": "hard",
      "note": "Mentions engineering, but it is a sales or security-review request, not an issue."
    }
  ]
}