{
  "$format": "thethinkloop.benchmark/v1",
  "version": "1.0.0",
  "created": "2026-10-08",
  "license": "CC BY 4.0",
  "provenance": "Written and labeled by TheThinkLoop for this benchmark. Every message is synthetic: no customer data. Labels were assigned by a single annotator following the labeling guide; inter-annotator agreement has not been measured yet.",
  "id": "ticket-urgency",
  "title": "Ticket urgency",
  "templateSlug": "ticket-urgency",
  "description": "Support tickets placed on a four-level urgency rubric.",
  "question": {
    "type": "score",
    "instructions": "How urgent is this support ticket?",
    "levels": [
      {
        "label": "Low",
        "description": "No time pressure. A question, feedback, or a minor cosmetic issue."
      },
      {
        "label": "Normal",
        "description": "A real problem with a workaround, or affecting one person."
      },
      {
        "label": "High",
        "description": "Blocks an important workflow or has a near deadline."
      },
      {
        "label": "Critical",
        "description": "Outage, data loss, or revenue impact affecting many users."
      }
    ]
  },
  "labels": [
    "Low",
    "Normal",
    "High",
    "Critical"
  ],
  "labelingGuide": "Labels follow the level descriptions in the question. Urgency is judged from the impact described, not from the customer's tone: shouting about a cosmetic issue is Low, and a polite report of a multi-day failure can be High. Security exposure of another customer's data is Critical.",
  "cases": [
    {
      "id": "urgency-001",
      "input": "Small thing: the date picker shows Sunday as the first day of the week. We'd prefer Monday. No rush.",
      "expected": "Low",
      "difficulty": "typical"
    },
    {
      "id": "urgency-002",
      "input": "Love the new dashboard! One suggestion: could the charts have a dark mode?",
      "expected": "Low",
      "difficulty": "typical"
    },
    {
      "id": "urgency-003",
      "input": "Is there a keyboard shortcut for creating a new project?",
      "expected": "Low",
      "difficulty": "typical"
    },
    {
      "id": "urgency-004",
      "input": "Typo on the billing page: 'recieve' should be 'receive'.",
      "expected": "Low",
      "difficulty": "typical"
    },
    {
      "id": "urgency-005",
      "input": "Exports to PDF are cutting off the last column. I can export to CSV instead for now, but it would be good to fix.",
      "expected": "Normal",
      "difficulty": "typical"
    },
    {
      "id": "urgency-006",
      "input": "I can't upload my profile picture. It says the file is too large, but it's only 800 KB.",
      "expected": "Normal",
      "difficulty": "typical"
    },
    {
      "id": "urgency-007",
      "input": "One of our users isn't receiving email notifications. Everyone else is fine.",
      "expected": "Normal",
      "difficulty": "typical"
    },
    {
      "id": "urgency-008",
      "input": "Search sometimes returns results from archived projects. Annoying, but we can filter them out manually.",
      "expected": "Normal",
      "difficulty": "typical"
    },
    {
      "id": "urgency-009",
      "input": "Our month-end close is tomorrow and the accounting export is failing with 'timeout'. Finance can't finish without it.",
      "expected": "High",
      "difficulty": "typical"
    },
    {
      "id": "urgency-010",
      "input": "None of our sales reps can log their calls since this morning; the Save button errors out. They're using a spreadsheet meanwhile, but it's a mess.",
      "expected": "High",
      "difficulty": "typical"
    },
    {
      "id": "urgency-011",
      "input": "We go live with our customers on Monday and webhook signatures are failing validation in our staging environment.",
      "expected": "High",
      "difficulty": "typical"
    },
    {
      "id": "urgency-012",
      "input": "SSO login is failing for our whole support team, about 15 agents. They can use password login as a fallback, but most don't remember their passwords.",
      "expected": "High",
      "difficulty": "typical"
    },
    {
      "id": "urgency-013",
      "input": "The app is completely down for all of our users. Every page returns a 502. We have 400 agents unable to work.",
      "expected": "Critical",
      "difficulty": "typical"
    },
    {
      "id": "urgency-014",
      "input": "Customers are reporting that payments on our storefront have been failing since your last release. We're losing orders every minute.",
      "expected": "Critical",
      "difficulty": "typical"
    },
    {
      "id": "urgency-015",
      "input": "We just noticed that records created in the last 6 hours are missing from every project. Did something delete our data?",
      "expected": "Critical",
      "difficulty": "typical"
    },
    {
      "id": "urgency-016",
      "input": "Security issue: one of our users can see another company's invoices in their dashboard.",
      "expected": "Critical",
      "difficulty": "typical"
    },
    {
      "id": "urgency-017",
      "input": "URGENT!!! The logo on our login page is slightly blurry on retina screens. Please fix ASAP.",
      "expected": "Low",
      "difficulty": "hard",
      "note": "Urgent language, cosmetic impact."
    },
    {
      "id": "urgency-018",
      "input": "Hi, sorry to bother you. I think our nightly data sync may not have run for a few days? Not sure if it matters; our reports are just showing last week's numbers. Whenever you have time.",
      "expected": "High",
      "difficulty": "hard",
      "note": "A polite tone hides a multi-day data pipeline failure."
    },
    {
      "id": "urgency-019",
      "input": "This is the third time I'm writing about this. The CSV import still skips rows with accents in names. I've been fixing them by hand every week.",
      "expected": "Normal",
      "difficulty": "hard",
      "note": "Frustration and repeat contact, but a workaround exists."
    },
    {
      "id": "urgency-020",
      "input": "Just a heads-up: your status page says all systems are operational, but our API calls have been returning 503 for about 20 minutes. Our checkout integration depends on it.",
      "expected": "Critical",
      "difficulty": "hard",
      "note": "Casual framing of a revenue-affecting outage."
    },
    {
      "id": "urgency-021",
      "input": "Can you extend our trial by a week? Our evaluation committee meets next Friday.",
      "expected": "Normal",
      "difficulty": "hard",
      "note": "Time-bound, but nothing is broken."
    },
    {
      "id": "urgency-022",
      "input": "We're presenting your product to our board in 2 hours and the demo workspace won't load.",
      "expected": "High",
      "difficulty": "hard",
      "note": "One workspace, but an immediate deadline."
    },
    {
      "id": "urgency-023",
      "input": "feature request: bulk edit tags. would save us hours every week",
      "expected": "Low",
      "difficulty": "typical"
    },
    {
      "id": "urgency-024",
      "input": "Two of our 300 users get a blank screen after login on Safari. Chrome works for them.",
      "expected": "Normal",
      "difficulty": "typical"
    }
  ]
}