{ "$format": "thethinkloop.benchmark/v1", "version": "1.0.0", "created": "2026-10-08", "license": "CC BY 4.0", "provenance": "Written and labeled by TheThinkLoop for this benchmark. Every message is synthetic: no customer data. Labels were assigned by a single annotator following the labeling guide; inter-annotator agreement has not been measured yet.", "id": "team-routing", "title": "Team routing", "templateSlug": "account-team-routing", "description": "Messages to a shared inbox at a B2B SaaS company, labeled with the team that should own them.", "question": { "type": "choice", "instructions": "Which team should own this customer message?", "choices": [ { "value": "sales", "description": "New purchases, pricing, demos, and quotes." }, { "value": "customer_success", "description": "Onboarding, adoption, renewals, and expansion." }, { "value": "support", "description": "Troubleshooting and technical problems with the product." }, { "value": "billing", "description": "Invoices, payments, and refunds." } ] }, "labels": [ "sales", "customer_success", "support", "billing" ], "labelingGuide": "sales: prospects and new contracts, including new entities. customer_success: onboarding, adoption, renewals, expansion of an existing account, and cancellations (customer success owns saves). support: anything where the product is not working as expected, even if it involves an invoice screen; also generic requests for help with no other signal. billing: payment methods, invoices, receipts, charges, refunds, and how an existing customer will be charged.", "cases": [ { "id": "route-001", "input": "Hi, we're a 40-person design agency evaluating tools like yours. Could someone walk us through pricing for the Team plan and whether you offer annual discounts?", "expected": "sales", "difficulty": "typical" }, { "id": "route-002", "input": "Do you have an enterprise plan with SAML SSO and a DPA? We'd need a quote for roughly 300 seats.", "expected": "sales", "difficulty": "typical" }, { "id": "route-003", "input": "I saw your talk at the conference. We're not customers yet, but I'd love a demo for our ops team next week.", "expected": "sales", "difficulty": "typical" }, { "id": "route-004", "input": "What's the difference between Pro and Business? Trying to decide before we buy.", "expected": "sales", "difficulty": "typical" }, { "id": "route-005", "input": "We're an existing customer on the Starter plan in one region. Our sister company in Germany wants its own separate contract. Who can quote them?", "expected": "sales", "difficulty": "hard", "note": "Mentions being a customer, but the request is a new contract for a new entity." }, { "id": "route-006", "input": "We signed last quarter and adoption on our support team is low. Can someone help us set up a training session and share best practices?", "expected": "customer_success", "difficulty": "typical" }, { "id": "route-007", "input": "Our renewal is coming up in 60 days and we'd like to review how the rollout went before we commit to another year.", "expected": "customer_success", "difficulty": "typical" }, { "id": "route-008", "input": "We want to add 25 more seats for our new office and roll the product out there. What's the best way to onboard them?", "expected": "customer_success", "difficulty": "typical" }, { "id": "route-009", "input": "Could we schedule a quarterly business review? Our new VP wants to see usage across teams.", "expected": "customer_success", "difficulty": "typical" }, { "id": "route-010", "input": "Honestly we're not getting much value. Half the team stopped logging in. Before we decide whether to renew, can someone help us figure out what we're doing wrong?", "expected": "customer_success", "difficulty": "hard", "note": "Frustrated tone, but the request is about adoption and renewal." }, { "id": "route-011", "input": "When I click Export to CSV nothing happens. I tried Chrome and Firefox. This started this morning.", "expected": "support", "difficulty": "typical" }, { "id": "route-012", "input": "The Slack integration stopped posting notifications to our channel after we reconnected it.", "expected": "support", "difficulty": "typical" }, { "id": "route-013", "input": "How do I set up SSO with Okta? The settings page shows an error when I upload our metadata file.", "expected": "support", "difficulty": "typical" }, { "id": "route-014", "input": "I'm locked out of my account after resetting my password. The reset link says it has expired every time.", "expected": "support", "difficulty": "typical" }, { "id": "route-015", "input": "Our invoice PDF won't download. The button just spins forever.", "expected": "support", "difficulty": "hard", "note": "Mentions an invoice, but the problem is a broken download." }, { "id": "route-016", "input": "We were charged for 50 seats, but the admin panel only lets us assign 35. It says 'seat limit reached'. Is this a bug?", "expected": "support", "difficulty": "hard", "note": "The charge matches the purchase; the product is blocking seat assignment." }, { "id": "route-017", "input": "Please update the billing address on our invoices to our new office: 12 Harbour St, Leeds.", "expected": "billing", "difficulty": "typical" }, { "id": "route-018", "input": "Our card was declined for this month's payment. Can we pay by bank transfer instead?", "expected": "billing", "difficulty": "typical" }, { "id": "route-019", "input": "We need our VAT number added to the invoice for our accounts payable team.", "expected": "billing", "difficulty": "typical" }, { "id": "route-020", "input": "I was charged twice for March. Can you refund the duplicate?", "expected": "billing", "difficulty": "typical" }, { "id": "route-021", "input": "Why did our bill go up from $400 to $520 this month? We didn't add anyone.", "expected": "billing", "difficulty": "typical" }, { "id": "route-022", "input": "We'd like to cancel our subscription effective at the end of the term and get a final invoice.", "expected": "customer_success", "difficulty": "hard", "note": "Under this labeling guide, cancellations go to customer success for a save attempt even though an invoice is mentioned." }, { "id": "route-023", "input": "Thinking about upgrading from Pro to Business next month. Will we be charged the full price or prorated?", "expected": "billing", "difficulty": "hard", "note": "An upgrade is mentioned, but the question is how an existing customer will be charged." }, { "id": "route-024", "input": "Your tool looks great, but can it integrate with Jira? We'd buy the Business plan if so.", "expected": "sales", "difficulty": "hard", "note": "Reads like a feature request; it comes from a prospect with buying intent." }, { "id": "route-025", "input": "hey can u help", "expected": "support", "difficulty": "hard", "note": "No signal at all. The guide sends generic help requests to support; a model should show low confidence." }, { "id": "route-026", "input": "Can we get copies of the receipts for all payments made in 2025?", "expected": "billing", "difficulty": "typical" }, { "id": "route-027", "input": "New admin here. I just took over our workspace from a colleague who left. Is there someone who can get me up to speed on how our team uses the product?", "expected": "customer_success", "difficulty": "typical" }, { "id": "route-028", "input": "The dashboard has been showing yesterday's numbers all day. The sync seems stuck.", "expected": "support", "difficulty": "typical" } ] }