{
  "$format": "thethinkloop.benchmark/v1",
  "version": "1.0.0",
  "created": "2026-10-08",
  "license": "CC BY 4.0",
  "provenance": "Written and labeled by TheThinkLoop for this benchmark. Every message is synthetic: no customer data. Labels were assigned by a single annotator following the labeling guide; inter-annotator agreement has not been measured yet.",
  "id": "refund-eligibility",
  "title": "Refund eligibility",
  "templateSlug": "refund-eligibility",
  "description": "Customer messages about refunds for a B2B SaaS subscription, labeled against a 30-day refund policy.",
  "question": {
    "type": "predicate",
    "instructions": "Should this refund request be approved under our 30-day refund policy?"
  },
  "labels": [
    "Yes",
    "No"
  ],
  "labelingGuide": "Yes: the message requests a refund and the purchase or charge falls within 30 days, or the charge is a duplicate or billing error (refundable at any time). No: the purchase is older than 30 days with no billing error, the message is not a refund request, the charge was already refunded, or nothing in the message establishes eligibility. Hardship and goodwill exceptions are a separate decision (see the policy exception template) and are labeled No here.",
  "cases": [
    {
      "id": "refund-001",
      "input": "Hi, I bought the annual plan 4 days ago but it turns out our team already has a license through our parent company. Could I get a refund please?",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "refund-002",
      "input": "I was billed twice this month for the same subscription: two identical charges of $49 on March 3. Please refund one of them.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "refund-003",
      "input": "Signed up yesterday, and the product isn't what I expected. I'd like my money back.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "refund-004",
      "input": "We purchased 10 seats last week by mistake; we only needed 1. Can you refund the other 9?",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "refund-005",
      "input": "Your checkout charged my card three times when the page froze. I only want one subscription. Refund the extra two, please.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "refund-006",
      "input": "I upgraded to Pro two days ago and it doesn't support the SSO provider we use, which your sales rep said it did. Requesting a full refund.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "refund-007",
      "input": "Requesting a refund on order 55102, purchased 12 days ago. The export feature we needed is only in the Enterprise tier.",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "refund-008",
      "input": "We've been on the Team plan for about 8 months and we're switching tools. Can we get a refund for this year's remaining months?",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "refund-009",
      "input": "Bought this 6 months ago and barely used it. Want a refund.",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "refund-010",
      "input": "Can you send me a copy of last month's invoice? Our finance team needs it for their records.",
      "expected": "No",
      "difficulty": "typical",
      "note": "Not a refund request."
    },
    {
      "id": "refund-011",
      "input": "We renewed our annual subscription in January. It's now May and we've decided to cancel. Please refund the unused portion.",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "refund-012",
      "input": "I purchased a lifetime license two years ago. The app is slower now than it used to be and I want my money back.",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "refund-013",
      "input": "How do I change the credit card on file?",
      "expected": "No",
      "difficulty": "typical",
      "note": "Not a refund request."
    },
    {
      "id": "refund-014",
      "input": "Please cancel my plan at the end of the billing period. No refund needed.",
      "expected": "No",
      "difficulty": "typical",
      "note": "Explicitly not a refund request."
    },
    {
      "id": "refund-015",
      "input": "I'd like a refund for last year's subscription; we didn't use it much after the first month.",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "refund-016",
      "input": "I was charged on the 2nd for a renewal I didn't expect; I thought I'd turned auto-renew off back in the spring. Today is the 9th. Can I get that renewal refunded?",
      "expected": "Yes",
      "difficulty": "hard",
      "note": "The renewal charge is a purchase made 7 days ago, so it is inside the window even though the account is old."
    },
    {
      "id": "refund-017",
      "input": "You already refunded me for the duplicate charge last week, thanks! But I still see a pending charge of $49 on my statement. Can you refund that too?",
      "expected": "No",
      "difficulty": "hard",
      "note": "Most likely the same charge whose reversal is still pending. Approving would refund twice; a person should verify."
    },
    {
      "id": "refund-018",
      "input": "Purchased 45 days ago. I know your policy is 30 days, but I was in hospital for most of that time and couldn't try the product. Is there anything you can do?",
      "expected": "No",
      "difficulty": "hard",
      "note": "Sympathetic, but outside the policy. A hardship exception is a separate decision."
    },
    {
      "id": "refund-019",
      "input": "Great product honestly, no complaints. I just bought a second subscription by accident on my work account, 3 days after the first. Can the second one be refunded?",
      "expected": "Yes",
      "difficulty": "hard",
      "note": "Positive sentiment and 'no complaints' sit alongside a valid accidental duplicate purchase."
    },
    {
      "id": "refund-020",
      "input": "Refund. Now.",
      "expected": "No",
      "difficulty": "hard",
      "note": "Nothing establishes eligibility. The right action is a clarifying question, not an approval."
    },
    {
      "id": "refund-021",
      "input": "My card was charged $588 on April 30 for the annual plan, but I'd already paid for the year by invoice on April 1. Today is June 20. Please refund the card charge.",
      "expected": "Yes",
      "difficulty": "hard",
      "note": "Outside 30 days, but a duplicate payment is a billing error and is refundable at any time."
    },
    {
      "id": "refund-022",
      "input": "I'm writing on behalf of a client who purchased your service. They want a refund. Please send it to my account instead of theirs.",
      "expected": "No",
      "difficulty": "hard",
      "note": "No timing information, and a request to redirect funds to a third party."
    },
    {
      "id": "refund-023",
      "input": "Your outage last Tuesday cost us a full day of sales. I want this month refunded. We paid on the 1st and it's the 14th.",
      "expected": "Yes",
      "difficulty": "hard",
      "note": "Angry tone and an outage claim, but the payment is 13 days old and inside the window."
    },
    {
      "id": "refund-024",
      "input": "Not happy. Subscribed in 2024 and nothing has improved. Refund everything.",
      "expected": "No",
      "difficulty": "typical"
    },
    {
      "id": "refund-025",
      "input": "I meant to buy the monthly plan but clicked annual by mistake ten minutes ago. Can you switch me and refund the difference?",
      "expected": "Yes",
      "difficulty": "typical"
    },
    {
      "id": "refund-026",
      "input": "Charged twice?? I see two charges of $29 from you on my bank app, one on the 3rd and one on the 4th of this month. I only have one account.",
      "expected": "Yes",
      "difficulty": "typical"
    }
  ]
}