{
  "slug": "ai-chatbot-eating-disorder-nonprofit-failure",
  "title": "A nonprofit's chatbot told eating disorder patients to lose weight",
  "date": "2026-02-12",
  "featuredClaim": "Vendor secretly upgraded eating disorder chatbot to generative AI, causing it to recommend dangerous weight loss.",
  "description": "A mental health charity deployed a clinically tested chatbot for eating disorder support, which was unexpectedly modified by a vendor to use generative AI. The new AI system began providing harmful weight loss advice, causing the chatbot to be pulled offline quickly.",
  "keyPoints": [
    "Vendor upgraded chatbot to generative AI without explicit approval",
    "Chatbot began recommending dangerous weight loss advice to eating disorder patients",
    "Contract lacked clear provisions about technology modifications",
    "No mechanism to prevent unilateral AI system changes"
  ],
  "topics": [
    {
      "id": "strategy",
      "slug": "ai-strategy",
      "label": "AI Strategy",
      "description": "Strategic planning and implementation approaches for AI adoption"
    },
    {
      "id": "business",
      "slug": "ai-business-applications",
      "label": "Business Applications",
      "description": "Real-world business use cases and applications"
    },
    {
      "id": "implementation",
      "slug": "ai-implementation",
      "label": "Implementation",
      "description": "Hands-on implementation techniques and frameworks"
    }
  ],
  "claims": [
    "A mental health charity's eating disorder chatbot underwent vendor upgrade to generative AI without explicit approval.",
    "The upgraded chatbot began advising eating disorder patients to reduce daily calorie intake by five hundred to one thousand.",
    "The charity's original chatbot underwent clinical testing with a seven hundred person trial showing measurable positive results.",
    "The vendor and charity disputed whether technology changes required approval, with neither party able to prove their case.",
    "The chatbot was removed from service within days while the human helpline it replaced had already shut down."
  ],
  "claimTitles": [
    "Unauthorized Generative AI Upgrade",
    "Dangerous Calorie Reduction Advice",
    "Clinically Validated Original System",
    "Contract Ambiguity Dispute",
    "Dual Service Elimination"
  ],
  "originalUrl": "https://aiadopters.club/p/a-nonprofits-chatbot-told-eating",
  "quote": "The vendor changed the AI without telling anyone. The contract had no clause to stop it.",
  "keyStatistics": [
    {
      "stat": "700-person trial",
      "context": "Clinical testing demonstrated real results before the vendor's unauthorized system upgrade"
    },
    {
      "stat": "500 to 1,000 calories per day",
      "context": "Dangerous reduction amount the upgraded chatbot recommended to eating disorder patients"
    },
    {
      "stat": "Incident 545",
      "context": "This failed chatbot is catalogued in the OECD AI Incident Database"
    },
    {
      "stat": "37 million users",
      "context": "A third organization successfully reached this scale using zero machine learning"
    }
  ],
  "supportingContext": "This case, documented as Incident 545 in the OECD AI Incident Database, demonstrates critical gaps in AI vendor governance for small and medium businesses. The charity's contract contained ambiguous language around system upgrades, allowing the vendor to substitute generative AI for the clinically-tested rule-based system. For practitioners, the incident highlights the necessity of explicit contractual clauses requiring written approval for model upgrades, version changes, and architectural modifications. The recommended immediate action is adding vendor notification requirements to all AI contracts before technology substitutions occur.",
  "canonicalUrl": "https://kbanc.com/claims-library/ai-chatbot-eating-disorder-nonprofit-failure",
  "markdownUrl": "https://kbanc.com/md/claims-library/ai-chatbot-eating-disorder-nonprofit-failure.md",
  "jsonUrl": "https://kbanc.com/api/claims/ai-chatbot-eating-disorder-nonprofit-failure.json",
  "source": {
    "publisher": "AI Adopters Club",
    "title": "A nonprofit's chatbot told eating disorder patients to lose weight",
    "url": "https://aiadopters.club/p/a-nonprofits-chatbot-told-eating"
  }
}