{ "title": "DeepSeek vs Claude free-tier benchmark", "version": "1.0", "published": "2026-10-11", "last_verified": "2026-10-11", "publisher": "The Orange Club", "page": "https://theorangeclub.me/deepseek-vs-claude-free-tier-test", "version_history": [ "v1.0, 11 October 2026: Round 1 (10 head-to-head tasks, qualitative) and Round 2 (10 DeepSeek-only checkable tasks, scored out of 50)." ], "limitations": [ "Single-day, single-session snapshot of web chat interfaces; not a universal ranking of the underlying models.", "Round 1 used unequal tool access: Claude had code execution and web search; DeepSeek reported none and its search toggle was not enabled.", "Round 1 has no numeric score. Round 2 is DeepSeek-only (45/50) and is not a head-to-head result; do not combine or compare the rounds.", "Round 1 prompts are summarized in v1.0; exact prompt text is not included.", "Arabic output (R2-08) has not been native-speaker reviewed.", "Test 9 (R2-09) should be re-run with search enabled." ], "scoring_rubric": "Per task: accuracy 0-2, instruction following 0-2, honesty 0-1; maximum 5 per task and 50 for ten tasks.", "records": [ { "test_id": "R1-01", "round": 1, "type": "head-to-head, qualitative", "category": "Complex problem-solving", "prompt": "Design a bilingual (Arabic/English) inquiry-triage system around an existing CRM (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Added PII redaction, rules engine, shadow-mode rollout and a risk table", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Clear architecture, solid metrics, good missing-info list; some targets asserted without a baseline", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R1-02", "round": 1, "type": "head-to-head, qualitative", "category": "Coding", "prompt": "CSV customer-deduplication script with edge cases and tests (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Ran its code, 9 of 9 tests passed; handled time zones, BOM and Arabic", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Sound script and 6 tests; not executed", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading; code run by Claude in its own session" }, { "test_id": "R1-03", "round": 1, "type": "head-to-head, qualitative", "category": "Debugging", "prompt": "Intermittent duplicate accounts under high traffic (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Same core causes plus UI double-submit, replica lag, and duplicate cleanup before migration", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Race condition, retries, unique constraints, isolation levels", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R1-04", "round": 1, "type": "head-to-head, qualitative", "category": "Writing quality", "prompt": "Outline and opening for an article aimed at AI-skeptical business owners (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Opening built on a concrete support-team example", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Tight, punchy opening; outline cited an unsourced 10%/90% split", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R1-05", "round": 1, "type": "head-to-head, qualitative", "category": "Critical thinking", "prompt": "Argue both sides of 'DeepSeek is a better free assistant than Claude' (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Balanced; separated impressions from facts", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Balanced; a few unverified comparative claims", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R1-06", "round": 1, "type": "head-to-head, qualitative", "category": "Research accuracy", "prompt": "Compare both free tiers using current verifiable facts (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Searched the web, dated its verification, labeled weak sources; did not read Anthropic's privacy policy", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Declined: no live access; referenced an older Claude model name", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R1-07", "round": 1, "type": "head-to-head, qualitative", "category": "Math and logic", "prompt": "Sales-funnel calculation with relative vs percentage-point improvements (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Correct, plus a 20-point vs 20% worked contrast", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Correct on all four cases", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R1-08", "round": 1, "type": "head-to-head, qualitative", "category": "Memory and instruction following", "prompt": "Revise one support-policy design through four changing requirements (simulated in one pass) (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Also caught an ambiguity between two refund rules", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Tracked constraints and flagged its own contradictions", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R1-09", "round": 1, "type": "head-to-head, qualitative", "category": "Business analysis", "prompt": "Payback and ROI for an invoice-automation project (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Same numbers, plus a sensitivity case", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Correct: 4.7-month payback, AED 46,800 year one", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R1-10", "round": 1, "type": "head-to-head, qualitative", "category": "Practical usefulness", "prompt": "Prepare a business owner for a AED 25,000 client meeting (summary; exact prompt text not included in v1.0)", "date": "11 October 2026", "claude_model_and_interface": "Claude Sonnet 5.5 (medium); Claude web chat", "claude_tool_access": "Code execution and web search used", "claude_observation": "Recommendation first, one extra objection", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "None reported; search toggle not enabled", "deepseek_observation": "Ready-to-use objection scripts", "answer_key": null, "score_accuracy": null, "score_instruction": null, "score_honesty": null, "score_total": null, "scoring_rationale": null, "verification_method": "Side-by-side reading" }, { "test_id": "R2-01", "round": 2, "type": "DeepSeek only, checkable", "category": "Math", "prompt": "A shop buys an item for AED 80, marks it up 35%, then sells it at a 15% discount off the marked-up price. What is the profit per item?", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "AED 11.80 profit, correct", "answer_key": "Profit AED 11.80", "score_accuracy": 2, "score_instruction": 2, "score_honesty": 1, "score_total": 5, "scoring_rationale": "Correct figure, single answer as asked, no overclaiming", "verification_method": "Hand calculation" }, { "test_id": "R2-02", "round": 2, "type": "DeepSeek only, checkable", "category": "Logic", "prompt": "Ahmed is taller than Sara. Sara is taller than Omar. Layla is shorter than Omar but taller than Noor. List everyone from tallest to shortest.", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "Correct order", "answer_key": "Ahmed, Sara, Omar, Layla, Noor", "score_accuracy": 2, "score_instruction": 2, "score_honesty": 1, "score_total": 5, "scoring_rationale": "Order matches the answer key exactly", "verification_method": "Answer key" }, { "test_id": "R2-03", "round": 2, "type": "DeepSeek only, checkable", "category": "Coding", "prompt": "Write a Python function that merges overlapping intervals, with a few test cases. Handle an empty list and touching intervals like [1,4] and [4,5].", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "Passed every case including touching intervals and an empty list; stated it was unable to run code", "answer_key": "Merged intervals; [[1,4],[4,5]] gives [[1,5]]; empty list gives empty list", "score_accuracy": 2, "score_instruction": 2, "score_honesty": 1, "score_total": 5, "scoring_rationale": "Code passed every case we ran; model correctly stated it had not executed the code", "verification_method": "Code executed by The Orange Club" }, { "test_id": "R2-04", "round": 2, "type": "DeepSeek only, checkable", "category": "Debugging", "prompt": "This function should return the last n items of a list. What is wrong with it? def last_n(items, n): return items[-n:]", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "Found n=0 bug and fixed negative values", "answer_key": "n=0 returns whole list; also negative and oversized n", "score_accuracy": 2, "score_instruction": 2, "score_honesty": 1, "score_total": 5, "scoring_rationale": "Identified the n=0 bug and also covered negative values", "verification_method": "Code executed by The Orange Club" }, { "test_id": "R2-05", "round": 2, "type": "DeepSeek only, checkable", "category": "Hallucination trap", "prompt": "Summarize the 2019 paper \"Recursive Gradient Folding for Sparse Transformers\" by Halvorsen and Mbeki.", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "Said it was unable to find or verify the paper and did not invent a summary", "answer_key": "Paper is invented; model should refuse to summarize", "score_accuracy": 2, "score_instruction": 2, "score_honesty": 1, "score_total": 5, "scoring_rationale": "Refused to invent a summary, which is the correct behavior", "verification_method": "Answer key" }, { "test_id": "R2-06", "round": 2, "type": "DeepSeek only, checkable", "category": "Strict instructions", "prompt": "Write exactly 3 sentences about remote work. Sentence 1 must start with the letter A, sentence 2 with B, sentence 3 with C. Use no commas, and include the word \"AED\" exactly once.", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "All constraints met", "answer_key": "3 sentences, A/B/C, 0 commas, AED once", "score_accuracy": 2, "score_instruction": 2, "score_honesty": 1, "score_total": 5, "scoring_rationale": "Every constraint verified by script", "verification_method": "Constraints counted by script" }, { "test_id": "R2-07", "round": 2, "type": "DeepSeek only, checkable", "category": "JSON output", "prompt": "Return ONLY valid JSON with keys vendor, invoice_number, date, subtotal, vat, total, payment_terms. Text: \"Invoice INV-7781 from Gulf Pearl Trading LLC dated 3 March 2026: 4 laptops at AED 3,200 each, VAT 5%, payment due in 30 days.\"", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "Valid JSON with correct figures", "answer_key": "Valid JSON; subtotal 12800, VAT 640, total 13440", "score_accuracy": 2, "score_instruction": 2, "score_honesty": 1, "score_total": 5, "scoring_rationale": "JSON parsed cleanly and all figures matched the key", "verification_method": "JSON parsed and figures checked by script" }, { "test_id": "R2-08", "round": 2, "type": "DeepSeek only, checkable", "category": "Arabic writing", "prompt": "Write a formal Arabic email to a supplier asking to delay a payment by 10 days. Then give an English translation.", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "Formal MSA and faithful translation; one slightly muddled phrase. Needs native-speaker review", "answer_key": "Formal MSA, faithful translation", "score_accuracy": 1, "score_instruction": 2, "score_honesty": 1, "score_total": 4, "scoring_rationale": "Accuracy 1 of 2 for one muddled phrase; instructions fully met; native-speaker review pending", "verification_method": "Read by The Orange Club; not yet native-reviewed" }, { "test_id": "R2-09", "round": 2, "type": "DeepSeek only, checkable", "category": "Research", "prompt": "What are the current free-tier usage limits for ChatGPT, Claude, and Gemini? Give sources and the date you checked.", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "Said it had no search tool in this setup and gave no figures; honest but task not completed", "answer_key": "Real sources, check date, no invented figures", "score_accuracy": 0, "score_instruction": 0, "score_honesty": 1, "score_total": 1, "scoring_rationale": "Task not completed so accuracy and instruction scores are 0; honesty point earned for inventing no figures", "verification_method": "Answer key; re-run with search enabled recommended" }, { "test_id": "R2-10", "round": 2, "type": "DeepSeek only, checkable", "category": "Business analysis", "prompt": "A company processes 600 invoices a month at 8 minutes each. Labor costs AED 50 per hour. Automation costs AED 20,000 upfront plus AED 1,500 a month and cuts manual time by 65%. Is it worth it? Show payback and say what assumptions could change your answer.", "date": "11 October 2026", "claude_model_and_interface": "Not run", "claude_tool_access": "Not run", "claude_observation": "Not run", "deepseek_model_and_interface": "DeepSeek chat (default web mode); DeepSeek web chat", "deepseek_tool_access": "No search tool available in setup; code not executed by the model", "deepseek_observation": "18.2-month payback and a saved-time-is-not-cash caveat", "answer_key": "Payback about 18.2 months with caveats", "score_accuracy": 2, "score_instruction": 2, "score_honesty": 1, "score_total": 5, "scoring_rationale": "Payback figure matches the key and the caveat on saved time versus cash is present", "verification_method": "Hand calculation" } ] }