{ "title": "Home RTX 3090 Qwen3.8-27B experiment: sanitized benchmark summary", "published": "2026-08-25", "updated": "2026-08-26", "measurement_window": "2026-08-18 to 2026-08-21", "hardware": { "setting": "after-hours experiment on a home PC; not production infrastructure", "gpu": "NVIDIA GeForce RTX 3090", "vram_gib": 24, "inference_slots": 1 }, "model": { "family": "Qwen3.8-27B", "parameters": "27B dense", "llama_cpp_weights": "Unsloth Dynamic V3 GGUF", "vllm_weights": "W4A16 AutoRound with a locally requantized fast variant" }, "long_context_fact_retrieval": [ { "profile": "60k", "prompt_tokens": 50059, "prefill_tokens_per_second": 890.7, "generation_tokens_per_second": 49.24, "wall_seconds": 59.3, "max_vram_mib": 22287, "facts_checked": 3, "facts_returned": 3 }, { "profile": "120k", "prompt_tokens": 115074, "prefill_tokens_per_second": 681.6, "generation_tokens_per_second": 40.46, "wall_seconds": 172.6, "max_vram_mib": 22649, "facts_checked": 3, "facts_returned": 3 }, { "profile": "250k", "prompt_tokens": 230085, "prefill_tokens_per_second": 412.4, "generation_tokens_per_second": 28.05, "wall_seconds": 563.4, "max_vram_mib": 23623, "facts_checked": 3, "facts_returned": 3 } ], "matched_go_json_error_repair": [ { "profile": "llama.cpp / Unsloth Dynamic V3 Q4_K_M", "harness": "Qwen Code 0.21.13, medium effort", "context_tokens": 120000, "score": 98, "score_max": 100, "score_assigned_by": "OpenAI Codex using the Syntalith rubric", "wall_seconds": 1468.76, "estimated_energy_wh": 100.36, "compactions": 1 }, { "profile": "patched vLLM / fast W4A16 AutoRound", "harness": "Qwen Code 0.21.13, medium effort", "context_tokens": 150000, "score": 100, "score_max": 100, "score_assigned_by": "OpenAI Codex using the Syntalith rubric", "wall_seconds": 502.14, "estimated_energy_wh": 33.88, "compactions": 0, "max_vram_mib": 22539 } ], "same_qwen_model_different_harnesses": [ { "harness": "Qwen Code", "score": 100, "score_max": 100, "score_assigned_by": "OpenAI Codex using the Syntalith rubric", "wall_seconds": 502.14, "compactions": 0, "judgment": "selected profile within the experiment" }, { "harness": "Codex", "score": 87, "score_max": 100, "score_assigned_by": "OpenAI Codex using the Syntalith rubric", "wall_seconds": 1150.42, "compactions": 1, "judgment": "material nondeterministic fallback defect" }, { "harness": "Claude Code compatibility experiment", "score": 98, "score_max": 100, "score_assigned_by": "OpenAI Codex using the Syntalith rubric", "wall_seconds": 1732.38, "compactions": 0, "judgment": "completed, but non-Claude routing is not supported by Anthropic" } ], "daily_driver_tasks": [ { "task": "CSV export", "harness": "Qwen Code", "effort": "medium", "hidden_checks": "3/3", "public_checks": "3/3", "wall_seconds": 38.24, "judgment": "best result in this task" }, { "task": "pagination", "harness": "Qwen Code", "effort": "medium", "hidden_checks": "3/3", "public_checks": "3/3", "wall_seconds": 264.53, "judgment": "smallest compatible production fix" }, { "task": "CSV import", "harness": "Qwen Code", "effort": "medium", "hidden_checks": "4/5", "public_checks": "12/12", "wall_seconds": 93.14, "judgment": "missed blank name and role validation" }, { "task": "CSV import", "harness": "Codex", "effort": "medium", "hidden_checks": "5/5", "public_checks": "11/11", "wall_seconds": 212.0, "judgment": "fully correct, larger patch" }, { "task": "CSV import", "harness": "OpenCode", "effort": "medium", "hidden_checks": "5/5", "public_checks": "9/9", "wall_seconds": 89.04, "judgment": "fastest fully correct result in this task" } ], "frontend_repair": { "data": "synthetic applicant records", "creation": { "prompt_scope": "one-sentence request to build and test an interactive ATS frontend from an empty folder", "harness": "Qwen Code 0.21.13", "effort": "low", "wall_seconds": 1563.22, "tool_calls": 55, "output_tokens": 104161, "core_text_files": 4, "core_lines": 2028, "core_bytes": 79588, "completed": false, "terminal_reason": "client loop detector stopped the run before complete browser QA" }, "before": { "matching_rows": 8652, "dom_elements": 251327, "document_scroll_height_px": 485065, "judgment": "the intended virtual list rendered the whole dataset" }, "repair": { "wall_seconds": 863.96, "tool_calls": 50, "judgment": "fixed two virtualization defects and four latent interaction or data defects" }, "visual_followup": { "wall_seconds": 477.8, "tool_calls": 20, "judgment": "fixed two CSS defects found by independent screenshot review" }, "after_independent_desktop_check": { "dom_nodes": 722, "rendered_rows": 15, "document_height_px": 900, "horizontal_page_overflow": false }, "after_independent_mobile_check": { "dom_nodes": 577, "rendered_rows": 10, "document_height_px": 844, "horizontal_page_overflow": false } }, "short_context_speculation": { "baseline": "150k MTP-3", "baseline_decode_tokens_per_second": 93.3, "dflash2_7_context_tokens": 65536, "dflash2_7_decode_tokens_per_second_runs": [123.3, 111.4, 120.9], "dflash2_7_median_decode_tokens_per_second": 120.9, "median_gain_percent": 29.6, "decision": "optional short-context profile; the 150k MTP profile remained the reference for heavier experiment cells" }, "limitations": [ "Each configuration was run once, so the results do not establish repeatability.", "The 100-point scores were assigned by OpenAI Codex under a Syntalith rubric, not by a human auditor or independent laboratory.", "The harness comparison used the same local Qwen3.8-27B model. It is not a comparison with hosted OpenAI or Anthropic models.", "The vLLM profile uses a third-party patched and frozen stack, not vanilla upstream vLLM.", "The experiment evaluated one GPU and one inference slot. It does not establish multi-user production throughput.", "The matched vLLM and llama.cpp cell compared whole profiles with different formats, engines, KV-cache types, speculation paths and context limits. It does not isolate quantization quality." ] }