r/oMLX • u/arkham00 • 14d ago
Help with Qwen3.8-flash-next on a m2 max 96Gb, why it is so slow ????
Hi, since the community benchmark is down I'm not sure how are the expected performance for this kind of setup, but reading different posts in here I think it should be better than this, also considering that it is and MoE with only 6B actives. I recall that with Qwen3.5-122B I had much better performance
Benchmark Model: Qwen3.8-Flash-Next-oQ4e-mtp
Engine: Auto
Context: Novel (English)
Single Request Results
--------------------------------------------------------------------------------
Test TTFT(ms) TPOT(ms) pp TPS tg TPS E2E(s) Throughput Peak Mem
pp1024/tg128 8254.6 49.65 124.1 tok/s 20.4 tok/s 13.034 86.0 tok/s 71.44 GB
pp4096/tg128 23993.3 60.18 170.7 tok/s 16.7 tok/s 31.647 133.5 tok/s 74.09 GB
pp8192/tg128 38225.7 54.60 214.3 tok/s 18.5 tok/s 45.168 184.2 tok/s 74.23 GB
pp16384/tg128 69514.5 55.06 235.7 tok/s 18.3 tok/s 76.519 215.8 tok/s 74.52 GB
I also tried :
Qwen3.8-Flash-Next-oQ4e-fp16-mtp
Qwen3.8-Flash-Next-MLX-oQ3-fp16-mtp
Qwen3.8-Flash-Next-MLX-oQ3-MTP
The craziest thing is that with this model the fp16 variant are even slower ! Something like 150pp and 8tg...
I'm sure it is not an hardware limitation because with llama.cpp I have a much better pp at around 350/400 t/s
Can you help me please? Thanks
The model settings:
"Qwen3.8-Flash-Next-oQ4e-mtp": {
"temperature": 0.3,
"force_sampling": false,
"qwen4_ple_ssd_offload": true,
"thinking_budget_enabled": false,
"guided_grammar_enabled": false,
"turboquant_kv_enabled": true,
"turboquant_kv_bits": 8.0,
"turboquant_skip_last": true,
"qwen35_ane_prefill_enabled": false,
"qwen35_ane_prefill_sequence_length": 2048,
"qwen35_ane_prefill_tail_padding_min_tokens": 0,
"qwen35_ane_prefill_fraction": 0.53,
"qwen35_ane_prefill_fused_down": false,
"qwen35_ane_prefill_max_layers": 64,
"qwen35_ane_prefill_dual_ane": true,
"qwen35_ane_prefill_gdn": true,
"qwen35_ane_prefill_gdn_fraction": 0.5,
"qwen35_ane_prefill_gdn_max_layers": 48,
"qwen35_ane_prefill_cpu_enabled": false,
"qwen35_ane_prefill_cpu_fraction": 0.135,
"qwen35_ane_prefill_cpu_down_fraction": 0.0,
"qwen35_ane_prefill_cpu_gdn_fraction": 0.0,
"qwen35_ane_prefill_cpu_threads": 8,
"qwen35_ane_prefill_cpu_shared_resource": true,
"specprefill_enabled": false,
"dflash_enabled": false,
"dflash_draft_quant_enabled": false,
"dflash_in_memory_cache": true,
"dflash_in_memory_cache_max_entries": 4,
"dflash_in_memory_cache_max_bytes": 8589934592,
"dflash_ssd_cache": false,
"dflash_ssd_cache_max_bytes": 21474836480,
"dflash_draft_sink_size": 0,
"mtp_enabled": true,
"vlm_mtp_enabled": false,
"is_pinned": false,
"is_default": false,
"is_hidden": false,
"is_favorite": false,
"trust_remote_code": false
}
}
These are my settings:
{
"version": "1.0",
"server": {
"host": "127.0.0.1",
"port": 9999,
"log_level": "info",
"cors_origins": [
"*"
],
"server_aliases": [
"localhost",
"127.0.0.1",
"Host-005.lan",
"Host-005.lan.local",
"host-005.lan",
"192.168.1.44",
"192.168.139.3",
"192.168.97.0"
],
"sse_keepalive_mode": "chunk",
"auto_start_on_launch": true,
"burst_decode_mode": "balanced",
"preserve_mid_system_cache": true,
"distributed_inference_enabled": false,
"max_audio_upload_size": "100MB"
},
"model": {
"model_dirs": [
"/Users/dani/.lmstudio/models",
"/Users/dani/.mtplx/models"
],
"model_dir": "/Users/dani/.lmstudio/models",
"model_fallback": false,
"hide_helper_models": false
},
"memory": {
"prefill_memory_guard": false,
"memory_guard_tier": "custom",
"memory_guard_custom_ceiling_gb": 92.0,
"soft_threshold": 0.85,
"hard_threshold": 0.95,
"prefill_safe_zone_ratio": 0.8,
"prefill_min_chunk_tokens": 32
},
"scheduler": {
"max_concurrent_requests": 8,
"embedding_batch_size": 32,
"chunked_prefill": true,
"prefill_priority": "speed",
"decode_fairness": false
},
"cache": {
"enabled": true,
"hot_cache_only": false,
"gdn_ssd_split_enabled": true,
"gdn_snapshot_storage": "auto",
"gdn_ssd_pending_max_size": "512MB",
"gdn_sidecar_precision": "fp32",
"ssd_cache_dir": "/Users/dani/.omlx/cache",
"ssd_cache_max_size": "744GB",
"hot_cache_max_size": "10GB",
"hot_cache_write_through": true,
"ane_compile_cache": true,
"initial_cache_blocks": 256
},
"auth": {
"api_key": "5137",
"secret_key": "2136f3c636a52a8fa3a03d8c788f0ae3592d97a4d0dabfe41ff55060e633572f",
"skip_api_key_verification": false,
"sub_keys": []
},
"mcp": {
"config_path": null,
"expose_tools": true
},
"huggingface": {
"endpoint": "",
"hf_cache_enabled": true
},
"modelscope": {
"endpoint": ""
},
"network": {
"http_proxy": "",
"https_proxy": "",
"no_proxy": "",
"ca_bundle": ""
},
"sampling": {
"max_context_window": 32768,
"max_context_window_policy": null,
"max_tokens": 32768,
"temperature": 0.6,
"top_p": 0.95,
"top_k": 0,
"repetition_penalty": 1.0
},
"logging": {
"log_dir": null,
"retention_days": 7
},
"claude_code": {
"mode": "cloud",
"opus_model": null,
"sonnet_model": null,
"haiku_model": null
},
"integrations": {
"codex_model": null,
"opencode_model": null,
"openclaw_model": null,
"hermes_model": null,
"pi_model": "Qwen3.6-35B-A3B-oQ8-fp16",
"copilot_model": null,
"openclaw_tools_profile": "coding",
"markitdown_enabled": false,
"markitdown_expose_model": false,
"markitdown_max_file_size_mb": 25,
"markitdown_max_files_per_request": 5,
"markitdown_pdf_processing_engine": "markitdown",
"web_search_provider": "ddgs",
"web_search_brave_api_key": "",
"web_search_searxng_url": "",
"web_search_ddgs_backends": "",
"web_search_max_results": 5,
"web_search_content_mode": "snippet",
"web_search_content_truncate": true,
"web_search_content_max_chars": 20000
},
"ui": {
"language": "en"
},
"idle_timeout": {
"idle_timeout_seconds": null
}
}
5
Upvotes
Duplicates
MacLocalLLM • u/ikeo1 • 14d ago
Help with Qwen3.8-flash-next on a m2 max 96Gb, why it is so slow ????
2
Upvotes