feat(moe): wire DiskProvider into MoE block loaders

When --expert-offload is set and tensor_storage is available:
- Qwen3MoeBlock loads router gate from VarBuilder (stays in RAM),
  expert weights streamed via DiskExpertProvider from safetensors
- Qwen3_5MoeBlock same pattern + shared expert stays in RAM
- Without --expert-offload, behavior is unchanged (ResidentProvider)

The full expert offloading pipeline is now functional end-to-end:
  CLI flag → Context → tensor_storage → DiskProvider → pread per expert

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: evilsocket

cake-core/src/models/qwen3_5_moe/block.rs

@@ -82,7 +82,31 @@ impl Forwarder for Qwen3_5MoeBlock {
         let rms_1 = load_rms_norm(h, eps, cfg.residual_rms_norm, vb.pp("input_layernorm"))?;
         let rms_2 =
             load_rms_norm(h, eps, cfg.residual_rms_norm, vb.pp("post_attention_layernorm"))?;
-        let moe = Qwen3_5MoeSparseMlp::load(vb.pp("mlp"), cfg, ctx.backend.clone())?;
+        let moe = if let Some(storage) = &ctx.tensor_storage {
+            // Expert offload: stream routed expert weights from disk
+            use candle_nn::linear_no_bias as linear;
+            let layer_prefix = format!("{name}.mlp");
+            let provider: std::sync::Arc<dyn crate::models::common::expert_provider::ExpertProvider> =
+                std::sync::Arc::new(crate::models::common::disk_expert_provider::DiskExpertProvider::new(
+                    storage.clone(), layer_prefix, cfg.num_experts, ctx.device.clone(), ctx.dtype,
+                ));
+            let mlp_vb = vb.pp("mlp");
+            let gate_w = mlp_vb.pp("gate").get((cfg.num_experts, h), "weight")?;
+            let gate = candle_nn::Linear::new(gate_w, None);
+            let si = cfg.shared_expert_intermediate_size.expect("shared_expert_intermediate_size");
+            let se = mlp_vb.pp("shared_expert");
+            let shared_gate_proj = linear(h, si, se.pp("gate_proj"))?;
+            let shared_up_proj = linear(h, si, se.pp("up_proj"))?;
+            let shared_down_proj = linear(si, h, se.pp("down_proj"))?;
+            let seg_w = mlp_vb.pp("shared_expert_gate").get((1, h), "weight")?;
+            let shared_expert_gate = candle_nn::Linear::new(seg_w, None);
+            Qwen3_5MoeSparseMlp::with_provider(
+                gate, provider, shared_gate_proj, shared_up_proj, shared_down_proj,
+                shared_expert_gate, cfg.num_experts, cfg.num_experts_per_tok, ctx.backend.clone(),
+            )
+        } else {
+            Qwen3_5MoeSparseMlp::load(vb.pp("mlp"), cfg, ctx.backend.clone())?
+        };
 
         if layer_type == "full_attention" {
             let attn = Qwen3_5FullAttention::load(vb.pp("self_attn"), cfg, ctx.backend.clone())?;

cake-core/src/models/qwen3_5_moe/moe.rs

@@ -100,6 +100,33 @@ impl Qwen3_5MoeSparseMlp {
         })
     }
 
+    /// Construct with a pre-built expert provider (for disk offloading).
+    /// Shared expert + router are loaded from VarBuilder (stay in RAM).
+    #[allow(clippy::too_many_arguments)]
+    pub fn with_provider(
+        gate: Linear,
+        expert_provider: SharedExpertProvider,
+        shared_gate_proj: Linear,
+        shared_up_proj: Linear,
+        shared_down_proj: Linear,
+        shared_expert_gate: Linear,
+        num_experts: usize,
+        num_experts_per_tok: usize,
+        backend: Arc<dyn ComputeBackend>,
+    ) -> Self {
+        Self {
+            gate,
+            expert_provider,
+            shared_gate_proj,
+            shared_up_proj,
+            shared_down_proj,
+            shared_expert_gate,
+            num_experts,
+            num_experts_per_tok,
+            backend,
+        }
+    }
+
     pub fn forward(&self, x: &Tensor) -> anyhow::Result<Tensor> {
         let (b, s, h) = x.dims3().map_err(|e| anyhow!("moe dims3: {e}"))?;
         let n_tok = b * s;

cake-core/src/models/qwen3_moe/block.rs

@@ -41,7 +41,28 @@ impl Forwarder for Qwen3MoeBlock {
         let cfg = ctx.config.as_ref().expect("No config specified");
 
         let attn = CausalSelfAttention::load(vb.pp("self_attn"), cfg, ctx.backend.clone())?;
-        let moe = SparseMoeMlp::load(vb.pp("mlp"), cfg, ctx.backend.clone())?;
+
+        let moe = if let Some(storage) = &ctx.tensor_storage {
+            // Expert offload: stream weights from disk via DiskProvider
+            let layer_prefix = format!("{name}.mlp");
+            let provider: std::sync::Arc<dyn crate::models::common::expert_provider::ExpertProvider> =
+                std::sync::Arc::new(crate::models::common::disk_expert_provider::DiskExpertProvider::new(
+                    storage.clone(),
+                    layer_prefix,
+                    cfg.num_experts,
+                    ctx.device.clone(),
+                    ctx.dtype,
+                ));
+            // Load router gate from VarBuilder (it's small, stays in RAM)
+            let gate_w = vb.pp("mlp").pp("gate").get((cfg.num_experts, cfg.hidden_size), "weight")?;
+            let gate = candle_nn::Linear::new(gate_w, None);
+            SparseMoeMlp::with_provider(
+                gate, provider, cfg.num_experts, cfg.num_experts_per_tok,
+                cfg.norm_topk_prob, ctx.backend.clone(),
+            )
+        } else {
+            SparseMoeMlp::load(vb.pp("mlp"), cfg, ctx.backend.clone())?
+        };
 
         let eps = cfg.rms_norm_eps;
         let h = cfg.hidden_size;