misc

fegin · fegin · commit 1b3d37dc2a08 · 2025-09-05T00:01:29.000-07:00
diff --git a/torchtitan/models/deepseek_v3/__init__.py b/torchtitan/models/deepseek_v3/__init__.py
@@ -75,6 +75,8 @@
         qk_rope_head_dim=64,
         v_head_dim=128,
         mscale=0.70,
+        use_flex_attn=True,
+        attn_mask_type="block_causal",
     ),
     "16B": DeepSeekV3ModelArgs(
         vocab_size=102400,
