feat(multi_head_attention.py): set bias=True

InternLM · Jan 19, 2024 · 05fa04a · 05fa04a
1 parent 71543b3
commit 05fa04a
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/internlm/model/multi_head_attention.py b/internlm/model/multi_head_attention.py
@@ -209,7 +209,7 @@ def __init__(
             embed_dim,
             3 * embed_dim,
             process_group,
-            bias=False,
+            bias=True,
             sequence_parallel=gpc.config.parallel.sequence_parallel,
             **factory_kwargs,
         )  # according to https://spaces.ac.cn/archives/9577
@@ -232,7 +232,7 @@ def __init__(
             embed_dim,
             embed_dim,
             process_group,
-            bias=False,
+            bias=True,
             sequence_parallel=gpc.config.parallel.sequence_parallel,
             **factory_kwargs,
         )