fix alibi in palm lite

lucidrains · Jun 20, 2022 · d78cd03 · d78cd03
1 parent 428c1e5
commit d78cd03
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/palm_pytorch/palm_lite.py b/palm_pytorch/palm_lite.py
@@ -54,7 +54,7 @@ def forward(self, qk_sim):
         h, i, j, device = *qk_sim.shape[-3:], qk_sim.device
 
         if exists(self.bias) and self.bias.shape[-1] >= j:
-            return qk_sim + self.bias[..., :i, :j]
+            return self.bias[..., :i, :j]
 
         bias = self.get_bias(i, j, device)
         bias = bias * self.slopes

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
     name="PaLM-pytorch",
     packages=find_packages(exclude=[]),
-    version="0.2.0",
+    version="0.2.1",
     license="MIT",
     description="PaLM: Scaling Language Modeling with Pathways - Pytorch",
     author="Phil Wang",