address #222

Author: lucidrains

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vector-quantize-pytorch"
-version = "1.23.1"
+version = "1.23.2"
 description = "Vector Quantization - Pytorch"
 authors = [
     { name = "Phil Wang", email = "[email protected]" }

vector_quantize_pytorch/vector_quantize_pytorch.py

@@ -1063,6 +1063,19 @@ def update_in_place_optimizer(self):
         if not exists(self.in_place_codebook_optimizer):
             return
 
+        # handle ddp, thanks to @gdoras
+
+        if self._codebook.use_ddp:
+
+            for param in self._codebook.parameters():
+                if not exists(param.grad):
+                    continue
+
+                distributed.all_reduce(param.grad)
+                param.grad /= distributed.get_world_size()
+
+        # optimizer step
+
         self.in_place_codebook_optimizer.step()
         self.in_place_codebook_optimizer.zero_grad()