MAX_TOKENS_IN_BATCH = 512

src/petals/client/sequential_autograd.py

@@ -20,7 +20,7 @@ from petals.utils.misc import DUMMY, is_dummy
 
 logger = get_logger(__name__)
 
-MAX_TOKENS_IN_BATCH = 1024
+MAX_TOKENS_IN_BATCH = 512
 
 
 async def sequential_forward(