-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathNeMo_v2.1.0rc0.patch
29 lines (28 loc) · 1.55 KB
/
NeMo_v2.1.0rc0.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
diff --git a/nemo/collections/nlp/parts/nlp_overrides.py b/nemo/collections/nlp/parts/nlp_overrides.py
index 68c934154..c0e288795 100644
--- a/nemo/collections/nlp/parts/nlp_overrides.py
+++ b/nemo/collections/nlp/parts/nlp_overrides.py
@@ -384,16 +384,17 @@ class NLPDDPStrategy(DDPStrategy):
"""
# check if using distributed checkpointing
if self.use_distributed_checkpointing:
- assert (
- len(checkpoint['optimizer_states']) == 1
- ), "Currently only support checkpointing 1 distributed optimizer per time!"
+ # assert (
+ # len(checkpoint['optimizer_states']) == 1
+ # ), "Currently only support checkpointing 1 distributed optimizer per time!"
# converts the optimizer states to their sharded equivalents
- sharded_optim_state = self.optimizer_sharded_state_dict(
- unsharded_optim_state=checkpoint['optimizer_states'][0]
- )
+ # sharded_optim_state = self.optimizer_sharded_state_dict(
+ # unsharded_optim_state=checkpoint['optimizer_states'][0]
+ # )
# Check whether to save optim states
- include_optimizer = True if not storage_options else storage_options.get('include_optimizer', True)
+ # include_optimizer = True if not storage_options else storage_options.get('include_optimizer', True)
+ include_optimizer = False
if include_optimizer:
checkpoint['optimizer_states'] = [sharded_optim_state]
else: