Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Oct 20, 2023
1 parent 3b9e3c0 commit ab3128d
Showing 1 changed file with 24 additions and 5 deletions.
29 changes: 24 additions & 5 deletions examples/nlp/language_modeling/megatron_ckpt_to_nemo.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,12 @@
from nemo.collections.nlp.models.language_modeling.megatron_retrieval_model import MegatronRetrievalModel
from nemo.collections.nlp.models.language_modeling.megatron_t5_model import MegatronT5Model
from nemo.collections.nlp.models.machine_translation.megatron_nmt_model import MegatronNMTModel
from nemo.collections.nlp.parts.nlp_overrides import NLPDDPStrategy, NLPSaveRestoreConnector, GradScaler, PipelineMixedPrecisionPlugin
from nemo.collections.nlp.parts.nlp_overrides import (
GradScaler,
NLPDDPStrategy,
NLPSaveRestoreConnector,
PipelineMixedPrecisionPlugin,
)
from nemo.utils import AppState, logging
from nemo.utils.distributed import initialize_distributed
from nemo.utils.model_utils import inject_model_parallel_rank
Expand Down Expand Up @@ -92,7 +97,14 @@ def get_args():
)
parser.add_argument("--local_rank", type=int, required=False, default=os.getenv('LOCAL_RANK', -1))
parser.add_argument("--bcp", action="store_true", help="Whether on BCP platform")
parser.add_argument("--precision", type=str, required=False, default='16-mixed', choices=['16-mixed', 'bf16-mixed'], help="Precision value for the trainer that matches with precision of the ckpt")
parser.add_argument(
"--precision",
type=str,
required=False,
default='16-mixed',
choices=['16-mixed', 'bf16-mixed'],
help="Precision value for the trainer that matches with precision of the ckpt",
)

args = parser.parse_args()
return args
Expand All @@ -110,11 +122,18 @@ def convert(local_rank, rank, world_size, args):
if args.model_type == 'gpt':
strategy = NLPDDPStrategy()

cfg = {'trainer':{'devices':args.gpus_per_node, 'num_nodes':num_nodes, 'accelerator':'gpu', 'precision':args.precision},
'model':{'native_amp_init_scale':2 ** 32, 'native_amp_growth_interval':1000, 'hysteresis':2}}
cfg = {
'trainer': {
'devices': args.gpus_per_node,
'num_nodes': num_nodes,
'accelerator': 'gpu',
'precision': args.precision,
},
'model': {'native_amp_init_scale': 2 ** 32, 'native_amp_growth_interval': 1000, 'hysteresis': 2},
}
cfg = OmegaConf.create(cfg)

scaler=None
scaler = None
# If FP16 create a GradScaler as the build_model_parallel_config of MegatronBaseModel expects it
if cfg.trainer.precision == '16-mixed':
scaler = GradScaler(
Expand Down

0 comments on commit ab3128d

Please sign in to comment.