Skip to content

Commit

Permalink
simplifications
Browse files Browse the repository at this point in the history
  • Loading branch information
anthdr committed Jan 16, 2025
1 parent 61f79da commit 68a0e6b
Showing 1 changed file with 1 addition and 3 deletions.
4 changes: 1 addition & 3 deletions eole/bin/tools/LM_scoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from eole.constants import DefaultTokens, CorpusTask
from eole.transforms import get_transforms_cls, make_transforms
from eole.models.model import BaseModel
from eole.decoders.ensemble import load_test_model as ensemble_load_test_model

from argparse import ArgumentParser
from eole.bin import BaseBin, register_bin
Expand Down Expand Up @@ -65,8 +64,7 @@ def run(cls, args):
if len(config.gpu_ranks) > 1:
logger.warning(f"gpu_ranks is {str(config.gpu_ranks)} but only the first one will be used.")

load_test_model = ensemble_load_test_model if len(config.model_path) > 1 else BaseModel.load_test_model
vocabs, model, model_opt = load_test_model(config, 0)
vocabs, model, model_opt = BaseModel.load_test_model(config, 0)
pad_token = vocabs["specials"].get("pad_token", DefaultTokens.PAD)
padding_idx = vocabs["tgt"].tokens_to_ids[pad_token]
criterion = torch.nn.CrossEntropyLoss(ignore_index=padding_idx, reduction="none")
Expand Down

0 comments on commit 68a0e6b

Please sign in to comment.