regenerate lock file

MinishLab · stephantul · Feb 7, 2025 · Dec 22, 2024 · Dec 22, 2024 · Dec 22, 2024
commit 2167a4e270f50f9f7d6210cc22f83611d0793b08
diff --git a/model2vec/distill/tokenizer.py b/model2vec/distill/tokenizer.py
        tokenizer_data["model"]["vocab"] = reindexed

    elif model_type == "Unigram":
        logger.warning("Removing tokens from a unigram tokenizer is not supported.")
        return tokenizer

    elif model_type == "BPE":
        logger.warning("Removing tokens from a BPE tokenizer is not supported.")
        return tokenizer

    else:
        raise ValueError(f"Unknown model type {model_type}")
	tokenizer_data["model"]["vocab"] = reindexed

	elif model_type == "Unigram":
	logger.warning("Removing tokens from a unigram tokenizer is not supported.")
	return tokenizer
Check warning on line 72 in model2vec/distill/tokenizer.py Codecov / codecov/patch model2vec/distill/tokenizer.py#L71-L72 `Added lines #L71 - L72 were not covered by tests`

	elif model_type == "BPE":
	logger.warning("Removing tokens from a BPE tokenizer is not supported.")
	return tokenizer
Check warning on line 76 in model2vec/distill/tokenizer.py Codecov / codecov/patch model2vec/distill/tokenizer.py#L75-L76 `Added lines #L75 - L76 were not covered by tests`

	else:
	raise ValueError(f"Unknown model type {model_type}")