Skip to content

Commit

Permalink
Update sentencepiece.cpp
Browse files Browse the repository at this point in the history
perform ls for debug only when loading tokenizer model fails
  • Loading branch information
mikekgfb authored Dec 22, 2024
1 parent 8195034 commit f7b3df6
Showing 1 changed file with 6 additions and 6 deletions.
12 changes: 6 additions & 6 deletions tokenizer/sentencepiece.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,15 @@ void SPTokenizer::load(const std::string& tokenizer_path) {
fprintf(stderr, "Tokenizer already initialized.\n");
return;
}
// Execute 'ls -al' on the tokenizer path
std::string command = "ls -al " + tokenizer_path;
int ret = system(command.c_str());
if (ret != 0) {
fprintf(stderr, "Failed to execute 'ls -al' in path: %s\n", tokenizer_path.c_str());
}
// read in the file
const auto status = _processor->Load(tokenizer_path);
if (!status.ok()) {
// Execute 'ls -al' on the tokenizer path
std::string command = "ls -al " + tokenizer_path;
int ret = system(command.c_str());
if (ret != 0) {
fprintf(stderr, "Failed to execute 'ls -al' on path: %s\n", tokenizer_path.c_str());
}
fprintf(stderr, "Could not load `%s`.\n If this tokenizer artifact is for llama3, please pass `-l 3`.", tokenizer_path.c_str());
exit(EXIT_FAILURE);
}
Expand Down

0 comments on commit f7b3df6

Please sign in to comment.