diff --git a/utils.cpp b/utils.cpp index a2195093b6ab5a..581f8dabf26778 100644 --- a/utils.cpp +++ b/utils.cpp @@ -303,7 +303,7 @@ struct llama_sp_symbol { struct llama_sp_bigram { struct comparator { - const bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) { + bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) { return (l.score < r.score) || (l.score == r.score && l.left > r.left); } }; @@ -409,7 +409,7 @@ struct llama_tokenizer { work_queue_.push(bigram); } - gpt_vocab const& vocab_; + const gpt_vocab & vocab_; std::vector symbols_; llama_sp_bigram::queue work_queue_; }; @@ -426,16 +426,7 @@ std::vector llama_tokenize(const gpt_vocab & vocab, std::string_v output.push_back(1); } - if (text[0] != ' ') { - std::string copy; - - copy += ' '; - copy += text; - tokenizer.tokenize(copy, output); - } else { - tokenizer.tokenize(text, output); - } - + tokenizer.tokenize(text, output); return output; }