RuntimeError when using generate.json() on llama 3.2 with llamaccp #1261
Labels
Comments
|
I noticed that condaforge was still on version 0.0.46. |
|
Can you try manually setting the tokenizer like they do here? import llama_cpp
from outlines import generate, models
model = models.llamacpp(
"bartowski/Llama-3.2-1B-Instruct-GGUF",
"Llama-3.2-1B-Instruct-Q4_K_M.gguf",
tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(
"meta-llama/Llama-3.2-1B"
),
) |
|
Thanks Since you need to use credentials to get things from the meta-llama repo |
|
Perfect. Closing for now, glad it worked! |
4 tasks
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Describe the issue as clearly as possible:
When trying to use
generate.jsonwithllama 3.2andllamaccpI get a runtime error:RuntimeError: Cannot convert token ` �` (30433) to bytes: �"Steps/code to reproduce the bug:
Expected result:
Error message:
{ "name": "RuntimeError", "message": "Cannot convert token ` �` (30433) to bytes: �", "stack": "--------------------------------------------------------------------------- RuntimeError Traceback (most recent call last) Cell In[102], line 9 7 class Sections(BaseModel): 8 sections: str ----> 9 generator = generate.json(model, Sections) File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/functools.py:907, in singledispatch.<locals>.wrapper(*args, **kw) 903 if not args: 904 raise TypeError(f'{funcname} requires at least ' 905 '1 positional argument') --> 907 return dispatch(args[0].__class__)(*args, **kw) File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/site-packages/outlines/generate/json.py:49, in json(model, schema_object, sampler, whitespace_pattern) 47 schema = pyjson.dumps(schema_object.model_json_schema()) 48 regex_str = build_regex_from_schema(schema, whitespace_pattern) ---> 49 generator = regex(model, regex_str, sampler) 50 generator.format_sequence = lambda x: schema_object.parse_raw(x) 51 elif callable(schema_object): File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/functools.py:907, in singledispatch.<locals>.wrapper(*args, **kw) 903 if not args: 904 raise TypeError(f'{funcname} requires at least ' 905 '1 positional argument') --> 907 return dispatch(args[0].__class__)(*args, **kw) File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/site-packages/outlines/generate/regex.py:61, in regex_llamacpp(model, regex_str, sampler) 53 @regex.register(LlamaCpp) 54 def regex_llamacpp( 55 model: LlamaCpp, 56 regex_str: str, 57 sampler: Sampler = multinomial(), 58 ): 59 from outlines.integrations.llamacpp import RegexLogitsProcessor ---> 61 logits_processor = RegexLogitsProcessor(regex_str, llm=model.model) 62 return SequenceGeneratorAdapter(model, logits_processor, sampler) File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/site-packages/outlines/integrations/llamacpp.py:129, in RegexLogitsProcessor.__init__(self, regex_string, llm) 119 \"\"\"Compile the FSM that drives the regex-guided generation. 120 121 Parameters (...) 126 The Llama model. 127 \"\"\" 128 tokenizer = LlamaCppTokenizer(model=llm) --> 129 fsm = RegexGuide(regex_string, tokenizer) 130 super().__init__(tokenizer=tokenizer, fsm=fsm) File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/site-packages/outlines/fsm/guide.py:145, in RegexGuide.__init__(self, regex_string, tokenizer) 140 def __init__(self, regex_string: str, tokenizer): 141 ( 142 self.states_to_token_maps, 143 self.empty_token_ids, 144 fsm_finals, --> 145 ) = create_states_mapping(regex_string, tokenizer) 146 self.eos_token_id = tokenizer.eos_token_id 147 self.final_states = fsm_finals | {-1} File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/site-packages/outlines/caching.py:122, in cache.<locals>.decorator.<locals>.wrapper(*args, **kwargs) 119 result = wrapper.__memory__.get(cache_key, default=ENOVAL, retry=True) 121 if result is ENOVAL: --> 122 result = cached_function(*args, **kwargs) 123 wrapper.__memory__.set(cache_key, result, expire, retry=True) 125 return result File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/site-packages/outlines/fsm/guide.py:118, in create_states_mapping(regex_string, tokenizer) 116 byte_fsm = make_byte_level_fsm(regex_pattern.to_fsm().reduce(), keep_utf8=True) 117 regex_fsm, _ = make_deterministic_fsm(byte_fsm) --> 118 states_to_token_maps, empty_token_ids = create_fsm_index_tokenizer( 119 regex_fsm, tokenizer 120 ) 122 # We make sure that it is possible to generate strings in the language 123 # of the regular expression with the tokens present in the model's 124 # vocabulary. 125 if not any( 126 regex_fsm.finals.intersection(v.values()) for v in states_to_token_maps.values() 127 ): File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/site-packages/outlines/fsm/regex.py:898, in create_fsm_index_tokenizer(fsm, tokenizer) 885 def create_fsm_index_tokenizer( 886 fsm: BetterFSM, 887 tokenizer: \"Tokenizer\", 888 ) -> Tuple[Dict[int, Dict[int, int]], Set[int]]: 889 \"\"\"Construct an FMS index from a tokenizer. 890 891 This uses the end-to-end approach of `create_fsm_index_end_to_end`. (...) 896 897 \"\"\" --> 898 vocabulary, empty_token_ids = reduced_vocabulary(tokenizer) 900 states_to_token_subsets = create_fsm_index_end_to_end(fsm.fsm_info, vocabulary) 902 # Allow transitions to EOS from all terminals FSM states that are 903 # reachable 904 # TODO: Do we really need this anymore? File ~/micromamba/envs/sanofi-immunogenicity-nlp-llm/lib/python3.12/site-packages/outlines/fsm/regex.py:861, in reduced_vocabulary(tokenizer) 857 token_bytes = cast( 858 List[int], [gpt2_unicode_to_bytes().get(c) for c in token] 859 ) 860 if None in token_bytes: --> 861 raise RuntimeError( 862 f\"Cannot convert token `{token}` ({token_idx}) to bytes: {token_str}\" 863 ) 864 token_str = \"\".join(byte_symbol(b) for b in token_bytes) 866 vocabulary.setdefault(token_str, []).append(token_idx) RuntimeError: Cannot convert token ` �` (30433) to bytes: �" }Outlines/Python version information:
Version information
0.0.46
Context for the issue:
No response
The text was updated successfully, but these errors were encountered: