Skip to content

Commit

Permalink
Fix rope scaling defaults (ggml-org#767)
Browse files Browse the repository at this point in the history
* Fix rope scale with backwards compatibility

* Fix defaults

* Fix op

* Remove backwards compatibility

* Check single val
  • Loading branch information
Josh-XT authored Sep 29, 2023
1 parent a72efc7 commit a945404
Showing 1 changed file with 8 additions and 7 deletions.
15 changes: 8 additions & 7 deletions llama_cpp/llama.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,8 +229,8 @@ def __init__(
n_batch: int = 512,
n_threads: Optional[int] = None,
n_threads_batch: Optional[int] = None,
rope_freq_base: float = 10000.0,
rope_freq_scale: float = 1.0,
rope_freq_base: float = 0.0,
rope_freq_scale: float = 0.0,
mul_mat_q: bool = True,
f16_kv: bool = True,
logits_all: bool = False,
Expand Down Expand Up @@ -282,7 +282,6 @@ def __init__(
Returns:
A Llama instance.
"""

self.verbose = verbose

self.numa = numa
Expand Down Expand Up @@ -320,16 +319,19 @@ def __init__(
self.n_threads_batch = n_threads_batch or max(
multiprocessing.cpu_count() // 2, 1
)

# Context Params
self.context_params = llama_cpp.llama_context_default_params()
self.context_params.seed = seed
self.context_params.n_ctx = n_ctx
self.context_params.n_batch = self.n_batch
self.context_params.n_threads = self.n_threads
self.context_params.n_threads_batch = self.n_threads_batch
self.context_params.rope_freq_base = rope_freq_base
self.context_params.rope_freq_scale = rope_freq_scale
self.context_params.rope_freq_base = (
rope_freq_base if rope_freq_base != 0.0 else 0
)
self.context_params.rope_freq_scale = (
rope_freq_scale if rope_freq_scale != 0.0 else 0
)
self.context_params.mul_mat_q = mul_mat_q
self.context_params.f16_kv = f16_kv
self.context_params.logits_all = logits_all
Expand All @@ -338,7 +340,6 @@ def __init__(
# Sampling Params
self.last_n_tokens_size = last_n_tokens_size


self.cache: Optional[BaseLlamaCache] = None

self.lora_base = lora_base
Expand Down

0 comments on commit a945404

Please sign in to comment.