ti.sil/pr-agent
1
0
Fork 0
mirror of https://github.com/qodo-ai/pr-agent.git synced 2025-12-11 18:35:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix tokenizer fallback to use o200k_base instead of cl100k_base

Browse source
This commit is contained in:
mrT23 2025-04-14 21:15:19 +03:00
parent 4b58a5488f
commit 08bf9593b2
No known key found for this signature in database
GPG key ID: D350490E39D5F5AD
1 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
pr_agent/algo/token_handler.py
View file

@ -19,8 +19,11 @@ class TokenEncoder:
with cls._lock: # Lock acquisition to ensure thread safety
if cls._encoder_instance is None or model != cls._model:
cls._model = model
cls._encoder_instance = encoding_for_model(cls._model) if "gpt" in cls._model else get_encoding(
"cl100k_base")
try:
cls._encoder_instance = encoding_for_model(cls._model) if "gpt" in cls._model else get_encoding(
"o200k_base")
except:
cls._encoder_instance = get_encoding("o200k_base")
return cls._encoder_instance