ti.sil/pr-agent
1
0
Fork 0
mirror of https://github.com/qodo-ai/pr-agent.git synced 2025-12-12 19:05:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Protect for empty description

Browse source
This commit is contained in:
Ori Kotek 2023-08-10 02:08:36 +03:00
parent e00500b90c
commit b206b1c5ff
No known key found for this signature in database
GPG key ID: 5AAFD0757D1EAC4C
1 changed files with 13 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
pr_agent/algo/pr_processing.py
View file

@ -298,12 +298,16 @@ def clip_tokens(text: str, max_tokens: int) -> str:
str: The clipped string.
"""
# We'll estimate the number of tokens by hueristically assuming 2.5 tokens per word
encoder = get_token_encoder()
num_input_tokens = len(encoder.encode(text))
if num_input_tokens <= max_tokens:
return text
num_chars = len(text)
chars_per_token = num_chars / num_input_tokens
num_output_chars = int(chars_per_token * max_tokens)
clipped_text = text[:num_output_chars]
return clipped_text
try:
encoder = get_token_encoder()
num_input_tokens = len(encoder.encode(text))
if num_input_tokens <= max_tokens:
return text
num_chars = len(text)
chars_per_token = num_chars / num_input_tokens
num_output_chars = int(chars_per_token * max_tokens)
clipped_text = text[:num_output_chars]
return clipped_text
except Exception as e:
logging.warning(f"Failed to clip tokens: {e}")
return text