Spaces:
Paused
Paused
File size: 2,472 Bytes
8d1819a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
# from . import files
import json
def truncate_text(agent, output, threshold=1000):
threshold = int(threshold)
if not threshold or len(output) <= threshold:
return output
# Adjust the file path as needed
placeholder = agent.read_prompt(
"fw.msg_truncated.md", length=(len(output) - threshold)
)
# placeholder = files.read_file("./prompts/default/fw.msg_truncated.md", length=(len(output) - threshold))
start_len = (threshold - len(placeholder)) // 2
end_len = threshold - len(placeholder) - start_len
truncated_output = output[:start_len] + placeholder + output[-end_len:]
return truncated_output
def truncate_dict_by_ratio(agent, data: dict|list|str, threshold_chars: int, truncate_to: int):
threshold_chars = int(threshold_chars)
truncate_to = int(truncate_to)
def process_item(item):
if isinstance(item, dict):
truncated_dict = {}
cumulative_size = 0
for key, value in item.items():
processed_value = process_item(value)
serialized_value = json.dumps(processed_value, ensure_ascii=False)
size = len(serialized_value)
if cumulative_size + size > threshold_chars:
truncated_dict[key] = truncate_text(
agent, serialized_value, truncate_to
)
else:
cumulative_size += size
truncated_dict[key] = processed_value
return truncated_dict
elif isinstance(item, list):
truncated_list = []
cumulative_size = 0
for value in item:
processed_value = process_item(value)
serialized_value = json.dumps(processed_value, ensure_ascii=False)
size = len(serialized_value)
if cumulative_size + size > threshold_chars:
truncated_list.append(
truncate_text(agent, serialized_value, truncate_to)
)
else:
cumulative_size += size
truncated_list.append(processed_value)
return truncated_list
elif isinstance(item, str):
if len(item) > threshold_chars:
return truncate_text(agent, item, truncate_to)
return item
else:
return item
return process_item(data)
|