File size: 3,992 Bytes
750b88d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | --- a/LightMem/src/lightmem/factory/memory_buffer/sensory_memory.py
+++ b/LightMem/src/lightmem/factory/memory_buffer/sensory_memory.py
@@ -10,7 +10,9 @@
self.token_count: int = 0
def _recount_tokens(self) -> None:
- self.token_count = sum(len(self.tokenizer.encode(m["content"])) for m in self.buffer if m["role"]=="user")
+ self.token_count = sum(len(self.tokenizer.encode(m["content"]))
+ for m in (self.buffer or [])
+ if m and m.get("role") == "user")
def add_messages(self, messages: List[Dict], segmenter, text_embedder) -> None:
all_segments = []
@@ -47,7 +49,8 @@
2. Fine-grained adjustment based on semantic similarity.
"""
segments = []
- buffer_texts = [m["content"] for m in self.buffer if m["role"] == "user"]
+ buffer_texts = [m["content"] for m in (self.buffer or [])
+ if m and m.get("role") == "user"]
boundaries = segmenter.propose_cut(buffer_texts)
if not boundaries:
--- a/LightMem/src/lightmem/factory/memory_buffer/short_term_memory.py
+++ b/LightMem/src/lightmem/factory/memory_buffer/short_term_memory.py
@@ -16,7 +16,8 @@
}
allowed_roles = role_map.get(messages_use, [])
- text_list = [msg["content"] for msg in messages if msg["role"] in allowed_roles]
+ text_list = [msg["content"] for msg in (messages or [])
+ if msg and msg.get("role") in allowed_roles]
text = " ".join(text_list)
--- a/LightMem/src/lightmem/memory/lightmem.py
+++ b/LightMem/src/lightmem/memory/lightmem.py
@@ -337,7 +337,9 @@
self.logger.info(f"[{call_id}] Extraction triggered {extract_trigger_num} times, extract_list length: {len(extract_list)}")
extract_list, timestamps_list, weekday_list, speaker_list, topic_id_map = assign_sequence_numbers_with_timestamps(extract_list, offset_ms=500, topic_id_mapping=topic_id_mapping)
self.logger.debug(f"[{call_id}] Extract list sample: {json.dumps(extract_list)}")
- max_source_ids = [sum(1 for seg in batch for msg in seg if msg.get("role") == "user") - 1 for batch in extract_list]
+ max_source_ids = [sum(1 for seg in batch for msg in (seg or [])
+ if msg and msg.get("role") == "user") - 1
+ for batch in extract_list]
self.logger.info(f"[{call_id}] Batch max_source_ids: {max_source_ids}")
if self.config.metadata_generate and self.config.text_summary:
self.logger.info(f"[{call_id}] Starting metadata generation")
--- a/LightMem/src/lightmem/memory/utils.py
+++ b/LightMem/src/lightmem/memory/utils.py
@@ -69,7 +69,9 @@
for segments in extract_list:
for seg in segments:
- for message in seg:
+ for message in (seg or []):
+ if not message:
+ continue
session_time = message.get('session_time', '')
message_refs.append((message, session_time))
@@ -110,7 +112,9 @@
for segments in extract_list:
for seg in segments:
- for message in seg:
+ for message in (seg or []):
+ if not message:
+ continue
message["sequence_number"] = current_index
timestamps_list.append(message["time_stamp"])
weekday_list.append(message["weekday"])
@@ -126,7 +130,9 @@
for api_idx, api_call_segments in enumerate(extract_list):
for topic_idx, topic_segment in enumerate(api_call_segments):
tid = topic_id_mapping[api_idx][topic_idx]
- for msg in topic_segment:
+ for msg in (topic_segment or []):
+ if not msg:
+ continue
seq = msg.get("sequence_number")
sequence_to_topic[seq] = tid
|