| |
| |
| @@ -10,7 +10,9 @@ |
| self.token_count: int = 0 |
| |
| def _recount_tokens(self) -> None: |
| - self.token_count = sum(len(self.tokenizer.encode(m["content"])) for m in self.buffer if m["role"]=="user") |
| + self.token_count = sum(len(self.tokenizer.encode(m["content"])) |
| + for m in (self.buffer or []) |
| + if m and m.get("role") == "user") |
| |
| def add_messages(self, messages: List[Dict], segmenter, text_embedder) -> None: |
| all_segments = [] |
| @@ -47,7 +49,8 @@ |
| 2. Fine-grained adjustment based on semantic similarity. |
| """ |
| segments = [] |
| - buffer_texts = [m["content"] for m in self.buffer if m["role"] == "user"] |
| + buffer_texts = [m["content"] for m in (self.buffer or []) |
| + if m and m.get("role") == "user"] |
| boundaries = segmenter.propose_cut(buffer_texts) |
| |
| if not boundaries: |
| |
| |
| @@ -16,7 +16,8 @@ |
| } |
| |
| allowed_roles = role_map.get(messages_use, []) |
| - text_list = [msg["content"] for msg in messages if msg["role"] in allowed_roles] |
| + text_list = [msg["content"] for msg in (messages or []) |
| + if msg and msg.get("role") in allowed_roles] |
| |
| text = " ".join(text_list) |
| |
| |
| |
| @@ -337,7 +337,9 @@ |
| self.logger.info(f"[{call_id}] Extraction triggered {extract_trigger_num} times, extract_list length: {len(extract_list)}") |
| extract_list, timestamps_list, weekday_list, speaker_list, topic_id_map = assign_sequence_numbers_with_timestamps(extract_list, offset_ms=500, topic_id_mapping=topic_id_mapping) |
| self.logger.debug(f"[{call_id}] Extract list sample: {json.dumps(extract_list)}") |
| - max_source_ids = [sum(1 for seg in batch for msg in seg if msg.get("role") == "user") - 1 for batch in extract_list] |
| + max_source_ids = [sum(1 for seg in batch for msg in (seg or []) |
| + if msg and msg.get("role") == "user") - 1 |
| + for batch in extract_list] |
| self.logger.info(f"[{call_id}] Batch max_source_ids: {max_source_ids}") |
| if self.config.metadata_generate and self.config.text_summary: |
| self.logger.info(f"[{call_id}] Starting metadata generation") |
| |
| |
| @@ -69,7 +69,9 @@ |
| |
| for segments in extract_list: |
| for seg in segments: |
| - for message in seg: |
| + for message in (seg or []): |
| + if not message: |
| + continue |
| session_time = message.get('session_time', '') |
| message_refs.append((message, session_time)) |
| |
| @@ -110,7 +112,9 @@ |
| |
| for segments in extract_list: |
| for seg in segments: |
| - for message in seg: |
| + for message in (seg or []): |
| + if not message: |
| + continue |
| message["sequence_number"] = current_index |
| timestamps_list.append(message["time_stamp"]) |
| weekday_list.append(message["weekday"]) |
| @@ -126,7 +130,9 @@ |
| for api_idx, api_call_segments in enumerate(extract_list): |
| for topic_idx, topic_segment in enumerate(api_call_segments): |
| tid = topic_id_mapping[api_idx][topic_idx] |
| - for msg in topic_segment: |
| + for msg in (topic_segment or []): |
| + if not msg: |
| + continue |
| seq = msg.get("sequence_number") |
| sequence_to_topic[seq] = tid |
| |
|
|
