Spaces:
Running
Running
Upload 2 files
Browse files- util/streaming_parser.py +4 -3
util/streaming_parser.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import json
|
| 2 |
from typing import Iterator, Dict, Any, Iterable, AsyncIterator
|
|
|
|
| 3 |
|
| 4 |
def parse_json_array_stream(line_iterator: Iterable[str]) -> Iterator[Dict[str, Any]]:
|
| 5 |
"""
|
|
@@ -30,13 +31,13 @@ def parse_json_array_stream(line_iterator: Iterable[str]) -> Iterator[Dict[str,
|
|
| 30 |
stripped_line = line.strip()
|
| 31 |
if not stripped_line:
|
| 32 |
continue
|
| 33 |
-
|
| 34 |
if stripped_line.startswith('['):
|
| 35 |
in_array = True
|
| 36 |
# 去掉起始的 '[' 字符,剩下的部分继续处理
|
| 37 |
line = stripped_line[1:]
|
| 38 |
-
#
|
| 39 |
-
line_iterator =
|
| 40 |
break
|
| 41 |
|
| 42 |
if not in_array:
|
|
|
|
| 1 |
import json
|
| 2 |
from typing import Iterator, Dict, Any, Iterable, AsyncIterator
|
| 3 |
+
from itertools import chain
|
| 4 |
|
| 5 |
def parse_json_array_stream(line_iterator: Iterable[str]) -> Iterator[Dict[str, Any]]:
|
| 6 |
"""
|
|
|
|
| 31 |
stripped_line = line.strip()
|
| 32 |
if not stripped_line:
|
| 33 |
continue
|
| 34 |
+
|
| 35 |
if stripped_line.startswith('['):
|
| 36 |
in_array = True
|
| 37 |
# 去掉起始的 '[' 字符,剩下的部分继续处理
|
| 38 |
line = stripped_line[1:]
|
| 39 |
+
# 使用 chain 连接剩余行,避免转换为列表(内存优化)
|
| 40 |
+
line_iterator = chain([line], line_iterator)
|
| 41 |
break
|
| 42 |
|
| 43 |
if not in_array:
|