xiaoyukkkk commited on
Commit
f8b6b2a
·
verified ·
1 Parent(s): 40a5b01

Upload 2 files

Browse files
Files changed (1) hide show
  1. util/streaming_parser.py +4 -3
util/streaming_parser.py CHANGED
@@ -1,5 +1,6 @@
1
  import json
2
  from typing import Iterator, Dict, Any, Iterable, AsyncIterator
 
3
 
4
  def parse_json_array_stream(line_iterator: Iterable[str]) -> Iterator[Dict[str, Any]]:
5
  """
@@ -30,13 +31,13 @@ def parse_json_array_stream(line_iterator: Iterable[str]) -> Iterator[Dict[str,
30
  stripped_line = line.strip()
31
  if not stripped_line:
32
  continue
33
-
34
  if stripped_line.startswith('['):
35
  in_array = True
36
  # 去掉起始的 '[' 字符,剩下的部分继续处理
37
  line = stripped_line[1:]
38
- # 将剩余部分和后续的迭代器重新组合成一个新的迭代器
39
- line_iterator = iter(list([line]) + list(line_iterator))
40
  break
41
 
42
  if not in_array:
 
1
  import json
2
  from typing import Iterator, Dict, Any, Iterable, AsyncIterator
3
+ from itertools import chain
4
 
5
  def parse_json_array_stream(line_iterator: Iterable[str]) -> Iterator[Dict[str, Any]]:
6
  """
 
31
  stripped_line = line.strip()
32
  if not stripped_line:
33
  continue
34
+
35
  if stripped_line.startswith('['):
36
  in_array = True
37
  # 去掉起始的 '[' 字符,剩下的部分继续处理
38
  line = stripped_line[1:]
39
+ # 使用 chain 连接剩余行,避免转换为列表(内存优化)
40
+ line_iterator = chain([line], line_iterator)
41
  break
42
 
43
  if not in_array: