File size: 627 Bytes
7c9f50a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | import sys
import pandas
import json
if len(sys.argv) < 2:
print("请提供文件路径作为参数")
sys.exit(1)
samples = []
file_path = sys.argv[1]
try:
with open(file_path, 'r', encoding='utf-8') as f:
fin = json.load(f)
except Exception:
fin = []
with open(file_path, 'r', encoding='utf-8') as f:
fin = [json.loads(d) for d in f.readlines()]
assert isinstance(fin, list)
for jdict in fin:
instruct = jdict["instruction"]
input = jdict["input"]
output = jdict["output"]
samples.append(instruct+input+output)
pd = pandas.Series(samples).map(len)
print(pd.describe())
|