eyad-silx commited on
Commit
c7534e3
·
verified ·
1 Parent(s): 0452e22

Upload podflow/filter_eval.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. podflow/filter_eval.py +17 -0
podflow/filter_eval.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import sys, json
2
+ from pathlib import Path
3
+ sys.path.insert(0, "/root/ingot/src")
4
+ from eval_service.run import load_tokenizer, _render
5
+ tok = load_tokenizer(Path("/root/models/teacher"))
6
+ rows = [json.loads(l) for l in open("/root/evalset.jsonl")]
7
+ keep = []
8
+ for r in rows:
9
+ try:
10
+ p = _render(tok, r["messages"], r.get("tools"))
11
+ except Exception:
12
+ continue
13
+ if len(p) <= 3000 and len(p) >= 8:
14
+ keep.append(r)
15
+ Path("/root/evalset.jsonl").write_text(
16
+ "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in keep))
17
+ print("kept", len(keep), "of", len(rows))