File size: 1,420 Bytes
6eed659 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | import glob,json,os,collections,soundfile as sf,torch
from transformers import AutoModelForAudioClassification, AutoFeatureExtractor
M="/root/models/lid_best"
fe=AutoFeatureExtractor.from_pretrained(M)
m=AutoModelForAudioClassification.from_pretrained(M,dtype=torch.bfloat16).cuda().eval()
i2l=m.config.id2label
files=sorted(glob.glob("/scratch/p2_16k/*.wav"))
gpu={}; conf={}
with torch.inference_mode():
for i in range(0,len(files),8):
b=files[i:i+8]
au=[sf.read(f,dtype="float32")[0][:16000*20] for f in b]
x=fe(au,sampling_rate=16000,return_tensors="pt",padding=True)
x={k:v.to("cuda",dtype=torch.bfloat16 if v.dtype==torch.float32 else v.dtype) for k,v in x.items()}
pr=m(**x).logits.float().softmax(-1)
for f,p in zip(b,pr):
k=int(p.argmax()); lab=i2l[k] if k in i2l else i2l[str(k)]
if lab=="lug": lab="lin"
i2=os.path.splitext(os.path.basename(f))[0]
gpu[i2]=lab; conf[i2]=float(p.max())
lex=json.load(open("/root/test_lang.json"))
print("GPU LID :",dict(collections.Counter(gpu.values())))
print("lexical :",dict(collections.Counter(lex.values())))
dis=[i for i in gpu if gpu[i]!=lex.get(i)]
print("DESACCORDS :",len(dis))
for i in dis: print(" ",i,"gpu=",gpu[i],"(conf %.3f)"%conf[i],"lex=",lex.get(i))
json.dump({"gpu":gpu,"conf":conf,"disagree":dis},open("/root/lid_test.json","w"))
print("LID_TEST_DONE")
|