File size: 3,553 Bytes
83ddd7e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | from __future__ import annotations
import sys, os, json, time
os.environ['CC'] = '/opt/rh/gcc-toolset-9/root/usr/bin/gcc'
os.environ['CXX'] = '/opt/rh/gcc-toolset-9/root/usr/bin/g++'
os.environ["PATH"] += os.pathsep + os.path.expanduser("~/.local/bin")
os.environ["PATH"] += os.pathsep + "/rhome/jyang311/shared/jyangfile/jyangfile/trainingmaterial/exllamav2"
os.environ["MAX_JOBS"] = '16'
from exllamav2 import ExLlamaV2, ExLlamaV2Config, ExLlamaV2Cache, ExLlamaV2Tokenizer
from exllamav2.generator import ExLlamaV2DynamicGenerator, ExLlamaV2DynamicJob, ExLlamaV2Sampler
import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm
import os
import math
from tqdm import tqdm
import json
def calculate_perplexity(model, tokenizer, input_ids) -> torch.Tensor:
# Tokenize the input text
target_ids = input_ids[:, 1:]
# Compute loss
with torch.no_grad():
logits = model.forward(input_ids).float()
logprobs = F.log_softmax(logits, dim=-1).to('cuda')
target_ids = target_ids.to('cuda')
target_logprobs = logprobs[:, :-1].gather(dim=-1, index=target_ids.unsqueeze(-1)).squeeze(-1)
nll = -target_logprobs
# Perplexity is exp(loss)
perplexity = nll.exp()
return nll
def calculate_stdev(numbers):
if len(numbers) < 2:
raise ValueError("At least two numbers are required to calculate the standard deviation.")
mean = sum(numbers) / len(numbers)
variance = sum((x - mean) ** 2 for x in numbers) / (len(numbers) - 1)
stdev = math.sqrt(variance)
return stdev
import json
with open('clustered_embeded_full.json') as tmpinputCollector:
col = json.load(tmpinputCollector)
modelDir = "../Mistral-Large-Instruct-2407-123B-exl2/"
max_response_len = 1024
cache_size = 100*1024 # Adjust as needed, 100k seems to be a safe size for L3-8B on a single 24 GB GPU
max_rows = 10000
config = ExLlamaV2Config(modelDir)
tokenizer = ExLlamaV2Tokenizer(config)
model = ExLlamaV2(config)
cache = ExLlamaV2Cache(model, max_seq_len = cache_size, lazy = True)
model.load_autosplit(cache)
def getPpl(input_text):
input_ids = tokenizer.encode(input_text)
# Calculate perplexity
perplexity = calculate_perplexity(model, tokenizer, input_ids)
vocab = tokenizer.get_id_to_piece_list()
tokens = []
ppls = []
for idx in range(input_ids.shape[-1]):
token = input_ids[0, idx].item()
text = vocab[token]
ppl = float("inf") if idx == 0 else perplexity[0, idx - 1].item()
tokens.append(text)
ppls.append(ppl)
return tokens, ppls
with open('isolatedMiddle.json') as tmpinputCollector:
res = json.load(tmpinputCollector)
i=0
for sample in tqdm(res):
premise = sample['premise']
conclusion = sample['conclusion']
for middle in sample['middle4conclusionppl']:
if sample['middle4conclusionppl'][middle] !='':
continue
#print(middle)
finalInput = premise.strip()+' '+middle.strip()+' '+conclusion
#print(finalInput)
tokens, ppls = getPpl(finalInput)
lenofConclusion = len(tokenizer.encode(conclusion)[0])
#print(tokenizer.encode(conclusion)[0])
concPpl = ppls[-lenofConclusion:]
# print(sum(concPpl) / len(concPpl))
sample['middle4conclusionppl'][middle] = concPpl
# print(len(concPpl))
i+=1
if i%3000 != 0:
continue
with open("isolatedMiddle.json", "w") as outfile:
json.dump(res, outfile)
|