encodingGen.py

import requests
import json

with open("embeddingData.json", "r") as f:
    data = json.loads(f.read())
    
for i in range(0,len(data),10):
    newData = []
    for j in range(i,i+10):
        try:
            newData.append(data[j]["text"])
        except:
            pass

    url = "https://api.deepinfra.com/v1/inference/BAAI/bge-large-en-v1.5"

    payload = json.dumps({
        "inputs": newData
    })
    headers = {
        'Accept': 'application/json, text/plain, */*',
        'Accept-Language': 'en-US,en;q=0.9,gu;q=0.8,ru;q=0.7,hi;q=0.6',
        'Connection': 'keep-alive',
        'Content-Type': 'application/json',
        'Origin': 'https://deepinfra.com',
        'Referer': 'https://deepinfra.com/',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-site',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
        'sec-ch-ua': '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"'
    }

    response = requests.request("POST", url, headers=headers, data=payload)
    for j in range(len(response.json()["embeddings"])):
        data[i+j]["embedding"] = response.json()["embeddings"][j]
        print(data[i+j]["text"])

with open("embeddingData.json", "w") as f:
    f.write(json.dumps(data, indent=4))