src/send_to_hub.py

from huggingface_hub import HfFileSystem
import os
import json
from dotenv import load_dotenv
load_dotenv()
from datetime import datetime


assert os.getenv("HF_TOKEN") is not None, "HF_TOKEN environment variable is not set"

import logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')


fs = HfFileSystem(token=os.getenv("HF_TOKEN"))

def send_message_buckt(messages, messages_uuid, model_info, REPO_ID = "CEIA-POSITIVO2/smol_chat" ):
    
    save_at = f"datasets/{REPO_ID}/{messages_uuid}.json"

    data = {
            'messages': messages,
            'date' : datetime.now().strftime("%d/%m/%y %H:%M:%S"),
            'model_info': model_info
            }
    
    #logging.info(f'[Messages ] {data}')
    
    with open(f'{messages_uuid}.json', 'w') as f:
        json.dump(data, f)
        
    fs.upload(f'{messages_uuid}.json', save_at)