scripts/data_versioning.py

import pickle, os, hashlib
CACHE_DIR = "data/rag_stores.pkl"
def get_rag_stores(build_func, force_rebuild=False):
    if not force_rebuild and os.path.exists(CACHE_DIR):
        with open(CACHE_DIR, "rb") as f:
            return pickle.load(f)
    stores = build_func()
    with open(CACHE_DIR, "wb") as f:
        pickle.dump(stores, f)
    return stores
def get_data_version():
    if os.path.exists(CACHE_DIR):
        with open(CACHE_DIR, "rb") as f:
            return hashlib.md5(f.read()).hexdigest()[:8]
    return "unversioned"