gen.py

import hashlib
import os
import json

sha1s = {}


for root, dirs, files in os.walk("."):
    if ".git" in dirs:
        dirs.remove(".git")          
    for name in files:
        file_path = os.path.join(root, name)
        with open(file_path, "rb") as f:
            data = f.read()
            sha1 = hashlib.sha1(data).hexdigest()
            sha1s[file_path] = sha1


with open("dl.py", "w") as f:
    f.write("#!/usr/bin/env python\n\nimport os\nimport hashlib\n")
    for file_path, sha1 in sha1s.items():
        f.write(f"print('Downloading {file_path}')\n")
        f.write(f"if not os.path.exists('{file_path}') or '{sha1}' != hashlib.sha1(open('{file_path}', 'rb').read()).hexdigest():\n")
        f.write(f"    os.system('curl -L \"https://huggingface.co/AI-Porn/AI-PORN/resolve/main/{file_path}\" --create-dirs -o \"./{file_path}\"')\n")

file_list = {}
for root, dirs, files in os.walk("."):
    if ".git" in dirs:
        dirs.remove(".git")
    folder_name = os.path.basename(root)
    file_list[folder_name] = files
    file_list[folder_name].extend(dirs)

#with open("file_list.json", "w") as f:
 #   json.dump(file_list, f)