Upload 3 files

claude_analysis.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import json
+from pathlib import Path
+
+try:
+    import anthropic
+except ImportError:
+    print("\nERROR: anthropic library is not installed")
+    print("To install, run: pip install anthropic")
+    print("or: python -m pip install anthropic\n")
+    print(json.dumps({"error": "Missing anthropic library. Please run: pip install anthropic"}))
+    sys.exit(1)
+
+# Проверяем версию anthropic
+try:
+    anthropic_version = anthropic.__version__
+    print(f"\nDEBUG - Using anthropic version: {anthropic_version}")
+except:
+    print("\nWARNING - Could not determine anthropic version")
+
+api_key = os.environ.get("ANTHROPIC_API_KEY", "")
+# api_key = "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"
+if not api_key:
+    print(json.dumps({"error": "No ANTHROPIC_API_KEY provided"}))
+    sys.exit(0)
+
+def get_txt_contents(local_path=None):
+    """
+    Получает содержимое .txt файлов либо из локальной папки, либо из аргументов командной строки
+    """
+    txt_contents = []
+    
+    # Пробуем использовать локальную папку
+    if local_path:
+        try:
+            path = Path(local_path)
+            if path.exists():
+                print(f"\nDEBUG - Reading files from local path: {path}")
+                for txt_file in path.glob("*.txt"):
+                    try:
+                        with open(txt_file, 'r', encoding='utf-8') as f:
+                            txt_contents.append(f.read())
+                            print(f"DEBUG - Read file: {txt_file.name}")
+                    except Exception as e:
+                        print(f"DEBUG - Error reading {txt_file}: {e}")
+                return txt_contents
+        except Exception as e:
+            print(f"\nDEBUG - Error accessing local path: {e}")
+    
+    # Если локальная папка недоступна, используем аргументы командной строки
+    txt_file_list = sys.argv[1:]
+    if not txt_file_list:
+        return txt_contents
+        
+    for f in txt_file_list:
+        try:
+            with open(f, 'r', encoding='utf-8') as ff:
+                txt_contents.append(ff.read())
+        except Exception as e:
+            print(f"DEBUG - Error reading {f}: {e}")
+            
+    return txt_contents
+
+# -----------------------------------------------------------------------------
+# 1. Собираем список файлов
+# -----------------------------------------------------------------------------
+local_path = r"G:\My Drive\Kohya_SS\Flux\SoloBand\IconsGray"
+txt_contents = get_txt_contents(local_path)
+
+if not txt_contents:
+    print("\nDEBUG - No text files found in local path or arguments")
+
+combined_text = "\n---\n".join(txt_contents)
+
+# -----------------------------------------------------------------------------
+# 2. Формируем промпт для Claude
+# -----------------------------------------------------------------------------
+prompt_content = f"""You are a highly creative AI art director. Your task is to generate an imaginative and specific JSON response. Never use "Unknown" values - instead, create interesting and unique artistic directions even without input files.
+
+You MUST ALWAYS return ONLY a valid JSON object in this exact format, with no additional text:
+
+{{
+  "token": "Choose exactly one: FD_AI (Fantasy & Dragons), SB_AI (Storybook), CM_AI (Comic), AN_AI (Anime), RT_AI (Realistic)",
+  "art_type": "Create a specific description like: 'Mystical Forest Portal', 'Neon Cyberpunk City', 'Cozy Cottage Interior', 'Ancient Dragon Lair', 'Futuristic Space Station'",
+  "style_name": "Define clear style like: 'Vibrant watercolor fantasy', 'Detailed pencil illustration', 'Dark gothic oil painting', 'Cute kawaii digital art', 'Cinematic 3D render'",
+  "model_name": "Will be auto-generated as {{token}}_{{art_type}}_V1",
+  "prompts": [
+    "Generate 6 detailed prompts that perfectly match the art_type and style_name",
+    "Each prompt should be unique and vivid",
+    "Include specific details about composition, lighting, mood, colors",
+    "Keep each prompt to 1-2 impactful sentences",
+    "Focus on visual elements rather than story",
+    "Maintain consistent style across all prompts"
+  ]
+}}
+
+Even with no input files, you must generate a creative and complete response using the available tokens and styles. Never return "Unknown" values.
+
+Input content to analyze (if any):
+---
+{combined_text}
+"""
+
+# Добавляем отладочный вывод промпта
+print("\nDEBUG - Full prompt being sent to Claude:")
+print("="*80)
+print(prompt_content)
+print("="*80)
+
+# -----------------------------------------------------------------------------
+# 3. Обращаемся к Anthropic (Claude) с указанной моделью
+# -----------------------------------------------------------------------------
+try:
+    client = anthropic.Anthropic(api_key=api_key)
+    message = client.messages.create(
+        model="claude-3-sonnet-20240229",
+        max_tokens=1024,
+        messages=[
+            {
+                "role": "user",
+                "content": prompt_content
+            }
+        ],
+        temperature=0.7,
+    )
+    raw_reply = message.content[0].text
+except Exception as e:
+    print(json.dumps({"error": f"Request to Claude failed: {str(e)}"}))
+    sys.exit(0)
+
+# -----------------------------------------------------------------------------
+# 4. Пытаемся интерпретировать ответ как JSON
+# -----------------------------------------------------------------------------
+try:
+    data = json.loads(raw_reply)
+except:
+    # Если парсить напрямую не получается, отправим ошибку
+    print(json.dumps({"error": "Claude response is not valid JSON", "raw_reply": raw_reply}))
+    sys.exit(0)
+
+# -----------------------------------------------------------------------------
+# 5. (опционально) Проверяем, что в ответе есть нужные поля
+# -----------------------------------------------------------------------------
+token = data.get("token", "SB_AI")
+art_type = data.get("art_type", "UnknownArtType")
+style_name = data.get("style_name", "UnknownStyle")
+model_name = data.get("model_name", f"{token}_{art_type}_V1")
+prompts = data.get("prompts", [])
+
+# -----------------------------------------------------------------------------
+# 6. Выводим финальный JSON в stdout
+# -----------------------------------------------------------------------------
+out = {
+  "token": token,
+  "art_type": art_type,
+  "style_name": style_name,
+  "model_name": model_name,
+  "prompts": prompts
+}
+print(json.dumps(out, ensure_ascii=False))

finetune_analysis.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+set -e
+
+echo ">>> [Script 2] This script will handle Claude analysis and config update."
+
+###############################################################################
+# 1. Ожидание нажатия Enter и получение дополнительных комментариев
+###############################################################################
+echo ">>> [Script 2] Нажмите Enter, чтобы продолжить распаковку и анализ .txt файлов ..."
+read
+
+echo ">>> [Script 2] Введите дополнительные комментарии по поводу арта (или нажмите Enter, чтобы пропустить):"
+read USER_COMMENTS
+
+###############################################################################
+# 2. Распаковываем все .zip в /workspace/MyLearningDataset/Images
+###############################################################################
+echo ">>> [Script 2] Unzipping all .zip in /workspace/MyLearningDataset/Images ..."
+find /workspace/MyLearningDataset/Images -type f -name '*.zip' | while read zipf; do
+    unzip -o "$zipf" -d /workspace/MyLearningDataset/Images
+    rm -f "$zipf"
+done
+
+###############################################################################
+# 3. Собираем до 35 .txt файлов
+###############################################################################
+echo ">>> [Script 2] Collecting up to 35 .txt files ..."
+TXT_FILES=$(find /workspace/MyLearningDataset/Images -type f -name '*.txt' | head -n 35)
+
+if [ -z "$TXT_FILES" ]; then
+  echo "[Script 2] WARNING: No .txt files found (up to 35)."
+fi
+
+###############################################################################
+# 4. Устанавливаем библиотеку anthropic (если не установлена)
+#    И указываем ANTHROPIC_API_KEY
+###############################################################################
+echo ">>> [Script 2] Installing anthropic library (if needed) ..."
+pip install anthropic
+
+# Подставьте ваш реальный ключ, если нужно
+ANTHROPIC_API_KEY="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"
+export ANTHROPIC_API_KEY
+
+###############################################################################
+# 5. Запускаем Python-скрипт claude_analysis.py, передавая список файлов
+###############################################################################
+echo ">>> [Script 2] Sending request to Claude via claude_analysis.py ..."
+PARSED_JSON=$(python /workspace/claude_analysis.py "$TXT_FILES" "$USER_COMMENTS" 2>/dev/null || true)
+
+if [ -z "$PARSED_JSON" ]; then
+  echo "[Script 2] ERROR: Claude response is empty or not found."
+  exit 1
+fi
+
+# Проверка на ошибку (на всякий случай)
+if [[ "$PARSED_JSON" == *"error"* ]]; then
+  echo "[Script 2] ERROR: JSON parse problem. See logs."
+  echo "$PARSED_JSON"
+  exit 1
+fi
+
+###############################################################################
+# 6. Из полученного JSON извлекаем нужные поля (token, art_type, style_name, model_name, prompts)
+###############################################################################
+# Можно извлекать с помощью Python, jq или других инструментов. Для универсальности:
+TOKEN=$(echo "$PARSED_JSON"      | python -c 'import sys, json; d=json.load(sys.stdin); print(d["token"])')
+ART_TYPE=$(echo "$PARSED_JSON"   | python -c 'import sys, json; d=json.load(sys.stdin); print(d["art_type"])')
+STYLE_NAME=$(echo "$PARSED_JSON" | python -c 'import sys, json; d=json.load(sys.stdin); print(d["style_name"])')
+MODEL_NAME=$(echo "$PARSED_JSON" | python -c 'import sys, json; d=json.load(sys.stdin); print(d["model_name"])')
+
+# Многострочные промпты (каждый элемент массива -> перенос строки)
+PROMPTS=$(echo "$PARSED_JSON"    | python -c '
+import sys, json
+d=json.load(sys.stdin)
+prompts = d.get("prompts", [])
+print("\n".join(prompts))
+')
+
+echo ">>> [Script 2] Claude parsed result:"
+echo "token      = $TOKEN"
+echo "art_type   = $ART_TYPE"
+echo "style_name = $STYLE_NAME"
+echo "model_name = $MODEL_NAME"
+echo "prompts:"
+echo "$PROMPTS"
+echo
+
+###############################################################################
+# 7. Переносим .txt и .png файлы в папку /workspace/MyLearningDataset/Images/1_{model_name}_{style_name}
+###############################################################################
+NEW_FOLDER="/workspace/MyLearningDataset/Images/1_${MODEL_NAME}_${STYLE_NAME}"
+mkdir -p "$NEW_FOLDER"
+
+echo ">>> [Script 2] Moving all .txt and .png files into $NEW_FOLDER ..."
+find /workspace/MyLearningDataset/Images -type f \( -name '*.txt' -o -name '*.png' \) -exec mv -f {} "$NEW_FOLDER" \; 2>/dev/null || true
+
+###############################################################################
+# 8. Ищем FluxDatasetConfig.json и редактируем нужные поля
+###############################################################################
+FLUX_CONFIG_PATH=$(find /workspace -name "FluxDatasetConfig.json" | head -n 1)
+if [ -z "$FLUX_CONFIG_PATH" ]; then
+  echo "[Script 2] ERROR: FluxDatasetConfig.json not found!"
+  exit 1
+fi
+
+echo ">>> [Script 2] Updating FluxDatasetConfig.json at $FLUX_CONFIG_PATH ..."
+cat <<EOF > /workspace/update_flux_config.py
+import json
+
+path = r"${FLUX_CONFIG_PATH}"
+with open(path, "r", encoding="utf-8") as f:
+    config = json.load(f)
+
+config["train_data_dir"] = "/workspace/MyLearningDataset/Images"
+config["output_dir"] = "/workspace/MyLearningDataset/Models"
+config["output_name"] = "${MODEL_NAME}"
+config["huggingface_repo_id"] = "Gerchegg/${MODEL_NAME}"
+config["logging_dir"] = "/workspace/MyLearningDataset/Logs"
+
+# Многострочные промпты
+config["sample_prompts"] = """${PROMPTS}"""
+
+with open(path, "w", encoding="utf-8") as f:
+    json.dump(config, f, ensure_ascii=False, indent=2)
+EOF
+
+python /workspace/update_flux_config.py
+
+echo ">>> [Script 2] Done. Analysis complete!"

finetune_pipeline.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -e
+
+###############################################################################
+# 0. Подготовка среды: обновление apt, установка unzip
+###############################################################################
+echo ">>> [Script 1] Step 0: Installing system dependencies ..."
+apt-get update && apt-get install -y unzip
+
+###############################################################################
+# 1. Скачиваем zip-файл с Hugging Face и распаковываем
+###############################################################################
+echo ">>> [Script 1] Step 1: Downloading MyRunpodFinetuneScripts.zip ..."
+wget -O /workspace/MyRunpodFinetuneScripts.zip \
+  "https://huggingface.co/Gerchegg/FeaturesFluxAI/resolve/main/MyRunpodFinetuneScripts.zip"
+
+echo ">>> [Script 1] Step 2: Unzipping MyRunpodFinetuneScripts.zip ..."
+unzip -o /workspace/MyRunpodFinetuneScripts.zip -d /workspace
+# Предполагается, что внутри появится папка Kohya_Flux*** с нужными файлами.
+
+###############################################################################
+# 2. Запускаем два "параллельных" потока (фона):
+#   - Thread 1: Устанавливает пакеты + python Download_Train_Models.py
+#   - Thread 2: Устанавливает нужные deps (kohya_ss, torch, xformers), запускает GUI
+###############################################################################
+echo ">>> [Script 1] Starting parallel threads ..."
+
+# ------------------ Поток 1 --------------------
+(
+  echo ">>> [Thread 1] Installing Python packages for huggingface ..."
+  pip install huggingface_hub ipywidgets hf_transfer
+
+  export HF_HUB_ENABLE_HF_TRANSFER=1
+
+  echo ">>> [Thread 1] Running Download_Train_Models.py ..."
+  dwn_path=$(find /workspace -name "Download_Train_Models.py" | head -n 1)
+  if [ -z "$dwn_path" ]; then
+    echo "[Thread 1] ERROR: Download_Train_Models.py not found!"
+    exit 1
+  fi
+
+  python "$dwn_path" --dir /workspace
+
+  echo ">>> [Thread 1] Done."
+) &
+
+# ------------------ Поток 2 --------------------
+(
+  echo ">>> [Thread 2] Installing Python and OS dependencies ..."
+  apt update --yes
+  yes | apt-get install python3.10-tk
+  apt-get install psmisc --yes
+
+  echo ">>> [Thread 2] Cloning kohya_ss ..."
+  cd /workspace
+  if [ ! -d "/workspace/kohya_ss" ]; then
+    git clone https://github.com/bmaltais/kohya_ss.git
+  fi
+
+  cd /workspace/kohya_ss
+  git checkout sd3-flux.1
+
+  echo ">>> [Thread 2] Creating Python venv ..."
+  python3 -m venv venv
+  source venv/bin/activate
+  yes | apt-get install python3.10-tk
+
+  echo ">>> [Thread 2] Running setup.sh ..."
+  ./setup.sh -n -u
+
+  apt update --yes
+  yes | apt-get install python3.10-tk
+  apt-get install psmisc --yes
+
+  pip install hf_transfer
+  export HF_HUB_ENABLE_HF_TRANSFER=0
+
+  echo ">>> [Thread 2] Killing processes on port 7860 ..."
+  fuser -k 7860/tcp || true
+
+  git checkout sd3-flux.1
+  source venv/bin/activate
+
+  echo ">>> [Thread 2] Updating torch & xformers ..."
+  pip uninstall -y xformers
+  pip install torch==2.5.1+cu124 torchvision --index-url https://download.pytorch.org/whl/cu124
+  pip install xformers==0.0.28.post3 --index-url https://download.pytorch.org/whl/cu124
+
+  echo ">>> [Thread 2] Launching kohya_ss GUI on 0.0.0.0:7860 ..."
+  ./gui.sh --listen=0.0.0.0 --share --noverify
+) &
+
+###############################################################################
+# 3. Создаём общую структуру папок (Models, Images, Logs)
+#    по заданию, делаем это в первой части
+###############################################################################
+echo ">>> [Script 1] Creating folder structure in /workspace/MyLearningDataset ..."
+mkdir -p /workspace/MyLearningDataset/Models
+mkdir -p /workspace/MyLearningDataset/Images
+mkdir -p /workspace/MyLearningDataset/Logs
+
+###############################################################################
+# 4. Ожидаем завершения потоков
+###############################################################################
+echo ">>> [Script 1] Waiting for threads to finish ..."
+wait
+echo ">>> [Script 1] ALL DONE! Now run 'finetune_analysis.sh' to proceed."