Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .dockerignore +9 -0
- .env +3 -0
- .gitattributes +5 -0
- Dockerfile +26 -0
- anticheat_hashes.json +1 -0
- app.py +116 -0
- labels.txt +10 -0
- mission_model.h5 +3 -0
- outputs/confusion_matrix.png +0 -0
- outputs/debug_bias_mitigation.jpg +0 -0
- requirements.txt +10 -0
- runtime.txt +1 -0
- scripts/data_prep/balance_dataset.py +85 -0
- scripts/data_prep/collect_data.py +221 -0
- scripts/data_prep/count_dataset.py +43 -0
- scripts/data_prep/fix_dataset.py +28 -0
- scripts/data_prep/organize_dataset.py +91 -0
- scripts/testing/split_dataset.py +73 -0
- scripts/testing/test_duplicate.py +16 -0
- scripts/testing/test_predictor.py +27 -0
- scripts/testing/test_server.py +0 -0
- scripts/testing/test_upload.html +61 -0
- scripts/training/download_pangasinan.py +46 -0
- scripts/training/evaluate_model.py +57 -0
- scripts/training/train_ai.py +209 -0
- scripts/training/train_ai_v2.py +170 -0
- utils/__init__.py +1 -0
- utils/__pycache__/__init__.cpython-311.pyc +0 -0
- utils/__pycache__/anticheat.cpython-311.pyc +0 -0
- utils/__pycache__/predictor.cpython-311.pyc +0 -0
- utils/__pycache__/verdict.cpython-311.pyc +0 -0
- utils/anticheat.py +98 -0
- utils/predictor.py +109 -0
- utils/verdict.py +42 -0
- venv/.gitignore +2 -0
- venv/Lib/site-packages/__pycache__/pylab.cpython-311.pyc +0 -0
- venv/Lib/site-packages/__pycache__/six.cpython-311.pyc +0 -0
- venv/Lib/site-packages/__pycache__/threadpoolctl.cpython-311.pyc +0 -0
- venv/Lib/site-packages/__pycache__/typing_extensions.cpython-311.pyc +3 -0
- venv/Lib/site-packages/_distutils_hack/__init__.py +222 -0
- venv/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-311.pyc +0 -0
- venv/Lib/site-packages/_distutils_hack/__pycache__/override.cpython-311.pyc +0 -0
- venv/Lib/site-packages/_distutils_hack/override.py +1 -0
- venv/Lib/site-packages/_multiprocess/__init__.py +8 -0
- venv/Lib/site-packages/_multiprocess/__pycache__/__init__.cpython-311.pyc +0 -0
- venv/Lib/site-packages/_yaml/__init__.py +33 -0
- venv/Lib/site-packages/_yaml/__pycache__/__init__.cpython-311.pyc +0 -0
- venv/Lib/site-packages/absl/__init__.py +15 -0
- venv/Lib/site-packages/absl/__pycache__/__init__.cpython-311.pyc +0 -0
- venv/Lib/site-packages/absl/__pycache__/app.cpython-311.pyc +0 -0
.dockerignore
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Ignore datasets and heavy files to keep deployment fast
|
| 2 |
+
dataset/
|
| 3 |
+
__pycache__/
|
| 4 |
+
*.pyc
|
| 5 |
+
.env
|
| 6 |
+
.git
|
| 7 |
+
.vscode
|
| 8 |
+
.gemini
|
| 9 |
+
scripts/testing/mission_dataset_split.zip
|
.env
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
OLLAMA_URL="http://localhost:11434"
|
| 2 |
+
OLLAMA_API_KEY="87b344ea09c540848abd777349d64466.PeZFbKB2Y03ddFyD4YXW0TpT"
|
| 3 |
+
OLLAMA_MODEL="llava-llama3"
|
.gitattributes
CHANGED
|
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
venv/images/imagehash.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
venv/Lib/site-packages/__pycache__/typing_extensions.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
venv/Lib/site-packages/absl/testing/__pycache__/absltest.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
venv/Lib/site-packages/aiohttp/_http_parser.cp311-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
venv/Lib/site-packages/aiohttp/_websocket/reader_c.cp311-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
Dockerfile
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Use official Python runtime as a parent image
|
| 2 |
+
FROM python:3.11-slim
|
| 3 |
+
|
| 4 |
+
# Set working directory in the container
|
| 5 |
+
WORKDIR /app
|
| 6 |
+
|
| 7 |
+
# Install system dependencies (needed for OpenCV/Pillow if required)
|
| 8 |
+
RUN apt-get update && apt-get install -y \
|
| 9 |
+
libgl1-mesa-glx \
|
| 10 |
+
libglib2.0-0 \
|
| 11 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 12 |
+
|
| 13 |
+
# Copy requirements file first (for caching)
|
| 14 |
+
COPY requirements.txt .
|
| 15 |
+
|
| 16 |
+
# Install dependencies
|
| 17 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 18 |
+
|
| 19 |
+
# Copy the rest of the application code
|
| 20 |
+
COPY . .
|
| 21 |
+
|
| 22 |
+
# Expose port 7860 for Hugging Face Spaces
|
| 23 |
+
EXPOSE 7860
|
| 24 |
+
|
| 25 |
+
# Run the application using gunicorn for production
|
| 26 |
+
CMD ["gunicorn", "--bind", "0.0.0.0:7860", "app:app"]
|
anticheat_hashes.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"hashes": ["46256446866b53d9", "e0e42531b264e6d6", "e24a3c7a839cef24", "ccdac541d91b9237", "dcdd9c43634f189a", "b3ccd03f4cb11758", "f48b6995d4cf02b1", "525bf524bcb65692", "198964c9cb682cae", "eff64b09e12f8680"]}
|
app.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import traceback
|
| 3 |
+
import logging
|
| 4 |
+
from flask import Flask, request, jsonify
|
| 5 |
+
from flask_cors import CORS
|
| 6 |
+
from werkzeug.utils import secure_filename
|
| 7 |
+
from dotenv import load_dotenv
|
| 8 |
+
|
| 9 |
+
from utils.anticheat import AntiCheatEngine
|
| 10 |
+
from utils.predictor import Predictor
|
| 11 |
+
from utils.verdict import get_verdict
|
| 12 |
+
|
| 13 |
+
load_dotenv()
|
| 14 |
+
|
| 15 |
+
# Setup logging
|
| 16 |
+
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
# Initialize components
|
| 20 |
+
app = Flask(__name__)
|
| 21 |
+
CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True)
|
| 22 |
+
|
| 23 |
+
ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'webp'}
|
| 24 |
+
app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 # Limit upload to 100MB
|
| 25 |
+
|
| 26 |
+
logger.info("🧠 Loading the MISSION 17 AI Brain (Ollama Vision)...")
|
| 27 |
+
anticheat = AntiCheatEngine()
|
| 28 |
+
predictor = Predictor()
|
| 29 |
+
|
| 30 |
+
def allowed_file(filename):
|
| 31 |
+
return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
|
| 32 |
+
|
| 33 |
+
@app.after_request
|
| 34 |
+
def after_request(response):
|
| 35 |
+
response.headers.add('Access-Control-Allow-Origin', '*')
|
| 36 |
+
response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
|
| 37 |
+
response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
|
| 38 |
+
return response
|
| 39 |
+
|
| 40 |
+
@app.route('/health', methods=['GET'])
|
| 41 |
+
def health():
|
| 42 |
+
return jsonify({
|
| 43 |
+
"status": "ok",
|
| 44 |
+
"model": predictor.get_model_name(),
|
| 45 |
+
"anticheat_hashes": anticheat.count()
|
| 46 |
+
}), 200
|
| 47 |
+
|
| 48 |
+
@app.route('/reset-anti-cheat', methods=['POST', 'GET'])
|
| 49 |
+
def reset_anti_cheat():
|
| 50 |
+
count = anticheat.clear()
|
| 51 |
+
logger.info("🛡️ Anti-cheat hash database cleared!")
|
| 52 |
+
return jsonify({"message": "Anti-cheat hash database cleared!", "count": count}), 200
|
| 53 |
+
|
| 54 |
+
@app.route('/predict', methods=['POST'])
|
| 55 |
+
def predict():
|
| 56 |
+
try:
|
| 57 |
+
# 🔒 CHECK 1: File Presence
|
| 58 |
+
if 'file' not in request.files:
|
| 59 |
+
return jsonify({'error': 'No file uploaded'}), 400
|
| 60 |
+
|
| 61 |
+
file = request.files['file']
|
| 62 |
+
|
| 63 |
+
# 🔒 CHECK 2: Empty File Detection (Bug Fix)
|
| 64 |
+
file.seek(0, os.SEEK_END)
|
| 65 |
+
if file.tell() == 0:
|
| 66 |
+
return jsonify({"error": "Processing failed: Empty file"}), 400
|
| 67 |
+
file.seek(0)
|
| 68 |
+
|
| 69 |
+
# 🔒 CHECK 3: Empty Filename
|
| 70 |
+
if file.filename == '':
|
| 71 |
+
return jsonify({'error': 'No selected file'}), 400
|
| 72 |
+
|
| 73 |
+
# 🔒 CHECK 4: File Type Validation
|
| 74 |
+
if not allowed_file(file.filename):
|
| 75 |
+
return jsonify({'error': 'Invalid file type. Only JPG/PNG allowed.'}), 400
|
| 76 |
+
|
| 77 |
+
# Read file bytes ONCE and reuse them
|
| 78 |
+
file_bytes = file.read()
|
| 79 |
+
|
| 80 |
+
# 🎯 MODULE 11: Calculate Hash and Check for Cheaters
|
| 81 |
+
if anticheat.is_duplicate(file_bytes):
|
| 82 |
+
logger.warning("🚨 ANTI-CHEAT: Duplicate image detected!")
|
| 83 |
+
return jsonify({
|
| 84 |
+
"status": "REJECTED",
|
| 85 |
+
"error": "Duplicate image detected. You cannot farm points!",
|
| 86 |
+
"prediction": "Anti-Cheat: Duplicate"
|
| 87 |
+
}), 400
|
| 88 |
+
|
| 89 |
+
# 🤖 AI Vision Prediction via Ollama
|
| 90 |
+
logger.info("🤖 Sending image to Ollama Vision...")
|
| 91 |
+
ai_result = predictor.predict(file_bytes)
|
| 92 |
+
|
| 93 |
+
category = ai_result.get('category', 'Non_SDG_Invalid')
|
| 94 |
+
confidence = ai_result.get('confidence', 0)
|
| 95 |
+
reason = ai_result.get('reason', '')
|
| 96 |
+
|
| 97 |
+
# ⚖️ Get formatted verdict based on AI output
|
| 98 |
+
verdict_response = get_verdict(category, confidence, threshold=55)
|
| 99 |
+
verdict_response['reason'] = reason
|
| 100 |
+
verdict_response['model'] = predictor.get_model_name()
|
| 101 |
+
|
| 102 |
+
# Only register hash if the image was VERIFIED (save memory/prevent false positives on bad images)
|
| 103 |
+
if verdict_response['is_verified']:
|
| 104 |
+
anticheat.register(file_bytes)
|
| 105 |
+
logger.info(f"✅ Unique verified image logged to anticheat.")
|
| 106 |
+
|
| 107 |
+
return jsonify(verdict_response)
|
| 108 |
+
|
| 109 |
+
except Exception as e:
|
| 110 |
+
logger.error(f"❌ Processing Error: {str(e)}")
|
| 111 |
+
traceback.print_exc()
|
| 112 |
+
return jsonify({'error': "Processing failed", 'detail': str(e)}), 500
|
| 113 |
+
|
| 114 |
+
if __name__ == '__main__':
|
| 115 |
+
# Hugging Face requires the app to listen on 0.0.0.0:7860
|
| 116 |
+
app.run(host='0.0.0.0', port=7860, debug=False)
|
labels.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Non_SDG_Invalid
|
| 2 |
+
SDG11_Sustainable_Cities
|
| 3 |
+
SDG12_Recycling
|
| 4 |
+
SDG13_15_Planting
|
| 5 |
+
SDG1_2_Donation
|
| 6 |
+
SDG3_Health_Wellbeing
|
| 7 |
+
SDG4_Quality_Education
|
| 8 |
+
SDG6_14_Cleanup
|
| 9 |
+
SDG7_Clean_Energy
|
| 10 |
+
SDG8_Support_Local
|
mission_model.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fb0e4373bc7f9a3d1a9e9d7cacbc1686942d39e2c1debdf2c1b67e1767ef28bc
|
| 3 |
+
size 20967176
|
outputs/confusion_matrix.png
ADDED
|
outputs/debug_bias_mitigation.jpg
ADDED
|
requirements.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Flask==2.2.2
|
| 2 |
+
Flask-Cors==3.0.10
|
| 3 |
+
numpy
|
| 4 |
+
Pillow
|
| 5 |
+
imagehash
|
| 6 |
+
gunicorn
|
| 7 |
+
tensorflow
|
| 8 |
+
scikit-learn
|
| 9 |
+
matplotlib
|
| 10 |
+
seaborn
|
runtime.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3.11.9
|
scripts/data_prep/balance_dataset.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
import random
|
| 4 |
+
|
| 5 |
+
# 👇 CONFIGURATION
|
| 6 |
+
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 7 |
+
DATASET_DIR = os.path.join(CURRENT_DIR, '..', '..', '..', 'dataset', 'mission_dataset')
|
| 8 |
+
|
| 9 |
+
# Target: balance ALL classes to this count
|
| 10 |
+
# Set to None to auto-detect the median (safe default)
|
| 11 |
+
TARGET_COUNT = 500
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def count_classes(dataset_dir):
|
| 15 |
+
counts = {}
|
| 16 |
+
for class_name in os.listdir(dataset_dir):
|
| 17 |
+
class_path = os.path.join(dataset_dir, class_name)
|
| 18 |
+
if os.path.isdir(class_path):
|
| 19 |
+
images = [
|
| 20 |
+
f for f in os.listdir(class_path)
|
| 21 |
+
if f.lower().endswith(('.jpg', '.jpeg', '.png', '.webp'))
|
| 22 |
+
]
|
| 23 |
+
counts[class_name] = len(images)
|
| 24 |
+
return counts
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def balance_dataset():
|
| 28 |
+
print(f"⚖️ Balancing ALL classes in: {DATASET_DIR}\n")
|
| 29 |
+
|
| 30 |
+
if not os.path.exists(DATASET_DIR):
|
| 31 |
+
print("❌ Error: Dataset folder not found.")
|
| 32 |
+
return
|
| 33 |
+
|
| 34 |
+
counts = count_classes(DATASET_DIR)
|
| 35 |
+
|
| 36 |
+
if not counts:
|
| 37 |
+
print("❌ No class folders found.")
|
| 38 |
+
return
|
| 39 |
+
|
| 40 |
+
# Determine target
|
| 41 |
+
target = TARGET_COUNT
|
| 42 |
+
if target is None:
|
| 43 |
+
sorted_counts = sorted(counts.values())
|
| 44 |
+
target = sorted_counts[len(sorted_counts) // 2] # median
|
| 45 |
+
print(f"📊 Auto-detected target (median): {target} images per class\n")
|
| 46 |
+
else:
|
| 47 |
+
print(f"📊 Target: {target} images per class\n")
|
| 48 |
+
|
| 49 |
+
print(f"{'Class':<35} {'Before':>8} {'After':>8} {'Action'}")
|
| 50 |
+
print("-" * 65)
|
| 51 |
+
|
| 52 |
+
for class_name, count in sorted(counts.items()):
|
| 53 |
+
class_path = os.path.join(DATASET_DIR, class_name)
|
| 54 |
+
images = [
|
| 55 |
+
f for f in os.listdir(class_path)
|
| 56 |
+
if f.lower().endswith(('.jpg', '.jpeg', '.png', '.webp'))
|
| 57 |
+
]
|
| 58 |
+
|
| 59 |
+
if count > target:
|
| 60 |
+
# Trim to target — shuffle first to keep a random selection
|
| 61 |
+
random.shuffle(images)
|
| 62 |
+
excess = images[target:]
|
| 63 |
+
for img in excess:
|
| 64 |
+
os.remove(os.path.join(class_path, img))
|
| 65 |
+
after = target
|
| 66 |
+
action = f"✂️ Trimmed -{len(excess)}"
|
| 67 |
+
|
| 68 |
+
elif count < target:
|
| 69 |
+
# Class is under-represented — warn user to add more images
|
| 70 |
+
after = count
|
| 71 |
+
action = f"⚠️ Under-represented (need +{target - count} more images)"
|
| 72 |
+
|
| 73 |
+
else:
|
| 74 |
+
after = count
|
| 75 |
+
action = "✅ OK"
|
| 76 |
+
|
| 77 |
+
print(f"{class_name:<35} {count:>8} {after:>8} {action}")
|
| 78 |
+
|
| 79 |
+
print("\n✨ Balancing complete! Now run train_ai.py to retrain the model.")
|
| 80 |
+
print(" 💡 TIP: For under-represented classes, collect more real photos")
|
| 81 |
+
print(" or use Google Images to download additional training data.")
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
if __name__ == "__main__":
|
| 85 |
+
balance_dataset()
|
scripts/data_prep/collect_data.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from bing_image_downloader import downloader
|
| 2 |
+
import os
|
| 3 |
+
import shutil
|
| 4 |
+
import math
|
| 5 |
+
|
| 6 |
+
# 👇 CONFIGURATION
|
| 7 |
+
BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', 'dataset', 'mission_dataset')
|
| 8 |
+
|
| 9 |
+
# Target images per class — everything will be brought up to this number
|
| 10 |
+
TARGET = 500
|
| 11 |
+
|
| 12 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 13 |
+
# CLASS DEFINITIONS
|
| 14 |
+
# Each class has:
|
| 15 |
+
# "current": how many images it already has
|
| 16 |
+
# "terms": list of 5 search terms to download from
|
| 17 |
+
#
|
| 18 |
+
# limit_per_term = ceil((TARGET - current) / len(terms))
|
| 19 |
+
# Classes already at TARGET are automatically skipped.
|
| 20 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 21 |
+
CLASSES = {
|
| 22 |
+
|
| 23 |
+
# 🍱 SDG 1/2: Donation — WEAKEST CLASS (150 → 500, needs +350, 70/term)
|
| 24 |
+
"SDG1_2_Donation": {
|
| 25 |
+
"current": 150,
|
| 26 |
+
"terms": [
|
| 27 |
+
"people donating food to community",
|
| 28 |
+
"clothes donation box charity",
|
| 29 |
+
"feeding program volunteers serving food",
|
| 30 |
+
"grocery donation drive event",
|
| 31 |
+
"charity relief goods distribution"
|
| 32 |
+
]
|
| 33 |
+
},
|
| 34 |
+
|
| 35 |
+
# 🏖️ SDG 6/14: Cleanup — WEAKEST CLASS (150 → 500, needs +350, 70/term)
|
| 36 |
+
"SDG6_14_Cleanup": {
|
| 37 |
+
"current": 150,
|
| 38 |
+
"terms": [
|
| 39 |
+
"beach cleanup volunteers collecting trash",
|
| 40 |
+
"river cleanup community activity",
|
| 41 |
+
"coastal cleanup garbage bags collected",
|
| 42 |
+
"people picking up litter shoreline",
|
| 43 |
+
"estero creek waterway cleanup"
|
| 44 |
+
]
|
| 45 |
+
},
|
| 46 |
+
|
| 47 |
+
# 📚 SDG 4: Education (253 → 500, needs +247, 50/term)
|
| 48 |
+
"SDG4_Quality_Education": {
|
| 49 |
+
"current": 253,
|
| 50 |
+
"terms": [
|
| 51 |
+
"student reading open book",
|
| 52 |
+
"teacher writing on whiteboard classroom",
|
| 53 |
+
"group study session library",
|
| 54 |
+
"hand writing notes in notebook",
|
| 55 |
+
"child using educational tablet learning"
|
| 56 |
+
]
|
| 57 |
+
},
|
| 58 |
+
|
| 59 |
+
# 🏃 SDG 3: Health (262 → 500, needs +238, 48/term)
|
| 60 |
+
"SDG3_Health_Wellbeing": {
|
| 61 |
+
"current": 262,
|
| 62 |
+
"terms": [
|
| 63 |
+
"people jogging in park",
|
| 64 |
+
"group yoga session outdoors",
|
| 65 |
+
"eating fresh fruit salad bowl",
|
| 66 |
+
"drinking glass of water healthy",
|
| 67 |
+
"washing hands with soap hygiene"
|
| 68 |
+
]
|
| 69 |
+
},
|
| 70 |
+
|
| 71 |
+
# 🛍️ SDG 8: Support Local (267 → 500, needs +233, 47/term)
|
| 72 |
+
"SDG8_Support_Local": {
|
| 73 |
+
"current": 267,
|
| 74 |
+
"terms": [
|
| 75 |
+
"buying from street food vendor",
|
| 76 |
+
"shopping at local farmers market",
|
| 77 |
+
"artisan crafting handmade goods",
|
| 78 |
+
"small bakery local shop front",
|
| 79 |
+
"supporting small business community"
|
| 80 |
+
]
|
| 81 |
+
},
|
| 82 |
+
|
| 83 |
+
# 🌱 SDG 13/15: Planting (270 → 500, needs +230, 46/term)
|
| 84 |
+
"SDG13_15_Planting": {
|
| 85 |
+
"current": 270,
|
| 86 |
+
"terms": [
|
| 87 |
+
"person planting tree sapling",
|
| 88 |
+
"community tree planting activity",
|
| 89 |
+
"garden seedling transplanting soil",
|
| 90 |
+
"plant growing hands holding soil",
|
| 91 |
+
"reforestation volunteers planting trees"
|
| 92 |
+
]
|
| 93 |
+
},
|
| 94 |
+
|
| 95 |
+
# 🏙️ SDG 11: Sustainable Cities (275 → 500, needs +225, 45/term)
|
| 96 |
+
"SDG11_Sustainable_Cities": {
|
| 97 |
+
"current": 275,
|
| 98 |
+
"terms": [
|
| 99 |
+
"riding bicycle on city road",
|
| 100 |
+
"passengers inside public city bus",
|
| 101 |
+
"waiting at train station platform",
|
| 102 |
+
"walking on pedestrian crossing street",
|
| 103 |
+
"segregated bike lane urban city"
|
| 104 |
+
]
|
| 105 |
+
},
|
| 106 |
+
|
| 107 |
+
# ⚡ SDG 7: Clean Energy (277 → 500, needs +223, 45/term)
|
| 108 |
+
"SDG7_Clean_Energy": {
|
| 109 |
+
"current": 277,
|
| 110 |
+
"terms": [
|
| 111 |
+
"solar panels on house roof",
|
| 112 |
+
"hand turning off light switch",
|
| 113 |
+
"electric vehicle charging station",
|
| 114 |
+
"wind turbine farm landscape",
|
| 115 |
+
"modern led light bulb energy saving"
|
| 116 |
+
]
|
| 117 |
+
},
|
| 118 |
+
|
| 119 |
+
# 🚫 Non-SDG Invalid (430 → 500, needs +70, 14/term)
|
| 120 |
+
"Non_SDG_Invalid": {
|
| 121 |
+
"current": 430,
|
| 122 |
+
"terms": [
|
| 123 |
+
"random indoor selfie photo",
|
| 124 |
+
"luxury sports car fast",
|
| 125 |
+
"video game screenshot gaming",
|
| 126 |
+
"cat sleeping on sofa",
|
| 127 |
+
"abstract digital art wallpaper"
|
| 128 |
+
]
|
| 129 |
+
},
|
| 130 |
+
|
| 131 |
+
# ♻️ SDG 12: Recycling — ALREADY AT TARGET (500), will be skipped
|
| 132 |
+
"SDG12_Recycling": {
|
| 133 |
+
"current": 500,
|
| 134 |
+
"terms": []
|
| 135 |
+
},
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 139 |
+
|
| 140 |
+
print(f"🚀 Smart Data Collection — Target: {TARGET} images per class")
|
| 141 |
+
print(f" Dataset path: {BASE_DIR}\n")
|
| 142 |
+
|
| 143 |
+
if not os.path.exists(BASE_DIR):
|
| 144 |
+
print(f"❌ ERROR: Could not find '{BASE_DIR}'. Check your folder structure.")
|
| 145 |
+
exit()
|
| 146 |
+
|
| 147 |
+
total_added = 0
|
| 148 |
+
|
| 149 |
+
for category, info in CLASSES.items():
|
| 150 |
+
current = info["current"]
|
| 151 |
+
terms = info["terms"]
|
| 152 |
+
needed = TARGET - current
|
| 153 |
+
|
| 154 |
+
# Skip classes already at or above target
|
| 155 |
+
if needed <= 0:
|
| 156 |
+
print(f"⏭️ [{category}] already at {current}/{TARGET} — SKIPPED\n")
|
| 157 |
+
continue
|
| 158 |
+
|
| 159 |
+
limit_per_term = math.ceil(needed / len(terms))
|
| 160 |
+
target_dir = os.path.join(BASE_DIR, category)
|
| 161 |
+
os.makedirs(target_dir, exist_ok=True)
|
| 162 |
+
|
| 163 |
+
print(f"📂 [{category}]")
|
| 164 |
+
print(f" {current} → {TARGET} | need +{needed} | {limit_per_term} images/term")
|
| 165 |
+
|
| 166 |
+
category_added = 0
|
| 167 |
+
|
| 168 |
+
for term in terms:
|
| 169 |
+
print(f" 🔍 '{term}' ({limit_per_term} images)...")
|
| 170 |
+
try:
|
| 171 |
+
downloader.download(
|
| 172 |
+
term,
|
| 173 |
+
limit=limit_per_term,
|
| 174 |
+
output_dir="temp_downloads",
|
| 175 |
+
adult_filter_off=True,
|
| 176 |
+
force_replace=False,
|
| 177 |
+
timeout=10,
|
| 178 |
+
verbose=False
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
source_folder = os.path.join("temp_downloads", term)
|
| 182 |
+
if os.path.exists(source_folder):
|
| 183 |
+
files = os.listdir(source_folder)
|
| 184 |
+
moved = 0
|
| 185 |
+
for file in files:
|
| 186 |
+
old_path = os.path.join(source_folder, file)
|
| 187 |
+
if not os.path.isfile(old_path):
|
| 188 |
+
continue
|
| 189 |
+
clean_term = term.replace(" ", "_")
|
| 190 |
+
new_filename = f"{clean_term}_{file}"
|
| 191 |
+
new_path = os.path.join(target_dir, new_filename)
|
| 192 |
+
if os.path.exists(new_path):
|
| 193 |
+
continue # Skip duplicates
|
| 194 |
+
try:
|
| 195 |
+
shutil.move(old_path, new_path)
|
| 196 |
+
moved += 1
|
| 197 |
+
except Exception:
|
| 198 |
+
pass
|
| 199 |
+
print(f" ✅ +{moved} images")
|
| 200 |
+
category_added += moved
|
| 201 |
+
|
| 202 |
+
except Exception as e:
|
| 203 |
+
print(f" ⚠️ Skipped '{term}': {e}")
|
| 204 |
+
|
| 205 |
+
# Clean up temp after each term
|
| 206 |
+
if os.path.exists("temp_downloads"):
|
| 207 |
+
try:
|
| 208 |
+
shutil.rmtree("temp_downloads")
|
| 209 |
+
except Exception:
|
| 210 |
+
pass
|
| 211 |
+
|
| 212 |
+
new_total = current + category_added
|
| 213 |
+
print(f" 📊 Result: {current} → {new_total} images (+{category_added})\n")
|
| 214 |
+
total_added += category_added
|
| 215 |
+
|
| 216 |
+
print("=" * 55)
|
| 217 |
+
print(f"✨ Done! Total new images added: {total_added}")
|
| 218 |
+
print(f" All classes should now be near {TARGET} images each.")
|
| 219 |
+
print("\n Next steps:")
|
| 220 |
+
print(" 1. python train_ai.py ← retrain the model")
|
| 221 |
+
print(" 2. python evaluate_model.py ← check accuracy")
|
scripts/data_prep/count_dataset.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
def count_images():
|
| 4 |
+
# Define the path to the dataset
|
| 5 |
+
# Based on your other scripts, it is in ../dataset/mission_dataset
|
| 6 |
+
base_dir = os.path.dirname(os.path.abspath(__file__))
|
| 7 |
+
dataset_dir = os.path.join(base_dir, '..', '..', '..', 'dataset', 'mission_dataset')
|
| 8 |
+
|
| 9 |
+
print(f"📊 Checking dataset at: {os.path.abspath(dataset_dir)}\n")
|
| 10 |
+
|
| 11 |
+
if not os.path.exists(dataset_dir):
|
| 12 |
+
print(f"❌ Error: Folder not found. Have you run 'organize_dataset.py'?")
|
| 13 |
+
return
|
| 14 |
+
|
| 15 |
+
total_images = 0
|
| 16 |
+
|
| 17 |
+
# Get all subfolders (classes)
|
| 18 |
+
try:
|
| 19 |
+
classes = [d for d in os.listdir(dataset_dir) if os.path.isdir(os.path.join(dataset_dir, d))]
|
| 20 |
+
classes.sort()
|
| 21 |
+
except Exception as e:
|
| 22 |
+
print(f"❌ Error reading directory: {e}")
|
| 23 |
+
return
|
| 24 |
+
|
| 25 |
+
print(f"{'CLASS NAME':<35} | {'COUNT':<10} | {'STATUS'}")
|
| 26 |
+
print("-" * 50)
|
| 27 |
+
|
| 28 |
+
for class_name in classes:
|
| 29 |
+
class_path = os.path.join(dataset_dir, class_name)
|
| 30 |
+
# Count files that look like images
|
| 31 |
+
images = [f for f in os.listdir(class_path) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp', '.bmp'))]
|
| 32 |
+
count = len(images)
|
| 33 |
+
|
| 34 |
+
status = "✅ Ready" if count >= 100 else "⚠️ Low Data" if count > 0 else "❌ Empty"
|
| 35 |
+
|
| 36 |
+
print(f"{class_name:<35} | {count:<10} | {status}")
|
| 37 |
+
total_images += count
|
| 38 |
+
|
| 39 |
+
print("-" * 50)
|
| 40 |
+
print(f"✅ TOTAL IMAGES: {total_images}")
|
| 41 |
+
|
| 42 |
+
if __name__ == "__main__":
|
| 43 |
+
count_images()
|
scripts/data_prep/fix_dataset.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
|
| 4 |
+
# Define paths
|
| 5 |
+
import os
|
| 6 |
+
base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', 'dataset', 'garbage_classification')
|
| 7 |
+
old_planting = os.path.join(base_dir, "planting")
|
| 8 |
+
new_planting = os.path.join(base_dir, "SDG13_15_Planting")
|
| 9 |
+
|
| 10 |
+
# Create new folder if it doesn't exist
|
| 11 |
+
if not os.path.exists(new_planting):
|
| 12 |
+
os.makedirs(new_planting)
|
| 13 |
+
|
| 14 |
+
# Move files from Old -> New
|
| 15 |
+
if os.path.exists(old_planting):
|
| 16 |
+
print(f"🔄 Moving files from '{old_planting}' to '{new_planting}'...")
|
| 17 |
+
files = os.listdir(old_planting)
|
| 18 |
+
for file in files:
|
| 19 |
+
old_path = os.path.join(old_planting, file)
|
| 20 |
+
new_path = os.path.join(new_planting, f"old_{file}") # Rename to avoid conflicts
|
| 21 |
+
shutil.move(old_path, new_path)
|
| 22 |
+
|
| 23 |
+
# Delete the empty old folder
|
| 24 |
+
os.rmdir(old_planting)
|
| 25 |
+
print("✅ Successfully merged folders!")
|
| 26 |
+
print("🗑️ Deleted old 'planting' folder.")
|
| 27 |
+
else:
|
| 28 |
+
print("⚠️ Old 'planting' folder not found. Already merged?")
|
scripts/data_prep/organize_dataset.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
|
| 4 |
+
# 👇 CONFIGURATION
|
| 5 |
+
# Current script directory
|
| 6 |
+
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 7 |
+
# The main project dataset folder (../dataset)
|
| 8 |
+
BASE_DIR = os.path.join(CURRENT_DIR, '..', '..', '..', 'dataset')
|
| 9 |
+
|
| 10 |
+
# 1. The "Correct" Destination
|
| 11 |
+
FINAL_DEST = os.path.join(BASE_DIR, "mission_dataset")
|
| 12 |
+
|
| 13 |
+
# 2. The "Old" Kaggle Dataset
|
| 14 |
+
OLD_GARBAGE_DIR = os.path.join(BASE_DIR, "garbage_classification")
|
| 15 |
+
|
| 16 |
+
# 3. The "Misplaced" Downloads (if any) inside mission17-ai/dataset
|
| 17 |
+
MISPLACED_DIR = os.path.join(CURRENT_DIR, '..', '..', "dataset", "mission_dataset")
|
| 18 |
+
|
| 19 |
+
# Map OLD folders to NEW SDG destinations
|
| 20 |
+
# We are putting ALL waste items into SDG12 (Responsible Consumption & Production)
|
| 21 |
+
MOVES = {
|
| 22 |
+
"SDG12_Recycling": [
|
| 23 |
+
"battery", "brown-glass", "cardboard",
|
| 24 |
+
"clothes", "green-glass", "metal", "paper",
|
| 25 |
+
"plastic", "shoes", "white-glass"
|
| 26 |
+
],
|
| 27 |
+
"Non_SDG_Invalid": [
|
| 28 |
+
"trash", "biological"
|
| 29 |
+
]
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
def organize_files():
|
| 33 |
+
print(f"📦 Organizing dataset...")
|
| 34 |
+
|
| 35 |
+
# Ensure destination exists
|
| 36 |
+
if not os.path.exists(FINAL_DEST):
|
| 37 |
+
os.makedirs(FINAL_DEST)
|
| 38 |
+
print(f" ✅ Created '{FINAL_DEST}'")
|
| 39 |
+
|
| 40 |
+
# --- STEP 1: Merge Kaggle Data ---
|
| 41 |
+
if os.path.exists(OLD_GARBAGE_DIR):
|
| 42 |
+
print(f" 🔄 Merging 'garbage_classification'...")
|
| 43 |
+
for dest_folder, source_folders in MOVES.items():
|
| 44 |
+
dest_path = os.path.join(FINAL_DEST, dest_folder)
|
| 45 |
+
if not os.path.exists(dest_path): os.makedirs(dest_path)
|
| 46 |
+
|
| 47 |
+
for folder in source_folders:
|
| 48 |
+
src_path = os.path.join(OLD_GARBAGE_DIR, folder)
|
| 49 |
+
if os.path.exists(src_path):
|
| 50 |
+
# Move files
|
| 51 |
+
for file in os.listdir(src_path):
|
| 52 |
+
try:
|
| 53 |
+
shutil.move(os.path.join(src_path, file), os.path.join(dest_path, f"{folder}_{file}"))
|
| 54 |
+
except Exception: pass
|
| 55 |
+
# Remove empty folder
|
| 56 |
+
try:
|
| 57 |
+
os.rmdir(src_path)
|
| 58 |
+
except: pass
|
| 59 |
+
|
| 60 |
+
# Try to remove root garbage dir
|
| 61 |
+
try: os.rmdir(OLD_GARBAGE_DIR)
|
| 62 |
+
except: pass
|
| 63 |
+
print(" ✅ Kaggle data merged.")
|
| 64 |
+
|
| 65 |
+
# --- STEP 2: Fix Misplaced Downloads ---
|
| 66 |
+
if os.path.exists(MISPLACED_DIR):
|
| 67 |
+
print(f" ⚠️ Found misplaced images in '{MISPLACED_DIR}'. Moving them...")
|
| 68 |
+
for category in os.listdir(MISPLACED_DIR):
|
| 69 |
+
src = os.path.join(MISPLACED_DIR, category)
|
| 70 |
+
dest = os.path.join(FINAL_DEST, category)
|
| 71 |
+
|
| 72 |
+
if os.path.isdir(src):
|
| 73 |
+
if not os.path.exists(dest): os.makedirs(dest)
|
| 74 |
+
for file in os.listdir(src):
|
| 75 |
+
try:
|
| 76 |
+
shutil.move(os.path.join(src, file), os.path.join(dest, file))
|
| 77 |
+
except: pass
|
| 78 |
+
try: os.rmdir(src)
|
| 79 |
+
except: pass
|
| 80 |
+
|
| 81 |
+
# Cleanup parent 'dataset' in mission17-ai if empty
|
| 82 |
+
try:
|
| 83 |
+
os.rmdir(MISPLACED_DIR)
|
| 84 |
+
os.rmdir(os.path.join(CURRENT_DIR, '..', '..', "dataset"))
|
| 85 |
+
except: pass
|
| 86 |
+
print(" ✅ Misplaced images moved to correct folder.")
|
| 87 |
+
|
| 88 |
+
print(f"\n✨ SUCCESS! Dataset is ready at: {FINAL_DEST}")
|
| 89 |
+
|
| 90 |
+
if __name__ == "__main__":
|
| 91 |
+
organize_files()
|
scripts/testing/split_dataset.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
import random
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
# --- CONFIGURATION ---
|
| 7 |
+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 8 |
+
DATASET_DIR = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset')
|
| 9 |
+
OUTPUT_DIR = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset_split')
|
| 10 |
+
|
| 11 |
+
# Split ratios
|
| 12 |
+
TRAIN_RATIO = 0.80
|
| 13 |
+
TEST_RATIO = 0.20
|
| 14 |
+
|
| 15 |
+
def split_dataset():
|
| 16 |
+
"""
|
| 17 |
+
Randomly splits the dataset into train/ and test/ folders.
|
| 18 |
+
This prevents 'Data Leakage' so your evaluate_model.py tests on truly unseen images.
|
| 19 |
+
"""
|
| 20 |
+
print(f"🚀 Splitting dataset: {DATASET_DIR}")
|
| 21 |
+
print(f" Outputting to: {OUTPUT_DIR}")
|
| 22 |
+
|
| 23 |
+
if not os.path.exists(DATASET_DIR):
|
| 24 |
+
print(f"❌ ERROR: Dataset not found at {DATASET_DIR}")
|
| 25 |
+
return
|
| 26 |
+
|
| 27 |
+
# Create output directories
|
| 28 |
+
train_dir = os.path.join(OUTPUT_DIR, 'train')
|
| 29 |
+
test_dir = os.path.join(OUTPUT_DIR, 'test')
|
| 30 |
+
|
| 31 |
+
os.makedirs(train_dir, exist_ok=True)
|
| 32 |
+
os.makedirs(test_dir, exist_ok=True)
|
| 33 |
+
|
| 34 |
+
classes = [d for d in os.listdir(DATASET_DIR) if os.path.isdir(os.path.join(DATASET_DIR, d))]
|
| 35 |
+
|
| 36 |
+
total_moved = 0
|
| 37 |
+
|
| 38 |
+
for class_name in classes:
|
| 39 |
+
class_path = os.path.join(DATASET_DIR, class_name)
|
| 40 |
+
images = [f for f in os.listdir(class_path) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))]
|
| 41 |
+
|
| 42 |
+
# Shuffle images randomly
|
| 43 |
+
random.shuffle(images)
|
| 44 |
+
|
| 45 |
+
# Calculate split index
|
| 46 |
+
split_idx = int(len(images) * TRAIN_RATIO)
|
| 47 |
+
|
| 48 |
+
train_images = images[:split_idx]
|
| 49 |
+
test_images = images[split_idx:]
|
| 50 |
+
|
| 51 |
+
# Create class folders in train/ and test/
|
| 52 |
+
os.makedirs(os.path.join(train_dir, class_name), exist_ok=True)
|
| 53 |
+
os.makedirs(os.path.join(test_dir, class_name), exist_ok=True)
|
| 54 |
+
|
| 55 |
+
print(f"📁 [{class_name}] Total: {len(images)} -> Train: {len(train_images)}, Test: {len(test_images)}")
|
| 56 |
+
|
| 57 |
+
# Copy files
|
| 58 |
+
for img in train_images:
|
| 59 |
+
shutil.copy2(os.path.join(class_path, img), os.path.join(train_dir, class_name, img))
|
| 60 |
+
total_moved += 1
|
| 61 |
+
|
| 62 |
+
for img in test_images:
|
| 63 |
+
shutil.copy2(os.path.join(class_path, img), os.path.join(test_dir, class_name, img))
|
| 64 |
+
total_moved += 1
|
| 65 |
+
|
| 66 |
+
print("=" * 50)
|
| 67 |
+
print(f"✅ Dataset split complete! {total_moved} images processed.")
|
| 68 |
+
print(" Next steps:")
|
| 69 |
+
print(" 1. Check the new folder 'mission_dataset_split'")
|
| 70 |
+
print(" 2. Run train_ai_v2.py (which now points to this new folder)")
|
| 71 |
+
|
| 72 |
+
if __name__ == '__main__':
|
| 73 |
+
split_dataset()
|
scripts/testing/test_duplicate.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import requests
|
| 2 |
+
|
| 3 |
+
url = "http://localhost:5000/predict"
|
| 4 |
+
file_path = "debug_bias_mitigation.jpg"
|
| 5 |
+
|
| 6 |
+
print("--- First Upload ---")
|
| 7 |
+
with open(file_path, "rb") as f:
|
| 8 |
+
r = requests.post(url, files={"file": f})
|
| 9 |
+
print(f"Status: {r.status_code}")
|
| 10 |
+
print(f"Response: {r.text}")
|
| 11 |
+
|
| 12 |
+
print("\n--- Second Upload ---")
|
| 13 |
+
with open(file_path, "rb") as f:
|
| 14 |
+
r = requests.post(url, files={"file": f})
|
| 15 |
+
print(f"Status: {r.status_code}")
|
| 16 |
+
print(f"Response: {r.text}")
|
scripts/testing/test_predictor.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import sys
|
| 2 |
+
import os
|
| 3 |
+
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
| 4 |
+
from utils.predictor import Predictor
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def test_ollama():
|
| 8 |
+
print("Testing Ollama Predictor...")
|
| 9 |
+
p = Predictor()
|
| 10 |
+
print(f"Model configured: {p.get_model_name()}")
|
| 11 |
+
|
| 12 |
+
# We will just see if we can reach the API
|
| 13 |
+
# Since we need an image, let's create a dummy 1x1 black pixel image
|
| 14 |
+
import io
|
| 15 |
+
from PIL import Image
|
| 16 |
+
|
| 17 |
+
img = Image.new('RGB', (10, 10), color = 'black')
|
| 18 |
+
img_byte_arr = io.BytesIO()
|
| 19 |
+
img.save(img_byte_arr, format='PNG')
|
| 20 |
+
img_bytes = img_byte_arr.getvalue()
|
| 21 |
+
|
| 22 |
+
print("Sending dummy image to Ollama...")
|
| 23 |
+
res = p.predict(img_bytes)
|
| 24 |
+
print("Response:", res)
|
| 25 |
+
|
| 26 |
+
if __name__ == "__main__":
|
| 27 |
+
test_ollama()
|
scripts/testing/test_server.py
ADDED
|
File without changes
|
scripts/testing/test_upload.html
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8">
|
| 5 |
+
<title>Mission 17 AI Scanner</title>
|
| 6 |
+
<style>
|
| 7 |
+
body { font-family: 'Segoe UI', sans-serif; text-align: center; padding: 50px; background-color: #f4f4f9; }
|
| 8 |
+
.card { background: white; padding: 40px; border-radius: 15px; box-shadow: 0 4px 15px rgba(0,0,0,0.1); display: inline-block; max-width: 400px; }
|
| 9 |
+
button { background: #007bff; color: white; border: none; padding: 12px 24px; border-radius: 5px; cursor: pointer; font-size: 16px; margin-top: 15px; }
|
| 10 |
+
button:hover { background: #0056b3; }
|
| 11 |
+
#result { margin-top: 25px; font-weight: bold; }
|
| 12 |
+
.verified { color: #28a745; background: #e6fffa; padding: 15px; border-radius: 8px; border: 1px solid #28a745; }
|
| 13 |
+
.rejected { color: #dc3545; background: #fff5f5; padding: 15px; border-radius: 8px; border: 1px solid #dc3545; }
|
| 14 |
+
</style>
|
| 15 |
+
</head>
|
| 16 |
+
<body>
|
| 17 |
+
<div class="card">
|
| 18 |
+
<h2>🤖 Mission 17 AI Scanner</h2>
|
| 19 |
+
<p>Upload a photo to verify your mission!</p>
|
| 20 |
+
|
| 21 |
+
<input type="file" id="fileInput" accept="image/*">
|
| 22 |
+
<br>
|
| 23 |
+
<button onclick="scanImage()">🔍 Scan Mission</button>
|
| 24 |
+
|
| 25 |
+
<div id="result"></div>
|
| 26 |
+
</div>
|
| 27 |
+
|
| 28 |
+
<script>
|
| 29 |
+
async function scanImage() {
|
| 30 |
+
const fileInput = document.getElementById('fileInput');
|
| 31 |
+
const resultDiv = document.getElementById('result');
|
| 32 |
+
|
| 33 |
+
if (!fileInput.files[0]) {
|
| 34 |
+
alert("Please select an image first!");
|
| 35 |
+
return;
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
resultDiv.innerHTML = "⏳ Scanning...";
|
| 39 |
+
|
| 40 |
+
const formData = new FormData();
|
| 41 |
+
formData.append("file", fileInput.files[0]);
|
| 42 |
+
|
| 43 |
+
try {
|
| 44 |
+
const response = await fetch("http://127.0.0.1:5000/predict", { method: "POST", body: formData });
|
| 45 |
+
const data = await response.json();
|
| 46 |
+
|
| 47 |
+
const colorClass = data.verdict === "VERIFIED" ? "verified" : "rejected";
|
| 48 |
+
resultDiv.innerHTML = `
|
| 49 |
+
<div class="${colorClass}">
|
| 50 |
+
<h3>${data.message}</h3>
|
| 51 |
+
<p><strong>📷 Type:</strong> ${data.source_check}</p>
|
| 52 |
+
<p><strong>🎯 Accuracy:</strong> ${data.confidence}</p>
|
| 53 |
+
<p><strong>🌍 SDG:</strong> ${data.sdg}</p>
|
| 54 |
+
</div>`;
|
| 55 |
+
} catch (error) {
|
| 56 |
+
resultDiv.innerHTML = "❌ Error connecting to server. Is app.py running?";
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
</script>
|
| 60 |
+
</body>
|
| 61 |
+
</html>
|
scripts/training/download_pangasinan.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
from datasets import load_dataset
|
| 4 |
+
|
| 5 |
+
print("Downloading dataset leklek02/pangasinan...")
|
| 6 |
+
ds = load_dataset("leklek02/pangasinan", split="train")
|
| 7 |
+
|
| 8 |
+
print(f"Total rows downloaded: {len(ds)}")
|
| 9 |
+
|
| 10 |
+
# 1. Create Few-Shot Examples (Top 100 to keep the prompt size reasonable)
|
| 11 |
+
examples = []
|
| 12 |
+
for i in range(min(100, len(ds))):
|
| 13 |
+
row = ds[i]
|
| 14 |
+
user_text = str(row['instruction'])
|
| 15 |
+
if row.get('input') and str(row['input']).strip():
|
| 16 |
+
user_text += "\n" + str(row['input'])
|
| 17 |
+
|
| 18 |
+
examples.append({
|
| 19 |
+
"User": user_text,
|
| 20 |
+
"Bot": str(row['output'])
|
| 21 |
+
})
|
| 22 |
+
|
| 23 |
+
backend_path = r"c:\Users\Kurt Perez\mission17\mission17-backend\utils\pangasinan_examples.json"
|
| 24 |
+
os.makedirs(os.path.dirname(backend_path), exist_ok=True)
|
| 25 |
+
with open(backend_path, "w", encoding="utf-8") as f:
|
| 26 |
+
json.dump(examples, f, indent=2, ensure_ascii=False)
|
| 27 |
+
print(f"Saved 100 examples to {backend_path} for immediate use in Chatbot.")
|
| 28 |
+
|
| 29 |
+
# 2. Create Gemini Tuning JSONL (All rows for future Fine-Tuning)
|
| 30 |
+
tuning_path = r"c:\Users\Kurt Perez\mission17\dataset\gemini_pangasinan_tuning.jsonl"
|
| 31 |
+
os.makedirs(os.path.dirname(tuning_path), exist_ok=True)
|
| 32 |
+
with open(tuning_path, "w", encoding="utf-8") as f:
|
| 33 |
+
for row in ds:
|
| 34 |
+
user_text = str(row['instruction'])
|
| 35 |
+
if row.get('input') and str(row['input']).strip():
|
| 36 |
+
user_text += "\n" + str(row['input'])
|
| 37 |
+
|
| 38 |
+
jsonl_obj = {
|
| 39 |
+
"contents": [
|
| 40 |
+
{"role": "user", "parts": [{"text": user_text}]},
|
| 41 |
+
{"role": "model", "parts": [{"text": str(row['output'])}]}
|
| 42 |
+
]
|
| 43 |
+
}
|
| 44 |
+
f.write(json.dumps(jsonl_obj, ensure_ascii=False) + "\n")
|
| 45 |
+
|
| 46 |
+
print(f"Saved full tuning dataset to {tuning_path} (Upload this to Google AI Studio to fine-tune Gemini!)")
|
scripts/training/evaluate_model.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import tensorflow as tf
|
| 3 |
+
from tensorflow.keras.preprocessing.image import ImageDataGenerator
|
| 4 |
+
from tensorflow.keras.applications.efficientnet import preprocess_input
|
| 5 |
+
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix
|
| 6 |
+
import matplotlib.pyplot as plt
|
| 7 |
+
import seaborn as sns
|
| 8 |
+
import os
|
| 9 |
+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 10 |
+
|
| 11 |
+
print("⏳ Loading AI Model...")
|
| 12 |
+
# 👇 Ensure this is your correct model name!
|
| 13 |
+
model = tf.keras.models.load_model(os.path.join(BASE_DIR, '..', '..', 'mission_model.h5'))
|
| 14 |
+
|
| 15 |
+
print("📁 Loading Test Dataset...")
|
| 16 |
+
# 👇 Pointing to the new TEST split folder
|
| 17 |
+
test_dir = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset_split', 'test')
|
| 18 |
+
|
| 19 |
+
test_datagen = ImageDataGenerator(preprocessing_function=preprocess_input)
|
| 20 |
+
test_generator = test_datagen.flow_from_directory(
|
| 21 |
+
test_dir,
|
| 22 |
+
target_size=(224, 224),
|
| 23 |
+
batch_size=32,
|
| 24 |
+
class_mode='categorical',
|
| 25 |
+
shuffle=False
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
print("🤖 Running Predictions (This may take a minute)...")
|
| 29 |
+
Y_pred = model.predict(test_generator)
|
| 30 |
+
y_pred_classes = np.argmax(Y_pred, axis=1) # 👈 FIXED: Grabs the top prediction per image
|
| 31 |
+
y_true = test_generator.classes
|
| 32 |
+
|
| 33 |
+
print("\n" + "="*50)
|
| 34 |
+
print("🏆 CAPSTONE AI PERFORMANCE METRICS 🏆")
|
| 35 |
+
print("="*50)
|
| 36 |
+
|
| 37 |
+
# 👈 FIXED: Added average='weighted' to handle all 10 classes correctly
|
| 38 |
+
accuracy = accuracy_score(y_true, y_pred_classes)
|
| 39 |
+
precision = precision_score(y_true, y_pred_classes, average='weighted', zero_division=0)
|
| 40 |
+
recall = recall_score(y_true, y_pred_classes, average='weighted', zero_division=0)
|
| 41 |
+
f1 = f1_score(y_true, y_pred_classes, average='weighted', zero_division=0)
|
| 42 |
+
|
| 43 |
+
print(f"✅ Accuracy: {accuracy * 100:.2f}%")
|
| 44 |
+
print(f"🎯 Precision: {precision * 100:.2f}%")
|
| 45 |
+
print(f"🔍 Recall: {recall * 100:.2f}%")
|
| 46 |
+
print(f"⚖️ F1-Score: {f1 * 100:.2f}%")
|
| 47 |
+
print("="*50)
|
| 48 |
+
|
| 49 |
+
# Make the confusion matrix chart larger to fit 10 classes
|
| 50 |
+
cm = confusion_matrix(y_true, y_pred_classes)
|
| 51 |
+
plt.figure(figsize=(10, 8))
|
| 52 |
+
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
|
| 53 |
+
plt.title('AI Confusion Matrix (10 Classes)')
|
| 54 |
+
plt.ylabel('Actual Image Class')
|
| 55 |
+
plt.xlabel('AI Prediction')
|
| 56 |
+
plt.savefig(os.path.join(BASE_DIR, '..', '..', 'outputs', 'confusion_matrix.png'))
|
| 57 |
+
print("\n📊 Saved 'confusion_matrix.png' to your outputs folder. Put this in your presentation!")
|
scripts/training/train_ai.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import tensorflow as tf
|
| 3 |
+
from tensorflow.keras.preprocessing.image import ImageDataGenerator
|
| 4 |
+
from tensorflow.keras.applications import EfficientNetB0
|
| 5 |
+
from tensorflow.keras.applications.efficientnet import preprocess_input
|
| 6 |
+
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout, BatchNormalization
|
| 7 |
+
from tensorflow.keras.models import Model
|
| 8 |
+
from tensorflow.keras.optimizers import Adam
|
| 9 |
+
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
|
| 10 |
+
|
| 11 |
+
# --- CONFIGURATION ---
|
| 12 |
+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 13 |
+
DATASET_DIR = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset')
|
| 14 |
+
MODEL_SAVE_PATH = os.path.join(BASE_DIR, '..', '..', 'mission_model.h5')
|
| 15 |
+
LABELS_SAVE_PATH = os.path.join(BASE_DIR, '..', '..', 'labels.txt')
|
| 16 |
+
|
| 17 |
+
# Hyperparameters
|
| 18 |
+
IMG_SIZE = (224, 224)
|
| 19 |
+
BATCH_SIZE = 32
|
| 20 |
+
EPOCHS_INITIAL = 25 # Phase 1: Train top layers only
|
| 21 |
+
EPOCHS_FINETUNE = 15 # Phase 2: Fine-tune top base layers
|
| 22 |
+
LR_INITIAL = 1e-3 # Higher LR for initial training
|
| 23 |
+
LR_FINETUNE = 1e-5 # Much lower LR for fine-tuning (prevents forgetting)
|
| 24 |
+
FINETUNE_FROM_LAYER = 150 # Unfreeze EfficientNetB0 from this layer onwards
|
| 25 |
+
|
| 26 |
+
def build_generators():
|
| 27 |
+
"""Create training and validation data generators with strong augmentation."""
|
| 28 |
+
print("📸 Preparing Image Generators with Strong Augmentation...")
|
| 29 |
+
|
| 30 |
+
# 🔥 EfficientNetB0 has its own internal preprocessing — do NOT use rescale=1./255!
|
| 31 |
+
# Using preprocess_input correctly scales raw 0-255 pixel values for EfficientNet.
|
| 32 |
+
train_datagen = ImageDataGenerator(
|
| 33 |
+
preprocessing_function=preprocess_input, # ✅ EfficientNetB0-compatible
|
| 34 |
+
rotation_range=30,
|
| 35 |
+
width_shift_range=0.2,
|
| 36 |
+
height_shift_range=0.2,
|
| 37 |
+
horizontal_flip=True,
|
| 38 |
+
brightness_range=[0.7, 1.3],
|
| 39 |
+
zoom_range=0.2,
|
| 40 |
+
shear_range=0.1,
|
| 41 |
+
channel_shift_range=20.0,
|
| 42 |
+
fill_mode='nearest',
|
| 43 |
+
validation_split=0.2
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
# Validation: only preprocess_input, NO augmentation
|
| 47 |
+
val_datagen = ImageDataGenerator(
|
| 48 |
+
preprocessing_function=preprocess_input, # ✅ Must match training
|
| 49 |
+
validation_split=0.2
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
train_generator = train_datagen.flow_from_directory(
|
| 53 |
+
DATASET_DIR,
|
| 54 |
+
target_size=IMG_SIZE,
|
| 55 |
+
batch_size=BATCH_SIZE,
|
| 56 |
+
class_mode='categorical',
|
| 57 |
+
subset='training',
|
| 58 |
+
shuffle=True
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
validation_generator = val_datagen.flow_from_directory(
|
| 62 |
+
DATASET_DIR,
|
| 63 |
+
target_size=IMG_SIZE,
|
| 64 |
+
batch_size=BATCH_SIZE,
|
| 65 |
+
class_mode='categorical',
|
| 66 |
+
subset='validation',
|
| 67 |
+
shuffle=False
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
return train_generator, validation_generator
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def build_model(num_classes):
|
| 74 |
+
"""
|
| 75 |
+
Build model using EfficientNetB0 (more accurate than MobileNetV2).
|
| 76 |
+
Phase 1 starts with all base layers FROZEN — only top layers train first.
|
| 77 |
+
"""
|
| 78 |
+
print("🧠 Building Model (EfficientNetB0 — upgraded from MobileNetV2)...")
|
| 79 |
+
|
| 80 |
+
base_model = EfficientNetB0(
|
| 81 |
+
weights='imagenet',
|
| 82 |
+
include_top=False,
|
| 83 |
+
input_shape=IMG_SIZE + (3,)
|
| 84 |
+
)
|
| 85 |
+
base_model.trainable = False # Freeze all base layers for Phase 1
|
| 86 |
+
|
| 87 |
+
x = base_model.output
|
| 88 |
+
x = GlobalAveragePooling2D()(x)
|
| 89 |
+
x = BatchNormalization()(x) # ✨ NEW — stabilizes training
|
| 90 |
+
x = Dropout(0.3)(x) # was 0.2 — slightly stronger regularization
|
| 91 |
+
x = Dense(256, activation='relu')(x) # ✨ NEW — extra dense layer for richer features
|
| 92 |
+
x = Dropout(0.2)(x)
|
| 93 |
+
predictions = Dense(num_classes, activation='softmax')(x)
|
| 94 |
+
|
| 95 |
+
model = Model(inputs=base_model.input, outputs=predictions)
|
| 96 |
+
return model, base_model
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def get_callbacks(phase_name):
|
| 100 |
+
"""Smart callbacks: stop early if no improvement, reduce LR on plateau."""
|
| 101 |
+
return [
|
| 102 |
+
EarlyStopping(
|
| 103 |
+
monitor='val_accuracy',
|
| 104 |
+
patience=5, # Stop if no improvement for 5 epochs
|
| 105 |
+
restore_best_weights=True,
|
| 106 |
+
verbose=1
|
| 107 |
+
),
|
| 108 |
+
ReduceLROnPlateau(
|
| 109 |
+
monitor='val_loss',
|
| 110 |
+
factor=0.5, # Halve LR if stuck
|
| 111 |
+
patience=3,
|
| 112 |
+
min_lr=1e-7,
|
| 113 |
+
verbose=1
|
| 114 |
+
),
|
| 115 |
+
ModelCheckpoint(
|
| 116 |
+
filepath=MODEL_SAVE_PATH,
|
| 117 |
+
monitor='val_accuracy',
|
| 118 |
+
save_best_only=True, # Always keep the best checkpoint
|
| 119 |
+
verbose=1
|
| 120 |
+
)
|
| 121 |
+
]
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def train_brain():
|
| 125 |
+
print("🚀 Initializing Mission 17 AI Training (Enhanced)...")
|
| 126 |
+
|
| 127 |
+
# 1. CHECK DATASET
|
| 128 |
+
if not os.path.exists(DATASET_DIR):
|
| 129 |
+
print(f"❌ ERROR: Dataset not found at {DATASET_DIR}")
|
| 130 |
+
return
|
| 131 |
+
|
| 132 |
+
# 2. BUILD GENERATORS
|
| 133 |
+
try:
|
| 134 |
+
train_generator, validation_generator = build_generators()
|
| 135 |
+
except Exception as e:
|
| 136 |
+
print(f"❌ Error loading data: {e}")
|
| 137 |
+
return
|
| 138 |
+
|
| 139 |
+
if train_generator.samples == 0:
|
| 140 |
+
print("❌ No images found! Check your dataset structure.")
|
| 141 |
+
return
|
| 142 |
+
|
| 143 |
+
# 3. SAVE LABELS
|
| 144 |
+
class_names = list(train_generator.class_indices.keys())
|
| 145 |
+
print(f"🏷️ Classes Detected: {class_names}")
|
| 146 |
+
with open(LABELS_SAVE_PATH, 'w') as f:
|
| 147 |
+
for name in class_names:
|
| 148 |
+
f.write(name + '\n')
|
| 149 |
+
print(f"✅ Labels saved to {LABELS_SAVE_PATH}")
|
| 150 |
+
|
| 151 |
+
num_classes = len(class_names)
|
| 152 |
+
|
| 153 |
+
# 4. BUILD MODEL
|
| 154 |
+
model, base_model = build_model(num_classes)
|
| 155 |
+
|
| 156 |
+
# ════════════════════════════════════════════
|
| 157 |
+
# PHASE 1: Train top layers only (fast)
|
| 158 |
+
# ════════════════════════════════════════════
|
| 159 |
+
print("\n" + "="*50)
|
| 160 |
+
print("🏋️ PHASE 1: Training Top Layers (Base Frozen)")
|
| 161 |
+
print("="*50)
|
| 162 |
+
|
| 163 |
+
model.compile(
|
| 164 |
+
optimizer=Adam(learning_rate=LR_INITIAL),
|
| 165 |
+
loss='categorical_crossentropy',
|
| 166 |
+
metrics=['accuracy']
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
model.fit(
|
| 170 |
+
train_generator,
|
| 171 |
+
epochs=EPOCHS_INITIAL,
|
| 172 |
+
validation_data=validation_generator,
|
| 173 |
+
callbacks=get_callbacks('phase1')
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
# ════════════════════════════════════════════
|
| 177 |
+
# PHASE 2: Fine-tune top layers of base model
|
| 178 |
+
# ════════════════════════════════════════════
|
| 179 |
+
print("\n" + "="*50)
|
| 180 |
+
print("🔬 PHASE 2: Fine-Tuning Top Base Layers")
|
| 181 |
+
print(f" Unfreezing EfficientNetB0 from layer {FINETUNE_FROM_LAYER}+")
|
| 182 |
+
print("="*50)
|
| 183 |
+
|
| 184 |
+
base_model.trainable = True
|
| 185 |
+
|
| 186 |
+
# Only unfreeze layers AFTER FINETUNE_FROM_LAYER — keep earlier layers frozen
|
| 187 |
+
for layer in base_model.layers[:FINETUNE_FROM_LAYER]:
|
| 188 |
+
layer.trainable = False
|
| 189 |
+
|
| 190 |
+
# CRITICAL: Recompile with much lower LR to avoid destroying pre-trained weights
|
| 191 |
+
model.compile(
|
| 192 |
+
optimizer=Adam(learning_rate=LR_FINETUNE),
|
| 193 |
+
loss='categorical_crossentropy',
|
| 194 |
+
metrics=['accuracy']
|
| 195 |
+
)
|
| 196 |
+
|
| 197 |
+
model.fit(
|
| 198 |
+
train_generator,
|
| 199 |
+
epochs=EPOCHS_FINETUNE,
|
| 200 |
+
validation_data=validation_generator,
|
| 201 |
+
callbacks=get_callbacks('phase2')
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
print(f"\n✅ Training complete! Best model saved to {MODEL_SAVE_PATH}")
|
| 205 |
+
print(" Run evaluate_model.py to check accuracy metrics & confusion matrix.")
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
if __name__ == '__main__':
|
| 209 |
+
train_brain()
|
scripts/training/train_ai_v2.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import numpy as np
|
| 3 |
+
import tensorflow as tf
|
| 4 |
+
from tensorflow.keras.preprocessing.image import ImageDataGenerator
|
| 5 |
+
from tensorflow.keras.applications import EfficientNetB0
|
| 6 |
+
from tensorflow.keras.applications.efficientnet import preprocess_input
|
| 7 |
+
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout, BatchNormalization
|
| 8 |
+
from tensorflow.keras.models import Model
|
| 9 |
+
from tensorflow.keras.optimizers import Adam
|
| 10 |
+
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
|
| 11 |
+
from sklearn.utils.class_weight import compute_class_weight
|
| 12 |
+
|
| 13 |
+
# --- CONFIGURATION ---
|
| 14 |
+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 15 |
+
# IMPORTANT: Pointing to the new split dataset folder
|
| 16 |
+
DATASET_DIR = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset_split', 'train')
|
| 17 |
+
MODEL_SAVE_PATH = os.path.join(BASE_DIR, '..', '..', 'mission_model.h5')
|
| 18 |
+
LABELS_SAVE_PATH = os.path.join(BASE_DIR, '..', '..', 'labels.txt')
|
| 19 |
+
|
| 20 |
+
IMG_SIZE = (224, 224)
|
| 21 |
+
BATCH_SIZE = 32
|
| 22 |
+
EPOCHS_INITIAL = 20
|
| 23 |
+
EPOCHS_FINETUNE = 15
|
| 24 |
+
LR_INITIAL = 1e-3
|
| 25 |
+
LR_FINETUNE = 1e-5
|
| 26 |
+
|
| 27 |
+
def build_generators():
|
| 28 |
+
print("📸 Preparing Image Generators...")
|
| 29 |
+
|
| 30 |
+
train_datagen = ImageDataGenerator(
|
| 31 |
+
preprocessing_function=preprocess_input,
|
| 32 |
+
rotation_range=30,
|
| 33 |
+
width_shift_range=0.2,
|
| 34 |
+
height_shift_range=0.2,
|
| 35 |
+
horizontal_flip=True,
|
| 36 |
+
brightness_range=[0.7, 1.3],
|
| 37 |
+
zoom_range=0.2,
|
| 38 |
+
validation_split=0.2 # 20% of the train/ folder becomes validation
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
train_generator = train_datagen.flow_from_directory(
|
| 42 |
+
DATASET_DIR,
|
| 43 |
+
target_size=IMG_SIZE,
|
| 44 |
+
batch_size=BATCH_SIZE,
|
| 45 |
+
class_mode='categorical',
|
| 46 |
+
subset='training',
|
| 47 |
+
shuffle=True
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
validation_generator = train_datagen.flow_from_directory(
|
| 51 |
+
DATASET_DIR,
|
| 52 |
+
target_size=IMG_SIZE,
|
| 53 |
+
batch_size=BATCH_SIZE,
|
| 54 |
+
class_mode='categorical',
|
| 55 |
+
subset='validation',
|
| 56 |
+
shuffle=False
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
return train_generator, validation_generator
|
| 60 |
+
|
| 61 |
+
def get_class_weights(train_generator):
|
| 62 |
+
"""
|
| 63 |
+
Calculates class weights to handle imbalanced datasets.
|
| 64 |
+
This stops the AI from being biased toward the majority class.
|
| 65 |
+
"""
|
| 66 |
+
print("⚖️ Calculating Class Weights for balanced training...")
|
| 67 |
+
class_indices = train_generator.class_indices
|
| 68 |
+
classes = train_generator.classes
|
| 69 |
+
|
| 70 |
+
weights = compute_class_weight(
|
| 71 |
+
class_weight='balanced',
|
| 72 |
+
classes=np.unique(classes),
|
| 73 |
+
y=classes
|
| 74 |
+
)
|
| 75 |
+
class_weights = dict(enumerate(weights))
|
| 76 |
+
|
| 77 |
+
print(" Weights applied:")
|
| 78 |
+
for cls_name, cls_idx in class_indices.items():
|
| 79 |
+
print(f" - {cls_name}: {class_weights[cls_idx]:.2f}")
|
| 80 |
+
|
| 81 |
+
return class_weights
|
| 82 |
+
|
| 83 |
+
def build_model(num_classes):
|
| 84 |
+
print("🧠 Building Model (EfficientNetB0)...")
|
| 85 |
+
|
| 86 |
+
base_model = EfficientNetB0(
|
| 87 |
+
weights='imagenet',
|
| 88 |
+
include_top=False,
|
| 89 |
+
input_shape=IMG_SIZE + (3,)
|
| 90 |
+
)
|
| 91 |
+
base_model.trainable = False
|
| 92 |
+
|
| 93 |
+
x = base_model.output
|
| 94 |
+
x = GlobalAveragePooling2D()(x)
|
| 95 |
+
x = BatchNormalization()(x)
|
| 96 |
+
x = Dropout(0.3)(x)
|
| 97 |
+
x = Dense(256, activation='relu')(x)
|
| 98 |
+
x = BatchNormalization()(x)
|
| 99 |
+
x = Dropout(0.3)(x)
|
| 100 |
+
predictions = Dense(num_classes, activation='softmax')(x)
|
| 101 |
+
|
| 102 |
+
model = Model(inputs=base_model.input, outputs=predictions)
|
| 103 |
+
return model, base_model
|
| 104 |
+
|
| 105 |
+
def get_callbacks(phase_name):
|
| 106 |
+
return [
|
| 107 |
+
EarlyStopping(monitor='val_accuracy', patience=5, restore_best_weights=True, verbose=1),
|
| 108 |
+
ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=1e-7, verbose=1),
|
| 109 |
+
ModelCheckpoint(filepath=MODEL_SAVE_PATH, monitor='val_accuracy', save_best_only=True, verbose=1)
|
| 110 |
+
]
|
| 111 |
+
|
| 112 |
+
def train_brain():
|
| 113 |
+
print("🚀 Initializing Mission 17 AI Training v2 (Optimized)...")
|
| 114 |
+
|
| 115 |
+
if not os.path.exists(DATASET_DIR):
|
| 116 |
+
print(f"❌ ERROR: Training Dataset not found at {DATASET_DIR}")
|
| 117 |
+
print(" Did you run scripts/testing/split_dataset.py first?")
|
| 118 |
+
return
|
| 119 |
+
|
| 120 |
+
train_generator, validation_generator = build_generators()
|
| 121 |
+
|
| 122 |
+
# Save Labels
|
| 123 |
+
class_names = list(train_generator.class_indices.keys())
|
| 124 |
+
with open(LABELS_SAVE_PATH, 'w') as f:
|
| 125 |
+
for name in class_names:
|
| 126 |
+
f.write(name + '\n')
|
| 127 |
+
|
| 128 |
+
num_classes = len(class_names)
|
| 129 |
+
|
| 130 |
+
# Get Class Weights
|
| 131 |
+
class_weights = get_class_weights(train_generator)
|
| 132 |
+
|
| 133 |
+
model, base_model = build_model(num_classes)
|
| 134 |
+
|
| 135 |
+
# --- PHASE 1 ---
|
| 136 |
+
print("\n" + "="*50)
|
| 137 |
+
print("🏋️ PHASE 1: Training Top Layers (Base Frozen)")
|
| 138 |
+
print("="*50)
|
| 139 |
+
|
| 140 |
+
model.compile(optimizer=Adam(learning_rate=LR_INITIAL), loss='categorical_crossentropy', metrics=['accuracy'])
|
| 141 |
+
model.fit(
|
| 142 |
+
train_generator,
|
| 143 |
+
epochs=EPOCHS_INITIAL,
|
| 144 |
+
validation_data=validation_generator,
|
| 145 |
+
class_weight=class_weights, # Apply weights!
|
| 146 |
+
callbacks=get_callbacks('phase1')
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
# --- PHASE 2 ---
|
| 150 |
+
print("\n" + "="*50)
|
| 151 |
+
print("🔬 PHASE 2: Fine-Tuning Top Base Layers")
|
| 152 |
+
print("="*50)
|
| 153 |
+
|
| 154 |
+
base_model.trainable = True
|
| 155 |
+
for layer in base_model.layers[:150]:
|
| 156 |
+
layer.trainable = False
|
| 157 |
+
|
| 158 |
+
model.compile(optimizer=Adam(learning_rate=LR_FINETUNE), loss='categorical_crossentropy', metrics=['accuracy'])
|
| 159 |
+
model.fit(
|
| 160 |
+
train_generator,
|
| 161 |
+
epochs=EPOCHS_FINETUNE,
|
| 162 |
+
validation_data=validation_generator,
|
| 163 |
+
class_weight=class_weights, # Apply weights!
|
| 164 |
+
callbacks=get_callbacks('phase2')
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
print(f"\n✅ Training complete! Model saved to {MODEL_SAVE_PATH}")
|
| 168 |
+
|
| 169 |
+
if __name__ == '__main__':
|
| 170 |
+
train_brain()
|
utils/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Utils module for Mission 17 AI."""
|
utils/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (214 Bytes). View file
|
|
|
utils/__pycache__/anticheat.cpython-311.pyc
ADDED
|
Binary file (5.83 kB). View file
|
|
|
utils/__pycache__/predictor.cpython-311.pyc
ADDED
|
Binary file (5.93 kB). View file
|
|
|
utils/__pycache__/verdict.cpython-311.pyc
ADDED
|
Binary file (1.88 kB). View file
|
|
|
utils/anticheat.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import json
|
| 3 |
+
import io
|
| 4 |
+
import imagehash
|
| 5 |
+
from PIL import Image
|
| 6 |
+
|
| 7 |
+
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
| 8 |
+
HASH_FILE = os.path.join(BASE_DIR, 'anticheat_hashes.json')
|
| 9 |
+
|
| 10 |
+
class AntiCheatEngine:
|
| 11 |
+
def __init__(self):
|
| 12 |
+
self.hashes = set()
|
| 13 |
+
self._load_hashes()
|
| 14 |
+
|
| 15 |
+
def _load_hashes(self):
|
| 16 |
+
if os.path.exists(HASH_FILE):
|
| 17 |
+
try:
|
| 18 |
+
with open(HASH_FILE, 'r') as f:
|
| 19 |
+
data = json.load(f)
|
| 20 |
+
self.hashes = set(data.get("hashes", []))
|
| 21 |
+
except Exception as e:
|
| 22 |
+
print(f"⚠️ Could not load anticheat hashes: {e}")
|
| 23 |
+
|
| 24 |
+
def _save_hashes(self):
|
| 25 |
+
try:
|
| 26 |
+
with open(HASH_FILE, 'w') as f:
|
| 27 |
+
json.dump({"hashes": list(self.hashes)}, f)
|
| 28 |
+
except Exception as e:
|
| 29 |
+
print(f"⚠️ Could not save anticheat hashes: {e}")
|
| 30 |
+
|
| 31 |
+
def get_hashes(self, file_bytes):
|
| 32 |
+
"""Calculates pHash and dHash for better duplicate detection."""
|
| 33 |
+
try:
|
| 34 |
+
img = Image.open(io.BytesIO(file_bytes)).convert('RGB')
|
| 35 |
+
p_hash = str(imagehash.phash(img))
|
| 36 |
+
d_hash = str(imagehash.dhash(img))
|
| 37 |
+
return p_hash, d_hash
|
| 38 |
+
except Exception:
|
| 39 |
+
return None, None
|
| 40 |
+
|
| 41 |
+
def is_duplicate(self, file_bytes, similarity_threshold=8):
|
| 42 |
+
"""
|
| 43 |
+
Checks if the image is a duplicate based on stored hashes.
|
| 44 |
+
similarity_threshold: the max hamming distance to be considered a duplicate.
|
| 45 |
+
"""
|
| 46 |
+
p_hash_str, d_hash_str = self.get_hashes(file_bytes)
|
| 47 |
+
|
| 48 |
+
if not p_hash_str or not d_hash_str:
|
| 49 |
+
return False
|
| 50 |
+
|
| 51 |
+
# Check exact matches first for speed
|
| 52 |
+
if p_hash_str in self.hashes or d_hash_str in self.hashes:
|
| 53 |
+
return True
|
| 54 |
+
|
| 55 |
+
p_hash = imagehash.hex_to_hash(p_hash_str)
|
| 56 |
+
d_hash = imagehash.hex_to_hash(d_hash_str)
|
| 57 |
+
|
| 58 |
+
# Check similarity (hamming distance)
|
| 59 |
+
for stored_hash_str in self.hashes:
|
| 60 |
+
try:
|
| 61 |
+
stored_hash = imagehash.hex_to_hash(stored_hash_str)
|
| 62 |
+
# Compare both pHash and dHash representation lengths isn't an issue since they are stored as strings
|
| 63 |
+
# but we should compare apples to apples. Let's simplify and just do exact match on dHash and pHash,
|
| 64 |
+
# but also check similarity if we parse them properly.
|
| 65 |
+
|
| 66 |
+
# For safety, let's just do an exact match on string representations for now,
|
| 67 |
+
# or a simple distance check if we assume all stored are pHashes.
|
| 68 |
+
# Since we store both, some might be dHash, some pHash.
|
| 69 |
+
# Let's just compare distances safely.
|
| 70 |
+
distance = p_hash - stored_hash
|
| 71 |
+
if distance < similarity_threshold:
|
| 72 |
+
return True
|
| 73 |
+
|
| 74 |
+
distance = d_hash - stored_hash
|
| 75 |
+
if distance < similarity_threshold:
|
| 76 |
+
return True
|
| 77 |
+
except Exception:
|
| 78 |
+
continue
|
| 79 |
+
|
| 80 |
+
return False
|
| 81 |
+
|
| 82 |
+
def register(self, file_bytes):
|
| 83 |
+
"""Registers a new image hash to prevent future duplicates."""
|
| 84 |
+
p_hash_str, d_hash_str = self.get_hashes(file_bytes)
|
| 85 |
+
if p_hash_str:
|
| 86 |
+
self.hashes.add(p_hash_str)
|
| 87 |
+
if d_hash_str:
|
| 88 |
+
self.hashes.add(d_hash_str)
|
| 89 |
+
self._save_hashes()
|
| 90 |
+
return p_hash_str
|
| 91 |
+
|
| 92 |
+
def clear(self):
|
| 93 |
+
self.hashes.clear()
|
| 94 |
+
self._save_hashes()
|
| 95 |
+
return len(self.hashes)
|
| 96 |
+
|
| 97 |
+
def count(self):
|
| 98 |
+
return len(self.hashes)
|
utils/predictor.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import io
|
| 3 |
+
import traceback
|
| 4 |
+
import numpy as np
|
| 5 |
+
from PIL import Image
|
| 6 |
+
from tensorflow.keras.models import load_model
|
| 7 |
+
from tensorflow.keras.applications.efficientnet import preprocess_input
|
| 8 |
+
|
| 9 |
+
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
| 10 |
+
MODEL_PATH = os.path.join(BASE_DIR, 'mission_model.h5')
|
| 11 |
+
LABELS_PATH = os.path.join(BASE_DIR, 'labels.txt')
|
| 12 |
+
|
| 13 |
+
class Predictor:
|
| 14 |
+
def __init__(self):
|
| 15 |
+
self.model = None
|
| 16 |
+
self.class_names = []
|
| 17 |
+
self._load_model()
|
| 18 |
+
|
| 19 |
+
def _load_model(self):
|
| 20 |
+
print("🧠 Loading TensorFlow CNN Brain...")
|
| 21 |
+
if not os.path.exists(MODEL_PATH):
|
| 22 |
+
print(f"❌ ERROR: {MODEL_PATH} not found. You need to train the model!")
|
| 23 |
+
return
|
| 24 |
+
|
| 25 |
+
try:
|
| 26 |
+
self.model = load_model(MODEL_PATH)
|
| 27 |
+
print("✅ Model loaded successfully!")
|
| 28 |
+
except Exception as e:
|
| 29 |
+
print(f"❌ Failed to load model: {e}")
|
| 30 |
+
|
| 31 |
+
# Load Labels
|
| 32 |
+
try:
|
| 33 |
+
with open(LABELS_PATH, 'r') as f:
|
| 34 |
+
self.class_names = [line.strip() for line in f.readlines()]
|
| 35 |
+
print(f"🏷️ Labels loaded: {self.class_names}")
|
| 36 |
+
except FileNotFoundError:
|
| 37 |
+
print("❌ ERROR: labels.txt not found.")
|
| 38 |
+
self.class_names = []
|
| 39 |
+
|
| 40 |
+
# 🔥 WARMUP STEP (Optimization)
|
| 41 |
+
if self.model:
|
| 42 |
+
print("🔥 Warming up model for instant first-prediction...")
|
| 43 |
+
dummy_image = np.zeros((1, 224, 224, 3), dtype=np.float32)
|
| 44 |
+
self.model.predict(dummy_image, verbose=0)
|
| 45 |
+
print("⚡ AI is fully optimized and ready!")
|
| 46 |
+
|
| 47 |
+
def predict(self, file_bytes):
|
| 48 |
+
"""
|
| 49 |
+
Runs the image through the custom EfficientNet CNN.
|
| 50 |
+
"""
|
| 51 |
+
if not self.model or not self.class_names:
|
| 52 |
+
return {"category": "Non_SDG_Invalid", "confidence": 0, "reason": "Model offline or missing."}
|
| 53 |
+
|
| 54 |
+
try:
|
| 55 |
+
# 1. Read image using PIL (just like in train_ai.py)
|
| 56 |
+
img = Image.open(io.BytesIO(file_bytes)).convert('RGB')
|
| 57 |
+
|
| 58 |
+
# 2. Resize to 224x224 (EfficientNetB0 input size)
|
| 59 |
+
img = img.resize((224, 224), Image.LANCZOS)
|
| 60 |
+
|
| 61 |
+
# 3. Apply EfficientNetB0 preprocess_input
|
| 62 |
+
img_array = np.array(img, dtype=np.float32)
|
| 63 |
+
img_array = preprocess_input(img_array)
|
| 64 |
+
img_array = np.expand_dims(img_array, axis=0)
|
| 65 |
+
|
| 66 |
+
# 4. Predict
|
| 67 |
+
predictions = self.model.predict(img_array)
|
| 68 |
+
score = predictions[0]
|
| 69 |
+
|
| 70 |
+
top_index = np.argmax(score)
|
| 71 |
+
label = self.class_names[top_index]
|
| 72 |
+
|
| 73 |
+
confidence = int(np.max(score) * 100)
|
| 74 |
+
|
| 75 |
+
# Clean up label if it has the SDG prefix (e.g. SDG12_Recycling -> Recycling)
|
| 76 |
+
# The verdict.py MISSION_MAP expects "Recycling", "Planting", etc.
|
| 77 |
+
category = label
|
| 78 |
+
if "_" in label and label.startswith("SDG"):
|
| 79 |
+
# E.g. "SDG12_Recycling" -> "Recycling"
|
| 80 |
+
category = label.split("_", 1)[1]
|
| 81 |
+
# If there are multiple underscores (like SDG13_15_Planting), take the last part
|
| 82 |
+
if "_" in category:
|
| 83 |
+
category = category.rsplit("_", 1)[-1]
|
| 84 |
+
elif label == "Non_SDG_Invalid":
|
| 85 |
+
category = "Non_SDG_Invalid"
|
| 86 |
+
|
| 87 |
+
# Quick check for combined strings
|
| 88 |
+
if "Planting" in label: category = "Planting"
|
| 89 |
+
if "Cleanup" in label: category = "Cleanup"
|
| 90 |
+
if "Donation" in label: category = "Donation"
|
| 91 |
+
if "Cities" in label or "Sustainable" in label: category = "Sustainable_Cities"
|
| 92 |
+
if "Local" in label: category = "Support_Local"
|
| 93 |
+
if "Health" in label: category = "Health"
|
| 94 |
+
if "Energy" in label: category = "Energy"
|
| 95 |
+
if "Education" in label: category = "Education"
|
| 96 |
+
|
| 97 |
+
return {
|
| 98 |
+
"category": category,
|
| 99 |
+
"confidence": confidence,
|
| 100 |
+
"reason": f"Predicted {label} with {confidence}% confidence"
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
except Exception as e:
|
| 104 |
+
traceback.print_exc()
|
| 105 |
+
print(f"⚠️ Predictor error: {e}")
|
| 106 |
+
return {"category": "Non_SDG_Invalid", "confidence": 0, "reason": str(e)}
|
| 107 |
+
|
| 108 |
+
def get_model_name(self):
|
| 109 |
+
return "Custom CNN (mission_model.h5)"
|
utils/verdict.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Maps AI prediction to (Verdict, Message, SDG)
|
| 2 |
+
MISSION_MAP = {
|
| 3 |
+
"Planting": ("VERIFIED", "✅ Valid Planting Mission (SDG 13/15)", "SDG 13/15"),
|
| 4 |
+
"Recycling": ("VERIFIED", "✅ Valid Recycling Mission (SDG 12)", "SDG 12"),
|
| 5 |
+
"Cleanup": ("VERIFIED", "✅ Valid Cleanup Mission (SDG 6/14)", "SDG 6/14"),
|
| 6 |
+
"Donation": ("VERIFIED", "✅ Valid Donation Mission (SDG 1/2)", "SDG 1/2"),
|
| 7 |
+
"Health": ("VERIFIED", "✅ Valid Health & Wellness (SDG 3)", "SDG 3"),
|
| 8 |
+
"Education": ("VERIFIED", "✅ Valid Education Activity (SDG 4)", "SDG 4"),
|
| 9 |
+
"Energy": ("VERIFIED", "✅ Valid Energy Saving Action (SDG 7)", "SDG 7"),
|
| 10 |
+
"Sustainable_Cities": ("VERIFIED", "✅ Valid Sustainable Commute (SDG 11)", "SDG 11"),
|
| 11 |
+
"Support_Local": ("VERIFIED", "✅ Valid Support for Local Biz (SDG 8)", "SDG 8"),
|
| 12 |
+
"Non_SDG_Invalid": ("REJECTED", "⚠️ Image does not match any mission.", "N/A"),
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
def get_verdict(category, confidence_percent, threshold=55):
|
| 16 |
+
"""
|
| 17 |
+
Returns the final verdict response dictionary.
|
| 18 |
+
Requires a confidence of at least `threshold` for a VERIFIED verdict.
|
| 19 |
+
"""
|
| 20 |
+
verdict, message, sdg = MISSION_MAP.get(category, ("REJECTED", "⚠️ Unknown Image Category.", "N/A"))
|
| 21 |
+
|
| 22 |
+
is_verified = (verdict == "VERIFIED")
|
| 23 |
+
|
| 24 |
+
# If it's technically a valid category but confidence is too low
|
| 25 |
+
if is_verified and confidence_percent < threshold:
|
| 26 |
+
verdict = "UNCERTAIN"
|
| 27 |
+
message = f"❓ Unclear Image ({confidence_percent}%). Please take a clearer photo."
|
| 28 |
+
is_verified = False
|
| 29 |
+
sdg = "N/A"
|
| 30 |
+
|
| 31 |
+
source_check = "📸 Raw Picture" if is_verified else "🤖 AI Generated / Invalid"
|
| 32 |
+
|
| 33 |
+
return {
|
| 34 |
+
'prediction': category,
|
| 35 |
+
'confidence': f"{confidence_percent}%",
|
| 36 |
+
'confidence_raw': confidence_percent,
|
| 37 |
+
'verdict': verdict,
|
| 38 |
+
'message': message,
|
| 39 |
+
'is_verified': is_verified,
|
| 40 |
+
'sdg': sdg,
|
| 41 |
+
'source_check': source_check
|
| 42 |
+
}
|
venv/.gitignore
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Created by venv; see https://docs.python.org/3/library/venv.html
|
| 2 |
+
*
|
venv/Lib/site-packages/__pycache__/pylab.cpython-311.pyc
ADDED
|
Binary file (305 Bytes). View file
|
|
|
venv/Lib/site-packages/__pycache__/six.cpython-311.pyc
ADDED
|
Binary file (46.6 kB). View file
|
|
|
venv/Lib/site-packages/__pycache__/threadpoolctl.cpython-311.pyc
ADDED
|
Binary file (64.7 kB). View file
|
|
|
venv/Lib/site-packages/__pycache__/typing_extensions.cpython-311.pyc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8579e0349d3763ed1ffe4345807900a184ffc9f6dacf47f1b52f7d8104f1640b
|
| 3 |
+
size 179469
|
venv/Lib/site-packages/_distutils_hack/__init__.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# don't import any costly modules
|
| 2 |
+
import sys
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
is_pypy = '__pypy__' in sys.builtin_module_names
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def warn_distutils_present():
|
| 10 |
+
if 'distutils' not in sys.modules:
|
| 11 |
+
return
|
| 12 |
+
if is_pypy and sys.version_info < (3, 7):
|
| 13 |
+
# PyPy for 3.6 unconditionally imports distutils, so bypass the warning
|
| 14 |
+
# https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250
|
| 15 |
+
return
|
| 16 |
+
import warnings
|
| 17 |
+
|
| 18 |
+
warnings.warn(
|
| 19 |
+
"Distutils was imported before Setuptools, but importing Setuptools "
|
| 20 |
+
"also replaces the `distutils` module in `sys.modules`. This may lead "
|
| 21 |
+
"to undesirable behaviors or errors. To avoid these issues, avoid "
|
| 22 |
+
"using distutils directly, ensure that setuptools is installed in the "
|
| 23 |
+
"traditional way (e.g. not an editable install), and/or make sure "
|
| 24 |
+
"that setuptools is always imported before distutils."
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def clear_distutils():
|
| 29 |
+
if 'distutils' not in sys.modules:
|
| 30 |
+
return
|
| 31 |
+
import warnings
|
| 32 |
+
|
| 33 |
+
warnings.warn("Setuptools is replacing distutils.")
|
| 34 |
+
mods = [
|
| 35 |
+
name
|
| 36 |
+
for name in sys.modules
|
| 37 |
+
if name == "distutils" or name.startswith("distutils.")
|
| 38 |
+
]
|
| 39 |
+
for name in mods:
|
| 40 |
+
del sys.modules[name]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def enabled():
|
| 44 |
+
"""
|
| 45 |
+
Allow selection of distutils by environment variable.
|
| 46 |
+
"""
|
| 47 |
+
which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local')
|
| 48 |
+
return which == 'local'
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def ensure_local_distutils():
|
| 52 |
+
import importlib
|
| 53 |
+
|
| 54 |
+
clear_distutils()
|
| 55 |
+
|
| 56 |
+
# With the DistutilsMetaFinder in place,
|
| 57 |
+
# perform an import to cause distutils to be
|
| 58 |
+
# loaded from setuptools._distutils. Ref #2906.
|
| 59 |
+
with shim():
|
| 60 |
+
importlib.import_module('distutils')
|
| 61 |
+
|
| 62 |
+
# check that submodules load as expected
|
| 63 |
+
core = importlib.import_module('distutils.core')
|
| 64 |
+
assert '_distutils' in core.__file__, core.__file__
|
| 65 |
+
assert 'setuptools._distutils.log' not in sys.modules
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def do_override():
|
| 69 |
+
"""
|
| 70 |
+
Ensure that the local copy of distutils is preferred over stdlib.
|
| 71 |
+
|
| 72 |
+
See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
|
| 73 |
+
for more motivation.
|
| 74 |
+
"""
|
| 75 |
+
if enabled():
|
| 76 |
+
warn_distutils_present()
|
| 77 |
+
ensure_local_distutils()
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class _TrivialRe:
|
| 81 |
+
def __init__(self, *patterns):
|
| 82 |
+
self._patterns = patterns
|
| 83 |
+
|
| 84 |
+
def match(self, string):
|
| 85 |
+
return all(pat in string for pat in self._patterns)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class DistutilsMetaFinder:
|
| 89 |
+
def find_spec(self, fullname, path, target=None):
|
| 90 |
+
# optimization: only consider top level modules and those
|
| 91 |
+
# found in the CPython test suite.
|
| 92 |
+
if path is not None and not fullname.startswith('test.'):
|
| 93 |
+
return
|
| 94 |
+
|
| 95 |
+
method_name = 'spec_for_{fullname}'.format(**locals())
|
| 96 |
+
method = getattr(self, method_name, lambda: None)
|
| 97 |
+
return method()
|
| 98 |
+
|
| 99 |
+
def spec_for_distutils(self):
|
| 100 |
+
if self.is_cpython():
|
| 101 |
+
return
|
| 102 |
+
|
| 103 |
+
import importlib
|
| 104 |
+
import importlib.abc
|
| 105 |
+
import importlib.util
|
| 106 |
+
|
| 107 |
+
try:
|
| 108 |
+
mod = importlib.import_module('setuptools._distutils')
|
| 109 |
+
except Exception:
|
| 110 |
+
# There are a couple of cases where setuptools._distutils
|
| 111 |
+
# may not be present:
|
| 112 |
+
# - An older Setuptools without a local distutils is
|
| 113 |
+
# taking precedence. Ref #2957.
|
| 114 |
+
# - Path manipulation during sitecustomize removes
|
| 115 |
+
# setuptools from the path but only after the hook
|
| 116 |
+
# has been loaded. Ref #2980.
|
| 117 |
+
# In either case, fall back to stdlib behavior.
|
| 118 |
+
return
|
| 119 |
+
|
| 120 |
+
class DistutilsLoader(importlib.abc.Loader):
|
| 121 |
+
def create_module(self, spec):
|
| 122 |
+
mod.__name__ = 'distutils'
|
| 123 |
+
return mod
|
| 124 |
+
|
| 125 |
+
def exec_module(self, module):
|
| 126 |
+
pass
|
| 127 |
+
|
| 128 |
+
return importlib.util.spec_from_loader(
|
| 129 |
+
'distutils', DistutilsLoader(), origin=mod.__file__
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
@staticmethod
|
| 133 |
+
def is_cpython():
|
| 134 |
+
"""
|
| 135 |
+
Suppress supplying distutils for CPython (build and tests).
|
| 136 |
+
Ref #2965 and #3007.
|
| 137 |
+
"""
|
| 138 |
+
return os.path.isfile('pybuilddir.txt')
|
| 139 |
+
|
| 140 |
+
def spec_for_pip(self):
|
| 141 |
+
"""
|
| 142 |
+
Ensure stdlib distutils when running under pip.
|
| 143 |
+
See pypa/pip#8761 for rationale.
|
| 144 |
+
"""
|
| 145 |
+
if self.pip_imported_during_build():
|
| 146 |
+
return
|
| 147 |
+
clear_distutils()
|
| 148 |
+
self.spec_for_distutils = lambda: None
|
| 149 |
+
|
| 150 |
+
@classmethod
|
| 151 |
+
def pip_imported_during_build(cls):
|
| 152 |
+
"""
|
| 153 |
+
Detect if pip is being imported in a build script. Ref #2355.
|
| 154 |
+
"""
|
| 155 |
+
import traceback
|
| 156 |
+
|
| 157 |
+
return any(
|
| 158 |
+
cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None)
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
@staticmethod
|
| 162 |
+
def frame_file_is_setup(frame):
|
| 163 |
+
"""
|
| 164 |
+
Return True if the indicated frame suggests a setup.py file.
|
| 165 |
+
"""
|
| 166 |
+
# some frames may not have __file__ (#2940)
|
| 167 |
+
return frame.f_globals.get('__file__', '').endswith('setup.py')
|
| 168 |
+
|
| 169 |
+
def spec_for_sensitive_tests(self):
|
| 170 |
+
"""
|
| 171 |
+
Ensure stdlib distutils when running select tests under CPython.
|
| 172 |
+
|
| 173 |
+
python/cpython#91169
|
| 174 |
+
"""
|
| 175 |
+
clear_distutils()
|
| 176 |
+
self.spec_for_distutils = lambda: None
|
| 177 |
+
|
| 178 |
+
sensitive_tests = (
|
| 179 |
+
[
|
| 180 |
+
'test.test_distutils',
|
| 181 |
+
'test.test_peg_generator',
|
| 182 |
+
'test.test_importlib',
|
| 183 |
+
]
|
| 184 |
+
if sys.version_info < (3, 10)
|
| 185 |
+
else [
|
| 186 |
+
'test.test_distutils',
|
| 187 |
+
]
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
for name in DistutilsMetaFinder.sensitive_tests:
|
| 192 |
+
setattr(
|
| 193 |
+
DistutilsMetaFinder,
|
| 194 |
+
f'spec_for_{name}',
|
| 195 |
+
DistutilsMetaFinder.spec_for_sensitive_tests,
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
DISTUTILS_FINDER = DistutilsMetaFinder()
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def add_shim():
|
| 203 |
+
DISTUTILS_FINDER in sys.meta_path or insert_shim()
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
class shim:
|
| 207 |
+
def __enter__(self):
|
| 208 |
+
insert_shim()
|
| 209 |
+
|
| 210 |
+
def __exit__(self, exc, value, tb):
|
| 211 |
+
remove_shim()
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def insert_shim():
|
| 215 |
+
sys.meta_path.insert(0, DISTUTILS_FINDER)
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def remove_shim():
|
| 219 |
+
try:
|
| 220 |
+
sys.meta_path.remove(DISTUTILS_FINDER)
|
| 221 |
+
except ValueError:
|
| 222 |
+
pass
|
venv/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (11.2 kB). View file
|
|
|
venv/Lib/site-packages/_distutils_hack/__pycache__/override.cpython-311.pyc
ADDED
|
Binary file (324 Bytes). View file
|
|
|
venv/Lib/site-packages/_distutils_hack/override.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
__import__('_distutils_hack').do_override()
|
venv/Lib/site-packages/_multiprocess/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
#
|
| 3 |
+
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
|
| 4 |
+
# Copyright (c) 2022-2026 The Uncertainty Quantification Foundation.
|
| 5 |
+
# License: 3-clause BSD. The full license text is available at:
|
| 6 |
+
# - https://github.com/uqfoundation/multiprocess/blob/master/LICENSE
|
| 7 |
+
|
| 8 |
+
from _multiprocessing import *
|
venv/Lib/site-packages/_multiprocess/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (243 Bytes). View file
|
|
|
venv/Lib/site-packages/_yaml/__init__.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# This is a stub package designed to roughly emulate the _yaml
|
| 2 |
+
# extension module, which previously existed as a standalone module
|
| 3 |
+
# and has been moved into the `yaml` package namespace.
|
| 4 |
+
# It does not perfectly mimic its old counterpart, but should get
|
| 5 |
+
# close enough for anyone who's relying on it even when they shouldn't.
|
| 6 |
+
import yaml
|
| 7 |
+
|
| 8 |
+
# in some circumstances, the yaml module we imoprted may be from a different version, so we need
|
| 9 |
+
# to tread carefully when poking at it here (it may not have the attributes we expect)
|
| 10 |
+
if not getattr(yaml, '__with_libyaml__', False):
|
| 11 |
+
from sys import version_info
|
| 12 |
+
|
| 13 |
+
exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError
|
| 14 |
+
raise exc("No module named '_yaml'")
|
| 15 |
+
else:
|
| 16 |
+
from yaml._yaml import *
|
| 17 |
+
import warnings
|
| 18 |
+
warnings.warn(
|
| 19 |
+
'The _yaml extension module is now located at yaml._yaml'
|
| 20 |
+
' and its location is subject to change. To use the'
|
| 21 |
+
' LibYAML-based parser and emitter, import from `yaml`:'
|
| 22 |
+
' `from yaml import CLoader as Loader, CDumper as Dumper`.',
|
| 23 |
+
DeprecationWarning
|
| 24 |
+
)
|
| 25 |
+
del warnings
|
| 26 |
+
# Don't `del yaml` here because yaml is actually an existing
|
| 27 |
+
# namespace member of _yaml.
|
| 28 |
+
|
| 29 |
+
__name__ = '_yaml'
|
| 30 |
+
# If the module is top-level (i.e. not a part of any specific package)
|
| 31 |
+
# then the attribute should be set to ''.
|
| 32 |
+
# https://docs.python.org/3.8/library/types.html
|
| 33 |
+
__package__ = ''
|
venv/Lib/site-packages/_yaml/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (941 Bytes). View file
|
|
|
venv/Lib/site-packages/absl/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2017 The Abseil Authors.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
__version__ = '2.4.0'
|
venv/Lib/site-packages/absl/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (213 Bytes). View file
|
|
|
venv/Lib/site-packages/absl/__pycache__/app.cpython-311.pyc
ADDED
|
Binary file (22.8 kB). View file
|
|
|