diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..561cc7642c5916ce630e2637dc1fda3d0cac838d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +# Ignore datasets and heavy files to keep deployment fast +dataset/ +__pycache__/ +*.pyc +.env +.git +.vscode +.gemini +scripts/testing/mission_dataset_split.zip diff --git a/.env b/.env new file mode 100644 index 0000000000000000000000000000000000000000..e7cd2c3e1001a00547ac5ca046d3a6a467fbe0f2 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +OLLAMA_URL="http://localhost:11434" +OLLAMA_API_KEY="87b344ea09c540848abd777349d64466.PeZFbKB2Y03ddFyD4YXW0TpT" +OLLAMA_MODEL="llava-llama3" diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..d77a8fc06eb84a6ba2702dda53ecbe9cac3f4a1c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +venv/images/imagehash.png filter=lfs diff=lfs merge=lfs -text +venv/Lib/site-packages/__pycache__/typing_extensions.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text +venv/Lib/site-packages/absl/testing/__pycache__/absltest.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text +venv/Lib/site-packages/aiohttp/_http_parser.cp311-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text +venv/Lib/site-packages/aiohttp/_websocket/reader_c.cp311-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..1237ca394363e2fa1694849d9a31e8cd3570ffa9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# Use official Python runtime as a parent image +FROM python:3.11-slim + +# Set working directory in the container +WORKDIR /app + +# Install system dependencies (needed for OpenCV/Pillow if required) +RUN apt-get update && apt-get install -y \ + libgl1-mesa-glx \ + libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements file first (for caching) +COPY requirements.txt . + +# Install dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the application code +COPY . . + +# Expose port 7860 for Hugging Face Spaces +EXPOSE 7860 + +# Run the application using gunicorn for production +CMD ["gunicorn", "--bind", "0.0.0.0:7860", "app:app"] diff --git a/anticheat_hashes.json b/anticheat_hashes.json new file mode 100644 index 0000000000000000000000000000000000000000..930cf306e954b6fb25b86aa20bfb0218acb2fa78 --- /dev/null +++ b/anticheat_hashes.json @@ -0,0 +1 @@ +{"hashes": ["46256446866b53d9", "e0e42531b264e6d6", "e24a3c7a839cef24", "ccdac541d91b9237", "dcdd9c43634f189a", "b3ccd03f4cb11758", "f48b6995d4cf02b1", "525bf524bcb65692", "198964c9cb682cae", "eff64b09e12f8680"]} \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..dadf1cc1238b36b72b29b532adcd412ef665fd2a --- /dev/null +++ b/app.py @@ -0,0 +1,116 @@ +import os +import traceback +import logging +from flask import Flask, request, jsonify +from flask_cors import CORS +from werkzeug.utils import secure_filename +from dotenv import load_dotenv + +from utils.anticheat import AntiCheatEngine +from utils.predictor import Predictor +from utils.verdict import get_verdict + +load_dotenv() + +# Setup logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Initialize components +app = Flask(__name__) +CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True) + +ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'webp'} +app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 # Limit upload to 100MB + +logger.info("🧠 Loading the MISSION 17 AI Brain (Ollama Vision)...") +anticheat = AntiCheatEngine() +predictor = Predictor() + +def allowed_file(filename): + return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + +@app.after_request +def after_request(response): + response.headers.add('Access-Control-Allow-Origin', '*') + response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization') + response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS') + return response + +@app.route('/health', methods=['GET']) +def health(): + return jsonify({ + "status": "ok", + "model": predictor.get_model_name(), + "anticheat_hashes": anticheat.count() + }), 200 + +@app.route('/reset-anti-cheat', methods=['POST', 'GET']) +def reset_anti_cheat(): + count = anticheat.clear() + logger.info("πŸ›‘οΈ Anti-cheat hash database cleared!") + return jsonify({"message": "Anti-cheat hash database cleared!", "count": count}), 200 + +@app.route('/predict', methods=['POST']) +def predict(): + try: + # πŸ”’ CHECK 1: File Presence + if 'file' not in request.files: + return jsonify({'error': 'No file uploaded'}), 400 + + file = request.files['file'] + + # πŸ”’ CHECK 2: Empty File Detection (Bug Fix) + file.seek(0, os.SEEK_END) + if file.tell() == 0: + return jsonify({"error": "Processing failed: Empty file"}), 400 + file.seek(0) + + # πŸ”’ CHECK 3: Empty Filename + if file.filename == '': + return jsonify({'error': 'No selected file'}), 400 + + # πŸ”’ CHECK 4: File Type Validation + if not allowed_file(file.filename): + return jsonify({'error': 'Invalid file type. Only JPG/PNG allowed.'}), 400 + + # Read file bytes ONCE and reuse them + file_bytes = file.read() + + # 🎯 MODULE 11: Calculate Hash and Check for Cheaters + if anticheat.is_duplicate(file_bytes): + logger.warning("🚨 ANTI-CHEAT: Duplicate image detected!") + return jsonify({ + "status": "REJECTED", + "error": "Duplicate image detected. You cannot farm points!", + "prediction": "Anti-Cheat: Duplicate" + }), 400 + + # πŸ€– AI Vision Prediction via Ollama + logger.info("πŸ€– Sending image to Ollama Vision...") + ai_result = predictor.predict(file_bytes) + + category = ai_result.get('category', 'Non_SDG_Invalid') + confidence = ai_result.get('confidence', 0) + reason = ai_result.get('reason', '') + + # βš–οΈ Get formatted verdict based on AI output + verdict_response = get_verdict(category, confidence, threshold=55) + verdict_response['reason'] = reason + verdict_response['model'] = predictor.get_model_name() + + # Only register hash if the image was VERIFIED (save memory/prevent false positives on bad images) + if verdict_response['is_verified']: + anticheat.register(file_bytes) + logger.info(f"βœ… Unique verified image logged to anticheat.") + + return jsonify(verdict_response) + + except Exception as e: + logger.error(f"❌ Processing Error: {str(e)}") + traceback.print_exc() + return jsonify({'error': "Processing failed", 'detail': str(e)}), 500 + +if __name__ == '__main__': + # Hugging Face requires the app to listen on 0.0.0.0:7860 + app.run(host='0.0.0.0', port=7860, debug=False) \ No newline at end of file diff --git a/labels.txt b/labels.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6de29a85e3dbc653602ee4672f2c91b6596f598 --- /dev/null +++ b/labels.txt @@ -0,0 +1,10 @@ +Non_SDG_Invalid +SDG11_Sustainable_Cities +SDG12_Recycling +SDG13_15_Planting +SDG1_2_Donation +SDG3_Health_Wellbeing +SDG4_Quality_Education +SDG6_14_Cleanup +SDG7_Clean_Energy +SDG8_Support_Local diff --git a/mission_model.h5 b/mission_model.h5 new file mode 100644 index 0000000000000000000000000000000000000000..4bc6f67bae979a625326515801dbf8d41c9acee3 --- /dev/null +++ b/mission_model.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb0e4373bc7f9a3d1a9e9d7cacbc1686942d39e2c1debdf2c1b67e1767ef28bc +size 20967176 diff --git a/outputs/confusion_matrix.png b/outputs/confusion_matrix.png new file mode 100644 index 0000000000000000000000000000000000000000..0701a9e69f0c7c4b95a40b4f5baff415a89279a1 Binary files /dev/null and b/outputs/confusion_matrix.png differ diff --git a/outputs/debug_bias_mitigation.jpg b/outputs/debug_bias_mitigation.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3117a940738d2c85653e58227f826e4cc3b84766 Binary files /dev/null and b/outputs/debug_bias_mitigation.jpg differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c4b887e63b2f75429c310605db8012db999574ed --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +Flask==2.2.2 +Flask-Cors==3.0.10 +numpy +Pillow +imagehash +gunicorn +tensorflow +scikit-learn +matplotlib +seaborn diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..2419ad5b0a329db360c988461d3ca3d7c73d6eef --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +3.11.9 diff --git a/scripts/data_prep/balance_dataset.py b/scripts/data_prep/balance_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..a3a562dc20a2039795fcdc4922ef572ebcea28ac --- /dev/null +++ b/scripts/data_prep/balance_dataset.py @@ -0,0 +1,85 @@ +import os +import shutil +import random + +# πŸ‘‡ CONFIGURATION +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +DATASET_DIR = os.path.join(CURRENT_DIR, '..', '..', '..', 'dataset', 'mission_dataset') + +# Target: balance ALL classes to this count +# Set to None to auto-detect the median (safe default) +TARGET_COUNT = 500 + + +def count_classes(dataset_dir): + counts = {} + for class_name in os.listdir(dataset_dir): + class_path = os.path.join(dataset_dir, class_name) + if os.path.isdir(class_path): + images = [ + f for f in os.listdir(class_path) + if f.lower().endswith(('.jpg', '.jpeg', '.png', '.webp')) + ] + counts[class_name] = len(images) + return counts + + +def balance_dataset(): + print(f"βš–οΈ Balancing ALL classes in: {DATASET_DIR}\n") + + if not os.path.exists(DATASET_DIR): + print("❌ Error: Dataset folder not found.") + return + + counts = count_classes(DATASET_DIR) + + if not counts: + print("❌ No class folders found.") + return + + # Determine target + target = TARGET_COUNT + if target is None: + sorted_counts = sorted(counts.values()) + target = sorted_counts[len(sorted_counts) // 2] # median + print(f"πŸ“Š Auto-detected target (median): {target} images per class\n") + else: + print(f"πŸ“Š Target: {target} images per class\n") + + print(f"{'Class':<35} {'Before':>8} {'After':>8} {'Action'}") + print("-" * 65) + + for class_name, count in sorted(counts.items()): + class_path = os.path.join(DATASET_DIR, class_name) + images = [ + f for f in os.listdir(class_path) + if f.lower().endswith(('.jpg', '.jpeg', '.png', '.webp')) + ] + + if count > target: + # Trim to target β€” shuffle first to keep a random selection + random.shuffle(images) + excess = images[target:] + for img in excess: + os.remove(os.path.join(class_path, img)) + after = target + action = f"βœ‚οΈ Trimmed -{len(excess)}" + + elif count < target: + # Class is under-represented β€” warn user to add more images + after = count + action = f"⚠️ Under-represented (need +{target - count} more images)" + + else: + after = count + action = "βœ… OK" + + print(f"{class_name:<35} {count:>8} {after:>8} {action}") + + print("\n✨ Balancing complete! Now run train_ai.py to retrain the model.") + print(" πŸ’‘ TIP: For under-represented classes, collect more real photos") + print(" or use Google Images to download additional training data.") + + +if __name__ == "__main__": + balance_dataset() diff --git a/scripts/data_prep/collect_data.py b/scripts/data_prep/collect_data.py new file mode 100644 index 0000000000000000000000000000000000000000..1ba66a2206ff7db0c5382e1e740fa899b3ca2df2 --- /dev/null +++ b/scripts/data_prep/collect_data.py @@ -0,0 +1,221 @@ +from bing_image_downloader import downloader +import os +import shutil +import math + +# πŸ‘‡ CONFIGURATION +BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', 'dataset', 'mission_dataset') + +# Target images per class β€” everything will be brought up to this number +TARGET = 500 + +# ───────────────────────────────────────────────────────────────────────────── +# CLASS DEFINITIONS +# Each class has: +# "current": how many images it already has +# "terms": list of 5 search terms to download from +# +# limit_per_term = ceil((TARGET - current) / len(terms)) +# Classes already at TARGET are automatically skipped. +# ───────────────────────────────────────────────────────────────────────────── +CLASSES = { + + # 🍱 SDG 1/2: Donation β€” WEAKEST CLASS (150 β†’ 500, needs +350, 70/term) + "SDG1_2_Donation": { + "current": 150, + "terms": [ + "people donating food to community", + "clothes donation box charity", + "feeding program volunteers serving food", + "grocery donation drive event", + "charity relief goods distribution" + ] + }, + + # πŸ–οΈ SDG 6/14: Cleanup β€” WEAKEST CLASS (150 β†’ 500, needs +350, 70/term) + "SDG6_14_Cleanup": { + "current": 150, + "terms": [ + "beach cleanup volunteers collecting trash", + "river cleanup community activity", + "coastal cleanup garbage bags collected", + "people picking up litter shoreline", + "estero creek waterway cleanup" + ] + }, + + # πŸ“š SDG 4: Education (253 β†’ 500, needs +247, 50/term) + "SDG4_Quality_Education": { + "current": 253, + "terms": [ + "student reading open book", + "teacher writing on whiteboard classroom", + "group study session library", + "hand writing notes in notebook", + "child using educational tablet learning" + ] + }, + + # πŸƒ SDG 3: Health (262 β†’ 500, needs +238, 48/term) + "SDG3_Health_Wellbeing": { + "current": 262, + "terms": [ + "people jogging in park", + "group yoga session outdoors", + "eating fresh fruit salad bowl", + "drinking glass of water healthy", + "washing hands with soap hygiene" + ] + }, + + # πŸ›οΈ SDG 8: Support Local (267 β†’ 500, needs +233, 47/term) + "SDG8_Support_Local": { + "current": 267, + "terms": [ + "buying from street food vendor", + "shopping at local farmers market", + "artisan crafting handmade goods", + "small bakery local shop front", + "supporting small business community" + ] + }, + + # 🌱 SDG 13/15: Planting (270 β†’ 500, needs +230, 46/term) + "SDG13_15_Planting": { + "current": 270, + "terms": [ + "person planting tree sapling", + "community tree planting activity", + "garden seedling transplanting soil", + "plant growing hands holding soil", + "reforestation volunteers planting trees" + ] + }, + + # πŸ™οΈ SDG 11: Sustainable Cities (275 β†’ 500, needs +225, 45/term) + "SDG11_Sustainable_Cities": { + "current": 275, + "terms": [ + "riding bicycle on city road", + "passengers inside public city bus", + "waiting at train station platform", + "walking on pedestrian crossing street", + "segregated bike lane urban city" + ] + }, + + # ⚑ SDG 7: Clean Energy (277 β†’ 500, needs +223, 45/term) + "SDG7_Clean_Energy": { + "current": 277, + "terms": [ + "solar panels on house roof", + "hand turning off light switch", + "electric vehicle charging station", + "wind turbine farm landscape", + "modern led light bulb energy saving" + ] + }, + + # 🚫 Non-SDG Invalid (430 β†’ 500, needs +70, 14/term) + "Non_SDG_Invalid": { + "current": 430, + "terms": [ + "random indoor selfie photo", + "luxury sports car fast", + "video game screenshot gaming", + "cat sleeping on sofa", + "abstract digital art wallpaper" + ] + }, + + # ♻️ SDG 12: Recycling β€” ALREADY AT TARGET (500), will be skipped + "SDG12_Recycling": { + "current": 500, + "terms": [] + }, +} + +# ───────────────────────────────────────────────────────────────────────────── + +print(f"πŸš€ Smart Data Collection β€” Target: {TARGET} images per class") +print(f" Dataset path: {BASE_DIR}\n") + +if not os.path.exists(BASE_DIR): + print(f"❌ ERROR: Could not find '{BASE_DIR}'. Check your folder structure.") + exit() + +total_added = 0 + +for category, info in CLASSES.items(): + current = info["current"] + terms = info["terms"] + needed = TARGET - current + + # Skip classes already at or above target + if needed <= 0: + print(f"⏭️ [{category}] already at {current}/{TARGET} β€” SKIPPED\n") + continue + + limit_per_term = math.ceil(needed / len(terms)) + target_dir = os.path.join(BASE_DIR, category) + os.makedirs(target_dir, exist_ok=True) + + print(f"πŸ“‚ [{category}]") + print(f" {current} β†’ {TARGET} | need +{needed} | {limit_per_term} images/term") + + category_added = 0 + + for term in terms: + print(f" πŸ” '{term}' ({limit_per_term} images)...") + try: + downloader.download( + term, + limit=limit_per_term, + output_dir="temp_downloads", + adult_filter_off=True, + force_replace=False, + timeout=10, + verbose=False + ) + + source_folder = os.path.join("temp_downloads", term) + if os.path.exists(source_folder): + files = os.listdir(source_folder) + moved = 0 + for file in files: + old_path = os.path.join(source_folder, file) + if not os.path.isfile(old_path): + continue + clean_term = term.replace(" ", "_") + new_filename = f"{clean_term}_{file}" + new_path = os.path.join(target_dir, new_filename) + if os.path.exists(new_path): + continue # Skip duplicates + try: + shutil.move(old_path, new_path) + moved += 1 + except Exception: + pass + print(f" βœ… +{moved} images") + category_added += moved + + except Exception as e: + print(f" ⚠️ Skipped '{term}': {e}") + + # Clean up temp after each term + if os.path.exists("temp_downloads"): + try: + shutil.rmtree("temp_downloads") + except Exception: + pass + + new_total = current + category_added + print(f" πŸ“Š Result: {current} β†’ {new_total} images (+{category_added})\n") + total_added += category_added + +print("=" * 55) +print(f"✨ Done! Total new images added: {total_added}") +print(f" All classes should now be near {TARGET} images each.") +print("\n Next steps:") +print(" 1. python train_ai.py ← retrain the model") +print(" 2. python evaluate_model.py ← check accuracy") \ No newline at end of file diff --git a/scripts/data_prep/count_dataset.py b/scripts/data_prep/count_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..8051655a0ec3ed00ddc3fb0835905a17b6dbd13a --- /dev/null +++ b/scripts/data_prep/count_dataset.py @@ -0,0 +1,43 @@ +import os + +def count_images(): + # Define the path to the dataset + # Based on your other scripts, it is in ../dataset/mission_dataset + base_dir = os.path.dirname(os.path.abspath(__file__)) + dataset_dir = os.path.join(base_dir, '..', '..', '..', 'dataset', 'mission_dataset') + + print(f"πŸ“Š Checking dataset at: {os.path.abspath(dataset_dir)}\n") + + if not os.path.exists(dataset_dir): + print(f"❌ Error: Folder not found. Have you run 'organize_dataset.py'?") + return + + total_images = 0 + + # Get all subfolders (classes) + try: + classes = [d for d in os.listdir(dataset_dir) if os.path.isdir(os.path.join(dataset_dir, d))] + classes.sort() + except Exception as e: + print(f"❌ Error reading directory: {e}") + return + + print(f"{'CLASS NAME':<35} | {'COUNT':<10} | {'STATUS'}") + print("-" * 50) + + for class_name in classes: + class_path = os.path.join(dataset_dir, class_name) + # Count files that look like images + images = [f for f in os.listdir(class_path) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp', '.bmp'))] + count = len(images) + + status = "βœ… Ready" if count >= 100 else "⚠️ Low Data" if count > 0 else "❌ Empty" + + print(f"{class_name:<35} | {count:<10} | {status}") + total_images += count + + print("-" * 50) + print(f"βœ… TOTAL IMAGES: {total_images}") + +if __name__ == "__main__": + count_images() \ No newline at end of file diff --git a/scripts/data_prep/fix_dataset.py b/scripts/data_prep/fix_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..ffafa37ba4d47b68c37f2060a8e441757acf847c --- /dev/null +++ b/scripts/data_prep/fix_dataset.py @@ -0,0 +1,28 @@ +import os +import shutil + +# Define paths +import os +base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', 'dataset', 'garbage_classification') +old_planting = os.path.join(base_dir, "planting") +new_planting = os.path.join(base_dir, "SDG13_15_Planting") + +# Create new folder if it doesn't exist +if not os.path.exists(new_planting): + os.makedirs(new_planting) + +# Move files from Old -> New +if os.path.exists(old_planting): + print(f"πŸ”„ Moving files from '{old_planting}' to '{new_planting}'...") + files = os.listdir(old_planting) + for file in files: + old_path = os.path.join(old_planting, file) + new_path = os.path.join(new_planting, f"old_{file}") # Rename to avoid conflicts + shutil.move(old_path, new_path) + + # Delete the empty old folder + os.rmdir(old_planting) + print("βœ… Successfully merged folders!") + print("πŸ—‘οΈ Deleted old 'planting' folder.") +else: + print("⚠️ Old 'planting' folder not found. Already merged?") \ No newline at end of file diff --git a/scripts/data_prep/organize_dataset.py b/scripts/data_prep/organize_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..17cc6f8aeb05c7c79c647359bcac8834e6c5234e --- /dev/null +++ b/scripts/data_prep/organize_dataset.py @@ -0,0 +1,91 @@ +import os +import shutil + +# πŸ‘‡ CONFIGURATION +# Current script directory +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +# The main project dataset folder (../dataset) +BASE_DIR = os.path.join(CURRENT_DIR, '..', '..', '..', 'dataset') + +# 1. The "Correct" Destination +FINAL_DEST = os.path.join(BASE_DIR, "mission_dataset") + +# 2. The "Old" Kaggle Dataset +OLD_GARBAGE_DIR = os.path.join(BASE_DIR, "garbage_classification") + +# 3. The "Misplaced" Downloads (if any) inside mission17-ai/dataset +MISPLACED_DIR = os.path.join(CURRENT_DIR, '..', '..', "dataset", "mission_dataset") + +# Map OLD folders to NEW SDG destinations +# We are putting ALL waste items into SDG12 (Responsible Consumption & Production) +MOVES = { + "SDG12_Recycling": [ + "battery", "brown-glass", "cardboard", + "clothes", "green-glass", "metal", "paper", + "plastic", "shoes", "white-glass" + ], + "Non_SDG_Invalid": [ + "trash", "biological" + ] +} + +def organize_files(): + print(f"πŸ“¦ Organizing dataset...") + + # Ensure destination exists + if not os.path.exists(FINAL_DEST): + os.makedirs(FINAL_DEST) + print(f" βœ… Created '{FINAL_DEST}'") + + # --- STEP 1: Merge Kaggle Data --- + if os.path.exists(OLD_GARBAGE_DIR): + print(f" πŸ”„ Merging 'garbage_classification'...") + for dest_folder, source_folders in MOVES.items(): + dest_path = os.path.join(FINAL_DEST, dest_folder) + if not os.path.exists(dest_path): os.makedirs(dest_path) + + for folder in source_folders: + src_path = os.path.join(OLD_GARBAGE_DIR, folder) + if os.path.exists(src_path): + # Move files + for file in os.listdir(src_path): + try: + shutil.move(os.path.join(src_path, file), os.path.join(dest_path, f"{folder}_{file}")) + except Exception: pass + # Remove empty folder + try: + os.rmdir(src_path) + except: pass + + # Try to remove root garbage dir + try: os.rmdir(OLD_GARBAGE_DIR) + except: pass + print(" βœ… Kaggle data merged.") + + # --- STEP 2: Fix Misplaced Downloads --- + if os.path.exists(MISPLACED_DIR): + print(f" ⚠️ Found misplaced images in '{MISPLACED_DIR}'. Moving them...") + for category in os.listdir(MISPLACED_DIR): + src = os.path.join(MISPLACED_DIR, category) + dest = os.path.join(FINAL_DEST, category) + + if os.path.isdir(src): + if not os.path.exists(dest): os.makedirs(dest) + for file in os.listdir(src): + try: + shutil.move(os.path.join(src, file), os.path.join(dest, file)) + except: pass + try: os.rmdir(src) + except: pass + + # Cleanup parent 'dataset' in mission17-ai if empty + try: + os.rmdir(MISPLACED_DIR) + os.rmdir(os.path.join(CURRENT_DIR, '..', '..', "dataset")) + except: pass + print(" βœ… Misplaced images moved to correct folder.") + + print(f"\n✨ SUCCESS! Dataset is ready at: {FINAL_DEST}") + +if __name__ == "__main__": + organize_files() \ No newline at end of file diff --git a/scripts/testing/split_dataset.py b/scripts/testing/split_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..63682fddba1b1d034c66ee731d8d85f6ce87358f --- /dev/null +++ b/scripts/testing/split_dataset.py @@ -0,0 +1,73 @@ +import os +import shutil +import random +from pathlib import Path + +# --- CONFIGURATION --- +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +DATASET_DIR = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset') +OUTPUT_DIR = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset_split') + +# Split ratios +TRAIN_RATIO = 0.80 +TEST_RATIO = 0.20 + +def split_dataset(): + """ + Randomly splits the dataset into train/ and test/ folders. + This prevents 'Data Leakage' so your evaluate_model.py tests on truly unseen images. + """ + print(f"πŸš€ Splitting dataset: {DATASET_DIR}") + print(f" Outputting to: {OUTPUT_DIR}") + + if not os.path.exists(DATASET_DIR): + print(f"❌ ERROR: Dataset not found at {DATASET_DIR}") + return + + # Create output directories + train_dir = os.path.join(OUTPUT_DIR, 'train') + test_dir = os.path.join(OUTPUT_DIR, 'test') + + os.makedirs(train_dir, exist_ok=True) + os.makedirs(test_dir, exist_ok=True) + + classes = [d for d in os.listdir(DATASET_DIR) if os.path.isdir(os.path.join(DATASET_DIR, d))] + + total_moved = 0 + + for class_name in classes: + class_path = os.path.join(DATASET_DIR, class_name) + images = [f for f in os.listdir(class_path) if f.lower().endswith(('.png', '.jpg', '.jpeg', '.webp'))] + + # Shuffle images randomly + random.shuffle(images) + + # Calculate split index + split_idx = int(len(images) * TRAIN_RATIO) + + train_images = images[:split_idx] + test_images = images[split_idx:] + + # Create class folders in train/ and test/ + os.makedirs(os.path.join(train_dir, class_name), exist_ok=True) + os.makedirs(os.path.join(test_dir, class_name), exist_ok=True) + + print(f"πŸ“ [{class_name}] Total: {len(images)} -> Train: {len(train_images)}, Test: {len(test_images)}") + + # Copy files + for img in train_images: + shutil.copy2(os.path.join(class_path, img), os.path.join(train_dir, class_name, img)) + total_moved += 1 + + for img in test_images: + shutil.copy2(os.path.join(class_path, img), os.path.join(test_dir, class_name, img)) + total_moved += 1 + + print("=" * 50) + print(f"βœ… Dataset split complete! {total_moved} images processed.") + print(" Next steps:") + print(" 1. Check the new folder 'mission_dataset_split'") + print(" 2. Run train_ai_v2.py (which now points to this new folder)") + +if __name__ == '__main__': + split_dataset() diff --git a/scripts/testing/test_duplicate.py b/scripts/testing/test_duplicate.py new file mode 100644 index 0000000000000000000000000000000000000000..60e94f3f75aca8bb5a604fea401a8156ce885ac8 --- /dev/null +++ b/scripts/testing/test_duplicate.py @@ -0,0 +1,16 @@ +import requests + +url = "http://localhost:5000/predict" +file_path = "debug_bias_mitigation.jpg" + +print("--- First Upload ---") +with open(file_path, "rb") as f: + r = requests.post(url, files={"file": f}) + print(f"Status: {r.status_code}") + print(f"Response: {r.text}") + +print("\n--- Second Upload ---") +with open(file_path, "rb") as f: + r = requests.post(url, files={"file": f}) + print(f"Status: {r.status_code}") + print(f"Response: {r.text}") diff --git a/scripts/testing/test_predictor.py b/scripts/testing/test_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..9ee066a47d5cfe1f42370ecd49d0f05b48b851bc --- /dev/null +++ b/scripts/testing/test_predictor.py @@ -0,0 +1,27 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +from utils.predictor import Predictor + + +def test_ollama(): + print("Testing Ollama Predictor...") + p = Predictor() + print(f"Model configured: {p.get_model_name()}") + + # We will just see if we can reach the API + # Since we need an image, let's create a dummy 1x1 black pixel image + import io + from PIL import Image + + img = Image.new('RGB', (10, 10), color = 'black') + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format='PNG') + img_bytes = img_byte_arr.getvalue() + + print("Sending dummy image to Ollama...") + res = p.predict(img_bytes) + print("Response:", res) + +if __name__ == "__main__": + test_ollama() diff --git a/scripts/testing/test_server.py b/scripts/testing/test_server.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/scripts/testing/test_upload.html b/scripts/testing/test_upload.html new file mode 100644 index 0000000000000000000000000000000000000000..698137fe1c0b5910efa82948960245215e272b7f --- /dev/null +++ b/scripts/testing/test_upload.html @@ -0,0 +1,61 @@ + + + + + Mission 17 AI Scanner + + + +
+

πŸ€– Mission 17 AI Scanner

+

Upload a photo to verify your mission!

+ + +
+ + +
+
+ + + + diff --git a/scripts/training/download_pangasinan.py b/scripts/training/download_pangasinan.py new file mode 100644 index 0000000000000000000000000000000000000000..24cf21e58dd18585caa1e0ca25b519a9f4922791 --- /dev/null +++ b/scripts/training/download_pangasinan.py @@ -0,0 +1,46 @@ +import json +import os +from datasets import load_dataset + +print("Downloading dataset leklek02/pangasinan...") +ds = load_dataset("leklek02/pangasinan", split="train") + +print(f"Total rows downloaded: {len(ds)}") + +# 1. Create Few-Shot Examples (Top 100 to keep the prompt size reasonable) +examples = [] +for i in range(min(100, len(ds))): + row = ds[i] + user_text = str(row['instruction']) + if row.get('input') and str(row['input']).strip(): + user_text += "\n" + str(row['input']) + + examples.append({ + "User": user_text, + "Bot": str(row['output']) + }) + +backend_path = r"c:\Users\Kurt Perez\mission17\mission17-backend\utils\pangasinan_examples.json" +os.makedirs(os.path.dirname(backend_path), exist_ok=True) +with open(backend_path, "w", encoding="utf-8") as f: + json.dump(examples, f, indent=2, ensure_ascii=False) +print(f"Saved 100 examples to {backend_path} for immediate use in Chatbot.") + +# 2. Create Gemini Tuning JSONL (All rows for future Fine-Tuning) +tuning_path = r"c:\Users\Kurt Perez\mission17\dataset\gemini_pangasinan_tuning.jsonl" +os.makedirs(os.path.dirname(tuning_path), exist_ok=True) +with open(tuning_path, "w", encoding="utf-8") as f: + for row in ds: + user_text = str(row['instruction']) + if row.get('input') and str(row['input']).strip(): + user_text += "\n" + str(row['input']) + + jsonl_obj = { + "contents": [ + {"role": "user", "parts": [{"text": user_text}]}, + {"role": "model", "parts": [{"text": str(row['output'])}]} + ] + } + f.write(json.dumps(jsonl_obj, ensure_ascii=False) + "\n") + +print(f"Saved full tuning dataset to {tuning_path} (Upload this to Google AI Studio to fine-tune Gemini!)") diff --git a/scripts/training/evaluate_model.py b/scripts/training/evaluate_model.py new file mode 100644 index 0000000000000000000000000000000000000000..b69313c823ede335a93b9e74079b47ebebb9f83f --- /dev/null +++ b/scripts/training/evaluate_model.py @@ -0,0 +1,57 @@ +import numpy as np +import tensorflow as tf +from tensorflow.keras.preprocessing.image import ImageDataGenerator +from tensorflow.keras.applications.efficientnet import preprocess_input +from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix +import matplotlib.pyplot as plt +import seaborn as sns +import os +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +print("⏳ Loading AI Model...") +# πŸ‘‡ Ensure this is your correct model name! +model = tf.keras.models.load_model(os.path.join(BASE_DIR, '..', '..', 'mission_model.h5')) + +print("πŸ“ Loading Test Dataset...") +# πŸ‘‡ Pointing to the new TEST split folder +test_dir = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset_split', 'test') + +test_datagen = ImageDataGenerator(preprocessing_function=preprocess_input) +test_generator = test_datagen.flow_from_directory( + test_dir, + target_size=(224, 224), + batch_size=32, + class_mode='categorical', + shuffle=False +) + +print("πŸ€– Running Predictions (This may take a minute)...") +Y_pred = model.predict(test_generator) +y_pred_classes = np.argmax(Y_pred, axis=1) # πŸ‘ˆ FIXED: Grabs the top prediction per image +y_true = test_generator.classes + +print("\n" + "="*50) +print("πŸ† CAPSTONE AI PERFORMANCE METRICS πŸ†") +print("="*50) + +# πŸ‘ˆ FIXED: Added average='weighted' to handle all 10 classes correctly +accuracy = accuracy_score(y_true, y_pred_classes) +precision = precision_score(y_true, y_pred_classes, average='weighted', zero_division=0) +recall = recall_score(y_true, y_pred_classes, average='weighted', zero_division=0) +f1 = f1_score(y_true, y_pred_classes, average='weighted', zero_division=0) + +print(f"βœ… Accuracy: {accuracy * 100:.2f}%") +print(f"🎯 Precision: {precision * 100:.2f}%") +print(f"πŸ” Recall: {recall * 100:.2f}%") +print(f"βš–οΈ F1-Score: {f1 * 100:.2f}%") +print("="*50) + +# Make the confusion matrix chart larger to fit 10 classes +cm = confusion_matrix(y_true, y_pred_classes) +plt.figure(figsize=(10, 8)) +sns.heatmap(cm, annot=True, fmt='d', cmap='Blues') +plt.title('AI Confusion Matrix (10 Classes)') +plt.ylabel('Actual Image Class') +plt.xlabel('AI Prediction') +plt.savefig(os.path.join(BASE_DIR, '..', '..', 'outputs', 'confusion_matrix.png')) +print("\nπŸ“Š Saved 'confusion_matrix.png' to your outputs folder. Put this in your presentation!") \ No newline at end of file diff --git a/scripts/training/train_ai.py b/scripts/training/train_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..68e43b7f22b3713e40c2e6af39f31c5eaf02a5b9 --- /dev/null +++ b/scripts/training/train_ai.py @@ -0,0 +1,209 @@ +import os +import tensorflow as tf +from tensorflow.keras.preprocessing.image import ImageDataGenerator +from tensorflow.keras.applications import EfficientNetB0 +from tensorflow.keras.applications.efficientnet import preprocess_input +from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout, BatchNormalization +from tensorflow.keras.models import Model +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint + +# --- CONFIGURATION --- +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +DATASET_DIR = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset') +MODEL_SAVE_PATH = os.path.join(BASE_DIR, '..', '..', 'mission_model.h5') +LABELS_SAVE_PATH = os.path.join(BASE_DIR, '..', '..', 'labels.txt') + +# Hyperparameters +IMG_SIZE = (224, 224) +BATCH_SIZE = 32 +EPOCHS_INITIAL = 25 # Phase 1: Train top layers only +EPOCHS_FINETUNE = 15 # Phase 2: Fine-tune top base layers +LR_INITIAL = 1e-3 # Higher LR for initial training +LR_FINETUNE = 1e-5 # Much lower LR for fine-tuning (prevents forgetting) +FINETUNE_FROM_LAYER = 150 # Unfreeze EfficientNetB0 from this layer onwards + +def build_generators(): + """Create training and validation data generators with strong augmentation.""" + print("πŸ“Έ Preparing Image Generators with Strong Augmentation...") + + # πŸ”₯ EfficientNetB0 has its own internal preprocessing β€” do NOT use rescale=1./255! + # Using preprocess_input correctly scales raw 0-255 pixel values for EfficientNet. + train_datagen = ImageDataGenerator( + preprocessing_function=preprocess_input, # βœ… EfficientNetB0-compatible + rotation_range=30, + width_shift_range=0.2, + height_shift_range=0.2, + horizontal_flip=True, + brightness_range=[0.7, 1.3], + zoom_range=0.2, + shear_range=0.1, + channel_shift_range=20.0, + fill_mode='nearest', + validation_split=0.2 + ) + + # Validation: only preprocess_input, NO augmentation + val_datagen = ImageDataGenerator( + preprocessing_function=preprocess_input, # βœ… Must match training + validation_split=0.2 + ) + + train_generator = train_datagen.flow_from_directory( + DATASET_DIR, + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + subset='training', + shuffle=True + ) + + validation_generator = val_datagen.flow_from_directory( + DATASET_DIR, + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + subset='validation', + shuffle=False + ) + + return train_generator, validation_generator + + +def build_model(num_classes): + """ + Build model using EfficientNetB0 (more accurate than MobileNetV2). + Phase 1 starts with all base layers FROZEN β€” only top layers train first. + """ + print("🧠 Building Model (EfficientNetB0 β€” upgraded from MobileNetV2)...") + + base_model = EfficientNetB0( + weights='imagenet', + include_top=False, + input_shape=IMG_SIZE + (3,) + ) + base_model.trainable = False # Freeze all base layers for Phase 1 + + x = base_model.output + x = GlobalAveragePooling2D()(x) + x = BatchNormalization()(x) # ✨ NEW β€” stabilizes training + x = Dropout(0.3)(x) # was 0.2 β€” slightly stronger regularization + x = Dense(256, activation='relu')(x) # ✨ NEW β€” extra dense layer for richer features + x = Dropout(0.2)(x) + predictions = Dense(num_classes, activation='softmax')(x) + + model = Model(inputs=base_model.input, outputs=predictions) + return model, base_model + + +def get_callbacks(phase_name): + """Smart callbacks: stop early if no improvement, reduce LR on plateau.""" + return [ + EarlyStopping( + monitor='val_accuracy', + patience=5, # Stop if no improvement for 5 epochs + restore_best_weights=True, + verbose=1 + ), + ReduceLROnPlateau( + monitor='val_loss', + factor=0.5, # Halve LR if stuck + patience=3, + min_lr=1e-7, + verbose=1 + ), + ModelCheckpoint( + filepath=MODEL_SAVE_PATH, + monitor='val_accuracy', + save_best_only=True, # Always keep the best checkpoint + verbose=1 + ) + ] + + +def train_brain(): + print("πŸš€ Initializing Mission 17 AI Training (Enhanced)...") + + # 1. CHECK DATASET + if not os.path.exists(DATASET_DIR): + print(f"❌ ERROR: Dataset not found at {DATASET_DIR}") + return + + # 2. BUILD GENERATORS + try: + train_generator, validation_generator = build_generators() + except Exception as e: + print(f"❌ Error loading data: {e}") + return + + if train_generator.samples == 0: + print("❌ No images found! Check your dataset structure.") + return + + # 3. SAVE LABELS + class_names = list(train_generator.class_indices.keys()) + print(f"🏷️ Classes Detected: {class_names}") + with open(LABELS_SAVE_PATH, 'w') as f: + for name in class_names: + f.write(name + '\n') + print(f"βœ… Labels saved to {LABELS_SAVE_PATH}") + + num_classes = len(class_names) + + # 4. BUILD MODEL + model, base_model = build_model(num_classes) + + # ════════════════════════════════════════════ + # PHASE 1: Train top layers only (fast) + # ════════════════════════════════════════════ + print("\n" + "="*50) + print("πŸ‹οΈ PHASE 1: Training Top Layers (Base Frozen)") + print("="*50) + + model.compile( + optimizer=Adam(learning_rate=LR_INITIAL), + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + model.fit( + train_generator, + epochs=EPOCHS_INITIAL, + validation_data=validation_generator, + callbacks=get_callbacks('phase1') + ) + + # ════════════════════════════════════════════ + # PHASE 2: Fine-tune top layers of base model + # ════════════════════════════════════════════ + print("\n" + "="*50) + print("πŸ”¬ PHASE 2: Fine-Tuning Top Base Layers") + print(f" Unfreezing EfficientNetB0 from layer {FINETUNE_FROM_LAYER}+") + print("="*50) + + base_model.trainable = True + + # Only unfreeze layers AFTER FINETUNE_FROM_LAYER β€” keep earlier layers frozen + for layer in base_model.layers[:FINETUNE_FROM_LAYER]: + layer.trainable = False + + # CRITICAL: Recompile with much lower LR to avoid destroying pre-trained weights + model.compile( + optimizer=Adam(learning_rate=LR_FINETUNE), + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + model.fit( + train_generator, + epochs=EPOCHS_FINETUNE, + validation_data=validation_generator, + callbacks=get_callbacks('phase2') + ) + + print(f"\nβœ… Training complete! Best model saved to {MODEL_SAVE_PATH}") + print(" Run evaluate_model.py to check accuracy metrics & confusion matrix.") + + +if __name__ == '__main__': + train_brain() \ No newline at end of file diff --git a/scripts/training/train_ai_v2.py b/scripts/training/train_ai_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..c52ccd4fec2f813b44cffb09e543a4a56803ee17 --- /dev/null +++ b/scripts/training/train_ai_v2.py @@ -0,0 +1,170 @@ +import os +import numpy as np +import tensorflow as tf +from tensorflow.keras.preprocessing.image import ImageDataGenerator +from tensorflow.keras.applications import EfficientNetB0 +from tensorflow.keras.applications.efficientnet import preprocess_input +from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout, BatchNormalization +from tensorflow.keras.models import Model +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint +from sklearn.utils.class_weight import compute_class_weight + +# --- CONFIGURATION --- +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +# IMPORTANT: Pointing to the new split dataset folder +DATASET_DIR = os.path.join(BASE_DIR, '..', '..', '..', 'dataset', 'mission_dataset_split', 'train') +MODEL_SAVE_PATH = os.path.join(BASE_DIR, '..', '..', 'mission_model.h5') +LABELS_SAVE_PATH = os.path.join(BASE_DIR, '..', '..', 'labels.txt') + +IMG_SIZE = (224, 224) +BATCH_SIZE = 32 +EPOCHS_INITIAL = 20 +EPOCHS_FINETUNE = 15 +LR_INITIAL = 1e-3 +LR_FINETUNE = 1e-5 + +def build_generators(): + print("πŸ“Έ Preparing Image Generators...") + + train_datagen = ImageDataGenerator( + preprocessing_function=preprocess_input, + rotation_range=30, + width_shift_range=0.2, + height_shift_range=0.2, + horizontal_flip=True, + brightness_range=[0.7, 1.3], + zoom_range=0.2, + validation_split=0.2 # 20% of the train/ folder becomes validation + ) + + train_generator = train_datagen.flow_from_directory( + DATASET_DIR, + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + subset='training', + shuffle=True + ) + + validation_generator = train_datagen.flow_from_directory( + DATASET_DIR, + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + subset='validation', + shuffle=False + ) + + return train_generator, validation_generator + +def get_class_weights(train_generator): + """ + Calculates class weights to handle imbalanced datasets. + This stops the AI from being biased toward the majority class. + """ + print("βš–οΈ Calculating Class Weights for balanced training...") + class_indices = train_generator.class_indices + classes = train_generator.classes + + weights = compute_class_weight( + class_weight='balanced', + classes=np.unique(classes), + y=classes + ) + class_weights = dict(enumerate(weights)) + + print(" Weights applied:") + for cls_name, cls_idx in class_indices.items(): + print(f" - {cls_name}: {class_weights[cls_idx]:.2f}") + + return class_weights + +def build_model(num_classes): + print("🧠 Building Model (EfficientNetB0)...") + + base_model = EfficientNetB0( + weights='imagenet', + include_top=False, + input_shape=IMG_SIZE + (3,) + ) + base_model.trainable = False + + x = base_model.output + x = GlobalAveragePooling2D()(x) + x = BatchNormalization()(x) + x = Dropout(0.3)(x) + x = Dense(256, activation='relu')(x) + x = BatchNormalization()(x) + x = Dropout(0.3)(x) + predictions = Dense(num_classes, activation='softmax')(x) + + model = Model(inputs=base_model.input, outputs=predictions) + return model, base_model + +def get_callbacks(phase_name): + return [ + EarlyStopping(monitor='val_accuracy', patience=5, restore_best_weights=True, verbose=1), + ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=1e-7, verbose=1), + ModelCheckpoint(filepath=MODEL_SAVE_PATH, monitor='val_accuracy', save_best_only=True, verbose=1) + ] + +def train_brain(): + print("πŸš€ Initializing Mission 17 AI Training v2 (Optimized)...") + + if not os.path.exists(DATASET_DIR): + print(f"❌ ERROR: Training Dataset not found at {DATASET_DIR}") + print(" Did you run scripts/testing/split_dataset.py first?") + return + + train_generator, validation_generator = build_generators() + + # Save Labels + class_names = list(train_generator.class_indices.keys()) + with open(LABELS_SAVE_PATH, 'w') as f: + for name in class_names: + f.write(name + '\n') + + num_classes = len(class_names) + + # Get Class Weights + class_weights = get_class_weights(train_generator) + + model, base_model = build_model(num_classes) + + # --- PHASE 1 --- + print("\n" + "="*50) + print("πŸ‹οΈ PHASE 1: Training Top Layers (Base Frozen)") + print("="*50) + + model.compile(optimizer=Adam(learning_rate=LR_INITIAL), loss='categorical_crossentropy', metrics=['accuracy']) + model.fit( + train_generator, + epochs=EPOCHS_INITIAL, + validation_data=validation_generator, + class_weight=class_weights, # Apply weights! + callbacks=get_callbacks('phase1') + ) + + # --- PHASE 2 --- + print("\n" + "="*50) + print("πŸ”¬ PHASE 2: Fine-Tuning Top Base Layers") + print("="*50) + + base_model.trainable = True + for layer in base_model.layers[:150]: + layer.trainable = False + + model.compile(optimizer=Adam(learning_rate=LR_FINETUNE), loss='categorical_crossentropy', metrics=['accuracy']) + model.fit( + train_generator, + epochs=EPOCHS_FINETUNE, + validation_data=validation_generator, + class_weight=class_weights, # Apply weights! + callbacks=get_callbacks('phase2') + ) + + print(f"\nβœ… Training complete! Model saved to {MODEL_SAVE_PATH}") + +if __name__ == '__main__': + train_brain() diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..da8e0b08bf26910d184ce402279af8012c4e6ccf --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1 @@ +"""Utils module for Mission 17 AI.""" diff --git a/utils/__pycache__/__init__.cpython-311.pyc b/utils/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88259ca104aac780fd91280548813c1a1422f92b Binary files /dev/null and b/utils/__pycache__/__init__.cpython-311.pyc differ diff --git a/utils/__pycache__/anticheat.cpython-311.pyc b/utils/__pycache__/anticheat.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6489e5e1ffe678fc09615fbda9fd26e3606593a0 Binary files /dev/null and b/utils/__pycache__/anticheat.cpython-311.pyc differ diff --git a/utils/__pycache__/predictor.cpython-311.pyc b/utils/__pycache__/predictor.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e889f8c6dc15dfea5f1d077710d30e8fef688964 Binary files /dev/null and b/utils/__pycache__/predictor.cpython-311.pyc differ diff --git a/utils/__pycache__/verdict.cpython-311.pyc b/utils/__pycache__/verdict.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e1175886425317c7d49e6be18962cb7cd5b006de Binary files /dev/null and b/utils/__pycache__/verdict.cpython-311.pyc differ diff --git a/utils/anticheat.py b/utils/anticheat.py new file mode 100644 index 0000000000000000000000000000000000000000..d0953f5f1eec34fa5898897559af63b240aa45fd --- /dev/null +++ b/utils/anticheat.py @@ -0,0 +1,98 @@ +import os +import json +import io +import imagehash +from PIL import Image + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HASH_FILE = os.path.join(BASE_DIR, 'anticheat_hashes.json') + +class AntiCheatEngine: + def __init__(self): + self.hashes = set() + self._load_hashes() + + def _load_hashes(self): + if os.path.exists(HASH_FILE): + try: + with open(HASH_FILE, 'r') as f: + data = json.load(f) + self.hashes = set(data.get("hashes", [])) + except Exception as e: + print(f"⚠️ Could not load anticheat hashes: {e}") + + def _save_hashes(self): + try: + with open(HASH_FILE, 'w') as f: + json.dump({"hashes": list(self.hashes)}, f) + except Exception as e: + print(f"⚠️ Could not save anticheat hashes: {e}") + + def get_hashes(self, file_bytes): + """Calculates pHash and dHash for better duplicate detection.""" + try: + img = Image.open(io.BytesIO(file_bytes)).convert('RGB') + p_hash = str(imagehash.phash(img)) + d_hash = str(imagehash.dhash(img)) + return p_hash, d_hash + except Exception: + return None, None + + def is_duplicate(self, file_bytes, similarity_threshold=8): + """ + Checks if the image is a duplicate based on stored hashes. + similarity_threshold: the max hamming distance to be considered a duplicate. + """ + p_hash_str, d_hash_str = self.get_hashes(file_bytes) + + if not p_hash_str or not d_hash_str: + return False + + # Check exact matches first for speed + if p_hash_str in self.hashes or d_hash_str in self.hashes: + return True + + p_hash = imagehash.hex_to_hash(p_hash_str) + d_hash = imagehash.hex_to_hash(d_hash_str) + + # Check similarity (hamming distance) + for stored_hash_str in self.hashes: + try: + stored_hash = imagehash.hex_to_hash(stored_hash_str) + # Compare both pHash and dHash representation lengths isn't an issue since they are stored as strings + # but we should compare apples to apples. Let's simplify and just do exact match on dHash and pHash, + # but also check similarity if we parse them properly. + + # For safety, let's just do an exact match on string representations for now, + # or a simple distance check if we assume all stored are pHashes. + # Since we store both, some might be dHash, some pHash. + # Let's just compare distances safely. + distance = p_hash - stored_hash + if distance < similarity_threshold: + return True + + distance = d_hash - stored_hash + if distance < similarity_threshold: + return True + except Exception: + continue + + return False + + def register(self, file_bytes): + """Registers a new image hash to prevent future duplicates.""" + p_hash_str, d_hash_str = self.get_hashes(file_bytes) + if p_hash_str: + self.hashes.add(p_hash_str) + if d_hash_str: + self.hashes.add(d_hash_str) + self._save_hashes() + return p_hash_str + + def clear(self): + self.hashes.clear() + self._save_hashes() + return len(self.hashes) + + def count(self): + return len(self.hashes) diff --git a/utils/predictor.py b/utils/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..ce011d50c356491fc5df39965cede27936f0deec --- /dev/null +++ b/utils/predictor.py @@ -0,0 +1,109 @@ +import os +import io +import traceback +import numpy as np +from PIL import Image +from tensorflow.keras.models import load_model +from tensorflow.keras.applications.efficientnet import preprocess_input + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +MODEL_PATH = os.path.join(BASE_DIR, 'mission_model.h5') +LABELS_PATH = os.path.join(BASE_DIR, 'labels.txt') + +class Predictor: + def __init__(self): + self.model = None + self.class_names = [] + self._load_model() + + def _load_model(self): + print("🧠 Loading TensorFlow CNN Brain...") + if not os.path.exists(MODEL_PATH): + print(f"❌ ERROR: {MODEL_PATH} not found. You need to train the model!") + return + + try: + self.model = load_model(MODEL_PATH) + print("βœ… Model loaded successfully!") + except Exception as e: + print(f"❌ Failed to load model: {e}") + + # Load Labels + try: + with open(LABELS_PATH, 'r') as f: + self.class_names = [line.strip() for line in f.readlines()] + print(f"🏷️ Labels loaded: {self.class_names}") + except FileNotFoundError: + print("❌ ERROR: labels.txt not found.") + self.class_names = [] + + # πŸ”₯ WARMUP STEP (Optimization) + if self.model: + print("πŸ”₯ Warming up model for instant first-prediction...") + dummy_image = np.zeros((1, 224, 224, 3), dtype=np.float32) + self.model.predict(dummy_image, verbose=0) + print("⚑ AI is fully optimized and ready!") + + def predict(self, file_bytes): + """ + Runs the image through the custom EfficientNet CNN. + """ + if not self.model or not self.class_names: + return {"category": "Non_SDG_Invalid", "confidence": 0, "reason": "Model offline or missing."} + + try: + # 1. Read image using PIL (just like in train_ai.py) + img = Image.open(io.BytesIO(file_bytes)).convert('RGB') + + # 2. Resize to 224x224 (EfficientNetB0 input size) + img = img.resize((224, 224), Image.LANCZOS) + + # 3. Apply EfficientNetB0 preprocess_input + img_array = np.array(img, dtype=np.float32) + img_array = preprocess_input(img_array) + img_array = np.expand_dims(img_array, axis=0) + + # 4. Predict + predictions = self.model.predict(img_array) + score = predictions[0] + + top_index = np.argmax(score) + label = self.class_names[top_index] + + confidence = int(np.max(score) * 100) + + # Clean up label if it has the SDG prefix (e.g. SDG12_Recycling -> Recycling) + # The verdict.py MISSION_MAP expects "Recycling", "Planting", etc. + category = label + if "_" in label and label.startswith("SDG"): + # E.g. "SDG12_Recycling" -> "Recycling" + category = label.split("_", 1)[1] + # If there are multiple underscores (like SDG13_15_Planting), take the last part + if "_" in category: + category = category.rsplit("_", 1)[-1] + elif label == "Non_SDG_Invalid": + category = "Non_SDG_Invalid" + + # Quick check for combined strings + if "Planting" in label: category = "Planting" + if "Cleanup" in label: category = "Cleanup" + if "Donation" in label: category = "Donation" + if "Cities" in label or "Sustainable" in label: category = "Sustainable_Cities" + if "Local" in label: category = "Support_Local" + if "Health" in label: category = "Health" + if "Energy" in label: category = "Energy" + if "Education" in label: category = "Education" + + return { + "category": category, + "confidence": confidence, + "reason": f"Predicted {label} with {confidence}% confidence" + } + + except Exception as e: + traceback.print_exc() + print(f"⚠️ Predictor error: {e}") + return {"category": "Non_SDG_Invalid", "confidence": 0, "reason": str(e)} + + def get_model_name(self): + return "Custom CNN (mission_model.h5)" diff --git a/utils/verdict.py b/utils/verdict.py new file mode 100644 index 0000000000000000000000000000000000000000..370df1dbbaf19f7dac2796adec3a72a580009a55 --- /dev/null +++ b/utils/verdict.py @@ -0,0 +1,42 @@ +# Maps AI prediction to (Verdict, Message, SDG) +MISSION_MAP = { + "Planting": ("VERIFIED", "βœ… Valid Planting Mission (SDG 13/15)", "SDG 13/15"), + "Recycling": ("VERIFIED", "βœ… Valid Recycling Mission (SDG 12)", "SDG 12"), + "Cleanup": ("VERIFIED", "βœ… Valid Cleanup Mission (SDG 6/14)", "SDG 6/14"), + "Donation": ("VERIFIED", "βœ… Valid Donation Mission (SDG 1/2)", "SDG 1/2"), + "Health": ("VERIFIED", "βœ… Valid Health & Wellness (SDG 3)", "SDG 3"), + "Education": ("VERIFIED", "βœ… Valid Education Activity (SDG 4)", "SDG 4"), + "Energy": ("VERIFIED", "βœ… Valid Energy Saving Action (SDG 7)", "SDG 7"), + "Sustainable_Cities": ("VERIFIED", "βœ… Valid Sustainable Commute (SDG 11)", "SDG 11"), + "Support_Local": ("VERIFIED", "βœ… Valid Support for Local Biz (SDG 8)", "SDG 8"), + "Non_SDG_Invalid": ("REJECTED", "⚠️ Image does not match any mission.", "N/A"), +} + +def get_verdict(category, confidence_percent, threshold=55): + """ + Returns the final verdict response dictionary. + Requires a confidence of at least `threshold` for a VERIFIED verdict. + """ + verdict, message, sdg = MISSION_MAP.get(category, ("REJECTED", "⚠️ Unknown Image Category.", "N/A")) + + is_verified = (verdict == "VERIFIED") + + # If it's technically a valid category but confidence is too low + if is_verified and confidence_percent < threshold: + verdict = "UNCERTAIN" + message = f"❓ Unclear Image ({confidence_percent}%). Please take a clearer photo." + is_verified = False + sdg = "N/A" + + source_check = "πŸ“Έ Raw Picture" if is_verified else "πŸ€– AI Generated / Invalid" + + return { + 'prediction': category, + 'confidence': f"{confidence_percent}%", + 'confidence_raw': confidence_percent, + 'verdict': verdict, + 'message': message, + 'is_verified': is_verified, + 'sdg': sdg, + 'source_check': source_check + } diff --git a/venv/.gitignore b/venv/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..49b53dcfd66bcbbee1f9f854a995842a8263d3d3 --- /dev/null +++ b/venv/.gitignore @@ -0,0 +1,2 @@ +# Created by venv; see https://docs.python.org/3/library/venv.html +* diff --git a/venv/Lib/site-packages/__pycache__/pylab.cpython-311.pyc b/venv/Lib/site-packages/__pycache__/pylab.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ebffadbd743eff9b2a92c659fe5af0f0ea89001a Binary files /dev/null and b/venv/Lib/site-packages/__pycache__/pylab.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/__pycache__/six.cpython-311.pyc b/venv/Lib/site-packages/__pycache__/six.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0bf8681c6a2a4cb7a20e8a736b6b06209d0f690f Binary files /dev/null and b/venv/Lib/site-packages/__pycache__/six.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/__pycache__/threadpoolctl.cpython-311.pyc b/venv/Lib/site-packages/__pycache__/threadpoolctl.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1146d5a8c9d23ec87a1dbcce10ad71333fcae863 Binary files /dev/null and b/venv/Lib/site-packages/__pycache__/threadpoolctl.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/__pycache__/typing_extensions.cpython-311.pyc b/venv/Lib/site-packages/__pycache__/typing_extensions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3bce6209eebf8912e94ae508e36b043372c43e62 --- /dev/null +++ b/venv/Lib/site-packages/__pycache__/typing_extensions.cpython-311.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8579e0349d3763ed1ffe4345807900a184ffc9f6dacf47f1b52f7d8104f1640b +size 179469 diff --git a/venv/Lib/site-packages/_distutils_hack/__init__.py b/venv/Lib/site-packages/_distutils_hack/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f987a5367fdfaa4f17cd4bf700d56f4b50992368 --- /dev/null +++ b/venv/Lib/site-packages/_distutils_hack/__init__.py @@ -0,0 +1,222 @@ +# don't import any costly modules +import sys +import os + + +is_pypy = '__pypy__' in sys.builtin_module_names + + +def warn_distutils_present(): + if 'distutils' not in sys.modules: + return + if is_pypy and sys.version_info < (3, 7): + # PyPy for 3.6 unconditionally imports distutils, so bypass the warning + # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 + return + import warnings + + warnings.warn( + "Distutils was imported before Setuptools, but importing Setuptools " + "also replaces the `distutils` module in `sys.modules`. This may lead " + "to undesirable behaviors or errors. To avoid these issues, avoid " + "using distutils directly, ensure that setuptools is installed in the " + "traditional way (e.g. not an editable install), and/or make sure " + "that setuptools is always imported before distutils." + ) + + +def clear_distutils(): + if 'distutils' not in sys.modules: + return + import warnings + + warnings.warn("Setuptools is replacing distutils.") + mods = [ + name + for name in sys.modules + if name == "distutils" or name.startswith("distutils.") + ] + for name in mods: + del sys.modules[name] + + +def enabled(): + """ + Allow selection of distutils by environment variable. + """ + which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local') + return which == 'local' + + +def ensure_local_distutils(): + import importlib + + clear_distutils() + + # With the DistutilsMetaFinder in place, + # perform an import to cause distutils to be + # loaded from setuptools._distutils. Ref #2906. + with shim(): + importlib.import_module('distutils') + + # check that submodules load as expected + core = importlib.import_module('distutils.core') + assert '_distutils' in core.__file__, core.__file__ + assert 'setuptools._distutils.log' not in sys.modules + + +def do_override(): + """ + Ensure that the local copy of distutils is preferred over stdlib. + + See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 + for more motivation. + """ + if enabled(): + warn_distutils_present() + ensure_local_distutils() + + +class _TrivialRe: + def __init__(self, *patterns): + self._patterns = patterns + + def match(self, string): + return all(pat in string for pat in self._patterns) + + +class DistutilsMetaFinder: + def find_spec(self, fullname, path, target=None): + # optimization: only consider top level modules and those + # found in the CPython test suite. + if path is not None and not fullname.startswith('test.'): + return + + method_name = 'spec_for_{fullname}'.format(**locals()) + method = getattr(self, method_name, lambda: None) + return method() + + def spec_for_distutils(self): + if self.is_cpython(): + return + + import importlib + import importlib.abc + import importlib.util + + try: + mod = importlib.import_module('setuptools._distutils') + except Exception: + # There are a couple of cases where setuptools._distutils + # may not be present: + # - An older Setuptools without a local distutils is + # taking precedence. Ref #2957. + # - Path manipulation during sitecustomize removes + # setuptools from the path but only after the hook + # has been loaded. Ref #2980. + # In either case, fall back to stdlib behavior. + return + + class DistutilsLoader(importlib.abc.Loader): + def create_module(self, spec): + mod.__name__ = 'distutils' + return mod + + def exec_module(self, module): + pass + + return importlib.util.spec_from_loader( + 'distutils', DistutilsLoader(), origin=mod.__file__ + ) + + @staticmethod + def is_cpython(): + """ + Suppress supplying distutils for CPython (build and tests). + Ref #2965 and #3007. + """ + return os.path.isfile('pybuilddir.txt') + + def spec_for_pip(self): + """ + Ensure stdlib distutils when running under pip. + See pypa/pip#8761 for rationale. + """ + if self.pip_imported_during_build(): + return + clear_distutils() + self.spec_for_distutils = lambda: None + + @classmethod + def pip_imported_during_build(cls): + """ + Detect if pip is being imported in a build script. Ref #2355. + """ + import traceback + + return any( + cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None) + ) + + @staticmethod + def frame_file_is_setup(frame): + """ + Return True if the indicated frame suggests a setup.py file. + """ + # some frames may not have __file__ (#2940) + return frame.f_globals.get('__file__', '').endswith('setup.py') + + def spec_for_sensitive_tests(self): + """ + Ensure stdlib distutils when running select tests under CPython. + + python/cpython#91169 + """ + clear_distutils() + self.spec_for_distutils = lambda: None + + sensitive_tests = ( + [ + 'test.test_distutils', + 'test.test_peg_generator', + 'test.test_importlib', + ] + if sys.version_info < (3, 10) + else [ + 'test.test_distutils', + ] + ) + + +for name in DistutilsMetaFinder.sensitive_tests: + setattr( + DistutilsMetaFinder, + f'spec_for_{name}', + DistutilsMetaFinder.spec_for_sensitive_tests, + ) + + +DISTUTILS_FINDER = DistutilsMetaFinder() + + +def add_shim(): + DISTUTILS_FINDER in sys.meta_path or insert_shim() + + +class shim: + def __enter__(self): + insert_shim() + + def __exit__(self, exc, value, tb): + remove_shim() + + +def insert_shim(): + sys.meta_path.insert(0, DISTUTILS_FINDER) + + +def remove_shim(): + try: + sys.meta_path.remove(DISTUTILS_FINDER) + except ValueError: + pass diff --git a/venv/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1727c90b13a8637df8ba7ed09b39bc9daec02ebe Binary files /dev/null and b/venv/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/_distutils_hack/__pycache__/override.cpython-311.pyc b/venv/Lib/site-packages/_distutils_hack/__pycache__/override.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..559a1ee993d22825ccae4af060a4c6a5a0719a12 Binary files /dev/null and b/venv/Lib/site-packages/_distutils_hack/__pycache__/override.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/_distutils_hack/override.py b/venv/Lib/site-packages/_distutils_hack/override.py new file mode 100644 index 0000000000000000000000000000000000000000..2cc433a4a55e3b41fa31089918fb62096092f89f --- /dev/null +++ b/venv/Lib/site-packages/_distutils_hack/override.py @@ -0,0 +1 @@ +__import__('_distutils_hack').do_override() diff --git a/venv/Lib/site-packages/_multiprocess/__init__.py b/venv/Lib/site-packages/_multiprocess/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a89f71736472fe74c8f3a5ffac26b9bb8dcc9a2 --- /dev/null +++ b/venv/Lib/site-packages/_multiprocess/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2022-2026 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/multiprocess/blob/master/LICENSE + +from _multiprocessing import * diff --git a/venv/Lib/site-packages/_multiprocess/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/_multiprocess/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..075cb1e0936c01a233d527f6378567cf36a6ac0c Binary files /dev/null and b/venv/Lib/site-packages/_multiprocess/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/_yaml/__init__.py b/venv/Lib/site-packages/_yaml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7baa8c4b68127d5cdf0be9a799429e61347c2694 --- /dev/null +++ b/venv/Lib/site-packages/_yaml/__init__.py @@ -0,0 +1,33 @@ +# This is a stub package designed to roughly emulate the _yaml +# extension module, which previously existed as a standalone module +# and has been moved into the `yaml` package namespace. +# It does not perfectly mimic its old counterpart, but should get +# close enough for anyone who's relying on it even when they shouldn't. +import yaml + +# in some circumstances, the yaml module we imoprted may be from a different version, so we need +# to tread carefully when poking at it here (it may not have the attributes we expect) +if not getattr(yaml, '__with_libyaml__', False): + from sys import version_info + + exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError + raise exc("No module named '_yaml'") +else: + from yaml._yaml import * + import warnings + warnings.warn( + 'The _yaml extension module is now located at yaml._yaml' + ' and its location is subject to change. To use the' + ' LibYAML-based parser and emitter, import from `yaml`:' + ' `from yaml import CLoader as Loader, CDumper as Dumper`.', + DeprecationWarning + ) + del warnings + # Don't `del yaml` here because yaml is actually an existing + # namespace member of _yaml. + +__name__ = '_yaml' +# If the module is top-level (i.e. not a part of any specific package) +# then the attribute should be set to ''. +# https://docs.python.org/3.8/library/types.html +__package__ = '' diff --git a/venv/Lib/site-packages/_yaml/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/_yaml/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4f51ef92fabe8ed032a61aa01cd1e19158e58fc Binary files /dev/null and b/venv/Lib/site-packages/_yaml/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/__init__.py b/venv/Lib/site-packages/absl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341e11f037891aa17ddeb7f0bfcac781d321daa6 --- /dev/null +++ b/venv/Lib/site-packages/absl/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = '2.4.0' diff --git a/venv/Lib/site-packages/absl/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/absl/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78480e58608897b12abacf92222f13b1c85e60c7 Binary files /dev/null and b/venv/Lib/site-packages/absl/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/__pycache__/app.cpython-311.pyc b/venv/Lib/site-packages/absl/__pycache__/app.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37ef862f7e74d18fdf9589640f1ebd9840ee2028 Binary files /dev/null and b/venv/Lib/site-packages/absl/__pycache__/app.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/__pycache__/command_name.cpython-311.pyc b/venv/Lib/site-packages/absl/__pycache__/command_name.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e08a53eeac9d8eb51eeb506fc2b9c7486aee41c Binary files /dev/null and b/venv/Lib/site-packages/absl/__pycache__/command_name.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/app.py b/venv/Lib/site-packages/absl/app.py new file mode 100644 index 0000000000000000000000000000000000000000..4656f96c2fefced69c13d87084c0049813b20e49 --- /dev/null +++ b/venv/Lib/site-packages/absl/app.py @@ -0,0 +1,539 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generic entry point for Abseil Python applications. + +To use this module, define a ``main`` function with a single ``argv`` argument +and call ``app.run(main)``. For example:: + + def main(argv): + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') + + if __name__ == '__main__': + app.run(main) +""" + +import collections +import errno +import importlib +import os +import pdb +import sys +import textwrap +import traceback + +from absl import command_name +from absl import flags +from absl import logging + +try: + import faulthandler +except ImportError: + faulthandler = None + +FLAGS = flags.FLAGS + +flags.DEFINE_boolean( + 'run_with_pdb', + False, + 'Set to true for debug mode. PDB is used by default; $PYTHONBREAKPOINT ' + '(https://docs.python.org/3/using/cmdline.html#envvar-PYTHONBREAKPOINT) ' + 'can be used to specify a custom debugger.', +) +flags.DEFINE_boolean( + 'pdb_post_mortem', + False, + 'Set to true to handle uncaught exceptions with the post mortem debugger. ' + 'PDB is used by default; $PYTHONBREAKPOINT ' + '(https://docs.python.org/3/using/cmdline.html#envvar-PYTHONBREAKPOINT) ' + 'can be used to specify a custom one.', +) +flags.DEFINE_alias('pdb', 'pdb_post_mortem') +flags.DEFINE_boolean('run_with_profiling', False, + 'Set to true for profiling the script. ' + 'Execution will be slower, and the output format might ' + 'change over time.') +flags.DEFINE_string('profile_file', None, + 'Dump profile information to a file (for python -m ' + 'pstats). Implies --run_with_profiling.') +flags.DEFINE_boolean('use_cprofile_for_profiling', True, + 'Use cProfile instead of the profile module for ' + 'profiling. This has no effect unless ' + '--run_with_profiling is set.') +flags.DEFINE_boolean('only_check_args', False, + 'Set to true to validate args and exit.', + allow_hide_cpp=True) + + +def _get_debugger_module_with_function(function_name): + """Provides the `$PYTHONBREAKPOINT` module if it contains `function_name`. + + Falls back to `pdb` otherwise. + + Args: + function_name: The name of the function required. + + Returns: + A debugger module providing `function_name`. + """ + python_breakpoint = os.getenv('PYTHONBREAKPOINT') + # The special value '0' for `$PYTHONBREAKPOINT` means "do not use a debugger". + # We don't respect it (if the user explicitly asks to debug) but shouldn't try + # to import a module with this name. + if python_breakpoint and python_breakpoint != '0': + debugger_module_import = python_breakpoint.rsplit('.', 1)[0] + try: + debugger_module = importlib.import_module(debugger_module_import) + except ImportError: + logging.warning( + ( + 'Could not import $PYTHONBREAKPOINT debugger module %r, ' + 'falling back to PDB' + ), + debugger_module_import, + ) + else: + if hasattr(debugger_module, function_name): + return debugger_module + logging.warning( + '$PYTHONBREAKPOINT debugger %r has no function %r, ' + 'falling back to PDB', + debugger_module_import, + function_name, + ) + return pdb + + +# If main() exits via an abnormal exception, call into these +# handlers before exiting. +EXCEPTION_HANDLERS = [] + + +class Error(Exception): + pass + + +class UsageError(Error): + """Exception raised when the arguments supplied by the user are invalid. + + Raise this when the arguments supplied are invalid from the point of + view of the application. For example when two mutually exclusive + flags have been supplied or when there are not enough non-flag + arguments. It is distinct from flags.Error which covers the lower + level of parsing and validating individual flags. + """ + + def __init__(self, message, exitcode=1): + super().__init__(message) + self.exitcode = exitcode + + +class HelpFlag(flags.BooleanFlag): + """Special boolean flag that displays usage and raises SystemExit.""" + NAME = 'help' + SHORT_NAME = '?' + + def __init__(self): + super().__init__( + self.NAME, + False, + 'show this help', + short_name=self.SHORT_NAME, + allow_hide_cpp=True, + ) + + def parse(self, arg): + if self._parse(arg): + usage(shorthelp=True, writeto_stdout=True) + # Advertise --helpfull on stdout, since usage() was on stdout. + print() + print('Try --helpfull to get a list of all flags.') + sys.exit(1) + + +class HelpshortFlag(HelpFlag): + """--helpshort is an alias for --help.""" + NAME = 'helpshort' + SHORT_NAME = None + + +class HelpfullFlag(flags.BooleanFlag): + """Display help for flags in the main module and all dependent modules.""" + + def __init__(self): + super().__init__('helpfull', False, 'show full help', allow_hide_cpp=True) + + def parse(self, arg): + if self._parse(arg): + usage(writeto_stdout=True) + sys.exit(1) + + +class HelpXMLFlag(flags.BooleanFlag): + """Similar to HelpfullFlag, but generates output in XML format.""" + + def __init__(self): + super().__init__( + 'helpxml', + False, + 'like --helpfull, but generates XML output', + allow_hide_cpp=True, + ) + + def parse(self, arg): + if self._parse(arg): + flags.FLAGS.write_help_in_xml_format(sys.stdout) + sys.exit(1) + + +def parse_flags_with_usage(args): + """Tries to parse the flags, print usage, and exit if unparsable. + + Args: + args: [str], a non-empty list of the command line arguments including + program name. + + Returns: + [str], a non-empty list of remaining command line arguments after parsing + flags, including program name. + """ + try: + return FLAGS(args) + except flags.Error as error: + message = str(error) + if '\n' in message: + final_message = 'FATAL Flags parsing error:\n%s\n' % textwrap.indent( + message, ' ') + else: + final_message = 'FATAL Flags parsing error: %s\n' % message + sys.stderr.write(final_message) + sys.stderr.write('Pass --helpshort or --helpfull to see help on flags.\n') + sys.exit(1) + + +_define_help_flags_called = False + + +def define_help_flags(): + """Registers help flags. Idempotent.""" + # Use a global to ensure idempotence. + global _define_help_flags_called + + if not _define_help_flags_called: + flags.DEFINE_flag(HelpFlag()) + flags.DEFINE_flag(HelpshortFlag()) # alias for --help + flags.DEFINE_flag(HelpfullFlag()) + flags.DEFINE_flag(HelpXMLFlag()) + _define_help_flags_called = True + + +def _register_and_parse_flags_with_usage( + argv=None, + flags_parser=parse_flags_with_usage, +): + """Registers help flags, parses arguments and shows usage if appropriate. + + This also calls sys.exit(0) if flag --only_check_args is True. + + Args: + argv: [str], a non-empty list of the command line arguments including + program name, sys.argv is used if None. + flags_parser: Callable[[List[str]], Any], the function used to parse flags. + The return value of this function is passed to `main` untouched. It must + guarantee FLAGS is parsed after this function is called. + + Returns: + The return value of `flags_parser`. When using the default `flags_parser`, + it returns the following: + [str], a non-empty list of remaining command line arguments after parsing + flags, including program name. + + Raises: + Error: Raised when flags_parser is called, but FLAGS is not parsed. + SystemError: Raised when it's called more than once. + """ + # fmt: on + if _register_and_parse_flags_with_usage.done: + raise SystemError('Flag registration can be done only once.') + + define_help_flags() + + original_argv = sys.argv if argv is None else argv + args_to_main = flags_parser(original_argv) + if not FLAGS.is_parsed(): + raise Error('FLAGS must be parsed after flags_parser is called.') + + # Exit when told so. + if FLAGS.only_check_args: + sys.exit(0) + # Immediately after flags are parsed, bump verbosity to INFO if the flag has + # not been set. + if FLAGS['verbosity'].using_default_value: + FLAGS.verbosity = 0 + _register_and_parse_flags_with_usage.done = True + + return args_to_main + +_register_and_parse_flags_with_usage.done = False + + +def _run_main(main, argv): + """Calls main, optionally with a debugger or profiler.""" + if FLAGS.run_with_pdb: + sys.exit(_get_debugger_module_with_function('runcall').runcall(main, argv)) + elif FLAGS.run_with_profiling or FLAGS.profile_file: + # Avoid import overhead since most apps (including performance-sensitive + # ones) won't be run with profiling. + # pylint: disable=g-import-not-at-top + import atexit + if FLAGS.use_cprofile_for_profiling: + import cProfile as profile + else: + import profile + profiler = profile.Profile() + if FLAGS.profile_file: + atexit.register(profiler.dump_stats, FLAGS.profile_file) + else: + atexit.register(profiler.print_stats) + sys.exit(profiler.runcall(main, argv)) + else: + sys.exit(main(argv)) + + +def _call_exception_handlers(exception): + """Calls any installed exception handlers.""" + for handler in EXCEPTION_HANDLERS: + try: + if handler.wants(exception): + handler.handle(exception) + except: # pylint: disable=bare-except + try: + # We don't want to stop for exceptions in the exception handlers but + # we shouldn't hide them either. + logging.error(traceback.format_exc()) + except: # pylint: disable=bare-except + # In case even the logging statement fails, ignore. + pass + + +def run( + main, + argv=None, + flags_parser=parse_flags_with_usage, +): + """Begins executing the program. + + Args: + main: The main function to execute. It takes an single argument "argv", + which is a list of command line arguments with parsed flags removed. + The return value is passed to `sys.exit`, and so for example + a return value of 0 or None results in a successful termination, whereas + a return value of 1 results in abnormal termination. + For more details, see https://docs.python.org/3/library/sys#sys.exit + argv: A non-empty list of the command line arguments including program name, + sys.argv is used if None. + flags_parser: Callable[[List[str]], Any], the function used to parse flags. + The return value of this function is passed to `main` untouched. + It must guarantee FLAGS is parsed after this function is called. + Should be passed as a keyword-only arg which will become mandatory in a + future release. + - Parses command line flags with the flag module. + - If there are any errors, prints usage(). + - Calls main() with the remaining arguments. + - If main() raises a UsageError, prints usage and the error message. + """ + # fmt: on + try: + args = _run_init( + sys.argv if argv is None else argv, + flags_parser, + ) + while _init_callbacks: + callback = _init_callbacks.popleft() + callback() + try: + _run_main(main, args) + except UsageError as error: + usage(shorthelp=True, detailed_error=error, exitcode=error.exitcode) + except: + exc = sys.exc_info()[1] + # Don't try to post-mortem debug successful SystemExits, since those + # mean there wasn't actually an error. In particular, the test framework + # raises SystemExit(False) even if all tests passed. + if isinstance(exc, SystemExit) and not exc.code: + raise + + # Check the tty so that we don't hang waiting for input in an + # non-interactive scenario. + if FLAGS.pdb_post_mortem and sys.stdout.isatty(): + traceback.print_exc() + print() + print(' *** Entering post-mortem debugging ***') + print() + _get_debugger_module_with_function('post_mortem').post_mortem() + raise + except Exception as e: + _call_exception_handlers(e) + raise + +# Callbacks which have been deferred until after _run_init has been called. +_init_callbacks = collections.deque() + + +def call_after_init(callback): + """Calls the given callback only once ABSL has finished initialization. + + If ABSL has already finished initialization when ``call_after_init`` is + called then the callback is executed immediately, otherwise `callback` is + stored to be executed after ``app.run`` has finished initializing (aka. just + before the main function is called). + + If called after ``app.run``, this is equivalent to calling ``callback()`` in + the caller thread. If called before ``app.run``, callbacks are run + sequentially (in an undefined order) in the same thread as ``app.run``. + + Args: + callback: a callable to be called once ABSL has finished initialization. + This may be immediate if initialization has already finished. It + takes no arguments and returns nothing. + """ + if _run_init.done: + callback() + else: + _init_callbacks.append(callback) + + +def _run_init( + argv, + flags_parser, +): + """Does one-time initialization and re-parses flags on rerun.""" + if _run_init.done: + return flags_parser(argv) + command_name.make_process_name_useful() + # Set up absl logging handler. + logging.use_absl_handler() + args = _register_and_parse_flags_with_usage( + argv=argv, + flags_parser=flags_parser, + ) + if faulthandler: + try: + faulthandler.enable() + except Exception: # pylint: disable=broad-except + # Some tests verify stderr output very closely, so don't print anything. + # Disabled faulthandler is a low-impact error. + pass + _run_init.done = True + return args + + +_run_init.done = False + + +def usage(shorthelp=False, writeto_stdout=False, detailed_error=None, + exitcode=None): + """Writes __main__'s docstring to stderr with some help text. + + Args: + shorthelp: bool, if True, prints only flags from the main module, + rather than all flags. + writeto_stdout: bool, if True, writes help message to stdout, + rather than to stderr. + detailed_error: str, additional detail about why usage info was presented. + exitcode: optional integer, if set, exits with this status code after + writing help. + """ + if writeto_stdout: + stdfile = sys.stdout + else: + stdfile = sys.stderr + + doc = sys.modules['__main__'].__doc__ + if not doc: + doc = '\nUSAGE: %s [flags]\n' % sys.argv[0] + doc = flags.text_wrap(doc, indent=' ', firstline_indent='') + else: + # Replace all '%s' with sys.argv[0], and all '%%' with '%'. + num_specifiers = doc.count('%') - 2 * doc.count('%%') + try: + doc %= (sys.argv[0],) * num_specifiers + except (OverflowError, TypeError, ValueError): + # Just display the docstring as-is. + pass + if shorthelp: + flag_str = FLAGS.main_module_help() + else: + flag_str = FLAGS.get_help() + try: + stdfile.write(doc) + if flag_str: + stdfile.write('\nflags:\n') + stdfile.write(flag_str) + stdfile.write('\n') + if detailed_error is not None: + stdfile.write('\n%s\n' % detailed_error) + except OSError as e: + # We avoid printing a huge backtrace if we get EPIPE, because + # "foo.par --help | less" is a frequent use case. + if e.errno != errno.EPIPE: + raise + if exitcode is not None: + sys.exit(exitcode) + + +class ExceptionHandler: + """Base exception handler from which other may inherit.""" + + def wants(self, exc): + """Returns whether this handler wants to handle the exception or not. + + This base class returns True for all exceptions by default. Override in + subclass if it wants to be more selective. + + Args: + exc: Exception, the current exception. + """ + del exc # Unused. + return True + + def handle(self, exc): + """Do something with the current exception. + + Args: + exc: Exception, the current exception + + This method must be overridden. + """ + raise NotImplementedError() + + +def install_exception_handler(handler): + """Installs an exception handler. + + Args: + handler: ExceptionHandler, the exception handler to install. + + Raises: + TypeError: Raised when the handler was not of the correct type. + + All installed exception handlers will be called if main() exits via + an abnormal exception, i.e. not one of SystemExit, KeyboardInterrupt, + FlagsError or UsageError. + """ + if not isinstance(handler, ExceptionHandler): + raise TypeError('handler of type %s does not inherit from ExceptionHandler' + % type(handler)) + EXCEPTION_HANDLERS.append(handler) diff --git a/venv/Lib/site-packages/absl/app.pyi b/venv/Lib/site-packages/absl/app.pyi new file mode 100644 index 0000000000000000000000000000000000000000..882ef2b9856231e09e65d0213abdd775a8efe13e --- /dev/null +++ b/venv/Lib/site-packages/absl/app.pyi @@ -0,0 +1,89 @@ +from collections.abc import Callable +from typing import Any, NoReturn, TypeVar, overload + +from absl.flags import _flag + +_MainArgs = TypeVar('_MainArgs') +_Exc = TypeVar('_Exc', bound=Exception) + +class ExceptionHandler: + + def wants(self, exc: _Exc) -> bool: + ... + + def handle(self, exc: _Exc): + ... + +EXCEPTION_HANDLERS: list[ExceptionHandler] = ... + +class HelpFlag(_flag.BooleanFlag): + def __init__(self): + ... + +class HelpshortFlag(HelpFlag): + ... + +class HelpfullFlag(_flag.BooleanFlag): + def __init__(self): + ... + +class HelpXMLFlag(_flag.BooleanFlag): + def __init__(self): + ... + +def define_help_flags() -> None: + ... + +@overload +def usage(shorthelp: bool | int = ..., + writeto_stdout: bool | int = ..., + detailed_error: Any | None = ..., + exitcode: None = ...) -> None: + ... + +@overload +def usage(shorthelp: bool | int, + writeto_stdout: bool | int, + detailed_error: Any | None, + exitcode: int) -> NoReturn: + ... + +@overload +def usage(shorthelp: bool | int = ..., + writeto_stdout: bool | int = ..., + detailed_error: Any | None = ..., + *, + exitcode: int) -> NoReturn: + ... + +def install_exception_handler(handler: ExceptionHandler) -> None: + ... + +class Error(Exception): + ... + +class UsageError(Error): + exitcode: int + +def parse_flags_with_usage(args: list[str]) -> list[str]: + ... + +def call_after_init(callback: Callable[[], Any]) -> None: + ... + +# Without the flag_parser argument, `main` should require a List[str]. +@overload +def run( + main: Callable[[list[str]], Any], + argv: list[str] | None = ..., +) -> NoReturn: + ... + +@overload +def run( + main: Callable[[_MainArgs], Any], + argv: list[str] | None = ..., + *, + flags_parser: Callable[[list[str]], _MainArgs], +) -> NoReturn: + ... diff --git a/venv/Lib/site-packages/absl/command_name.py b/venv/Lib/site-packages/absl/command_name.py new file mode 100644 index 0000000000000000000000000000000000000000..86f81af8ed9fbc98dd46a3171631163ae8f693a5 --- /dev/null +++ b/venv/Lib/site-packages/absl/command_name.py @@ -0,0 +1,63 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A tiny stand alone library to change the kernel process name on Linux.""" + +import os +import sys + +# This library must be kept small and stand alone. It is used by small things +# that require no extension modules. + + +def make_process_name_useful(): + """Sets the process name to something better than 'python' if possible.""" + set_kernel_process_name(os.path.basename(sys.argv[0])) + + +def set_kernel_process_name(name): + """Changes the Kernel's /proc/self/status process name on Linux. + + The kernel name is NOT what will be shown by the ps or top command. + It is a 15 character string stored in the kernel's process table that + is included in the kernel log when a process is OOM killed. + The first 15 bytes of name are used. Non-ASCII unicode is replaced with '?'. + + Does nothing if /proc/self/comm cannot be written or prctl() fails. + + Args: + name: bytes|unicode, the Linux kernel's command name to set. + """ + if not isinstance(name, bytes): + name = name.encode('ascii', 'replace') + try: + # This is preferred to using ctypes to try and call prctl() when possible. + with open('/proc/self/comm', 'wb') as proc_comm: + proc_comm.write(name[:15]) + except OSError: + try: + import ctypes # pylint: disable=g-import-not-at-top + except ImportError: + return # No ctypes. + try: + libc = ctypes.CDLL('libc.so.6') + except OSError: + return # No libc.so.6. + pr_set_name = ctypes.c_ulong(15) # linux/prctl.h PR_SET_NAME value. + zero = ctypes.c_ulong(0) + try: + libc.prctl(pr_set_name, name, zero, zero, zero) + # Ignore the prctl return value. Nothing we can do if it errored. + except AttributeError: + return # No prctl. diff --git a/venv/Lib/site-packages/absl/flags/__init__.py b/venv/Lib/site-packages/absl/flags/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..06401900d2d4c6da61f6c532e330eb6a3fcc39e7 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/__init__.py @@ -0,0 +1,220 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This package is used to define and parse command line flags. + +This package defines a *distributed* flag-definition policy: rather than +an application having to define all flags in or near main(), each Python +module defines flags that are useful to it. When one Python module +imports another, it gains access to the other's flags. (This is +implemented by having all modules share a common, global registry object +containing all the flag information.) + +Flags are defined through the use of one of the DEFINE_xxx functions. +The specific function used determines how the flag is parsed, checked, +and optionally type-converted, when it's seen on the command line. +""" + +import sys + +from absl.flags import _argument_parser +from absl.flags import _defines +from absl.flags import _exceptions +from absl.flags import _flag +from absl.flags import _flagvalues +from absl.flags import _helpers +from absl.flags import _validators + +__all__ = ( + 'DEFINE', + 'DEFINE_flag', + 'DEFINE_string', + 'DEFINE_boolean', + 'DEFINE_bool', + 'DEFINE_float', + 'DEFINE_integer', + 'DEFINE_enum', + 'DEFINE_enum_class', + 'DEFINE_list', + 'DEFINE_spaceseplist', + 'DEFINE_multi', + 'DEFINE_multi_string', + 'DEFINE_multi_integer', + 'DEFINE_multi_float', + 'DEFINE_multi_enum', + 'DEFINE_multi_enum_class', + 'DEFINE_alias', + # Flag validators. + 'register_validator', + 'validator', + 'register_multi_flags_validator', + 'multi_flags_validator', + 'mark_flag_as_required', + 'mark_flags_as_required', + 'mark_flags_as_mutual_exclusive', + 'mark_bool_flags_as_mutual_exclusive', + # Flag modifiers. + 'set_default', + 'override_value', + # Key flag related functions. + 'declare_key_flag', + 'adopt_module_key_flags', + 'disclaim_key_flags', + # Module exceptions. + 'Error', + 'CantOpenFlagFileError', + 'DuplicateFlagError', + 'IllegalFlagValueError', + 'UnrecognizedFlagError', + 'UnparsedFlagAccessError', + 'ValidationError', + 'FlagNameConflictsWithMethodError', + # Public classes. + 'Flag', + 'BooleanFlag', + 'EnumFlag', + 'EnumClassFlag', + 'MultiFlag', + 'MultiEnumClassFlag', + 'FlagHolder', + 'FlagValues', + 'ArgumentParser', + 'BooleanParser', + 'EnumParser', + 'EnumClassParser', + 'ArgumentSerializer', + 'FloatParser', + 'IntegerParser', + 'BaseListParser', + 'ListParser', + 'ListSerializer', + 'EnumClassListSerializer', + 'CsvListSerializer', + 'WhitespaceSeparatedListParser', + 'EnumClassSerializer', + # Helper functions. + 'get_help_width', + 'text_wrap', + 'flag_dict_to_args', + 'doc_to_help', + # The global FlagValues instance. + 'FLAGS', +) + +# Initialize the FLAGS_MODULE as early as possible. +# It's only used by adopt_module_key_flags to take SPECIAL_FLAGS into account. +_helpers.FLAGS_MODULE = sys.modules[__name__] + +# Add current module to disclaimed module ids. +_helpers.disclaim_module_ids.add(id(sys.modules[__name__])) + +# DEFINE functions. They are explained in more details in the module doc string. +# pylint: disable=invalid-name +DEFINE = _defines.DEFINE +DEFINE_flag = _defines.DEFINE_flag +DEFINE_string = _defines.DEFINE_string +DEFINE_boolean = _defines.DEFINE_boolean +DEFINE_bool = DEFINE_boolean # Match C++ API. +DEFINE_float = _defines.DEFINE_float +DEFINE_integer = _defines.DEFINE_integer +DEFINE_enum = _defines.DEFINE_enum +DEFINE_enum_class = _defines.DEFINE_enum_class +DEFINE_list = _defines.DEFINE_list +DEFINE_spaceseplist = _defines.DEFINE_spaceseplist +DEFINE_multi = _defines.DEFINE_multi +DEFINE_multi_string = _defines.DEFINE_multi_string +DEFINE_multi_integer = _defines.DEFINE_multi_integer +DEFINE_multi_float = _defines.DEFINE_multi_float +DEFINE_multi_enum = _defines.DEFINE_multi_enum +DEFINE_multi_enum_class = _defines.DEFINE_multi_enum_class +DEFINE_alias = _defines.DEFINE_alias +# pylint: enable=invalid-name + +# Flag validators. +register_validator = _validators.register_validator +validator = _validators.validator +register_multi_flags_validator = _validators.register_multi_flags_validator +multi_flags_validator = _validators.multi_flags_validator +mark_flag_as_required = _validators.mark_flag_as_required +mark_flags_as_required = _validators.mark_flags_as_required +mark_flags_as_mutual_exclusive = _validators.mark_flags_as_mutual_exclusive +mark_bool_flags_as_mutual_exclusive = _validators.mark_bool_flags_as_mutual_exclusive + +# Flag modifiers. +set_default = _defines.set_default +override_value = _defines.override_value + +# Key flag related functions. +declare_key_flag = _defines.declare_key_flag +adopt_module_key_flags = _defines.adopt_module_key_flags +disclaim_key_flags = _defines.disclaim_key_flags + +# Module exceptions. +# pylint: disable=invalid-name +Error = _exceptions.Error +CantOpenFlagFileError = _exceptions.CantOpenFlagFileError +DuplicateFlagError = _exceptions.DuplicateFlagError +IllegalFlagValueError = _exceptions.IllegalFlagValueError +UnrecognizedFlagError = _exceptions.UnrecognizedFlagError +UnparsedFlagAccessError = _exceptions.UnparsedFlagAccessError +ValidationError = _exceptions.ValidationError +FlagNameConflictsWithMethodError = _exceptions.FlagNameConflictsWithMethodError + +# Public classes. +Flag = _flag.Flag +BooleanFlag = _flag.BooleanFlag +EnumFlag = _flag.EnumFlag +EnumClassFlag = _flag.EnumClassFlag +MultiFlag = _flag.MultiFlag +MultiEnumClassFlag = _flag.MultiEnumClassFlag +FlagHolder = _flagvalues.FlagHolder +FlagValues = _flagvalues.FlagValues +ArgumentParser = _argument_parser.ArgumentParser +BooleanParser = _argument_parser.BooleanParser +EnumParser = _argument_parser.EnumParser +EnumClassParser = _argument_parser.EnumClassParser +ArgumentSerializer = _argument_parser.ArgumentSerializer +FloatParser = _argument_parser.FloatParser +IntegerParser = _argument_parser.IntegerParser +BaseListParser = _argument_parser.BaseListParser +ListParser = _argument_parser.ListParser +ListSerializer = _argument_parser.ListSerializer +EnumClassListSerializer = _argument_parser.EnumClassListSerializer +CsvListSerializer = _argument_parser.CsvListSerializer +WhitespaceSeparatedListParser = _argument_parser.WhitespaceSeparatedListParser +EnumClassSerializer = _argument_parser.EnumClassSerializer +# pylint: enable=invalid-name + +# Helper functions. +get_help_width = _helpers.get_help_width +text_wrap = _helpers.text_wrap +flag_dict_to_args = _helpers.flag_dict_to_args +doc_to_help = _helpers.doc_to_help + +# Special flags. +_helpers.SPECIAL_FLAGS = FlagValues() + +DEFINE_string( + 'flagfile', '', + 'Insert flag definitions from the given file into the command line.', + _helpers.SPECIAL_FLAGS) # pytype: disable=wrong-arg-types + +DEFINE_string('undefok', '', + 'comma-separated list of flag names that it is okay to specify ' + 'on the command line even if the program does not define a flag ' + 'with that name. IMPORTANT: flags in this list that have ' + 'arguments MUST use the --flag=value format.', + _helpers.SPECIAL_FLAGS) # pytype: disable=wrong-arg-types + +#: The global FlagValues instance. +FLAGS = _flagvalues.FLAGS diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8eeca49fe99f8560df42acb41696434fbb2a37b0 Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/_argument_parser.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/_argument_parser.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef94b8251754b96e2e717bb52170f058a80c913e Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/_argument_parser.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/_defines.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/_defines.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d37bb9f35f540605591f1ddedccfc3e0deada1a Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/_defines.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/_exceptions.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/_exceptions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77d48abb5dd5d74bbf4ead365dd57d81a4dd8c3c Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/_exceptions.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/_flag.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/_flag.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..06499713c775480f3b9af053afa17f3fe817e39d Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/_flag.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/_flagvalues.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/_flagvalues.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ecd7097353b79f5bd56fa2a38f9aed2dc4ef8393 Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/_flagvalues.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/_helpers.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/_helpers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c95c644b8abba52752089e8127f85a76bb5c73f6 Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/_helpers.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/_validators.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/_validators.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59fb86c2592d32b5d9242ca8a0fbb45ce29a7d89 Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/_validators.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/_validators_classes.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/_validators_classes.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0817a2d89cad9356940f84d21eb3ce4479f0ea28 Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/_validators_classes.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/__pycache__/argparse_flags.cpython-311.pyc b/venv/Lib/site-packages/absl/flags/__pycache__/argparse_flags.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..206e438810dff9660fb0334e83ef0fce2ca9d84d Binary files /dev/null and b/venv/Lib/site-packages/absl/flags/__pycache__/argparse_flags.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/flags/_argument_parser.py b/venv/Lib/site-packages/absl/flags/_argument_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..9aa7a0fb6f2f6b37e7a9f82cfc906427ed6e89e7 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/_argument_parser.py @@ -0,0 +1,632 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contains base classes used to parse and convert arguments. + +Do NOT import this module directly. Import the flags package and use the +aliases defined at the package level instead. +""" + +import collections +from collections.abc import Iterable, Sequence +import csv +import enum +import io +import string +from typing import Any, Generic, TypeVar +from xml.dom import minidom + +from absl.flags import _helpers + +_T = TypeVar('_T') +_ET = TypeVar('_ET', bound=enum.Enum) +_N = TypeVar('_N', int, float) + + +class _ArgumentParserCache(type): + """Metaclass used to cache and share argument parsers among flags.""" + + _instances: dict[Any, Any] = {} + + def __call__(cls, *args, **kwargs): + """Returns an instance of the argument parser cls. + + This method overrides behavior of the __new__ methods in + all subclasses of ArgumentParser (inclusive). If an instance + for cls with the same set of arguments exists, this instance is + returned, otherwise a new instance is created. + + If any keyword arguments are defined, or the values in args + are not hashable, this method always returns a new instance of + cls. + + Args: + *args: Positional initializer arguments. + **kwargs: Initializer keyword arguments. + + Returns: + An instance of cls, shared or new. + """ + if kwargs: + return type.__call__(cls, *args, **kwargs) + else: + instances = cls._instances + key = (cls,) + tuple(args) + try: + return instances[key] + except KeyError: + # No cache entry for key exists, create a new one. + return instances.setdefault(key, type.__call__(cls, *args)) + except TypeError: + # An object in args cannot be hashed, always return + # a new instance. + return type.__call__(cls, *args) + + +class ArgumentParser(Generic[_T], metaclass=_ArgumentParserCache): + """Base class used to parse and convert arguments. + + The :meth:`parse` method checks to make sure that the string argument is a + legal value and convert it to a native type. If the value cannot be + converted, it should throw a ``ValueError`` exception with a human + readable explanation of why the value is illegal. + + Subclasses should also define a syntactic_help string which may be + presented to the user to describe the form of the legal values. + + Argument parser classes must be stateless, since instances are cached + and shared between flags. Initializer arguments are allowed, but all + member variables must be derived from initializer arguments only. + """ + + syntactic_help: str = '' + + def parse(self, argument: str) -> _T | None: + """Parses the string argument and returns the native value. + + By default it returns its argument unmodified. + + Args: + argument: string argument passed in the commandline. + + Raises: + ValueError: Raised when it fails to parse the argument. + TypeError: Raised when the argument has the wrong type. + + Returns: + The parsed value in native type. + """ + if not isinstance(argument, str): + raise TypeError('flag value must be a string, found "{}"'.format( + type(argument))) + return argument # type: ignore[return-value] + + def flag_type(self) -> str: + """Returns a string representing the type of the flag.""" + return 'string' + + def _custom_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + """Returns a list of minidom.Element to add additional flag information. + + Args: + doc: minidom.Document, the DOM document it should create nodes from. + """ + del doc # Unused. + return [] + + +class ArgumentSerializer(Generic[_T]): + """Base class for generating string representations of a flag value.""" + + def serialize(self, value: _T) -> str: + """Returns a serialized string of the value.""" + return str(value) + + +class NumericParser(ArgumentParser[_N]): + """Parser of numeric values. + + Parsed value may be bounded to a given upper and lower bound. + """ + + lower_bound: _N | None + upper_bound: _N | None + + def is_outside_bounds(self, val: _N) -> bool: + """Returns whether the value is outside the bounds or not.""" + return ((self.lower_bound is not None and val < self.lower_bound) or + (self.upper_bound is not None and val > self.upper_bound)) + + def parse(self, argument: str | _N) -> _N: + """See base class.""" + val = self.convert(argument) + if self.is_outside_bounds(val): + raise ValueError('%s is not %s' % (val, self.syntactic_help)) + return val + + def _custom_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + elements = [] + if self.lower_bound is not None: + elements.append(_helpers.create_xml_dom_element( + doc, 'lower_bound', self.lower_bound)) + if self.upper_bound is not None: + elements.append(_helpers.create_xml_dom_element( + doc, 'upper_bound', self.upper_bound)) + return elements + + def convert(self, argument: str | _N) -> _N: + """Returns the correct numeric value of argument. + + Subclass must implement this method, and raise TypeError if argument is not + string or has the right numeric type. + + Args: + argument: string argument passed in the commandline, or the numeric type. + + Raises: + TypeError: Raised when argument is not a string or the right numeric type. + ValueError: Raised when failed to convert argument to the numeric value. + """ + raise NotImplementedError + + +class FloatParser(NumericParser[float]): + """Parser of floating point values. + + Parsed value may be bounded to a given upper and lower bound. + """ + number_article = 'a' + number_name = 'number' + syntactic_help = ' '.join((number_article, number_name)) + + def __init__( + self, + lower_bound: float | None = None, + upper_bound: float | None = None, + ) -> None: + super().__init__() + self.lower_bound = lower_bound + self.upper_bound = upper_bound + sh = self.syntactic_help + if lower_bound is not None and upper_bound is not None: + sh = ('%s in the range [%s, %s]' % (sh, lower_bound, upper_bound)) + elif lower_bound == 0: + sh = 'a non-negative %s' % self.number_name + elif upper_bound == 0: + sh = 'a non-positive %s' % self.number_name + elif upper_bound is not None: + sh = '%s <= %s' % (self.number_name, upper_bound) + elif lower_bound is not None: + sh = '%s >= %s' % (self.number_name, lower_bound) + self.syntactic_help = sh + + def convert(self, argument: int | float | str) -> float: + """Returns the float value of argument.""" + if ( + (isinstance(argument, int) and not isinstance(argument, bool)) + or isinstance(argument, float) + or isinstance(argument, str) + ): + return float(argument) + else: + raise TypeError( + 'Expect argument to be a string, int, or float, found {}'.format( + type(argument))) + + def flag_type(self) -> str: + """See base class.""" + return 'float' + + +class IntegerParser(NumericParser[int]): + """Parser of an integer value. + + Parsed value may be bounded to a given upper and lower bound. + """ + number_article = 'an' + number_name = 'integer' + syntactic_help = ' '.join((number_article, number_name)) + + def __init__( + self, lower_bound: int | None = None, upper_bound: int | None = None + ) -> None: + super().__init__() + self.lower_bound = lower_bound + self.upper_bound = upper_bound + sh = self.syntactic_help + if lower_bound is not None and upper_bound is not None: + sh = ('%s in the range [%s, %s]' % (sh, lower_bound, upper_bound)) + elif lower_bound == 1: + sh = 'a positive %s' % self.number_name + elif upper_bound == -1: + sh = 'a negative %s' % self.number_name + elif lower_bound == 0: + sh = 'a non-negative %s' % self.number_name + elif upper_bound == 0: + sh = 'a non-positive %s' % self.number_name + elif upper_bound is not None: + sh = '%s <= %s' % (self.number_name, upper_bound) + elif lower_bound is not None: + sh = '%s >= %s' % (self.number_name, lower_bound) + self.syntactic_help = sh + + def convert(self, argument: int | str) -> int: + """Returns the int value of argument.""" + if isinstance(argument, int) and not isinstance(argument, bool): + return argument + elif isinstance(argument, str): + base = 10 + if len(argument) > 2 and argument[0] == '0': + if argument[1] == 'o': + base = 8 + elif argument[1] == 'x': + base = 16 + return int(argument, base) + else: + raise TypeError('Expect argument to be a string or int, found {}'.format( + type(argument))) + + def flag_type(self) -> str: + """See base class.""" + return 'int' + + +class BooleanParser(ArgumentParser[bool]): + """Parser of boolean values.""" + + def parse(self, argument: str | int) -> bool: + """See base class.""" + if isinstance(argument, str): + if argument.lower() in ('true', 't', '1'): + return True + elif argument.lower() in ('false', 'f', '0'): + return False + else: + raise ValueError('Non-boolean argument to boolean flag', argument) + elif isinstance(argument, int): + # Only allow bool or integer 0, 1. + # Note that float 1.0 == True, 0.0 == False. + bool_value = bool(argument) + if argument == bool_value: + return bool_value + else: + raise ValueError('Non-boolean argument to boolean flag', argument) + + raise TypeError('Non-boolean argument to boolean flag', argument) + + def flag_type(self) -> str: + """See base class.""" + return 'bool' + + +class EnumParser(ArgumentParser[str]): + """Parser of a string enum value (a string value from a given set).""" + + def __init__( + self, enum_values: Iterable[str], case_sensitive: bool = True + ) -> None: + """Initializes EnumParser. + + Args: + enum_values: [str], a non-empty list of string values in the enum. + case_sensitive: bool, whether or not the enum is to be case-sensitive. + + Raises: + ValueError: When enum_values is empty. + """ + if not enum_values: + raise ValueError(f'enum_values cannot be empty, found "{enum_values}"') + if isinstance(enum_values, str): + raise ValueError(f'enum_values cannot be a str, found "{enum_values}"') + super().__init__() + self.enum_values = list(enum_values) + self.case_sensitive = case_sensitive + + def parse(self, argument: str) -> str: + """Determines validity of argument and returns the correct element of enum. + + Args: + argument: str, the supplied flag value. + + Returns: + The first matching element from enum_values. + + Raises: + ValueError: Raised when argument didn't match anything in enum. + """ + if self.case_sensitive: + if argument not in self.enum_values: + raise ValueError('value should be one of <%s>' % + '|'.join(self.enum_values)) + else: + return argument + else: + if argument.upper() not in [value.upper() for value in self.enum_values]: + raise ValueError('value should be one of <%s>' % + '|'.join(self.enum_values)) + else: + return [value for value in self.enum_values + if value.upper() == argument.upper()][0] + + def flag_type(self) -> str: + """See base class.""" + return 'string enum' + + +class EnumClassParser(ArgumentParser[_ET]): + """Parser of an Enum class member.""" + + def __init__( + self, enum_class: type[_ET], case_sensitive: bool = True + ) -> None: + """Initializes EnumParser. + + Args: + enum_class: class, the Enum class with all possible flag values. + case_sensitive: bool, whether or not the enum is to be case-sensitive. If + False, all member names must be unique when case is ignored. + + Raises: + TypeError: When enum_class is not a subclass of Enum. + ValueError: When enum_class is empty. + """ + if not issubclass(enum_class, enum.Enum): + raise TypeError(f'{enum_class} is not a subclass of Enum.') + if not enum_class.__members__: + raise ValueError('enum_class cannot be empty, but "{}" is empty.' + .format(enum_class)) + if not case_sensitive: + members = collections.Counter( + name.lower() for name in enum_class.__members__) + duplicate_keys = { + member for member, count in members.items() if count > 1 + } + if duplicate_keys: + raise ValueError( + 'Duplicate enum values for {} using case_sensitive=False'.format( + duplicate_keys)) + + super().__init__() + self.enum_class = enum_class + self._case_sensitive = case_sensitive + if case_sensitive: + self._member_names = tuple(enum_class.__members__) + else: + self._member_names = tuple( + name.lower() for name in enum_class.__members__) + + @property + def member_names(self) -> Sequence[str]: + """The accepted enum names, in lowercase if not case sensitive.""" + return self._member_names + + def parse(self, argument: _ET | str) -> _ET: + """Determines validity of argument and returns the correct element of enum. + + Args: + argument: str or Enum class member, the supplied flag value. + + Returns: + The first matching Enum class member in Enum class. + + Raises: + ValueError: Raised when argument didn't match anything in enum. + """ + if isinstance(argument, self.enum_class): + return argument # pytype: disable=bad-return-type + elif not isinstance(argument, str): + raise ValueError( + '{} is not an enum member or a name of a member in {}'.format( + argument, self.enum_class)) + key = EnumParser( + self._member_names, case_sensitive=self._case_sensitive).parse(argument) + if self._case_sensitive: + return self.enum_class[key] + else: + # If EnumParser.parse() return a value, we're guaranteed to find it + # as a member of the class + return next(value for name, value in self.enum_class.__members__.items() + if name.lower() == key.lower()) + + def flag_type(self) -> str: + """See base class.""" + return 'enum class' + + +class ListSerializer(Generic[_T], ArgumentSerializer[list[_T]]): + + def __init__(self, list_sep: str) -> None: + self.list_sep = list_sep + + def serialize(self, value: list[_T]) -> str: + """See base class.""" + return self.list_sep.join([str(x) for x in value]) + + +class EnumClassListSerializer(ListSerializer[_ET]): + """A serializer for :class:`MultiEnumClass` flags. + + This serializer simply joins the output of `EnumClassSerializer` using a + provided separator. + """ + + _element_serializer: 'EnumClassSerializer' + + def __init__(self, list_sep: str, **kwargs) -> None: + """Initializes EnumClassListSerializer. + + Args: + list_sep: String to be used as a separator when serializing + **kwargs: Keyword arguments to the `EnumClassSerializer` used to serialize + individual values. + """ + super().__init__(list_sep) + self._element_serializer = EnumClassSerializer(**kwargs) + + def serialize(self, value: _ET | list[_ET]) -> str: + """See base class.""" + if isinstance(value, list): + return self.list_sep.join( + self._element_serializer.serialize(x) for x in value) + else: + return self._element_serializer.serialize(value) + + +class CsvListSerializer(ListSerializer[str]): + + def serialize(self, value: list[str]) -> str: + """Serializes a list as a CSV string or unicode.""" + output = io.StringIO() + writer = csv.writer(output, delimiter=self.list_sep) + writer.writerow([str(x) for x in value]) + serialized_value = output.getvalue().strip() + + # We need the returned value to be pure ascii or Unicodes so that + # when the xml help is generated they are usefully encodable. + return str(serialized_value) + + +class EnumClassSerializer(ArgumentSerializer[_ET]): + """Class for generating string representations of an enum class flag value.""" + + def __init__(self, lowercase: bool) -> None: + """Initializes EnumClassSerializer. + + Args: + lowercase: If True, enum member names are lowercased during serialization. + """ + self._lowercase = lowercase + + def serialize(self, value: _ET) -> str: + """Returns a serialized string of the Enum class value.""" + as_string = str(value.name) + return as_string.lower() if self._lowercase else as_string + + +class BaseListParser(ArgumentParser): + """Base class for a parser of lists of strings. + + To extend, inherit from this class; from the subclass ``__init__``, call:: + + super().__init__(token, name) + + where token is a character used to tokenize, and name is a description + of the separator. + """ + + def __init__(self, token: str | None = None, name: str | None = None) -> None: + assert name + super().__init__() + self._token = token + self._name = name + self.syntactic_help = 'a %s separated list' % self._name + + def parse(self, argument: str) -> list[str]: + """See base class.""" + if isinstance(argument, list): + return argument + elif not argument: + return [] + else: + return [s.strip() for s in argument.split(self._token)] + + def flag_type(self) -> str: + """See base class.""" + return '%s separated list of strings' % self._name + + +class ListParser(BaseListParser): + """Parser for a comma-separated list of strings.""" + + def __init__(self) -> None: + super().__init__(',', 'comma') + + def parse(self, argument: str | list[str]) -> list[str]: + """Parses argument as comma-separated list of strings.""" + if isinstance(argument, list): + return argument + elif not argument: + return [] + else: + try: + return [s.strip() for s in list(csv.reader([argument], strict=True))[0]] + except csv.Error as e: + # Provide a helpful report for case like + # --listflag="$(printf 'hello,\nworld')" + # IOW, list flag values containing naked newlines. This error + # was previously "reported" by allowing csv.Error to + # propagate. + raise ValueError('Unable to parse the value %r as a %s: %s' + % (argument, self.flag_type(), e)) + + def _custom_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + elements = super()._custom_xml_dom_elements(doc) + elements.append(_helpers.create_xml_dom_element( + doc, 'list_separator', repr(','))) + return elements + + +class WhitespaceSeparatedListParser(BaseListParser): + """Parser for a whitespace-separated list of strings.""" + + def __init__(self, comma_compat: bool = False) -> None: + """Initializer. + + Args: + comma_compat: bool, whether to support comma as an additional separator. + If False then only whitespace is supported. This is intended only for + backwards compatibility with flags that used to be comma-separated. + """ + self._comma_compat = comma_compat + name = 'whitespace or comma' if self._comma_compat else 'whitespace' + super().__init__(None, name) + + def parse(self, argument: str | list[str]) -> list[str]: + """Parses argument as whitespace-separated list of strings. + + It also parses argument as comma-separated list of strings if requested. + + Args: + argument: string argument passed in the commandline. + + Returns: + [str], the parsed flag value. + """ + if isinstance(argument, list): + return argument + elif not argument: + return [] + else: + if self._comma_compat: + argument = argument.replace(',', ' ') + return argument.split() + + def _custom_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + elements = super()._custom_xml_dom_elements(doc) + separators = list(string.whitespace) + if self._comma_compat: + separators.append(',') + separators.sort() + for sep_char in separators: + elements.append(_helpers.create_xml_dom_element( + doc, 'list_separator', repr(sep_char))) + return elements diff --git a/venv/Lib/site-packages/absl/flags/_defines.py b/venv/Lib/site-packages/absl/flags/_defines.py new file mode 100644 index 0000000000000000000000000000000000000000..4914764cdeb9a43eb8dc16b88939b129a177e9e3 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/_defines.py @@ -0,0 +1,1702 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This modules contains flags DEFINE functions. + +Do NOT import this module directly. Import the flags package and use the +aliases defined at the package level instead. +""" + +from collections.abc import Iterable +import enum +import sys +import types +from typing import Any, Literal, TypeVar, overload + +from absl.flags import _argument_parser +from absl.flags import _exceptions +from absl.flags import _flag +from absl.flags import _flagvalues +from absl.flags import _helpers +from absl.flags import _validators + +_helpers.disclaim_module_ids.add(id(sys.modules[__name__])) + +_T = TypeVar('_T') +_ET = TypeVar('_ET', bound=enum.Enum) + + +def _register_bounds_validator_if_needed(parser, name, flag_values): + """Enforces lower and upper bounds for numeric flags. + + Args: + parser: NumericParser (either FloatParser or IntegerParser), provides lower + and upper bounds, and help text to display. + name: str, name of the flag + flag_values: FlagValues. + """ + if parser.lower_bound is not None or parser.upper_bound is not None: + + def checker(value): + if value is not None and parser.is_outside_bounds(value): + message = '%s is not %s' % (value, parser.syntactic_help) + raise _exceptions.ValidationError(message) + return True + + _validators.register_validator(name, checker, flag_values=flag_values) + + +@overload +def DEFINE( # pylint: disable=invalid-name + parser: _argument_parser.ArgumentParser[_T], + name: str, + default: Any, + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + serializer: _argument_parser.ArgumentSerializer[_T] | None = ..., + module_name: str | None = ..., + required: Literal[True] = ..., + **args: Any +) -> _flagvalues.FlagHolder[_T]: + ... + + +@overload +def DEFINE( # pylint: disable=invalid-name + parser: _argument_parser.ArgumentParser[_T], + name: str, + default: Any | None, + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + serializer: _argument_parser.ArgumentSerializer[_T] | None = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[_T | None]: + ... + + +def DEFINE( # pylint: disable=invalid-name + parser, + name, + default, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + serializer=None, + module_name=None, + required=False, + **args): + """Registers a generic Flag object. + + NOTE: in the docstrings of all DEFINE* functions, "registers" is short + for "creates a new flag and registers it". + + Auxiliary function: clients should use the specialized ``DEFINE_`` + function instead. + + Args: + parser: :class:`ArgumentParser`, used to parse the flag arguments. + name: str, the flag name. + default: The default value of the flag. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + serializer: :class:`ArgumentSerializer`, the flag serializer instance. + module_name: str, the name of the Python module declaring this flag. If not + provided, it will be computed using the stack trace of this call. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: dict, the extra keyword args that are passed to ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + return DEFINE_flag( + _flag.Flag(parser, serializer, name, default, help, **args), + flag_values, + module_name, + required=True if required else False, + ) + + +@overload +def DEFINE_flag( # pylint: disable=invalid-name + flag: _flag.Flag[_T], + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: Literal[True] = ..., +) -> _flagvalues.FlagHolder[_T]: + ... + + +@overload +def DEFINE_flag( # pylint: disable=invalid-name + flag: _flag.Flag[_T], + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., +) -> _flagvalues.FlagHolder[_T | None]: + ... + + +def DEFINE_flag( # pylint: disable=invalid-name + flag, + flag_values=_flagvalues.FLAGS, + module_name=None, + required=False): + """Registers a :class:`Flag` object with a :class:`FlagValues` object. + + By default, the global :const:`FLAGS` ``FlagValue`` object is used. + + Typical users will use one of the more specialized DEFINE_xxx + functions, such as :func:`DEFINE_string` or :func:`DEFINE_integer`. But + developers who need to create :class:`Flag` objects themselves should use + this function to register their flags. + + Args: + flag: :class:`Flag`, a flag that is key to the module. + flag_values: :class:`FlagValues`, the ``FlagValues`` instance with which the + flag will be registered. This should almost never need to be overridden. + module_name: str, the name of the Python module declaring this flag. If not + provided, it will be computed using the stack trace of this call. + required: bool, is this a required flag. This must be used as a keyword + argument. + + Returns: + a handle to defined flag. + """ + if required and flag.default is not None: + raise ValueError( + 'Required flag --%s needs to have None as default' % flag.name + ) + # Copying the reference to flag_values prevents pychecker warnings. + fv = flag_values + fv[flag.name] = flag + # Tell flag_values who's defining the flag. + if module_name: + module = sys.modules.get(module_name) + else: + module, module_name = _helpers.get_calling_module_object_and_name() + flag_values.register_flag_by_module(module_name, flag) + flag_values.register_flag_by_module_id(id(module), flag) + if required: + _validators.mark_flag_as_required(flag.name, fv) + ensure_non_none_value = (flag.default is not None) or required + return _flagvalues.FlagHolder( + fv, flag, ensure_non_none_value=ensure_non_none_value) + + +def set_default(flag_holder: _flagvalues.FlagHolder[_T], value: _T) -> None: + """Changes the default value of the provided flag object. + + The flag's current value is also updated if the flag is currently using + the default value, i.e. not specified in the command line, and not set + by FLAGS.name = value. + + Args: + flag_holder: FlagHolder, the flag to modify. + value: The new default value. + + Raises: + IllegalFlagValueError: Raised when value is not valid. + """ + flag_holder._flagvalues.set_default(flag_holder.name, value) # pylint: disable=protected-access + + +def override_value(flag_holder: _flagvalues.FlagHolder[_T], value: _T) -> None: + """Overrides the value of the provided flag. + + This value takes precedent over the default value and, when called after flag + parsing, any value provided at the command line. + + Args: + flag_holder: FlagHolder, the flag to modify. + value: The new value. + + Raises: + IllegalFlagValueError: The value did not pass the flag parser or validators. + """ + fv = flag_holder._flagvalues # pylint: disable=protected-access + # Ensure the new value satisfies the flag's parser while avoiding side + # effects of calling parse(). + parsed = fv[flag_holder.name]._parse(value) # pylint: disable=protected-access + if parsed != value: + raise _exceptions.IllegalFlagValueError( + 'flag %s: parsed value %r not equal to original %r' + % (flag_holder.name, parsed, value) + ) + setattr(fv, flag_holder.name, value) + + +def _internal_declare_key_flags( + flag_names: list[str], + flag_values: _flagvalues.FlagValues = _flagvalues.FLAGS, + key_flag_values: _flagvalues.FlagValues | None = None, +) -> None: + """Declares a flag as key for the calling module. + + Internal function. User code should call declare_key_flag or + adopt_module_key_flags instead. + + Args: + flag_names: [str], a list of names of already-registered Flag objects. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flags listed in flag_names have registered (the value of the flag_values + argument from the ``DEFINE_*`` calls that defined those flags). This + should almost never need to be overridden. + key_flag_values: :class:`FlagValues`, the FlagValues instance that (among + possibly many other things) keeps track of the key flags for each module. + Default ``None`` means "same as flag_values". This should almost never + need to be overridden. + + Raises: + UnrecognizedFlagError: Raised when the flag is not defined. + """ + key_flag_values = key_flag_values or flag_values + + module = _helpers.get_calling_module() + + for flag_name in flag_names: + key_flag_values.register_key_flag_for_module(module, flag_values[flag_name]) + + +def declare_key_flag( + flag_name: str | _flagvalues.FlagHolder, + flag_values: _flagvalues.FlagValues = _flagvalues.FLAGS, +) -> None: + """Declares one flag as key to the current module. + + Key flags are flags that are deemed really important for a module. + They are important when listing help messages; e.g., if the + --helpshort command-line flag is used, then only the key flags of the + main module are listed (instead of all flags, as in the case of + --helpfull). + + Sample usage:: + + flags.declare_key_flag('flag_1') + + Args: + flag_name: str | :class:`FlagHolder`, the name or holder of an already + declared flag. (Redeclaring flags as key, including flags implicitly key + because they were declared in this module, is a no-op.) + Positional-only parameter. + flag_values: :class:`FlagValues`, the FlagValues instance in which the + flag will be declared as a key flag. This should almost never need to be + overridden. + + Raises: + ValueError: Raised if flag_name not defined as a Python flag. + """ + flag_name, flag_values = _flagvalues.resolve_flag_ref(flag_name, flag_values) + if flag_name in _helpers.SPECIAL_FLAGS: + # Take care of the special flags, e.g., --flagfile, --undefok. + # These flags are defined in SPECIAL_FLAGS, and are treated + # specially during flag parsing, taking precedence over the + # user-defined flags. + _internal_declare_key_flags([flag_name], + flag_values=_helpers.SPECIAL_FLAGS, + key_flag_values=flag_values) + return + try: + _internal_declare_key_flags([flag_name], flag_values=flag_values) + except KeyError: + raise ValueError('Flag --%s is undefined. To set a flag as a key flag ' + 'first define it in Python.' % flag_name) + + +def adopt_module_key_flags( + module: Any, flag_values: _flagvalues.FlagValues = _flagvalues.FLAGS +) -> None: + """Declares that all flags key to a module are key to the current module. + + Args: + module: module, the module object from which all key flags will be declared + as key flags to the current module. + flag_values: :class:`FlagValues`, the FlagValues instance in which the + flags will be declared as key flags. This should almost never need to be + overridden. + + Raises: + Error: Raised when given an argument that is a module name (a string), + instead of a module object. + """ + if not isinstance(module, types.ModuleType): + raise _exceptions.Error('Expected a module object, not %r.' % (module,)) + _internal_declare_key_flags( + [f.name for f in flag_values.get_key_flags_for_module(module.__name__)], + flag_values=flag_values) + # If module is this flag module, take _helpers.SPECIAL_FLAGS into account. + if module == _helpers.FLAGS_MODULE: + _internal_declare_key_flags( + # As we associate flags with get_calling_module_object_and_name(), the + # special flags defined in this module are incorrectly registered with + # a different module. So, we can't use get_key_flags_for_module. + # Instead, we take all flags from _helpers.SPECIAL_FLAGS (a private + # FlagValues, where no other module should register flags). + [_helpers.SPECIAL_FLAGS[name].name for name in _helpers.SPECIAL_FLAGS], + flag_values=_helpers.SPECIAL_FLAGS, + key_flag_values=flag_values) + + +def disclaim_key_flags() -> None: + """Declares that the current module will not define any more key flags. + + Normally, the module that calls the DEFINE_xxx functions claims the + flag to be its key flag. This is undesirable for modules that + define additional DEFINE_yyy functions with its own flag parsers and + serializers, since that module will accidentally claim flags defined + by DEFINE_yyy as its key flags. After calling this function, the + module disclaims flag definitions thereafter, so the key flags will + be correctly attributed to the caller of DEFINE_yyy. + + After calling this function, the module will not be able to define + any more flags. This function will affect all FlagValues objects. + """ + globals_for_caller = sys._getframe(1).f_globals # pylint: disable=protected-access + module = _helpers.get_module_object_and_name(globals_for_caller) + if module is not None: + _helpers.disclaim_module_ids.add(id(module.module)) + + +@overload +def DEFINE_string( # pylint: disable=invalid-name + name: str, + default: str | None, + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[str]: + ... + + +@overload +def DEFINE_string( # pylint: disable=invalid-name + name: str, + default: None, + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[str | None]: + ... + + +@overload +def DEFINE_string( # pylint: disable=invalid-name + name: str, + default: str, + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[str]: + ... + + +def DEFINE_string( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + required=False, + **args +): + """Registers a flag whose value can be any string.""" + parser = _argument_parser.ArgumentParser[str]() + serializer = _argument_parser.ArgumentSerializer[str]() + return DEFINE( + parser, + name, + default, + help, + flag_values, + serializer, + required=True if required else False, + **args, + ) + + +@overload +def DEFINE_boolean( # pylint: disable=invalid-name + name: str, + default: None | str | bool | int, + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[bool]: + ... + + +@overload +def DEFINE_boolean( # pylint: disable=invalid-name + name: str, + default: None, + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[bool | None]: + ... + + +@overload +def DEFINE_boolean( # pylint: disable=invalid-name + name: str, + default: str | bool | int, + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[bool]: + ... + + +def DEFINE_boolean( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + module_name=None, + required=False, + **args +): + """Registers a boolean flag. + + Such a boolean flag does not take an argument. If a user wants to + specify a false value explicitly, the long option beginning with 'no' + must be used: i.e. --noflag + + This flag will have a value of None, True or False. None is possible + if default=None and the user does not specify the flag on the command + line. + + Args: + name: str, the flag name. + default: bool|str|None, the default value of the flag. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + module_name: str, the name of the Python module declaring this flag. If not + provided, it will be computed using the stack trace of this call. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: dict, the extra keyword args that are passed to ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + return DEFINE_flag( # pytype: disable=bad-return-type + _flag.BooleanFlag(name, default, help, **args), + flag_values, + module_name, + required=True if required else False, + ) + + +@overload +def DEFINE_float( # pylint: disable=invalid-name + name: str, + default: None | float | str, + help: str | None, # pylint: disable=redefined-builtin + lower_bound: float | None = ..., + upper_bound: float | None = ..., + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[float]: + ... + + +@overload +def DEFINE_float( # pylint: disable=invalid-name + name: str, + default: None, + help: str | None, # pylint: disable=redefined-builtin + lower_bound: float | None = ..., + upper_bound: float | None = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[float | None]: + ... + + +@overload +def DEFINE_float( # pylint: disable=invalid-name + name: str, + default: float | str, + help: str | None, # pylint: disable=redefined-builtin + lower_bound: float | None = ..., + upper_bound: float | None = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[float]: + ... + + +def DEFINE_float( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + lower_bound=None, + upper_bound=None, + flag_values=_flagvalues.FLAGS, + required=False, + **args +): + """Registers a flag whose value must be a float. + + If ``lower_bound`` or ``upper_bound`` are set, then this flag must be + within the given range. + + Args: + name: str, the flag name. + default: float|str|None, the default value of the flag. + help: str, the help message. + lower_bound: float, min value of the flag. + upper_bound: float, max value of the flag. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: dict, the extra keyword args that are passed to :func:`DEFINE`. + + Returns: + a handle to defined flag. + """ + parser = _argument_parser.FloatParser(lower_bound, upper_bound) + serializer = _argument_parser.ArgumentSerializer() + result = DEFINE( + parser, + name, + default, + help, # pylint: disable=redefined-builtin + flag_values, + serializer, + required=True if required else False, + **args, + ) + _register_bounds_validator_if_needed(parser, name, flag_values=flag_values) + return result + + +@overload +def DEFINE_integer( # pylint: disable=invalid-name + name: str, + default: None | int | str, + help: str | None, # pylint: disable=redefined-builtin + lower_bound: int | None = ..., + upper_bound: int | None = ..., + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[int]: + ... + + +@overload +def DEFINE_integer( # pylint: disable=invalid-name + name: str, + default: None, + help: str | None, # pylint: disable=redefined-builtin + lower_bound: int | None = ..., + upper_bound: int | None = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[int | None]: + ... + + +@overload +def DEFINE_integer( # pylint: disable=invalid-name + name: str, + default: int | str, + help: str | None, # pylint: disable=redefined-builtin + lower_bound: int | None = ..., + upper_bound: int | None = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[int]: + ... + + +def DEFINE_integer( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + lower_bound=None, + upper_bound=None, + flag_values=_flagvalues.FLAGS, + required=False, + **args +): + """Registers a flag whose value must be an integer. + + If ``lower_bound``, or ``upper_bound`` are set, then this flag must be + within the given range. + + Args: + name: str, the flag name. + default: int|str|None, the default value of the flag. + help: str, the help message. + lower_bound: int, min value of the flag. + upper_bound: int, max value of the flag. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: dict, the extra keyword args that are passed to :func:`DEFINE`. + + Returns: + a handle to defined flag. + """ + parser = _argument_parser.IntegerParser(lower_bound, upper_bound) + serializer = _argument_parser.ArgumentSerializer() + result = DEFINE( + parser, + name, + default, + help, # pylint: disable=redefined-builtin + flag_values, + serializer, + required=True if required else False, + **args, + ) + _register_bounds_validator_if_needed(parser, name, flag_values=flag_values) + return result + + +@overload +def DEFINE_enum( # pylint: disable=invalid-name + name: str, + default: str | None, + enum_values: Iterable[str], + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[str]: + ... + + +@overload +def DEFINE_enum( # pylint: disable=invalid-name + name: str, + default: None, + enum_values: Iterable[str], + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[str | None]: + ... + + +@overload +def DEFINE_enum( # pylint: disable=invalid-name + name: str, + default: str, + enum_values: Iterable[str], + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[str]: + ... + + +def DEFINE_enum( # pylint: disable=invalid-name + name, + default, + enum_values, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + module_name=None, + required=False, + **args +): + """Registers a flag whose value can be any string from enum_values. + + Instead of a string enum, prefer `DEFINE_enum_class`, which allows + defining enums from an `enum.Enum` class. + + Args: + name: str, the flag name. + default: str|None, the default value of the flag. + enum_values: [str], a non-empty list of strings with the possible values for + the flag. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + module_name: str, the name of the Python module declaring this flag. If not + provided, it will be computed using the stack trace of this call. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: dict, the extra keyword args that are passed to ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + result = DEFINE_flag( + _flag.EnumFlag(name, default, help, enum_values, **args), + flag_values, + module_name, + required=True if required else False, + ) + return result + + +@overload +def DEFINE_enum_class( # pylint: disable=invalid-name + name: str, + default: None | _ET | str, + enum_class: type[_ET], + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + case_sensitive: bool = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[_ET]: + ... + + +@overload +def DEFINE_enum_class( # pylint: disable=invalid-name + name: str, + default: None, + enum_class: type[_ET], + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + case_sensitive: bool = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[_ET | None]: + ... + + +@overload +def DEFINE_enum_class( # pylint: disable=invalid-name + name: str, + default: _ET | str, + enum_class: type[_ET], + help: str | None, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + case_sensitive: bool = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[_ET]: + ... + + +def DEFINE_enum_class( # pylint: disable=invalid-name + name, + default, + enum_class, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + module_name=None, + case_sensitive=False, + required=False, + **args +): + """Registers a flag whose value can be the name of enum members. + + Args: + name: str, the flag name. + default: Enum|str|None, the default value of the flag. + enum_class: class, the Enum class with all the possible values for the flag. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + module_name: str, the name of the Python module declaring this flag. If not + provided, it will be computed using the stack trace of this call. + case_sensitive: bool, whether to map strings to members of the enum_class + without considering case. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: dict, the extra keyword args that are passed to ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + # NOTE: pytype fails if this is a direct return. + result = DEFINE_flag( + _flag.EnumClassFlag( + name, default, help, enum_class, case_sensitive=case_sensitive, **args + ), + flag_values, + module_name, + required=True if required else False, + ) + return result + + +@overload +def DEFINE_list( # pylint: disable=invalid-name + name: str, + default: None | Iterable[str] | str, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[str]]: + ... + + +@overload +def DEFINE_list( # pylint: disable=invalid-name + name: str, + default: None, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[str] | None]: + ... + + +@overload +def DEFINE_list( # pylint: disable=invalid-name + name: str, + default: Iterable[str] | str, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[str]]: + ... + + +def DEFINE_list( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + required=False, + **args +): + """Registers a flag whose value is a comma-separated list of strings. + + The flag value is parsed with a CSV parser. + + Args: + name: str, the flag name. + default: list|str|None, the default value of the flag. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: Dictionary with extra keyword args that are passed to the + ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + parser = _argument_parser.ListParser() + serializer = _argument_parser.CsvListSerializer(',') + return DEFINE( + parser, + name, + default, + help, + flag_values, + serializer, + required=True if required else False, + **args, + ) + + +@overload +def DEFINE_spaceseplist( # pylint: disable=invalid-name + name: str, + default: None | Iterable[str] | str, + help: str, # pylint: disable=redefined-builtin + comma_compat: bool = ..., + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[str]]: + ... + + +@overload +def DEFINE_spaceseplist( # pylint: disable=invalid-name + name: str, + default: None, + help: str, # pylint: disable=redefined-builtin + comma_compat: bool = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[str] | None]: + ... + + +@overload +def DEFINE_spaceseplist( # pylint: disable=invalid-name + name: str, + default: Iterable[str] | str, + help: str, # pylint: disable=redefined-builtin + comma_compat: bool = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[str]]: + ... + + +def DEFINE_spaceseplist( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + comma_compat=False, + flag_values=_flagvalues.FLAGS, + required=False, + **args +): + """Registers a flag whose value is a whitespace-separated list of strings. + + Any whitespace can be used as a separator. + + Args: + name: str, the flag name. + default: list|str|None, the default value of the flag. + help: str, the help message. + comma_compat: bool - Whether to support comma as an additional separator. If + false then only whitespace is supported. This is intended only for + backwards compatibility with flags that used to be comma-separated. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: Dictionary with extra keyword args that are passed to the + ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + parser = _argument_parser.WhitespaceSeparatedListParser( + comma_compat=comma_compat) + serializer = _argument_parser.ListSerializer(' ') + return DEFINE( + parser, + name, + default, + help, + flag_values, + serializer, + required=True if required else False, + **args, + ) + + +@overload +def DEFINE_multi( # pylint: disable=invalid-name + parser: _argument_parser.ArgumentParser[_T], + serializer: _argument_parser.ArgumentSerializer[_T], + name: str, + default: Iterable[_T], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[_T]]: + ... + + +@overload +def DEFINE_multi( # pylint: disable=invalid-name + parser: _argument_parser.ArgumentParser[_T], + serializer: _argument_parser.ArgumentSerializer[_T], + name: str, + default: None | _T, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[_T]]: + ... + + +@overload +def DEFINE_multi( # pylint: disable=invalid-name + parser: _argument_parser.ArgumentParser[_T], + serializer: _argument_parser.ArgumentSerializer[_T], + name: str, + default: None, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[_T] | None]: + ... + + +@overload +def DEFINE_multi( # pylint: disable=invalid-name + parser: _argument_parser.ArgumentParser[_T], + serializer: _argument_parser.ArgumentSerializer[_T], + name: str, + default: Iterable[_T], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[_T]]: + ... + + +@overload +def DEFINE_multi( # pylint: disable=invalid-name + parser: _argument_parser.ArgumentParser[_T], + serializer: _argument_parser.ArgumentSerializer[_T], + name: str, + default: _T, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[_T]]: + ... + + +def DEFINE_multi( # pylint: disable=invalid-name + parser, + serializer, + name, + default, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + module_name=None, + required=False, + **args +): + """Registers a generic MultiFlag that parses its args with a given parser. + + Auxiliary function. Normal users should NOT use it directly. + + Developers who need to create their own 'Parser' classes for options + which can appear multiple times can call this module function to + register their flags. + + Args: + parser: ArgumentParser, used to parse the flag arguments. + serializer: ArgumentSerializer, the flag serializer instance. + name: str, the flag name. + default: Union[Iterable[T], str, None], the default value of the flag. If + the value is text, it will be parsed as if it was provided from the + command line. If the value is a non-string iterable, it will be iterated + over to create a shallow copy of the values. If it is None, it is left + as-is. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + module_name: A string, the name of the Python module declaring this flag. If + not provided, it will be computed using the stack trace of this call. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: Dictionary with extra keyword args that are passed to the + ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + result = DEFINE_flag( + _flag.MultiFlag(parser, serializer, name, default, help, **args), + flag_values, + module_name, + required=True if required else False, + ) + return result + + +@overload +def DEFINE_multi_string( # pylint: disable=invalid-name + name: str, + default: None | Iterable[str] | str, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[str]]: + ... + + +@overload +def DEFINE_multi_string( # pylint: disable=invalid-name + name: str, + default: None, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[str] | None]: + ... + + +@overload +def DEFINE_multi_string( # pylint: disable=invalid-name + name: str, + default: Iterable[str] | str, + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[str]]: + ... + + +def DEFINE_multi_string( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + required=False, + **args +): + """Registers a flag whose value can be a list of any strings. + + Use the flag on the command line multiple times to place multiple + string values into the list. The 'default' may be a single string + (which will be converted into a single-element list) or a list of + strings. + + + Args: + name: str, the flag name. + default: Union[Iterable[str], str, None], the default value of the flag; see + :func:`DEFINE_multi`. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: Dictionary with extra keyword args that are passed to the + ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + parser = _argument_parser.ArgumentParser() + serializer = _argument_parser.ArgumentSerializer() + return DEFINE_multi( + parser, + serializer, + name, + default, + help, # pylint: disable=redefined-builtin + flag_values, + required=True if required else False, + **args, + ) + + +@overload +def DEFINE_multi_integer( # pylint: disable=invalid-name + name: str, + default: None | Iterable[int] | int | str, + help: str, # pylint: disable=redefined-builtin + lower_bound: int | None = ..., + upper_bound: int | None = ..., + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[int]]: + ... + + +@overload +def DEFINE_multi_integer( # pylint: disable=invalid-name + name: str, + default: None, + help: str, # pylint: disable=redefined-builtin + lower_bound: int | None = ..., + upper_bound: int | None = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[int] | None]: + ... + + +@overload +def DEFINE_multi_integer( # pylint: disable=invalid-name + name: str, + default: Iterable[int] | int | str, + help: str, # pylint: disable=redefined-builtin + lower_bound: int | None = ..., + upper_bound: int | None = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[int]]: + ... + + +def DEFINE_multi_integer( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + lower_bound=None, + upper_bound=None, + flag_values=_flagvalues.FLAGS, + required=False, + **args +): + """Registers a flag whose value can be a list of arbitrary integers. + + Use the flag on the command line multiple times to place multiple + integer values into the list. The 'default' may be a single integer + (which will be converted into a single-element list) or a list of + integers. + + Args: + name: str, the flag name. + default: Union[Iterable[int], str, None], the default value of the flag; see + `DEFINE_multi`. + help: str, the help message. + lower_bound: int, min values of the flag. + upper_bound: int, max values of the flag. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: Dictionary with extra keyword args that are passed to the + ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + parser = _argument_parser.IntegerParser(lower_bound, upper_bound) + serializer = _argument_parser.ArgumentSerializer() + return DEFINE_multi( + parser, + serializer, + name, + default, + help, # pylint: disable=redefined-builtin + flag_values, + required=True if required else False, + **args, + ) + + +@overload +def DEFINE_multi_float( # pylint: disable=invalid-name + name: str, + default: None | Iterable[float] | float | str, + help: str, # pylint: disable=redefined-builtin + lower_bound: float | None = ..., + upper_bound: float | None = ..., + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[float]]: + ... + + +@overload +def DEFINE_multi_float( # pylint: disable=invalid-name + name: str, + default: None, + help: str, # pylint: disable=redefined-builtin + lower_bound: float | None = ..., + upper_bound: float | None = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[float] | None]: + ... + + +@overload +def DEFINE_multi_float( # pylint: disable=invalid-name + name: str, + default: Iterable[float] | float | str, + help: str, # pylint: disable=redefined-builtin + lower_bound: float | None = ..., + upper_bound: float | None = ..., + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[float]]: + ... + + +def DEFINE_multi_float( # pylint: disable=invalid-name + name, + default, + help, # pylint: disable=redefined-builtin + lower_bound=None, + upper_bound=None, + flag_values=_flagvalues.FLAGS, + required=False, + **args +): + """Registers a flag whose value can be a list of arbitrary floats. + + Use the flag on the command line multiple times to place multiple + float values into the list. The 'default' may be a single float + (which will be converted into a single-element list) or a list of + floats. + + Args: + name: str, the flag name. + default: Union[Iterable[float], str, None], the default value of the flag; + see `DEFINE_multi`. + help: str, the help message. + lower_bound: float, min values of the flag. + upper_bound: float, max values of the flag. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: Dictionary with extra keyword args that are passed to the + ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + parser = _argument_parser.FloatParser(lower_bound, upper_bound) + serializer = _argument_parser.ArgumentSerializer() + return DEFINE_multi( + parser, + serializer, + name, + default, + help, + flag_values, + required=True if required else False, + **args, + ) + + +@overload +def DEFINE_multi_enum( # pylint: disable=invalid-name + name: str, + default: None | Iterable[str] | str, + enum_values: Iterable[str], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[str]]: + ... + + +@overload +def DEFINE_multi_enum( # pylint: disable=invalid-name + name: str, + default: None, + enum_values: Iterable[str], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[str] | None]: + ... + + +@overload +def DEFINE_multi_enum( # pylint: disable=invalid-name + name: str, + default: Iterable[str] | str, + enum_values: Iterable[str], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[str]]: + ... + + +def DEFINE_multi_enum( # pylint: disable=invalid-name + name, + default, + enum_values, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + case_sensitive=True, + required=False, + **args +): + """Registers a flag whose value can be a list strings from enum_values. + + Use the flag on the command line multiple times to place multiple + enum values into the list. The 'default' may be a single string + (which will be converted into a single-element list) or a list of + strings. + + Args: + name: str, the flag name. + default: Union[Iterable[str], str, None], the default value of the flag; see + `DEFINE_multi`. + enum_values: [str], a non-empty list of strings with the possible values for + the flag. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + case_sensitive: Whether or not the enum is to be case-sensitive. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: Dictionary with extra keyword args that are passed to the + ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + parser = _argument_parser.EnumParser(enum_values, case_sensitive) + serializer = _argument_parser.ArgumentSerializer() + return DEFINE_multi( + parser, + serializer, + name, + default, + '<%s>: %s' % ('|'.join(enum_values), help), + flag_values, + required=True if required else False, + **args, + ) + + +@overload +def DEFINE_multi_enum_class( # pylint: disable=invalid-name + name: str, + # This is separate from `Union[None, _ET, Iterable[str], str]` to avoid a + # Pytype issue inferring the return value to + # FlagHolder[List[Union[_ET, enum.Enum]]] when an iterable of concrete enum + # subclasses are used. + default: Iterable[_ET], + enum_class: type[_ET], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[_ET]]: + ... + + +@overload +def DEFINE_multi_enum_class( # pylint: disable=invalid-name + name: str, + default: None | _ET | Iterable[str] | str, + enum_class: type[_ET], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + *, + required: Literal[True], + **args: Any +) -> _flagvalues.FlagHolder[list[_ET]]: + ... + + +@overload +def DEFINE_multi_enum_class( # pylint: disable=invalid-name + name: str, + default: None, + enum_class: type[_ET], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[_ET] | None]: + ... + + +@overload +def DEFINE_multi_enum_class( # pylint: disable=invalid-name + name: str, + # This is separate from `Union[None, _ET, Iterable[str], str]` to avoid a + # Pytype issue inferring the return value to + # FlagHolder[List[Union[_ET, enum.Enum]]] when an iterable of concrete enum + # subclasses are used. + default: Iterable[_ET], + enum_class: type[_ET], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[_ET]]: + ... + + +@overload +def DEFINE_multi_enum_class( # pylint: disable=invalid-name + name: str, + default: _ET | Iterable[str] | str, + enum_class: type[_ET], + help: str, # pylint: disable=redefined-builtin + flag_values: _flagvalues.FlagValues = ..., + module_name: str | None = ..., + required: bool = ..., + **args: Any +) -> _flagvalues.FlagHolder[list[_ET]]: + ... + + +def DEFINE_multi_enum_class( # pylint: disable=invalid-name + name, + default, + enum_class, + help, # pylint: disable=redefined-builtin + flag_values=_flagvalues.FLAGS, + module_name=None, + case_sensitive=False, + required=False, + **args +): + """Registers a flag whose value can be a list of enum members. + + Use the flag on the command line multiple times to place multiple + enum values into the list. + + Args: + name: str, the flag name. + default: Union[Iterable[Enum], Iterable[str], Enum, str, None], the default + value of the flag; see `DEFINE_multi`; only differences are documented + here. If the value is a single Enum, it is treated as a single-item list + of that Enum value. If it is an iterable, text values within the iterable + will be converted to the equivalent Enum objects. + enum_class: class, the Enum class with all the possible values for the flag. + help: str, the help message. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + module_name: A string, the name of the Python module declaring this flag. If + not provided, it will be computed using the stack trace of this call. + case_sensitive: bool, whether to map strings to members of the enum_class + without considering case. + required: bool, is this a required flag. This must be used as a keyword + argument. + **args: Dictionary with extra keyword args that are passed to the + ``Flag.__init__``. + + Returns: + a handle to defined flag. + """ + # NOTE: pytype fails if this is a direct return. + result = DEFINE_flag( + _flag.MultiEnumClassFlag( + name, + default, + help, + enum_class, + case_sensitive=case_sensitive, + **args, + ), + flag_values, + module_name, + required=True if required else False, + ) + return result + + +def DEFINE_alias( # pylint: disable=invalid-name + name: str, + original_name: str, + flag_values: _flagvalues.FlagValues = _flagvalues.FLAGS, + module_name: str | None = None, +) -> _flagvalues.FlagHolder[Any]: + """Defines an alias flag for an existing one. + + Args: + name: str, the flag name. + original_name: str, the original flag name. + flag_values: :class:`FlagValues`, the FlagValues instance with which the + flag will be registered. This should almost never need to be overridden. + module_name: A string, the name of the module that defines this flag. + + Returns: + a handle to defined flag. + + Raises: + flags.FlagError: + UnrecognizedFlagError: if the referenced flag doesn't exist. + DuplicateFlagError: if the alias name has been used by some existing flag. + """ + if original_name not in flag_values: + raise _exceptions.UnrecognizedFlagError(original_name) + flag = flag_values[original_name] + + class _FlagAlias(_flag.Flag): + """Overrides Flag class so alias value is copy of original flag value.""" + + def parse(self, argument): + flag.parse(argument) + self.present += 1 + + def _parse_from_default(self, value): + # The value was already parsed by the aliased flag, so there is no + # need to call the parser on it a second time. + # Additionally, because of how MultiFlag parses and merges values, + # it isn't possible to delegate to the aliased flag and still get + # the correct values. + return value + + @property + def value(self): + return flag.value + + @value.setter + def value(self, value): + flag.value = value + + help_msg = 'Alias for --%s.' % flag.name + # If alias_name has been used, flags.DuplicatedFlag will be raised. + return DEFINE_flag( + _FlagAlias( + flag.parser, + flag.serializer, + name, + flag.default, + help_msg, + boolean=flag.boolean), flag_values, module_name) diff --git a/venv/Lib/site-packages/absl/flags/_exceptions.py b/venv/Lib/site-packages/absl/flags/_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..e9edfce684a7ea351be35eb96b32c441b2ccda2a --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/_exceptions.py @@ -0,0 +1,107 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exception classes in ABSL flags library. + +Do NOT import this module directly. Import the flags package and use the +aliases defined at the package level instead. +""" + +import sys + +from absl.flags import _helpers + + +_helpers.disclaim_module_ids.add(id(sys.modules[__name__])) + + +class Error(Exception): + """The base class for all flags errors.""" + + +class CantOpenFlagFileError(Error): + """Raised when flagfile fails to open. + + E.g. the file doesn't exist, or has wrong permissions. + """ + + +class DuplicateFlagError(Error): + """Raised if there is a flag naming conflict.""" + + @classmethod + def from_flag(cls, flagname, flag_values, other_flag_values=None): + """Creates a DuplicateFlagError by providing flag name and values. + + Args: + flagname: str, the name of the flag being redefined. + flag_values: :class:`FlagValues`, the FlagValues instance containing the + first definition of flagname. + other_flag_values: :class:`FlagValues`, if it is not None, it should be + the FlagValues object where the second definition of flagname occurs. + If it is None, we assume that we're being called when attempting to + create the flag a second time, and we use the module calling this one + as the source of the second definition. + + Returns: + An instance of DuplicateFlagError. + """ + first_module = flag_values.find_module_defining_flag( + flagname, default='') + if other_flag_values is None: + second_module = _helpers.get_calling_module() + else: + second_module = other_flag_values.find_module_defining_flag( + flagname, default='') + flag_summary = flag_values[flagname].help + msg = ("The flag '%s' is defined twice. First from %s, Second from %s. " + "Description from first occurrence: %s") % ( + flagname, first_module, second_module, flag_summary) + return cls(msg) + + +class IllegalFlagValueError(Error): + """Raised when the flag command line argument is illegal.""" + + +class UnrecognizedFlagError(Error): + """Raised when a flag is unrecognized. + + Attributes: + flagname: str, the name of the unrecognized flag. + flagvalue: The value of the flag, empty if the flag is not defined. + """ + + def __init__(self, flagname, flagvalue='', suggestions=None): + self.flagname = flagname + self.flagvalue = flagvalue + if suggestions: + # Space before the question mark is intentional to not include it in the + # selection when copy-pasting the suggestion from (some) terminals. + tip = '. Did you mean: %s ?' % ', '.join(suggestions) + else: + tip = '' + super().__init__("Unknown command line flag '%s'%s" % (flagname, tip)) + + +class UnparsedFlagAccessError(Error): + """Raised when accessing the flag value from unparsed :class:`FlagValues`.""" + + +class ValidationError(Error): + """Raised when flag validator constraint is not satisfied.""" + + +class FlagNameConflictsWithMethodError(Error): + """Raised when a flag name conflicts with :class:`FlagValues` methods.""" diff --git a/venv/Lib/site-packages/absl/flags/_flag.py b/venv/Lib/site-packages/absl/flags/_flag.py new file mode 100644 index 0000000000000000000000000000000000000000..21c10ef4e9fb4355cf0616c8d804e7a3c445bac6 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/_flag.py @@ -0,0 +1,566 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contains Flag class - information about single command-line flag. + +Do NOT import this module directly. Import the flags package and use the +aliases defined at the package level instead. +""" + +from collections.abc import Iterable +import copy +import enum +import functools +from typing import Any, Generic, TypeVar +from xml.dom import minidom + +from absl.flags import _argument_parser +from absl.flags import _exceptions +from absl.flags import _helpers + +_T = TypeVar('_T') +_ET = TypeVar('_ET', bound=enum.Enum) + + +@functools.total_ordering +class Flag(Generic[_T]): + """Information about a command-line flag. + + Attributes: + name: the name for this flag + default: the default value for this flag + default_unparsed: the unparsed default value for this flag. + default_as_str: default value as repr'd string, e.g., "'true'" + (or None) + value: the most recent parsed value of this flag set by :meth:`parse` + help: a help string or None if no help is available + short_name: the single letter alias for this flag (or None) + boolean: if 'true', this flag does not accept arguments + present: true if this flag was parsed from command line flags + parser: an :class:`~absl.flags.ArgumentParser` object + serializer: an ArgumentSerializer object + allow_override: the flag may be redefined without raising an error, + and newly defined flag overrides the old one. + allow_override_cpp: use the flag from C++ if available the flag + definition is replaced by the C++ flag after init + allow_hide_cpp: use the Python flag despite having a C++ flag with + the same name (ignore the C++ flag) + using_default_value: the flag value has not been set by user + allow_overwrite: the flag may be parsed more than once without + raising an error, the last set value will be used + allow_using_method_names: whether this flag can be defined even if + it has a name that conflicts with a FlagValues method. + validators: list of the flag validators. + + The only public method of a ``Flag`` object is :meth:`parse`, but it is + typically only called by a :class:`~absl.flags.FlagValues` object. The + :meth:`parse` method is a thin wrapper around the + :meth:`ArgumentParser.parse()` method. The + parsed value is saved in ``.value``, and the ``.present`` attribute is + updated. If this flag was already present, an Error is raised. + + :meth:`parse` is also called during ``__init__`` to parse the default value + and initialize the ``.value`` attribute. This enables other python modules to + safely use flags even if the ``__main__`` module neglects to parse the + command line arguments. The ``.present`` attribute is cleared after + ``__init__`` parsing. If the default value is set to ``None``, then the + ``__init__`` parsing step is skipped and the ``.value`` attribute is + initialized to None. + + Note: The default value is also presented to the user in the help + string, so it is important that it be a legal value for this flag. + """ + + # NOTE: pytype doesn't find defaults without this. + default: _T | None + default_as_str: str | None + default_unparsed: _T | None | str + + parser: _argument_parser.ArgumentParser[_T] + + def __init__( + self, + parser: _argument_parser.ArgumentParser[_T], + serializer: _argument_parser.ArgumentSerializer[_T] | None, + name: str, + default: _T | None | str, + help_string: str | None, + short_name: str | None = None, + boolean: bool = False, + allow_override: bool = False, + allow_override_cpp: bool = False, + allow_hide_cpp: bool = False, + allow_overwrite: bool = True, + allow_using_method_names: bool = False, + ) -> None: + self.name = name + + if not help_string: + help_string = '(no help available)' + + self.help = help_string + self.short_name = short_name + self.boolean = boolean + self.present = 0 + self.parser = parser # type: ignore[annotation-type-mismatch] + self.serializer = serializer + self.allow_override = allow_override + self.allow_override_cpp = allow_override_cpp + self.allow_hide_cpp = allow_hide_cpp + self.allow_overwrite = allow_overwrite + self.allow_using_method_names = allow_using_method_names + + self.using_default_value = True + self._value: _T | None = None + self.validators: list[Any] = [] + if self.allow_hide_cpp and self.allow_override_cpp: + raise _exceptions.Error( + "Can't have both allow_hide_cpp (means use Python flag) and " + 'allow_override_cpp (means use C++ flag after InitGoogle)') + + self._set_default(default) + + @property + def value(self) -> _T | None: + return self._value + + @value.setter + def value(self, value: _T | None): + self._value = value + + def __hash__(self): + return hash(id(self)) + + def __eq__(self, other): + return self is other + + def __lt__(self, other): + if isinstance(other, Flag): + return id(self) < id(other) + return NotImplemented + + def __bool__(self): + raise TypeError('A Flag instance would always be True. ' + 'Did you mean to test the `.value` attribute?') + + def __getstate__(self): + raise TypeError("can't pickle Flag objects") + + def __copy__(self): + raise TypeError('%s does not support shallow copies. ' + 'Use copy.deepcopy instead.' % type(self).__name__) + + def __deepcopy__(self, memo: dict[int, Any]) -> 'Flag[_T]': + result = object.__new__(type(self)) + result.__dict__ = copy.deepcopy(self.__dict__, memo) + return result + + def _get_parsed_value_as_string(self, value: _T | None) -> str | None: + """Returns parsed flag value as string.""" + if value is None: + return None + if self.serializer: + return repr(self.serializer.serialize(value)) + if self.boolean: + if value: + return repr('true') + else: + return repr('false') + return repr(str(value)) + + def parse(self, argument: str | _T) -> None: + """Parses string and sets flag value. + + Args: + argument: str or the correct flag value type, argument to be parsed. + """ + if self.present and not self.allow_overwrite: + raise _exceptions.IllegalFlagValueError( + 'flag --%s=%s: already defined as %s' % ( + self.name, argument, self.value)) + self.value = self._parse(argument) + self.present += 1 + + def _parse(self, argument: str | _T) -> _T | None: + """Internal parse function. + + It returns the parsed value, and does not modify class states. + + Args: + argument: str or the correct flag value type, argument to be parsed. + + Returns: + The parsed value. + """ + try: + return self.parser.parse(argument) # type: ignore[arg-type] + except (TypeError, ValueError, OverflowError) as e: + # Recast as IllegalFlagValueError. + raise _exceptions.IllegalFlagValueError( + 'flag --%s=%s: %s' % (self.name, argument, e)) + + def unparse(self) -> None: + self.value = self.default + self.using_default_value = True + self.present = 0 + + def serialize(self) -> str: + """Serializes the flag.""" + return self._serialize(self.value) + + def _serialize(self, value: _T | None) -> str: + """Internal serialize function.""" + if value is None: + return '' + if self.boolean: + if value: + return '--%s' % self.name + else: + return '--no%s' % self.name + else: + if not self.serializer: + raise _exceptions.Error( + 'Serializer not present for flag %s' % self.name) + return '--%s=%s' % (self.name, self.serializer.serialize(value)) + + def _set_default(self, value: _T | None | str) -> None: + """Changes the default value (and current value too) for this Flag.""" + self.default_unparsed = value + if value is None: + self.default = None + else: + self.default = self._parse_from_default(value) + self.default_as_str = self._get_parsed_value_as_string(self.default) + if self.using_default_value: + self.value = self.default + + # This is split out so that aliases can skip regular parsing of the default + # value. + def _parse_from_default(self, value: str | _T) -> _T | None: + return self._parse(value) + + def flag_type(self) -> str: + """Returns a str that describes the type of the flag. + + NOTE: we use strings, and not the types.*Type constants because + our flags can have more exotic types, e.g., 'comma separated list + of strings', 'whitespace separated list of strings', etc. + """ + return self.parser.flag_type() + + def _create_xml_dom_element( + self, doc: minidom.Document, module_name: str, is_key: bool = False + ) -> minidom.Element: + """Returns an XML element that contains this flag's information. + + This is information that is relevant to all flags (e.g., name, + meaning, etc.). If you defined a flag that has some other pieces of + info, then please override _ExtraXMLInfo. + + Please do NOT override this method. + + Args: + doc: minidom.Document, the DOM document it should create nodes from. + module_name: str,, the name of the module that defines this flag. + is_key: boolean, True iff this flag is key for main module. + + Returns: + A minidom.Element instance. + """ + element = doc.createElement('flag') + if is_key: + element.appendChild(_helpers.create_xml_dom_element(doc, 'key', 'yes')) + element.appendChild(_helpers.create_xml_dom_element( + doc, 'file', module_name)) + # Adds flag features that are relevant for all flags. + element.appendChild(_helpers.create_xml_dom_element(doc, 'name', self.name)) + if self.short_name: + element.appendChild(_helpers.create_xml_dom_element( + doc, 'short_name', self.short_name)) + if self.help: + element.appendChild(_helpers.create_xml_dom_element( + doc, 'meaning', self.help)) + # The default flag value can either be represented as a string like on the + # command line, or as a Python object. We serialize this value in the + # latter case in order to remain consistent. + if self.serializer and not isinstance(self.default, str): + if self.default is not None: + default_serialized = self.serializer.serialize(self.default) + else: + default_serialized = '' + else: + default_serialized = self.default # type: ignore[assignment] + element.appendChild(_helpers.create_xml_dom_element( + doc, 'default', default_serialized)) + value_serialized = self._serialize_value_for_xml(self.value) + element.appendChild(_helpers.create_xml_dom_element( + doc, 'current', value_serialized)) + element.appendChild(_helpers.create_xml_dom_element( + doc, 'type', self.flag_type())) + # Adds extra flag features this flag may have. + for e in self._extra_xml_dom_elements(doc): + element.appendChild(e) + return element + + def _serialize_value_for_xml(self, value: _T | None) -> Any: + """Returns the serialized value, for use in an XML help text.""" + return value + + def _extra_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + """Returns extra info about this flag in XML. + + "Extra" means "not already included by _create_xml_dom_element above." + + Args: + doc: minidom.Document, the DOM document it should create nodes from. + + Returns: + A list of minidom.Element. + """ + # Usually, the parser knows the extra details about the flag, so + # we just forward the call to it. + return self.parser._custom_xml_dom_elements(doc) # pylint: disable=protected-access + + +class BooleanFlag(Flag[bool]): + """Basic boolean flag. + + Boolean flags do not take any arguments, and their value is either + ``True`` (1) or ``False`` (0). The false value is specified on the command + line by prepending the word ``'no'`` to either the long or the short flag + name. + + For example, if a Boolean flag was created whose long name was + ``'update'`` and whose short name was ``'x'``, then this flag could be + explicitly unset through either ``--noupdate`` or ``--nox``. + """ + + def __init__( + self, + name: str, + default: bool | None | str, + help: str | None, # pylint: disable=redefined-builtin + short_name: str | None = None, + **args + ) -> None: + p = _argument_parser.BooleanParser() + super().__init__(p, None, name, default, help, short_name, True, **args) + + +class EnumFlag(Flag[str]): + """Basic enum flag; its value can be any string from list of enum_values.""" + + parser: _argument_parser.EnumParser + + def __init__( + self, + name: str, + default: str | None, + help: str | None, # pylint: disable=redefined-builtin + enum_values: Iterable[str], + short_name: str | None = None, + case_sensitive: bool = True, + **args + ): + p = _argument_parser.EnumParser(enum_values, case_sensitive) + g: _argument_parser.ArgumentSerializer[str] + g = _argument_parser.ArgumentSerializer() + super().__init__(p, g, name, default, help, short_name, **args) + self.parser = p + self.help = '<%s>: %s' % ('|'.join(p.enum_values), self.help) + + def _extra_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + elements = [] + for enum_value in self.parser.enum_values: + elements.append(_helpers.create_xml_dom_element( + doc, 'enum_value', enum_value)) + return elements + + +class EnumClassFlag(Flag[_ET]): + """Basic enum flag; its value is an enum class's member.""" + + parser: _argument_parser.EnumClassParser + + def __init__( + self, + name: str, + default: _ET | None | str, + help: str | None, # pylint: disable=redefined-builtin + enum_class: type[_ET], + short_name: str | None = None, + case_sensitive: bool = False, + **args + ): + p = _argument_parser.EnumClassParser( + enum_class, case_sensitive=case_sensitive + ) + g: _argument_parser.EnumClassSerializer[_ET] + g = _argument_parser.EnumClassSerializer(lowercase=not case_sensitive) + super().__init__(p, g, name, default, help, short_name, **args) + self.parser = p + self.help = '<%s>: %s' % ('|'.join(p.member_names), self.help) + + def _extra_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + elements = [] + for enum_value in self.parser.enum_class.__members__.keys(): + elements.append(_helpers.create_xml_dom_element( + doc, 'enum_value', enum_value)) + return elements + + +class MultiFlag(Generic[_T], Flag[list[_T]]): + """A flag that can appear multiple time on the command-line. + + The value of such a flag is a list that contains the individual values + from all the appearances of that flag on the command-line. + + See the __doc__ for Flag for most behavior of this class. Only + differences in behavior are described here: + + * The default value may be either a single value or an iterable of values. + A single value is transformed into a single-item list of that value. + + * The value of the flag is always a list, even if the option was + only supplied once, and even if the default value is a single + value + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.help += ';\n repeat this option to specify a list of values' + + def parse(self, arguments: str | _T | Iterable[_T]): # pylint: disable=arguments-renamed + """Parses one or more arguments with the installed parser. + + Args: + arguments: a single argument or a list of arguments (typically a + list of default values); a single argument is converted + internally into a list containing one item. + """ + new_values = self._parse(arguments) + if self.present: + assert self.value is not None + self.value.extend(new_values) + else: + self.value = new_values + self.present += len(new_values) + + def _parse(self, arguments: str | _T | Iterable[_T]) -> list[_T]: # pylint: disable=arguments-renamed + arguments_list: list[str | _T] + + if isinstance(arguments, str): + arguments_list = [arguments] + + elif isinstance(arguments, Iterable): + arguments_list = list(arguments) + + else: + # Default value may be a list of values. Most other arguments + # will not be, so convert them into a single-item list to make + # processing simpler below. + arguments_list = [arguments] + + return [super(MultiFlag, self)._parse(item) for item in arguments_list] # type: ignore + + def _serialize(self, value: list[_T] | None) -> str: + """See base class.""" + if not self.serializer: + raise _exceptions.Error( + 'Serializer not present for flag %s' % self.name) + if value is None: + return '' + + serialized_items = [ + super(MultiFlag, self)._serialize(value_item) # type: ignore[arg-type] + for value_item in value + ] + + return '\n'.join(serialized_items) + + def flag_type(self): + """See base class.""" + return 'multi ' + self.parser.flag_type() + + def _extra_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + elements = [] + if hasattr(self.parser, 'enum_values'): + for enum_value in self.parser.enum_values: # pytype: disable=attribute-error + elements.append(_helpers.create_xml_dom_element( + doc, 'enum_value', enum_value)) + return elements + + +class MultiEnumClassFlag(MultiFlag[_ET]): # pytype: disable=not-indexable + """A multi_enum_class flag. + + See the __doc__ for MultiFlag for most behaviors of this class. In addition, + this class knows how to handle enum.Enum instances as values for this flag + type. + """ + + parser: _argument_parser.EnumClassParser[_ET] # type: ignore[assignment] + + def __init__( + self, + name: str, + default: None | Iterable[_ET] | _ET | Iterable[str] | str, + help_string: str, + enum_class: type[_ET], + case_sensitive: bool = False, + **args + ): + p = _argument_parser.EnumClassParser( + enum_class, case_sensitive=case_sensitive) + g: _argument_parser.EnumClassListSerializer + g = _argument_parser.EnumClassListSerializer( + list_sep=',', lowercase=not case_sensitive) + super().__init__(p, g, name, default, help_string, **args) + # NOTE: parser should be typed EnumClassParser[_ET] but the constructor + # restricts the available interface to ArgumentParser[str]. + self.parser = p + # NOTE: serializer should be non-Optional but this isn't inferred. + self.serializer = g + self.help = ( + '<%s>: %s;\n repeat this option to specify a list of values' % + ('|'.join(p.member_names), help_string or '(no help available)')) + + def _extra_xml_dom_elements( + self, doc: minidom.Document + ) -> list[minidom.Element]: + elements = [] + for enum_value in self.parser.enum_class.__members__.keys(): # pytype: disable=attribute-error + elements.append(_helpers.create_xml_dom_element( + doc, 'enum_value', enum_value)) + return elements + + def _serialize_value_for_xml(self, value): + """See base class.""" + if value is not None: + if not self.serializer: + raise _exceptions.Error( + 'Serializer not present for flag %s' % self.name + ) + value_serialized = self.serializer.serialize(value) + else: + value_serialized = '' + return value_serialized diff --git a/venv/Lib/site-packages/absl/flags/_flagvalues.py b/venv/Lib/site-packages/absl/flags/_flagvalues.py new file mode 100644 index 0000000000000000000000000000000000000000..fa0e9f68ec7bac69e1bffc7ceb1bc664ddf2eee5 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/_flagvalues.py @@ -0,0 +1,1552 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Defines the FlagValues class - registry of 'Flag' objects. + +Do NOT import this module directly. Import the flags package and use the +aliases defined at the package level instead. +""" + +from collections.abc import Callable, Iterable, Iterator, Sequence +import copy +from importlib import abc +import logging +import os +import sys +from typing import Any, Generic, TextIO, TypeVar +from xml.dom import minidom + +from absl.flags import _exceptions +from absl.flags import _flag +from absl.flags import _helpers +from absl.flags import _validators_classes +from absl.flags._flag import Flag + +# Add flagvalues module to disclaimed module ids. +_helpers.disclaim_module_ids.add(id(sys.modules[__name__])) + +_T = TypeVar('_T') +_T_co = TypeVar('_T_co', covariant=True) # pytype: disable=not-supported-yet + + +class ReloadDetector(abc.MetaPathFinder): + """Helper class for detecting reloads.""" + + def __init__(self): + self.reloading_modules = set() + + def find_spec(self, fullname, path, target=None): + if fullname in sys.modules: # Indicates a reload. + self.reloading_modules.add(fullname) + return None + + +reload_detector = ReloadDetector() + +# Register the hook by inserting it right before the last path finder. +# This should play nicely with lazy imports. +sys.meta_path.insert(-1, reload_detector) + + +class FlagValues: + """Registry of :class:`~absl.flags.Flag` objects. + + A :class:`FlagValues` can then scan command line arguments, passing flag + arguments through to the 'Flag' objects that it owns. It also + provides easy access to the flag values. Typically only one + :class:`FlagValues` object is needed by an application: + :const:`FLAGS`. + + This class is heavily overloaded: + + :class:`Flag` objects are registered via ``__setitem__``:: + + FLAGS['longname'] = x # register a new flag + + The ``.value`` attribute of the registered :class:`~absl.flags.Flag` objects + can be accessed as attributes of this :class:`FlagValues` object, through + ``__getattr__``. Both the long and short name of the original + :class:`~absl.flags.Flag` objects can be used to access its value:: + + FLAGS.longname # parsed flag value + FLAGS.x # parsed flag value (short name) + + Command line arguments are scanned and passed to the registered + :class:`~absl.flags.Flag` objects through the ``__call__`` method. Unparsed + arguments, including ``argv[0]`` (e.g. the program name) are returned:: + + argv = FLAGS(sys.argv) # scan command line arguments + + The original registered :class:`~absl.flags.Flag` objects can be retrieved + through the use of the dictionary-like operator, ``__getitem__``:: + + x = FLAGS['longname'] # access the registered Flag object + + The ``str()`` operator of a :class:`absl.flags.FlagValues` object provides + help for all of the registered :class:`~absl.flags.Flag` objects. + """ + + _HAS_DYNAMIC_ATTRIBUTES = True + + # A note on collections.abc.Mapping: + # FlagValues defines __getitem__, __iter__, and __len__. It makes perfect + # sense to let it be a collections.abc.Mapping class. However, we are not + # able to do so. The mixin methods, e.g. keys, values, are not uncommon flag + # names. Those flag values would not be accessible via the FLAGS.xxx form. + + __dict__: dict[str, Any] + + def __init__(self): + # Since everything in this class is so heavily overloaded, the only + # way of defining and using fields is to access __dict__ directly. + + # Dictionary: flag name (string) -> Flag object. + self.__dict__['__flags'] = {} + + # Set: name of hidden flag (string). + # Holds flags that should not be directly accessible from Python. + self.__dict__['__hiddenflags'] = set() + + # Dictionary: module name (string) -> list of Flag objects that are defined + # by that module. + self.__dict__['__flags_by_module'] = {} + # Dictionary: module id (int) -> list of Flag objects that are defined by + # that module. + self.__dict__['__flags_by_module_id'] = {} + # Dictionary: module name (string) -> list of Flag objects that are + # key for that module. + self.__dict__['__key_flags_by_module'] = {} + + # Bool: True if flags were parsed. + self.__dict__['__flags_parsed'] = False + + # Bool: True if unparse_flags() was called. + self.__dict__['__unparse_flags_called'] = False + + # None or Method(name, value) to call from __setattr__ for an unknown flag. + self.__dict__['__set_unknown'] = None + + # A set of banned flag names. This is to prevent users from accidentally + # defining a flag that has the same name as a method on this class. + # Users can still allow defining the flag by passing + # allow_using_method_names=True in DEFINE_xxx functions. + self.__dict__['__banned_flag_names'] = frozenset(dir(FlagValues)) + + # Bool: Whether to use GNU style scanning. + self.__dict__['__use_gnu_getopt'] = True + + # Bool: Whether use_gnu_getopt has been explicitly set by the user. + self.__dict__['__use_gnu_getopt_explicitly_set'] = False + + # Function: Takes a flag name as parameter, returns a tuple + # (is_retired, type_is_bool). + self.__dict__['__is_retired_flag_func'] = None + + def set_gnu_getopt(self, gnu_getopt: bool = True) -> None: + """Sets whether or not to use GNU style scanning. + + GNU style allows mixing of flag and non-flag arguments. See + http://docs.python.org/library/getopt.html#getopt.gnu_getopt + + Args: + gnu_getopt: bool, whether or not to use GNU style scanning. + """ + self.__dict__['__use_gnu_getopt'] = gnu_getopt + self.__dict__['__use_gnu_getopt_explicitly_set'] = True + + def is_gnu_getopt(self) -> bool: + return self.__dict__['__use_gnu_getopt'] + + def _flags(self) -> dict[str, Flag]: + return self.__dict__['__flags'] + + def flags_by_module_dict(self) -> dict[str, list[Flag]]: + """Returns the dictionary of module_name -> list of defined flags. + + Returns: + A dictionary. Its keys are module names (strings). Its values + are lists of Flag objects. + """ + return self.__dict__['__flags_by_module'] + + def flags_by_module_id_dict(self) -> dict[int, list[Flag]]: + """Returns the dictionary of module_id -> list of defined flags. + + Returns: + A dictionary. Its keys are module IDs (ints). Its values + are lists of Flag objects. + """ + return self.__dict__['__flags_by_module_id'] + + def key_flags_by_module_dict(self) -> dict[str, list[Flag]]: + """Returns the dictionary of module_name -> list of key flags. + + Returns: + A dictionary. Its keys are module names (strings). Its values + are lists of Flag objects. + """ + return self.__dict__['__key_flags_by_module'] + + def register_flag_by_module(self, module_name: str, flag: Flag) -> None: + """Records the module that defines a specific flag. + + We keep track of which flag is defined by which module so that we + can later sort the flags by module. + + Args: + module_name: str, the name of a Python module. + flag: Flag, the Flag instance that is key to the module. + """ + flags_by_module = self.flags_by_module_dict() + flags_by_module.setdefault(module_name, []).append(flag) + + def register_flag_by_module_id(self, module_id: int, flag: Flag) -> None: + """Records the module that defines a specific flag. + + Args: + module_id: int, the ID of the Python module. + flag: Flag, the Flag instance that is key to the module. + """ + flags_by_module_id = self.flags_by_module_id_dict() + flags_by_module_id.setdefault(module_id, []).append(flag) + + def register_key_flag_for_module(self, module_name: str, flag: Flag) -> None: + """Specifies that a flag is a key flag for a module. + + Args: + module_name: str, the name of a Python module. + flag: Flag, the Flag instance that is key to the module. + """ + key_flags_by_module = self.key_flags_by_module_dict() + # The list of key flags for the module named module_name. + key_flags = key_flags_by_module.setdefault(module_name, []) + # Add flag, but avoid duplicates. + if flag not in key_flags: + key_flags.append(flag) + + def _flag_is_registered(self, flag_obj: Flag) -> bool: + """Checks whether a Flag object is registered under long name or short name. + + Args: + flag_obj: Flag, the Flag instance to check for. + + Returns: + bool, True iff flag_obj is registered under long name or short name. + """ + flag_dict = self._flags() + # Check whether flag_obj is registered under its long name. + name = flag_obj.name + if name in flag_dict and flag_dict[name] == flag_obj: + return True + # Check whether flag_obj is registered under its short name. + short_name = flag_obj.short_name + if ( + short_name is not None + and short_name in flag_dict + and flag_dict[short_name] == flag_obj + ): + return True + return False + + def _cleanup_unregistered_flag_from_module_dicts( + self, flag_obj: Flag + ) -> None: + """Cleans up unregistered flags from all module -> [flags] dictionaries. + + If flag_obj is registered under either its long name or short name, it + won't be removed from the dictionaries. + + Args: + flag_obj: Flag, the Flag instance to clean up for. + """ + if self._flag_is_registered(flag_obj): + return + # Materialize dict values to list to avoid concurrent modification. + for flags_in_module in [ + *self.flags_by_module_dict().values(), + *self.flags_by_module_id_dict().values(), + *self.key_flags_by_module_dict().values(), + ]: + # While (as opposed to if) takes care of multiple occurrences of a + # flag in the list for the same module. + while flag_obj in flags_in_module: + flags_in_module.remove(flag_obj) + + def get_flags_for_module(self, module: str | Any) -> list[Flag]: + """Returns the list of flags defined by a module. + + Args: + module: module|str, the module to get flags from. + + Returns: + [Flag], a new list of Flag instances. Caller may update this list as + desired: none of those changes will affect the internals of this + FlagValue instance. + """ + if not isinstance(module, str): + module = module.__name__ + if module == '__main__': + module = sys.argv[0] + + return list(self.flags_by_module_dict().get(module, [])) + + def get_key_flags_for_module(self, module: str | Any) -> list[Flag]: + """Returns the list of key flags for a module. + + Args: + module: module|str, the module to get key flags from. + + Returns: + [Flag], a new list of Flag instances. Caller may update this list as + desired: none of those changes will affect the internals of this + FlagValue instance. + """ + if not isinstance(module, str): + module = module.__name__ + if module == '__main__': + module = sys.argv[0] + + # Any flag is a key flag for the module that defined it. NOTE: + # key_flags is a fresh list: we can update it without affecting the + # internals of this FlagValues object. + key_flags = self.get_flags_for_module(module) + + # Take into account flags explicitly declared as key for a module. + for flag in self.key_flags_by_module_dict().get(module, []): + if flag not in key_flags: + key_flags.append(flag) + return key_flags + + # TODO(yileiyang): Restrict default to Optional[str]. + def find_module_defining_flag( + self, flagname: str, default: _T | None = None + ) -> str | _T | None: + """Return the name of the module defining this flag, or default. + + Args: + flagname: str, name of the flag to lookup. + default: Value to return if flagname is not defined. Defaults to None. + + Returns: + The name of the module which registered the flag with this name. + If no such module exists (i.e. no flag with this name exists), + we return default. + """ + registered_flag = self._flags().get(flagname) + if registered_flag is None: + return default + for module, flags in self.flags_by_module_dict().items(): + for flag in flags: + # It must compare the flag with the one in _flags. This is because a + # flag might be overridden only for its long name (or short name), + # and only its short name (or long name) is considered registered. + if ( + flag.name == registered_flag.name + and flag.short_name == registered_flag.short_name + ): + return module + return default + + # TODO(yileiyang): Restrict default to Optional[str]. + def find_module_id_defining_flag( + self, flagname: str, default: _T | None = None + ) -> int | _T | None: + """Return the ID of the module defining this flag, or default. + + Args: + flagname: str, name of the flag to lookup. + default: Value to return if flagname is not defined. Defaults to None. + + Returns: + The ID of the module which registered the flag with this name. + If no such module exists (i.e. no flag with this name exists), + we return default. + """ + registered_flag = self._flags().get(flagname) + if registered_flag is None: + return default + for module_id, flags in self.flags_by_module_id_dict().items(): + for flag in flags: + # It must compare the flag with the one in _flags. This is because a + # flag might be overridden only for its long name (or short name), + # and only its short name (or long name) is considered registered. + if ( + flag.name == registered_flag.name + and flag.short_name == registered_flag.short_name + ): + return module_id + return default + + def _register_unknown_flag_setter( + self, setter: Callable[[str, Any], None] + ) -> None: + """Allow set default values for undefined flags. + + Args: + setter: Method(name, value) to call to __setattr__ an unknown flag. Must + raise NameError or ValueError for invalid name/value. + """ + self.__dict__['__set_unknown'] = setter + + def _set_unknown_flag(self, name: str, value: _T) -> _T: + """Returns value if setting flag |name| to |value| returned True. + + Args: + name: str, name of the flag to set. + value: Value to set. + + Returns: + Flag value on successful call. + + Raises: + UnrecognizedFlagError + IllegalFlagValueError + """ + setter = self.__dict__['__set_unknown'] + if setter: + try: + setter(name, value) + return value + except (TypeError, ValueError): # Flag value is not valid. + raise _exceptions.IllegalFlagValueError( + f'"{value}" is not valid for --{name}' + ) + except NameError: # Flag name is not valid. + pass + raise _exceptions.UnrecognizedFlagError(name, value) + + def append_flag_values(self, flag_values: 'FlagValues') -> None: + """Appends flags registered in another FlagValues instance. + + Args: + flag_values: FlagValues, the FlagValues instance from which to copy flags. + """ + for flag_name, flag in flag_values._flags().items(): # pylint: disable=protected-access + # Each flags with short_name appears here twice (once under its + # normal name, and again with its short name). To prevent + # problems (DuplicateFlagError) with double flag registration, we + # perform a check to make sure that the entry we're looking at is + # for its normal name. + if flag_name == flag.name: + try: + self[flag_name] = flag + except _exceptions.DuplicateFlagError: + raise _exceptions.DuplicateFlagError.from_flag( + flag_name, self, other_flag_values=flag_values + ) + + def remove_flag_values( + self, flag_values: 'FlagValues | Iterable[str]' + ) -> None: + """Remove flags that were previously appended from another FlagValues. + + Args: + flag_values: FlagValues, the FlagValues instance containing flags to + remove. + """ + for flag_name in flag_values: + self.__delattr__(flag_name) + + def __setitem__(self, name: str, flag: Flag) -> None: + """Registers a new flag variable.""" + fl = self._flags() + if not isinstance(flag, _flag.Flag): + raise _exceptions.IllegalFlagValueError( + f'Expect Flag instances, found type {type(flag)}. ' + "Maybe you didn't mean to use FlagValue.__setitem__?" + ) + if not isinstance(name, str): + raise _exceptions.Error('Flag name must be a string') + if not name: + raise _exceptions.Error('Flag name cannot be empty') + if ' ' in name: + raise _exceptions.Error('Flag name cannot contain a space') + self._check_method_name_conflicts(name, flag) + if name in fl and not flag.allow_override and not fl[name].allow_override: + module, module_name = _helpers.get_calling_module_object_and_name() + if self.find_module_defining_flag(name) == module_name and ( + id(module) != self.find_module_id_defining_flag(name) + or module_name in reload_detector.reloading_modules + ): + # If the flag has already been defined by a module with the same name, + # but a different ID, we can stop here because it indicates that the + # module is simply being imported a subsequent time. + # In case the module is being reloaded (using `importlib.reload`), it'll + # have the same ID, so we detect it using reload_detector. + return + raise _exceptions.DuplicateFlagError.from_flag(name, self) + # If a new flag overrides an old one, we need to cleanup the old flag's + # modules if it's not registered. + flags_to_cleanup = set() + short_name: str | None = flag.short_name + if short_name is not None: + if ( + short_name in fl + and not flag.allow_override + and not fl[short_name].allow_override + ): + raise _exceptions.DuplicateFlagError.from_flag(short_name, self) + if short_name in fl and fl[short_name] != flag: + flags_to_cleanup.add(fl[short_name]) + fl[short_name] = flag + if ( + name not in fl # new flag + or fl[name].using_default_value + or not flag.using_default_value + ): + if name in fl and fl[name] != flag: + flags_to_cleanup.add(fl[name]) + fl[name] = flag + for f in flags_to_cleanup: + self._cleanup_unregistered_flag_from_module_dicts(f) + + def __dir__(self) -> list[str]: + """Returns list of names of all defined flags. + + Useful for TAB-completion in ipython. + + Returns: + [str], a list of names of all defined flags. + """ + return sorted(self._flags()) + + def __getitem__(self, name: str) -> Flag: + """Returns the Flag object for the flag --name.""" + return self._flags()[name] + + def _hide_flag(self, name): + """Marks the flag --name as hidden.""" + self.__dict__['__hiddenflags'].add(name) + + def __getattr__(self, name: str) -> Any: + """Retrieves the 'value' attribute of the flag --name.""" + flag_entry = self._flags().get(name) + if flag_entry is None: + raise AttributeError(name) + if name in self.__dict__['__hiddenflags']: + raise AttributeError(name) + + if self.__dict__['__flags_parsed'] or flag_entry.present: + return flag_entry.value + else: + raise _exceptions.UnparsedFlagAccessError( + 'Trying to access flag --%s before flags were parsed.' % name + ) + + def __setattr__(self, name: str, value: _T) -> _T: + """Sets the 'value' attribute of the flag --name.""" + self._set_attributes(**{name: value}) + return value + + def _set_attributes(self, **attributes: Any) -> None: + """Sets multiple flag values together, triggers validators afterwards.""" + fl = self._flags() + known_flag_vals = {} + known_flag_used_defaults = {} + try: + for name, value in attributes.items(): + if name in self.__dict__['__hiddenflags']: + raise AttributeError(name) + flag_entry = fl.get(name) + if flag_entry is not None: + orig = flag_entry.value + flag_entry.value = value + known_flag_vals[name] = orig + else: + self._set_unknown_flag(name, value) + for name in known_flag_vals: + self._assert_validators(fl[name].validators) + known_flag_used_defaults[name] = fl[name].using_default_value + fl[name].using_default_value = False + except: + for name, orig in known_flag_vals.items(): + fl[name].value = orig + for name, orig in known_flag_used_defaults.items(): + fl[name].using_default_value = orig + # NOTE: We do not attempt to undo unknown flag side effects because we + # cannot reliably undo the user-configured behavior. + raise + + def validate_all_flags(self) -> None: + """Verifies whether all flags pass validation. + + Raises: + AttributeError: Raised if validators work with a non-existing flag. + IllegalFlagValueError: Raised if validation fails for at least one + validator. + """ + all_validators = set() + for flag in self._flags().values(): + all_validators.update(flag.validators) + self._assert_validators(all_validators) + + def _assert_validators( + self, validators: Iterable[_validators_classes.Validator] + ) -> None: + """Asserts if all validators in the list are satisfied. + + It asserts validators in the order they were created. + + Args: + validators: Iterable(validators.Validator), validators to be verified. + + Raises: + AttributeError: Raised if validators work with a non-existing flag. + IllegalFlagValueError: Raised if validation fails for at least one + validator. + """ + messages = [] + bad_flags: set[str] = set() + for validator in sorted( + validators, key=lambda validator: validator.insertion_index + ): + try: + if isinstance(validator, _validators_classes.SingleFlagValidator): + if validator.flag_name in bad_flags: + continue + elif isinstance(validator, _validators_classes.MultiFlagsValidator): + if bad_flags & set(validator.flag_names): + continue + validator.verify(self) + except _exceptions.ValidationError as e: + if isinstance(validator, _validators_classes.SingleFlagValidator): + bad_flags.add(validator.flag_name) + elif isinstance(validator, _validators_classes.MultiFlagsValidator): + bad_flags.update(set(validator.flag_names)) + message = validator.print_flags_with_values(self) + messages.append('%s: %s' % (message, str(e))) + if messages: + raise _exceptions.IllegalFlagValueError('\n'.join(messages)) + + def __delattr__(self, flag_name: str) -> None: + """Deletes a previously-defined flag from a flag object. + + This method makes sure we can delete a flag by using + + del FLAGS. + + E.g., + + flags.DEFINE_integer('foo', 1, 'Integer flag.') + del flags.FLAGS.foo + + If a flag is also registered by its the other name (long name or short + name), the other name won't be deleted. + + Args: + flag_name: str, the name of the flag to be deleted. + + Raises: + AttributeError: Raised when there is no registered flag named flag_name. + """ + fl = self._flags() + flag_entry = fl.get(flag_name) + if flag_entry is None: + raise AttributeError(flag_name) + del fl[flag_name] + + self._cleanup_unregistered_flag_from_module_dicts(flag_entry) + + def set_default(self, name: str, value: Any) -> None: + """Changes the default value of the named flag object. + + The flag's current value is also updated if the flag is currently using + the default value, i.e. not specified in the command line, and not set + by FLAGS.name = value. + + Args: + name: str, the name of the flag to modify. + value: The new default value. + + Raises: + UnrecognizedFlagError: Raised when there is no registered flag named name. + IllegalFlagValueError: Raised when value is not valid. + """ + fl = self._flags() + flag_entry = fl.get(name) + if flag_entry is None: + self._set_unknown_flag(name, value) + return + flag_entry._set_default(value) # pylint: disable=protected-access + self._assert_validators(flag_entry.validators) + + def __contains__(self, name: str) -> bool: + """Returns True if name is a value (flag) in the dict.""" + return name in self._flags() + + def __len__(self) -> int: + return len(self.__dict__['__flags']) + + def __iter__(self) -> Iterator[str]: + return iter(self._flags()) + + def __call__( + self, argv: Sequence[str], known_only: bool = False + ) -> list[str]: + """Parses flags from argv; stores parsed flags into this FlagValues object. + + All unparsed arguments are returned. + + Args: + argv: a tuple/list of strings. + known_only: bool, if True, parse and remove known flags; return the rest + untouched. Unknown flags specified by --undefok are not returned. + + Returns: + The list of arguments not parsed as options, including argv[0]. + + Raises: + Error: Raised on any parsing error. + TypeError: Raised on passing wrong type of arguments. + ValueError: Raised on flag value parsing error. + """ + if isinstance(argv, (str, bytes)): + raise TypeError( + 'argv should be a tuple/list of strings, not bytes or string.' + ) + if not argv: + raise ValueError( + 'argv cannot be an empty list, and must contain the program name as ' + 'the first element.' + ) + + # This pre parses the argv list for --flagfile=<> options. + program_name = argv[0] + args = self.read_flags_from_files(argv[1:], force_gnu=False) + + # Parse the arguments. + unknown_flags, unparsed_args = self._parse_args(args, known_only) + + # Handle unknown flags by raising UnrecognizedFlagError. + # Note some users depend on us raising this particular error. + for name, value in unknown_flags: + suggestions = _helpers.get_flag_suggestions(name, list(self)) + raise _exceptions.UnrecognizedFlagError( + name, value, suggestions=suggestions + ) + + self.mark_as_parsed() + self.validate_all_flags() + return [program_name] + unparsed_args + + def __getstate__(self) -> Any: + raise TypeError("can't pickle FlagValues") + + def __copy__(self) -> Any: + raise TypeError( + 'FlagValues does not support shallow copies. ' + 'Use absl.testing.flagsaver or copy.deepcopy instead.' + ) + + def __deepcopy__(self, memo) -> Any: + result = object.__new__(type(self)) + result.__dict__.update(copy.deepcopy(self.__dict__, memo)) + return result + + def _set_is_retired_flag_func(self, is_retired_flag_func): + """Sets a function for checking retired flags. + + Do not use it. This is a private absl API used to check retired flags + registered by the absl C++ flags library. + + Args: + is_retired_flag_func: Callable(str) -> (bool, bool), a function takes flag + name as parameter, returns a tuple (is_retired, type_is_bool). + """ + self.__dict__['__is_retired_flag_func'] = is_retired_flag_func + + def _parse_args( + self, args: list[str], known_only: bool + ) -> tuple[list[tuple[str, Any]], list[str]]: + """Helper function to do the main argument parsing. + + This function goes through args and does the bulk of the flag parsing. + It will find the corresponding flag in our flag dictionary, and call its + .parse() method on the flag value. + + Args: + args: [str], a list of strings with the arguments to parse. + known_only: bool, if True, parse and remove known flags; return the rest + untouched. Unknown flags specified by --undefok are not returned. + + Returns: + A tuple with the following: + unknown_flags: List of (flag name, arg) for flags we don't know about. + unparsed_args: List of arguments we did not parse. + + Raises: + Error: Raised on any parsing error. + ValueError: Raised on flag value parsing error. + """ + unparsed_names_and_args: list[tuple[str | None, str]] = [] + undefok: set[str] = set() + retired_flag_func = self.__dict__['__is_retired_flag_func'] + + flag_dict = self._flags() + args_it = iter(args) + del args + for arg in args_it: + value = None + + def get_value() -> str: + try: + return next(args_it) if value is None else value # pylint: disable=cell-var-from-loop + except StopIteration: + raise _exceptions.Error('Missing value for flag ' + arg) from None # pylint: disable=cell-var-from-loop + + if not arg.startswith('-'): + # A non-argument: default is break, GNU is skip. + unparsed_names_and_args.append((None, arg)) + if self.is_gnu_getopt(): + continue + else: + break + + if arg == '--': + if known_only: + unparsed_names_and_args.append((None, arg)) + break + + # At this point, arg must start with '-'. + if arg.startswith('--'): + arg_without_dashes = arg[2:] + else: + arg_without_dashes = arg[1:] + + if '=' in arg_without_dashes: + name, value = arg_without_dashes.split('=', 1) + else: + name, value = arg_without_dashes, None + + if not name: + # The argument is all dashes (including one dash). + unparsed_names_and_args.append((None, arg)) + if self.is_gnu_getopt(): + continue + else: + break + + # --undefok is a special case. + if name == 'undefok': + value = get_value() + undefok.update(v.strip() for v in value.split(',')) + undefok.update('no' + v.strip() for v in value.split(',')) + continue + + flag = flag_dict.get(name) + if flag is not None: + if flag.boolean and value is None: + value = 'true' + else: + value = get_value() + elif name.startswith('no') and len(name) > 2: + # Boolean flags can take the form of --noflag, with no value. + noflag = flag_dict.get(name[2:]) + if noflag is not None and noflag.boolean: + if value is not None: + raise ValueError(arg + ' does not take an argument') + flag = noflag + value = 'false' + + if retired_flag_func and flag is None: + is_retired, is_bool = retired_flag_func(name) + + # If we didn't recognize that flag, but it starts with + # "no" then maybe it was a boolean flag specified in the + # --nofoo form. + if not is_retired and name.startswith('no'): + is_retired, is_bool = retired_flag_func(name[2:]) + is_retired = is_retired and is_bool + + if is_retired: + if not is_bool and value is None: + # This happens when a non-bool retired flag is specified + # in format of "--flag value". + get_value() + logging.error( + 'Flag "%s" is retired and should no longer be specified. See ' + 'https://abseil.io/tips/90.', + name, + ) + continue + + if flag is not None: + # LINT.IfChange + flag.parse(value) + flag.using_default_value = False + # LINT.ThenChange(../testing/flagsaver.py:flag_override_parsing) + else: + unparsed_names_and_args.append((name, arg)) + + unknown_flags = [] + unparsed_args = [] + for arg_name, arg in unparsed_names_and_args: + if arg_name is None: + # Positional arguments. + unparsed_args.append(arg) + elif arg_name in undefok: + # Remove undefok flags. + continue + else: + # This is an unknown flag. + if known_only: + unparsed_args.append(arg) + else: + unknown_flags.append((arg_name, arg)) + + unparsed_args.extend(list(args_it)) + return unknown_flags, unparsed_args + + def is_parsed(self) -> bool: + """Returns whether flags were parsed.""" + return self.__dict__['__flags_parsed'] + + def mark_as_parsed(self) -> None: + """Explicitly marks flags as parsed. + + Use this when the caller knows that this FlagValues has been parsed as if + a ``__call__()`` invocation has happened. This is only a public method for + use by things like appcommands which do additional command like parsing. + """ + self.__dict__['__flags_parsed'] = True + + def unparse_flags(self) -> None: + """Unparses all flags to the point before any FLAGS(argv) was called.""" + for f in self._flags().values(): + f.unparse() + # We log this message before marking flags as unparsed to avoid a + # problem when the logging library causes flags access. + logging.info('unparse_flags() called; flags access will now raise errors.') + self.__dict__['__flags_parsed'] = False + self.__dict__['__unparse_flags_called'] = True + + def flag_values_dict(self) -> dict[str, Any]: + """Returns a dictionary that maps flag names to flag values.""" + return {name: flag.value for name, flag in list(self._flags().items())} + + def __str__(self): + """Returns a help string for all known flags.""" + return self.get_help() + + def get_help( + self, prefix: str = '', include_special_flags: bool = True + ) -> str: + """Returns a help string for all known flags. + + Args: + prefix: str, per-line output prefix. + include_special_flags: bool, whether to include description of + SPECIAL_FLAGS, i.e. --flagfile and --undefok. + + Returns: + str, formatted help message. + """ + flags_by_module = self.flags_by_module_dict() + if flags_by_module: + modules = sorted(flags_by_module) + # Print the help for the main module first, if possible. + main_module = sys.argv[0] + if main_module in modules: + modules.remove(main_module) + modules = [main_module] + modules + return self._get_help_for_modules(modules, prefix, include_special_flags) + else: + output_lines: list[str] = [] + # Just print one long list of flags. + values = list(self._flags().values()) + if include_special_flags: + values.extend(_helpers.SPECIAL_FLAGS._flags().values()) # pylint: disable=protected-access + self._render_flag_list(values, output_lines, prefix) + return '\n'.join(output_lines) + + def _get_help_for_modules(self, modules, prefix, include_special_flags): + """Returns the help string for a list of modules. + + Private to absl.flags package. + + Args: + modules: List[str], a list of modules to get the help string for. + prefix: str, a string that is prepended to each generated help line. + include_special_flags: bool, whether to include description of + SPECIAL_FLAGS, i.e. --flagfile and --undefok. + """ + output_lines = [] + for module in modules: + self._render_our_module_flags(module, output_lines, prefix) + if include_special_flags: + self._render_module_flags( + 'absl.flags', + _helpers.SPECIAL_FLAGS._flags().values(), # pylint: disable=protected-access # pytype: disable=attribute-error + output_lines, + prefix, + ) + return '\n'.join(output_lines) + + def _render_module_flags(self, module, flags, output_lines, prefix=''): + """Returns a help string for a given module.""" + if not isinstance(module, str): + module = module.__name__ + output_lines.append('\n%s%s:' % (prefix, module)) + self._render_flag_list(flags, output_lines, prefix + ' ') + + def _render_our_module_flags(self, module, output_lines, prefix=''): + """Returns a help string for a given module.""" + flags = self.get_flags_for_module(module) + if flags: + self._render_module_flags(module, flags, output_lines, prefix) + + def _render_our_module_key_flags(self, module, output_lines, prefix=''): + """Returns a help string for the key flags of a given module. + + Args: + module: module|str, the module to render key flags for. + output_lines: [str], a list of strings. The generated help message lines + will be appended to this list. + prefix: str, a string that is prepended to each generated help line. + """ + key_flags = self.get_key_flags_for_module(module) + if key_flags: + self._render_module_flags(module, key_flags, output_lines, prefix) + + def module_help(self, module: Any) -> str: + """Describes the key flags of a module. + + Args: + module: module|str, the module to describe the key flags for. + + Returns: + str, describing the key flags of a module. + """ + helplist: list[str] = [] + self._render_our_module_key_flags(module, helplist) + return '\n'.join(helplist) + + def main_module_help(self) -> str: + """Describes the key flags of the main module. + + Returns: + str, describing the key flags of the main module. + """ + return self.module_help(sys.argv[0]) + + def _render_flag_list(self, flaglist, output_lines, prefix=' '): + fl = self._flags() + special_fl = _helpers.SPECIAL_FLAGS._flags() # pylint: disable=protected-access # pytype: disable=attribute-error + flaglist = [(flag.name, flag) for flag in flaglist] + flaglist.sort() + flagset = {} + for name, flag in flaglist: + # It's possible this flag got deleted or overridden since being + # registered in the per-module flaglist. Check now against the + # canonical source of current flag information, the _flags. + if fl.get(name, None) != flag and special_fl.get(name, None) != flag: + # a different flag is using this name now + continue + # only print help once + if flag in flagset: + continue + flagset[flag] = 1 + flaghelp = '' + if flag.short_name: + flaghelp += '-%s,' % flag.short_name + if flag.boolean: + flaghelp += '--[no]%s:' % flag.name + else: + flaghelp += '--%s:' % flag.name + flaghelp += ' ' + if flag.help: + flaghelp += flag.help + flaghelp = _helpers.text_wrap( + flaghelp, indent=prefix + ' ', firstline_indent=prefix + ) + if flag.default_as_str: + flaghelp += '\n' + flaghelp += _helpers.text_wrap( + '(default: %s)' % flag.default_as_str, indent=prefix + ' ' + ) + if flag.parser.syntactic_help: + flaghelp += '\n' + flaghelp += _helpers.text_wrap( + '(%s)' % flag.parser.syntactic_help, indent=prefix + ' ' + ) + output_lines.append(flaghelp) + + def get_flag_value(self, name: str, default: Any) -> Any: # pylint: disable=invalid-name + """Returns the value of a flag (if not None) or a default value. + + Args: + name: str, the name of a flag. + default: Default value to use if the flag value is None. + + Returns: + Requested flag value or default. + """ + + value = self.__getattr__(name) + if value is not None: # Can't do if not value, b/c value might be '0' or "" + return value + else: + return default + + def _is_flag_file_directive(self, flag_string): + """Checks whether flag_string contain a --flagfile= directive.""" + if isinstance(flag_string, str): + if flag_string.startswith('--flagfile='): + return 1 + elif flag_string == '--flagfile': + return 1 + elif flag_string.startswith('-flagfile='): + return 1 + elif flag_string == '-flagfile': + return 1 + else: + return 0 + return 0 + + def _extract_filename(self, flagfile_str): + """Returns filename from a flagfile_str of form -[-]flagfile=filename. + + The cases of --flagfile foo and -flagfile foo shouldn't be hitting + this function, as they are dealt with in the level above this + function. + + Args: + flagfile_str: str, the flagfile string. + + Returns: + str, the filename from a flagfile_str of form -[-]flagfile=filename. + + Raises: + Error: Raised when illegal --flagfile is provided. + """ + if flagfile_str.startswith('--flagfile='): + return os.path.expanduser((flagfile_str[(len('--flagfile=')) :]).strip()) + elif flagfile_str.startswith('-flagfile='): + return os.path.expanduser((flagfile_str[(len('-flagfile=')) :]).strip()) + else: + raise _exceptions.Error('Hit illegal --flagfile type: %s' % flagfile_str) + + def _get_flag_file_lines(self, filename, parsed_file_stack=None): + """Returns the useful (!=comments, etc) lines from a file with flags. + + Args: + filename: str, the name of the flag file. + parsed_file_stack: [str], a list of the names of the files that we have + recursively encountered at the current depth. MUTATED BY THIS FUNCTION + (but the original value is preserved upon successfully returning from + function call). + + Returns: + List of strings. See the note below. + + NOTE(springer): This function checks for a nested --flagfile= + tag and handles the lower file recursively. It returns a list of + all the lines that _could_ contain command flags. This is + EVERYTHING except whitespace lines and comments (lines starting + with '#' or '//'). + """ + # For consistency with the cpp version, ignore empty values. + if not filename: + return [] + if parsed_file_stack is None: + parsed_file_stack = [] + # We do a little safety check for reparsing a file we've already encountered + # at a previous depth. + if filename in parsed_file_stack: + sys.stderr.write( + 'Warning: Hit circular flagfile dependency. Ignoring flagfile: %s\n' + % (filename,) + ) + return [] + else: + parsed_file_stack.append(filename) + + line_list = [] # All line from flagfile. + flag_line_list = [] # Subset of lines w/o comments, blanks, flagfile= tags. + try: + file_obj = open(filename) + except OSError as e_msg: + raise _exceptions.CantOpenFlagFileError( + 'ERROR:: Unable to open flagfile: %s' % e_msg + ) + + with file_obj: + line_list = file_obj.readlines() + + # This is where we check each line in the file we just read. + for line in line_list: + if line.isspace(): + pass + # Checks for comment (a line that starts with '#'). + elif line.startswith('#') or line.startswith('//'): + pass + # Checks for a nested "--flagfile=" flag in the current file. + # If we find one, recursively parse down into that file. + elif self._is_flag_file_directive(line): + sub_filename = self._extract_filename(line) + included_flags = self._get_flag_file_lines( + sub_filename, parsed_file_stack=parsed_file_stack + ) + flag_line_list.extend(included_flags) + else: + # Any line that's not a comment or a nested flagfile should get + # copied into 2nd position. This leaves earlier arguments + # further back in the list, thus giving them higher priority. + flag_line_list.append(line.strip()) + + parsed_file_stack.pop() + return flag_line_list + + def read_flags_from_files( + self, argv: Sequence[str], force_gnu: bool = True + ) -> list[str]: + """Processes command line args, but also allow args to be read from file. + + Args: + argv: [str], a list of strings, usually sys.argv[1:], which may contain + one or more flagfile directives of the form --flagfile="./filename". + Note that the name of the program (sys.argv[0]) should be omitted. + force_gnu: bool, if False, --flagfile parsing obeys the + FLAGS.is_gnu_getopt() value. If True, ignore the value and always follow + gnu_getopt semantics. + + Returns: + A new list which has the original list combined with what we read + from any flagfile(s). + + Raises: + IllegalFlagValueError: Raised when --flagfile is provided with no + argument. + + This function is called by FLAGS(argv). + It scans the input list for a flag that looks like: + --flagfile=. Then it opens , reads all valid key + and value pairs and inserts them into the input list in exactly the + place where the --flagfile arg is found. + + Note that your application's flags are still defined the usual way + using absl.flags DEFINE_flag() type functions. + + Notes (assuming we're getting a commandline of some sort as our input): + + * For duplicate flags, the last one we hit should "win". + * Since flags that appear later win, a flagfile's settings can be "weak" + if the --flagfile comes at the beginning of the argument sequence, + and it can be "strong" if the --flagfile comes at the end. + * A further "--flagfile=" CAN be nested in a flagfile. + It will be expanded in exactly the spot where it is found. + * In a flagfile, a line beginning with # or // is a comment. + * Entirely blank lines _should_ be ignored. + """ + rest_of_args = argv + new_argv = [] + while rest_of_args: + current_arg = rest_of_args[0] + rest_of_args = rest_of_args[1:] + if self._is_flag_file_directive(current_arg): + # This handles the case of -(-)flagfile foo. In this case the + # next arg really is part of this one. + if current_arg == '--flagfile' or current_arg == '-flagfile': + if not rest_of_args: + raise _exceptions.IllegalFlagValueError( + '--flagfile with no argument' + ) + flag_filename = os.path.expanduser(rest_of_args[0]) + rest_of_args = rest_of_args[1:] + else: + # This handles the case of (-)-flagfile=foo. + flag_filename = self._extract_filename(current_arg) + new_argv.extend(self._get_flag_file_lines(flag_filename)) + else: + new_argv.append(current_arg) + # Stop parsing after '--', like getopt and gnu_getopt. + if current_arg == '--': + break + # Stop parsing after a non-flag, like getopt. + if not current_arg.startswith('-'): + if not force_gnu and not self.__dict__['__use_gnu_getopt']: + break + else: + if ( + '=' not in current_arg + and rest_of_args + and not rest_of_args[0].startswith('-') + ): + # If this is an occurrence of a legitimate --x y, skip the value + # so that it won't be mistaken for a standalone arg. + fl = self._flags() + name = current_arg.lstrip('-') + if name in fl and not fl[name].boolean: + current_arg = rest_of_args[0] + rest_of_args = rest_of_args[1:] + new_argv.append(current_arg) + + if rest_of_args: + new_argv.extend(rest_of_args) + + return new_argv + + def flags_into_string(self) -> str: + """Returns a string with the flags assignments from this FlagValues object. + + This function ignores flags whose value is None. Each flag + assignment is separated by a newline. + + NOTE: MUST mirror the behavior of the C++ CommandlineFlagsIntoString + from https://github.com/gflags/gflags. + + Returns: + str, the string with the flags assignments from this FlagValues object. + The flags are ordered by (module_name, flag_name). + """ + module_flags = sorted(self.flags_by_module_dict().items()) + s = '' + for unused_module_name, flags in module_flags: + flags = sorted(flags, key=lambda f: f.name) + for flag in flags: + if flag.value is not None: + s += flag.serialize() + '\n' + return s + + def append_flags_into_file(self, filename: str) -> None: + """Appends all flags assignments from this FlagInfo object to a file. + + Output will be in the format of a flagfile. + + NOTE: MUST mirror the behavior of the C++ AppendFlagsIntoFile + from https://github.com/gflags/gflags. + + Args: + filename: str, name of the file. + """ + with open(filename, 'a') as out_file: + out_file.write(self.flags_into_string()) + + def write_help_in_xml_format(self, outfile: TextIO | None = None) -> None: + """Outputs flag documentation in XML format. + + NOTE: We use element names that are consistent with those used by + the C++ command-line flag library, from + https://github.com/gflags/gflags. + We also use a few new elements (e.g., ), but we do not + interfere / overlap with existing XML elements used by the C++ + library. Please maintain this consistency. + + Args: + outfile: File object we write to. Default None means sys.stdout. + """ + doc = minidom.Document() + all_flag = doc.createElement('AllFlags') + doc.appendChild(all_flag) + + all_flag.appendChild( + _helpers.create_xml_dom_element( + doc, 'program', os.path.basename(sys.argv[0]) + ) + ) + + usage_doc = sys.modules['__main__'].__doc__ + if not usage_doc: + usage_doc = '\nUSAGE: %s [flags]\n' % sys.argv[0] + else: + usage_doc = usage_doc.replace('%s', sys.argv[0]) + all_flag.appendChild( + _helpers.create_xml_dom_element(doc, 'usage', usage_doc) + ) + + # Get list of key flags for the main module. + key_flags = self.get_key_flags_for_module(sys.argv[0]) + + flags_by_module = self.flags_by_module_dict() + # Sort flags by declaring module name and next by flag name. + for module_name in sorted(flags_by_module.keys()): + flag_list = [(f.name, f) for f in flags_by_module[module_name]] + flag_list.sort() + for unused_flag_name, flag in flag_list: + is_key = flag in key_flags + all_flag.appendChild( + flag._create_xml_dom_element( # pylint: disable=protected-access + doc, module_name, is_key=is_key + ) + ) + + outfile = outfile or sys.stdout + outfile.write( + doc.toprettyxml(indent=' ', encoding='utf-8').decode('utf-8') + ) + outfile.flush() + + def _check_method_name_conflicts(self, name: str, flag: Flag): + if flag.allow_using_method_names: + return + short_name = flag.short_name + flag_names = {name} if short_name is None else {name, short_name} + for flag_name in flag_names: + if flag_name in self.__dict__['__banned_flag_names']: + raise _exceptions.FlagNameConflictsWithMethodError( + 'Cannot define a flag named "{name}". It conflicts with a method ' + 'on class "{class_name}". To allow defining it, use ' + 'allow_using_method_names and access the flag value with ' + "FLAGS['{name}'].value. FLAGS.{name} returns the method, " + 'not the flag value.'.format( + name=flag_name, class_name=type(self).__name__ + ) + ) + + +FLAGS = FlagValues() + + +class FlagHolder(Generic[_T_co]): + """Holds a defined flag. + + This facilitates a cleaner api around global state. Instead of:: + + flags.DEFINE_integer('foo', ...) + flags.DEFINE_integer('bar', ...) + + def method(): + # prints parsed value of 'bar' flag + print(flags.FLAGS.foo) + # runtime error due to typo or possibly bad coding style. + print(flags.FLAGS.baz) + + it encourages code like:: + + _FOO_FLAG = flags.DEFINE_integer('foo', ...) + _BAR_FLAG = flags.DEFINE_integer('bar', ...) + + def method(): + print(_FOO_FLAG.value) + print(_BAR_FLAG.value) + + since the name of the flag appears only once in the source code. + """ + + value: _T_co + + def __init__( + self, + flag_values: FlagValues, + flag: Flag[_T_co], + ensure_non_none_value: bool = False, + ): + """Constructs a FlagHolder instance providing typesafe access to flag. + + Args: + flag_values: The container the flag is registered to. + flag: The flag object for this flag. + ensure_non_none_value: Is the value of the flag allowed to be None. + """ + self._flagvalues = flag_values + # We take the entire flag object, but only keep the name. Why? + # - We want FlagHolder[T] to be generic container + # - flag_values contains all flags, so has no reference to T. + # - typecheckers don't like to see a generic class where none of the ctor + # arguments refer to the generic type. + self._name = flag.name + # We intentionally do NOT check if the default value is None. + # This allows future use of this for "required flags with None default" + self._ensure_non_none_value = ensure_non_none_value + + def __eq__(self, other): + raise TypeError( + "unsupported operand type(s) for ==: '{0}' and '{1}' " + "(did you mean to use '{0}.value' instead?)".format( + type(self).__name__, type(other).__name__ + ) + ) + + def __bool__(self): + raise TypeError( + "bool() not supported for instances of type '{0}' " + "(did you mean to use '{0}.value' instead?)".format(type(self).__name__) + ) + + __nonzero__ = __bool__ + + @property + def name(self) -> str: + return self._name + + @property # type: ignore[no-redef] + def value(self) -> _T_co: + """Returns the value of the flag. + + If ``_ensure_non_none_value`` is ``True``, then return value is not + ``None``. + + Raises: + UnparsedFlagAccessError: if flag parsing has not finished. + IllegalFlagValueError: if value is None unexpectedly. + """ + val = getattr(self._flagvalues, self._name) + if self._ensure_non_none_value and val is None: + raise _exceptions.IllegalFlagValueError( + 'Unexpected None value for flag %s' % self._name + ) + return val + + @property + def default(self) -> _T_co: + """Returns the default value of the flag.""" + return self._flagvalues[self._name].default # type: ignore[return-value] + + @property + def present(self) -> bool: + """Returns True if the flag was parsed from command-line flags.""" + return bool(self._flagvalues[self._name].present) + + def serialize(self) -> str: + """Returns a serialized representation of the flag.""" + return self._flagvalues[self._name].serialize() + + +def resolve_flag_ref( + flag_ref: str | FlagHolder, flag_values: FlagValues +) -> tuple[str, FlagValues]: + """Helper to validate and resolve a flag reference argument.""" + if isinstance(flag_ref, FlagHolder): + new_flag_values = flag_ref._flagvalues # pylint: disable=protected-access + if flag_values != FLAGS and flag_values != new_flag_values: + raise ValueError( + 'flag_values must not be customized when operating on a FlagHolder' + ) + return flag_ref.name, new_flag_values + return flag_ref, flag_values + + +def resolve_flag_refs( + flag_refs: Sequence[str | FlagHolder], flag_values: FlagValues +) -> tuple[list[str], FlagValues]: + """Helper to validate and resolve flag reference list arguments.""" + fv = None + names = [] + for ref in flag_refs: + if isinstance(ref, FlagHolder): + newfv = ref._flagvalues # pylint: disable=protected-access + name = ref.name + else: + newfv = flag_values + name = ref + if fv and fv != newfv: + raise ValueError( + 'multiple FlagValues instances used in invocation. ' + 'FlagHolders must be registered to the same FlagValues instance as ' + 'do flag names, if provided.' + ) + fv = newfv + names.append(name) + if fv is None: + raise ValueError('flag_refs argument must not be empty') + return names, fv diff --git a/venv/Lib/site-packages/absl/flags/_helpers.py b/venv/Lib/site-packages/absl/flags/_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..0de8153d766e99b12975a85c27e998985fd5fa96 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/_helpers.py @@ -0,0 +1,403 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Internal helper functions for Abseil Python flags library.""" + +from collections.abc import Iterable, Sequence +import re +import shutil +import sys +import textwrap +import types +from typing import Any, NamedTuple +from xml.dom import minidom + + +_DEFAULT_HELP_WIDTH = 80 # Default width of help output. +# Minimal "sane" width of help output. We assume that any value below 40 is +# unreasonable. +_MIN_HELP_WIDTH = 40 + +# Define the allowed error rate in an input string to get suggestions. +# +# We lean towards a high threshold because we tend to be matching a phrase, +# and the simple algorithm used here is geared towards correcting word +# spellings. +# +# For manual testing, consider " --list" which produced a large number +# of spurious suggestions when we used "least_errors > 0.5" instead of +# "least_erros >= 0.5". +_SUGGESTION_ERROR_RATE_THRESHOLD = 0.50 + +# Characters that cannot appear or are highly discouraged in an XML 1.0 +# document. (See http://www.w3.org/TR/REC-xml/#charsets or +# https://en.wikipedia.org/wiki/Valid_characters_in_XML#XML_1.0) +_ILLEGAL_XML_CHARS_REGEX = re.compile( + '[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]' +) + +# This is a set of module ids for the modules that disclaim key flags. +# This module is explicitly added to this set so that we never consider it to +# define key flag. +disclaim_module_ids: set[int] = {id(sys.modules[__name__])} + + +# Define special flags here so that help may be generated for them. +# NOTE: Please do NOT use SPECIAL_FLAGS from outside flags module. +# Initialized inside flagvalues.py. +# NOTE: This cannot be annotated as its actual FlagValues type since this would +# create a circular dependency. +SPECIAL_FLAGS: Any = None + + +# This points to the flags module, initialized in flags/__init__.py. +# This should only be used in adopt_module_key_flags to take SPECIAL_FLAGS into +# account. +FLAGS_MODULE: types.ModuleType | None = None + + +class _ModuleObjectAndName(NamedTuple): + """Module object and name. + + Fields: + - module: object, module object. + - module_name: str, module name. + """ + module: types.ModuleType + module_name: str + + +def get_module_object_and_name( + globals_dict: dict[str, Any], +) -> _ModuleObjectAndName | None: + """Returns the module that defines a global environment, and its name. + + Args: + globals_dict: A dictionary that should correspond to an environment + providing the values of the globals. + + Returns: + _ModuleObjectAndName - pair of module object & module name. + Returns None if the module could not be identified. + """ + try: + name = globals_dict['__name__'] + module = sys.modules[name] + except KeyError: + return None + # Pick a more informative name for the main module. + return _ModuleObjectAndName( + module, sys.argv[0] if name == '__main__' else name + ) + + +def get_calling_module_object_and_name() -> _ModuleObjectAndName: + """Returns the module that's calling into this module. + + We generally use this function to get the name of the module calling a + DEFINE_foo... function. + + Returns: + The module object that called into this one. + + Raises: + AssertionError: Raised when no calling module could be identified. + """ + for depth in range(1, sys.getrecursionlimit()): + # sys._getframe is the right thing to use here, as it's the best + # way to walk up the call stack. + globals_for_frame = sys._getframe(depth).f_globals # pylint: disable=protected-access + module = get_module_object_and_name(globals_for_frame) + if module is not None and id(module.module) not in disclaim_module_ids: + return module + raise AssertionError('No module was found') + + +def get_calling_module() -> str: + """Returns the name of the module that's calling into this module.""" + return get_calling_module_object_and_name().module_name + + +def create_xml_dom_element( + doc: minidom.Document, name: str, value: Any +) -> minidom.Element: + """Returns an XML DOM element with name and text value. + + Args: + doc: minidom.Document, the DOM document it should create nodes from. + name: str, the tag of XML element. + value: object, whose string representation will be used + as the value of the XML element. Illegal or highly discouraged xml 1.0 + characters are stripped. + + Returns: + An instance of minidom.Element. + """ + s = str(value) + if isinstance(value, bool): + # Display boolean values as the C++ flag library does: no caps. + s = s.lower() + # Remove illegal xml characters. + s = _ILLEGAL_XML_CHARS_REGEX.sub('', s) + + e = doc.createElement(name) + e.appendChild(doc.createTextNode(s)) + return e + + +def get_help_width() -> int: + """Returns the integer width of help lines that is used in TextWrap.""" + size = shutil.get_terminal_size(fallback=(_DEFAULT_HELP_WIDTH, 1)) + return size.columns + + +def get_flag_suggestions( + attempt: str, longopt_list: Sequence[str] +) -> list[str]: + """Returns helpful similar matches for an invalid flag.""" + # Don't suggest on very short strings, or if no longopts are specified. + if len(attempt) <= 2 or not longopt_list: + return [] + + option_names = [v.split('=')[0] for v in longopt_list] + + # Find close approximations in flag prefixes. + # This also handles the case where the flag is spelled right but ambiguous. + distances = [(_damerau_levenshtein(attempt, option[0:len(attempt)]), option) + for option in option_names] + # t[0] is distance, and sorting by t[1] allows us to have stable output. + distances.sort() + + least_errors, _ = distances[0] + # Don't suggest excessively bad matches. + if least_errors >= _SUGGESTION_ERROR_RATE_THRESHOLD * len(attempt): + return [] + + suggestions = [] + for errors, name in distances: + if errors == least_errors: + suggestions.append(name) + else: + break + return suggestions + + +def _damerau_levenshtein(a, b): + """Returns Damerau-Levenshtein edit distance from a to b.""" + memo = {} + + def distance(x, y): + """Recursively defined string distance with memoization.""" + if (x, y) in memo: + return memo[x, y] + if not x: + d = len(y) + elif not y: + d = len(x) + else: + d = min( + distance(x[1:], y) + 1, # correct an insertion error + distance(x, y[1:]) + 1, # correct a deletion error + distance(x[1:], y[1:]) + (x[0] != y[0])) # correct a wrong character + if len(x) >= 2 and len(y) >= 2 and x[0] == y[1] and x[1] == y[0]: + # Correct a transposition. + t = distance(x[2:], y[2:]) + 1 + if d > t: + d = t + + memo[x, y] = d + return d + return distance(a, b) + + +def text_wrap( + text: str, + length: int | None = None, + indent: str = '', + firstline_indent: str | None = None, +) -> str: + """Wraps a given text to a maximum line length and returns it. + + It turns lines that only contain whitespace into empty lines, keeps new lines, + and expands tabs using 4 spaces. + + Args: + text: Text to wrap. + length: Maximum length of a line, includes indentation. If this is `None` + then use `get_help_width()`. + indent: Indent for all but first line. + firstline_indent: Indent for first line. If `None`, fall back to `indent`. + + Returns: + The wrapped text. + + Raises: + ValueError: Raised if indent or firstline_indent not shorter than length. + """ + # Get defaults where callee used None + if length is None: + length = get_help_width() + if indent is None: + indent = '' + if firstline_indent is None: + firstline_indent = indent + + if len(indent) >= length: + raise ValueError('Length of indent exceeds length') + if len(firstline_indent) >= length: + raise ValueError('Length of first line indent exceeds length') + + text = text.expandtabs(4) + + result = [] + # Create one wrapper for the first paragraph and one for subsequent + # paragraphs that does not have the initial wrapping. + wrapper = textwrap.TextWrapper( + width=length, initial_indent=firstline_indent, subsequent_indent=indent) + subsequent_wrapper = textwrap.TextWrapper( + width=length, initial_indent=indent, subsequent_indent=indent) + + # textwrap does not have any special treatment for newlines. From the docs: + # "...newlines may appear in the middle of a line and cause strange output. + # For this reason, text should be split into paragraphs (using + # str.splitlines() or similar) which are wrapped separately." + for paragraph in (p.strip() for p in text.splitlines()): + if paragraph: + result.extend(wrapper.wrap(paragraph)) + else: + result.append('') # Keep empty lines. + # Replace initial wrapper with wrapper for subsequent paragraphs. + wrapper = subsequent_wrapper + + return '\n'.join(result) + + +def flag_dict_to_args( + flag_map: dict[str, Any], multi_flags: set[str] | None = None +) -> Iterable[str]: + """Convert a dict of values into process call parameters. + + This method is used to convert a dictionary into a sequence of parameters + for a binary that parses arguments using this module. + + Args: + flag_map: dict, a mapping where the keys are flag names (strings). + values are treated according to their type: + + * If value is ``None``, then only the name is emitted. + * If value is ``True``, then only the name is emitted. + * If value is ``False``, then only the name prepended with 'no' is + emitted. + * If value is a string then ``--name=value`` is emitted. + * If value is a collection, this will emit + ``--name=value1,value2,value3``, unless the flag name is in + ``multi_flags``, in which case this will emit + ``--name=value1 --name=value2 --name=value3``. + * Everything else is converted to string an passed as such. + + multi_flags: set, names (strings) of flags that should be treated as + multi-flags. + Yields: + sequence of string suitable for a subprocess execution. + """ + for key, value in flag_map.items(): + if value is None: + yield '--%s' % key + elif isinstance(value, bool): + if value: + yield '--%s' % key + else: + yield '--no%s' % key + elif isinstance(value, (bytes, str)): + # We don't want strings to be handled like python collections. + yield '--%s=%s' % (key, value) # type: ignore[str-bytes-safe] + else: + # Now we attempt to deal with collections. + try: + if multi_flags and key in multi_flags: + for item in value: + yield '--%s=%s' % (key, str(item)) + else: + yield '--%s=%s' % (key, ','.join(str(item) for item in value)) + except TypeError: + # Default case. + yield '--%s=%s' % (key, value) + + +def trim_docstring(docstring: str) -> str: + """Removes indentation from triple-quoted strings. + + This is the function specified in PEP 257 to handle docstrings: + https://www.python.org/dev/peps/pep-0257/. + + Args: + docstring: str, a python docstring. + + Returns: + str, docstring with indentation removed. + """ + if not docstring: + return '' + + # If you've got a line longer than this you have other problems... + max_indent = 1 << 29 + + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + + # Determine minimum indentation (first line doesn't count): + indent = max_indent + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < max_indent: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + + +def doc_to_help(doc: str) -> str: + """Takes a __doc__ string and reformats it as help.""" + + # Get rid of starting and ending white space. Using lstrip() or even + # strip() could drop more than maximum of first line and right space + # of last line. + doc = doc.strip() + + # Get rid of all empty lines. + whitespace_only_line = re.compile('^[ \t]+$', re.M) + doc = whitespace_only_line.sub('', doc) + + # Cut out common space at line beginnings. + doc = trim_docstring(doc) + + # Just like this module's comment, comments tend to be aligned somehow. + # In other words they all start with the same amount of white space. + # 1) keep double new lines; + # 2) keep ws after new lines if not empty line; + # 3) all other new lines shall be changed to a space; + # Solution: Match new lines between non white space and replace with space. + doc = re.sub(r'(?<=\S)\n(?=\S)', ' ', doc, flags=re.M) + + return doc diff --git a/venv/Lib/site-packages/absl/flags/_validators.py b/venv/Lib/site-packages/absl/flags/_validators.py new file mode 100644 index 0000000000000000000000000000000000000000..d4a937acd6184a02366c11384f7e8421cdbc1af1 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/_validators.py @@ -0,0 +1,353 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to enforce different constraints on flags. + +Flags validators can be registered using following functions / decorators:: + + flags.register_validator + @flags.validator + flags.register_multi_flags_validator + @flags.multi_flags_validator + +Three convenience functions are also provided for common flag constraints:: + + flags.mark_flag_as_required + flags.mark_flags_as_required + flags.mark_flags_as_mutual_exclusive + flags.mark_bool_flags_as_mutual_exclusive + +See their docstring in this module for a usage manual. + +Do NOT import this module directly. Import the flags package and use the +aliases defined at the package level instead. +""" + +import warnings + +from absl.flags import _exceptions +from absl.flags import _flagvalues +from absl.flags import _validators_classes + + +def register_validator(flag_name, + checker, + message='Flag validation failed', + flag_values=_flagvalues.FLAGS): + """Adds a constraint, which will be enforced during program execution. + + The constraint is validated when flags are initially parsed, and after each + change of the corresponding flag's value. + + Args: + flag_name: str | FlagHolder, name or holder of the flag to be checked. + Positional-only parameter. + checker: callable, a function to validate the flag. + + * input - A single positional argument: The value of the corresponding + flag (string, boolean, etc. This value will be passed to checker + by the library). + * output - bool, True if validator constraint is satisfied. + If constraint is not satisfied, it should either ``return False`` or + ``raise flags.ValidationError(desired_error_message)``. + + message: str, error text to be shown to the user if checker returns False. + If checker raises flags.ValidationError, message from the raised + error will be shown. + flag_values: flags.FlagValues, optional FlagValues instance to validate + against. + + Raises: + AttributeError: Raised when flag_name is not registered as a valid flag + name. + ValueError: Raised when flag_values is non-default and does not match the + FlagValues of the provided FlagHolder instance. + """ + flag_name, flag_values = _flagvalues.resolve_flag_ref(flag_name, flag_values) + v = _validators_classes.SingleFlagValidator(flag_name, checker, message) + _add_validator(flag_values, v) + + +def validator(flag_name, message='Flag validation failed', + flag_values=_flagvalues.FLAGS): + """A function decorator for defining a flag validator. + + Registers the decorated function as a validator for flag_name, e.g.:: + + @flags.validator('foo') + def _CheckFoo(foo): + ... + + See :func:`register_validator` for the specification of checker function. + + Args: + flag_name: str | FlagHolder, name or holder of the flag to be checked. + Positional-only parameter. + message: str, error text to be shown to the user if checker returns False. + If checker raises flags.ValidationError, message from the raised + error will be shown. + flag_values: flags.FlagValues, optional FlagValues instance to validate + against. + Returns: + A function decorator that registers its function argument as a validator. + Raises: + AttributeError: Raised when flag_name is not registered as a valid flag + name. + """ + + def decorate(function): + register_validator(flag_name, function, + message=message, + flag_values=flag_values) + return function + return decorate + + +def register_multi_flags_validator(flag_names, + multi_flags_checker, + message='Flags validation failed', + flag_values=_flagvalues.FLAGS): + """Adds a constraint to multiple flags. + + The constraint is validated when flags are initially parsed, and after each + change of the corresponding flag's value. + + Args: + flag_names: [str | FlagHolder], a list of the flag names or holders to be + checked. Positional-only parameter. + multi_flags_checker: callable, a function to validate the flag. + + * input - dict, with keys() being flag_names, and value for each key + being the value of the corresponding flag (string, boolean, etc). + * output - bool, True if validator constraint is satisfied. + If constraint is not satisfied, it should either return False or + raise flags.ValidationError. + + message: str, error text to be shown to the user if checker returns False. + If checker raises flags.ValidationError, message from the raised + error will be shown. + flag_values: flags.FlagValues, optional FlagValues instance to validate + against. + + Raises: + AttributeError: Raised when a flag is not registered as a valid flag name. + ValueError: Raised when multiple FlagValues are used in the same + invocation. This can occur when FlagHolders have different `_flagvalues` + or when str-type flag_names entries are present and the `flag_values` + argument does not match that of provided FlagHolder(s). + """ + flag_names, flag_values = _flagvalues.resolve_flag_refs( + flag_names, flag_values) + v = _validators_classes.MultiFlagsValidator( + flag_names, multi_flags_checker, message) + _add_validator(flag_values, v) + + +def multi_flags_validator(flag_names, + message='Flag validation failed', + flag_values=_flagvalues.FLAGS): + """A function decorator for defining a multi-flag validator. + + Registers the decorated function as a validator for flag_names, e.g.:: + + @flags.multi_flags_validator(['foo', 'bar']) + def _CheckFooBar(flags_dict): + ... + + See :func:`register_multi_flags_validator` for the specification of checker + function. + + Args: + flag_names: [str | FlagHolder], a list of the flag names or holders to be + checked. Positional-only parameter. + message: str, error text to be shown to the user if checker returns False. + If checker raises flags.ValidationError, message from the raised + error will be shown. + flag_values: flags.FlagValues, optional FlagValues instance to validate + against. + + Returns: + A function decorator that registers its function argument as a validator. + + Raises: + AttributeError: Raised when a flag is not registered as a valid flag name. + """ + + def decorate(function): + register_multi_flags_validator(flag_names, + function, + message=message, + flag_values=flag_values) + return function + + return decorate + + +def mark_flag_as_required(flag_name, flag_values=_flagvalues.FLAGS): + """Ensures that flag is not None during program execution. + + Registers a flag validator, which will follow usual validator rules. + Important note: validator will pass for any non-``None`` value, such as + ``False``, ``0`` (zero), ``''`` (empty string) and so on. + + If your module might be imported by others, and you only wish to make the flag + required when the module is directly executed, call this method like this:: + + if __name__ == '__main__': + flags.mark_flag_as_required('your_flag_name') + app.run() + + Args: + flag_name: str | FlagHolder, name or holder of the flag. + Positional-only parameter. + flag_values: flags.FlagValues, optional :class:`~absl.flags.FlagValues` + instance where the flag is defined. + Raises: + AttributeError: Raised when flag_name is not registered as a valid flag + name. + ValueError: Raised when flag_values is non-default and does not match the + FlagValues of the provided FlagHolder instance. + """ + flag_name, flag_values = _flagvalues.resolve_flag_ref(flag_name, flag_values) + if flag_values[flag_name].default is not None: + warnings.warn( + 'Flag --%s has a non-None default value; therefore, ' + 'mark_flag_as_required will pass even if flag is not specified in the ' + 'command line!' % flag_name, + stacklevel=2) + register_validator( + flag_name, + lambda value: value is not None, + message=f'Flag --{flag_name} must have a value other than None.', + flag_values=flag_values, + ) + + +def mark_flags_as_required(flag_names, flag_values=_flagvalues.FLAGS): + """Ensures that flags are not None during program execution. + + If your module might be imported by others, and you only wish to make the flag + required when the module is directly executed, call this method like this:: + + if __name__ == '__main__': + flags.mark_flags_as_required(['flag1', 'flag2', 'flag3']) + app.run() + + Args: + flag_names: Sequence[str | FlagHolder], names or holders of the flags. + flag_values: flags.FlagValues, optional FlagValues instance where the flags + are defined. + Raises: + AttributeError: If any of flag name has not already been defined as a flag. + """ + for flag_name in flag_names: + mark_flag_as_required(flag_name, flag_values) + + +def mark_flags_as_mutual_exclusive(flag_names, required=False, + flag_values=_flagvalues.FLAGS): + """Ensures that only one flag among flag_names is not None. + + Important note: This validator checks if flag values are ``None``, and it does + not distinguish between default and explicit values. Therefore, this validator + does not make sense when applied to flags with default values other than None, + including other false values (e.g. ``False``, ``0``, ``''``, ``[]``). That + includes multi flags with a default value of ``[]`` instead of None. + + Args: + flag_names: [str | FlagHolder], names or holders of flags. + Positional-only parameter. + required: bool. If true, exactly one of the flags must have a value other + than None. Otherwise, at most one of the flags can have a value other + than None, and it is valid for all of the flags to be None. + flag_values: flags.FlagValues, optional FlagValues instance where the flags + are defined. + + Raises: + ValueError: Raised when multiple FlagValues are used in the same + invocation. This can occur when FlagHolders have different `_flagvalues` + or when str-type flag_names entries are present and the `flag_values` + argument does not match that of provided FlagHolder(s). + """ + flag_names, flag_values = _flagvalues.resolve_flag_refs( + flag_names, flag_values) + for flag_name in flag_names: + if flag_values[flag_name].default is not None: + warnings.warn( + 'Flag --{} has a non-None default value. That does not make sense ' + 'with mark_flags_as_mutual_exclusive, which checks whether the ' + 'listed flags have a value other than None.'.format(flag_name), + stacklevel=2) + + def validate_mutual_exclusion(flags_dict): + flag_count = sum(1 for val in flags_dict.values() if val is not None) + if flag_count == 1 or (not required and flag_count == 0): + return True + raise _exceptions.ValidationError( + '{} one of ({}) must have a value other than None.'.format( + 'Exactly' if required else 'At most', ', '.join(flag_names))) + + register_multi_flags_validator( + flag_names, validate_mutual_exclusion, flag_values=flag_values) + + +def mark_bool_flags_as_mutual_exclusive(flag_names, required=False, + flag_values=_flagvalues.FLAGS): + """Ensures that only one flag among flag_names is True. + + Args: + flag_names: [str | FlagHolder], names or holders of flags. + Positional-only parameter. + required: bool. If true, exactly one flag must be True. Otherwise, at most + one flag can be True, and it is valid for all flags to be False. + flag_values: flags.FlagValues, optional FlagValues instance where the flags + are defined. + + Raises: + ValueError: Raised when multiple FlagValues are used in the same + invocation. This can occur when FlagHolders have different `_flagvalues` + or when str-type flag_names entries are present and the `flag_values` + argument does not match that of provided FlagHolder(s). + """ + flag_names, flag_values = _flagvalues.resolve_flag_refs( + flag_names, flag_values) + for flag_name in flag_names: + if not flag_values[flag_name].boolean: + raise _exceptions.ValidationError( + 'Flag --{} is not Boolean, which is required for flags used in ' + 'mark_bool_flags_as_mutual_exclusive.'.format(flag_name)) + + def validate_boolean_mutual_exclusion(flags_dict): + flag_count = sum(bool(val) for val in flags_dict.values()) + if flag_count == 1 or (not required and flag_count == 0): + return True + raise _exceptions.ValidationError( + '{} one of ({}) must be True.'.format( + 'Exactly' if required else 'At most', ', '.join(flag_names))) + + register_multi_flags_validator( + flag_names, validate_boolean_mutual_exclusion, flag_values=flag_values) + + +def _add_validator(fv, validator_instance): + """Register new flags validator to be checked. + + Args: + fv: flags.FlagValues, the FlagValues instance to add the validator. + validator_instance: validators.Validator, the validator to add. + Raises: + KeyError: Raised when validators work with a non-existing flag. + """ + for flag_name in validator_instance.get_flags_names(): + fv[flag_name].validators.append(validator_instance) diff --git a/venv/Lib/site-packages/absl/flags/_validators_classes.py b/venv/Lib/site-packages/absl/flags/_validators_classes.py new file mode 100644 index 0000000000000000000000000000000000000000..cf978bfd1bf837895310ae0b5e1b3eed90fef375 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/_validators_classes.py @@ -0,0 +1,172 @@ +# Copyright 2021 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines *private* classes used for flag validators. + +Do NOT import this module. DO NOT use anything from this module. They are +private APIs. +""" + +from absl.flags import _exceptions + + +class Validator: + """Base class for flags validators. + + Users should NOT overload these classes, and use flags.Register... + methods instead. + """ + + # Used to assign each validator an unique insertion_index + validators_count = 0 + + def __init__(self, checker, message): + """Constructor to create all validators. + + Args: + checker: function to verify the constraint. + Input of this method varies, see SingleFlagValidator and + multi_flags_validator for a detailed description. + message: str, error message to be shown to the user. + """ + self.checker = checker + self.message = message + Validator.validators_count += 1 + # Used to assert validators in the order they were registered. + self.insertion_index = Validator.validators_count + + def verify(self, flag_values): + """Verifies that constraint is satisfied. + + flags library calls this method to verify Validator's constraint. + + Args: + flag_values: flags.FlagValues, the FlagValues instance to get flags from. + Raises: + Error: Raised if constraint is not satisfied. + """ + param = self._get_input_to_checker_function(flag_values) + if not self.checker(param): + raise _exceptions.ValidationError(self.message) + + def get_flags_names(self): + """Returns the names of the flags checked by this validator. + + Returns: + [string], names of the flags. + """ + raise NotImplementedError('This method should be overloaded') + + def print_flags_with_values(self, flag_values): + raise NotImplementedError('This method should be overloaded') + + def _get_input_to_checker_function(self, flag_values): + """Given flag values, returns the input to be given to checker. + + Args: + flag_values: flags.FlagValues, containing all flags. + Returns: + The input to be given to checker. The return type depends on the specific + validator. + """ + raise NotImplementedError('This method should be overloaded') + + +class SingleFlagValidator(Validator): + """Validator behind register_validator() method. + + Validates that a single flag passes its checker function. The checker function + takes the flag value and returns True (if value looks fine) or, if flag value + is not valid, either returns False or raises an Exception. + """ + + def __init__(self, flag_name, checker, message): + """Constructor. + + Args: + flag_name: string, name of the flag. + checker: function to verify the validator. + input - value of the corresponding flag (string, boolean, etc). + output - bool, True if validator constraint is satisfied. + If constraint is not satisfied, it should either return False or + raise flags.ValidationError(desired_error_message). + message: str, error message to be shown to the user if validator's + condition is not satisfied. + """ + super().__init__(checker, message) + self.flag_name = flag_name + + def get_flags_names(self): + return [self.flag_name] + + def print_flags_with_values(self, flag_values): + return 'flag --%s=%s' % (self.flag_name, flag_values[self.flag_name].value) + + def _get_input_to_checker_function(self, flag_values): + """Given flag values, returns the input to be given to checker. + + Args: + flag_values: flags.FlagValues, the FlagValues instance to get flags from. + Returns: + object, the input to be given to checker. + """ + return flag_values[self.flag_name].value + + +class MultiFlagsValidator(Validator): + """Validator behind register_multi_flags_validator method. + + Validates that flag values pass their common checker function. The checker + function takes flag values and returns True (if values look fine) or, + if values are not valid, either returns False or raises an Exception. + """ + + def __init__(self, flag_names, checker, message): + """Constructor. + + Args: + flag_names: [str], containing names of the flags used by checker. + checker: function to verify the validator. + input - dict, with keys() being flag_names, and value for each + key being the value of the corresponding flag (string, boolean, + etc). + output - bool, True if validator constraint is satisfied. + If constraint is not satisfied, it should either return False or + raise flags.ValidationError(desired_error_message). + message: str, error message to be shown to the user if validator's + condition is not satisfied + """ + super().__init__(checker, message) + self.flag_names = flag_names + + def _get_input_to_checker_function(self, flag_values): + """Given flag values, returns the input to be given to checker. + + Args: + flag_values: flags.FlagValues, the FlagValues instance to get flags from. + Returns: + dict, with keys() being self.flag_names, and value for each key + being the value of the corresponding flag (string, boolean, etc). + """ + return {key: flag_values[key].value for key in self.flag_names} + + def print_flags_with_values(self, flag_values): + prefix = 'flags ' + flags_with_values = [] + for key in self.flag_names: + flags_with_values.append('%s=%s' % (key, flag_values[key].value)) + return prefix + ', '.join(flags_with_values) + + def get_flags_names(self): + return self.flag_names diff --git a/venv/Lib/site-packages/absl/flags/argparse_flags.py b/venv/Lib/site-packages/absl/flags/argparse_flags.py new file mode 100644 index 0000000000000000000000000000000000000000..9f77690f10c1e0fc9619b857ba6169636f30e6c3 --- /dev/null +++ b/venv/Lib/site-packages/absl/flags/argparse_flags.py @@ -0,0 +1,390 @@ +# Copyright 2018 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module provides argparse integration with absl.flags. + +``argparse_flags.ArgumentParser`` is a drop-in replacement for +:class:`argparse.ArgumentParser`. It takes care of collecting and defining absl +flags in :mod:`argparse`. + +Here is a simple example:: + + # Assume the following absl.flags is defined in another module: + # + # from absl import flags + # flags.DEFINE_string('echo', None, 'The echo message.') + # + parser = argparse_flags.ArgumentParser( + description='A demo of absl.flags and argparse integration.') + parser.add_argument('--header', help='Header message to print.') + + # The parser will also accept the absl flag `--echo`. + # The `header` value is available as `args.header` just like a regular + # argparse flag. The absl flag `--echo` continues to be available via + # `absl.flags.FLAGS` if you want to access it. + args = parser.parse_args() + + # Example usages: + # ./program --echo='A message.' --header='A header' + # ./program --header 'A header' --echo 'A message.' + + +Here is another example demonstrates subparsers:: + + parser = argparse_flags.ArgumentParser(description='A subcommands demo.') + parser.add_argument('--header', help='The header message to print.') + + subparsers = parser.add_subparsers(help='The command to execute.') + + roll_dice_parser = subparsers.add_parser( + 'roll_dice', help='Roll a dice.', + # By default, absl flags can also be specified after the sub-command. + # To only allow them before sub-command, pass + # `inherited_absl_flags=None`. + inherited_absl_flags=None) + roll_dice_parser.add_argument('--num_faces', type=int, default=6) + roll_dice_parser.set_defaults(command=roll_dice) + + shuffle_parser = subparsers.add_parser('shuffle', help='Shuffle inputs.') + shuffle_parser.add_argument( + 'inputs', metavar='I', nargs='+', help='Inputs to shuffle.') + shuffle_parser.set_defaults(command=shuffle) + + args = parser.parse_args(argv[1:]) + args.command(args) + + # Example usages: + # ./program --echo='A message.' roll_dice --num_faces=6 + # ./program shuffle --echo='A message.' 1 2 3 4 + + +There are several differences between :mod:`absl.flags` and +:mod:`~absl.flags.argparse_flags`: + +1. Flags defined with absl.flags are parsed differently when using the + argparse parser. Notably: + + 1) absl.flags allows both single-dash and double-dash for any flag, and + doesn't distinguish them; argparse_flags only allows double-dash for + flag's regular name, and single-dash for flag's ``short_name``. + 2) Boolean flags in absl.flags can be specified with ``--bool``, + ``--nobool``, as well as ``--bool=true/false`` (though not recommended); + in argparse_flags, it only allows ``--bool``, ``--nobool``. + +2. Help related flag differences: + + 1) absl.flags does not define help flags, absl.app does that; argparse_flags + defines help flags unless passed with ``add_help=False``. + 2) absl.app supports ``--helpxml``; argparse_flags does not. + 3) argparse_flags supports ``-h``; absl.app does not. +""" + +import argparse +import sys + +from absl import flags + + +_BUILT_IN_FLAGS = frozenset({ + 'help', + 'helpshort', + 'helpfull', + 'helpxml', + 'flagfile', + 'undefok', +}) + + +class ArgumentParser(argparse.ArgumentParser): + """Custom ArgumentParser class to support special absl flags.""" + + def __init__(self, **kwargs): + """Initializes ArgumentParser. + + Args: + **kwargs: same as argparse.ArgumentParser, except: + 1. It also accepts `inherited_absl_flags`: the absl flags to inherit. + The default is the global absl.flags.FLAGS instance. Pass None to + ignore absl flags. + 2. The `prefix_chars` argument must be the default value '-'. + + Raises: + ValueError: Raised when prefix_chars is not '-'. + """ + prefix_chars = kwargs.get('prefix_chars', '-') + if prefix_chars != '-': + raise ValueError( + 'argparse_flags.ArgumentParser only supports "-" as the prefix ' + 'character, found "{}".'.format(prefix_chars)) + + # Remove inherited_absl_flags before calling super. + self._inherited_absl_flags = kwargs.pop('inherited_absl_flags', flags.FLAGS) + # Now call super to initialize argparse.ArgumentParser before calling + # add_argument in _define_absl_flags. + super().__init__(**kwargs) + + if self.add_help: + # -h and --help are defined in super. + # Also add the --helpshort and --helpfull flags. + self.add_argument( + # Action 'help' defines a similar flag to -h/--help. + '--helpshort', action='help', + default=argparse.SUPPRESS, help=argparse.SUPPRESS) + self.add_argument( + '--helpfull', action=_HelpFullAction, + default=argparse.SUPPRESS, help='show full help message and exit') + + if self._inherited_absl_flags is not None: + self.add_argument( + '--undefok', default=argparse.SUPPRESS, help=argparse.SUPPRESS) + self._define_absl_flags(self._inherited_absl_flags) + + def parse_known_args(self, args=None, namespace=None): + if args is None: + args = sys.argv[1:] + if self._inherited_absl_flags is not None: + # Handle --flagfile. + # Explicitly specify force_gnu=True, since argparse behaves like + # gnu_getopt: flags can be specified after positional arguments. + args = self._inherited_absl_flags.read_flags_from_files( + args, force_gnu=True) + + undefok_missing = object() + undefok = getattr(namespace, 'undefok', undefok_missing) + + namespace, args = super().parse_known_args(args, namespace) + + # For Python <= 2.7.8: https://bugs.python.org/issue9351, a bug where + # sub-parsers don't preserve existing namespace attributes. + # Restore the undefok attribute if a sub-parser dropped it. + if undefok is not undefok_missing: + namespace.undefok = undefok + + if self._inherited_absl_flags is not None: + # Handle --undefok. At this point, `args` only contains unknown flags, + # so it won't strip defined flags that are also specified with --undefok. + # For Python <= 2.7.8: https://bugs.python.org/issue9351, a bug where + # sub-parsers don't preserve existing namespace attributes. The undefok + # attribute might not exist because a subparser dropped it. + if hasattr(namespace, 'undefok'): + args = _strip_undefok_args(namespace.undefok, args) + # absl flags are not exposed in the Namespace object. See Namespace: + # https://docs.python.org/3/library/argparse.html#argparse.Namespace. + del namespace.undefok + self._inherited_absl_flags.mark_as_parsed() + try: + self._inherited_absl_flags.validate_all_flags() + except flags.IllegalFlagValueError as e: + self.error(str(e)) + + return namespace, args + + def _define_absl_flags(self, absl_flags): + """Defines flags from absl_flags.""" + key_flags = set(absl_flags.get_key_flags_for_module(sys.argv[0])) + for name in absl_flags: + if name in _BUILT_IN_FLAGS: + # Do not inherit built-in flags. + continue + flag_instance = absl_flags[name] + # Each flags with short_name appears in FLAGS twice, so only define + # when the dictionary key is equal to the regular name. + if name == flag_instance.name: + # Suppress the flag in the help short message if it's not a main + # module's key flag. + suppress = flag_instance not in key_flags + self._define_absl_flag(flag_instance, suppress) + + def _define_absl_flag(self, flag_instance, suppress): + """Defines a flag from the flag_instance.""" + flag_name = flag_instance.name + short_name = flag_instance.short_name + argument_names = ['--' + flag_name] + if short_name: + argument_names.insert(0, '-' + short_name) + if suppress: + helptext = argparse.SUPPRESS + else: + # argparse help string uses %-formatting. Escape the literal %'s. + helptext = flag_instance.help.replace('%', '%%') + if flag_instance.boolean: + # Only add the `no` form to the long name. + argument_names.append('--no' + flag_name) + self.add_argument( + *argument_names, action=_BooleanFlagAction, help=helptext, + metavar=flag_instance.name.upper(), + flag_instance=flag_instance) + else: + self.add_argument( + *argument_names, action=_FlagAction, help=helptext, + metavar=flag_instance.name.upper(), + flag_instance=flag_instance) + + +class _FlagAction(argparse.Action): + """Action class for Abseil non-boolean flags.""" + + def __init__( + self, + option_strings, + dest, + help, # pylint: disable=redefined-builtin + metavar, + flag_instance, + default=argparse.SUPPRESS): + """Initializes _FlagAction. + + Args: + option_strings: See argparse.Action. + dest: Ignored. The flag is always defined with dest=argparse.SUPPRESS. + help: See argparse.Action. + metavar: See argparse.Action. + flag_instance: absl.flags.Flag, the absl flag instance. + default: Ignored. The flag always uses dest=argparse.SUPPRESS so it + doesn't affect the parsing result. + """ + del dest + self._flag_instance = flag_instance + super().__init__( + option_strings=option_strings, + dest=argparse.SUPPRESS, + help=help, + metavar=metavar, + ) + + def __call__(self, parser, namespace, values, option_string=None): + """See https://docs.python.org/3/library/argparse.html#action-classes.""" + self._flag_instance.parse(values) + self._flag_instance.using_default_value = False + + +class _BooleanFlagAction(argparse.Action): + """Action class for Abseil boolean flags.""" + + def __init__( + self, + option_strings, + dest, + help, # pylint: disable=redefined-builtin + metavar, + flag_instance, + default=argparse.SUPPRESS): + """Initializes _BooleanFlagAction. + + Args: + option_strings: See argparse.Action. + dest: Ignored. The flag is always defined with dest=argparse.SUPPRESS. + help: See argparse.Action. + metavar: See argparse.Action. + flag_instance: absl.flags.Flag, the absl flag instance. + default: Ignored. The flag always uses dest=argparse.SUPPRESS so it + doesn't affect the parsing result. + """ + del dest, default + self._flag_instance = flag_instance + flag_names = [self._flag_instance.name] + if self._flag_instance.short_name: + flag_names.append(self._flag_instance.short_name) + self._flag_names = frozenset(flag_names) + super().__init__( + option_strings=option_strings, + dest=argparse.SUPPRESS, + nargs=0, # Does not accept values, only `--bool` or `--nobool`. + help=help, + metavar=metavar, + ) + + def __call__(self, parser, namespace, values, option_string=None): + """See https://docs.python.org/3/library/argparse.html#action-classes.""" + if not isinstance(values, list) or values: + raise ValueError('values must be an empty list.') + if option_string.startswith('--'): + option = option_string[2:] + else: + option = option_string[1:] + if option in self._flag_names: + self._flag_instance.parse('true') + else: + if not option.startswith('no') or option[2:] not in self._flag_names: + raise ValueError('invalid option_string: ' + option_string) + self._flag_instance.parse('false') + self._flag_instance.using_default_value = False + + +class _HelpFullAction(argparse.Action): + """Action class for --helpfull flag.""" + + def __init__(self, option_strings, dest, default, help): # pylint: disable=redefined-builtin + """Initializes _HelpFullAction. + + Args: + option_strings: See argparse.Action. + dest: Ignored. The flag is always defined with dest=argparse.SUPPRESS. + default: Ignored. + help: See argparse.Action. + """ + del dest, default + super().__init__( + option_strings=option_strings, + dest=argparse.SUPPRESS, + default=argparse.SUPPRESS, + nargs=0, + help=help, + ) + + def __call__(self, parser, namespace, values, option_string=None): + """See https://docs.python.org/3/library/argparse.html#action-classes.""" + # This only prints flags when help is not argparse.SUPPRESS. + # It includes user defined argparse flags, as well as main module's + # key absl flags. Other absl flags use argparse.SUPPRESS, so they aren't + # printed here. + parser.print_help() + + absl_flags = parser._inherited_absl_flags # pylint: disable=protected-access + if absl_flags is not None: + modules = sorted(absl_flags.flags_by_module_dict()) + main_module = sys.argv[0] + if main_module in modules: + # The main module flags are already printed in parser.print_help(). + modules.remove(main_module) + print(absl_flags._get_help_for_modules( # pylint: disable=protected-access + modules, prefix='', include_special_flags=True)) + parser.exit() + + +def _strip_undefok_args(undefok, args): + """Returns a new list of args after removing flags in --undefok.""" + if undefok: + undefok_names = {name.strip() for name in undefok.split(',')} + undefok_names |= {'no' + name for name in undefok_names} + # Remove undefok flags. + args = [arg for arg in args if not _is_undefok(arg, undefok_names)] + return args + + +def _is_undefok(arg, undefok_names): + """Returns whether we can ignore arg based on a set of undefok flag names.""" + if not arg.startswith('-'): + return False + if arg.startswith('--'): + arg_without_dash = arg[2:] + else: + arg_without_dash = arg[1:] + if '=' in arg_without_dash: + name, _ = arg_without_dash.split('=', 1) + else: + name = arg_without_dash + if name in undefok_names: + return True + return False diff --git a/venv/Lib/site-packages/absl/logging/__init__.py b/venv/Lib/site-packages/absl/logging/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..532bd758d56e75778fca62437a6192c5a36531bc --- /dev/null +++ b/venv/Lib/site-packages/absl/logging/__init__.py @@ -0,0 +1,1335 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abseil Python logging module implemented on top of standard logging. + +Simple usage:: + + from absl import logging + + logging.info('Interesting Stuff') + logging.info('Interesting Stuff with Arguments: %d', 42) + + logging.set_verbosity(logging.INFO) + logging.log(logging.DEBUG, 'This will *not* be printed') + logging.set_verbosity(logging.DEBUG) + logging.log(logging.DEBUG, 'This will be printed') + + logging.warning('Worrying Stuff') + logging.error('Alarming Stuff') + logging.fatal('AAAAHHHHH!!!!') # Process exits. + +Usage note: Do not pre-format the strings in your program code. +Instead, let the logging module perform argument interpolation. +This saves cycles because strings that don't need to be printed +are never formatted. Note that this module does not attempt to +interpolate arguments when no arguments are given. In other words:: + + logging.info('Interesting Stuff: %s') + +does not raise an exception because logging.info() has only one +argument, the message string. + +"Lazy" evaluation for debugging +------------------------------- + +If you do something like this:: + + logging.debug('Thing: %s', thing.ExpensiveOp()) + +then the ExpensiveOp will be evaluated even if nothing +is printed to the log. To avoid this, use the level_debug() function:: + + if logging.level_debug(): + logging.debug('Thing: %s', thing.ExpensiveOp()) + +Per file level logging is supported by logging.vlog() and +logging.vlog_is_on(). For example:: + + if logging.vlog_is_on(2): + logging.vlog(2, very_expensive_debug_message()) + +Notes on Unicode +---------------- + +The log output is encoded as UTF-8. Don't pass data in other encodings in +bytes() instances -- instead pass unicode string instances when you need to +(for both the format string and arguments). + +Note on critical and fatal: +Standard logging module defines fatal as an alias to critical, but it's not +documented, and it does NOT actually terminate the program. +This module only defines fatal but not critical, and it DOES terminate the +program. + +The differences in behavior are historical and unfortunate. +""" + +import collections +from collections.abc import Mapping +import getpass +import inspect +import io +import itertools +import logging +import os +import socket +import struct +import sys +import tempfile +import threading +import time +import timeit +import traceback +import warnings + +from absl import flags +from absl.logging import converter + +# pylint: disable=g-import-not-at-top +try: + from typing import NoReturn +except ImportError: + pass + +# pylint: enable=g-import-not-at-top + +FLAGS = flags.FLAGS + + +# Logging levels. +FATAL = converter.ABSL_FATAL +ERROR = converter.ABSL_ERROR +WARNING = converter.ABSL_WARNING +WARN = converter.ABSL_WARNING # Deprecated name. +INFO = converter.ABSL_INFO +DEBUG = converter.ABSL_DEBUG + +# Regex to match/parse log line prefixes. +ABSL_LOGGING_PREFIX_REGEX = ( + r'^(?P[IWEF])' + r'(?P\d\d)(?P\d\d) ' + r'(?P\d\d):(?P\d\d):(?P\d\d)' + r'\.(?P\d\d\d\d\d\d) +' + r'(?P-?\d+) ' + r'(?P[a-zA-Z<][\w._<>-]+):(?P\d+)') + + +# Mask to convert integer thread ids to unsigned quantities for logging purposes +_THREAD_ID_MASK = 2 ** (struct.calcsize('L') * 8) - 1 + +# Extra property set on the LogRecord created by ABSLLogger when its level is +# CRITICAL/FATAL. +_ABSL_LOG_FATAL = '_absl_log_fatal' +# Extra prefix added to the log message when a non-absl logger logs a +# CRITICAL/FATAL message. +_CRITICAL_PREFIX = 'CRITICAL - ' + +# Used by findCaller to skip callers from */logging/__init__.py. +_LOGGING_FILE_PREFIX = os.path.join('logging', '__init__.') + +# The ABSL logger instance, initialized in _initialize(). +_absl_logger = None +# The ABSL handler instance, initialized in _initialize(). +_absl_handler = None + + +_CPP_NAME_TO_LEVELS = { + 'debug': '0', # Abseil C++ has no DEBUG level, mapping it to INFO here. + 'info': '0', + 'warning': '1', + 'warn': '1', + 'error': '2', + 'fatal': '3' +} + +_CPP_LEVEL_TO_NAMES = { + '0': 'info', + '1': 'warning', + '2': 'error', + '3': 'fatal', +} + + +class _VerbosityFlag(flags.Flag): + """Flag class for -v/--verbosity.""" + + def __init__(self, *args, **kwargs): + super().__init__( + flags.IntegerParser(), flags.ArgumentSerializer(), *args, **kwargs + ) + + @property + def value(self): + return self._value + + @value.setter + def value(self, v): + self._value = v + self._update_logging_levels() + + def _update_logging_levels(self): + """Updates absl logging levels to the current verbosity. + + Visibility: module-private + """ + if not _absl_logger: + return + + if self._value <= converter.ABSL_DEBUG: + standard_verbosity = converter.absl_to_standard(self._value) + else: + # --verbosity is set to higher than 1 for vlog. + standard_verbosity = logging.DEBUG - (self._value - 1) + + # Also update root level when absl_handler is used. + if _absl_handler in logging.root.handlers: + # Make absl logger inherit from the root logger. absl logger might have + # a non-NOTSET value if logging.set_verbosity() is called at import time. + _absl_logger.setLevel(logging.NOTSET) + logging.root.setLevel(standard_verbosity) + else: + _absl_logger.setLevel(standard_verbosity) + + +class _LoggerLevelsFlag(flags.Flag): + """Flag class for --logger_levels.""" + + def __init__(self, *args, **kwargs): + super().__init__( + _LoggerLevelsParser(), _LoggerLevelsSerializer(), *args, **kwargs + ) + + @property + def value(self): + # For lack of an immutable type, be defensive and return a copy. + # Modifications to the dict aren't supported and won't have any affect. + # While Py3 could use MappingProxyType, that isn't deepcopy friendly, so + # just return a copy. + return self._value.copy() + + @value.setter + def value(self, v): + self._value = {} if v is None else v + self._update_logger_levels() + + def _update_logger_levels(self): + # Visibility: module-private. + # This is called by absl.app.run() during initialization. + for name, level in self._value.items(): + logging.getLogger(name).setLevel(level) + + +class _LoggerLevelsParser(flags.ArgumentParser): + """Parser for --logger_levels flag.""" + + def parse(self, value): + if isinstance(value, Mapping): + return value + + pairs = [pair.strip() for pair in value.split(',') if pair.strip()] + + # Preserve the order so that serialization is deterministic. + levels = collections.OrderedDict() + for name_level in pairs: + name, level = name_level.split(':', 1) + name = name.strip() + level = level.strip() + levels[name] = level + return levels + + +class _LoggerLevelsSerializer: + """Serializer for --logger_levels flag.""" + + def serialize(self, value): + if isinstance(value, str): + return value + return ','.join(f'{name}:{level}' for name, level in value.items()) + + +class _StderrthresholdFlag(flags.Flag): + """Flag class for --stderrthreshold.""" + + def __init__(self, *args, **kwargs): + super().__init__( + flags.ArgumentParser(), flags.ArgumentSerializer(), *args, **kwargs + ) + + @property + def value(self): + return self._value + + @value.setter + def value(self, v): + if v in _CPP_LEVEL_TO_NAMES: + # --stderrthreshold also accepts numeric strings whose values are + # Abseil C++ log levels. + cpp_value = int(v) + v = _CPP_LEVEL_TO_NAMES[v] # Normalize to strings. + elif v.lower() in _CPP_NAME_TO_LEVELS: + v = v.lower() + if v == 'warn': + v = 'warning' # Use 'warning' as the canonical name. + cpp_value = int(_CPP_NAME_TO_LEVELS[v]) + else: + raise ValueError( + '--stderrthreshold must be one of (case-insensitive) ' + "'debug', 'info', 'warning', 'error', 'fatal', " + "or '0', '1', '2', '3', not '%s'" % v) + + self._value = v + + +LOGTOSTDERR = flags.DEFINE_boolean( + 'logtostderr', + False, + 'Should only log to stderr?', + allow_override_cpp=True, +) +ALSOLOGTOSTDERR = flags.DEFINE_boolean( + 'alsologtostderr', + False, + 'also log to stderr?', + allow_override_cpp=True, +) +LOG_DIR = flags.DEFINE_string( + 'log_dir', + os.getenv('TEST_TMPDIR', ''), + 'directory to write logfiles into', + allow_override_cpp=True, +) +VERBOSITY = flags.DEFINE_flag( + _VerbosityFlag( + 'verbosity', + -1, + ( + 'Logging verbosity level. Messages logged at this level or lower' + ' will be included. Set to 1 for debug logging. If the flag was not' + ' set or supplied, the value will be changed from the default of -1' + ' (warning) to 0 (info) after flags are parsed.' + ), + short_name='v', + allow_hide_cpp=True, + ) +) +LOGGER_LEVELS = flags.DEFINE_flag( + _LoggerLevelsFlag( + 'logger_levels', + {}, + ( + 'Specify log level of loggers. The format is a CSV list of ' + '`name:level`. Where `name` is the logger name used with ' + '`logging.getLogger()`, and `level` is a level name (INFO, DEBUG, ' + 'etc). e.g. `myapp.foo:INFO,other.logger:DEBUG`' + ), + ) +) +STDERRTHRESHOLD = flags.DEFINE_flag( + _StderrthresholdFlag( + 'stderrthreshold', + 'fatal', + ( + 'log messages at this level, or more severe, to stderr in ' + 'addition to the logfile. Possible values are ' + "'debug', 'info', 'warning', 'error', and 'fatal'. " + 'Obsoletes --alsologtostderr. Using --alsologtostderr ' + 'cancels the effect of this flag. Please also note that ' + 'this flag is subject to --verbosity and requires logfile ' + 'not be stderr.' + ), + allow_hide_cpp=True, + ) +) +SHOWPREFIXFORINFO = flags.DEFINE_boolean( + 'showprefixforinfo', + True, + ( + 'If False, do not prepend prefix to info messages ' + "when it's logged to stderr, " + '--verbosity is set to INFO level, ' + 'and python logging is used.' + ), +) + + +def get_verbosity(): + """Returns the logging verbosity.""" + return FLAGS['verbosity'].value + + +def set_verbosity(v): + """Sets the logging verbosity. + + Causes all messages of level <= v to be logged, + and all messages of level > v to be silently discarded. + + Args: + v: int|str, the verbosity level as an integer or string. Legal string values + are those that can be coerced to an integer as well as case-insensitive + 'debug', 'info', 'warning', 'error', and 'fatal'. + """ + try: + new_level = int(v) + except ValueError: + new_level = converter.ABSL_NAMES[v.upper()] + FLAGS.verbosity = new_level + + +def set_stderrthreshold(s): + """Sets the stderr threshold to the value passed in. + + Args: + s: str|int, valid strings values are case-insensitive 'debug', + 'info', 'warning', 'error', and 'fatal'; valid integer values are + logging.DEBUG|INFO|WARNING|ERROR|FATAL. + + Raises: + ValueError: Raised when s is an invalid value. + """ + if s in converter.ABSL_LEVELS: + FLAGS.stderrthreshold = converter.ABSL_LEVELS[s] + elif isinstance(s, str) and s.upper() in converter.ABSL_NAMES: + FLAGS.stderrthreshold = s + else: + raise ValueError( + 'set_stderrthreshold only accepts integer absl logging level ' + 'from -3 to 1, or case-insensitive string values ' + "'debug', 'info', 'warning', 'error', and 'fatal'. " + 'But found "{}" ({}).'.format(s, type(s))) + + +def fatal(msg, *args, **kwargs): + # type: (Any, Any, Any) -> NoReturn + """Logs a fatal message.""" + log(FATAL, msg, *args, **kwargs) + + +def error(msg, *args, **kwargs): + """Logs an error message.""" + log(ERROR, msg, *args, **kwargs) + + +def warning(msg, *args, **kwargs): + """Logs a warning message.""" + log(WARNING, msg, *args, **kwargs) + + +def warn(msg, *args, **kwargs): + """Deprecated, use 'warning' instead.""" + warnings.warn("The 'warn' function is deprecated, use 'warning' instead", + DeprecationWarning, 2) + log(WARNING, msg, *args, **kwargs) + + +def info(msg, *args, **kwargs): + """Logs an info message.""" + log(INFO, msg, *args, **kwargs) + + +def debug(msg, *args, **kwargs): + """Logs a debug message.""" + log(DEBUG, msg, *args, **kwargs) + + +def exception(msg, *args, exc_info=True, **kwargs): + """Logs an exception, with traceback and message.""" + error(msg, *args, exc_info=exc_info, **kwargs) + + +def _fast_stack_trace(): + """A fast stack trace that gets us the minimal information we need. + + Compared to using `get_absl_logger().findCaller(stack_info=True)`, this + function is ~100x faster. + + Returns: + A tuple of tuples of (filename, line_number, last_instruction_offset). + """ + cur_stack = inspect.currentframe() + if cur_stack is None or cur_stack.f_back is None: + return tuple() + # We drop the first frame, which is this function itself. + cur_stack = cur_stack.f_back + call_stack = [] + while cur_stack.f_back: + cur_stack = cur_stack.f_back + call_stack.append( + (cur_stack.f_code.co_filename, cur_stack.f_lineno, cur_stack.f_lasti) + ) + return tuple(call_stack) + + +# Counter to keep track of number of log entries per token. +_log_counter_per_token = {} + + +def _get_next_log_count_per_token(token): + """Wrapper for _log_counter_per_token. Thread-safe. + + Args: + token: The token for which to look up the count. + + Returns: + The number of times this function has been called with + *token* as an argument (starting at 0). + """ + # Can't use a defaultdict because defaultdict isn't atomic, whereas + # setdefault is. + return next(_log_counter_per_token.setdefault(token, itertools.count())) + + +def log_every_n(level, msg, n, *args, use_call_stack=False, **kwargs): + """Logs ``msg % args`` at level 'level' once per 'n' times. + + Logs the 1st call, (N+1)st call, (2N+1)st call, etc. + Not threadsafe. + + Args: + level: int, the absl logging level at which to log. + msg: str, the message to be logged. + n: int, the number of times this should be called before it is logged. + *args: The args to be substituted into the msg. + use_call_stack: bool, whether to include the call stack when counting the + number of times the message is logged. + **kwargs: May contain exc_info to add exception traceback to message. + """ + caller_info = get_absl_logger().findCaller() + if use_call_stack: + # To reduce storage costs, we hash the call stack. + caller_info = (*caller_info[0:3], hash(_fast_stack_trace())) + count = _get_next_log_count_per_token(caller_info) + log_if(level, msg, not (count % n), *args, **kwargs) + + +# Keeps track of the last log time of the given token. +# Note: must be a dict since set/get is atomic in CPython. +# Note: entries are never released as their number is expected to be low. +_log_timer_per_token = {} + + +def _seconds_have_elapsed(token, num_seconds): + """Tests if 'num_seconds' have passed since 'token' was requested. + + Not strictly thread-safe - may log with the wrong frequency if called + concurrently from multiple threads. Accuracy depends on resolution of + 'timeit.default_timer()'. + + Always returns True on the first call for a given 'token'. + + Args: + token: The token for which to look up the count. + num_seconds: The number of seconds to test for. + + Returns: + Whether it has been >= 'num_seconds' since 'token' was last requested. + """ + now = timeit.default_timer() + then = _log_timer_per_token.get(token, None) + if then is None or (now - then) >= num_seconds: + _log_timer_per_token[token] = now + return True + else: + return False + + +def log_every_n_seconds( + level, msg, n_seconds, *args, use_call_stack=False, **kwargs +): + """Logs ``msg % args`` at level ``level`` iff ``n_seconds`` elapsed since last call. + + Logs the first call, logs subsequent calls if 'n' seconds have elapsed since + the last logging call from the same call site (file + line). Not thread-safe. + + Args: + level: int, the absl logging level at which to log. + msg: str, the message to be logged. + n_seconds: float or int, seconds which should elapse before logging again. + *args: The args to be substituted into the msg. + use_call_stack: bool, whether to include the call stack when counting the + number of times the message is logged. + **kwargs: May contain exc_info to add exception traceback to message. + """ + caller_info = get_absl_logger().findCaller() + if use_call_stack: + # To reduce storage costs, we hash the call stack. + caller_info = (*caller_info[0:3], hash(_fast_stack_trace())) + should_log = _seconds_have_elapsed(caller_info, n_seconds) + log_if(level, msg, should_log, *args, **kwargs) + + +def log_first_n(level, msg, n, *args, use_call_stack=False, **kwargs): + """Logs ``msg % args`` at level ``level`` only first ``n`` times. + + Not threadsafe. + + Args: + level: int, the absl logging level at which to log. + msg: str, the message to be logged. + n: int, the maximal number of times the message is logged. + *args: The args to be substituted into the msg. + use_call_stack: bool, whether to include the call stack when counting the + number of times the message is logged. + **kwargs: May contain exc_info to add exception traceback to message. + """ + caller_info = get_absl_logger().findCaller() + if use_call_stack: + # To reduce storage costs, we hash the call stack. + caller_info = (*caller_info[0:3], hash(_fast_stack_trace())) + count = _get_next_log_count_per_token(caller_info) + log_if(level, msg, count < n, *args, **kwargs) + + +def log_if(level, msg, condition, *args, **kwargs): + """Logs ``msg % args`` at level ``level`` only if condition is fulfilled.""" + if condition: + log(level, msg, *args, **kwargs) + + +def log(level, msg, *args, **kwargs): + """Logs ``msg % args`` at absl logging level ``level``. + + If no args are given just print msg, ignoring any interpolation specifiers. + + Args: + level: int, the absl logging level at which to log the message + (logging.DEBUG|INFO|WARNING|ERROR|FATAL). While some C++ verbose logging + level constants are also supported, callers should prefer explicit + logging.vlog() calls for such purpose. + + msg: str, the message to be logged. + *args: The args to be substituted into the msg. + **kwargs: May contain exc_info to add exception traceback to message. + """ + if level > converter.ABSL_DEBUG: + # Even though this function supports level that is greater than 1, users + # should use logging.vlog instead for such cases. + # Treat this as vlog, 1 is equivalent to DEBUG. + standard_level = converter.STANDARD_DEBUG - (level - 1) + else: + if level < converter.ABSL_FATAL: + level = converter.ABSL_FATAL + standard_level = converter.absl_to_standard(level) + + # Match standard logging's behavior. Before use_absl_handler() and + # logging is configured, there is no handler attached on _absl_logger nor + # logging.root. So logs go no where. + if not logging.root.handlers: + logging.basicConfig() + + _absl_logger.log(standard_level, msg, *args, **kwargs) + + +def vlog(level, msg, *args, **kwargs): + """Log ``msg % args`` at C++ vlog level ``level``. + + Args: + level: int, the C++ verbose logging level at which to log the message, + e.g. 1, 2, 3, 4... While absl level constants are also supported, + callers should prefer logging.log|debug|info|... calls for such purpose. + msg: str, the message to be logged. + *args: The args to be substituted into the msg. + **kwargs: May contain exc_info to add exception traceback to message. + """ + log(level, msg, *args, **kwargs) + + +def vlog_is_on(level): + """Checks if vlog is enabled for the given level in caller's source file. + + Args: + level: int, the C++ verbose logging level at which to log the message, + e.g. 1, 2, 3, 4... While absl level constants are also supported, + callers should prefer level_debug|level_info|... calls for + checking those. + + Returns: + True if logging is turned on for that level. + """ + + if level > converter.ABSL_DEBUG: + # Even though this function supports level that is greater than 1, users + # should use logging.vlog instead for such cases. + # Treat this as vlog, 1 is equivalent to DEBUG. + standard_level = converter.STANDARD_DEBUG - (level - 1) + else: + if level < converter.ABSL_FATAL: + level = converter.ABSL_FATAL + standard_level = converter.absl_to_standard(level) + return _absl_logger.isEnabledFor(standard_level) + + +def flush(): + """Flushes all log files.""" + get_absl_handler().flush() + + +def level_debug(): + """Returns True if debug logging is turned on.""" + return get_verbosity() >= DEBUG + + +def level_info(): + """Returns True if info logging is turned on.""" + return get_verbosity() >= INFO + + +def level_warning(): + """Returns True if warning logging is turned on.""" + return get_verbosity() >= WARNING + + +level_warn = level_warning # Deprecated function. + + +def level_error(): + """Returns True if error logging is turned on.""" + return get_verbosity() >= ERROR + + +def get_log_file_name(level=INFO): + """Returns the name of the log file. + + For Python logging, only one file is used and level is ignored. And it returns + empty string if it logs to stderr/stdout or the log stream has no `name` + attribute. + + Args: + level: int, the absl.logging level. + + Raises: + ValueError: Raised when `level` has an invalid value. + """ + if level not in converter.ABSL_LEVELS: + raise ValueError(f'Invalid absl.logging level {level}') + stream = get_absl_handler().python_handler.stream + if (stream == sys.stderr or stream == sys.stdout or + not hasattr(stream, 'name')): + return '' + else: + return stream.name + + +def find_log_dir_and_names(program_name=None, log_dir=None): + """Computes the directory and filename prefix for log file. + + Args: + program_name: str|None, the filename part of the path to the program that is + running without its extension. e.g: if your program is called + ``usr/bin/foobar.py`` this method should probably be called with + ``program_name='foobar`` However, this is just a convention, you can pass + in any string you want, and it will be used as part of the log filename. + If you don't pass in anything, the default behavior is as described in the + example. In python standard logging mode, the program_name will be + prepended with ``py_`` if it is the ``program_name`` argument is omitted. + log_dir: str|None, the desired log directory. + + Returns: + (log_dir, file_prefix, symlink_prefix) + + Raises: + FileNotFoundError: raised when it cannot find a log directory. + """ + if not program_name: + # Strip the extension (foobar.par becomes foobar, and + # fubar.py becomes fubar). We do this so that the log + # file names are similar to C++ log file names. + program_name = os.path.splitext(os.path.basename(sys.argv[0]))[0] + + # Prepend py_ to files so that python code gets a unique file, and + # so that C++ libraries do not try to write to the same log files as us. + program_name = 'py_%s' % program_name + + actual_log_dir = find_log_dir(log_dir=log_dir) + + try: + username = getpass.getuser() + except KeyError: + # This can happen, e.g. when running under docker w/o passwd file. + if hasattr(os, 'getuid'): + # Windows doesn't have os.getuid + username = str(os.getuid()) + else: + username = 'unknown' + hostname = socket.gethostname() + file_prefix = '%s.%s.%s.log' % (program_name, hostname, username) + + return actual_log_dir, file_prefix, program_name + + +def find_log_dir(log_dir=None): + """Returns the most suitable directory to put log files into. + + Args: + log_dir: str|None, if specified, the logfile(s) will be created in that + directory. Otherwise if the --log_dir command-line flag is provided, the + logfile will be created in that directory. Otherwise the logfile will be + created in a standard location. + + Raises: + FileNotFoundError: raised when it cannot find a log directory. + """ + # Get a list of possible log dirs (will try to use them in order). + # NOTE: Google's internal implementation has a special handling for Google + # machines, which uses a list of directories. Hence the following uses `dirs` + # instead of a single directory. + if log_dir: + # log_dir was explicitly specified as an arg, so use it and it alone. + dirs = [log_dir] + elif FLAGS['log_dir'].value: + # log_dir flag was provided, so use it and it alone (this mimics the + # behavior of the same flag in logging.cc). + dirs = [FLAGS['log_dir'].value] + else: + dirs = [tempfile.gettempdir()] + + # Find the first usable log dir. + for d in dirs: + if os.path.isdir(d) and os.access(d, os.W_OK): + return d + raise FileNotFoundError( + "Can't find a writable directory for logs, tried %s" % dirs) + + +def get_absl_log_prefix(record): + """Returns the absl log prefix for the log record. + + Args: + record: logging.LogRecord, the record to get prefix for. + """ + created_tuple = time.localtime(record.created) + created_microsecond = int(record.created % 1.0 * 1e6) + + critical_prefix = '' + level = record.levelno + if _is_non_absl_fatal_record(record): + # When the level is FATAL, but not logged from absl, lower the level so + # it's treated as ERROR. + level = logging.ERROR + critical_prefix = _CRITICAL_PREFIX + severity = converter.get_initial_for_level(level) + + return '%c%02d%02d %02d:%02d:%02d.%06d %5d %s:%d] %s' % ( + severity, + created_tuple.tm_mon, + created_tuple.tm_mday, + created_tuple.tm_hour, + created_tuple.tm_min, + created_tuple.tm_sec, + created_microsecond, + _get_thread_id(), + record.filename, + record.lineno, + critical_prefix) + + +def skip_log_prefix(func): + """Skips reporting the prefix of a given function or name by :class:`~absl.logging.ABSLLogger`. + + This is a convenience wrapper function / decorator for + :meth:`~absl.logging.ABSLLogger.register_frame_to_skip`. + + If a callable function is provided, only that function will be skipped. + If a function name is provided, all functions with the same name in the + file that this is called in will be skipped. + + This can be used as a decorator of the intended function to be skipped. + + Args: + func: Callable function or its name as a string. + + Returns: + func (the input, unchanged). + + Raises: + ValueError: The input is callable but does not have a function code object. + TypeError: The input is neither callable nor a string. + """ + if callable(func): + func_code = getattr(func, '__code__', None) + if func_code is None: + raise ValueError('Input callable does not have a function code object.') + file_name = func_code.co_filename + func_name = func_code.co_name + func_lineno = func_code.co_firstlineno + elif isinstance(func, str): + file_name = get_absl_logger().findCaller()[0] + func_name = func + func_lineno = None + else: + raise TypeError('Input is neither callable nor a string.') + ABSLLogger.register_frame_to_skip(file_name, func_name, func_lineno) + return func + + +def _is_non_absl_fatal_record(log_record): + return (log_record.levelno >= logging.FATAL and + not log_record.__dict__.get(_ABSL_LOG_FATAL, False)) + + +def _is_absl_fatal_record(log_record): + return (log_record.levelno >= logging.FATAL and + log_record.__dict__.get(_ABSL_LOG_FATAL, False)) + + +# Indicates if we still need to warn about pre-init logs going to stderr. +_warn_preinit_stderr = True + + +class PythonHandler(logging.StreamHandler): + """The handler class used by Abseil Python logging implementation.""" + + def __init__(self, stream=None, formatter=None): + super().__init__(stream) + self.setFormatter(formatter or PythonFormatter()) + + def start_logging_to_file(self, program_name=None, log_dir=None): + """Starts logging messages to files instead of standard error.""" + FLAGS.logtostderr = False + + actual_log_dir, file_prefix, symlink_prefix = find_log_dir_and_names( + program_name=program_name, log_dir=log_dir) + + basename = '%s.INFO.%s.%d' % ( + file_prefix, + time.strftime('%Y%m%d-%H%M%S', time.localtime(time.time())), + os.getpid()) + filename = os.path.join(actual_log_dir, basename) + + self.stream = open(filename, 'a', encoding='utf-8') + + # os.symlink is not available on Windows Python 2. + if getattr(os, 'symlink', None): + # Create a symlink to the log file with a canonical name. + symlink = os.path.join(actual_log_dir, symlink_prefix + '.INFO') + try: + if os.path.islink(symlink): + os.unlink(symlink) + os.symlink(os.path.basename(filename), symlink) + except OSError: + # If it fails, we're sad but it's no error. Commonly, this + # fails because the symlink was created by another user and so + # we can't modify it + pass + + def use_absl_log_file(self, program_name=None, log_dir=None): + """Conditionally logs to files, based on --logtostderr.""" + if FLAGS['logtostderr'].value: + self.stream = sys.stderr + else: + self.start_logging_to_file(program_name=program_name, log_dir=log_dir) + + def flush(self): + """Flushes all log files.""" + self.acquire() + try: + if self.stream and hasattr(self.stream, 'flush'): + self.stream.flush() + except (OSError, ValueError): + # A ValueError is thrown if we try to flush a closed file. + pass + finally: + self.release() + + def _log_to_stderr(self, record): + """Emits the record to stderr. + + This temporarily sets the handler stream to stderr, calls + StreamHandler.emit, then reverts the stream back. + + Args: + record: logging.LogRecord, the record to log. + """ + # emit() is protected by a lock in logging.Handler, so we don't need to + # protect here again. + old_stream = self.stream + self.stream = sys.stderr + try: + super().emit(record) + finally: + self.stream = old_stream + + def emit(self, record): + """Prints a record out to some streams. + + 1. If ``FLAGS.logtostderr`` is set, it will print to ``sys.stderr`` ONLY. + 2. If ``FLAGS.alsologtostderr`` is set, it will print to ``sys.stderr``. + 3. If ``FLAGS.logtostderr`` is not set, it will log to the stream + associated with the current thread. + + Args: + record: :class:`logging.LogRecord`, the record to emit. + """ + # People occasionally call logging functions at import time before + # our flags may have even been defined yet, let alone even parsed, as we + # rely on the C++ side to define some flags for us and app init to + # deal with parsing. Match the C++ library behavior of notify and emit + # such messages to stderr. It encourages people to clean-up and does + # not hide the message. + level = record.levelno + if not FLAGS.is_parsed(): # Also implies "before flag has been defined". + global _warn_preinit_stderr + if _warn_preinit_stderr: + sys.stderr.write( + 'WARNING: Logging before flag parsing goes to stderr.\n') + _warn_preinit_stderr = False + self._log_to_stderr(record) + elif FLAGS['logtostderr'].value: + self._log_to_stderr(record) + else: + super().emit(record) + stderr_threshold = converter.string_to_standard( + FLAGS['stderrthreshold'].value) + if ((FLAGS['alsologtostderr'].value or level >= stderr_threshold) and + self.stream != sys.stderr): + self._log_to_stderr(record) + # Die when the record is created from ABSLLogger and level is FATAL. + if _is_absl_fatal_record(record): + self.flush() # Flush the log before dying. + + # In threaded python, sys.exit() from a non-main thread only + # exits the thread in question. + os.abort() + + def close(self): + """Closes the stream to which we are writing.""" + self.acquire() + try: + self.flush() + try: + # Do not close the stream if it's sys.stderr|stdout. They may be + # redirected or overridden to files, which should be managed by users + # explicitly. + user_managed = sys.stderr, sys.stdout, sys.__stderr__, sys.__stdout__ + if self.stream not in user_managed and ( + not hasattr(self.stream, 'isatty') or not self.stream.isatty()): + self.stream.close() + except ValueError: + # A ValueError is thrown if we try to run isatty() on a closed file. + pass + super().close() + finally: + self.release() + + +class ABSLHandler(logging.Handler): + """Abseil Python logging module's log handler.""" + + def __init__(self, python_logging_formatter): + super().__init__() + + self._python_handler = PythonHandler(formatter=python_logging_formatter) + self.activate_python_handler() + + def format(self, record): + return self._current_handler.format(record) + + def setFormatter(self, fmt): + self._current_handler.setFormatter(fmt) + + def emit(self, record): + self._current_handler.emit(record) + + def flush(self): + self._current_handler.flush() + + def close(self): + super().close() + self._current_handler.close() + + def handle(self, record): + rv = self.filter(record) + if rv: + return self._current_handler.handle(record) + return rv + + @property + def python_handler(self): + return self._python_handler + + def activate_python_handler(self): + """Uses the Python logging handler as the current logging handler.""" + self._current_handler = self._python_handler + + def use_absl_log_file(self, program_name=None, log_dir=None): + self._current_handler.use_absl_log_file(program_name, log_dir) + + def start_logging_to_file(self, program_name=None, log_dir=None): + self._current_handler.start_logging_to_file(program_name, log_dir) + + +class PythonFormatter(logging.Formatter): + """Formatter class used by :class:`~absl.logging.PythonHandler`.""" + + def format(self, record): + """Appends the message from the record to the results of the prefix. + + Args: + record: logging.LogRecord, the record to be formatted. + + Returns: + The formatted string representing the record. + """ + if (not FLAGS['showprefixforinfo'].value and + FLAGS['verbosity'].value == converter.ABSL_INFO and + record.levelno == logging.INFO and + _absl_handler.python_handler.stream == sys.stderr): + prefix = '' + else: + prefix = get_absl_log_prefix(record) + return prefix + super().format(record) + + +class ABSLLogger(logging.getLoggerClass()): + """A logger that will create LogRecords while skipping some stack frames. + + This class maintains an internal list of filenames and method names + for use when determining who called the currently executing stack + frame. Any method names from specific source files are skipped when + walking backwards through the stack. + + Client code should use the register_frame_to_skip method to let the + ABSLLogger know which method from which file should be + excluded from the walk backwards through the stack. + """ + _frames_to_skip = set() + + def findCaller(self, stack_info=False, stacklevel=1): + """Finds the frame of the calling method on the stack. + + This method skips any frames registered with the + ABSLLogger and any methods from this file, and whatever + method is currently being used to generate the prefix for the log + line. Then it returns the file name, line number, and method name + of the calling method. An optional fourth item may be returned, + callers who only need things from the first three are advised to + always slice or index the result rather than using direct unpacking + assignment. + + Args: + stack_info: bool, when True, include the stack trace as a fourth item + returned. On Python 3 there are always four items returned - the fourth + will be None when this is False. On Python 2 the stdlib base class API + only returns three items. We do the same when this new parameter is + unspecified or False for compatibility. + stacklevel: int, if greater than 1, that number of frames will be skipped. + + Returns: + (filename, lineno, methodname[, sinfo]) of the calling method. + """ + f_to_skip = ABSLLogger._frames_to_skip + # Use sys._getframe(2) instead of logging.currentframe(), it's slightly + # faster because there is one less frame to traverse. + frame = sys._getframe(2) # pylint: disable=protected-access + frame_to_return = None + + while frame: + code = frame.f_code + if (_LOGGING_FILE_PREFIX not in code.co_filename and + (code.co_filename, code.co_name, + code.co_firstlineno) not in f_to_skip and + (code.co_filename, code.co_name) not in f_to_skip): + frame_to_return = frame + stacklevel -= 1 + if stacklevel <= 0: + break + frame = frame.f_back + + if frame_to_return is not None: + sinfo = None + if stack_info: + out = io.StringIO() + out.write('Stack (most recent call last):\n') + traceback.print_stack(frame, file=out) + sinfo = out.getvalue().rstrip('\n') + return ( + frame_to_return.f_code.co_filename, + frame_to_return.f_lineno, + frame_to_return.f_code.co_name, + sinfo, + ) + + return None + + def critical(self, msg, *args, **kwargs): + """Logs ``msg % args`` with severity ``CRITICAL``.""" + self.log(logging.CRITICAL, msg, *args, **kwargs) + + def fatal(self, msg, *args, **kwargs): + """Logs ``msg % args`` with severity ``FATAL``.""" + self.log(logging.FATAL, msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + """Logs ``msg % args`` with severity ``ERROR``.""" + self.log(logging.ERROR, msg, *args, **kwargs) + + def warn(self, msg, *args, **kwargs): + """Logs ``msg % args`` with severity ``WARN``.""" + warnings.warn("The 'warn' method is deprecated, use 'warning' instead", + DeprecationWarning, 2) + self.log(logging.WARN, msg, *args, **kwargs) + + def warning(self, msg, *args, **kwargs): + """Logs ``msg % args`` with severity ``WARNING``.""" + self.log(logging.WARNING, msg, *args, **kwargs) + + def info(self, msg, *args, **kwargs): + """Logs ``msg % args`` with severity ``INFO``.""" + self.log(logging.INFO, msg, *args, **kwargs) + + def debug(self, msg, *args, **kwargs): + """Logs ``msg % args`` with severity ``DEBUG``.""" + self.log(logging.DEBUG, msg, *args, **kwargs) + + def log(self, level, msg, *args, **kwargs): + """Logs a message at a certain level substituting in the supplied arguments. + + This method behaves differently in python and c++ modes. + + Args: + level: int, the standard logging level at which to log the message. + msg: str, the text of the message to log. + *args: The arguments to substitute in the message. + **kwargs: The keyword arguments to substitute in the message. + """ + if level >= logging.FATAL: + # Add property to the LogRecord created by this logger. + # This will be used by the ABSLHandler to determine whether it should + # treat CRITICAL/FATAL logs as really FATAL. + extra = kwargs.setdefault('extra', {}) + extra[_ABSL_LOG_FATAL] = True + super().log(level, msg, *args, **kwargs) + + def handle(self, record): + """Calls handlers without checking ``Logger.disabled``. + + Non-root loggers are set to disabled after setup with :func:`logging.config` + if it's not explicitly specified. Historically, absl logging will not be + disabled by that. To maintaining this behavior, this function skips + checking the ``Logger.disabled`` bit. + + This logger can still be disabled by adding a filter that filters out + everything. + + Args: + record: logging.LogRecord, the record to handle. + """ + if self.filter(record): + self.callHandlers(record) + + @classmethod + def register_frame_to_skip(cls, file_name, function_name, line_number=None): + """Registers a function name to skip when walking the stack. + + The :class:`~absl.logging.ABSLLogger` sometimes skips method calls on the + stack to make the log messages meaningful in their appropriate context. + This method registers a function from a particular file as one + which should be skipped. + + Args: + file_name: str, the name of the file that contains the function. + function_name: str, the name of the function to skip. + line_number: int, if provided, only the function with this starting line + number will be skipped. Otherwise, all functions with the same name + in the file will be skipped. + """ + if line_number is not None: + cls._frames_to_skip.add((file_name, function_name, line_number)) + else: + cls._frames_to_skip.add((file_name, function_name)) + + +def _get_thread_id(): + """Gets id of current thread, suitable for logging as an unsigned quantity. + + If pywrapbase is linked, returns GetTID() for the thread ID to be + consistent with C++ logging. Otherwise, returns the numeric thread id. + The quantities are made unsigned by masking with 2*sys.maxint + 1. + + Returns: + Thread ID unique to this process (unsigned) + """ + thread_id = threading.get_ident() + return thread_id & _THREAD_ID_MASK + + +def get_absl_logger(): + """Returns the absl logger instance.""" + assert _absl_logger is not None + return _absl_logger + + +def get_absl_handler(): + """Returns the absl handler instance.""" + assert _absl_handler is not None + return _absl_handler + + +def use_python_logging(quiet=False): + """Uses the python implementation of the logging code. + + Args: + quiet: No logging message about switching logging type. + """ + get_absl_handler().activate_python_handler() + if not quiet: + info('Restoring pure python logging') + + +_attempted_to_remove_stderr_stream_handlers = False + + +def use_absl_handler(): + """Uses the ABSL logging handler for logging. + + This method is called in :func:`app.run()` so the absl handler + is used in absl apps. + """ + global _attempted_to_remove_stderr_stream_handlers + if not _attempted_to_remove_stderr_stream_handlers: + # The absl handler logs to stderr by default. To prevent double logging to + # stderr, the following code tries its best to remove other handlers that + # emit to stderr. Those handlers are most commonly added when + # logging.info/debug is called before calling use_absl_handler(). + handlers = [ + h for h in logging.root.handlers + if isinstance(h, logging.StreamHandler) and h.stream == sys.stderr] + for h in handlers: + logging.root.removeHandler(h) + _attempted_to_remove_stderr_stream_handlers = True + + absl_handler = get_absl_handler() + if absl_handler not in logging.root.handlers: + logging.root.addHandler(absl_handler) + FLAGS['verbosity']._update_logging_levels() # pylint: disable=protected-access + FLAGS['logger_levels']._update_logger_levels() # pylint: disable=protected-access + + +def _initialize(): + """Initializes loggers and handlers.""" + global _absl_logger, _absl_handler + + if _absl_logger: + return + + original_logger_class = logging.getLoggerClass() + logging.setLoggerClass(ABSLLogger) + _absl_logger = logging.getLogger('absl') + logging.setLoggerClass(original_logger_class) + + python_logging_formatter = PythonFormatter() + _absl_handler = ABSLHandler(python_logging_formatter) + + +_initialize() diff --git a/venv/Lib/site-packages/absl/logging/__init__.pyi b/venv/Lib/site-packages/absl/logging/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..7e69ecb17177381accb5640fa580b7abb00183bf --- /dev/null +++ b/venv/Lib/site-packages/absl/logging/__init__.pyi @@ -0,0 +1,278 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable +import logging +from typing import Any, NoReturn, TypeVar + +from absl import flags + +# Logging levels. +FATAL: int +ERROR: int +WARNING: int +WARN: int # Deprecated name. +INFO: int +DEBUG: int + +ABSL_LOGGING_PREFIX_REGEX: str + +LOGTOSTDERR: flags.FlagHolder[bool] +ALSOLOGTOSTDERR: flags.FlagHolder[bool] +LOG_DIR: flags.FlagHolder[str] +VERBOSITY: flags.FlagHolder[int] +LOGGER_LEVELS: flags.FlagHolder[dict[str, str]] +STDERRTHRESHOLD: flags.FlagHolder[str] +SHOWPREFIXFORINFO: flags.FlagHolder[bool] + +def get_verbosity() -> int: + ... + +def set_verbosity(v: int | str) -> None: + ... + +def set_stderrthreshold(s: int | str) -> None: + ... + +# TODO(b/277607978): Provide actual args+kwargs shadowing stdlib's logging functions. +def fatal(msg: Any, *args: Any, **kwargs: Any) -> NoReturn: + ... + +def error(msg: Any, *args: Any, **kwargs: Any) -> None: + ... + +def warning(msg: Any, *args: Any, **kwargs: Any) -> None: + ... + +def warn(msg: Any, *args: Any, **kwargs: Any) -> None: + ... + +def info(msg: Any, *args: Any, **kwargs: Any) -> None: + ... + +def debug(msg: Any, *args: Any, **kwargs: Any) -> None: + ... + +def exception(msg: Any, *args: Any, **kwargs: Any) -> None: + ... + +def log_every_n( + level: int, + msg: Any, + n: int, + *args: Any, + use_call_stack: bool = ..., + **kwargs: Any, +) -> None: + ... + +def log_every_n_seconds( + level: int, + msg: Any, + n_seconds: float, + *args: Any, + use_call_stack: bool = ..., + **kwargs: Any, +) -> None: + ... + +def log_first_n( + level: int, + msg: Any, + n: int, + *args: Any, + use_call_stack: bool = ..., + **kwargs: Any, +) -> None: + ... + +def log_if(level: int, + msg: Any, + condition: Any, + *args: Any, + **kwargs: Any, +) -> None: + ... + +def log(level: int, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + +def vlog(level: int, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + +def vlog_is_on(level: int) -> bool: + ... + +def flush() -> None: + ... + +def level_debug() -> bool: + ... + +def level_info() -> bool: + ... + +def level_warning() -> bool: + ... + +level_warn = level_warning # Deprecated function. + +def level_error() -> bool: + ... + +def get_log_file_name(level: int = ...) -> str: + ... + +def find_log_dir_and_names( + program_name: str | None = ..., log_dir: str | None = ... +) -> tuple[str, str, str]: + ... + +def find_log_dir(log_dir: str | None = ...) -> str: + ... + +def get_absl_log_prefix(record: logging.LogRecord) -> str: + ... + +_SkipLogT = TypeVar('_SkipLogT', str, Callable[..., Any]) + +def skip_log_prefix(func: _SkipLogT) -> _SkipLogT: + ... + +_StreamT = TypeVar('_StreamT') + +class PythonHandler(logging.StreamHandler[_StreamT]): # type: ignore[type-var] + + def __init__( + self, + stream: _StreamT | None = ..., + formatter: logging.Formatter | None = ..., + ) -> None: + ... + + def start_logging_to_file( + self, program_name: str | None = ..., log_dir: str | None = ... + ) -> None: + ... + + def use_absl_log_file( + self, program_name: str | None = ..., log_dir: str | None = ... + ) -> None: + ... + + def flush(self) -> None: + ... + + def emit(self, record: logging.LogRecord) -> None: + ... + + def close(self) -> None: + ... + +class ABSLHandler(logging.Handler): + + def __init__(self, python_logging_formatter: PythonFormatter) -> None: + ... + + def format(self, record: logging.LogRecord) -> str: + ... + + def setFormatter(self, fmt) -> None: + ... + + def emit(self, record: logging.LogRecord) -> None: + ... + + def flush(self) -> None: + ... + + def close(self) -> None: + ... + + def handle(self, record: logging.LogRecord) -> bool: + ... + + @property + def python_handler(self) -> PythonHandler: + ... + + def activate_python_handler(self) -> None: + ... + + def use_absl_log_file( + self, program_name: str | None = ..., log_dir: str | None = ... + ) -> None: + ... + + def start_logging_to_file(self, program_name=None, log_dir=None) -> None: + ... + +class PythonFormatter(logging.Formatter): + + def format(self, record: logging.LogRecord) -> str: + ... + +class ABSLLogger(logging.Logger): + + def findCaller( + self, stack_info: bool = ..., stacklevel: int = ... + ) -> tuple[str, int, str, str | None]: + ... + + def critical(self, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + + def fatal(self, msg: Any, *args: Any, **kwargs: Any) -> NoReturn: # type: ignore[override] + ... + + def error(self, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + + def warn(self, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + + def warning(self, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + + def info(self, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + + def debug(self, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + + def log(self, level: int, msg: Any, *args: Any, **kwargs: Any) -> None: + ... + + def handle(self, record: logging.LogRecord) -> None: + ... + + @classmethod + def register_frame_to_skip( + cls, file_name: str, function_name: str, line_number: int | None = ... + ) -> None: + ... + +# NOTE: Returns None before _initialize called but shouldn't occur after import. +def get_absl_logger() -> ABSLLogger: + ... + +# NOTE: Returns None before _initialize called but shouldn't occur after import. +def get_absl_handler() -> ABSLHandler: + ... + +def use_python_logging(quiet: bool = ...) -> None: + ... + +def use_absl_handler() -> None: + ... diff --git a/venv/Lib/site-packages/absl/logging/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/absl/logging/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bcfa7f9594afbc81685f6206183ea80e1c8eb435 Binary files /dev/null and b/venv/Lib/site-packages/absl/logging/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/logging/__pycache__/converter.cpython-311.pyc b/venv/Lib/site-packages/absl/logging/__pycache__/converter.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dfc6d82ededc60dc80c76927c4f1ca1faf8ef8c2 Binary files /dev/null and b/venv/Lib/site-packages/absl/logging/__pycache__/converter.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/logging/converter.py b/venv/Lib/site-packages/absl/logging/converter.py new file mode 100644 index 0000000000000000000000000000000000000000..ad3fcd5045edc665cc3118a9caa4f44fc1607be2 --- /dev/null +++ b/venv/Lib/site-packages/absl/logging/converter.py @@ -0,0 +1,214 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to convert log levels between Abseil Python, C++, and Python standard. + +This converter has to convert (best effort) between three different +logging level schemes: + + * **cpp**: The C++ logging level scheme used in Abseil C++. + * **absl**: The absl.logging level scheme used in Abseil Python. + * **standard**: The python standard library logging level scheme. + +Here is a handy ascii chart for easy mental mapping:: + + LEVEL | cpp | absl | standard | + ---------+-----+--------+----------+ + DEBUG | 0 | 1 | 10 | + INFO | 0 | 0 | 20 | + WARNING | 1 | -1 | 30 | + ERROR | 2 | -2 | 40 | + CRITICAL | 3 | -3 | 50 | + FATAL | 3 | -3 | 50 | + +Note: standard logging ``CRITICAL`` is mapped to absl/cpp ``FATAL``. +However, only ``CRITICAL`` logs from the absl logger (or absl.logging.fatal) +will terminate the program. ``CRITICAL`` logs from non-absl loggers are treated +as error logs with a message prefix ``"CRITICAL - "``. + +Converting from standard to absl or cpp is a lossy conversion. +Converting back to standard will lose granularity. For this reason, +users should always try to convert to standard, the richest +representation, before manipulating the levels, and then only to cpp +or absl if those level schemes are absolutely necessary. +""" + +import logging + +STANDARD_CRITICAL = logging.CRITICAL +STANDARD_ERROR = logging.ERROR +STANDARD_WARNING = logging.WARNING +STANDARD_INFO = logging.INFO +STANDARD_DEBUG = logging.DEBUG + +# These levels are also used to define the constants +# FATAL, ERROR, WARNING, INFO, and DEBUG in the +# absl.logging module. +ABSL_FATAL = -3 +ABSL_ERROR = -2 +ABSL_WARNING = -1 +ABSL_WARN = -1 # Deprecated name. +ABSL_INFO = 0 +ABSL_DEBUG = 1 + +ABSL_LEVELS = {ABSL_FATAL: 'FATAL', + ABSL_ERROR: 'ERROR', + ABSL_WARNING: 'WARNING', + ABSL_INFO: 'INFO', + ABSL_DEBUG: 'DEBUG'} + +# Inverts the ABSL_LEVELS dictionary +ABSL_NAMES = {'FATAL': ABSL_FATAL, + 'ERROR': ABSL_ERROR, + 'WARNING': ABSL_WARNING, + 'WARN': ABSL_WARNING, # Deprecated name. + 'INFO': ABSL_INFO, + 'DEBUG': ABSL_DEBUG} + +ABSL_TO_STANDARD = {ABSL_FATAL: STANDARD_CRITICAL, + ABSL_ERROR: STANDARD_ERROR, + ABSL_WARNING: STANDARD_WARNING, + ABSL_INFO: STANDARD_INFO, + ABSL_DEBUG: STANDARD_DEBUG} + +# Inverts the ABSL_TO_STANDARD +STANDARD_TO_ABSL = {v: k for (k, v) in ABSL_TO_STANDARD.items()} + + +def get_initial_for_level(level): + """Gets the initial that should start the log line for the given level. + + It returns: + + * ``'I'`` when: ``level < STANDARD_WARNING``. + * ``'W'`` when: ``STANDARD_WARNING <= level < STANDARD_ERROR``. + * ``'E'`` when: ``STANDARD_ERROR <= level < STANDARD_CRITICAL``. + * ``'F'`` when: ``level >= STANDARD_CRITICAL``. + + Args: + level: int, a Python standard logging level. + + Returns: + The first initial as it would be logged by the C++ logging module. + """ + if level < STANDARD_WARNING: + return 'I' + elif level < STANDARD_ERROR: + return 'W' + elif level < STANDARD_CRITICAL: + return 'E' + else: + return 'F' + + +def absl_to_cpp(level): + """Converts an absl log level to a cpp log level. + + Args: + level: int, an absl.logging level. + + Raises: + TypeError: Raised when level is not an integer. + + Returns: + The corresponding integer level for use in Abseil C++. + """ + if not isinstance(level, int): + raise TypeError(f'Expect an int level, found {type(level)}') + if level >= 0: + # C++ log levels must be >= 0 + return 0 + else: + return -level + + +def absl_to_standard(level): + """Converts an integer level from the absl value to the standard value. + + Args: + level: int, an absl.logging level. + + Raises: + TypeError: Raised when level is not an integer. + + Returns: + The corresponding integer level for use in standard logging. + """ + if not isinstance(level, int): + raise TypeError(f'Expect an int level, found {type(level)}') + if level < ABSL_FATAL: + level = ABSL_FATAL + if level <= ABSL_DEBUG: + return ABSL_TO_STANDARD[level] + # Maps to vlog levels. + return STANDARD_DEBUG - level + 1 + + +def string_to_standard(level): + """Converts a string level to standard logging level value. + + Args: + level: str, case-insensitive ``'debug'``, ``'info'``, ``'warning'``, + ``'error'``, ``'fatal'``. + + Returns: + The corresponding integer level for use in standard logging. + """ + return absl_to_standard(ABSL_NAMES.get(level.upper())) + + +def standard_to_absl(level): + """Converts an integer level from the standard value to the absl value. + + Args: + level: int, a Python standard logging level. + + Raises: + TypeError: Raised when level is not an integer. + + Returns: + The corresponding integer level for use in absl logging. + """ + if not isinstance(level, int): + raise TypeError(f'Expect an int level, found {type(level)}') + if level < 0: + level = 0 + if level < STANDARD_DEBUG: + # Maps to vlog levels. + return STANDARD_DEBUG - level + 1 + elif level < STANDARD_INFO: + return ABSL_DEBUG + elif level < STANDARD_WARNING: + return ABSL_INFO + elif level < STANDARD_ERROR: + return ABSL_WARNING + elif level < STANDARD_CRITICAL: + return ABSL_ERROR + else: + return ABSL_FATAL + + +def standard_to_cpp(level): + """Converts an integer level from the standard value to the cpp value. + + Args: + level: int, a Python standard logging level. + + Raises: + TypeError: Raised when level is not an integer. + + Returns: + The corresponding integer level for use in cpp logging. + """ + return absl_to_cpp(standard_to_absl(level)) diff --git a/venv/Lib/site-packages/absl/py.typed b/venv/Lib/site-packages/absl/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/Lib/site-packages/absl/testing/__init__.py b/venv/Lib/site-packages/absl/testing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a3bd1cd51810385ca0e5e9fed3fb9a804febf27e --- /dev/null +++ b/venv/Lib/site-packages/absl/testing/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/venv/Lib/site-packages/absl/testing/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/absl/testing/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..34b9c76939d659ad4ff466f8ce13e4d30f214eab Binary files /dev/null and b/venv/Lib/site-packages/absl/testing/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/testing/__pycache__/_bazelize_command.cpython-311.pyc b/venv/Lib/site-packages/absl/testing/__pycache__/_bazelize_command.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a7cff0bc3f9c7679df913c63ef46cbddf54862d9 Binary files /dev/null and b/venv/Lib/site-packages/absl/testing/__pycache__/_bazelize_command.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/testing/__pycache__/_pretty_print_reporter.cpython-311.pyc b/venv/Lib/site-packages/absl/testing/__pycache__/_pretty_print_reporter.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..75ac5eb18d5386c02b5fa9cef778b964e568561a Binary files /dev/null and b/venv/Lib/site-packages/absl/testing/__pycache__/_pretty_print_reporter.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/testing/__pycache__/absltest.cpython-311.pyc b/venv/Lib/site-packages/absl/testing/__pycache__/absltest.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de599e7bfe9a7784260ad52d45c86af81d31dda8 --- /dev/null +++ b/venv/Lib/site-packages/absl/testing/__pycache__/absltest.cpython-311.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d1c5fed7f42a6dfeb011b9c8e963690eb2862da7df63ef5434c1b884fe43242 +size 134812 diff --git a/venv/Lib/site-packages/absl/testing/__pycache__/flagsaver.cpython-311.pyc b/venv/Lib/site-packages/absl/testing/__pycache__/flagsaver.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6023e30e2cc581b80dbfa690d8bc1033147b0cc Binary files /dev/null and b/venv/Lib/site-packages/absl/testing/__pycache__/flagsaver.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/testing/__pycache__/parameterized.cpython-311.pyc b/venv/Lib/site-packages/absl/testing/__pycache__/parameterized.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38e241805ab66b2dd6c3bcc265d9ec0a9df58f91 Binary files /dev/null and b/venv/Lib/site-packages/absl/testing/__pycache__/parameterized.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/testing/__pycache__/xml_reporter.cpython-311.pyc b/venv/Lib/site-packages/absl/testing/__pycache__/xml_reporter.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50cea44fc0482a2ad4e7dd9a1c52478b83c23bfd Binary files /dev/null and b/venv/Lib/site-packages/absl/testing/__pycache__/xml_reporter.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/absl/testing/_bazelize_command.py b/venv/Lib/site-packages/absl/testing/_bazelize_command.py new file mode 100644 index 0000000000000000000000000000000000000000..acde3d2bd674a148ab3ec2a50984219c42b58a34 --- /dev/null +++ b/venv/Lib/site-packages/absl/testing/_bazelize_command.py @@ -0,0 +1,68 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Internal helper for running tests on Windows Bazel.""" + +import os + +from absl import flags + +FLAGS = flags.FLAGS + + +def get_executable_path(py_binary_name): + """Returns the executable path of a py_binary. + + This returns the executable path of a py_binary that is in another Bazel + target's data dependencies. + + On Linux/macOS, the path and __file__ has the same root directory. + On Windows, bazel builds an .exe file and we need to use the MANIFEST file + the location the actual binary. + + Args: + py_binary_name: string, the name of a py_binary that is in another Bazel + target's data dependencies. + + Raises: + RuntimeError: Raised when it cannot locate the executable path. + """ + + if os.name == 'nt': + py_binary_name += '.exe' + manifest_file = os.path.join(FLAGS.test_srcdir, 'MANIFEST') + workspace_name = os.environ['TEST_WORKSPACE'] + manifest_entry = f'{workspace_name}/{py_binary_name}' + with open(manifest_file) as manifest_fd: + for line in manifest_fd: + tokens = line.strip().split(' ') + if len(tokens) != 2: + continue + if manifest_entry == tokens[0]: + return tokens[1] + raise RuntimeError( + 'Cannot locate executable path for {}, MANIFEST file: {}.'.format( + py_binary_name, manifest_file)) + else: + # NOTE: __file__ may be .py or .pyc, depending on how the module was + # loaded and executed. + path = __file__ + + # Use the package name to find the root directory: every dot is + # a directory, plus one for ourselves. + for _ in range(__name__.count('.') + 1): + path = os.path.dirname(path) + + root_directory = path + return os.path.join(root_directory, py_binary_name) diff --git a/venv/Lib/site-packages/absl/testing/_pretty_print_reporter.py b/venv/Lib/site-packages/absl/testing/_pretty_print_reporter.py new file mode 100644 index 0000000000000000000000000000000000000000..af2bedbc6558b53c0d4bbd54a71859138cddb0c5 --- /dev/null +++ b/venv/Lib/site-packages/absl/testing/_pretty_print_reporter.py @@ -0,0 +1,92 @@ +# Copyright 2018 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TestResult implementing default output for test execution status.""" + +import unittest + + +class TextTestResult(unittest.TextTestResult): + """TestResult class that provides the default text result formatting.""" + + def __init__(self, stream, descriptions, verbosity): + # Disable the verbose per-test output from the superclass, since it would + # conflict with our customized output. + super().__init__(stream, descriptions, 0) + self._per_test_output = verbosity > 0 + + def _print_status(self, tag, test, reason=None): + if self._per_test_output: + test_id = test.id() + if test_id.startswith('__main__.'): + test_id = test_id[len('__main__.'):] + if reason: + print('[%s] %s - %s' % (tag, test_id, reason), file=self.stream) + else: + print('[%s] %s' % (tag, test_id), file=self.stream) + self.stream.flush() + + def startTest(self, test): + super().startTest(test) + self._print_status(' RUN ', test) + + def addSuccess(self, test): + super().addSuccess(test) + self._print_status(' OK ', test) + + def addError(self, test, err): + super().addError(test, err) + self._print_status(' FAILED ', test) + + def addFailure(self, test, err): + super().addFailure(test, err) + self._print_status(' FAILED ', test) + + def addSkip(self, test, reason): + super().addSkip(test, reason) + self._print_status(' SKIPPED ', test, reason) + + def addExpectedFailure(self, test, err): + super().addExpectedFailure(test, err) + self._print_status(' OK ', test) + + def addUnexpectedSuccess(self, test): + super().addUnexpectedSuccess(test) + self._print_status(' FAILED ', test) + + +class TextTestRunner(unittest.TextTestRunner): + """A test runner that produces formatted text results.""" + + _TEST_RESULT_CLASS = TextTestResult + + # Set this to true at the class or instance level to run tests using a + # debug-friendly method (e.g, one that doesn't catch exceptions and interacts + # better with debuggers). + # Usually this is set using --pdb_post_mortem. + run_for_debugging = False + + def run(self, test) -> unittest.TextTestResult: + if self.run_for_debugging: + return self._run_debug(test) + else: + return super().run(test) + + def _run_debug(self, test) -> unittest.TextTestResult: + test.debug() + # Return an empty result to indicate success. + return self._makeResult() + + def _makeResult(self): + return TextTestResult(self.stream, self.descriptions, self.verbosity) diff --git a/venv/Lib/site-packages/absl/testing/absltest.py b/venv/Lib/site-packages/absl/testing/absltest.py new file mode 100644 index 0000000000000000000000000000000000000000..a56da04233cb7d78cdb6f731ede9ca37311ff27d --- /dev/null +++ b/venv/Lib/site-packages/absl/testing/absltest.py @@ -0,0 +1,2865 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base functionality for Abseil Python tests. + +This module contains base classes and high-level functions for Abseil-style +tests. +""" + +from collections import abc +import contextlib +import dataclasses +import difflib +import enum +import errno +import faulthandler +import getpass +import inspect +import io +import itertools +import json +import numbers +import os +import random +import re +import shlex +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import textwrap +import typing +from typing import Any, AnyStr, BinaryIO, IO, NoReturn, TextIO +import unittest +from unittest import mock # pylint: disable=unused-import Allow absltest.mock. +import unittest.case +from urllib import parse + +from absl import app # pylint: disable=g-import-not-at-top +from absl import flags +from absl import logging +from absl.testing import _pretty_print_reporter +from absl.testing import xml_reporter + + +# Re-export a bunch of unittest functions we support so that people don't +# have to import unittest to get them +# pylint: disable=invalid-name +skip = unittest.skip +skipIf = unittest.skipIf +skipUnless = unittest.skipUnless +SkipTest = unittest.SkipTest +expectedFailure = unittest.expectedFailure +# pylint: enable=invalid-name + +# End unittest re-exports + +FLAGS = flags.FLAGS + +# Private typing symbols. +_T = typing.TypeVar('_T') # Unbounded TypeVar for general usage +_TEXT_OR_BINARY_TYPES = (str, bytes) + +# Suppress surplus entries in AssertionError stack traces. +__unittest = True # pylint: disable=invalid-name + + +def expectedFailureIf(condition, reason): # pylint: disable=invalid-name + """Expects the test to fail if the run condition is True. + + Example usage:: + + @expectedFailureIf(sys.version.major == 2, "Not yet working in py2") + def test_foo(self): + ... + + Args: + condition: bool, whether to expect failure or not. + reason: str, the reason to expect failure. + + Returns: + Decorator function + """ + del reason # Unused + if condition: + return unittest.expectedFailure + else: + return lambda f: f + + +class TempFileCleanup(enum.Enum): + # Always cleanup temp files when the test completes. + ALWAYS = 'always' + # Only cleanup temp file if the test passes. This allows easier inspection + # of tempfile contents on test failure. absltest.TEST_TMPDIR.value determines + # where tempfiles are created. + SUCCESS = 'success' + # Never cleanup temp files. + OFF = 'never' + + +# Many of the methods in this module have names like assertSameElements. +# This kind of name does not comply with PEP8 style, +# but it is consistent with the naming of methods in unittest.py. +# pylint: disable=invalid-name + + +def _get_default_test_random_seed() -> int: + random_seed = 301 + value = os.environ.get('TEST_RANDOM_SEED', '') + try: + random_seed = int(value) + except ValueError: + pass + return random_seed + + +def get_default_test_srcdir() -> str: + """Returns default test source dir.""" + return os.environ.get('TEST_SRCDIR', '') + + +def get_default_test_tmpdir() -> str: + """Returns default test temp dir.""" + tmpdir = os.environ.get('TEST_TMPDIR', '') + if not tmpdir: + tmpdir = os.path.join(tempfile.gettempdir(), 'absl_testing') + + return tmpdir + + +def _get_default_randomize_ordering_seed() -> int: + """Returns default seed to use for randomizing test order. + + This function first checks the --test_randomize_ordering_seed flag, and then + the TEST_RANDOMIZE_ORDERING_SEED environment variable. If the first value + we find is: + * (not set): disable test randomization + * 0: disable test randomization + * 'random': choose a random seed in [1, 4294967295] for test order + randomization + * positive integer: use this seed for test order randomization + + (The values used are patterned after + https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHASHSEED). + + In principle, it would be simpler to return None if no override is provided; + however, the python random module has no `get_seed()`, only `getstate()`, + which returns far more data than we want to pass via an environment variable + or flag. + + Returns: + A default value for test case randomization (int). 0 means do not randomize. + + Raises: + ValueError: Raised when the flag or env value is not one of the options + above. + """ + if FLAGS['test_randomize_ordering_seed'].present: + randomize = FLAGS.test_randomize_ordering_seed + elif 'TEST_RANDOMIZE_ORDERING_SEED' in os.environ: + randomize = os.environ['TEST_RANDOMIZE_ORDERING_SEED'] + else: + randomize = '' + if not randomize: + return 0 + if randomize == 'random': + return random.Random().randint(1, 4294967295) + if randomize == '0': + return 0 + try: + seed = int(randomize) + if seed > 0: + return seed + except ValueError: + pass + raise ValueError(f'Unknown test randomization seed value: {randomize}') + + +TEST_SRCDIR = flags.DEFINE_string( + 'test_srcdir', + get_default_test_srcdir(), + 'Root of directory tree where source files live', + allow_override_cpp=True) +TEST_TMPDIR = flags.DEFINE_string( + 'test_tmpdir', + get_default_test_tmpdir(), + 'Directory for temporary testing files', + allow_override_cpp=True) + +flags.DEFINE_integer( + 'test_random_seed', + _get_default_test_random_seed(), + 'Random seed for testing. Some test frameworks may ' + 'change the default value of this flag between runs, so ' + 'it is not appropriate for seeding probabilistic tests.', + allow_override_cpp=True) +flags.DEFINE_string( + 'test_randomize_ordering_seed', + '', + 'If positive, use this as a seed to randomize the ' + 'execution order for test cases. If "random", pick a ' + 'random seed to use. If 0 or not set, do not randomize ' + 'test case execution order. This flag also overrides ' + 'the TEST_RANDOMIZE_ORDERING_SEED environment variable.', + allow_override_cpp=True) +flags.DEFINE_string('xml_output_file', '', 'File to store XML test results') + + +def _open( + filepath: str, mode: str, _open_func: abc.Callable[..., IO[AnyStr]] = open +) -> IO[AnyStr]: + """Opens a file. + + Like open(), but ensure that we can open real files even if tests stub out + open(). + + Args: + filepath: A filepath. + mode: A mode. + _open_func: A built-in open() function. + + Returns: + The opened file object. + """ + return _open_func(filepath, mode, encoding='utf-8') + + +class _TempDir: + """Represents a temporary directory for tests. + + Creation of this class is internal. Using its public methods is OK. + + This class implements the `os.PathLike[str]` interface. This means it can + be directly passed to e.g. `os.path.join()`. + """ + + def __init__(self, path: str) -> None: + """Module-private: do not instantiate outside module.""" + self._path = path + + @property + def full_path(self) -> str: + """Returns the path, as a string, for the directory. + + TIP: Instead of e.g. `os.path.join(temp_dir.full_path, some_file_name)`, + you can simply do `os.path.join(temp_dir, some_file_name)` because + `__fspath__()` is implemented. + """ + return self._path + + def __fspath__(self) -> str: + """See os.PathLike.""" + return self.full_path + + def create_file( + self, + file_path: str | None = None, + content: AnyStr | None = None, + mode: str = 'w', + encoding: str = 'utf8', + errors: str = 'strict', + ) -> '_TempFile': + """Create a file in the directory. + + NOTE: If the file already exists, it will be made writable and overwritten. + + Args: + file_path: Optional file path for the temp file. If not given, a unique + file name will be generated and used. Slashes are allowed in the name; + any missing intermediate directories will be created. NOTE: This path + is the path that will be cleaned up, including any directories in the + path, e.g., 'foo/bar/baz.txt' will `rm -r foo` + content: Optional string or bytes to initially write to the file. If not + specified, then an empty file is created. + mode: Mode string to use when writing content. Only used if `content` is + non-empty. + encoding: Encoding to use when writing string content. Only used if + `content` is text. + errors: How to handle text to bytes encoding errors. Only used if + `content` is text. + + Returns: + A _TempFile representing the created file. + """ + tf, _ = _TempFile._create(self._path, file_path, content, mode, encoding, + errors) + return tf + + def mkdir(self, dir_path: str | None = None) -> '_TempDir': + """Create a directory in the directory. + + Args: + dir_path: Optional path to the directory to create. If not given, + a unique name will be generated and used. + + Returns: + A _TempDir representing the created directory. + """ + if dir_path: + path = os.path.join(self._path, dir_path) + else: + path = tempfile.mkdtemp(dir=self._path) + + # Note: there's no need to clear the directory since the containing + # dir was cleared by the tempdir() function. + os.makedirs(path, exist_ok=True) + return _TempDir(path) + + +class _TempFile: + """Represents a tempfile for tests. + + Creation of this class is internal. Using its public methods is OK. + + This class implements the `os.PathLike[str]` interface. This means it can + be directly passed to e.g. `os.path.join()`. + """ + + def __init__(self, path: str) -> None: + """Private: use _create instead.""" + self._path = path + + @classmethod + def _create( + cls, + base_path: str, + file_path: str | None, + content: AnyStr | None, + mode: str, + encoding: str, + errors: str, + ) -> tuple['_TempFile', str]: + """Module-private: create a tempfile instance.""" + if file_path: + cleanup_path = os.path.join(base_path, _get_first_part(file_path)) + path = os.path.join(base_path, file_path) + os.makedirs(os.path.dirname(path), exist_ok=True) + # The file may already exist, in which case, ensure it's writable so that + # it can be truncated. + if os.path.exists(path) and not os.access(path, os.W_OK): + stat_info = os.stat(path) + os.chmod(path, stat_info.st_mode | stat.S_IWUSR) + else: + os.makedirs(base_path, exist_ok=True) + fd, path = tempfile.mkstemp(dir=str(base_path)) + os.close(fd) + cleanup_path = path + + tf = cls(path) + + if content: + if isinstance(content, str): + tf.write_text(content, mode=mode, encoding=encoding, errors=errors) + else: + tf.write_bytes(content, mode) + + else: + tf.write_bytes(b'') + + return tf, cleanup_path + + @property + def full_path(self) -> str: + """Returns the path, as a string, for the file. + + TIP: Instead of e.g. `os.path.join(temp_file.full_path, some_file_name)`, + you can simply do `os.path.join(temp_file, some_file_name)` because + `__fspath__()` is implemented. + """ + return self._path + + def __fspath__(self) -> str: + """See os.PathLike.""" + return self.full_path + + def read_text(self, encoding: str = 'utf8', errors: str = 'strict') -> str: + """Return the contents of the file as text.""" + with self.open_text(encoding=encoding, errors=errors) as fp: + return fp.read() + + def read_bytes(self) -> bytes: + """Return the content of the file as bytes.""" + with self.open_bytes() as fp: + return fp.read() + + def write_text( + self, + text: str, + mode: str = 'w', + encoding: str = 'utf8', + errors: str = 'strict', + ) -> None: + """Write text to the file. + + Args: + text: Text to write. + mode: The mode to open the file for writing. + encoding: The encoding to use when writing the text to the file. + errors: The error handling strategy to use when converting text to bytes. + """ + with self.open_text(mode, encoding=encoding, errors=errors) as fp: + fp.write(text) + + def write_bytes(self, data: bytes, mode: str = 'wb') -> None: + """Write bytes to the file. + + Args: + data: bytes to write. + mode: Mode to open the file for writing. The "b" flag is implicit if + not already present. It must not have the "t" flag. + """ + with self.open_bytes(mode) as fp: + fp.write(data) + + def open_text( + self, mode: str = 'rt', encoding: str = 'utf8', errors: str = 'strict' + ) -> contextlib.AbstractContextManager[TextIO]: + """Return a context manager for opening the file in text mode. + + Args: + mode: The mode to open the file in. The "t" flag is implicit if not + already present. It must not have the "b" flag. + encoding: The encoding to use when opening the file. + errors: How to handle decoding errors. + + Returns: + Context manager that yields an open file. + + Raises: + ValueError: if invalid inputs are provided. + """ + if 'b' in mode: + raise ValueError('Invalid mode {!r}: "b" flag not allowed when opening ' + 'file in text mode'.format(mode)) + if 't' not in mode: + mode += 't' + cm = self._open(mode, encoding, errors) + return cm + + def open_bytes( + self, mode: str = 'rb' + ) -> contextlib.AbstractContextManager[BinaryIO]: + """Return a context manager for opening the file in binary mode. + + Args: + mode: The mode to open the file in. The "b" mode is implicit if not + already present. It must not have the "t" flag. + + Returns: + Context manager that yields an open file. + + Raises: + ValueError: if invalid inputs are provided. + """ + if 't' in mode: + raise ValueError('Invalid mode {!r}: "t" flag not allowed when opening ' + 'file in binary mode'.format(mode)) + if 'b' not in mode: + mode += 'b' + cm = self._open(mode, encoding=None, errors=None) + return cm + + # TODO(b/123775699): Once pytype supports typing.Literal, use overload and + # Literal to express more precise return types. The contained type is + # currently `Any` to avoid [bad-return-type] errors in the open_* methods. + @contextlib.contextmanager + def _open( + self, + mode: str, + encoding: str | None = 'utf8', + errors: str | None = 'strict', + ) -> abc.Iterator[Any]: + with open( + self.full_path, mode=mode, encoding=encoding, errors=errors + ) as fp: + yield fp + + +class _method: + """A decorator that supports both instance and classmethod invocations. + + Using similar semantics to the @property builtin, this decorator can augment + an instance method to support conditional logic when invoked on a class + object. This breaks support for invoking an instance method via the class + (e.g. Cls.method(self, ...)) but is still situationally useful. + """ + + _finstancemethod: Any + _fclassmethod: Any | None + + def __init__(self, finstancemethod: abc.Callable[..., Any]) -> None: + self._finstancemethod = finstancemethod + self._fclassmethod = None + + def classmethod(self, fclassmethod: abc.Callable[..., Any]) -> '_method': + if isinstance(fclassmethod, classmethod): + self._fclassmethod = fclassmethod + else: + self._fclassmethod = classmethod(fclassmethod) + return self + + def __doc__(self) -> str: # type: ignore[override] + return ( + getattr(self._finstancemethod, '__doc__') + or getattr(self._fclassmethod, '__doc__') + or '' + ) + + def __get__( + self, obj: Any | None, type_: type[Any] | None + ) -> abc.Callable[..., Any]: + func = self._fclassmethod if obj is None else self._finstancemethod + return func.__get__(obj, type_) # type: ignore[attribute-error, union-attr] + + +class TestCase(unittest.TestCase): + """Extension of unittest.TestCase providing more power.""" + + # When to cleanup files/directories created by our `create_tempfile()` and + # `create_tempdir()` methods after each test case completes. This does *not* + # affect e.g., files created outside of those methods, e.g., using the stdlib + # tempfile module. This can be overridden at the class level, instance level, + # or with the `cleanup` arg of `create_tempfile()` and `create_tempdir()`. See + # `TempFileCleanup` for details on the different values. + tempfile_cleanup: TempFileCleanup = TempFileCleanup.ALWAYS + + maxDiff = 80 * 20 + longMessage = True + + # Exit stacks for per-test and per-class scopes. + if sys.version_info < (3, 11): + _exit_stack = None + _cls_exit_stack = None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # This is to work around missing type stubs in unittest.pyi + self._outcome: Any | None = getattr(self, '_outcome') + + def setUp(self): + super().setUp() + # NOTE: Only Python 3 contextlib has ExitStack and + # Python 3.11+ already has enterContext. + if hasattr(contextlib, 'ExitStack') and sys.version_info < (3, 11): + self._exit_stack = contextlib.ExitStack() + self.addCleanup(self._exit_stack.close) + + @classmethod + def setUpClass(cls): + super().setUpClass() + # NOTE: Only Python 3 contextlib has ExitStack, only Python 3.8+ has + # addClassCleanup and Python 3.11+ already has enterClassContext. + if ( + hasattr(contextlib, 'ExitStack') + and hasattr(cls, 'addClassCleanup') + and sys.version_info < (3, 11) + ): + cls._cls_exit_stack = contextlib.ExitStack() + cls.addClassCleanup(cls._cls_exit_stack.close) + + def create_tempdir( + self, + name: str | None = None, + cleanup: TempFileCleanup | None = None, + ) -> _TempDir: + """Create a temporary directory specific to the test. + + NOTE: The directory and its contents will be recursively cleared before + creation. This ensures that there is no pre-existing state. + + This creates a named directory on disk that is isolated to this test, and + will be properly cleaned up by the test. This avoids several pitfalls of + creating temporary directories for test purposes, as well as makes it easier + to setup directories and verify their contents. For example:: + + def test_foo(self): + out_dir = self.create_tempdir() + out_log = out_dir.create_file('output.log') + expected_outputs = [ + os.path.join(out_dir, 'data-0.txt'), + os.path.join(out_dir, 'data-1.txt'), + ] + code_under_test(out_dir) + self.assertTrue(os.path.exists(expected_paths[0])) + self.assertTrue(os.path.exists(expected_paths[1])) + self.assertEqual('foo', out_log.read_text()) + + See also: :meth:`create_tempfile` for creating temporary files. + + Args: + name: Optional name of the directory. If not given, a unique + name will be generated and used. + cleanup: Optional cleanup policy on when/if to remove the directory (and + all its contents) at the end of the test. If None, then uses + :attr:`tempfile_cleanup`. + + Returns: + A _TempDir representing the created directory; see _TempDir class docs + for usage. + """ + test_path = self._get_tempdir_path_test() + + if name: + path = os.path.join(test_path, name) + cleanup_path = os.path.join(test_path, _get_first_part(name)) + else: + os.makedirs(test_path, exist_ok=True) + path = tempfile.mkdtemp(dir=test_path) + cleanup_path = path + + _rmtree_ignore_errors(cleanup_path) + os.makedirs(path, exist_ok=True) + + self._maybe_add_temp_path_cleanup(cleanup_path, cleanup) + + return _TempDir(path) + + def create_tempfile( + self, + file_path: str | None = None, + content: AnyStr | None = None, + mode: str = 'w', + encoding: str = 'utf8', + errors: str = 'strict', + cleanup: TempFileCleanup | None = None, + ) -> _TempFile: + """Create a temporary file specific to the test. + + This creates a named file on disk that is isolated to this test, and will + be properly cleaned up by the test. This avoids several pitfalls of + creating temporary files for test purposes, as well as makes it easier + to setup files, their data, read them back, and inspect them when + a test fails. For example:: + + def test_foo(self): + output = self.create_tempfile() + code_under_test(output) + self.assertGreater(os.path.getsize(output), 0) + self.assertEqual('foo', output.read_text()) + + NOTE: This will zero-out the file. This ensures there is no pre-existing + state. + NOTE: If the file already exists, it will be made writable and overwritten. + + See also: :meth:`create_tempdir` for creating temporary directories, and + ``_TempDir.create_file`` for creating files within a temporary directory. + + Args: + file_path: Optional file path for the temp file. If not given, a unique + file name will be generated and used. Slashes are allowed in the name; + any missing intermediate directories will be created. NOTE: This path is + the path that will be cleaned up, including any directories in the path, + e.g., ``'foo/bar/baz.txt'`` will ``rm -r foo``. + content: Optional string or + bytes to initially write to the file. If not + specified, then an empty file is created. + mode: Mode string to use when writing content. Only used if `content` is + non-empty. + encoding: Encoding to use when writing string content. Only used if + `content` is text. + errors: How to handle text to bytes encoding errors. Only used if + `content` is text. + cleanup: Optional cleanup policy on when/if to remove the directory (and + all its contents) at the end of the test. If None, then uses + :attr:`tempfile_cleanup`. + + Returns: + A _TempFile representing the created file; see _TempFile class docs for + usage. + """ + test_path = self._get_tempdir_path_test() + tf, cleanup_path = _TempFile._create(test_path, file_path, content=content, + mode=mode, encoding=encoding, + errors=errors) + self._maybe_add_temp_path_cleanup(cleanup_path, cleanup) + return tf + + @_method + def enter_context(self, manager: contextlib.AbstractContextManager[_T]) -> _T: + """Returns the CM's value after registering it with the exit stack. + + Entering a context pushes it onto a stack of contexts. When `enter_context` + is called on the test instance (e.g. `self.enter_context`), the context is + exited after the test case's tearDown call. When called on the test class + (e.g. `TestCase.enter_context`), the context is exited after the test + class's tearDownClass call. + + Contexts are exited in the reverse order of entering. They will always + be exited, regardless of test failure/success. + + This is useful to eliminate per-test boilerplate when context managers + are used. For example, instead of decorating every test with `@mock.patch`, + simply do `self.foo = self.enter_context(mock.patch(...))' in `setUp()`. + + NOTE: The context managers will always be exited without any error + information. This is an unfortunate implementation detail due to some + internals of how unittest runs tests. + + Args: + manager: The context manager to enter. + """ + if sys.version_info >= (3, 11): + return self.enterContext(manager) + + if not self._exit_stack: + raise AssertionError( + 'self._exit_stack is not set: enter_context is Py3-only; also make ' + 'sure that AbslTest.setUp() is called.') + return self._exit_stack.enter_context(manager) + + @enter_context.classmethod + @classmethod + def _enter_context_cls( + cls, manager: contextlib.AbstractContextManager[_T] + ) -> _T: + if sys.version_info >= (3, 11): + return cls.enterClassContext(manager) + + if not cls._cls_exit_stack: + raise AssertionError( + 'cls._cls_exit_stack is not set: cls.enter_context requires ' + 'Python 3.8+; also make sure that AbslTest.setUpClass() is called.') + return cls._cls_exit_stack.enter_context(manager) + + @classmethod + def _get_tempdir_path_cls(cls) -> str: + return os.path.join(TEST_TMPDIR.value, + cls.__qualname__.replace('__main__.', '')) + + def _get_tempdir_path_test(self) -> str: + return os.path.join(self._get_tempdir_path_cls(), self._testMethodName) + + def _get_tempfile_cleanup( + self, override: TempFileCleanup | None + ) -> TempFileCleanup: + if override is not None: + return override + return self.tempfile_cleanup + + def _maybe_add_temp_path_cleanup( + self, path: str, cleanup: TempFileCleanup | None + ) -> None: + cleanup = self._get_tempfile_cleanup(cleanup) + if cleanup == TempFileCleanup.OFF: + return + elif cleanup == TempFileCleanup.ALWAYS: + self.addCleanup(_rmtree_ignore_errors, path) + elif cleanup == TempFileCleanup.SUCCESS: + self._internal_add_cleanup_on_success(_rmtree_ignore_errors, path) + else: + raise AssertionError(f'Unexpected cleanup value: {cleanup}') + + def _internal_add_cleanup_on_success( + self, + function: abc.Callable[..., Any], + *args: Any, + **kwargs: Any, + ) -> None: + """Adds `function` as cleanup when the test case succeeds.""" + outcome = self._outcome + assert outcome is not None + previous_failure_count = ( + len(outcome.result.failures) + + len(outcome.result.errors) + + len(outcome.result.unexpectedSuccesses) + ) + def _call_cleaner_on_success(*args, **kwargs): + if not self._internal_ran_and_passed_when_called_during_cleanup( + previous_failure_count): + return + function(*args, **kwargs) + self.addCleanup(_call_cleaner_on_success, *args, **kwargs) + + def _internal_ran_and_passed_when_called_during_cleanup( + self, + previous_failure_count: int, + ) -> bool: + """Returns whether test is passed. Expected to be called during cleanup.""" + outcome = self._outcome + if sys.version_info[:2] >= (3, 11): + assert outcome is not None + current_failure_count = ( + len(outcome.result.failures) + + len(outcome.result.errors) + + len(outcome.result.unexpectedSuccesses) + ) + return current_failure_count == previous_failure_count + else: + # Before Python 3.11 https://github.com/python/cpython/pull/28180, errors + # were bufferred in _Outcome before calling cleanup. + result = self.defaultTestResult() + self._feedErrorsToResult(result, outcome.errors) # pytype: disable=attribute-error + return result.wasSuccessful() + + def shortDescription(self) -> str: + """Formats both the test method name and the first line of its docstring. + + If no docstring is given, only returns the method name. + + This method overrides unittest.TestCase.shortDescription(), which + only returns the first line of the docstring, obscuring the name + of the test upon failure. + + Returns: + desc: A short description of a test method. + """ + desc = self.id() + + # Omit the main name so that test name can be directly copy/pasted to + # the command line. + if desc.startswith('__main__.'): + desc = desc[len('__main__.'):] + + # NOTE: super() is used here instead of directly invoking + # unittest.TestCase.shortDescription(self), because of the + # following line that occurs later on: + # unittest.TestCase = TestCase + # Because of this, direct invocation of what we think is the + # superclass will actually cause infinite recursion. + doc_first_line = super().shortDescription() + if doc_first_line is not None: + desc = '\n'.join((desc, doc_first_line)) + return desc + + def assertStartsWith(self, actual, expected_start, msg=None): + """Asserts that actual.startswith(expected_start) is True. + + Args: + actual: str + expected_start: str + msg: Optional message to report on failure. + """ + if not actual.startswith(expected_start): + self.fail('%r does not start with %r' % (actual, expected_start), msg) + + def assertNotStartsWith(self, actual, unexpected_start, msg=None): + """Asserts that actual.startswith(unexpected_start) is False. + + Args: + actual: str + unexpected_start: str + msg: Optional message to report on failure. + """ + if actual.startswith(unexpected_start): + self.fail('%r does start with %r' % (actual, unexpected_start), msg) + + def assertEndsWith(self, actual, expected_end, msg=None): + """Asserts that actual.endswith(expected_end) is True. + + Args: + actual: str + expected_end: str + msg: Optional message to report on failure. + """ + if not actual.endswith(expected_end): + self.fail('%r does not end with %r' % (actual, expected_end), msg) + + def assertNotEndsWith(self, actual, unexpected_end, msg=None): + """Asserts that actual.endswith(unexpected_end) is False. + + Args: + actual: str + unexpected_end: str + msg: Optional message to report on failure. + """ + if actual.endswith(unexpected_end): + self.fail('%r does end with %r' % (actual, unexpected_end), msg) + + def assertSequenceStartsWith(self, prefix, whole, msg=None): + """An equality assertion for the beginning of ordered sequences. + + If prefix is an empty sequence, it will raise an error unless whole is also + an empty sequence. + + If prefix is not a sequence, it will raise an error if the first element of + whole does not match. + + Args: + prefix: A sequence expected at the beginning of the whole parameter. + whole: The sequence in which to look for prefix. + msg: Optional message to report on failure. + """ + try: + prefix_len = len(prefix) + except (TypeError, NotImplementedError): + prefix = [prefix] + prefix_len = 1 + + if isinstance(whole, abc.Mapping) or isinstance(whole, abc.Set): + self.fail( + 'For whole: Mapping or Set objects are not supported, found type: %s' + % type(whole), + msg, + ) + try: + whole_len = len(whole) + except (TypeError, NotImplementedError): + self.fail('For whole: len(%s) is not supported, it appears to be type: ' + '%s' % (whole, type(whole)), msg) + + assert prefix_len <= whole_len, self._formatMessage( + msg, + 'Prefix length (%d) is longer than whole length (%d).' % + (prefix_len, whole_len) + ) + + if not prefix_len and whole_len: + self.fail('Prefix length is 0 but whole length is %d: %s' % + (len(whole), whole), msg) + + try: + self.assertSequenceEqual(prefix, whole[:prefix_len], msg) + except AssertionError: + self.fail('prefix: %s not found at start of whole: %s.' % + (prefix, whole), msg) + + def assertEmpty(self, container, msg=None): + """Asserts that an object has zero length. + + Args: + container: Anything that implements the collections.abc.Sized interface. + msg: Optional message to report on failure. + """ + if not isinstance(container, abc.Sized): + self.fail('Expected a Sized object, got: ' + '{!r}'.format(type(container).__name__), msg) + + # explicitly check the length since some Sized objects (e.g. numpy.ndarray) + # have strange __nonzero__/__bool__ behavior. + if len(container): # pylint: disable=g-explicit-length-test + self.fail(f'{container!r} has length of {len(container)}.', msg) + + def assertNotEmpty(self, container, msg=None): + """Asserts that an object has non-zero length. + + Args: + container: Anything that implements the collections.abc.Sized interface. + msg: Optional message to report on failure. + """ + if not isinstance(container, abc.Sized): + self.fail('Expected a Sized object, got: ' + '{!r}'.format(type(container).__name__), msg) + + # explicitly check the length since some Sized objects (e.g. numpy.ndarray) + # have strange __nonzero__/__bool__ behavior. + if not len(container): # pylint: disable=g-explicit-length-test + self.fail(f'{container!r} has length of 0.', msg) + + def assertLen(self, container, expected_len, msg=None): + """Asserts that an object has the expected length. + + Args: + container: Anything that implements the collections.abc.Sized interface. + expected_len: The expected length of the container. + msg: Optional message to report on failure. + """ + if not isinstance(container, abc.Sized): + self.fail('Expected a Sized object, got: ' + '{!r}'.format(type(container).__name__), msg) + if len(container) != expected_len: + container_repr = unittest.util.safe_repr(container) # pytype: disable=module-attr + self.fail('{} has length of {}, expected {}.'.format( + container_repr, len(container), expected_len), msg) + + def assertSequenceAlmostEqual(self, expected_seq, actual_seq, places=None, + msg=None, delta=None): + """An approximate equality assertion for ordered sequences. + + Fail if the two sequences are unequal as determined by their value + differences rounded to the given number of decimal places (default 7) and + comparing to zero, or by comparing that the difference between each value + in the two sequences is more than the given delta. + + Note that decimal places (from zero) are usually not the same as significant + digits (measured from the most significant digit). + + If the two sequences compare equal then they will automatically compare + almost equal. + + Args: + expected_seq: A sequence containing elements we are expecting. + actual_seq: The sequence that we are testing. + places: The number of decimal places to compare. + msg: The message to be printed if the test fails. + delta: The OK difference between compared values. + """ + if len(expected_seq) != len(actual_seq): + self.fail('Sequence size mismatch: {} vs {}'.format( + len(expected_seq), len(actual_seq)), msg) + + err_list = [] + for idx, (exp_elem, act_elem) in enumerate(zip(expected_seq, actual_seq)): + try: + # assertAlmostEqual should be called with at most one of `places` and + # `delta`. However, it's okay for assertSequenceAlmostEqual to pass + # both because we want the latter to fail if the former does. + # pytype: disable=wrong-keyword-args + self.assertAlmostEqual(exp_elem, act_elem, places=places, msg=msg, + delta=delta) + # pytype: enable=wrong-keyword-args + except self.failureException as err: + err_list.append(f'At index {idx}: {err}') + + if err_list: + if len(err_list) > 30: + err_list = err_list[:30] + ['...'] + msg = self._formatMessage(msg, '\n'.join(err_list)) + self.fail(msg) + + def assertContainsSubset(self, expected_subset, actual_set, msg=None): + """Checks whether actual iterable is a superset of expected iterable.""" + missing = set(expected_subset) - set(actual_set) + if not missing: + return + + self.fail('Missing elements %s\nExpected: %s\nActual: %s' % ( + missing, expected_subset, actual_set), msg) + + def assertNoCommonElements(self, expected_seq, actual_seq, msg=None): + """Checks whether actual iterable and expected iterable are disjoint.""" + common = set(expected_seq) & set(actual_seq) + if not common: + return + + self.fail('Common elements %s\nExpected: %s\nActual: %s' % ( + common, expected_seq, actual_seq), msg) + + def assertItemsEqual(self, expected_seq, actual_seq, msg=None): + """Deprecated, please use assertCountEqual instead. + + This is equivalent to assertCountEqual. + + Args: + expected_seq: A sequence containing elements we are expecting. + actual_seq: The sequence that we are testing. + msg: The message to be printed if the test fails. + """ + super().assertCountEqual(expected_seq, actual_seq, msg) + + def assertSameElements(self, expected_seq, actual_seq, msg=None): + """Asserts that two sequences have the same elements (in any order). + + This method, unlike assertCountEqual, doesn't care about any + duplicates in the expected and actual sequences:: + + # Doesn't raise an AssertionError + assertSameElements([1, 1, 1, 0, 0, 0], [0, 1]) + + If possible, you should use assertCountEqual instead of + assertSameElements. + + Args: + expected_seq: A sequence containing elements we are expecting. + actual_seq: The sequence that we are testing. + msg: The message to be printed if the test fails. + """ + # `unittest2.TestCase` used to have assertSameElements, but it was + # removed in favor of assertItemsEqual. As there's a unit test + # that explicitly checks this behavior, I am leaving this method + # alone. + # Fail on strings: empirically, passing strings to this test method + # is almost always a bug. If comparing the character sets of two strings + # is desired, cast the inputs to sets or lists explicitly. + if (isinstance(expected_seq, _TEXT_OR_BINARY_TYPES) or + isinstance(actual_seq, _TEXT_OR_BINARY_TYPES)): + self.fail('Passing string/bytes to assertSameElements is usually a bug. ' + 'Did you mean to use assertEqual?\n' + 'Expected: %s\nActual: %s' % (expected_seq, actual_seq)) + try: + expected = {element: None for element in expected_seq} + actual = {element: None for element in actual_seq} + missing = [element for element in expected if element not in actual] + unexpected = [element for element in actual if element not in expected] + missing.sort() + unexpected.sort() + except TypeError: + # Fall back to slower list-compare if any of the objects are + # not hashable. + expected = list(expected_seq) + actual = list(actual_seq) + expected.sort() + actual.sort() + missing, unexpected = _sorted_list_difference(expected, actual) + errors = [] + if msg: + errors.extend((msg, ':\n')) + if missing: + errors.append('Expected, but missing:\n %r\n' % missing) + if unexpected: + errors.append('Unexpected, but present:\n %r\n' % unexpected) + if missing or unexpected: + self.fail(''.join(errors)) + + # unittest.TestCase.assertMultiLineEqual works very similarly, but it + # has a different error format. However, I find this slightly more readable. + def assertMultiLineEqual(self, first, second, msg=None, **kwargs): + """Asserts that two multi-line strings are equal.""" + assert isinstance(first, + str), ('First argument is not a string: %r' % (first,)) + assert isinstance(second, + str), ('Second argument is not a string: %r' % (second,)) + line_limit = kwargs.pop('line_limit', 0) + if kwargs: + raise TypeError(f'Unexpected keyword args {tuple(kwargs)}') + + if first == second: + return + if msg: + failure_message = [msg + ':\n'] + else: + failure_message = ['\n'] + if line_limit: + line_limit += len(failure_message) + for line in difflib.ndiff(first.splitlines(True), second.splitlines(True)): + failure_message.append(line) + if not line.endswith('\n'): + failure_message.append('\n') + if line_limit and len(failure_message) > line_limit: + n_omitted = len(failure_message) - line_limit + failure_message = failure_message[:line_limit] + failure_message.append( + '(... and {} more delta lines omitted for brevity.)\n'.format( + n_omitted)) + + raise self.failureException(''.join(failure_message)) + + def assertBetween(self, value, minv, maxv, msg=None): + """Asserts that value is between minv and maxv (inclusive).""" + msg = self._formatMessage(msg, + '"%r" unexpectedly not between "%r" and "%r"' % + (value, minv, maxv)) + self.assertTrue(minv <= value, msg) + self.assertTrue(maxv >= value, msg) + + def assertRegexMatch(self, actual_str, regexes, message=None): + r"""Asserts that at least one regex in regexes matches str. + + If possible you should use `assertRegex`, which is a simpler + version of this method. `assertRegex` takes a single regular + expression (a string or re compiled object) instead of a list. + + Notes: + + 1. This function uses substring matching, i.e. the matching + succeeds if *any* substring of the error message matches *any* + regex in the list. This is more convenient for the user than + full-string matching. + + 2. If regexes is the empty list, the matching will always fail. + + 3. Use regexes=[''] for a regex that will always pass. + + 4. '.' matches any single character *except* the newline. To + match any character, use '(.|\n)'. + + 5. '^' matches the beginning of each line, not just the beginning + of the string. Similarly, '$' matches the end of each line. + + 6. An exception will be thrown if regexes contains an invalid + regex. + + Args: + actual_str: The string we try to match with the items in regexes. + regexes: The regular expressions we want to match against str. + See "Notes" above for detailed notes on how this is interpreted. + message: The message to be printed if the test fails. + """ + if isinstance(regexes, _TEXT_OR_BINARY_TYPES): + self.fail('regexes is string or bytes; use assertRegex instead.', + message) + if not regexes: + self.fail('No regexes specified.', message) + + regex_type = type(regexes[0]) + for regex in regexes[1:]: + if type(regex) is not regex_type: # pylint: disable=unidiomatic-typecheck + self.fail('regexes list must all be the same type.', message) + + if regex_type is bytes and isinstance(actual_str, str): + regexes = [regex.decode('utf-8') for regex in regexes] + regex_type = str + elif regex_type is str and isinstance(actual_str, bytes): + regexes = [regex.encode('utf-8') for regex in regexes] + regex_type = bytes + + if regex_type is str: + regex = '(?:%s)' % ')|(?:'.join(regexes) + elif regex_type is bytes: + regex = b'(?:' + (b')|(?:'.join(regexes)) + b')' + else: + self.fail('Only know how to deal with unicode str or bytes regexes.', + message) + + if not re.search(regex, actual_str, re.MULTILINE): + self.fail('"%s" does not contain any of these regexes: %s.' % + (actual_str, regexes), message) + + def assertCommandSucceeds(self, command, regexes=(b'',), env=None, + close_fds=True, msg=None): + """Asserts that a shell command succeeds (i.e. exits with code 0). + + Args: + command: List or string representing the command to run. + regexes: List of regular expression byte strings that match success. + env: Dictionary of environment variable settings. If None, no environment + variables will be set for the child process. This is to make tests + more hermetic. NOTE: this behavior is different than the standard + subprocess module. + close_fds: Whether or not to close all open fd's in the child after + forking. + msg: Optional message to report on failure. + """ + (ret_code, err) = get_command_stderr(command, env, close_fds) + + # We need bytes regexes here because `err` is bytes. + # Accommodate code which listed their output regexes w/o the b'' prefix by + # converting them to bytes for the user. + if isinstance(regexes[0], str): + regexes = [regex.encode('utf-8') for regex in regexes] + + command_string = get_command_string(command) + self.assertEqual( + ret_code, 0, + self._formatMessage(msg, + 'Running command\n' + '%s failed with error code %s and message\n' + '%s' % (_quote_long_string(command_string), + ret_code, + _quote_long_string(err))) + ) + self.assertRegexMatch( + err, + regexes, + message=self._formatMessage( + msg, + 'Running command\n' + '%s failed with error code %s and message\n' + '%s which matches no regex in %s' % ( + _quote_long_string(command_string), + ret_code, + _quote_long_string(err), + regexes))) + + def assertCommandFails(self, command, regexes, env=None, close_fds=True, + msg=None): + """Asserts a shell command fails and the error matches a regex in a list. + + Args: + command: List or string representing the command to run. + regexes: the list of regular expression strings. + env: Dictionary of environment variable settings. If None, no environment + variables will be set for the child process. This is to make tests + more hermetic. NOTE: this behavior is different than the standard + subprocess module. + close_fds: Whether or not to close all open fd's in the child after + forking. + msg: Optional message to report on failure. + """ + (ret_code, err) = get_command_stderr(command, env, close_fds) + + # We need bytes regexes here because `err` is bytes. + # Accommodate code which listed their output regexes w/o the b'' prefix by + # converting them to bytes for the user. + if isinstance(regexes[0], str): + regexes = [regex.encode('utf-8') for regex in regexes] + + command_string = get_command_string(command) + self.assertNotEqual( + ret_code, 0, + self._formatMessage(msg, 'The following command succeeded ' + 'while expected to fail:\n%s' % + _quote_long_string(command_string))) + self.assertRegexMatch( + err, + regexes, + message=self._formatMessage( + msg, + 'Running command\n' + '%s failed with error code %s and message\n' + '%s which matches no regex in %s' % ( + _quote_long_string(command_string), + ret_code, + _quote_long_string(err), + regexes))) + + class _AssertRaisesContext: + + def __init__(self, expected_exception, test_case, test_func, msg=None): + self.expected_exception = expected_exception + self.test_case = test_case + self.test_func = test_func + self.msg = msg + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, tb): + if exc_type is None: + self.test_case.fail(self.expected_exception.__name__ + ' not raised', + self.msg) + if not issubclass(exc_type, self.expected_exception): + return False + self.test_func(exc_value) + if exc_value: + self.exception = exc_value.with_traceback(None) + return True + + @typing.overload + def assertRaisesWithPredicateMatch( + self, expected_exception, predicate) -> _AssertRaisesContext: + # The purpose of this return statement is to work around + # https://github.com/PyCQA/pylint/issues/5273; it is otherwise ignored. + return self._AssertRaisesContext(None, None, None) + + @typing.overload + def assertRaisesWithPredicateMatch( + self, + expected_exception, + predicate, + callable_obj: abc.Callable[..., Any], + *args, + **kwargs, + ) -> None: + # The purpose of this return statement is to work around + # https://github.com/PyCQA/pylint/issues/5273; it is otherwise ignored. + return self._AssertRaisesContext(None, None, None) # type: ignore[return-value] + + def assertRaisesWithPredicateMatch(self, expected_exception, predicate, + callable_obj=None, *args, **kwargs): + """Asserts that exception is thrown and predicate(exception) is true. + + Args: + expected_exception: Exception class expected to be raised. + predicate: Function of one argument that inspects the passed-in exception + and returns True (success) or False (please fail the test). + callable_obj: Function to be called. + *args: Extra args. + **kwargs: Extra keyword args. + + Returns: + A context manager if callable_obj is None. Otherwise, None. + + Raises: + self.failureException if callable_obj does not raise a matching exception. + """ + def Check(err): + self.assertTrue(predicate(err), + '%r does not match predicate %r' % (err, predicate)) + + context = self._AssertRaisesContext(expected_exception, self, Check) + if callable_obj is None: + return context + with context: + callable_obj(*args, **kwargs) + + @typing.overload + def assertRaisesWithLiteralMatch( + self, expected_exception, expected_exception_message + ) -> _AssertRaisesContext: + # The purpose of this return statement is to work around + # https://github.com/PyCQA/pylint/issues/5273; it is otherwise ignored. + return self._AssertRaisesContext(None, None, None) + + @typing.overload + def assertRaisesWithLiteralMatch( + self, + expected_exception, + expected_exception_message, + callable_obj: abc.Callable[..., Any], + *args, + **kwargs, + ) -> None: + # The purpose of this return statement is to work around + # https://github.com/PyCQA/pylint/issues/5273; it is otherwise ignored. + return self._AssertRaisesContext(None, None, None) # type: ignore[return-value] + + def assertRaisesWithLiteralMatch(self, expected_exception, + expected_exception_message, + callable_obj=None, *args, **kwargs): + """Asserts that the message in a raised exception equals the given string. + + Unlike assertRaisesRegex, this method takes a literal string, not + a regular expression. + + with self.assertRaisesWithLiteralMatch(ExType, 'message'): + DoSomething() + + Args: + expected_exception: Exception class expected to be raised. + expected_exception_message: String message expected in the raised + exception. For a raise exception e, expected_exception_message must + equal str(e). + callable_obj: Function to be called, or None to return a context. + *args: Extra args. + **kwargs: Extra kwargs. + + Returns: + A context manager if callable_obj is None. Otherwise, None. + + Raises: + self.failureException if callable_obj does not raise a matching exception. + """ + def Check(err): + actual_exception_message = str(err) + self.assertTrue(expected_exception_message == actual_exception_message, + 'Exception message does not match.\n' + 'Expected: %r\n' + 'Actual: %r' % (expected_exception_message, + actual_exception_message)) + + context = self._AssertRaisesContext(expected_exception, self, Check) + if callable_obj is None: + return context + with context: + callable_obj(*args, **kwargs) + + def assertContainsInOrder(self, strings, target, msg=None): + """Asserts that the strings provided are found in the target in order. + + This may be useful for checking HTML output. + + Args: + strings: A list of strings, such as [ 'fox', 'dog' ] + target: A target string in which to look for the strings, such as + 'The quick brown fox jumped over the lazy dog'. + msg: Optional message to report on failure. + """ + if isinstance(strings, (bytes, str)): + strings = (strings,) + + current_index = 0 + last_string = None + for string in strings: + index = target.find(str(string), current_index) + if index == -1 and current_index == 0: + self.fail("Did not find '%s' in '%s'" % + (string, target), msg) + elif index == -1: + self.fail("Did not find '%s' after '%s' in '%s'" % + (string, last_string, target), msg) + last_string = string + current_index = index + + def assertContainsSubsequence(self, container, subsequence, msg=None): + """Asserts that "container" contains "subsequence" as a subsequence. + + Asserts that "container" contains all the elements of "subsequence", in + order, but possibly with other elements interspersed. For example, [1, 2, 3] + is a subsequence of [0, 0, 1, 2, 0, 3, 0] but not of [0, 0, 1, 3, 0, 2, 0]. + + Args: + container: the list we're testing for subsequence inclusion. + subsequence: the list we hope will be a subsequence of container. + msg: Optional message to report on failure. + """ + first_nonmatching = None + reversed_container = list(reversed(container)) + subsequence = list(subsequence) + + for e in subsequence: + if e not in reversed_container: + first_nonmatching = e + break + while e != reversed_container.pop(): + pass + + if first_nonmatching is not None: + self.fail('%s not a subsequence of %s. First non-matching element: %s' % + (subsequence, container, first_nonmatching), msg) + + def assertContainsExactSubsequence(self, container, subsequence, msg=None): + """Asserts that "container" contains "subsequence" as an exact subsequence. + + Asserts that "container" contains all the elements of "subsequence", in + order, and without other elements interspersed. For example, [1, 2, 3] is an + exact subsequence of [0, 0, 1, 2, 3, 0] but not of [0, 0, 1, 2, 0, 3, 0]. + + Args: + container: the list we're testing for subsequence inclusion. + subsequence: the list we hope will be an exact subsequence of container. + msg: Optional message to report on failure. + """ + container = list(container) + subsequence = list(subsequence) + longest_match = 0 + + for start in range(1 + len(container) - len(subsequence)): + if longest_match == len(subsequence): + break + index = 0 + while (index < len(subsequence) and + subsequence[index] == container[start + index]): + index += 1 + longest_match = max(longest_match, index) + + if longest_match < len(subsequence): + self.fail('%s not an exact subsequence of %s. ' + 'Longest matching prefix: %s' % + (subsequence, container, subsequence[:longest_match]), msg) + + def assertTotallyOrdered(self, *groups, **kwargs): + """Asserts that total ordering has been implemented correctly. + + For example, say you have a class A that compares only on its attribute x. + Comparators other than ``__lt__`` are omitted for brevity:: + + class A(object): + def __init__(self, x, y): + self.x = x + self.y = y + + def __hash__(self): + return hash(self.x) + + def __lt__(self, other): + try: + return self.x < other.x + except AttributeError: + return NotImplemented + + assertTotallyOrdered will check that instances can be ordered correctly. + For example:: + + self.assertTotallyOrdered( + [None], # None should come before everything else. + [1], # Integers sort earlier. + [A(1, 'a')], + [A(2, 'b')], # 2 is after 1. + [A(3, 'c'), A(3, 'd')], # The second argument is irrelevant. + [A(4, 'z')], + ['foo']) # Strings sort last. + + Args: + *groups: A list of groups of elements. Each group of elements is a list + of objects that are equal. The elements in each group must be less + than the elements in the group after it. For example, these groups are + totally ordered: ``[None]``, ``[1]``, ``[2, 2]``, ``[3]``. + **kwargs: optional msg keyword argument can be passed. + """ + + def CheckOrder(small, big): + """Ensures small is ordered before big.""" + self.assertFalse(small == big, + self._formatMessage(msg, '%r unexpectedly equals %r' % + (small, big))) + self.assertTrue(small != big, + self._formatMessage(msg, '%r unexpectedly equals %r' % + (small, big))) + self.assertLess(small, big, msg) + self.assertFalse(big < small, + self._formatMessage(msg, + '%r unexpectedly less than %r' % + (big, small))) + self.assertLessEqual(small, big, msg) + self.assertFalse(big <= small, self._formatMessage( + '%r unexpectedly less than or equal to %r' % (big, small), msg + )) + self.assertGreater(big, small, msg) + self.assertFalse(small > big, + self._formatMessage(msg, + '%r unexpectedly greater than %r' % + (small, big))) + self.assertGreaterEqual(big, small) + self.assertFalse(small >= big, self._formatMessage( + msg, + '%r unexpectedly greater than or equal to %r' % (small, big))) + + def CheckEqual(a, b): + """Ensures that a and b are equal.""" + self.assertEqual(a, b, msg) + self.assertFalse(a != b, + self._formatMessage(msg, '%r unexpectedly unequals %r' % + (a, b))) + + # Objects that compare equal must hash to the same value, but this only + # applies if both objects are hashable. + if (isinstance(a, abc.Hashable) and + isinstance(b, abc.Hashable)): + self.assertEqual( + hash(a), hash(b), + self._formatMessage( + msg, 'hash %d of %r unexpectedly not equal to hash %d of %r' % + (hash(a), a, hash(b), b))) + + self.assertFalse(a < b, + self._formatMessage(msg, + '%r unexpectedly less than %r' % + (a, b))) + self.assertFalse(b < a, + self._formatMessage(msg, + '%r unexpectedly less than %r' % + (b, a))) + self.assertLessEqual(a, b, msg) + self.assertLessEqual(b, a, msg) # pylint: disable=arguments-out-of-order + self.assertFalse(a > b, + self._formatMessage(msg, + '%r unexpectedly greater than %r' % + (a, b))) + self.assertFalse(b > a, + self._formatMessage(msg, + '%r unexpectedly greater than %r' % + (b, a))) + self.assertGreaterEqual(a, b, msg) + self.assertGreaterEqual(b, a, msg) # pylint: disable=arguments-out-of-order + + msg = kwargs.get('msg') + + # For every combination of elements, check the order of every pair of + # elements. + for elements in itertools.product(*groups): + elements = list(elements) + for index, small in enumerate(elements[:-1]): + for big in elements[index + 1:]: + CheckOrder(small, big) + + # Check that every element in each group is equal. + for group in groups: + for a in group: + CheckEqual(a, a) + for a, b in itertools.product(group, group): + CheckEqual(a, b) + + def assertDictContainsSubset( + self, + subset: abc.Mapping[Any, Any], + dictionary: abc.Mapping[Any, Any], + msg=None, + ): + """Raises AssertionError if "dictionary" is not a superset of "subset". + + Args: + subset: A dict, the expected subset of the "dictionary". + dictionary: A dict, the actual value. + msg: An optional str, the associated message. + + Raises: + AssertionError: if "dictionary" is not a superset of "subset". + """ + self.assertDictEqual({**dictionary}, {**dictionary, **subset}, msg) + + def assertDictEqual(self, a, b, msg=None): + """Raises AssertionError if a and b are not equal dictionaries. + + Args: + a: A dict, the expected value. + b: A dict, the actual value. + msg: An optional str, the associated message. + + Raises: + AssertionError: if the dictionaries are not equal. + """ + self.assertMappingEqual(a, b, msg, mapping_type=dict) + + def assertDictAlmostEqual( + self, + a, + b, + places=None, + msg=None, + delta=None, + ): + """Raises AssertionError if a and b are not equal or almost equal dicts. + + This is like assertDictEqual, except for float values which are compared + using assertAlmostEqual. Almost equality is determined for float values by: + - have numeric difference less than the given delta, + or + - equal if rounded to the given number of decimal places after the decimal + point (default 7). + + Args: + a: A dict, the expected value. + b: A dict, the actual value. + places: The number of decimal places to compare for floats. + msg: An optional str, the associated message. + delta: The OK difference between compared values for floats. + + Raises: + AssertionError: if the dictionaries are not equal or almost equal. + ValueError: if both places and delta are specified. + """ + + # Almost equality with preset places and delta. + def almost_equal_compare(a_value, b_value): + if isinstance(a_value, numbers.Number) and isinstance( + b_value, numbers.Number + ): + try: + # assertAlmostEqual should be called with at most one of `places` + # and `delta`. However, it's okay for assertMappingEqual to pass + # both because we want the latter to fail if the former does. + # pytype: disable=wrong-keyword-args + self.assertAlmostEqual( + a_value, + b_value, + places=places, + delta=delta, + ) + # pytype: enable=wrong-keyword-args + except self.failureException as err: + return False, err + return True, None + else: + # Fall back to regular equality check if the values are not numbers. + try: + self.assertEqual(a_value, b_value) + except self.failureException as err: + return False, err + return True, None + + if delta is not None and places is not None: + raise ValueError('specify delta or places not both\n') + + self.assertMappingEqual( + a, + b, + msg, + mapping_type=dict, + check_values_equality=almost_equal_compare, + ) + + def assertMappingEqual( + self, + a, + b, + msg=None, + mapping_type=abc.Mapping, + check_values_equality=lambda x, y: (x == y, None), + ): + """Raises AssertionError if a and b differ in keys or values. + + Key sets must be exectly the same, the corresponding values should satisfy + the provided equality function. + + Args: + a: A mapping, the expected value. + b: A mapping, the actual value. + msg: An optional str, the associated message. + mapping_type: The expected type of the mappings. + check_values_equality: A function that takes two values and returns a + tuple of (bool, BaseException), where the bool is True if the values are + equal and the BaseException is an optional exception occured during the + equality check. + + Raises: + AssertionError: if the dictionaries are not equal. + """ + + if not isinstance(a, mapping_type): + self.fail( + f'a should be a {mapping_type.__name__}, found type:' + f' {type(a).__name__}', + msg, + ) + if not isinstance(b, mapping_type): + self.fail( + f'b should be a {mapping_type.__name__}, found type:' + f' {type(b).__name__}', + msg, + ) + if a == b: + return + + def Sorted(list_of_items): + try: + return sorted(list_of_items) # In 3.3, unordered are possible. + except TypeError: + return list_of_items + + a_items = Sorted(list(a.items())) + b_items = Sorted(list(b.items())) + + unexpected = [] + missing = [] + different = [] + + # The standard library default output confounds lexical difference with + # value difference; treat them separately. + for a_key, a_value in a_items: + if a_key not in b: + missing.append((a_key, a_value)) + continue + b_value = b[a_key] + is_equal, err = check_values_equality(a_value, b_value) + if not is_equal: + different.append((a_key, a_value, b_value, err)) + + for b_key, b_value in b_items: + if b_key not in a: + unexpected.append((b_key, b_value)) + + # If all difference buckets are empty, then mappings are considered equal. + if not unexpected and not different and not missing: + return + + safe_repr = unittest.util.safe_repr # pytype: disable=module-attr + + def Repr(dikt): + """Deterministic repr for dict.""" + # Sort the entries based on their repr, not based on their sort order, + # which will be non-deterministic across executions, for many types. + entries = sorted((safe_repr(k), safe_repr(v)) for k, v in dikt.items()) + return '{' + ', '.join(f'{k}: {v}' for k, v in entries) + '}' + + message = [f'{Repr(a)} != {Repr(b)}{"("+msg+")" if msg else ""}'] + + if unexpected: + message.append( + 'Unexpected, but present entries:\n' + + ''.join(f'{safe_repr(k)}: {safe_repr(v)}\n' for k, v in unexpected) + ) + + if different: + message.append( + 'repr() of differing entries:\n' + + ''.join( + f'{safe_repr(k)}: ' + f'{err if err else f"{safe_repr(a_value)} != {safe_repr(b_value)}"}\n' + for k, a_value, b_value, err in different + ) + ) + + if missing: + message.append( + 'Missing entries:\n' + + ''.join(f'{safe_repr(k)}: {safe_repr(v)}\n' for k, v in missing) + ) + + raise self.failureException('\n'.join(message)) + + def assertDataclassEqual(self, first, second, msg=None): + """Asserts two dataclasses are equal with more informative errors. + + Arguments must both be dataclasses. This compares equality of individual + fields and takes care to not compare fields that are marked as + non-comparable. It gives per field differences, which are easier to parse + than the comparison of the string representations from assertEqual. + + In cases where the dataclass has a custom __eq__, and it is defined in a + way that is inconsistent with equality of comparable fields, we raise an + exception without further trying to figure out how they are different. + + Args: + first: A dataclass, the first value. + second: A dataclass, the second value. + msg: An optional str, the associated message. + + Raises: + AssertionError: if the dataclasses are not equal. + """ + + if not dataclasses.is_dataclass(first) or isinstance(first, type): + raise self.failureException('First argument is not a dataclass instance.') + if not dataclasses.is_dataclass(second) or isinstance(second, type): + raise self.failureException( + 'Second argument is not a dataclass instance.' + ) + + if first == second: + return + + if type(first) is not type(second): + self.fail( + 'Found different dataclass types: %s != %s' + % (type(first), type(second)), + msg, + ) + + # Make sure to skip fields that are marked compare=False. + different = [ + (f.name, getattr(first, f.name), getattr(second, f.name)) + for f in dataclasses.fields(first) + if f.compare and getattr(first, f.name) != getattr(second, f.name) + ] + + safe_repr = unittest.util.safe_repr # pytype: disable=module-attr + message = ['%s != %s' % (safe_repr(first), safe_repr(second))] + if different: + message.append('Fields that differ:') + message.extend( + '%s: %s != %s' % (k, safe_repr(first_v), safe_repr(second_v)) + for k, first_v, second_v in different + ) + else: + message.append( + 'Cannot detect difference by examining the fields of the dataclass.' + ) + + self.fail('\n'.join(message), msg) + + def assertUrlEqual(self, a, b, msg=None): + """Asserts that urls are equal, ignoring ordering of query params.""" + parsed_a = parse.urlparse(a) + parsed_b = parse.urlparse(b) + self.assertEqual(parsed_a.scheme, parsed_b.scheme, msg) + self.assertEqual(parsed_a.netloc, parsed_b.netloc, msg) + self.assertEqual(parsed_a.path, parsed_b.path, msg) + self.assertEqual(parsed_a.fragment, parsed_b.fragment, msg) + self.assertEqual(sorted(parsed_a.params.split(';')), + sorted(parsed_b.params.split(';')), msg) + self.assertDictEqual( + parse.parse_qs(parsed_a.query, keep_blank_values=True), + parse.parse_qs(parsed_b.query, keep_blank_values=True), msg) + + def assertSameStructure(self, a, b, aname='a', bname='b', msg=None): + """Asserts that two values contain the same structural content. + + The two arguments should be data trees consisting of trees of dicts and + lists. They will be deeply compared by walking into the contents of dicts + and lists; other items will be compared using the == operator. + If the two structures differ in content, the failure message will indicate + the location within the structures where the first difference is found. + This may be helpful when comparing large structures. + + Mixed Sequence and Set types are supported. Mixed Mapping types are + supported, but the order of the keys will not be considered in the + comparison. + + Args: + a: The first structure to compare. + b: The second structure to compare. + aname: Variable name to use for the first structure in assertion messages. + bname: Variable name to use for the second structure. + msg: Additional text to include in the failure message. + """ + + # Accumulate all the problems found so we can report all of them at once + # rather than just stopping at the first + problems = [] + + _walk_structure_for_problems(a, b, aname, bname, problems, + self.assertEqual, self.failureException) + + # Avoid spamming the user toooo much + if self.maxDiff is not None: + max_problems_to_show = self.maxDiff // 80 + if len(problems) > max_problems_to_show: + problems = problems[0:max_problems_to_show-1] + ['...'] + + if problems: + self.fail('; '.join(problems), msg) + + def assertJsonEqual(self, first, second, msg=None): + """Asserts that the JSON objects defined in two strings are equal. + + A summary of the differences will be included in the failure message + using assertSameStructure. + + Args: + first: A string containing JSON to decode and compare to second. + second: A string containing JSON to decode and compare to first. + msg: Additional text to include in the failure message. + """ + try: + first_structured = json.loads(first) + except ValueError as e: + raise ValueError(self._formatMessage( + msg, + 'could not decode first JSON value %s: %s' % (first, e))) + + try: + second_structured = json.loads(second) + except ValueError as e: + raise ValueError(self._formatMessage( + msg, + 'could not decode second JSON value %s: %s' % (second, e))) + + self.assertSameStructure(first_structured, second_structured, + aname='first', bname='second', msg=msg) + + def _getAssertEqualityFunc( + self, first: Any, second: Any + ) -> abc.Callable[..., None]: + try: + return super()._getAssertEqualityFunc(first, second) + except AttributeError: + # This is a workaround if unittest.TestCase.__init__ was never run. + # It usually means that somebody created a subclass just for the + # assertions and has overridden __init__. "assertTrue" is a safe + # value that will not make __init__ raise a ValueError. + test_method = getattr(self, '_testMethodName', 'assertTrue') + super().__init__(test_method) + + return super()._getAssertEqualityFunc(first, second) + + def fail(self, msg=None, user_msg=None) -> NoReturn: + """Fail immediately with the given standard message and user message.""" + super().fail(self._formatMessage(user_msg, msg)) + + +def _sorted_list_difference( + expected: list[_T], actual: list[_T] +) -> tuple[list[_T], list[_T]]: + """Finds elements in only one or the other of two, sorted input lists. + + Returns a two-element tuple of lists. The first list contains those + elements in the "expected" list but not in the "actual" list, and the + second contains those elements in the "actual" list but not in the + "expected" list. Duplicate elements in either input list are ignored. + + Args: + expected: The list we expected. + actual: The list we actually got. + Returns: + (missing, unexpected) + missing: items in expected that are not in actual. + unexpected: items in actual that are not in expected. + """ + i = j = 0 + missing = [] + unexpected = [] + while True: + try: + e = expected[i] + a = actual[j] + if e < a: # type: ignore[operator] + missing.append(e) + i += 1 + while expected[i] == e: + i += 1 + elif e > a: # type: ignore[operator] + unexpected.append(a) + j += 1 + while actual[j] == a: + j += 1 + else: + i += 1 + try: + while expected[i] == e: + i += 1 + finally: + j += 1 + while actual[j] == a: + j += 1 + except IndexError: + missing.extend(expected[i:]) + unexpected.extend(actual[j:]) + break + return missing, unexpected + + +def _are_both_of_integer_type(a: object, b: object) -> bool: + return isinstance(a, int) and isinstance(b, int) + + +def _are_both_of_sequence_type(a: object, b: object) -> bool: + return isinstance(a, abc.Sequence) and isinstance( + b, abc.Sequence) and not isinstance( + a, _TEXT_OR_BINARY_TYPES) and not isinstance(b, _TEXT_OR_BINARY_TYPES) + + +def _are_both_of_set_type(a: object, b: object) -> bool: + return isinstance(a, abc.Set) and isinstance(b, abc.Set) + + +def _are_both_of_mapping_type(a: object, b: object) -> bool: + return isinstance(a, abc.Mapping) and isinstance( + b, abc.Mapping) + + +def _walk_structure_for_problems( + a, b, aname, bname, problem_list, leaf_assert_equal_func, failure_exception +): + """The recursive comparison behind assertSameStructure.""" + if type(a) != type(b) and not ( # pylint: disable=unidiomatic-typecheck + _are_both_of_integer_type(a, b) or _are_both_of_sequence_type(a, b) or + _are_both_of_set_type(a, b) or _are_both_of_mapping_type(a, b)): + # We do not distinguish between int and long types as 99.99% of Python 2 + # code should never care. They collapse into a single type in Python 3. + problem_list.append('%s is a %r but %s is a %r' % + (aname, type(a), bname, type(b))) + # If they have different types there's no point continuing + return + + if isinstance(a, abc.Set): + for k in a: + if k not in b: + problem_list.append( + '%s has %r but %s does not' % (aname, k, bname)) + for k in b: + if k not in a: + problem_list.append('%s lacks %r but %s has it' % (aname, k, bname)) + + # NOTE: a or b could be a defaultdict, so we must take care that the traversal + # doesn't modify the data. + elif isinstance(a, abc.Mapping): + for k in a: + if k in b: + _walk_structure_for_problems( + a[k], b[k], '%s[%r]' % (aname, k), '%s[%r]' % (bname, k), + problem_list, leaf_assert_equal_func, failure_exception) + else: + problem_list.append( + "%s has [%r] with value %r but it's missing in %s" % + (aname, k, a[k], bname)) + for k in b: + if k not in a: + problem_list.append( + '%s lacks [%r] but %s has it with value %r' % + (aname, k, bname, b[k])) + + # Strings/bytes are Sequences but we'll just do those with regular != + elif (isinstance(a, abc.Sequence) and + not isinstance(a, _TEXT_OR_BINARY_TYPES)): + minlen = min(len(a), len(b)) + for i in range(minlen): + _walk_structure_for_problems( + a[i], b[i], '%s[%d]' % (aname, i), '%s[%d]' % (bname, i), + problem_list, leaf_assert_equal_func, failure_exception) + for i in range(minlen, len(a)): + problem_list.append('%s has [%i] with value %r but %s does not' % + (aname, i, a[i], bname)) + for i in range(minlen, len(b)): + problem_list.append('%s lacks [%i] but %s has it with value %r' % + (aname, i, bname, b[i])) + + else: + try: + leaf_assert_equal_func(a, b) + except failure_exception: + problem_list.append('%s is %r but %s is %r' % (aname, a, bname, b)) + + +def get_command_string(command): + """Returns an escaped string that can be used as a shell command. + + Args: + command: List or string representing the command to run. + Returns: + A string suitable for use as a shell command. + """ + if isinstance(command, str): + return command + else: + if os.name == 'nt': + return ' '.join(command) + else: + # The following is identical to Python 3's shlex.quote function. + command_string = '' + for word in command: + # Single quote word, and replace each ' in word with '"'"' + command_string += "'" + word.replace("'", "'\"'\"'") + "' " + return command_string[:-1] + + +def get_command_stderr(command, env=None, close_fds=True): + """Runs the given shell command and returns a tuple. + + Args: + command: List or string representing the command to run. + env: Dictionary of environment variable settings. If None, no environment + variables will be set for the child process. This is to make tests + more hermetic. NOTE: this behavior is different than the standard + subprocess module. + close_fds: Whether or not to close all open fd's in the child after forking. + On Windows, this is ignored and close_fds is always False. + + Returns: + Tuple of (exit status, text printed to stdout and stderr by the command). + """ + if env is None: env = {} + if os.name == 'nt': + # Windows does not support setting close_fds to True while also redirecting + # standard handles. + close_fds = False + + use_shell = isinstance(command, str) + # Pass the shell command as stdin to /bin/sh rather than using Python's + # behavior of passing it in as a command line argument. That can save us when + # the shell command exceeds the maximum command line length but the actual + # individual process invocations within it don't. + if os.name != 'nt' and use_shell: + stdin_input = command.encode() + command = ['/bin/sh'] + use_shell = False + else: + stdin_input = None + + result = subprocess.run( + command, + close_fds=close_fds, + env=env, + shell=use_shell, + input=stdin_input, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + check=False, + ) + return (result.returncode, result.stdout) + + +def _quote_long_string(s: str | bytes | bytearray) -> str: + """Quotes a potentially multi-line string to make the start and end obvious. + + Args: + s: A string. + + Returns: + The quoted string. + """ + if isinstance(s, (bytes, bytearray)): + try: + s = s.decode('utf-8') + except UnicodeDecodeError: + s = str(s) + return ('8<-----------\n' + + s + '\n' + + '----------->8\n') + + +def print_python_version() -> None: + # Having this in the test output logs by default helps debugging when all + # you've got is the log and no other idea of which Python was used. + sys.stderr.write('Running tests under Python {0[0]}.{0[1]}.{0[2]}: ' + '{1}\n'.format( + sys.version_info, + sys.executable if sys.executable else 'embedded.')) + + +def main(*args: str, **kwargs: Any) -> None: + """Executes a set of Python unit tests. + + Usually this function is called without arguments, so the + unittest.TestProgram instance will get created with the default settings, + so it will run all test methods of all TestCase classes in the ``__main__`` + module. + + Args: + *args: Positional arguments passed through to + ``unittest.TestProgram.__init__``. + **kwargs: Keyword arguments passed through to + ``unittest.TestProgram.__init__``. + """ + print_python_version() + _run_in_app(run_tests, args, kwargs) + + +def _is_in_app_main() -> bool: + """Returns True iff app.run is active.""" + f = sys._getframe().f_back # pylint: disable=protected-access + while f: + if f.f_code == app.run.__code__: + return True + f = f.f_back + return False + + +def _register_sigterm_with_faulthandler() -> None: + """Have faulthandler dump stacks on SIGTERM. Useful to diagnose timeouts.""" + if getattr(faulthandler, 'register', None): + # faulthandler.register is not available on Windows. + # faulthandler.enable() is already called by app.run. + try: + faulthandler.register(signal.SIGTERM, chain=True) # pytype: disable=module-attr + except Exception as e: # pylint: disable=broad-except + sys.stderr.write('faulthandler.register(SIGTERM) failed ' + '%r; ignoring.\n' % e) + + +def _run_in_app( + function: abc.Callable[..., None], + args: abc.Sequence[str], + kwargs: abc.Mapping[str, Any], +) -> None: + """Executes a set of Python unit tests, ensuring app.run. + + This is a private function, users should call absltest.main(). + + _run_in_app calculates argv to be the command-line arguments of this program + (without the flags), sets the default of FLAGS.alsologtostderr to True, + then it calls function(argv, args, kwargs), making sure that `function' + will get called within app.run(). _run_in_app does this by checking whether + it is called by app.run(), or by calling app.run() explicitly. + + The reason why app.run has to be ensured is to make sure that + flags are parsed and stripped properly, and other initializations done by + the app module are also carried out, no matter if absltest.run() is called + from within or outside app.run(). + + If _run_in_app is called from within app.run(), then it will reparse + sys.argv and pass the result without command-line flags into the argv + argument of `function'. The reason why this parsing is needed is that + __main__.main() calls absltest.main() without passing its argv. So the + only way _run_in_app could get to know the argv without the flags is that + it reparses sys.argv. + + _run_in_app changes the default of FLAGS.alsologtostderr to True so that the + test program's stderr will contain all the log messages unless otherwise + specified on the command-line. This overrides any explicit assignment to + FLAGS.alsologtostderr by the test program prior to the call to _run_in_app() + (e.g. in __main__.main). + + Please note that _run_in_app (and the function it calls) is allowed to make + changes to kwargs. + + Args: + function: absltest.run_tests or a similar function. It will be called as + function(argv, args, kwargs) where argv is a list containing the + elements of sys.argv without the command-line flags. + args: Positional arguments passed through to unittest.TestProgram.__init__. + kwargs: Keyword arguments passed through to unittest.TestProgram.__init__. + """ + if _is_in_app_main(): + _register_sigterm_with_faulthandler() + + # Change the default of alsologtostderr from False to True, so the test + # programs's stderr will contain all the log messages. + # If --alsologtostderr=false is specified in the command-line, or user + # has called FLAGS.alsologtostderr = False before, then the value is kept + # False. + FLAGS.set_default('alsologtostderr', True) + + # Here we only want to get the `argv` without the flags. To avoid any + # side effects of parsing flags, we temporarily stub out the `parse` method + stored_parse_methods = {} + noop_parse = lambda _: None + for name in FLAGS: + # Avoid any side effects of parsing flags. + stored_parse_methods[name] = FLAGS[name].parse + # This must be a separate loop since multiple flag names (short_name=) can + # point to the same flag object. + for name in FLAGS: + FLAGS[name].parse = noop_parse # type: ignore[method-assign] + try: + argv = FLAGS(sys.argv) + finally: + for name in FLAGS: + FLAGS[name].parse = stored_parse_methods[name] # type: ignore[method-assign] + sys.stdout.flush() + + function(argv, args, kwargs) + else: + # Send logging to stderr. Use --alsologtostderr instead of --logtostderr + # in case tests are reading their own logs. + FLAGS.set_default('alsologtostderr', True) + + def main_function(argv): + _register_sigterm_with_faulthandler() + function(argv, args, kwargs) + + app.run(main=main_function) + + +def _is_suspicious_attribute( + testCaseClass: type[unittest.TestCase], name: str +) -> bool: + """Returns True if an attribute is a method named like a test method.""" + if name.startswith('Test') and len(name) > 4 and name[4].isupper(): + attr = getattr(testCaseClass, name) + if inspect.isfunction(attr) or inspect.ismethod(attr): + args = inspect.getfullargspec(attr) + return (len(args.args) == 1 and args.args[0] == 'self' and + args.varargs is None and args.varkw is None and + not args.kwonlyargs) + return False + + +def skipThisClass( + reason: str, +) -> abc.Callable[[type[_T]], type[_T]]: + """Skip tests in the decorated TestCase, but not any of its subclasses. + + This decorator indicates that this class should skip all its tests, but not + any of its subclasses. Useful for if you want to share testMethod or setUp + implementations between a number of concrete testcase classes. + + Example usage, showing how you can share some common test methods between + subclasses. In this example, only ``BaseTest`` will be marked as skipped, and + not RealTest or SecondRealTest:: + + @absltest.skipThisClass("Shared functionality") + class BaseTest(absltest.TestCase): + def test_simple_functionality(self): + self.assertEqual(self.system_under_test.method(), 1) + + class RealTest(BaseTest): + def setUp(self): + super().setUp() + self.system_under_test = MakeSystem(argument) + + def test_specific_behavior(self): + ... + + class SecondRealTest(BaseTest): + def setUp(self): + super().setUp() + self.system_under_test = MakeSystem(other_arguments) + + def test_other_behavior(self): + ... + + Args: + reason: The reason we have a skip in place. For instance: 'shared test + methods' or 'shared assertion methods'. + + Returns: + Decorator function that will cause a class to be skipped. + """ + if isinstance(reason, type): + raise TypeError(f'Got {reason!r}, expected reason as string') + + def _skip_class(test_case_class): + if not issubclass(test_case_class, unittest.TestCase): + raise TypeError( + f'Decorating {test_case_class!r}, expected TestCase subclass' + ) + + # Only shadow the setUpClass method if it is directly defined. If it is + # in the parent class we invoke it via a super() call instead of holding + # a reference to it. + shadowed_setupclass = test_case_class.__dict__.get('setUpClass', None) + + @classmethod + def replacement_setupclass(cls, *args, **kwargs): + # Skip this class if it is the one that was decorated with @skipThisClass + if cls is test_case_class: + raise SkipTest(reason) + if shadowed_setupclass: + # Pass along `cls` so the MRO chain doesn't break. + # The original method is a `classmethod` descriptor, which can't + # be directly called, but `__func__` has the underlying function. + return shadowed_setupclass.__func__(cls, *args, **kwargs) + else: + # Because there's no setUpClass() defined directly on test_case_class, + # we call super() ourselves to continue execution of the inheritance + # chain. + return super(test_case_class, cls).setUpClass(*args, **kwargs) + + test_case_class.setUpClass = replacement_setupclass + return test_case_class + + return _skip_class + + +class TestLoader(unittest.TestLoader): + """A test loader which supports common test features. + + Supported features include: + * Banning untested methods with test-like names: methods attached to this + testCase with names starting with `Test` are ignored by the test runner, + and often represent mistakenly-omitted test cases. This loader will raise + a TypeError when attempting to load a TestCase with such methods. + * Randomization of test case execution order (optional). + """ + + _ERROR_MSG = textwrap.dedent("""Method '%s' is named like a test case but + is not one. This is often a bug. If you want it to be a test method, + name it with 'test' in lowercase. If not, rename the method to not begin + with 'Test'.""") + + def __init__(self, *args, **kwds): + super().__init__(*args, **kwds) + seed = _get_default_randomize_ordering_seed() + if seed: + self._randomize_ordering_seed = seed + self._random = random.Random(self._randomize_ordering_seed) + else: + self._randomize_ordering_seed = None + self._random = None + + def getTestCaseNames(self, testCaseClass): # pylint:disable=invalid-name + """Validates and returns a (possibly randomized) list of test case names.""" + for name in dir(testCaseClass): + if _is_suspicious_attribute(testCaseClass, name): + raise TypeError(TestLoader._ERROR_MSG % name) + names = list(super().getTestCaseNames(testCaseClass)) + if self._randomize_ordering_seed is not None and self._random is not None: + logging.info( + 'Randomizing test order with seed: %d', self._randomize_ordering_seed) + logging.info( + 'To reproduce this order, re-run with ' + '--test_randomize_ordering_seed=%d', self._randomize_ordering_seed) + self._random.shuffle(names) + return names + + def shardTestCaseNames( + self, + iterator: abc.Iterator[Any], + ordered_names: abc.Sequence[str], + shard_index: int, + ) -> abc.Sequence[str]: + """Filters and returns test case names for a specific shard. + + This method is intended to be used in conjunction with test sharding + (e.g., when running tests on a distributed system or when running tests + with bazel's test sharding feature). It will return a subset of the + input test case names, based on the shard index and total shard count. + + Args: + iterator: An iterator over the shards, where each iteration returns the + next shard index. + ordered_names: A sequence of test case names. + shard_index: The index of the current shard. + + Returns: + A sequence of test case names for the current shard. + """ + filtered_names = [] + # We need to sort the list of tests in order to determine which tests this + # shard is responsible for; however, it's important to preserve the order + # returned by the base loader, e.g. in the case of randomized test ordering. + for testcase in sorted(ordered_names): + bucket = next(iterator) + if bucket == shard_index: + filtered_names.append(testcase) + return [x for x in ordered_names if x in filtered_names] + + +def get_default_xml_output_filename() -> str | None: + if os.environ.get('XML_OUTPUT_FILE'): + return os.environ['XML_OUTPUT_FILE'] + elif os.environ.get('RUNNING_UNDER_TEST_DAEMON'): + return os.path.join(os.path.dirname(TEST_TMPDIR.value), 'test_detail.xml') + elif os.environ.get('TEST_XMLOUTPUTDIR'): + return os.path.join( + os.environ['TEST_XMLOUTPUTDIR'], + os.path.splitext(os.path.basename(sys.argv[0]))[0] + '.xml') + return None + + +def _setup_filtering(argv: abc.MutableSequence[str]) -> bool: + """Implements the bazel test filtering protocol. + + The following environment variable is used in this method: + + TESTBRIDGE_TEST_ONLY: string, if set, is forwarded to the unittest + framework to use as a test filter. Its value is split with shlex, then: + 1. On Python 3.6 and before, split values are passed as positional + arguments on argv. + 2. On Python 3.7+, split values are passed to unittest's `-k` flag. Tests + are matched by glob patterns or substring. See + https://docs.python.org/3/library/unittest.html#cmdoption-unittest-k + + Args: + argv: the argv to mutate in-place. + + Returns: + Whether test filtering is requested. + """ + test_filter = os.environ.get('TESTBRIDGE_TEST_ONLY') + if argv is None or not test_filter: + return False + + filters = ['-k=' + test_filter for test_filter in shlex.split(test_filter)] + + argv[1:1] = filters + return True + + +def _setup_test_runner_fail_fast(argv: abc.MutableSequence[str]) -> None: + """Implements the bazel test fail fast protocol. + + The following environment variable is used in this method: + + TESTBRIDGE_TEST_RUNNER_FAIL_FAST=<1|0> + + If set to 1, --failfast is passed to the unittest framework to return upon + first failure. + + Args: + argv: the argv to mutate in-place. + """ + + if argv is None: + return + + if os.environ.get('TESTBRIDGE_TEST_RUNNER_FAIL_FAST') != '1': + return + + argv[1:1] = ['--failfast'] + + +def _setup_sharding( + custom_loader: unittest.TestLoader | None = None, +) -> tuple[unittest.TestLoader, int | None]: + """Implements the bazel sharding protocol. + + The following environment variables are used in this method: + + TEST_SHARD_STATUS_FILE: string, if set, points to a file. We write a blank + file to tell the test runner that this test implements the test sharding + protocol. + + TEST_TOTAL_SHARDS: int, if set, sharding is requested. + + TEST_SHARD_INDEX: int, must be set if TEST_TOTAL_SHARDS is set. Specifies + the shard index for this instance of the test process. Must satisfy: + 0 <= TEST_SHARD_INDEX < TEST_TOTAL_SHARDS. + + Args: + custom_loader: A TestLoader to be made sharded. + + Returns: + A tuple of ``(test_loader, shard_index)``. ``test_loader`` is for + shard-filtering or the standard test loader depending on the sharding + environment variables. ``shard_index`` is the shard index, or ``None`` when + sharding is not used. + """ + + # It may be useful to write the shard file even if the other sharding + # environment variables are not set. Test runners may use this functionality + # to query whether a test binary implements the test sharding protocol. + if 'TEST_SHARD_STATUS_FILE' in os.environ: + try: + with open(os.environ['TEST_SHARD_STATUS_FILE'], 'w') as f: + f.write('') + except OSError: + sys.stderr.write('Error opening TEST_SHARD_STATUS_FILE (%s). Exiting.' + % os.environ['TEST_SHARD_STATUS_FILE']) + sys.exit(1) + + base_loader = custom_loader or TestLoader() + if 'TEST_TOTAL_SHARDS' not in os.environ: + # Not using sharding, use the expected test loader. + return base_loader, None + + total_shards = int(os.environ['TEST_TOTAL_SHARDS']) + shard_index = int(os.environ['TEST_SHARD_INDEX']) + + if shard_index < 0 or shard_index >= total_shards: + sys.stderr.write('ERROR: Bad sharding values. index=%d, total=%d\n' % + (shard_index, total_shards)) + sys.exit(1) + + # Replace the original getTestCaseNames with one that returns + # the test case names for this shard. + delegate_get_names = base_loader.getTestCaseNames + + bucket_iterator = itertools.cycle(range(total_shards)) + + def getSharedTestCaseNames(testCaseClass): + has_shard_test_case_names = hasattr(base_loader, 'shardTestCaseNames') + if has_shard_test_case_names: + sharder = getattr(base_loader, 'shardTestCaseNames') + else: + sharder = lambda *args: TestLoader.shardTestCaseNames(base_loader, *args) + + names = sharder( + bucket_iterator, delegate_get_names(testCaseClass), shard_index + ) + return names + + base_loader.getTestCaseNames = getSharedTestCaseNames # type: ignore[method-assign] + return base_loader, shard_index + + +def _run_and_get_tests_result( + argv: abc.MutableSequence[str], + args: abc.Sequence[Any], + kwargs: abc.MutableMapping[str, Any], + xml_test_runner_class: type[unittest.TextTestRunner], +) -> tuple[unittest.TestResult, bool]: + """Same as run_tests, but it doesn't exit. + + Args: + argv: sys.argv with the command-line flags removed from the front, i.e. the + argv with which :func:`app.run()` has called + ``__main__.main``. It is passed to + ``unittest.TestProgram.__init__(argv=)``, which does its own flag parsing. + It is ignored if kwargs contains an argv entry. + args: Positional arguments passed through to + ``unittest.TestProgram.__init__``. + kwargs: Keyword arguments passed through to + ``unittest.TestProgram.__init__``. + xml_test_runner_class: The type of the test runner class. + + Returns: + A tuple of ``(test_result, fail_when_no_tests_ran)``. + ``fail_when_no_tests_ran`` indicates whether the test should fail when + no tests ran. + """ + + # The entry from kwargs overrides argv. + argv = kwargs.pop('argv', argv) + + if sys.version_info[:2] >= (3, 12): + # Python 3.12 unittest changed the behavior from PASS to FAIL in + # https://github.com/python/cpython/pull/102051. absltest follows this. + fail_when_no_tests_ran = True + else: + # Historically, absltest and unittest before Python 3.12 passes if no tests + # ran. + fail_when_no_tests_ran = False + + # Set up test filtering if requested in environment. + if _setup_filtering(argv): + # When test filtering is requested, ideally we also want to fail when no + # tests ran. However, the test filters are usually done when running bazel. + # When you run multiple targets, e.g. `bazel test //my_dir/... + # --test_filter=MyTest`, you don't necessarily want individual tests to fail + # because no tests match in that particular target. + # Due to this use case, we don't fail when test filtering is requested via + # the environment variable from bazel. + fail_when_no_tests_ran = False + + # Set up --failfast as requested in environment + _setup_test_runner_fail_fast(argv) + + # Shard the (default or custom) loader if sharding is turned on. + kwargs['testLoader'], shard_index = _setup_sharding( + kwargs.get('testLoader', None) + ) + if shard_index is not None and shard_index > 0: + # When sharding is requested, all the shards except the first one shall not + # fail when no tests ran. This happens when the shard count is greater than + # the test case count. + fail_when_no_tests_ran = False + + # XML file name is based upon (sorted by priority): + # --xml_output_file flag, XML_OUTPUT_FILE variable, + # TEST_XMLOUTPUTDIR variable or RUNNING_UNDER_TEST_DAEMON variable. + if FLAGS.xml_output_file: + xml_output_file = FLAGS.xml_output_file + else: + xml_output_file = get_default_xml_output_filename() + if xml_output_file: + FLAGS.xml_output_file = xml_output_file # type: ignore[assignment] + + xml_buffer = None + if xml_output_file: + xml_output_dir = os.path.dirname(xml_output_file) + if xml_output_dir and not os.path.isdir(xml_output_dir): + try: + os.makedirs(xml_output_dir) + except OSError as e: + # File exists error can occur with concurrent tests + if e.errno != errno.EEXIST: + raise + # Fail early if we can't write to the XML output file. This is so that we + # don't waste people's time running tests that will just fail anyways. + with _open(xml_output_file, 'w'): + pass + + # We can reuse testRunner if it supports XML output (e. g. by inheriting + # from xml_reporter.TextAndXMLTestRunner). Otherwise we need to use + # xml_reporter.TextAndXMLTestRunner. + if (kwargs.get('testRunner') is not None + and not hasattr(kwargs['testRunner'], 'set_default_xml_stream')): + sys.stderr.write('WARNING: XML_OUTPUT_FILE or --xml_output_file setting ' + 'overrides testRunner=%r setting (possibly from --pdb)' + % (kwargs['testRunner'])) + # Passing a class object here allows TestProgram to initialize + # instances based on its kwargs and/or parsed command-line args. + kwargs['testRunner'] = xml_test_runner_class + if kwargs.get('testRunner') is None: + kwargs['testRunner'] = xml_test_runner_class + # Use an in-memory buffer (not backed by the actual file) to store the XML + # report, because some tools modify the file (e.g., create a placeholder + # with partial information, in case the test process crashes). + xml_buffer = io.StringIO() + kwargs['testRunner'].set_default_xml_stream(xml_buffer) # pytype: disable=attribute-error + + # If we've used a seed to randomize test case ordering, we want to record it + # as a top-level attribute in the `testsuites` section of the XML output. + randomize_ordering_seed = getattr( + kwargs['testLoader'], '_randomize_ordering_seed', None) + setter = getattr(kwargs['testRunner'], 'set_testsuites_property', None) + if randomize_ordering_seed and setter: + setter('test_randomize_ordering_seed', randomize_ordering_seed) + elif kwargs.get('testRunner') is None: + kwargs['testRunner'] = _pretty_print_reporter.TextTestRunner + + if FLAGS.pdb_post_mortem: + runner = kwargs['testRunner'] + # testRunner can be a class or an instance, which must be tested for + # differently. + # Overriding testRunner isn't uncommon, so only enable the debugging + # integration if the runner claims it does; we don't want to accidentally + # clobber something on the runner. + if ((isinstance(runner, type) and + issubclass(runner, _pretty_print_reporter.TextTestRunner)) or + isinstance(runner, _pretty_print_reporter.TextTestRunner)): + runner.run_for_debugging = True + + # Make sure tmpdir exists. + if not os.path.isdir(TEST_TMPDIR.value): + try: + os.makedirs(TEST_TMPDIR.value) + except OSError as e: + # Concurrent test might have created the directory. + if e.errno != errno.EEXIST: + raise + + # Let unittest.TestProgram.__init__ do its own argv parsing, e.g. for '-v', + # on argv, which is sys.argv without the command-line flags. + kwargs['argv'] = argv + + # Request unittest.TestProgram to not exit. The exit will be handled by + # `absltest.run_tests`. + kwargs['exit'] = False + + try: + test_program = unittest.TestProgram(*args, **kwargs) + return test_program.result, fail_when_no_tests_ran + finally: + if xml_buffer: + try: + with _open(xml_output_file, 'w') as f: + f.write(xml_buffer.getvalue()) + finally: + xml_buffer.close() + + +def run_tests( + argv: abc.MutableSequence[str], + args: abc.Sequence[Any], + kwargs: abc.MutableMapping[str, Any], +) -> None: + """Executes a set of Python unit tests. + + Most users should call absltest.main() instead of run_tests. + + Please note that run_tests should be called from app.run. + Calling absltest.main() would ensure that. + + Please note that run_tests is allowed to make changes to kwargs. + + Args: + argv: sys.argv with the command-line flags removed from the front, i.e. the + argv with which :func:`app.run()` has called + ``__main__.main``. It is passed to + ``unittest.TestProgram.__init__(argv=)``, which does its own flag parsing. + It is ignored if kwargs contains an argv entry. + args: Positional arguments passed through to + ``unittest.TestProgram.__init__``. + kwargs: Keyword arguments passed through to + ``unittest.TestProgram.__init__``. + """ + result, fail_when_no_tests_ran = _run_and_get_tests_result( + argv, args, kwargs, xml_reporter.TextAndXMLTestRunner + ) + if fail_when_no_tests_ran and result.testsRun == 0 and not result.skipped: + # Python 3.12 unittest exits with 5 when no tests ran. The exit code 5 comes + # from pytest which does the same thing. + sys.exit(5) + sys.exit(not result.wasSuccessful()) + + +def _rmtree_ignore_errors(path: str) -> None: + if os.path.isfile(path): + try: + os.unlink(path) + except OSError: + pass + else: + shutil.rmtree(path, ignore_errors=True) + + +def _get_first_part(path: str) -> str: + parts = path.split(os.sep, 1) + return parts[0] diff --git a/venv/Lib/site-packages/absl/testing/flagsaver.py b/venv/Lib/site-packages/absl/testing/flagsaver.py new file mode 100644 index 0000000000000000000000000000000000000000..508491808daf25b5242d923fa3f88978b30c3730 --- /dev/null +++ b/venv/Lib/site-packages/absl/testing/flagsaver.py @@ -0,0 +1,403 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Decorator and context manager for saving and restoring flag values. + +There are many ways to save and restore. Always use the most convenient method +for a given use case. + +Here are examples of each method. They all call ``do_stuff()`` while +``FLAGS.someflag`` is temporarily set to ``'foo'``:: + + from absl.testing import flagsaver + + # Use a decorator which can optionally override flags via arguments. + @flagsaver.flagsaver(someflag='foo') + def some_func(): + do_stuff() + + # Use a decorator which can optionally override flags with flagholders. + @flagsaver.flagsaver((module.FOO_FLAG, 'foo'), (other_mod.BAR_FLAG, 23)) + def some_func(): + do_stuff() + + # Use a decorator which does not override flags itself. + @flagsaver.flagsaver + def some_func(): + FLAGS.someflag = 'foo' + do_stuff() + + # Use a context manager which can optionally override flags via arguments. + with flagsaver.flagsaver(someflag='foo'): + do_stuff() + + # Save and restore the flag values yourself. + saved_flag_values = flagsaver.save_flag_values() + try: + FLAGS.someflag = 'foo' + do_stuff() + finally: + flagsaver.restore_flag_values(saved_flag_values) + + # Use the parsing version to emulate users providing the flags. + # Note that all flags must be provided as strings (unparsed). + @flagsaver.as_parsed(some_int_flag='123') + def some_func(): + # Because the flag was parsed it is considered "present". + assert FLAGS.some_int_flag.present + do_stuff() + + # flagsaver.as_parsed() can also be used as a context manager just like + # flagsaver.flagsaver() + with flagsaver.as_parsed(some_int_flag='123'): + do_stuff() + + # The flagsaver.as_parsed() interface also supports FlagHolder objects. + @flagsaver.as_parsed((module.FOO_FLAG, 'foo'), (other_mod.BAR_FLAG, '23')) + def some_func(): + do_stuff() + + # Using as_parsed with a multi_X flag requires a sequence of strings. + @flagsaver.as_parsed(some_multi_int_flag=['123', '456']) + def some_func(): + assert FLAGS.some_multi_int_flag.present + do_stuff() + + # If a flag name includes non-identifier characters it can be specified like + # so: + @flagsaver.as_parsed(**{'i-like-dashes': 'true'}) + def some_func(): + do_stuff() + +We save and restore a shallow copy of each Flag object's ``__dict__`` attribute. +This preserves all attributes of the flag, such as whether or not it was +overridden from its default value. + +WARNING: Currently a flag that is saved and then deleted cannot be restored. An +exception will be raised. However if you *add* a flag after saving flag values, +and then restore flag values, the added flag will be deleted with no errors. +""" + +from collections.abc import Callable, Mapping, Sequence +import functools +import inspect +from typing import Any, TypeVar, overload + +from absl import flags + +FLAGS = flags.FLAGS + + +# The type of pre/post wrapped functions. +_CallableT = TypeVar('_CallableT', bound=Callable) + + +@overload +def flagsaver(func: _CallableT) -> _CallableT: + ... + + +@overload +def flagsaver( + *args: tuple[flags.FlagHolder, Any], **kwargs: Any +) -> '_FlagOverrider': + ... + + +def flagsaver(*args, **kwargs): + """The main flagsaver interface. See module doc for usage.""" + return _construct_overrider(_FlagOverrider, *args, **kwargs) # type: ignore[bad-return-type] + + +@overload +def as_parsed( + *args: tuple[flags.FlagHolder, str | Sequence[str]], + **kwargs: str | Sequence[str], +) -> '_ParsingFlagOverrider': + ... + + +@overload +def as_parsed(func: _CallableT) -> _CallableT: + ... + + +def as_parsed(*args, **kwargs): + """Overrides flags by parsing strings, saves flag state similar to flagsaver. + + This function can be used as either a decorator or context manager similar to + flagsaver.flagsaver(). However, where flagsaver.flagsaver() directly sets the + flags to new values, this function will parse the provided arguments as if + they were provided on the command line. Among other things, this will cause + `FLAGS['flag_name'].present == True`. + + A note on unparsed input: For many flag types, the unparsed version will be + a single string. However for multi_x (multi_string, multi_integer, multi_enum) + the unparsed version will be a Sequence of strings. + + Args: + *args: Tuples of FlagHolders and their unparsed value. + **kwargs: The keyword args are flag names, and the values are unparsed + values. + + Returns: + _ParsingFlagOverrider that serves as a context manager or decorator. Will + save previous flag state and parse new flags, then on cleanup it will + restore the previous flag state. + """ + return _construct_overrider(_ParsingFlagOverrider, *args, **kwargs) + + +# NOTE: the order of these overload declarations matters. The type checker will +# pick the first match which could be incorrect. +@overload +def _construct_overrider( + flag_overrider_cls: type['_ParsingFlagOverrider'], + *args: tuple[flags.FlagHolder, str | Sequence[str]], + **kwargs: str | Sequence[str], +) -> '_ParsingFlagOverrider': + ... + + +@overload +def _construct_overrider( + flag_overrider_cls: type['_FlagOverrider'], func: _CallableT +) -> _CallableT: + ... + + +@overload +def _construct_overrider( + flag_overrider_cls: type['_FlagOverrider'], + *args: tuple[flags.FlagHolder, Any], + **kwargs: Any, +) -> '_FlagOverrider': + ... + + +def _construct_overrider(flag_overrider_cls, *args, **kwargs): + """Handles the args/kwargs returning an instance of flag_overrider_cls. + + If flag_overrider_cls is _FlagOverrider then values should be native python + types matching the python types. Otherwise if flag_overrider_cls is + _ParsingFlagOverrider the values should be strings or sequences of strings. + + Args: + flag_overrider_cls: The class that will do the overriding. + *args: Tuples of FlagHolder and the new flag value. + **kwargs: Keword args mapping flag name to new flag value. + + Returns: + A _FlagOverrider to be used as a decorator or context manager. + """ + if not args: + return flag_overrider_cls(**kwargs) + # args can be [func] if used as `@flagsaver` instead of `@flagsaver(...)` + if len(args) == 1 and callable(args[0]): + if kwargs: + raise ValueError( + "It's invalid to specify both positional and keyword parameters.") + func = args[0] + if inspect.isclass(func): + raise TypeError('@flagsaver.flagsaver cannot be applied to a class.') + return _wrap(flag_overrider_cls, func, {}) + # args can be a list of (FlagHolder, value) pairs. + # In which case they augment any specified kwargs. + for arg in args: + if not isinstance(arg, tuple) or len(arg) != 2: + raise ValueError('Expected (FlagHolder, value) pair, found %r' % (arg,)) + holder, value = arg + if not isinstance(holder, flags.FlagHolder): + raise ValueError('Expected (FlagHolder, value) pair, found %r' % (arg,)) + if holder.name in kwargs: + raise ValueError('Cannot set --%s multiple times' % holder.name) + kwargs[holder.name] = value + return flag_overrider_cls(**kwargs) + + +def save_flag_values( + flag_values: flags.FlagValues = FLAGS, +) -> dict[str, dict[str, Any]]: + """Returns copy of flag values as a dict. + + Args: + flag_values: FlagValues, the FlagValues instance with which the flag will be + saved. This should almost never need to be overridden. + + Returns: + Dictionary mapping keys to values. Keys are flag names, values are + corresponding ``__dict__`` members. E.g. ``{'key': value_dict, ...}``. + """ + return {name: _copy_flag_dict(flag_values[name]) for name in flag_values} + + +def restore_flag_values( + saved_flag_values: Mapping[str, dict[str, Any]], + flag_values: flags.FlagValues = FLAGS, +) -> None: + """Restores flag values based on the dictionary of flag values. + + Args: + saved_flag_values: {'flag_name': value_dict, ...} + flag_values: FlagValues, the FlagValues instance from which the flag will be + restored. This should almost never need to be overridden. + """ + new_flag_names = list(flag_values) + for name in new_flag_names: + saved = saved_flag_values.get(name) + if saved is None: + # If __dict__ was not saved delete "new" flag. + delattr(flag_values, name) + else: + if flag_values[name].value != saved['_value']: + flag_values[name].value = saved['_value'] # Ensure C++ value is set. + flag_values[name].__dict__ = saved + + +@overload +def _wrap( + flag_overrider_cls: type['_FlagOverrider'], + func: _CallableT, + overrides: Mapping[str, Any], +) -> _CallableT: + ... + + +@overload +def _wrap( + flag_overrider_cls: type['_ParsingFlagOverrider'], + func: _CallableT, + overrides: Mapping[str, str | Sequence[str]], +) -> _CallableT: + ... + + +def _wrap(flag_overrider_cls, func, overrides): + """Creates a wrapper function that saves/restores flag values. + + Args: + flag_overrider_cls: The class that will be used as a context manager. + func: This will be called between saving flags and restoring flags. + overrides: Flag names mapped to their values. These flags will be set after + saving the original flag state. The type of the values depends on if + _FlagOverrider or _ParsingFlagOverrider was specified. + + Returns: + A wrapped version of func. + """ + + @functools.wraps(func) + def _flagsaver_wrapper(*args, **kwargs): + """Wrapper function that saves and restores flags.""" + with flag_overrider_cls(**overrides): + return func(*args, **kwargs) + + return _flagsaver_wrapper + + +class _FlagOverrider: + """Overrides flags for the duration of the decorated function call. + + It also restores all original values of flags after decorated method + completes. + """ + + def __init__(self, **overrides: Any): + self._overrides = overrides + self._saved_flag_values = None + + def __call__(self, func: _CallableT) -> _CallableT: + if inspect.isclass(func): + raise TypeError('flagsaver cannot be applied to a class.') + return _wrap(self.__class__, func, self._overrides) + + def __enter__(self): + self._saved_flag_values = save_flag_values(FLAGS) + try: + FLAGS._set_attributes(**self._overrides) + except: + # It may fail because of flag validators. + restore_flag_values(self._saved_flag_values, FLAGS) + raise + + def __exit__(self, exc_type, exc_value, traceback): + restore_flag_values(self._saved_flag_values, FLAGS) + + +class _ParsingFlagOverrider(_FlagOverrider): + """Context manager for overriding flags. + + Simulates command line parsing. + + This is simlar to _FlagOverrider except that all **overrides should be + strings or sequences of strings, and when context is entered this class calls + .parse(value) + + This results in the flags having .present set properly. + """ + + def __init__(self, **overrides: str | Sequence[str]): + for flag_name, new_value in overrides.items(): + if isinstance(new_value, str): + continue + if isinstance(new_value, Sequence) and all( + isinstance(single_value, str) for single_value in new_value + ): + continue + raise TypeError( + f'flagsaver.as_parsed() cannot parse {flag_name}. Expected a single ' + f'string or sequence of strings but {type(new_value)} was provided.') + super().__init__(**overrides) + + def __enter__(self): + self._saved_flag_values = save_flag_values(FLAGS) + try: + for flag_name, unparsed_value in self._overrides.items(): + # LINT.IfChange(flag_override_parsing) + FLAGS[flag_name].parse(unparsed_value) + FLAGS[flag_name].using_default_value = False + # LINT.ThenChange() + + # Perform the validation on all modified flags. This is something that + # FLAGS._set_attributes() does for you in _FlagOverrider. + for flag_name in self._overrides: + FLAGS._assert_validators(FLAGS[flag_name].validators) + + except KeyError as e: + # If a flag doesn't exist, an UnrecognizedFlagError is more specific. + restore_flag_values(self._saved_flag_values, FLAGS) + raise flags.UnrecognizedFlagError('Unknown command line flag.') from e + + except: + # It may fail because of flag validators or general parsing issues. + restore_flag_values(self._saved_flag_values, FLAGS) + raise + + +def _copy_flag_dict(flag: flags.Flag) -> dict[str, Any]: + """Returns a copy of the flag object's ``__dict__``. + + It's mostly a shallow copy of the ``__dict__``, except it also does a shallow + copy of the validator list. + + Args: + flag: flags.Flag, the flag to copy. + + Returns: + A copy of the flag object's ``__dict__``. + """ + copy = flag.__dict__.copy() + copy['_value'] = flag.value # Ensure correct restore for C++ flags. + copy['validators'] = list(flag.validators) + return copy diff --git a/venv/Lib/site-packages/absl/testing/parameterized.py b/venv/Lib/site-packages/absl/testing/parameterized.py new file mode 100644 index 0000000000000000000000000000000000000000..1ee91258a702aa6db489edc0d276492f3afd5386 --- /dev/null +++ b/venv/Lib/site-packages/absl/testing/parameterized.py @@ -0,0 +1,726 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Adds support for parameterized tests to Python's unittest TestCase class. + +A parameterized test is a method in a test case that is invoked with different +argument tuples. + +A simple example:: + + class AdditionExample(parameterized.TestCase): + @parameterized.parameters( + (1, 2, 3), + (4, 5, 9), + (1, 1, 3)) + def testAddition(self, op1, op2, result): + self.assertEqual(result, op1 + op2) + +Each invocation is a separate test case and properly isolated just +like a normal test method, with its own setUp/tearDown cycle. In the +example above, there are three separate testcases, one of which will +fail due to an assertion error (1 + 1 != 3). + +Parameters for individual test cases can be tuples (with positional parameters) +or dictionaries (with named parameters):: + + class AdditionExample(parameterized.TestCase): + @parameterized.parameters( + {'op1': 1, 'op2': 2, 'result': 3}, + {'op1': 4, 'op2': 5, 'result': 9}, + ) + def testAddition(self, op1, op2, result): + self.assertEqual(result, op1 + op2) + +If a parameterized test fails, the error message will show the +original test name and the parameters for that test. + +The id method of the test, used internally by the unittest framework, is also +modified to show the arguments (but note that the name reported by `id()` +doesn't match the actual test name, see below). To make sure that test names +stay the same across several invocations, object representations like:: + + >>> class Foo(object): + ... pass + >>> repr(Foo()) + '<__main__.Foo object at 0x23d8610>' + +are turned into ``__main__.Foo``. When selecting a subset of test cases to run +on the command-line, the test cases contain an index suffix for each argument +in the order they were passed to :func:`parameters` (eg. testAddition0, +testAddition1, etc.) This naming scheme is subject to change; for more reliable +and stable names, especially in test logs, use :func:`named_parameters` instead. + +Tests using :func:`named_parameters` are similar to :func:`parameters`, except +only tuples or dicts of args are supported. For tuples, the first parameter arg +has to be a string (or an object that returns an apt name when converted via +``str()``). For dicts, a value for the key ``testcase_name`` must be present and +must be a string (or an object that returns an apt name when converted via +``str()``):: + + class NamedExample(parameterized.TestCase): + @parameterized.named_parameters( + ('Normal', 'aa', 'aaa', True), + ('EmptyPrefix', '', 'abc', True), + ('BothEmpty', '', '', True)) + def testStartsWith(self, prefix, string, result): + self.assertEqual(result, string.startswith(prefix)) + + class NamedExample(parameterized.TestCase): + @parameterized.named_parameters( + {'testcase_name': 'Normal', + 'result': True, 'string': 'aaa', 'prefix': 'aa'}, + {'testcase_name': 'EmptyPrefix', + 'result': True, 'string': 'abc', 'prefix': ''}, + {'testcase_name': 'BothEmpty', + 'result': True, 'string': '', 'prefix': ''}) + def testStartsWith(self, prefix, string, result): + self.assertEqual(result, string.startswith(prefix)) + +Named tests also have the benefit that they can be run individually +from the command line:: + + $ testmodule.py NamedExample.testStartsWithNormal + . + -------------------------------------------------------------------- + Ran 1 test in 0.000s + + OK + +Parameterized Classes +===================== + +If invocation arguments are shared across test methods in a single +TestCase class, instead of decorating all test methods +individually, the class itself can be decorated:: + + @parameterized.parameters( + (1, 2, 3), + (4, 5, 9)) + class ArithmeticTest(parameterized.TestCase): + def testAdd(self, arg1, arg2, result): + self.assertEqual(arg1 + arg2, result) + + def testSubtract(self, arg1, arg2, result): + self.assertEqual(result - arg1, arg2) + +Inputs from Iterables +===================== + +If parameters should be shared across several test cases, or are dynamically +created from other sources, a single non-tuple iterable can be passed into +the decorator. This iterable will be used to obtain the test cases:: + + class AdditionExample(parameterized.TestCase): + @parameterized.parameters( + c.op1, c.op2, c.result for c in testcases + ) + def testAddition(self, op1, op2, result): + self.assertEqual(result, op1 + op2) + + +Single-Argument Test Methods +============================ + +If a test method takes only one argument, the single arguments must not be +wrapped into a tuple:: + + class NegativeNumberExample(parameterized.TestCase): + @parameterized.parameters( + -1, -3, -4, -5 + ) + def testIsNegative(self, arg): + self.assertTrue(IsNegative(arg)) + + +List/tuple as a Single Argument +=============================== + +If a test method takes a single argument of a list/tuple, it must be wrapped +inside a tuple:: + + class ZeroSumExample(parameterized.TestCase): + @parameterized.parameters( + ([-1, 0, 1], ), + ([-2, 0, 2], ), + ) + def testSumIsZero(self, arg): + self.assertEqual(0, sum(arg)) + + +Cartesian product of Parameter Values as Parameterized Test Cases +================================================================= + +If required to test method over a cartesian product of parameters, +`parameterized.product` may be used to facilitate generation of parameters +test combinations:: + + class TestModuloExample(parameterized.TestCase): + @parameterized.product( + num=[0, 20, 80], + modulo=[2, 4], + expected=[0] + ) + def testModuloResult(self, num, modulo, expected): + self.assertEqual(expected, num % modulo) + +This results in 6 test cases being created - one for each combination of the +parameters. It is also possible to supply sequences of keyword argument dicts +as elements of the cartesian product:: + + @parameterized.product( + (dict(num=5, modulo=3, expected=2), + dict(num=7, modulo=4, expected=3)), + dtype=(int, float) + ) + def testModuloResult(self, num, modulo, expected, dtype): + self.assertEqual(expected, dtype(num) % modulo) + +This results in 4 test cases being created - for each of the two sets of test +data (supplied as kwarg dicts) and for each of the two data types (supplied as +a named parameter). Multiple keyword argument dicts may be supplied if required. + +Async Support +============= + +If a test needs to call async functions, it can inherit from both +parameterized.TestCase and another TestCase that supports async calls, such +as [asynctest](https://github.com/Martiusweb/asynctest):: + + import asynctest + + class AsyncExample(parameterized.TestCase, asynctest.TestCase): + @parameterized.parameters( + ('a', 1), + ('b', 2), + ) + async def testSomeAsyncFunction(self, arg, expected): + actual = await someAsyncFunction(arg) + self.assertEqual(actual, expected) +""" + +from collections import abc +import functools +import inspect +import itertools +import re +import types +import unittest +import warnings + +from absl.testing import absltest + + +_ADDR_RE = re.compile(r'\<([a-zA-Z0-9_\-\.]+) object at 0x[a-fA-F0-9]+\>') +_NAMED = object() +_ARGUMENT_REPR = object() +_NAMED_DICT_KEY = 'testcase_name' + + +class NoTestsError(Exception): + """Raised when parameterized decorators do not generate any tests.""" + + +class DuplicateTestNameError(Exception): + """Raised when a parameterized test has the same test name multiple times.""" + + def __init__(self, test_class_name, new_test_name, original_test_name): + super().__init__( + 'Duplicate parameterized test name in {}: generated test name {!r} ' + '(generated from {!r}) already exists. Consider using ' + 'named_parameters() to give your tests unique names and/or renaming ' + 'the conflicting test method.'.format( + test_class_name, new_test_name, original_test_name + ) + ) + + +def _clean_repr(obj): + return _ADDR_RE.sub(r'<\1>', repr(obj)) + + +def _non_string_or_bytes_iterable(obj): + return (isinstance(obj, abc.Iterable) and not isinstance(obj, str) and + not isinstance(obj, bytes)) + + +def _format_parameter_list(testcase_params): + if isinstance(testcase_params, abc.Mapping): + return ', '.join('%s=%s' % (argname, _clean_repr(value)) + for argname, value in testcase_params.items()) + elif _non_string_or_bytes_iterable(testcase_params): + return ', '.join(map(_clean_repr, testcase_params)) + else: + return _format_parameter_list((testcase_params,)) + + +def _async_wrapped(func): + @functools.wraps(func) + async def wrapper(*args, **kwargs): + return await func(*args, **kwargs) + return wrapper + + +class _ParameterizedTestIter: + """Callable and iterable class for producing new test cases.""" + + def __init__(self, test_method, testcases, naming_type, original_name=None): + """Returns concrete test functions for a test and a list of parameters. + + The naming_type is used to determine the name of the concrete + functions as reported by the unittest framework. If naming_type is + _FIRST_ARG, the testcases must be tuples, and the first element must + have a string representation that is a valid Python identifier. + + Args: + test_method: The decorated test method. + testcases: (list of tuple/dict) A list of parameter tuples/dicts for + individual test invocations. + naming_type: The test naming type, either _NAMED or _ARGUMENT_REPR. + original_name: The original test method name. When decorated on a test + method, None is passed to __init__ and test_method.__name__ is used. + Note test_method.__name__ might be different than the original defined + test method because of the use of other decorators. A more accurate + value is set by TestGeneratorMetaclass.__new__ later. + """ + self._test_method = test_method + self.testcases = testcases + self._naming_type = naming_type + if original_name is None: + original_name = test_method.__name__ + self._original_name = original_name + self.__name__ = _ParameterizedTestIter.__name__ + + def __call__(self, *args, **kwargs): + raise RuntimeError('You appear to be running a parameterized test case ' + 'without having inherited from parameterized.' + 'TestCase. This is bad because none of ' + 'your test cases are actually being run. You may also ' + 'be using another decorator before the parameterized ' + 'one, in which case you should reverse the order.') + + def __iter__(self): + test_method = self._test_method + naming_type = self._naming_type + + def make_bound_param_test(testcase_params): + @functools.wraps(test_method) + def bound_param_test(self): + if isinstance(testcase_params, abc.Mapping): + return test_method(self, **testcase_params) + elif _non_string_or_bytes_iterable(testcase_params): + return test_method(self, *testcase_params) + else: + return test_method(self, testcase_params) + + if naming_type is _NAMED: + # Signal the metaclass that the name of the test function is unique + # and descriptive. + bound_param_test.__x_use_name__ = True + + testcase_name = None + if isinstance(testcase_params, abc.Mapping): + if _NAMED_DICT_KEY not in testcase_params: + raise RuntimeError( + 'Dict for named tests must contain key "%s"' % _NAMED_DICT_KEY) + # Create a new dict to avoid modifying the supplied testcase_params. + testcase_name = testcase_params[_NAMED_DICT_KEY] + testcase_params = { + k: v for k, v in testcase_params.items() if k != _NAMED_DICT_KEY + } + elif _non_string_or_bytes_iterable(testcase_params): + if not isinstance(testcase_params[0], str): + raise RuntimeError( + 'The first element of named test parameters is the test name ' + 'suffix and must be a string') + testcase_name = testcase_params[0] + testcase_params = testcase_params[1:] + else: + raise RuntimeError( + 'Named tests must be passed a dict or non-string iterable.') + + test_method_name = self._original_name + # Support PEP-8 underscore style for test naming if used. + if (test_method_name.startswith('test_') + and testcase_name + and not testcase_name.startswith('_')): + test_method_name += '_' + + bound_param_test.__name__ = test_method_name + str(testcase_name) + elif naming_type is _ARGUMENT_REPR: + # If it's a generator, convert it to a tuple and treat them as + # parameters. + if isinstance(testcase_params, types.GeneratorType): + testcase_params = tuple(testcase_params) + # The metaclass creates a unique, but non-descriptive method name for + # _ARGUMENT_REPR tests using an indexed suffix. + # To keep test names descriptive, only the original method name is used. + # To make sure test names are unique, we add a unique descriptive suffix + # __x_params_repr__ for every test. + params_repr = '(%s)' % (_format_parameter_list(testcase_params),) + bound_param_test.__x_params_repr__ = params_repr + else: + raise RuntimeError('%s is not a valid naming type.' % (naming_type,)) + + bound_param_test.__doc__ = '%s(%s)' % ( + bound_param_test.__name__, _format_parameter_list(testcase_params)) + if test_method.__doc__: + bound_param_test.__doc__ += '\n%s' % (test_method.__doc__,) + if inspect.iscoroutinefunction(test_method): + return _async_wrapped(bound_param_test) + return bound_param_test + + return (make_bound_param_test(c) for c in self.testcases) + + +def _modify_class(class_object, testcases, naming_type): + assert not getattr(class_object, '_test_params_reprs', None), ( + 'Cannot add parameters to %s. Either it already has parameterized ' + 'methods, or its super class is also a parameterized class.' % ( + class_object,)) + # NOTE: _test_params_repr is private to parameterized.TestCase and it's + # metaclass; do not use it outside of those classes. + class_object._test_params_reprs = test_params_reprs = {} + for name, obj in class_object.__dict__.copy().items(): + if (name.startswith(unittest.TestLoader.testMethodPrefix) + and isinstance(obj, types.FunctionType)): + delattr(class_object, name) + methods = {} + _update_class_dict_for_param_test_case( + class_object.__name__, methods, test_params_reprs, name, + _ParameterizedTestIter(obj, testcases, naming_type, name)) + for meth_name, meth in methods.items(): + setattr(class_object, meth_name, meth) + + +def _parameter_decorator(naming_type, testcases): + """Implementation of the parameterization decorators. + + Args: + naming_type: The naming type. + testcases: Testcase parameters. + + Raises: + NoTestsError: Raised when the decorator generates no tests. + + Returns: + A function for modifying the decorated object. + """ + def _apply(obj): + if isinstance(obj, type): + _modify_class(obj, testcases, naming_type) + return obj + else: + return _ParameterizedTestIter(obj, testcases, naming_type) + + if (len(testcases) == 1 and + not isinstance(testcases[0], tuple) and + not isinstance(testcases[0], abc.Mapping)): + # Support using a single non-tuple parameter as a list of test cases. + # Note that the single non-tuple parameter can't be Mapping either, which + # means a single dict parameter case. + assert _non_string_or_bytes_iterable(testcases[0]), ( + 'Single parameter argument must be a non-string non-Mapping iterable') + testcases = testcases[0] + + if not isinstance(testcases, abc.Sequence): + testcases = list(testcases) + if not testcases: + raise NoTestsError( + 'parameterized test decorators did not generate any tests. ' + 'Make sure you specify non-empty parameters, ' + 'and do not reuse generators more than once.') + + return _apply + + +def parameters(*testcases): + """A decorator for creating parameterized tests. + + See the module docstring for a usage example. + + Args: + *testcases: Parameters for the decorated method, either a single + iterable, or a list of tuples/dicts/objects (for tests with only one + argument). + + Raises: + NoTestsError: Raised when the decorator generates no tests. + + Returns: + A test generator to be handled by TestGeneratorMetaclass. + """ + return _parameter_decorator(_ARGUMENT_REPR, testcases) + + +def named_parameters(*testcases): + """A decorator for creating parameterized tests. + + See the module docstring for a usage example. For every parameter tuple + passed, the first element of the tuple should be a string and will be appended + to the name of the test method. Each parameter dict passed must have a value + for the key "testcase_name", the string representation of that value will be + appended to the name of the test method. + + Args: + *testcases: Parameters for the decorated method, either a single iterable, + or a list of tuples or dicts. + + Raises: + NoTestsError: Raised when the decorator generates no tests. + + Returns: + A test generator to be handled by TestGeneratorMetaclass. + """ + return _parameter_decorator(_NAMED, testcases) + + +def product(*kwargs_seqs, **testgrid): + """A decorator for running tests over cartesian product of parameters values. + + See the module docstring for a usage example. The test will be run for every + possible combination of the parameters. + + Args: + *kwargs_seqs: Each positional parameter is a sequence of keyword arg dicts; + every test case generated will include exactly one kwargs dict from each + positional parameter; these will then be merged to form an overall list + of arguments for the test case. + **testgrid: A mapping of parameter names and their possible values. Possible + values should given as either a list or a tuple. + + Raises: + NoTestsError: Raised when the decorator generates no tests. + + Returns: + A test generator to be handled by TestGeneratorMetaclass. + """ + + for name, values in testgrid.items(): + assert isinstance(values, (list, tuple)), ( + 'Values of {} must be given as list or tuple, found {}'.format( + name, type(values))) + + prior_arg_names = set() + for kwargs_seq in kwargs_seqs: + assert ((isinstance(kwargs_seq, (list, tuple))) and + all(isinstance(kwargs, dict) for kwargs in kwargs_seq)), ( + 'Positional parameters must be a sequence of keyword arg' + 'dicts, found {}' + .format(kwargs_seq)) + if kwargs_seq: + arg_names = set(kwargs_seq[0]) + assert all(set(kwargs) == arg_names for kwargs in kwargs_seq), ( + 'Keyword argument dicts within a single parameter must all have the ' + 'same keys, found {}'.format(kwargs_seq)) + assert not (arg_names & prior_arg_names), ( + 'Keyword argument dict sequences must all have distinct argument ' + 'names, found duplicate(s) {}' + .format(sorted(arg_names & prior_arg_names))) + prior_arg_names |= arg_names + + assert not (prior_arg_names & set(testgrid)), ( + 'Arguments supplied in kwargs dicts in positional parameters must not ' + 'overlap with arguments supplied as named parameters; found duplicate ' + 'argument(s) {}'.format(sorted(prior_arg_names & set(testgrid)))) + + # Convert testgrid into a sequence of sequences of kwargs dicts and combine + # with the positional parameters. + # So foo=[1,2], bar=[3,4] --> [[{foo: 1}, {foo: 2}], [{bar: 3, bar: 4}]] + testgrid = (tuple({k: v} for v in vs) for k, vs in testgrid.items()) + testgrid = tuple(kwargs_seqs) + tuple(testgrid) + + # Create all possible combinations of parameters as a cartesian product + # of parameter values. + testcases = [ + dict(itertools.chain.from_iterable(case.items() + for case in cases)) + for cases in itertools.product(*testgrid) + ] + return _parameter_decorator(_ARGUMENT_REPR, testcases) + + +class TestGeneratorMetaclass(type): + """Metaclass for adding tests generated by parameterized decorators.""" + + def __new__(cls, class_name, bases, dct): + # NOTE: _test_params_repr is private to parameterized.TestCase and it's + # metaclass; do not use it outside of those classes. + test_params_reprs = dct.setdefault('_test_params_reprs', {}) + for name, obj in dct.copy().items(): + if (name.startswith(unittest.TestLoader.testMethodPrefix) and + _non_string_or_bytes_iterable(obj)): + # NOTE: `obj` might not be a _ParameterizedTestIter in two cases: + # 1. a class-level iterable named test* that isn't a test, such as + # a list of something. Such attributes get deleted from the class. + # + # 2. If a decorator is applied to the parameterized test, e.g. + # @morestuff + # @parameterized.parameters(...) + # def test_foo(...): ... + # + # This is OK so long as the underlying parameterized function state + # is forwarded (e.g. using functool.wraps() and **without** + # accessing explicitly accessing the internal attributes. + if isinstance(obj, _ParameterizedTestIter): + # Update the original test method name so it's more accurate. + # The mismatch might happen when another decorator is used inside + # the parameterized decrators, and the inner decorator doesn't + # preserve its __name__. + obj._original_name = name + iterator = iter(obj) + dct.pop(name) + _update_class_dict_for_param_test_case( + class_name, dct, test_params_reprs, name, iterator) + # If the base class is a subclass of parameterized.TestCase, inherit its + # _test_params_reprs too. + for base in bases: + # Check if the base has _test_params_reprs first, then check if it's a + # subclass of parameterized.TestCase. Otherwise when this is called for + # the parameterized.TestCase definition itself, this raises because + # itself is not defined yet. This works as long as absltest.TestCase does + # not define _test_params_reprs. + base_test_params_reprs = getattr(base, '_test_params_reprs', None) + if base_test_params_reprs and issubclass(base, TestCase): + for test_method, test_method_id in base_test_params_reprs.items(): + # test_method may both exists in base and this class. + # This class's method overrides base class's. + # That's why it should only inherit it if it does not exist. + test_params_reprs.setdefault(test_method, test_method_id) + + return type.__new__(cls, class_name, bases, dct) + + +def _update_class_dict_for_param_test_case( + test_class_name, dct, test_params_reprs, name, iterator): + """Adds individual test cases to a dictionary. + + Args: + test_class_name: The name of the class tests are added to. + dct: The target dictionary. + test_params_reprs: The dictionary for mapping names to test IDs. + name: The original name of the test case. + iterator: The iterator generating the individual test cases. + + Raises: + DuplicateTestNameError: Raised when a test name occurs multiple times. + RuntimeError: If non-parameterized functions are generated. + """ + for idx, func in enumerate(iterator): + assert callable(func), 'Test generators must yield callables, got %r' % ( + func,) + if not (getattr(func, '__x_use_name__', None) or + getattr(func, '__x_params_repr__', None)): + raise RuntimeError( + '{}.{} generated a test function without using the parameterized ' + 'decorators. Only tests generated using the decorators are ' + 'supported.'.format(test_class_name, name)) + + if getattr(func, '__x_use_name__', False): + original_name = func.__name__ + new_name = original_name + else: + original_name = name + new_name = '%s%d' % (original_name, idx) + + if new_name in dct: + raise DuplicateTestNameError(test_class_name, new_name, original_name) + + dct[new_name] = func + test_params_reprs[new_name] = getattr(func, '__x_params_repr__', '') + + +class TestCase(absltest.TestCase, metaclass=TestGeneratorMetaclass): + """Base class for test cases using the parameters decorator.""" + + # visibility: private; do not call outside this class. + def _get_params_repr(self): + return self._test_params_reprs.get(self._testMethodName, '') + + def __str__(self): + params_repr = self._get_params_repr() + if params_repr: + params_repr = ' ' + params_repr + return '{}{} ({})'.format( + self._testMethodName, params_repr, + unittest.util.strclass(self.__class__)) + + def id(self): + """Returns the descriptive ID of the test. + + This is used internally by the unittesting framework to get a name + for the test to be used in reports. + + Returns: + The test id. + """ + base = super().id() + params_repr = self._get_params_repr() + if params_repr: + # We include the params in the id so that, when reported in the + # test.xml file, the value is more informative than just "test_foo0". + # Use a space to separate them so that it's copy/paste friendly and + # easy to identify the actual test id. + return f'{base} {params_repr}' + else: + return base + + +# This function is kept CamelCase because it's used as a class's base class. +def CoopTestCase(other_base_class) -> type: # pylint: disable=invalid-name, g-bare-generic + """Returns a new base class with a cooperative metaclass base. + + This enables the TestCase to be used in combination + with other base classes that have custom metaclasses, such as + ``mox.MoxTestBase``. + + Only works with metaclasses that do not override ``type.__new__``. + + Example:: + + from absl.testing import parameterized + + class ExampleTest(parameterized.CoopTestCase(OtherTestCase)): + ... + + Args: + other_base_class: (class) A test case base class. + + Returns: + A new class object. + """ + # If the other base class has a metaclass of 'type' then trying to combine + # the metaclasses will result in an MRO error. So simply combine them and + # return. + if type(other_base_class) == type: # pylint: disable=unidiomatic-typecheck + warnings.warn( + 'CoopTestCase is only necessary when combining with a class that uses' + ' a metaclass. Use multiple inheritance like this instead: class' + f' ExampleTest(paramaterized.TestCase, {other_base_class.__name__}):', + stacklevel=2, + ) + + class CoopTestCaseBase(other_base_class, TestCase): + pass + + return CoopTestCaseBase + else: + + class CoopMetaclass(type(other_base_class), TestGeneratorMetaclass): # type: ignore # pylint: disable=unused-variable + pass + + class CoopTestCaseBase(other_base_class, TestCase, metaclass=CoopMetaclass): # type: ignore + pass + + return CoopTestCaseBase diff --git a/venv/Lib/site-packages/absl/testing/xml_reporter.py b/venv/Lib/site-packages/absl/testing/xml_reporter.py new file mode 100644 index 0000000000000000000000000000000000000000..25114e94b1cb6a2565375ae861bb7be4a0344bf7 --- /dev/null +++ b/venv/Lib/site-packages/absl/testing/xml_reporter.py @@ -0,0 +1,570 @@ +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Python test reporter that generates test reports in JUnit XML format.""" + +import datetime +import re +import sys +import threading +import time +import traceback +from typing import Any +import unittest +from xml.sax import saxutils +from absl.testing import _pretty_print_reporter + + +# See http://www.w3.org/TR/REC-xml/#NT-Char +_bad_control_character_codes = set(range(0, 0x20)) - {0x9, 0xA, 0xD} + + +_control_character_conversions = { + chr(i): f'\\x{i:02x}' for i in _bad_control_character_codes +} + + +_escape_xml_attr_conversions = { + '"': '"', + "'": ''', + '\n': ' ', + '\t': ' ', + '\r': ' ', + ' ': ' '} +_escape_xml_attr_conversions.update(_control_character_conversions) + + +# When class or module level function fails, unittest/suite.py adds a +# _ErrorHolder instance instead of a real TestCase, and it has a description +# like "setUpClass (__main__.MyTestCase)". +_CLASS_OR_MODULE_LEVEL_TEST_DESC_REGEX = re.compile(r'^(\w+) \((\S+)\)$') + + +# NOTE: while saxutils.quoteattr() theoretically does the same thing; it +# seems to often end up being too smart for it's own good not escaping properly. +# This function is much more reliable. +def _escape_xml_attr(content): + """Escapes xml attributes.""" + # Note: saxutils doesn't escape the quotes. + return saxutils.escape(content, _escape_xml_attr_conversions) + + +def _escape_cdata(s): + """Escapes a string to be used as XML CDATA. + + CDATA characters are treated strictly as character data, not as XML markup, + but there are still certain restrictions on them. + + Args: + s: the string to be escaped. + Returns: + An escaped version of the input string. + """ + for char, escaped in _control_character_conversions.items(): + s = s.replace(char, escaped) + return s.replace(']]>', ']] >') + + +def _iso8601_timestamp(timestamp): + """Produces an ISO8601 datetime. + + Args: + timestamp: an Epoch based timestamp in seconds. + + Returns: + A iso8601 format timestamp if the input is a valid timestamp, None otherwise + """ + if timestamp is None or timestamp < 0: + return None + return datetime.datetime.fromtimestamp( + timestamp, tz=datetime.timezone.utc).isoformat() + + +def _print_xml_element_header(element, attributes, stream, indentation=''): + """Prints an XML header of an arbitrary element. + + Args: + element: element name (testsuites, testsuite, testcase) + attributes: 2-tuple list with (attributes, values) already escaped + stream: output stream to write test report XML to + indentation: indentation added to the element header + """ + stream.write('%s<%s' % (indentation, element)) + for attribute in attributes: + if (len(attribute) == 2 and attribute[0] is not None and + attribute[1] is not None): + stream.write(' %s="%s"' % (attribute[0], attribute[1])) + stream.write('>\n') + +# Copy time.time which ensures the real time is used internally. +# This prevents bad interactions with tests that stub out time. +_time_copy = time.time + + +def _safe_str(obj: object) -> str: + """Returns a string representation of an object.""" + try: + return str(obj) + except Exception: # pylint: disable=broad-except + return '' % ( + type(obj).__module__, + type(obj).__name__, + ) + + +class _TestCaseResult: + """Private helper for _TextAndXMLTestResult that represents a test result. + + Attributes: + test: A TestCase instance of an individual test method. + name: The name of the individual test method. + full_class_name: The full name of the test class. + run_time: The duration (in seconds) it took to run the test. + start_time: Epoch relative timestamp of when test started (in seconds) + errors: A list of error 4-tuples. Error tuple entries are + 1) a string identifier of either "failure" or "error" + 2) an exception_type + 3) an exception_message + 4) a string version of a sys.exc_info()-style tuple of values + ('error', err[0], err[1], self._exc_info_to_string(err)) + If the length of errors is 0, then the test is either passed or + skipped. + skip_reason: A string explaining why the test was skipped. + """ + + def __init__(self, test): + self.run_time = -1 + self.start_time = -1 + self.skip_reason = None + self.errors = [] + self.test = test + + # Parse the test id to get its test name and full class path. + # Unfortunately there is no better way of knowning the test and class. + # Worse, unittest uses _ErrorHandler instances to represent class / module + # level failures. + test_desc = test.id() or str(test) + # Check if it's something like "setUpClass (__main__.TestCase)". + match = _CLASS_OR_MODULE_LEVEL_TEST_DESC_REGEX.match(test_desc) + if match: + name = match.group(1) + full_class_name = match.group(2) + else: + class_name = unittest.util.strclass(test.__class__) + if isinstance(test, unittest.case._SubTest): + # If the test case is a _SubTest, the real TestCase instance is + # available as _SubTest.test_case. + class_name = unittest.util.strclass(test.test_case.__class__) + if test_desc.startswith(class_name + '.'): + # In a typical unittest.TestCase scenario, test.id() returns with + # a class name formatted using unittest.util.strclass. + name = test_desc[len(class_name)+1:] + full_class_name = class_name + else: + # Otherwise make a best effort to guess the test name and full class + # path. + parts = test_desc.rsplit('.', 1) + name = parts[-1] + full_class_name = parts[0] if len(parts) == 2 else '' + self.name = _escape_xml_attr(name) + self.full_class_name = _escape_xml_attr(full_class_name) + + def set_run_time(self, time_in_secs): + self.run_time = time_in_secs + + def set_start_time(self, time_in_secs): + self.start_time = time_in_secs + + def print_xml_summary(self, stream): + """Prints an XML Summary of a TestCase. + + Status and result are populated as per JUnit XML test result reporter. + A test that has been skipped will always have a skip reason, + as every skip method in Python's unittest requires the reason arg to be + passed. + + Args: + stream: output stream to write test report XML to + """ + + if self.skip_reason is None: + status = 'run' + result = 'completed' + else: + status = 'notrun' + result = 'suppressed' + + test_case_attributes = [ + ('name', '%s' % self.name), + ('status', '%s' % status), + ('result', '%s' % result), + ('time', '%.3f' % self.run_time), + ('classname', self.full_class_name), + ('timestamp', _iso8601_timestamp(self.start_time)), + ] + _print_xml_element_header('testcase', test_case_attributes, stream, ' ') + self._print_testcase_details(stream) + stream.write(' \n') + + def _print_testcase_details(self, stream): + for error in self.errors: + outcome, exception_type, message, error_msg = error # pylint: disable=unpacking-non-sequence + message = _escape_xml_attr(_safe_str(message)) + exception_type = _escape_xml_attr(str(exception_type)) + error_msg = _escape_cdata(error_msg) + stream.write(' <%s message="%s" type="%s">\n' + % (outcome, message, exception_type, error_msg, outcome)) + + +class _TestSuiteResult: + """Private helper for _TextAndXMLTestResult.""" + + def __init__(self): + self.suites = {} + self.failure_counts = {} + self.error_counts = {} + self.overall_start_time = -1 + self.overall_end_time = -1 + self._testsuites_properties = {} + + def add_test_case_result(self, test_case_result): + suite_name = type(test_case_result.test).__name__ + if suite_name == '_ErrorHolder': + # _ErrorHolder is a special case created by unittest for class / module + # level functions. + suite_name = test_case_result.full_class_name.rsplit('.')[-1] + if isinstance(test_case_result.test, unittest.case._SubTest): + # If the test case is a _SubTest, the real TestCase instance is + # available as _SubTest.test_case. + suite_name = type(test_case_result.test.test_case).__name__ + + self._setup_test_suite(suite_name) + self.suites[suite_name].append(test_case_result) + for error in test_case_result.errors: + # Only count the first failure or error so that the sum is equal to the + # total number of *testcases* that have failures or errors. + if error[0] == 'failure': + self.failure_counts[suite_name] += 1 + break + elif error[0] == 'error': + self.error_counts[suite_name] += 1 + break + + def print_xml_summary(self, stream): + overall_test_count = sum(len(x) for x in self.suites.values()) + overall_failures = sum(self.failure_counts.values()) + overall_errors = sum(self.error_counts.values()) + overall_attributes = [ + ('name', ''), + ('tests', '%d' % overall_test_count), + ('failures', '%d' % overall_failures), + ('errors', '%d' % overall_errors), + ('time', '%.3f' % (self.overall_end_time - self.overall_start_time)), + ('timestamp', _iso8601_timestamp(self.overall_start_time)), + ] + _print_xml_element_header('testsuites', overall_attributes, stream) + if self._testsuites_properties: + stream.write(' \n') + for name, value in sorted(self._testsuites_properties.items()): + stream.write(' \n' % + (_escape_xml_attr(name), _escape_xml_attr(str(value)))) + stream.write(' \n') + + for suite_name in self.suites: + suite = self.suites[suite_name] + suite_end_time = max(x.start_time + x.run_time for x in suite) + suite_start_time = min(x.start_time for x in suite) + failures = self.failure_counts[suite_name] + errors = self.error_counts[suite_name] + suite_attributes = [ + ('name', '%s' % suite_name), + ('tests', '%d' % len(suite)), + ('failures', '%d' % failures), + ('errors', '%d' % errors), + ('time', '%.3f' % (suite_end_time - suite_start_time)), + ('timestamp', _iso8601_timestamp(suite_start_time)), + ] + _print_xml_element_header('testsuite', suite_attributes, stream) + + # test_case_result entries are not guaranteed to be in any user-friendly + # order, especially when using subtests. So sort them. + for test_case_result in sorted(suite, key=lambda t: t.name): + test_case_result.print_xml_summary(stream) + stream.write('\n') + stream.write('\n') + + def _setup_test_suite(self, suite_name): + """Adds a test suite to the set of suites tracked by this test run. + + Args: + suite_name: string, The name of the test suite being initialized. + """ + if suite_name in self.suites: + return + self.suites[suite_name] = [] + self.failure_counts[suite_name] = 0 + self.error_counts[suite_name] = 0 + + def set_end_time(self, timestamp_in_secs): + """Sets the start timestamp of this test suite. + + Args: + timestamp_in_secs: timestamp in seconds since epoch + """ + self.overall_end_time = timestamp_in_secs + + def set_start_time(self, timestamp_in_secs): + """Sets the end timestamp of this test suite. + + Args: + timestamp_in_secs: timestamp in seconds since epoch + """ + self.overall_start_time = timestamp_in_secs + + +class _TextAndXMLTestResult(_pretty_print_reporter.TextTestResult): + """Private TestResult class that produces both formatted text results and XML. + + Used by TextAndXMLTestRunner. + """ + + _TEST_SUITE_RESULT_CLASS = _TestSuiteResult + _TEST_CASE_RESULT_CLASS = _TestCaseResult + + def __init__(self, xml_stream, stream, descriptions, verbosity, + time_getter=_time_copy, testsuites_properties=None): + super().__init__(stream, descriptions, verbosity) + self.xml_stream = xml_stream + self.pending_test_case_results = {} + self.suite = self._TEST_SUITE_RESULT_CLASS() + if testsuites_properties: + self.suite._testsuites_properties = testsuites_properties + self.time_getter = time_getter + + # This lock guards any mutations on pending_test_case_results. + self._pending_test_case_results_lock = threading.RLock() + + def startTest(self, test): + self.start_time = self.time_getter() + super().startTest(test) + + def stopTest(self, test): + # Grabbing the write lock to avoid conflicting with stopTestRun. + with self._pending_test_case_results_lock: + super().stopTest(test) + result = self.get_pending_test_case_result(test) + if not result: + test_name = test.id() or str(test) + sys.stderr.write('No pending test case: %s\n' % test_name) + return + if getattr(self, 'start_time', None) is None: + # startTest may not be called for skipped tests since Python 3.12.1. + self.start_time = self.time_getter() + test_id = id(test) + run_time = self.time_getter() - self.start_time + result.set_run_time(run_time) + result.set_start_time(self.start_time) + self.suite.add_test_case_result(result) + del self.pending_test_case_results[test_id] + + def startTestRun(self): + self.suite.set_start_time(self.time_getter()) + super().startTestRun() + + def stopTestRun(self): + self.suite.set_end_time(self.time_getter()) + # All pending_test_case_results will be added to the suite and removed from + # the pending_test_case_results dictionary. Grabbing the write lock to avoid + # results from being added during this process to avoid duplicating adds or + # accidentally erasing newly appended pending results. + with self._pending_test_case_results_lock: + # Errors in the test fixture (setUpModule, tearDownModule, + # setUpClass, tearDownClass) can leave a pending result which + # never gets added to the suite. The runner calls stopTestRun + # which gives us an opportunity to add these errors for + # reporting here. + for test_id in self.pending_test_case_results: + result = self.pending_test_case_results[test_id] + if getattr(self, 'start_time', None) is not None: + run_time = self.suite.overall_end_time - self.start_time + result.set_run_time(run_time) + result.set_start_time(self.start_time) + self.suite.add_test_case_result(result) + self.pending_test_case_results.clear() + + def _exc_info_to_string(self, err, test=None): + """Converts a sys.exc_info()-style tuple of values into a string. + + This method must be overridden because the method signature in + unittest.TestResult changed between Python 2.2 and 2.4. + + Args: + err: A sys.exc_info() tuple of values for an error. + test: The test method. + + Returns: + A formatted exception string. + """ + if test: + return super()._exc_info_to_string(err, test) + return ''.join(traceback.format_exception(*err)) + + def add_pending_test_case_result(self, test, error_summary=None, + skip_reason=None): + """Adds result information to a test case result which may still be running. + + If a result entry for the test already exists, add_pending_test_case_result + will add error summary tuples and/or overwrite skip_reason for the result. + If it does not yet exist, a result entry will be created. + Note that a test result is considered to have been run and passed + only if there are no errors or skip_reason. + + Args: + test: A test method as defined by unittest + error_summary: A 4-tuple with the following entries: + 1) a string identifier of either "failure" or "error" + 2) an exception_type + 3) an exception_message + 4) a string version of a sys.exc_info()-style tuple of values + ('error', err[0], err[1], self._exc_info_to_string(err)) + If the length of errors is 0, then the test is either passed or + skipped. + skip_reason: a string explaining why the test was skipped + """ + with self._pending_test_case_results_lock: + test_id = id(test) + if test_id not in self.pending_test_case_results: + self.pending_test_case_results[test_id] = self._TEST_CASE_RESULT_CLASS( + test) + if error_summary: + self.pending_test_case_results[test_id].errors.append(error_summary) + if skip_reason: + self.pending_test_case_results[test_id].skip_reason = skip_reason + + def delete_pending_test_case_result(self, test): + with self._pending_test_case_results_lock: + test_id = id(test) + del self.pending_test_case_results[test_id] + + def get_pending_test_case_result(self, test): + test_id = id(test) + return self.pending_test_case_results.get(test_id, None) + + def addSuccess(self, test): + super().addSuccess(test) + self.add_pending_test_case_result(test) + + def addError(self, test, err): + super().addError(test, err) + error_summary = ('error', err[0], err[1], + self._exc_info_to_string(err, test=test)) + self.add_pending_test_case_result(test, error_summary=error_summary) + + def addFailure(self, test, err): + super().addFailure(test, err) + error_summary = ('failure', err[0], err[1], + self._exc_info_to_string(err, test=test)) + self.add_pending_test_case_result(test, error_summary=error_summary) + + def addSkip(self, test, reason): + super().addSkip(test, reason) + self.add_pending_test_case_result(test, skip_reason=reason) + + def addExpectedFailure(self, test, err): + super().addExpectedFailure(test, err) + if callable(getattr(test, 'recordProperty', None)): + test.recordProperty('EXPECTED_FAILURE', + self._exc_info_to_string(err, test=test)) + self.add_pending_test_case_result(test) + + def addUnexpectedSuccess(self, test): + super().addUnexpectedSuccess(test) + test_name = test.id() or str(test) + error_summary = ('error', '', '', + 'Test case %s should have failed, but passed.' + % (test_name)) + self.add_pending_test_case_result(test, error_summary=error_summary) + + def addSubTest(self, test, subtest, err): # pylint: disable=invalid-name + super().addSubTest(test, subtest, err) + if err is not None: + if issubclass(err[0], test.failureException): + error_summary = ('failure', err[0], err[1], + self._exc_info_to_string(err, test=test)) + else: + error_summary = ('error', err[0], err[1], + self._exc_info_to_string(err, test=test)) + else: + error_summary = None + self.add_pending_test_case_result(subtest, error_summary=error_summary) + + def printErrors(self): + super().printErrors() + self.xml_stream.write('\n') + self.suite.print_xml_summary(self.xml_stream) + + +class TextAndXMLTestRunner(unittest.TextTestRunner): + """A test runner that produces both formatted text results and XML. + + It prints out the names of tests as they are run, errors as they + occur, and a summary of the results at the end of the test run. + """ + + _TEST_RESULT_CLASS = _TextAndXMLTestResult + + _xml_stream = None + _testsuites_properties: dict[Any, Any] = {} + + def __init__(self, xml_stream=None, *args, **kwargs): + """Initialize a TextAndXMLTestRunner. + + Args: + xml_stream: file-like or None; XML-formatted test results are output + via this object's write() method. If None (the default), the + new instance behaves as described in the set_default_xml_stream method + documentation below. + *args: passed unmodified to unittest.TextTestRunner.__init__. + **kwargs: passed unmodified to unittest.TextTestRunner.__init__. + """ + super().__init__(*args, **kwargs) + if xml_stream is not None: + self._xml_stream = xml_stream + # else, do not set self._xml_stream to None -- this allows implicit fallback + # to the class attribute's value. + + @classmethod + def set_default_xml_stream(cls, xml_stream): + """Sets the default XML stream for the class. + + Args: + xml_stream: file-like or None; used for instances when xml_stream is None + or not passed to their constructors. If None is passed, instances + created with xml_stream=None will act as ordinary TextTestRunner + instances; this is the default state before any calls to this method + have been made. + """ + cls._xml_stream = xml_stream + + def _makeResult(self): + if self._xml_stream is None: + return super()._makeResult() + else: + return self._TEST_RESULT_CLASS( + self._xml_stream, self.stream, self.descriptions, self.verbosity, + testsuites_properties=self._testsuites_properties) + + @classmethod + def set_testsuites_property(cls, key, value): + cls._testsuites_properties[key] = value diff --git a/venv/Lib/site-packages/absl_py-2.4.0.dist-info/INSTALLER b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/venv/Lib/site-packages/absl_py-2.4.0.dist-info/METADATA b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..0cf2d721b265bf32a2ff733b34f8d2915c2c1aa6 --- /dev/null +++ b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/METADATA @@ -0,0 +1,100 @@ +Metadata-Version: 2.4 +Name: absl-py +Version: 2.4.0 +Summary: Abseil Python Common Libraries, see https://github.com/abseil/abseil-py. +Project-URL: Changelog, https://github.com/abseil/abseil-py/blob/main/CHANGELOG.md +Project-URL: Documentation, https://abseil.io/docs/python/ +Project-URL: Issues, https://github.com/abseil/abseil-py/issues +Project-URL: Source, https://github.com/abseil/abseil-py +Author: The Abseil Authors +License-Expression: Apache-2.0 +License-File: AUTHORS +License-File: LICENSE +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.10 +Description-Content-Type: text/markdown + +[![Package version](https://img.shields.io/pypi/v/absl-py)](https://pypi.org/project/absl-py) +[![Supported Python versions](https://img.shields.io/pypi/pyversions/absl-py.svg?style=flat-square)](https://pypi.org/project/absl-py) +[![License](https://img.shields.io/github/license/abseil/abseil-py)](https://github.com/abseil/abseil-py/blob/main/LICENSE) +[![Build Status](https://github.com/abseil/abseil-py/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/abseil/abseil-py/actions) +[![Overall downloads](https://pepy.tech/badge/absl-py)](https://pepy.tech/project/absl-py) +[![Last month downloads](https://pepy.tech/badge/absl-py/month)](https://pepy.tech/project/absl-py) + +# Abseil Python Common Libraries + +This repository is a collection of Python library code for building Python +applications. The code is collected from Google's own Python code base, and has +been extensively tested and used in production. + +## Features + +* Simple application startup +* Distributed commandline flags system +* Custom logging module with additional features +* Testing utilities + +## Getting Started + +### Installation + +To install the package, simply run: + +```bash +pip install absl-py +``` + +Or install from source: + +```bash +pip install . +``` + +### Running Tests + +To run Abseil tests, you can clone the git repo and run +[bazel](https://bazel.build/): + +```bash +git clone https://github.com/abseil/abseil-py.git +cd abseil-py +bazel test absl/... +``` + +Please also validate the type annotations against the latest mypy: + +```bash +pip install mypy +mypy absl +``` + +### Example Code + +Please refer to +[smoke_tests/sample_app.py](https://github.com/abseil/abseil-py/blob/main/smoke_tests/sample_app.py) +as an example to get started. + +## Documentation + +See the [Abseil Python Developer Guide](https://abseil.io/docs/python/). + +## Future Releases + +The current repository includes an initial set of libraries for early adoption. +More components and interoperability with Abseil C++ Common Libraries +will come in future releases. + +## License + +The Abseil Python library is licensed under the terms of the Apache +license. See [LICENSE](LICENSE) for more information. diff --git a/venv/Lib/site-packages/absl_py-2.4.0.dist-info/RECORD b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..1408a4a15328ee1d8bf7d52037c57ca5596f1ef7 --- /dev/null +++ b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/RECORD @@ -0,0 +1,53 @@ +absl/__init__.py,sha256=tQJRn_5rbQ5gWoXdkGAV-fJSGFCOGH6fFBI1NxLx5ho,607 +absl/__pycache__/__init__.cpython-311.pyc,, +absl/__pycache__/app.cpython-311.pyc,, +absl/__pycache__/command_name.cpython-311.pyc,, +absl/app.py,sha256=zC4tAKcK054F7kSrR7pzSCEDqNcJKLVb2AkvjIlmvl8,17065 +absl/app.pyi,sha256=YTcQS7o1DBFyFNgMpdykmvP3g4OljSwp3BuX9sSSLIM,1814 +absl/command_name.py,sha256=FgWUMHmlX0yQxEuMPXxFxn5ayWWZSLB0cq8Vx361TpU,2283 +absl/flags/__init__.py,sha256=n_uLeSK-15_1DKLb0wXivmt6A8xvADDC87D1yoJdqVU,7665 +absl/flags/__pycache__/__init__.cpython-311.pyc,, +absl/flags/__pycache__/_argument_parser.cpython-311.pyc,, +absl/flags/__pycache__/_defines.cpython-311.pyc,, +absl/flags/__pycache__/_exceptions.cpython-311.pyc,, +absl/flags/__pycache__/_flag.cpython-311.pyc,, +absl/flags/__pycache__/_flagvalues.cpython-311.pyc,, +absl/flags/__pycache__/_helpers.cpython-311.pyc,, +absl/flags/__pycache__/_validators.cpython-311.pyc,, +absl/flags/__pycache__/_validators_classes.cpython-311.pyc,, +absl/flags/__pycache__/argparse_flags.cpython-311.pyc,, +absl/flags/_argument_parser.py,sha256=XFIDxu1I8NjgBZhUdzSLUwDHhXFvOC-SbI8FsVmWZ_0,20531 +absl/flags/_defines.py,sha256=P3DLHYAaSgXSBlRgBN6QOS6oRcNUYNtzeapL5EQJ5LA,52499 +absl/flags/_exceptions.py,sha256=FZzlzhvkjqPImTxXqbS1pSPYKr_TvtOd5ellvoiVLDI,3619 +absl/flags/_flag.py,sha256=fOv_KC7t1e-BoabTP3iYHR1dUkTf2yhgfAWonjhByEQ,19891 +absl/flags/_flagvalues.py,sha256=QqCMTxHIJdwqYwlYhZ76nuXzsJQtF20RTzq-x0S_tPg,55319 +absl/flags/_helpers.py,sha256=0cWFV4XFLkcHWu-du6es8sCsWl26k1dZ6fqCPsWZ68k,13301 +absl/flags/_validators.py,sha256=VcsJtZzohliNxsI974NECYpeozD8rswHNHXggrQ4BLo,14140 +absl/flags/_validators_classes.py,sha256=PGUWzO7v3wPOHb9leIKKzry3q-pPeKCoMB_O7prLdnY,6093 +absl/flags/argparse_flags.py,sha256=usJudgMpy3P6Vvq7-LmJNa2Rj3ygHM3hwDTGd1mbAzc,14386 +absl/logging/__init__.py,sha256=3quWtll8Cb949CYs-VOC1AnxdBqClX8ab55VB28knVs,43883 +absl/logging/__init__.pyi,sha256=7yyq5rAJfkBFRaD8iqOTsLW-xhMAIDJ5ub4r3Has04U,5984 +absl/logging/__pycache__/__init__.cpython-311.pyc,, +absl/logging/__pycache__/converter.cpython-311.pyc,, +absl/logging/converter.py,sha256=6eBymfv9UNkog0BGat4HPWlxC_oSqvHcQ46jnSdtaMg,6323 +absl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +absl/testing/__init__.py,sha256=7cM57swk2T1Hc5wxmt-JpcaR6xfdPJyL_lyRqgODvuM,584 +absl/testing/__pycache__/__init__.cpython-311.pyc,, +absl/testing/__pycache__/_bazelize_command.cpython-311.pyc,, +absl/testing/__pycache__/_pretty_print_reporter.cpython-311.pyc,, +absl/testing/__pycache__/absltest.cpython-311.pyc,, +absl/testing/__pycache__/flagsaver.cpython-311.pyc,, +absl/testing/__pycache__/parameterized.cpython-311.pyc,, +absl/testing/__pycache__/xml_reporter.cpython-311.pyc,, +absl/testing/_bazelize_command.py,sha256=qpioV02ln2sBBJ9kdlHgNpKk8_wxdz2hJGKbG6EWZMI,2287 +absl/testing/_pretty_print_reporter.py,sha256=PZh9NXSXBbXDi0FOk-BOmpse8LXa92Er16tgyBRogMs,3065 +absl/testing/absltest.py,sha256=CkUgJ54S4LJ1RwjNISv4pEW_N9kZBrNDpnzQ1YGPOlc,104586 +absl/testing/flagsaver.py,sha256=BeftLpNxJctYtd041tECgCSglQVFKSsQ4KBFT9D_m5k,13291 +absl/testing/parameterized.py,sha256=TpTlWTUXjikGeUDE45AgubvnmHsPbFNj-wUtAwT-e6E,27817 +absl/testing/xml_reporter.py,sha256=Q0keNHVOh3L9zXwNIzWCjXcWYlCM9ZJxdjMaYyHF3dw,21424 +absl_py-2.4.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +absl_py-2.4.0.dist-info/METADATA,sha256=Eh8VboTinee1rj1-UDeBjw_KiicKBxJBi6H0zB1Sq0g,3283 +absl_py-2.4.0.dist-info/RECORD,, +absl_py-2.4.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87 +absl_py-2.4.0.dist-info/licenses/AUTHORS,sha256=YoLudsylaQg7W5mLn4FroQMuEnuNx8RpQrhkd_xvv6U,296 +absl_py-2.4.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358 diff --git a/venv/Lib/site-packages/absl_py-2.4.0.dist-info/WHEEL b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..ae8ec1bdaa94d726ceb907542d76cbd5d38cafcd --- /dev/null +++ b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.28.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/venv/Lib/site-packages/absl_py-2.4.0.dist-info/licenses/AUTHORS b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/licenses/AUTHORS new file mode 100644 index 0000000000000000000000000000000000000000..23b11ada16bb8e69695cf52e5994784d98054e0d --- /dev/null +++ b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/licenses/AUTHORS @@ -0,0 +1,7 @@ +# This is the list of Abseil authors for copyright purposes. +# +# This does not necessarily list everyone who has contributed code, since in +# some cases, their employer may be the copyright holder. To see the full list +# of contributors, see the revision history in source control. + +Google Inc. diff --git a/venv/Lib/site-packages/absl_py-2.4.0.dist-info/licenses/LICENSE b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/venv/Lib/site-packages/absl_py-2.4.0.dist-info/licenses/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/INSTALLER b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/METADATA b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..c8a61b093c877752873e1eea4ae05af62b0171e3 --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/METADATA @@ -0,0 +1,123 @@ +Metadata-Version: 2.4 +Name: aiohappyeyeballs +Version: 2.6.2 +Summary: Happy Eyeballs for asyncio +License: PSF-2.0 +License-File: LICENSE +Author: J. Nick Koston +Author-email: nick@koston.org +Requires-Python: >=3.10 +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Topic :: Software Development :: Libraries +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: License :: OSI Approved :: Python Software Foundation License +Project-URL: Bug Tracker, https://github.com/aio-libs/aiohappyeyeballs/issues +Project-URL: Changelog, https://github.com/aio-libs/aiohappyeyeballs/blob/main/CHANGELOG.md +Project-URL: Documentation, https://aiohappyeyeballs.readthedocs.io +Project-URL: Repository, https://github.com/aio-libs/aiohappyeyeballs +Description-Content-Type: text/markdown + +# aiohappyeyeballs + +

+ + CI Status + + + Documentation Status + + + Test coverage percentage + +

+

+ + Poetry + + + Ruff + + + pre-commit + +

+

+ + PyPI Version + + Supported Python versions + License +

+ +--- + +**Documentation**: https://aiohappyeyeballs.readthedocs.io + +**Source Code**: https://github.com/aio-libs/aiohappyeyeballs + +--- + +[Happy Eyeballs](https://en.wikipedia.org/wiki/Happy_Eyeballs) +([RFC 8305](https://www.rfc-editor.org/rfc/rfc8305.html)) + +## Use case + +This library exists to allow connecting with +[Happy Eyeballs](https://en.wikipedia.org/wiki/Happy_Eyeballs) +([RFC 8305](https://www.rfc-editor.org/rfc/rfc8305.html)) +when you +already have a list of addrinfo and not a DNS name. + +The stdlib version of `loop.create_connection()` +will only work when you pass in an unresolved name which +is not a good fit when using DNS caching or resolving +names via another method such as `zeroconf`. + +## Installation + +Install this via pip (or your favourite package manager): + +`pip install aiohappyeyeballs` + +## License + +[aiohappyeyeballs is licensed under the same terms as cpython itself.](https://github.com/python/cpython/blob/main/LICENSE) + +## Example usage + +```python + +addr_infos = await loop.getaddrinfo("example.org", 80) + +socket = await start_connection(addr_infos) +socket = await start_connection(addr_infos, local_addr_infos=local_addr_infos, happy_eyeballs_delay=0.2) + +transport, protocol = await loop.create_connection( + MyProtocol, sock=socket, ...) + +# Remove the first address for each family from addr_info +pop_addr_infos_interleave(addr_info, 1) + +# Remove all matching address from addr_info +remove_addr_infos(addr_info, "dead::beef::") + +# Convert a local_addr to local_addr_infos +local_addr_infos = addr_to_addr_infos(("127.0.0.1",0)) +``` + +## Credits + +This package contains code from cpython and is licensed under the same terms as cpython itself. + +This package was created with +[Copier](https://copier.readthedocs.io/) and the +[browniebroke/pypackage-template](https://github.com/browniebroke/pypackage-template) +project template. + diff --git a/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/RECORD b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..dd64ef3355c9619f47cc63e549cdeed6c0f72eab --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/RECORD @@ -0,0 +1,16 @@ +aiohappyeyeballs-2.6.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +aiohappyeyeballs-2.6.2.dist-info/METADATA,sha256=cqs2VY8TwE2e_4qqnW409Si3suvj-aM-8dCiWG2Angk,5888 +aiohappyeyeballs-2.6.2.dist-info/RECORD,, +aiohappyeyeballs-2.6.2.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88 +aiohappyeyeballs-2.6.2.dist-info/licenses/LICENSE,sha256=Oy-B_iHRgcSZxZolbI4ZaEVdZonSaaqFNzv7avQdo78,13936 +aiohappyeyeballs/__init__.py,sha256=Af9ADZj3BWLfGaA7ITOFPlKIenh3ozSNWL3yezSZ2Jw,361 +aiohappyeyeballs/__pycache__/__init__.cpython-311.pyc,, +aiohappyeyeballs/__pycache__/_staggered.cpython-311.pyc,, +aiohappyeyeballs/__pycache__/impl.cpython-311.pyc,, +aiohappyeyeballs/__pycache__/types.cpython-311.pyc,, +aiohappyeyeballs/__pycache__/utils.cpython-311.pyc,, +aiohappyeyeballs/_staggered.py,sha256=aj3cSwHEDX88UMfO9bUau9tfrRAszhjg99dpEMiAOGM,6698 +aiohappyeyeballs/impl.py,sha256=TIkAK4xfACvKBp1s7DAKSobHefUTOC2HGE-n0tOthRk,9667 +aiohappyeyeballs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +aiohappyeyeballs/types.py,sha256=_8JmHFix6MeM1e7hRP7BleEaGy93GswGtzQv068zKY8,288 +aiohappyeyeballs/utils.py,sha256=dPAcNcrU_VhaTolTEEA94hgh9ONDN9_dYT4Xo9ORqQw,2922 diff --git a/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/WHEEL b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..6a352135a5549b2935f6d31e32ddf0f3f62b7d85 --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: poetry-core 2.4.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/licenses/LICENSE b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f26bcf4d2de6eb136e31006ca3ab447d5e488adf --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/licenses/LICENSE @@ -0,0 +1,279 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see https://opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +Python software and documentation are licensed under the +Python Software Foundation License Version 2. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Version 2 +and the Zero-Clause BSD license. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. + + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; +All Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION +---------------------------------------------------------------------- + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/venv/Lib/site-packages/aiohappyeyeballs/__init__.py b/venv/Lib/site-packages/aiohappyeyeballs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ad6bfb5a073f17d54395af7696b8c3fa312e1abb --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs/__init__.py @@ -0,0 +1,14 @@ +__version__ = "2.6.2" + +from .impl import start_connection +from .types import AddrInfoType, SocketFactoryType +from .utils import addr_to_addr_infos, pop_addr_infos_interleave, remove_addr_infos + +__all__ = ( + "AddrInfoType", + "SocketFactoryType", + "addr_to_addr_infos", + "pop_addr_infos_interleave", + "remove_addr_infos", + "start_connection", +) diff --git a/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3db943f9573eabe48add405e6d8960177f445926 Binary files /dev/null and b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/_staggered.cpython-311.pyc b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/_staggered.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b777d3ebcc4a8f24883a4390bf23216f192e21d Binary files /dev/null and b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/_staggered.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/impl.cpython-311.pyc b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/impl.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf0ee828b85f9e13818e293cfe6fc0f6b140b5fb Binary files /dev/null and b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/impl.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/types.cpython-311.pyc b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/types.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a2fb25bdc6ef6111d6449dff229e0232aac3261 Binary files /dev/null and b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/types.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/utils.cpython-311.pyc b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4dac9ae8c5c3a6ff0bbf3682307b4233fbb6ca0c Binary files /dev/null and b/venv/Lib/site-packages/aiohappyeyeballs/__pycache__/utils.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohappyeyeballs/_staggered.py b/venv/Lib/site-packages/aiohappyeyeballs/_staggered.py new file mode 100644 index 0000000000000000000000000000000000000000..bb745dc0a81460327d6cc19e5bd38cc44f296fe5 --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs/_staggered.py @@ -0,0 +1,197 @@ +import asyncio +import contextlib +from collections.abc import Awaitable, Callable, Iterable +from typing import ( + TYPE_CHECKING, + Any, + TypeVar, +) + +_T = TypeVar("_T") + +RE_RAISE_EXCEPTIONS = (SystemExit, KeyboardInterrupt) + + +def _set_result(wait_next: "asyncio.Future[None]") -> None: + """Set the result of a future if it is not already done.""" + if not wait_next.done(): + wait_next.set_result(None) + + +async def _wait_one( + futures: "Iterable[asyncio.Future[Any]]", + loop: asyncio.AbstractEventLoop, +) -> _T: + """Wait for the first future to complete.""" + wait_next = loop.create_future() + + def _on_completion(fut: "asyncio.Future[Any]") -> None: + if not wait_next.done(): + wait_next.set_result(fut) + + for f in futures: + f.add_done_callback(_on_completion) + + try: + return await wait_next + finally: + for f in futures: + f.remove_done_callback(_on_completion) + + +async def staggered_race( + coro_fns: Iterable[Callable[[], Awaitable[_T]]], + delay: float | None, + *, + loop: asyncio.AbstractEventLoop | None = None, +) -> tuple[_T | None, int | None, list[BaseException | None]]: + """ + Run coroutines with staggered start times and take the first to finish. + + This method takes an iterable of coroutine functions. The first one is + started immediately. From then on, whenever the immediately preceding one + fails (raises an exception), or when *delay* seconds has passed, the next + coroutine is started. This continues until one of the coroutines complete + successfully, in which case all others are cancelled, or until all + coroutines fail. + + The coroutines provided should be well-behaved in the following way: + + * They should only ``return`` if completed successfully. + + * They should always raise an exception if they did not complete + successfully. In particular, if they handle cancellation, they should + probably reraise, like this:: + + try: + # do work + except asyncio.CancelledError: + # undo partially completed work + raise + + Args: + ---- + coro_fns: an iterable of coroutine functions, i.e. callables that + return a coroutine object when called. Use ``functools.partial`` or + lambdas to pass arguments. + + delay: amount of time, in seconds, between starting coroutines. If + ``None``, the coroutines will run sequentially. + + loop: the event loop to use. If ``None``, the running loop is used. + + Returns: + ------- + tuple *(winner_result, winner_index, exceptions)* where + + - *winner_result*: the result of the winning coroutine, or ``None`` + if no coroutines won. + + - *winner_index*: the index of the winning coroutine in + ``coro_fns``, or ``None`` if no coroutines won. If the winning + coroutine may return None on success, *winner_index* can be used + to definitively determine whether any coroutine won. + + - *exceptions*: list of exceptions returned by the coroutines. + ``len(exceptions)`` is equal to the number of coroutines actually + started, and the order is the same as in ``coro_fns``. The winning + coroutine's entry is ``None``. + + """ + loop = loop or asyncio.get_running_loop() + exceptions: list[BaseException | None] = [] + tasks: set[asyncio.Task[tuple[_T, int] | None]] = set() + + async def run_one_coro( + coro_fn: Callable[[], Awaitable[_T]], + this_index: int, + start_next: "asyncio.Future[None]", + ) -> tuple[_T, int] | None: + """ + Run a single coroutine. + + If the coroutine fails, set the exception in the exceptions list and + start the next coroutine by setting the result of the start_next. + + If the coroutine succeeds, return the result and the index of the + coroutine in the coro_fns list. + + If SystemExit or KeyboardInterrupt is raised, re-raise it. + """ + try: + result = await coro_fn() + except RE_RAISE_EXCEPTIONS: + raise + except BaseException as e: + exceptions[this_index] = e + _set_result(start_next) # Kickstart the next coroutine + return None + + return result, this_index + + start_next_timer: asyncio.TimerHandle | None = None + start_next: asyncio.Future[None] | None + task: asyncio.Task[tuple[_T, int] | None] + done: asyncio.Future[None] | asyncio.Task[tuple[_T, int] | None] + coro_iter = iter(coro_fns) + this_index = -1 + try: + while True: + if coro_fn := next(coro_iter, None): + this_index += 1 + exceptions.append(None) + start_next = loop.create_future() + task = loop.create_task(run_one_coro(coro_fn, this_index, start_next)) + tasks.add(task) + start_next_timer = ( + loop.call_later(delay, _set_result, start_next) if delay else None + ) + elif not tasks: + # We exhausted the coro_fns list and no tasks are running + # so we have no winner and all coroutines failed. + break + + while tasks or start_next: + done = await _wait_one( + (*tasks, start_next) if start_next else tasks, loop + ) + if done is start_next: + # The current task has failed or the timer has expired + # so we need to start the next task. + start_next = None + if start_next_timer: + start_next_timer.cancel() + start_next_timer = None + + # Break out of the task waiting loop to start the next + # task. + break + + if TYPE_CHECKING: + assert isinstance(done, asyncio.Task) + + tasks.remove(done) + if winner := done.result(): + return *winner, exceptions + finally: + # We either have: + # - a winner + # - all tasks failed + # - a KeyboardInterrupt or SystemExit. + + # + # If the timer is still running, cancel it. + # + if start_next_timer: + start_next_timer.cancel() + + # + # If there are any tasks left, cancel them and than + # wait them so they fill the exceptions list. + # + for task in tasks: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + return None, None, exceptions diff --git a/venv/Lib/site-packages/aiohappyeyeballs/impl.py b/venv/Lib/site-packages/aiohappyeyeballs/impl.py new file mode 100644 index 0000000000000000000000000000000000000000..4112ab4d1b2bdeec6da543eac4433af175699c7d --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs/impl.py @@ -0,0 +1,261 @@ +"""Base implementation.""" + +import asyncio +import collections +import contextlib +import functools +import itertools +import socket +from collections.abc import Sequence + +from . import _staggered +from .types import AddrInfoType, SocketFactoryType + + +async def start_connection( + addr_infos: Sequence[AddrInfoType], + *, + local_addr_infos: Sequence[AddrInfoType] | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + loop: asyncio.AbstractEventLoop | None = None, + socket_factory: SocketFactoryType | None = None, +) -> socket.socket: + """ + Connect to a TCP server. + + Create a socket connection to a specified destination. The + destination is specified as a list of AddrInfoType tuples as + returned from getaddrinfo(). + + The arguments are, in order: + + * ``family``: the address family, e.g. ``socket.AF_INET`` or + ``socket.AF_INET6``. + * ``type``: the socket type, e.g. ``socket.SOCK_STREAM`` or + ``socket.SOCK_DGRAM``. + * ``proto``: the protocol, e.g. ``socket.IPPROTO_TCP`` or + ``socket.IPPROTO_UDP``. + * ``canonname``: the canonical name of the address, e.g. + ``"www.python.org"``. + * ``sockaddr``: the socket address + + This method is a coroutine which will try to establish the connection + in the background. When successful, the coroutine returns a + socket. + + The expected use case is to use this method in conjunction with + loop.create_connection() to establish a connection to a server:: + + socket = await start_connection(addr_infos) + transport, protocol = await loop.create_connection( + MyProtocol, sock=socket, ...) + """ + if not addr_infos: + raise ValueError("addr_infos must not be empty") + + current_loop = loop or asyncio.get_running_loop() + + single_addr_info = len(addr_infos) == 1 + + if happy_eyeballs_delay is not None and interleave is None: + # If using happy eyeballs, default to interleave addresses by family + interleave = 1 + + if interleave and not single_addr_info: + addr_infos = _interleave_addrinfos(addr_infos, interleave) + + sock: socket.socket | None = None + # uvloop can raise RuntimeError instead of OSError + exceptions: list[list[OSError | RuntimeError]] = [] + if happy_eyeballs_delay is None or single_addr_info: + # not using happy eyeballs + for addrinfo in addr_infos: + try: + sock = await _connect_sock( + current_loop, + exceptions, + addrinfo, + local_addr_infos, + None, + socket_factory, + ) + break + except (RuntimeError, OSError): + continue + else: # using happy eyeballs + open_sockets: set[socket.socket] = set() + try: + sock, _, _ = await _staggered.staggered_race( + ( + functools.partial( + _connect_sock, + current_loop, + exceptions, + addrinfo, + local_addr_infos, + open_sockets, + socket_factory, + ) + for addrinfo in addr_infos + ), + happy_eyeballs_delay, + ) + finally: + # If we have a winner, staggered_race will + # cancel the other tasks, however there is a + # small race window where any of the other tasks + # can be done before they are cancelled which + # will leave the socket open. To avoid this problem + # we pass a set to _connect_sock to keep track of + # the open sockets and close them here if there + # are any "runner up" sockets. + for s in open_sockets: + if s is not sock: + with contextlib.suppress(OSError): + s.close() + open_sockets = None # type: ignore[assignment] + + if sock is None: + all_exceptions = [exc for sub in exceptions for exc in sub] + try: + first_exception = all_exceptions[0] + if len(all_exceptions) == 1: + raise first_exception + else: + # If they all have the same str(), raise one. + model = str(first_exception) + if all(str(exc) == model for exc in all_exceptions): + raise first_exception + # Raise a combined exception so the user can see all + # the various error messages. + msg = "Multiple exceptions: {}".format( + ", ".join(str(exc) for exc in all_exceptions) + ) + # If the errno is the same for all exceptions, raise + # an OSError with that errno. + if isinstance(first_exception, OSError): + first_errno = first_exception.errno + if all( + isinstance(exc, OSError) and exc.errno == first_errno + for exc in all_exceptions + ): + raise OSError(first_errno, msg) + elif isinstance(first_exception, RuntimeError) and all( + isinstance(exc, RuntimeError) for exc in all_exceptions + ): + raise RuntimeError(msg) + # We have a mix of OSError and RuntimeError + # so we have to pick which one to raise. + # and we raise OSError for compatibility + raise OSError(msg) + finally: + all_exceptions = None # type: ignore[assignment] + exceptions = None # type: ignore[assignment] + + return sock + + +async def _connect_sock( + loop: asyncio.AbstractEventLoop, + exceptions: list[list[OSError | RuntimeError]], + addr_info: AddrInfoType, + local_addr_infos: Sequence[AddrInfoType] | None = None, + open_sockets: set[socket.socket] | None = None, + socket_factory: SocketFactoryType | None = None, +) -> socket.socket: + """ + Create, bind and connect one socket. + + If open_sockets is passed, add the socket to the set of open sockets. + Any failure caught here will remove the socket from the set and close it. + + Callers can use this set to close any sockets that are not the winner + of all staggered tasks in the result there are runner up sockets aka + multiple winners. + """ + my_exceptions: list[OSError | RuntimeError] = [] + exceptions.append(my_exceptions) + family, type_, proto, _, address = addr_info + sock = None + try: + if socket_factory is not None: + sock = socket_factory(addr_info) + else: + sock = socket.socket(family=family, type=type_, proto=proto) + if open_sockets is not None: + open_sockets.add(sock) + sock.setblocking(False) + if local_addr_infos is not None: + for lfamily, _, _, _, laddr in local_addr_infos: + # skip local addresses of different family + if lfamily != family: + continue + try: + sock.bind(laddr) + break + except OSError as exc: + msg = ( + f"error while attempting to bind on " + f"address {laddr!r}: " + f"{(exc.strerror or '').lower()}" + ) + exc = OSError(exc.errno, msg) + my_exceptions.append(exc) + else: # all bind attempts failed + if my_exceptions: + raise my_exceptions.pop() + else: + raise OSError(f"no matching local address with {family=} found") + await loop.sock_connect(sock, address) + return sock + except (RuntimeError, OSError) as exc: + my_exceptions.append(exc) + if sock is not None: + if open_sockets is not None: + open_sockets.remove(sock) + try: + sock.close() + except OSError as e: + my_exceptions.append(e) + raise + raise + except: + if sock is not None: + if open_sockets is not None: + open_sockets.remove(sock) + try: + sock.close() + except OSError as e: + my_exceptions.append(e) + raise + raise + finally: + exceptions = my_exceptions = None # type: ignore[assignment] + + +def _interleave_addrinfos( + addrinfos: Sequence[AddrInfoType], first_address_family_count: int = 1 +) -> list[AddrInfoType]: + """Interleave list of addrinfo tuples by family.""" + # Group addresses by family + addrinfos_by_family: collections.OrderedDict[int, list[AddrInfoType]] = ( + collections.OrderedDict() + ) + for addr in addrinfos: + family = addr[0] + if family not in addrinfos_by_family: + addrinfos_by_family[family] = [] + addrinfos_by_family[family].append(addr) + addrinfos_lists = list(addrinfos_by_family.values()) + + reordered: list[AddrInfoType] = [] + if first_address_family_count > 1: + reordered.extend(addrinfos_lists[0][: first_address_family_count - 1]) + del addrinfos_lists[0][: first_address_family_count - 1] + reordered.extend( + a + for a in itertools.chain.from_iterable(itertools.zip_longest(*addrinfos_lists)) + if a is not None + ) + return reordered diff --git a/venv/Lib/site-packages/aiohappyeyeballs/py.typed b/venv/Lib/site-packages/aiohappyeyeballs/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/Lib/site-packages/aiohappyeyeballs/types.py b/venv/Lib/site-packages/aiohappyeyeballs/types.py new file mode 100644 index 0000000000000000000000000000000000000000..80f09afdd6d5f4cca4d9552653867e2de8e2d6a5 --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs/types.py @@ -0,0 +1,14 @@ +"""Types for aiohappyeyeballs.""" + +import socket +from collections.abc import Callable + +AddrInfoType = tuple[ + int | socket.AddressFamily, + int | socket.SocketKind, + int, + str, + tuple, # type: ignore[type-arg] +] + +SocketFactoryType = Callable[[AddrInfoType], socket.socket] diff --git a/venv/Lib/site-packages/aiohappyeyeballs/utils.py b/venv/Lib/site-packages/aiohappyeyeballs/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5969691ce3627dd64b9ca539331b3ef4af5da6de --- /dev/null +++ b/venv/Lib/site-packages/aiohappyeyeballs/utils.py @@ -0,0 +1,92 @@ +"""Utility functions for aiohappyeyeballs.""" + +import ipaddress +import socket + +from .types import AddrInfoType + + +def addr_to_addr_infos( + addr: tuple[str, int, int, int] | tuple[str, int, int] | tuple[str, int] | None, +) -> list[AddrInfoType] | None: + """Convert an address tuple to a list of addr_info tuples.""" + if addr is None: + return None + host = addr[0] + port = addr[1] + is_ipv6 = ":" in host + if is_ipv6: + flowinfo = 0 + scopeid = 0 + addr_len = len(addr) + if addr_len >= 4: + scopeid = addr[3] # type: ignore[misc] + if addr_len >= 3: + flowinfo = addr[2] # type: ignore[misc] + addr = (host, port, flowinfo, scopeid) + family = socket.AF_INET6 + else: + addr = (host, port) + family = socket.AF_INET + return [(family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", addr)] + + +def pop_addr_infos_interleave( + addr_infos: list[AddrInfoType], interleave: int | None = None +) -> None: + """ + Pop addr_info from the list of addr_infos by family up to interleave times. + + The interleave parameter is used to know how many addr_infos for + each family should be popped of the top of the list. + """ + seen: dict[int, int] = {} + if interleave is None: + interleave = 1 + to_remove: list[AddrInfoType] = [] + for addr_info in addr_infos: + family = addr_info[0] + if family not in seen: + seen[family] = 0 + if seen[family] < interleave: + to_remove.append(addr_info) + seen[family] += 1 + for addr_info in to_remove: + addr_infos.remove(addr_info) + + +def _addr_tuple_to_ip_address( + addr: tuple[str, int] | tuple[str, int, int, int], +) -> tuple[ipaddress.IPv4Address, int] | tuple[ipaddress.IPv6Address, int, int, int]: + """Convert an address tuple to an IPv4Address.""" + return (ipaddress.ip_address(addr[0]), *addr[1:]) + + +def remove_addr_infos( + addr_infos: list[AddrInfoType], + addr: tuple[str, int] | tuple[str, int, int, int], +) -> None: + """ + Remove an address from the list of addr_infos. + + The addr value is typically the return value of + sock.getpeername(). + """ + bad_addrs_infos: list[AddrInfoType] = [] + for addr_info in addr_infos: + if addr_info[-1] == addr: + bad_addrs_infos.append(addr_info) + if bad_addrs_infos: + for bad_addr_info in bad_addrs_infos: + addr_infos.remove(bad_addr_info) + return + # Slow path in case addr is formatted differently + match_addr = _addr_tuple_to_ip_address(addr) + for addr_info in addr_infos: + if match_addr == _addr_tuple_to_ip_address(addr_info[-1]): + bad_addrs_infos.append(addr_info) + if bad_addrs_infos: + for bad_addr_info in bad_addrs_infos: + addr_infos.remove(bad_addr_info) + return + raise ValueError(f"Address {addr} not found in addr_infos") diff --git a/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/INSTALLER b/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/licenses/LICENSE.txt b/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..ebfae372d79404742257a1bc396eacf1f8d9926f --- /dev/null +++ b/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/licenses/LICENSE.txt @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright aio-libs contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/licenses/vendor/llhttp/LICENSE b/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/licenses/vendor/llhttp/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..4f7f18df79730c836bff3e5d68c851b22cac252c --- /dev/null +++ b/venv/Lib/site-packages/aiohttp-3.14.1.dist-info/licenses/vendor/llhttp/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright Β© 2018 Fedor Indutny + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/venv/Lib/site-packages/aiohttp/.hash/_cparser.pxd.hash b/venv/Lib/site-packages/aiohttp/.hash/_cparser.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..061bccc5c228b0a588ce21df0c27acfc9ffe2556 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/.hash/_cparser.pxd.hash @@ -0,0 +1 @@ +535e028ee43275a6ceca0817f6ceb90ab44e8a73f20a3a8f5a0c036447687f79 *D:/a/aiohttp/aiohttp/aiohttp/_cparser.pxd diff --git a/venv/Lib/site-packages/aiohttp/.hash/_find_header.pxd.hash b/venv/Lib/site-packages/aiohttp/.hash/_find_header.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..8af9f81d71e4e4ee436f4bf3e6f95940e5f3666e --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/.hash/_find_header.pxd.hash @@ -0,0 +1 @@ +0455129b185e981b5b96ac738f31f7c74dc57f1696953cae0083b3f18679fe73 *D:/a/aiohttp/aiohttp/aiohttp/_find_header.pxd diff --git a/venv/Lib/site-packages/aiohttp/.hash/_http_parser.pyx.hash b/venv/Lib/site-packages/aiohttp/.hash/_http_parser.pyx.hash new file mode 100644 index 0000000000000000000000000000000000000000..d621a8b5a0fa43e0f534c8d0ca8c7e8f596466e7 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/.hash/_http_parser.pyx.hash @@ -0,0 +1 @@ +00fc87643dd46deca4e9ebe3f7d6cd42e118b5f0581552da5e4c418dc0900cb7 *D:/a/aiohttp/aiohttp/aiohttp/_http_parser.pyx diff --git a/venv/Lib/site-packages/aiohttp/.hash/_http_writer.pyx.hash b/venv/Lib/site-packages/aiohttp/.hash/_http_writer.pyx.hash new file mode 100644 index 0000000000000000000000000000000000000000..dd4e9e01a403243622e36c3598e47a09d592ff5a --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/.hash/_http_writer.pyx.hash @@ -0,0 +1 @@ +1ad6a84f394f80760b2f3599b170c56f6515ae464cc03ee20c79db2ee16ecb1d *D:/a/aiohttp/aiohttp/aiohttp/_http_writer.pyx diff --git a/venv/Lib/site-packages/aiohttp/.hash/hdrs.py.hash b/venv/Lib/site-packages/aiohttp/.hash/hdrs.py.hash new file mode 100644 index 0000000000000000000000000000000000000000..ab9d45fb42434f205eb83b18b6fe78ad8a1a9bee --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/.hash/hdrs.py.hash @@ -0,0 +1 @@ +8eaf9275f91702a2b2ceb2b3168f2965edcf347435a352039c24ed9bb083e887 *D:/a/aiohttp/aiohttp/aiohttp/hdrs.py diff --git a/venv/Lib/site-packages/aiohttp/__init__.py b/venv/Lib/site-packages/aiohttp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..181077de15aab905aa53457a824b37b0ca7b1d5a --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/__init__.py @@ -0,0 +1,279 @@ +__version__ = "3.14.1" + +from typing import TYPE_CHECKING + +from . import hdrs as hdrs +from .client import ( + BaseConnector, + ClientConnectionError, + ClientConnectionResetError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientError, + ClientHttpProxyError, + ClientOSError, + ClientPayloadError, + ClientProxyConnectionError, + ClientRequest, + ClientResponse, + ClientResponseError, + ClientSession, + ClientSSLError, + ClientTimeout, + ClientWebSocketResponse, + ClientWSTimeout, + ConnectionTimeoutError, + ContentTypeError, + Fingerprint, + InvalidURL, + InvalidUrlClientError, + InvalidUrlRedirectClientError, + NamedPipeConnector, + NonHttpUrlClientError, + NonHttpUrlRedirectClientError, + RedirectClientError, + RequestInfo, + ServerConnectionError, + ServerDisconnectedError, + ServerFingerprintMismatch, + ServerTimeoutError, + SocketTimeoutError, + TCPConnector, + TooManyRedirects, + UnixConnector, + WSMessageTypeError, + WSServerHandshakeError, + request, +) +from .client_middleware_digest_auth import DigestAuthMiddleware +from .client_middlewares import ClientHandlerType, ClientMiddlewareType +from .compression_utils import set_zlib_backend +from .connector import ( + AddrInfoType as AddrInfoType, + SocketFactoryType as SocketFactoryType, +) +from .cookiejar import CookieJar as CookieJar, DummyCookieJar as DummyCookieJar +from .formdata import FormData as FormData +from .helpers import BasicAuth, ChainMapProxy, ETag, encode_basic_auth +from .http import ( + HttpVersion as HttpVersion, + HttpVersion10 as HttpVersion10, + HttpVersion11 as HttpVersion11, + WebSocketError as WebSocketError, + WSCloseCode as WSCloseCode, + WSMessage as WSMessage, + WSMsgType as WSMsgType, +) +from .multipart import ( + BadContentDispositionHeader as BadContentDispositionHeader, + BadContentDispositionParam as BadContentDispositionParam, + BodyPartReader as BodyPartReader, + MultipartReader as MultipartReader, + MultipartWriter as MultipartWriter, + content_disposition_filename as content_disposition_filename, + parse_content_disposition as parse_content_disposition, +) +from .payload import ( + PAYLOAD_REGISTRY as PAYLOAD_REGISTRY, + AsyncIterablePayload as AsyncIterablePayload, + BufferedReaderPayload as BufferedReaderPayload, + BytesIOPayload as BytesIOPayload, + BytesPayload as BytesPayload, + IOBasePayload as IOBasePayload, + JsonPayload as JsonPayload, + Payload as Payload, + StringIOPayload as StringIOPayload, + StringPayload as StringPayload, + TextIOPayload as TextIOPayload, + get_payload as get_payload, + payload_type as payload_type, +) +from .payload_streamer import streamer as streamer +from .resolver import ( + AsyncResolver as AsyncResolver, + DefaultResolver as DefaultResolver, + ThreadedResolver as ThreadedResolver, +) +from .streams import ( + EMPTY_PAYLOAD as EMPTY_PAYLOAD, + DataQueue as DataQueue, + EofStream as EofStream, + FlowControlDataQueue as FlowControlDataQueue, + StreamReader as StreamReader, +) +from .tracing import ( + TraceConfig as TraceConfig, + TraceConnectionCreateEndParams as TraceConnectionCreateEndParams, + TraceConnectionCreateStartParams as TraceConnectionCreateStartParams, + TraceConnectionQueuedEndParams as TraceConnectionQueuedEndParams, + TraceConnectionQueuedStartParams as TraceConnectionQueuedStartParams, + TraceConnectionReuseconnParams as TraceConnectionReuseconnParams, + TraceDnsCacheHitParams as TraceDnsCacheHitParams, + TraceDnsCacheMissParams as TraceDnsCacheMissParams, + TraceDnsResolveHostEndParams as TraceDnsResolveHostEndParams, + TraceDnsResolveHostStartParams as TraceDnsResolveHostStartParams, + TraceRequestChunkSentParams as TraceRequestChunkSentParams, + TraceRequestEndParams as TraceRequestEndParams, + TraceRequestExceptionParams as TraceRequestExceptionParams, + TraceRequestHeadersSentParams as TraceRequestHeadersSentParams, + TraceRequestRedirectParams as TraceRequestRedirectParams, + TraceRequestStartParams as TraceRequestStartParams, + TraceResponseChunkReceivedParams as TraceResponseChunkReceivedParams, +) + +if TYPE_CHECKING: + # At runtime these are lazy-loaded at the bottom of the file. + from .worker import ( + GunicornUVLoopWebWorker as GunicornUVLoopWebWorker, + GunicornWebWorker as GunicornWebWorker, + ) + +__all__: tuple[str, ...] = ( + "hdrs", + # client + "AddrInfoType", + "BaseConnector", + "ClientConnectionError", + "ClientConnectionResetError", + "ClientConnectorCertificateError", + "ClientConnectorDNSError", + "ClientConnectorError", + "ClientConnectorSSLError", + "ClientError", + "ClientHttpProxyError", + "ClientOSError", + "ClientPayloadError", + "ClientProxyConnectionError", + "ClientResponse", + "ClientRequest", + "ClientResponseError", + "ClientSSLError", + "ClientSession", + "ClientTimeout", + "ClientWebSocketResponse", + "ClientWSTimeout", + "ConnectionTimeoutError", + "ContentTypeError", + "Fingerprint", + "FlowControlDataQueue", + "InvalidURL", + "InvalidUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlClientError", + "NonHttpUrlRedirectClientError", + "RedirectClientError", + "RequestInfo", + "ServerConnectionError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ServerTimeoutError", + "SocketFactoryType", + "SocketTimeoutError", + "TCPConnector", + "TooManyRedirects", + "UnixConnector", + "NamedPipeConnector", + "WSServerHandshakeError", + "request", + # client_middleware + "ClientMiddlewareType", + "ClientHandlerType", + # cookiejar + "CookieJar", + "DummyCookieJar", + # formdata + "FormData", + # helpers + "BasicAuth", + "ChainMapProxy", + "DigestAuthMiddleware", + "ETag", + "encode_basic_auth", + "set_zlib_backend", + # http + "HttpVersion", + "HttpVersion10", + "HttpVersion11", + "WSMsgType", + "WSCloseCode", + "WSMessage", + "WebSocketError", + # multipart + "BadContentDispositionHeader", + "BadContentDispositionParam", + "BodyPartReader", + "MultipartReader", + "MultipartWriter", + "content_disposition_filename", + "parse_content_disposition", + # payload + "AsyncIterablePayload", + "BufferedReaderPayload", + "BytesIOPayload", + "BytesPayload", + "IOBasePayload", + "JsonPayload", + "PAYLOAD_REGISTRY", + "Payload", + "StringIOPayload", + "StringPayload", + "TextIOPayload", + "get_payload", + "payload_type", + # payload_streamer + "streamer", + # resolver + "AsyncResolver", + "DefaultResolver", + "ThreadedResolver", + # streams + "DataQueue", + "EMPTY_PAYLOAD", + "EofStream", + "StreamReader", + # tracing + "TraceConfig", + "TraceConnectionCreateEndParams", + "TraceConnectionCreateStartParams", + "TraceConnectionQueuedEndParams", + "TraceConnectionQueuedStartParams", + "TraceConnectionReuseconnParams", + "TraceDnsCacheHitParams", + "TraceDnsCacheMissParams", + "TraceDnsResolveHostEndParams", + "TraceDnsResolveHostStartParams", + "TraceRequestChunkSentParams", + "TraceRequestEndParams", + "TraceRequestExceptionParams", + "TraceRequestHeadersSentParams", + "TraceRequestRedirectParams", + "TraceRequestStartParams", + "TraceResponseChunkReceivedParams", + # workers (imported lazily with __getattr__) + "GunicornUVLoopWebWorker", + "GunicornWebWorker", + "WSMessageTypeError", +) + + +def __dir__() -> tuple[str, ...]: + return __all__ + ("__doc__",) + + +def __getattr__(name: str) -> object: + global GunicornUVLoopWebWorker, GunicornWebWorker + + # Importing gunicorn takes a long time (>100ms), so only import if actually needed. + if name in ("GunicornUVLoopWebWorker", "GunicornWebWorker"): + try: + from .worker import GunicornUVLoopWebWorker as guv, GunicornWebWorker as gw + except ImportError: + return None + + GunicornUVLoopWebWorker = guv # type: ignore[misc] + GunicornWebWorker = gw # type: ignore[misc] + return guv if name == "GunicornUVLoopWebWorker" else gw + + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbabcbb5ec0a7146b3e8090d2969fe494f126c69 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/_cookie_helpers.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/_cookie_helpers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74b25859da508183ec9e2ca822b3ab56ed12ca7a Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/_cookie_helpers.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/abc.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/abc.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ca331a1fa525142734ba50874d704b861b0b7c4 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/abc.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/base_protocol.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/base_protocol.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd3f4abf552fdb3218781d38b3cf2c9a43c415c3 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/base_protocol.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/client.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/client.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..36614392cb8c20ee867e685f514f0ce6567ebb29 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/client.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/client_exceptions.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/client_exceptions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07609c31fdd699bca5c6c955977ec701fe73c608 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/client_exceptions.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/client_middleware_digest_auth.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/client_middleware_digest_auth.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e670b351636c0f383cef67b4c20db69228427fe Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/client_middleware_digest_auth.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/client_middlewares.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/client_middlewares.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53ed37abacfb7e2368463f9e784de4e4204f244d Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/client_middlewares.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/client_proto.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/client_proto.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..952d5694b95e291579b1030df8aebab782ecb17a Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/client_proto.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/client_reqrep.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/client_reqrep.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..747efee414ad579923c51dbba5f6d25ed055ed91 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/client_reqrep.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/client_ws.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/client_ws.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9b7b87b9be6e28453c0dffa7ad2dde6ee331cd7e Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/client_ws.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/compression_utils.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/compression_utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41465f4e63252a5cd644554207c66a7c77edccad Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/compression_utils.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/connector.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/connector.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d7c22d78d94be5d1d5fddce7717b2ca6f0d2179 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/connector.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/cookiejar.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/cookiejar.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1ccf9ac2a0d60d9aa0fb7c95d058695481fb351d Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/cookiejar.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/formdata.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/formdata.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2abe82e1913ba75aac0b985dc644f6087345d532 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/formdata.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/hdrs.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/hdrs.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ecff4d9c997baf6705fd97bbb80013cd59aa3f11 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/hdrs.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/helpers.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/helpers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fcd3c8d22b8dfe5fee66cd537dcd70c803bc6ff8 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/helpers.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/http.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/http.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b70c6f35cfe881749aa75286412364d807a43ef Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/http.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/http_exceptions.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/http_exceptions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..44ef100946a1bb719b9420d139671fa2c2a3c919 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/http_exceptions.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/http_parser.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/http_parser.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97e46ef0a755b2aaad8929415d55de14f7132ad0 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/http_parser.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/http_websocket.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/http_websocket.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7df6d66c4512d124145c8337b81f036b6d743ec6 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/http_websocket.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/http_writer.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/http_writer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0a5166177f72f608d60e0f32788440f7980c985 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/http_writer.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/log.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/log.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa51ac7d16d8cd6715fb302c77cfe45f7b3fc840 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/log.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/multipart.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/multipart.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f9c859018d4370409246bc3c40df3030d87a322 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/multipart.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/payload.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/payload.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b95e46278a1de6eae35eb7e8410406d08f9a6bf Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/payload.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/payload_streamer.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/payload_streamer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9818214d1764f78c7257845e41bcdd2ee7fb2ca Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/payload_streamer.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/pytest_plugin.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/pytest_plugin.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..45f4d09dd1a6325bad8f1aa67cbf4cbd244de522 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/pytest_plugin.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/resolver.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/resolver.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62cd232259b5801586dae6432ca74263a2b0d50f Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/resolver.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/streams.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/streams.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81bdda0892db1b8aae315aeb7898be800f0605e0 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/streams.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/tcp_helpers.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/tcp_helpers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa1cb578dc0f43d8628ab7050b8a8365598a92f1 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/tcp_helpers.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/test_utils.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/test_utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..404d56ef2a04668e1283f6c4fdd60b83c0e7d4e0 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/test_utils.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/tracing.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/tracing.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2155a1dbaa2c9e7ae0754743807211fd7178e209 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/tracing.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/typedefs.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/typedefs.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef1225d55070a9236f6c161cb5f0bcb3c2e6fa48 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/typedefs.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bab5911d46f9e90241016547b521c491da819b2f Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_app.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_app.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..caf0d8807a31dc63db0e0924c84847f3beae7efc Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_app.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_exceptions.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_exceptions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53e51669a510577d8c2641545e04e6e52b847c4f Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_exceptions.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_fileresponse.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_fileresponse.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7025d103328fb67fbf225b91291886b7c60c782 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_fileresponse.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_log.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_log.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..779d02d3f2e44f1032832ba4701f6579e9f09f1d Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_log.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_middlewares.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_middlewares.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0bde712c2641729780cd5e73cf34776eabf847cd Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_middlewares.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_protocol.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_protocol.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..70df3c97e2fdc1e1ecd4d4309e63d171900c9d7a Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_protocol.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_request.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_request.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1547987dc234251710976b50122b3ac910dc2da2 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_request.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_response.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_response.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a881f2e38332729f279528e377503c5cc1b216c0 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_response.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_routedef.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_routedef.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..becad1bc342d50d8789141194c52f586d6d8b3f0 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_routedef.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_runner.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_runner.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01b14fcec1d69bf76052037c96bf36f55afb7e3e Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_runner.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_server.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_server.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e0a3fc9b3f4e9490d20248e2cac9a3fdaf5d8ca Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_server.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_urldispatcher.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_urldispatcher.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25760b2e5ad2cab8c84771aa5b0ad941be3e9e91 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_urldispatcher.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/web_ws.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/web_ws.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87bb9e515c2107c1aac6ec7d9f0b3b9e8fdd0861 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/web_ws.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/__pycache__/worker.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/__pycache__/worker.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e33ba7ff88a1f2a59ff97b2c47222cd75926ebe4 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/__pycache__/worker.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/_cookie_helpers.py b/venv/Lib/site-packages/aiohttp/_cookie_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..523404eaf73dc42579d0134f72aa1c15f49c14f2 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_cookie_helpers.py @@ -0,0 +1,361 @@ +""" +Internal cookie handling helpers. + +This module contains internal utilities for cookie parsing and manipulation. +These are not part of the public API and may change without notice. +""" + +import re +from collections.abc import Sequence +from http.cookies import CookieError, Morsel +from typing import cast + +from .log import internal_logger + +__all__ = ( + "parse_set_cookie_headers", + "parse_cookie_header", + "preserve_morsel_with_coded_value", +) + +# Cookie parsing constants +# Allow more characters in cookie names to handle real-world cookies +# that don't strictly follow RFC standards (fixes #2683) +# RFC 6265 defines cookie-name token as per RFC 2616 Section 2.2, +# but many servers send cookies with characters like {} [] () etc. +# This makes the cookie parser more tolerant of real-world cookies +# while still providing some validation to catch obviously malformed names. +_COOKIE_NAME_RE = re.compile(r"^[!#$%&\'()*+\-./0-9:<=>?@A-Z\[\]^_`a-z{|}~]+$") +_COOKIE_KNOWN_ATTRS = frozenset( # AKA Morsel._reserved + ( + "path", + "domain", + "max-age", + "expires", + "secure", + "httponly", + "samesite", + "partitioned", + "version", + "comment", + ) +) +_COOKIE_BOOL_ATTRS = frozenset( # AKA Morsel._flags + ("secure", "httponly", "partitioned") +) + +# SimpleCookie's pattern for parsing cookies with relaxed validation +# Based on http.cookies pattern but extended to allow more characters in cookie names +# to handle real-world cookies (fixes #2683) +_COOKIE_PATTERN = re.compile( + r""" + \s* # Optional whitespace at start of cookie + (?P # Start of group 'key' + # aiohttp has extended to include [] for compatibility with real-world cookies + [\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\[\]]+ # Any word of at least one letter + ) # End of group 'key' + ( # Optional group: there may not be a value. + \s*=\s* # Equal Sign + (?P # Start of group 'val' + "(?:[^\\"]|\\.)*" # Any double-quoted string (properly closed) + | # or + "[^";]* # Unmatched opening quote (differs from SimpleCookie - issue #7993) + | # or + # Special case for "expires" attr - RFC 822, RFC 850, RFC 1036, RFC 1123 + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day (with comma) + [\w\d\s-]{9,11}\s[\d:]{8}\s # Date and time in specific format + (GMT|[+-]\d{4}) # Timezone: GMT or RFC 2822 offset like -0000, +0100 + # NOTE: RFC 2822 timezone support is an aiohttp extension + # for issue #4493 - SimpleCookie does NOT support this + | # or + # ANSI C asctime() format: "Wed Jun 9 10:18:14 2021" + # NOTE: This is an aiohttp extension for issue #4327 - SimpleCookie does NOT support this format + \w{3}\s+\w{3}\s+[\s\d]\d\s+\d{2}:\d{2}:\d{2}\s+\d{4} + | # or + [\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=\[\]]* # Any word or empty string + ) # End of group 'val' + )? # End of optional value group + \s* # Any number of spaces. + (\s+|;|$) # Ending either at space, semicolon, or EOS. + """, + re.VERBOSE | re.ASCII, +) + + +def preserve_morsel_with_coded_value(cookie: Morsel[str]) -> Morsel[str]: + """ + Preserve a Morsel's coded_value exactly as received from the server. + + This function ensures that cookie encoding is preserved exactly as sent by + the server, which is critical for compatibility with old servers that have + strict requirements about cookie formats. + + This addresses the issue described in https://github.com/aio-libs/aiohttp/pull/1453 + where Python's SimpleCookie would re-encode cookies, breaking authentication + with certain servers. + + Args: + cookie: A Morsel object from SimpleCookie + + Returns: + A Morsel object with preserved coded_value + + """ + mrsl_val = cast("Morsel[str]", cookie.get(cookie.key, Morsel())) + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + try: + mrsl_val.__setstate__( # type: ignore[attr-defined] + { + "key": cookie.key, + "value": cookie.value, + "coded_value": cookie.coded_value, + } + ) + except CookieError: + return cookie + return mrsl_val + + +_unquote_sub = re.compile(r"\\(?:([0-3][0-7][0-7])|(.))").sub + + +def _unquote_replace(m: re.Match[str]) -> str: + """ + Replace function for _unquote_sub regex substitution. + + Handles escaped characters in cookie values: + - Octal sequences are converted to their character representation + - Other escaped characters are unescaped by removing the backslash + """ + if m[1]: + return chr(int(m[1], 8)) + return m[2] + + +def _unquote(value: str) -> str: + """ + Unquote a cookie value. + + Vendored from http.cookies._unquote to ensure compatibility. + + Note: The original implementation checked for None, but we've removed + that check since all callers already ensure the value is not None. + """ + # If there aren't any doublequotes, + # then there can't be any special characters. See RFC 2109. + if len(value) < 2: + return value + if value[0] != '"' or value[-1] != '"': + return value + + # We have to assume that we must decode this string. + # Down to work. + + # Remove the "s + value = value[1:-1] + + # Check for special sequences. Examples: + # \012 --> \n + # \" --> " + # + return _unquote_sub(_unquote_replace, value) + + +def parse_cookie_header(header: str) -> list[tuple[str, Morsel[str]]]: + """ + Parse a Cookie header according to RFC 6265 Section 5.4. + + Cookie headers contain only name-value pairs separated by semicolons. + There are no attributes in Cookie headers - even names that match + attribute names (like 'path' or 'secure') should be treated as cookies. + + This parser uses the same regex-based approach as parse_set_cookie_headers + to properly handle quoted values that may contain semicolons. When the + regex fails to match a malformed cookie, it falls back to simple parsing + to ensure subsequent cookies are not lost + https://github.com/aio-libs/aiohttp/issues/11632 + + Args: + header: The Cookie header value to parse + + Returns: + List of (name, Morsel) tuples for compatibility with SimpleCookie.update() + """ + if not header: + return [] + + cookies: list[tuple[str, Morsel[str]]] = [] + morsel: Morsel[str] + i = 0 + n = len(header) + + invalid_names = [] + while i < n: + # Use the same pattern as parse_set_cookie_headers to find cookies + match = _COOKIE_PATTERN.match(header, i) + if not match: + # Fallback for malformed cookies https://github.com/aio-libs/aiohttp/issues/11632 + # Find next semicolon to skip or attempt simple key=value parsing + next_semi = header.find(";", i) + eq_pos = header.find("=", i) + + # Try to extract key=value if '=' comes before ';' + if eq_pos != -1 and (next_semi == -1 or eq_pos < next_semi): + end_pos = next_semi if next_semi != -1 else n + key = header[i:eq_pos].strip() + value = header[eq_pos + 1 : end_pos].strip() + + # Validate the name (same as regex path) + if not _COOKIE_NAME_RE.match(key): + invalid_names.append(key) + else: + morsel = Morsel() + try: + morsel.__setstate__( # type: ignore[attr-defined] + { + "key": key, + "value": _unquote(value), + "coded_value": value, + } + ) + except CookieError: + pass + else: + cookies.append((key, morsel)) + + # Move to next cookie or end + i = next_semi + 1 if next_semi != -1 else n + continue + + key = match.group("key") + value = match.group("val") or "" + i = match.end(0) + + # Validate the name + if not key or not _COOKIE_NAME_RE.match(key): + invalid_names.append(key) + continue + + # Create new morsel + morsel = Morsel() + # Preserve the original value as coded_value (with quotes if present) + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + try: + morsel.__setstate__( # type: ignore[attr-defined] + {"key": key, "value": _unquote(value), "coded_value": value} + ) + except CookieError: + continue + + cookies.append((key, morsel)) + + if invalid_names: + internal_logger.debug( + "Cannot load cookie. Illegal cookie names: %r", invalid_names + ) + + return cookies + + +def parse_set_cookie_headers(headers: Sequence[str]) -> list[tuple[str, Morsel[str]]]: + """ + Parse cookie headers using a vendored version of SimpleCookie parsing. + + This implementation is based on SimpleCookie.__parse_string to ensure + compatibility with how SimpleCookie parses cookies, including handling + of malformed cookies with missing semicolons. + + This function is used for both Cookie and Set-Cookie headers in order to be + forgiving. Ideally we would have followed RFC 6265 Section 5.2 (for Cookie + headers) and RFC 6265 Section 4.2.1 (for Set-Cookie headers), but the + real world data makes it impossible since we need to be a bit more forgiving. + + NOTE: This implementation differs from SimpleCookie in handling unmatched quotes. + SimpleCookie will stop parsing when it encounters a cookie value with an unmatched + quote (e.g., 'cookie="value'), causing subsequent cookies to be silently dropped. + This implementation handles unmatched quotes more gracefully to prevent cookie loss. + See https://github.com/aio-libs/aiohttp/issues/7993 + """ + parsed_cookies: list[tuple[str, Morsel[str]]] = [] + + for header in headers: + if not header: + continue + + # Parse cookie string using SimpleCookie's algorithm + i = 0 + n = len(header) + current_morsel: Morsel[str] | None = None + morsel_seen = False + + while 0 <= i < n: + # Start looking for a cookie + match = _COOKIE_PATTERN.match(header, i) + if not match: + # No more cookies + break + + key, value = match.group("key"), match.group("val") + i = match.end(0) + lower_key = key.lower() + + if key[0] == "$": + if not morsel_seen: + # We ignore attributes which pertain to the cookie + # mechanism as a whole, such as "$Version". + continue + # Process as attribute + if current_morsel is not None: + attr_lower_key = lower_key[1:] + if attr_lower_key in _COOKIE_KNOWN_ATTRS: + current_morsel[attr_lower_key] = value or "" + elif lower_key in _COOKIE_KNOWN_ATTRS: + if not morsel_seen: + # Invalid cookie string - attribute before cookie + break + if lower_key in _COOKIE_BOOL_ATTRS: + # Boolean attribute with any value should be True + if current_morsel is not None and current_morsel.isReservedKey(key): + current_morsel[lower_key] = True + elif value is None: + # Invalid cookie string - non-boolean attribute without value + break + elif current_morsel is not None: + # Regular attribute with value + current_morsel[lower_key] = _unquote(value) + elif value is not None: + # This is a cookie name=value pair + # Validate the name + if key in _COOKIE_KNOWN_ATTRS or not _COOKIE_NAME_RE.match(key): + internal_logger.warning( + "Can not load cookies: Illegal cookie name %r", key + ) + current_morsel = None + else: + # Create new morsel + current_morsel = Morsel() + # Preserve the original value as coded_value (with quotes if present) + try: + current_morsel.__setstate__( # type: ignore[attr-defined] + { + "key": key, + "value": _unquote(value), + "coded_value": value, + } + ) + except CookieError: + current_morsel = None + else: + parsed_cookies.append((key, current_morsel)) + morsel_seen = True + else: + # Invalid cookie string - no value for non-attribute + break + + return parsed_cookies diff --git a/venv/Lib/site-packages/aiohttp/_cparser.pxd b/venv/Lib/site-packages/aiohttp/_cparser.pxd new file mode 100644 index 0000000000000000000000000000000000000000..b28adb490487ddd5fb9a44760c853a8cb3116bdd --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_cparser.pxd @@ -0,0 +1,159 @@ +from libc.stdint cimport int32_t, uint8_t, uint16_t, uint64_t + + +cdef extern from "llhttp.h": + + struct llhttp__internal_s: + int32_t _index + void* _span_pos0 + void* _span_cb0 + int32_t error + const char* reason + const char* error_pos + void* data + void* _current + uint64_t content_length + uint8_t type + uint8_t method + uint8_t http_major + uint8_t http_minor + uint8_t header_state + uint8_t lenient_flags + uint8_t upgrade + uint8_t finish + uint16_t flags + uint16_t status_code + void* settings + + ctypedef llhttp__internal_s llhttp__internal_t + ctypedef llhttp__internal_t llhttp_t + + ctypedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length) except -1 + ctypedef int (*llhttp_cb)(llhttp_t*) except -1 + + struct llhttp_settings_s: + llhttp_cb on_message_begin + llhttp_data_cb on_url + llhttp_data_cb on_status + llhttp_data_cb on_header_field + llhttp_data_cb on_header_value + llhttp_cb on_headers_complete + llhttp_data_cb on_body + llhttp_cb on_message_complete + llhttp_cb on_chunk_header + llhttp_cb on_chunk_complete + + llhttp_cb on_url_complete + llhttp_cb on_status_complete + llhttp_cb on_header_field_complete + llhttp_cb on_header_value_complete + + ctypedef llhttp_settings_s llhttp_settings_t + + enum llhttp_errno: + HPE_OK, + HPE_INTERNAL, + HPE_STRICT, + HPE_LF_EXPECTED, + HPE_UNEXPECTED_CONTENT_LENGTH, + HPE_CLOSED_CONNECTION, + HPE_INVALID_METHOD, + HPE_INVALID_URL, + HPE_INVALID_CONSTANT, + HPE_INVALID_VERSION, + HPE_INVALID_HEADER_TOKEN, + HPE_INVALID_CONTENT_LENGTH, + HPE_INVALID_CHUNK_SIZE, + HPE_INVALID_STATUS, + HPE_INVALID_EOF_STATE, + HPE_INVALID_TRANSFER_ENCODING, + HPE_CB_MESSAGE_BEGIN, + HPE_CB_HEADERS_COMPLETE, + HPE_CB_MESSAGE_COMPLETE, + HPE_CB_CHUNK_HEADER, + HPE_CB_CHUNK_COMPLETE, + HPE_PAUSED, + HPE_PAUSED_UPGRADE, + HPE_USER + + ctypedef llhttp_errno llhttp_errno_t + + enum llhttp_flags: + F_CHUNKED, + F_CONTENT_LENGTH + + enum llhttp_type: + HTTP_REQUEST, + HTTP_RESPONSE, + HTTP_BOTH + + enum llhttp_method: + HTTP_DELETE, + HTTP_GET, + HTTP_HEAD, + HTTP_POST, + HTTP_PUT, + HTTP_CONNECT, + HTTP_OPTIONS, + HTTP_TRACE, + HTTP_COPY, + HTTP_LOCK, + HTTP_MKCOL, + HTTP_MOVE, + HTTP_PROPFIND, + HTTP_PROPPATCH, + HTTP_SEARCH, + HTTP_UNLOCK, + HTTP_BIND, + HTTP_REBIND, + HTTP_UNBIND, + HTTP_ACL, + HTTP_REPORT, + HTTP_MKACTIVITY, + HTTP_CHECKOUT, + HTTP_MERGE, + HTTP_MSEARCH, + HTTP_NOTIFY, + HTTP_SUBSCRIBE, + HTTP_UNSUBSCRIBE, + HTTP_PATCH, + HTTP_PURGE, + HTTP_MKCALENDAR, + HTTP_LINK, + HTTP_UNLINK, + HTTP_SOURCE, + HTTP_PRI, + HTTP_DESCRIBE, + HTTP_ANNOUNCE, + HTTP_SETUP, + HTTP_PLAY, + HTTP_PAUSE, + HTTP_TEARDOWN, + HTTP_GET_PARAMETER, + HTTP_SET_PARAMETER, + HTTP_REDIRECT, + HTTP_RECORD, + HTTP_FLUSH + + ctypedef llhttp_method llhttp_method_t; + + void llhttp_settings_init(llhttp_settings_t* settings) + void llhttp_init(llhttp_t* parser, llhttp_type type, + const llhttp_settings_t* settings) + + llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len) + + int llhttp_should_keep_alive(const llhttp_t* parser) + + void llhttp_resume(llhttp_t* parser) + void llhttp_resume_after_upgrade(llhttp_t* parser) + + llhttp_errno_t llhttp_get_errno(const llhttp_t* parser) + const char* llhttp_get_error_reason(const llhttp_t* parser) + const char* llhttp_get_error_pos(const llhttp_t* parser) + + const char* llhttp_method_name(llhttp_method_t method) + + void llhttp_set_lenient_headers(llhttp_t* parser, int enabled) + void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled) + void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled) diff --git a/venv/Lib/site-packages/aiohttp/_find_header.pxd b/venv/Lib/site-packages/aiohttp/_find_header.pxd new file mode 100644 index 0000000000000000000000000000000000000000..9305cb61da27a827c280b3c7ea3ca65e96c8793b --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_find_header.pxd @@ -0,0 +1,2 @@ +cdef extern from "_find_header.h": + int find_header(char *, int) diff --git a/venv/Lib/site-packages/aiohttp/_headers.pxi b/venv/Lib/site-packages/aiohttp/_headers.pxi new file mode 100644 index 0000000000000000000000000000000000000000..45c50f15e8c5e77996bf29315c13027b7b8f66ef --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_headers.pxi @@ -0,0 +1,83 @@ +# The file is autogenerated from aiohttp/hdrs.py +# Run ./tools/gen.py to update it after the origin changing. + +from . import hdrs +cdef tuple headers = ( + hdrs.ACCEPT, + hdrs.ACCEPT_CHARSET, + hdrs.ACCEPT_ENCODING, + hdrs.ACCEPT_LANGUAGE, + hdrs.ACCEPT_RANGES, + hdrs.ACCESS_CONTROL_ALLOW_CREDENTIALS, + hdrs.ACCESS_CONTROL_ALLOW_HEADERS, + hdrs.ACCESS_CONTROL_ALLOW_METHODS, + hdrs.ACCESS_CONTROL_ALLOW_ORIGIN, + hdrs.ACCESS_CONTROL_EXPOSE_HEADERS, + hdrs.ACCESS_CONTROL_MAX_AGE, + hdrs.ACCESS_CONTROL_REQUEST_HEADERS, + hdrs.ACCESS_CONTROL_REQUEST_METHOD, + hdrs.AGE, + hdrs.ALLOW, + hdrs.AUTHORIZATION, + hdrs.CACHE_CONTROL, + hdrs.CONNECTION, + hdrs.CONTENT_DISPOSITION, + hdrs.CONTENT_ENCODING, + hdrs.CONTENT_LANGUAGE, + hdrs.CONTENT_LENGTH, + hdrs.CONTENT_LOCATION, + hdrs.CONTENT_MD5, + hdrs.CONTENT_RANGE, + hdrs.CONTENT_TRANSFER_ENCODING, + hdrs.CONTENT_TYPE, + hdrs.COOKIE, + hdrs.DATE, + hdrs.DESTINATION, + hdrs.DIGEST, + hdrs.ETAG, + hdrs.EXPECT, + hdrs.EXPIRES, + hdrs.FORWARDED, + hdrs.FROM, + hdrs.HOST, + hdrs.IF_MATCH, + hdrs.IF_MODIFIED_SINCE, + hdrs.IF_NONE_MATCH, + hdrs.IF_RANGE, + hdrs.IF_UNMODIFIED_SINCE, + hdrs.KEEP_ALIVE, + hdrs.LAST_EVENT_ID, + hdrs.LAST_MODIFIED, + hdrs.LINK, + hdrs.LOCATION, + hdrs.MAX_FORWARDS, + hdrs.ORIGIN, + hdrs.PRAGMA, + hdrs.PROXY_AUTHENTICATE, + hdrs.PROXY_AUTHORIZATION, + hdrs.RANGE, + hdrs.REFERER, + hdrs.RETRY_AFTER, + hdrs.SEC_WEBSOCKET_ACCEPT, + hdrs.SEC_WEBSOCKET_EXTENSIONS, + hdrs.SEC_WEBSOCKET_KEY, + hdrs.SEC_WEBSOCKET_KEY1, + hdrs.SEC_WEBSOCKET_PROTOCOL, + hdrs.SEC_WEBSOCKET_VERSION, + hdrs.SERVER, + hdrs.SET_COOKIE, + hdrs.TE, + hdrs.TRAILER, + hdrs.TRANSFER_ENCODING, + hdrs.URI, + hdrs.UPGRADE, + hdrs.USER_AGENT, + hdrs.VARY, + hdrs.VIA, + hdrs.WWW_AUTHENTICATE, + hdrs.WANT_DIGEST, + hdrs.WARNING, + hdrs.X_FORWARDED_FOR, + hdrs.X_FORWARDED_HOST, + hdrs.X_FORWARDED_PROTO, +) diff --git a/venv/Lib/site-packages/aiohttp/_http_parser.cp311-win_amd64.pyd b/venv/Lib/site-packages/aiohttp/_http_parser.cp311-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..3cf904e8077bffe6634075fee5537a988f603090 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_http_parser.cp311-win_amd64.pyd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:511cd7062151ae724f1f194d009caa38cf0c2bbfac709b5476dcc8354a0575ed +size 252928 diff --git a/venv/Lib/site-packages/aiohttp/_http_parser.pyx b/venv/Lib/site-packages/aiohttp/_http_parser.pyx new file mode 100644 index 0000000000000000000000000000000000000000..010f968ebabf7ce59bc4c80133715220d5da96e7 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_http_parser.pyx @@ -0,0 +1,965 @@ +# Based on https://github.com/MagicStack/httptools +# + +from cpython cimport ( + Py_buffer, + PyBUF_SIMPLE, + PyBuffer_Release, + PyBytes_AsString, + PyBytes_AsStringAndSize, + PyObject_GetBuffer, +) +from cpython.mem cimport PyMem_Free, PyMem_Malloc +from libc.limits cimport ULLONG_MAX +from libc.string cimport memcpy + +from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiDictProxy +from yarl import URL as _URL + +from aiohttp import hdrs +from aiohttp.helpers import DEBUG, set_exception + +from .http_exceptions import ( + BadHttpMessage, + BadHttpMethod, + BadStatusLine, + ContentLengthError, + InvalidHeader, + InvalidURLError, + LineTooLong, + PayloadEncodingError, + TransferEncodingError, +) +from .http_parser import DeflateBuffer as _DeflateBuffer +from .http_writer import ( + HttpVersion as _HttpVersion, + HttpVersion10 as _HttpVersion10, + HttpVersion11 as _HttpVersion11, +) +from .streams import EMPTY_PAYLOAD as _EMPTY_PAYLOAD, StreamReader as _StreamReader + +cimport cython + +from aiohttp cimport _cparser as cparser + +include "_headers.pxi" + +from aiohttp cimport _find_header + + +cdef frozenset ALLOWED_UPGRADES = frozenset({"websocket"}) +DEF DEFAULT_FREELIST_SIZE = 250 + +cdef extern from "Python.h": + int PyByteArray_Resize(object, Py_ssize_t) except -1 + Py_ssize_t PyByteArray_Size(object) except -1 + char* PyByteArray_AsString(object) + +__all__ = ('HttpRequestParser', 'HttpResponseParser', + 'RawRequestMessage', 'RawResponseMessage') + +cdef object URL = _URL +cdef object URL_build = URL.build +cdef object CIMultiDict = _CIMultiDict +cdef object CIMultiDictProxy = _CIMultiDictProxy +cdef object HttpVersion = _HttpVersion +cdef object HttpVersion10 = _HttpVersion10 +cdef object HttpVersion11 = _HttpVersion11 +cdef object SEC_WEBSOCKET_KEY1 = hdrs.SEC_WEBSOCKET_KEY1 +cdef object CONTENT_ENCODING = hdrs.CONTENT_ENCODING +cdef object EMPTY_PAYLOAD = _EMPTY_PAYLOAD +cdef object StreamReader = _StreamReader +cdef object DeflateBuffer = _DeflateBuffer +cdef tuple EMPTY_FEED_DATA_RESULT = ((), False, b"") + +# RFC 9110 singleton headers β€” duplicates are rejected in strict mode. +# In lax mode (response parser default), the check is skipped entirely +# since real-world servers (e.g. Google APIs, Werkzeug) commonly send +# duplicate headers like Content-Type or Server. +cdef frozenset SINGLETON_HEADERS = frozenset({ + hdrs.CONTENT_LENGTH, + hdrs.CONTENT_LOCATION, + hdrs.CONTENT_RANGE, + hdrs.CONTENT_TYPE, + hdrs.ETAG, + hdrs.HOST, + hdrs.MAX_FORWARDS, + hdrs.SERVER, + hdrs.TRANSFER_ENCODING, + hdrs.USER_AGENT, +}) + +cdef inline object extend(object buf, const char* at, size_t length): + cdef Py_ssize_t s + cdef char* ptr + s = PyByteArray_Size(buf) + PyByteArray_Resize(buf, s + length) + ptr = PyByteArray_AsString(buf) + memcpy(ptr + s, at, length) + + +DEF METHODS_COUNT = 46; + +cdef list _http_method = [] + +for i in range(METHODS_COUNT): + _http_method.append( + cparser.llhttp_method_name( i).decode('ascii')) + + +cdef inline str http_method_str(int i): + if i < METHODS_COUNT: + return _http_method[i] + else: + return "" + +cdef inline object find_header(bytes raw_header): + cdef Py_ssize_t size + cdef char *buf + cdef int idx + PyBytes_AsStringAndSize(raw_header, &buf, &size) + idx = _find_header.find_header(buf, size) + if idx == -1: + return raw_header.decode('utf-8', 'surrogateescape') + return headers[idx] + + +@cython.freelist(DEFAULT_FREELIST_SIZE) +cdef class RawRequestMessage: + cdef readonly str method + cdef readonly str path + cdef readonly object version # HttpVersion + cdef readonly object headers # CIMultiDict + cdef readonly object raw_headers # tuple + cdef readonly object should_close + cdef readonly object compression + cdef readonly object upgrade + cdef readonly object chunked + cdef readonly object url # yarl.URL + + def __init__(self, method, path, version, headers, raw_headers, + should_close, compression, upgrade, chunked, url): + self.method = method + self.path = path + self.version = version + self.headers = headers + self.raw_headers = raw_headers + self.should_close = should_close + self.compression = compression + self.upgrade = upgrade + self.chunked = chunked + self.url = url + + def __repr__(self): + info = [] + info.append(("method", self.method)) + info.append(("path", self.path)) + info.append(("version", self.version)) + info.append(("headers", self.headers)) + info.append(("raw_headers", self.raw_headers)) + info.append(("should_close", self.should_close)) + info.append(("compression", self.compression)) + info.append(("upgrade", self.upgrade)) + info.append(("chunked", self.chunked)) + info.append(("url", self.url)) + sinfo = ', '.join(name + '=' + repr(val) for name, val in info) + return '' + + def _replace(self, **dct): + cdef RawRequestMessage ret + ret = _new_request_message(self.method, + self.path, + self.version, + self.headers, + self.raw_headers, + self.should_close, + self.compression, + self.upgrade, + self.chunked, + self.url) + if "method" in dct: + ret.method = dct["method"] + if "path" in dct: + ret.path = dct["path"] + if "version" in dct: + ret.version = dct["version"] + if "headers" in dct: + ret.headers = dct["headers"] + if "raw_headers" in dct: + ret.raw_headers = dct["raw_headers"] + if "should_close" in dct: + ret.should_close = dct["should_close"] + if "compression" in dct: + ret.compression = dct["compression"] + if "upgrade" in dct: + ret.upgrade = dct["upgrade"] + if "chunked" in dct: + ret.chunked = dct["chunked"] + if "url" in dct: + ret.url = dct["url"] + return ret + +cdef _new_request_message(str method, + str path, + object version, + object headers, + object raw_headers, + bint should_close, + object compression, + bint upgrade, + bint chunked, + object url): + cdef RawRequestMessage ret + ret = RawRequestMessage.__new__(RawRequestMessage) + ret.method = method + ret.path = path + ret.version = version + ret.headers = headers + ret.raw_headers = raw_headers + ret.should_close = should_close + ret.compression = compression + ret.upgrade = upgrade + ret.chunked = chunked + ret.url = url + return ret + + +@cython.freelist(DEFAULT_FREELIST_SIZE) +cdef class RawResponseMessage: + cdef readonly object version # HttpVersion + cdef readonly int code + cdef readonly str reason + cdef readonly object headers # CIMultiDict + cdef readonly object raw_headers # tuple + cdef readonly object should_close + cdef readonly object compression + cdef readonly object upgrade + cdef readonly object chunked + + def __init__(self, version, code, reason, headers, raw_headers, + should_close, compression, upgrade, chunked): + self.version = version + self.code = code + self.reason = reason + self.headers = headers + self.raw_headers = raw_headers + self.should_close = should_close + self.compression = compression + self.upgrade = upgrade + self.chunked = chunked + + def __repr__(self): + info = [] + info.append(("version", self.version)) + info.append(("code", self.code)) + info.append(("reason", self.reason)) + info.append(("headers", self.headers)) + info.append(("raw_headers", self.raw_headers)) + info.append(("should_close", self.should_close)) + info.append(("compression", self.compression)) + info.append(("upgrade", self.upgrade)) + info.append(("chunked", self.chunked)) + sinfo = ', '.join(name + '=' + repr(val) for name, val in info) + return '' + + +cdef _new_response_message(object version, + int code, + str reason, + object headers, + object raw_headers, + bint should_close, + object compression, + bint upgrade, + bint chunked): + cdef RawResponseMessage ret + ret = RawResponseMessage.__new__(RawResponseMessage) + ret.version = version + ret.code = code + ret.reason = reason + ret.headers = headers + ret.raw_headers = raw_headers + ret.should_close = should_close + ret.compression = compression + ret.upgrade = upgrade + ret.chunked = chunked + return ret + + +@cython.internal +cdef class HttpParser: + + cdef: + cparser.llhttp_t* _cparser + cparser.llhttp_settings_t* _csettings + + bytes _raw_name + object _name + bytes _raw_value + bint _has_value + int _header_name_size + + readonly object protocol + object _loop + object _timer + + size_t _max_line_size + size_t _max_field_size + size_t _max_headers + bint _response_with_body + bint _read_until_eof + bint _lax + + bytes _tail + bint _started + object _url + bytearray _buf + str _path + str _reason + list _headers + set _seen_singletons + list _raw_headers + bint _upgraded + list _messages + bint _more_data_available + bint _paused + Py_ssize_t _msg_in_flight + Py_ssize_t _max_msg_queue_size + bint _eof_pending + object _payload + unsigned long long _content_length_expected + bint _payload_error + object _payload_exception + object _last_error + bint _auto_decompress + int _limit + + str _content_encoding + + Py_buffer py_buf + + def __cinit__(self): + self._cparser = \ + PyMem_Malloc(sizeof(cparser.llhttp_t)) + if self._cparser is NULL: + raise MemoryError() + + self._csettings = \ + PyMem_Malloc(sizeof(cparser.llhttp_settings_t)) + if self._csettings is NULL: + raise MemoryError() + + def __dealloc__(self): + PyMem_Free(self._cparser) + PyMem_Free(self._csettings) + + cdef _init( + self, cparser.llhttp_type mode, + object protocol, object loop, int limit, + object timer=None, + size_t max_line_size=8190, size_t max_headers=128, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True, + Py_ssize_t max_msg_queue_size=0, + ): + cparser.llhttp_settings_init(self._csettings) + cparser.llhttp_init(self._cparser, mode, self._csettings) + self._cparser.data = self + self._cparser.content_length = 0 + self._content_length_expected = 0 + + self.protocol = protocol + self._loop = loop + self._timer = timer + + self._buf = bytearray() + self._more_data_available = False + self._paused = False + self._msg_in_flight = 0 + self._max_msg_queue_size = max_msg_queue_size + self._eof_pending = False + self._payload = None + self._payload_error = 0 + self._payload_exception = payload_exception + self._messages = [] + + self._raw_name = b"" + self._raw_value = b"" + self._tail = b"" + self._has_value = False + self._header_name_size = 0 + + self._max_line_size = max_line_size + self._max_headers = max_headers + self._max_field_size = max_field_size + self._response_with_body = response_with_body + self._read_until_eof = read_until_eof + self._upgraded = False + self._auto_decompress = auto_decompress + self._content_encoding = None + self._lax = False + self._seen_singletons = set() + + self._csettings.on_url = cb_on_url + self._csettings.on_status = cb_on_status + self._csettings.on_header_field = cb_on_header_field + self._csettings.on_header_value = cb_on_header_value + self._csettings.on_headers_complete = cb_on_headers_complete + self._csettings.on_body = cb_on_body + self._csettings.on_message_begin = cb_on_message_begin + self._csettings.on_message_complete = cb_on_message_complete + self._csettings.on_chunk_header = cb_on_chunk_header + self._csettings.on_chunk_complete = cb_on_chunk_complete + + self._last_error = None + self._limit = limit + + cdef _process_header(self): + cdef str value + if self._raw_name != b"": + name = find_header(self._raw_name) + value = self._raw_value.decode('utf-8', 'surrogateescape') + + # reject null bytes in header values - matches the Python parser + # check at http_parser.py. llhttp in lenient mode doesn't reject + # these itself, so we need to catch them here. + # ref: RFC 9110 section 5.5 (CTL chars forbidden in field values) + if "\x00" in value: + raise InvalidHeader(self._raw_value) + + if not self._lax and name in SINGLETON_HEADERS: + if name in self._seen_singletons: + raise BadHttpMessage(f"Duplicate '{name}' header found.") + self._seen_singletons.add(name) + self._headers.append((name, value)) + if len(self._headers) > self._max_headers: + raise BadHttpMessage("Too many headers received") + + if name is CONTENT_ENCODING: + self._content_encoding = value + + self._has_value = False + self._header_name_size = 0 + self._raw_headers.append((self._raw_name, self._raw_value)) + self._raw_name = b"" + self._raw_value = b"" + + cdef _on_header_field(self, char* at, size_t length): + if self._has_value: + self._process_header() + + if self._raw_name == b"": + self._raw_name = at[:length] + else: + self._raw_name += at[:length] + + cdef _on_header_value(self, char* at, size_t length): + if self._raw_value == b"": + self._raw_value = at[:length] + else: + self._raw_value += at[:length] + self._has_value = True + + cdef _on_headers_complete(self): + cdef str h_upg + cdef str enc + + self._process_header() + + http_version = self.http_version() + should_close = not cparser.llhttp_should_keep_alive(self._cparser) + upgrade = self._cparser.upgrade + chunked = self._cparser.flags & cparser.F_CHUNKED + + raw_headers = tuple(self._raw_headers) + headers = CIMultiDictProxy(CIMultiDict(self._headers)) + + if self._cparser.type == cparser.HTTP_REQUEST: + if http_version == HttpVersion11 and hdrs.HOST not in headers: + raise BadHttpMessage("Missing 'Host' header in request.") + h_upg = headers.get("upgrade", "") + if (upgrade and h_upg.isascii() and h_upg.lower() in ALLOWED_UPGRADES) or self._cparser.method == cparser.HTTP_CONNECT: + self._upgraded = True + else: + if upgrade and self._cparser.status_code == 101: + self._upgraded = True + + # do not support old websocket spec + if SEC_WEBSOCKET_KEY1 in headers: + raise InvalidHeader(SEC_WEBSOCKET_KEY1) + + encoding = None + enc = self._content_encoding + if enc is not None: + self._content_encoding = None + if enc.isascii() and enc.lower() in {"gzip", "deflate", "br", "zstd"}: + encoding = enc + + if self._cparser.type == cparser.HTTP_REQUEST: + method = http_method_str(self._cparser.method) + msg = _new_request_message( + method, self._path, + http_version, headers, raw_headers, + should_close, encoding, upgrade, chunked, self._url) + else: + msg = _new_response_message( + http_version, self._cparser.status_code, self._reason, + headers, raw_headers, should_close, encoding, + upgrade, chunked) + + if ( + self._response_with_body + and ( + ULLONG_MAX > self._cparser.content_length > 0 or chunked or + self._cparser.method == cparser.HTTP_CONNECT or + (self._cparser.status_code >= 199 and + self._cparser.content_length == 0 and + self._read_until_eof) + ) + ): + payload = StreamReader( + self.protocol, timer=self._timer, loop=self._loop, + limit=self._limit) + else: + payload = EMPTY_PAYLOAD + + self._payload = payload + self._content_length_expected = self._cparser.content_length + if encoding is not None and self._auto_decompress: + self._payload = DeflateBuffer(payload, encoding, max_decompress_size=self._limit) + + self._messages.append((msg, payload)) + + cdef _on_message_complete(self): + self._payload.feed_eof() + self._payload = None + + cdef _on_chunk_header(self): + self._payload.begin_http_chunk_receiving() + + cdef _on_chunk_complete(self): + self._payload.end_http_chunk_receiving() + + cdef object _on_status_complete(self): + pass + + cdef inline http_version(self): + cdef cparser.llhttp_t* parser = self._cparser + + if parser.http_major == 1: + if parser.http_minor == 0: + return HttpVersion10 + elif parser.http_minor == 1: + return HttpVersion11 + + return HttpVersion(parser.http_major, parser.http_minor) + + ### Public API ### + + def pause_reading(self): + assert self._payload is not None + self._paused = True + + def message_consumed(self): + # Protocol drained a queued message; free a slot for parsing. + if self._msg_in_flight > 0: + self._msg_in_flight -= 1 + + def feed_eof(self): + cdef bytes desc + + if self._payload is not None: + if self._cparser.flags & cparser.F_CHUNKED: + raise TransferEncodingError( + "Not enough data to satisfy transfer length header.") + elif self._cparser.flags & cparser.F_CONTENT_LENGTH: + received = self._content_length_expected - self._cparser.content_length + raise ContentLengthError( + f"Not enough data to satisfy content length header " + f"(received {received} of {self._content_length_expected} bytes).") + elif cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK: + desc = cparser.llhttp_get_error_reason(self._cparser) + raise PayloadEncodingError(desc.decode('latin-1')) + else: + self._eof_pending = True + while self._more_data_available: + if self._paused: + self._paused = False + return # Will resume via feed_data(b"") later + self._more_data_available = self._payload.feed_data(b"", 0) + self._payload.feed_eof() + self._payload = None + self._more_data_available = False + self._eof_pending = False + elif self._started: + self._on_headers_complete() + if self._messages: + return self._messages[-1][0] + + def feed_data(self, incoming_data): + cdef: + size_t data_len + size_t nb + char* base + cdef cparser.llhttp_errno_t errno + cdef bytes data + + # Proactor loop sends bytearray. + # Ensure cython sees `data` as bytes + if type(incoming_data) is not bytes: + data = bytes(incoming_data) + else: + data = incoming_data + + if self._tail: + data, self._tail = self._tail + data, b"" + + if self._more_data_available: + result = cb_on_body(self._cparser, b"", 0) + if result is cparser.HPE_PAUSED: + self._tail = data + return EMPTY_FEED_DATA_RESULT + + if self._eof_pending: + self._payload.feed_eof() + self._payload = None + self._eof_pending = False + # We can't have new messages here, otherwise we wouldn't have + # received EOF. + return EMPTY_FEED_DATA_RESULT + + PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) + # Cache buffer pointer before PyBuffer_Release to avoid use-after-release. + base = self.py_buf.buf + data_len = self.py_buf.len + + errno = cparser.llhttp_execute( + self._cparser, + base, + data_len) + + if errno is cparser.HPE_PAUSED_UPGRADE: + cparser.llhttp_resume_after_upgrade(self._cparser) + nb = cparser.llhttp_get_error_pos(self._cparser) - base + elif errno is cparser.HPE_PAUSED: + cparser.llhttp_resume(self._cparser) + pos = cparser.llhttp_get_error_pos(self._cparser) - base + self._tail = data[pos:] + + PyBuffer_Release(&self.py_buf) + + if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED, cparser.HPE_PAUSED_UPGRADE): + if self._payload_error == 0: + if self._last_error is not None: + ex = self._last_error + self._last_error = None + else: + after = cparser.llhttp_get_error_pos(self._cparser) + before = data[:after - base] + after_b = after.split(b"\r\n", 1)[0] + before = before.rsplit(b"\r\n", 1)[-1] + data = before + after_b + pointer = " " * (len(repr(before))-1) + "^" + ex = parser_error_from_errno(self._cparser, data, pointer) + self._payload = None + raise ex + + if self._messages: + messages = self._messages + self._messages = [] + else: + messages = () + + if self._upgraded: + return messages, True, data[nb:] + if not messages: # Shortcut to reduce Python overhead + return EMPTY_FEED_DATA_RESULT + return messages, False, b"" + + def set_upgraded(self, val): + self._upgraded = val + + +cdef class HttpRequestParser(HttpParser): + + def __init__( + self, protocol, loop, int limit, timer=None, + size_t max_line_size=8190, size_t max_headers=128, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True, Py_ssize_t max_msg_queue_size=0, + ): + self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer, + max_line_size, max_headers, max_field_size, + payload_exception, response_with_body, read_until_eof, + auto_decompress, max_msg_queue_size) + + cdef object _on_status_complete(self): + cdef int idx1, idx2 + if not self._buf: + return + self._path = self._buf.decode('utf-8', 'surrogateescape') + try: + idx3 = len(self._path) + if self._cparser.method == cparser.HTTP_CONNECT: + # authority-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3 + self._url = URL.build(authority=self._path, encoded=True) + elif idx3 > 1 and self._path[0] == '/': + # origin-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1 + idx1 = self._path.find("?") + if idx1 == -1: + query = "" + idx2 = self._path.find("#") + if idx2 == -1: + path = self._path + fragment = "" + else: + path = self._path[0: idx2] + fragment = self._path[idx2+1:] + + else: + path = self._path[0:idx1] + idx1 += 1 + idx2 = self._path.find("#", idx1+1) + if idx2 == -1: + query = self._path[idx1:] + fragment = "" + else: + query = self._path[idx1: idx2] + fragment = self._path[idx2+1:] + + self._url = URL.build( + path=path, + query_string=query, + fragment=fragment, + encoded=True, + ) + else: + # absolute-form for proxy maybe, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2 + self._url = URL(self._path, encoded=True) + finally: + PyByteArray_Resize(self._buf, 0) + + +cdef class HttpResponseParser(HttpParser): + + def __init__( + self, protocol, loop, int limit, timer=None, + size_t max_line_size=8190, size_t max_headers=128, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True + ): + self._init(cparser.HTTP_RESPONSE, protocol, loop, limit, timer, + max_line_size, max_headers, max_field_size, + payload_exception, response_with_body, read_until_eof, + auto_decompress) + # Use strict parsing on dev mode, so users are warned about broken servers. + if not DEBUG: + cparser.llhttp_set_lenient_headers(self._cparser, 1) + cparser.llhttp_set_lenient_optional_cr_before_lf(self._cparser, 1) + cparser.llhttp_set_lenient_spaces_after_chunk_size(self._cparser, 1) + self._lax = True + + cdef object _on_status_complete(self): + if self._buf: + self._reason = self._buf.decode('utf-8', 'surrogateescape') + PyByteArray_Resize(self._buf, 0) + else: + self._reason = self._reason or '' + +cdef int cb_on_message_begin(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + + pyparser._started = True + pyparser._headers = [] + pyparser._seen_singletons = set() + pyparser._raw_headers = [] + PyByteArray_Resize(pyparser._buf, 0) + pyparser._path = None + pyparser._reason = None + return 0 + + +cdef int cb_on_url(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + try: + if len(pyparser._buf) + length > pyparser._max_line_size: + status = pyparser._buf + at[:length] + raise LineTooLong(status[:100] + b"...", pyparser._max_line_size) + extend(pyparser._buf, at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_status(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + try: + if len(pyparser._buf) + length > pyparser._max_line_size: + reason = pyparser._buf + at[:length] + raise LineTooLong(reason[:100] + b"...", pyparser._max_line_size) + extend(pyparser._buf, at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_header_field(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef Py_ssize_t size + try: + pyparser._on_status_complete() + size = len(pyparser._raw_name) + length + if size > pyparser._max_field_size: + name = pyparser._raw_name + at[:length] + raise LineTooLong(name[:100] + b"...", pyparser._max_field_size) + pyparser._header_name_size = size + pyparser._on_header_field(at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_header_value(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef Py_ssize_t size + try: + size = len(pyparser._raw_value) + length + if pyparser._header_name_size + size > pyparser._max_field_size: + value = pyparser._raw_value + at[:length] + raise LineTooLong(value[:100] + b"...", pyparser._max_field_size) + pyparser._on_header_value(at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_status_complete() + pyparser._on_headers_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + if pyparser._upgraded or pyparser._cparser.method == cparser.HTTP_CONNECT: + return 2 + if not pyparser._response_with_body: + return 1 + return 0 + + +cdef int cb_on_body(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef bytes body = at[:length] + while body or pyparser._more_data_available: + try: + pyparser._more_data_available = pyparser._payload.feed_data(body, length) + except BaseException as underlying_exc: + reraised_exc = underlying_exc + if pyparser._payload_exception is not None: + reraised_exc = pyparser._payload_exception(str(underlying_exc)) + + set_exception(pyparser._payload, reraised_exc, underlying_exc) + + pyparser._payload_error = 1 + pyparser._paused = False + return -1 + body = b"" + length = 0 + + if pyparser._paused: + pyparser._paused = False + return cparser.HPE_PAUSED + pyparser._paused = False + return 0 + + +cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._started = False + pyparser._on_message_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + if pyparser._max_msg_queue_size: + pyparser._msg_in_flight += 1 + if pyparser._msg_in_flight >= pyparser._max_msg_queue_size: + # Queue full: pause llhttp between messages. feed_data() buffers + # the remainder as tail; resumes once the queue drains. + return cparser.HPE_PAUSED + return 0 + + +cdef int cb_on_chunk_header(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_chunk_header() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + return 0 + + +cdef int cb_on_chunk_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_chunk_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + return 0 + + +cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer): + cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(parser) + cdef bytes desc = cparser.llhttp_get_error_reason(parser) + + err_msg = "{}:\n\n {!r}\n {}".format(desc.decode("latin-1"), data, pointer) + + if errno in {cparser.HPE_CB_MESSAGE_BEGIN, + cparser.HPE_CB_HEADERS_COMPLETE, + cparser.HPE_CB_MESSAGE_COMPLETE, + cparser.HPE_CB_CHUNK_HEADER, + cparser.HPE_CB_CHUNK_COMPLETE, + cparser.HPE_INVALID_HEADER_TOKEN, + cparser.HPE_INVALID_CONTENT_LENGTH, + cparser.HPE_INVALID_CHUNK_SIZE, + cparser.HPE_INVALID_EOF_STATE, + cparser.HPE_INVALID_TRANSFER_ENCODING}: + return BadHttpMessage(err_msg) + elif errno == cparser.HPE_INVALID_METHOD: + if data.startswith(b"\x16\x03"): + return BadHttpMethod(error="Received HTTPS traffic on an HTTP port") + return BadHttpMethod(error=err_msg) + elif errno in {cparser.HPE_INVALID_STATUS, + cparser.HPE_INVALID_VERSION, + cparser.HPE_INVALID_CONSTANT}: + return BadStatusLine(error=f"Bad status line:\n {err_msg}") + elif errno == cparser.HPE_INVALID_URL: + return InvalidURLError(err_msg) + + return BadHttpMessage(err_msg) diff --git a/venv/Lib/site-packages/aiohttp/_http_writer.cp311-win_amd64.pyd b/venv/Lib/site-packages/aiohttp/_http_writer.cp311-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..c2a9d1cbffbd7be7610c6d36e45bd2e37956bfca Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_http_writer.cp311-win_amd64.pyd differ diff --git a/venv/Lib/site-packages/aiohttp/_http_writer.pyx b/venv/Lib/site-packages/aiohttp/_http_writer.pyx new file mode 100644 index 0000000000000000000000000000000000000000..8a86c900ddfcfedbcb839ae0e17b37eb8e6efc94 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_http_writer.pyx @@ -0,0 +1,164 @@ +from cpython.bytes cimport PyBytes_FromStringAndSize +from cpython.exc cimport PyErr_NoMemory +from cpython.mem cimport PyMem_Free, PyMem_Malloc, PyMem_Realloc +from cpython.object cimport PyObject_Str +from libc.stdint cimport uint8_t, uint64_t +from libc.string cimport memcpy + +from multidict import istr + +DEF BUF_SIZE = 16 * 1024 # 16KiB + +cdef object _istr = istr + + +# ----------------- writer --------------------------- + +cdef struct Writer: + char *buf + Py_ssize_t size + Py_ssize_t pos + bint heap_allocated + +cdef inline void _init_writer(Writer* writer, char *buf): + writer.buf = buf + writer.size = BUF_SIZE + writer.pos = 0 + writer.heap_allocated = 0 + + +cdef inline void _release_writer(Writer* writer): + if writer.heap_allocated: + PyMem_Free(writer.buf) + + +cdef inline int _write_byte(Writer* writer, uint8_t ch): + cdef char * buf + cdef Py_ssize_t size + + if writer.pos == writer.size: + # reallocate + size = writer.size + BUF_SIZE + if not writer.heap_allocated: + buf = PyMem_Malloc(size) + if buf == NULL: + PyErr_NoMemory() + return -1 + memcpy(buf, writer.buf, writer.size) + else: + buf = PyMem_Realloc(writer.buf, size) + if buf == NULL: + PyErr_NoMemory() + return -1 + writer.buf = buf + writer.size = size + writer.heap_allocated = 1 + writer.buf[writer.pos] = ch + writer.pos += 1 + return 0 + + +cdef inline int _write_utf8(Writer* writer, Py_UCS4 symbol): + cdef uint64_t utf = symbol + + if utf < 0x80: + return _write_byte(writer, utf) + elif utf < 0x800: + if _write_byte(writer, (0xc0 | (utf >> 6))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + elif 0xD800 <= utf <= 0xDFFF: + # surogate pair, ignored + return 0 + elif utf < 0x10000: + if _write_byte(writer, (0xe0 | (utf >> 12))) < 0: + return -1 + if _write_byte(writer, (0x80 | ((utf >> 6) & 0x3f))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + elif utf > 0x10FFFF: + # symbol is too large + return 0 + else: + if _write_byte(writer, (0xf0 | (utf >> 18))) < 0: + return -1 + if _write_byte(writer, + (0x80 | ((utf >> 12) & 0x3f))) < 0: + return -1 + if _write_byte(writer, + (0x80 | ((utf >> 6) & 0x3f))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + + +cdef inline int _write_str(Writer* writer, str s): + cdef Py_UCS4 ch + for ch in s: + if _write_utf8(writer, ch) < 0: + return -1 + + +cdef inline int _write_str_raise_on_nlcr(Writer* writer, object s): + cdef Py_UCS4 ch + cdef str out_str + if type(s) is str: + out_str = s + elif type(s) is _istr: + out_str = PyObject_Str(s) + elif not isinstance(s, str): + raise TypeError("Cannot serialize non-str key {!r}".format(s)) + else: + out_str = str(s) + + for ch in out_str: + # https://www.rfc-editor.org/info/rfc9110/#section-5.5-5 + # https://www.rfc-editor.org/info/rfc9112/#section-4-3 + if (ch < 0x20 and ch != 0x09) or ch == 0x7F: + raise ValueError( + "Forbidden control character detected in headers. " + "Potential header injection attack." + ) + if _write_utf8(writer, ch) < 0: + return -1 + + +# --------------- _serialize_headers ---------------------- + +def _serialize_headers(str status_line, headers): + cdef Writer writer + cdef object key + cdef object val + cdef char buf[BUF_SIZE] + + _init_writer(&writer, buf) + + try: + if _write_str_raise_on_nlcr(&writer, status_line) < 0: + raise + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + for key, val in headers.items(): + if _write_str_raise_on_nlcr(&writer, key) < 0: + raise + if _write_byte(&writer, b':') < 0: + raise + if _write_byte(&writer, b' ') < 0: + raise + if _write_str_raise_on_nlcr(&writer, val) < 0: + raise + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + return PyBytes_FromStringAndSize(writer.buf, writer.pos) + finally: + _release_writer(&writer) diff --git a/venv/Lib/site-packages/aiohttp/_websocket/.hash/mask.pxd.hash b/venv/Lib/site-packages/aiohttp/_websocket/.hash/mask.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..59ec1233e51ca80d766bc036bbfcc1fded2bb7dc --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/.hash/mask.pxd.hash @@ -0,0 +1 @@ +e354dd499be171b6125bf56bc3b6c5e2bff2a28af69e3b5d699ddb9af2bafa3c *D:/a/aiohttp/aiohttp/aiohttp/_websocket/mask.pxd diff --git a/venv/Lib/site-packages/aiohttp/_websocket/.hash/mask.pyx.hash b/venv/Lib/site-packages/aiohttp/_websocket/.hash/mask.pyx.hash new file mode 100644 index 0000000000000000000000000000000000000000..025189ec4bd962d7b10b160b40f6faab3db37c9b --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/.hash/mask.pyx.hash @@ -0,0 +1 @@ +468edd38ebf8dc7000a8d333df1c82035d69a5c9febc0448be3c9c4ad4c4630c *D:/a/aiohttp/aiohttp/aiohttp/_websocket/mask.pyx diff --git a/venv/Lib/site-packages/aiohttp/_websocket/.hash/reader_c.pxd.hash b/venv/Lib/site-packages/aiohttp/_websocket/.hash/reader_c.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..c7df26c67c9cf1272b830fbc5bf93c04806b1abc --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/.hash/reader_c.pxd.hash @@ -0,0 +1 @@ +fad990691939b116b302179e938eef47e6cbabf5fe5a30bb465c1626de5eb614 *D:/a/aiohttp/aiohttp/aiohttp/_websocket/reader_c.pxd diff --git a/venv/Lib/site-packages/aiohttp/_websocket/__init__.py b/venv/Lib/site-packages/aiohttp/_websocket/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a00ebcf833d69d49c9f0b2167f1b31623c6cf6d1 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/__init__.py @@ -0,0 +1 @@ +"""WebSocket protocol versions 13 and 8.""" diff --git a/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46c74a431deb10b20e6a77f1fae93354b49ae35b Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/__init__.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/helpers.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/helpers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d704ba31846e9f22afb0fd78064fd01e6a47e79 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/helpers.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/models.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/models.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b530f8132dfc151e1d2bb40d2422f4bdcfa95dc6 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/models.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59063d0c14fc1eb589e639411c8c2e7bf659706e Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader_c.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader_c.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13f9b2f325b552f901e32c24ecde895ab680a9d0 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader_c.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader_py.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader_py.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe2ad228cc1f53273fe0f6d3ad0e49ac6bf18d54 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/reader_py.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/writer.cpython-311.pyc b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/writer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a727dc1818c11169348b1da0b95694d739bc52e6 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_websocket/__pycache__/writer.cpython-311.pyc differ diff --git a/venv/Lib/site-packages/aiohttp/_websocket/helpers.py b/venv/Lib/site-packages/aiohttp/_websocket/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..950c799eded0ebb54581f6376ef6a628b0ff3e89 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/helpers.py @@ -0,0 +1,148 @@ +"""Helpers for WebSocket protocol versions 13 and 8.""" + +import functools +import re +from re import Pattern +from struct import Struct +from typing import TYPE_CHECKING, Final + +from ..helpers import NO_EXTENSIONS +from .models import WSHandshakeError + +UNPACK_LEN3 = Struct("!Q").unpack_from +UNPACK_CLOSE_CODE = Struct("!H").unpack +PACK_LEN1 = Struct("!BB").pack +PACK_LEN2 = Struct("!BBH").pack +PACK_LEN3 = Struct("!BBQ").pack +PACK_CLOSE_CODE = Struct("!H").pack +PACK_RANDBITS = Struct("!L").pack +MSG_SIZE: Final[int] = 2**14 +MASK_LEN: Final[int] = 4 + +WS_KEY: Final[bytes] = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + +# Used by _websocket_mask_python +@functools.lru_cache +def _xor_table() -> list[bytes]: + return [bytes(a ^ b for a in range(256)) for b in range(256)] + + +def _websocket_mask_python(mask: bytes, data: bytearray) -> None: + """Websocket masking function. + + `mask` is a `bytes` object of length 4; `data` is a `bytearray` + object of any length. The contents of `data` are masked with `mask`, + as specified in section 5.3 of RFC 6455. + + Note that this function mutates the `data` argument. + + This pure-python implementation may be replaced by an optimized + version when available. + + """ + assert isinstance(data, bytearray), data + assert len(mask) == 4, mask + + if data: + _XOR_TABLE = _xor_table() + a, b, c, d = (_XOR_TABLE[n] for n in mask) + data[::4] = data[::4].translate(a) + data[1::4] = data[1::4].translate(b) + data[2::4] = data[2::4].translate(c) + data[3::4] = data[3::4].translate(d) + + +if TYPE_CHECKING or NO_EXTENSIONS: # pragma: no cover + websocket_mask = _websocket_mask_python +else: + try: + from .mask import _websocket_mask_cython # type: ignore[import-not-found] + + websocket_mask = _websocket_mask_cython + except ImportError: # pragma: no cover + websocket_mask = _websocket_mask_python + + +_WS_EXT_RE: Final[Pattern[str]] = re.compile( + r"^(?:;\s*(?:" + r"(server_no_context_takeover)|" + r"(client_no_context_takeover)|" + r"(server_max_window_bits(?:=(\d+))?)|" + r"(client_max_window_bits(?:=(\d+))?)))*$" +) + +_WS_EXT_RE_SPLIT: Final[Pattern[str]] = re.compile(r"permessage-deflate([^,]+)?") + + +def ws_ext_parse(extstr: str | None, isserver: bool = False) -> tuple[int, bool]: + if not extstr: + return 0, False + + compress = 0 + notakeover = False + for ext in _WS_EXT_RE_SPLIT.finditer(extstr): + defext = ext.group(1) + # Return compress = 15 when get `permessage-deflate` + if not defext: + compress = 15 + break + match = _WS_EXT_RE.match(defext) + if match: + compress = 15 + if isserver: + # Server never fail to detect compress handshake. + # Server does not need to send max wbit to client + if match.group(4): + compress = int(match.group(4)) + # Group3 must match if group4 matches + # Compress wbit 8 does not support in zlib + # If compress level not support, + # CONTINUE to next extension + if compress > 15 or compress < 9: + compress = 0 + continue + if match.group(1): + notakeover = True + # Ignore regex group 5 & 6 for client_max_window_bits + break + else: + if match.group(6): + compress = int(match.group(6)) + # Group5 must match if group6 matches + # Compress wbit 8 does not support in zlib + # If compress level not support, + # FAIL the parse progress + if compress > 15 or compress < 9: + raise WSHandshakeError("Invalid window size") + if match.group(2): + notakeover = True + # Ignore regex group 5 & 6 for client_max_window_bits + break + # Return Fail if client side and not match + elif not isserver: + raise WSHandshakeError("Extension for deflate not supported" + ext.group(1)) + + return compress, notakeover + + +def ws_ext_gen( + compress: int = 15, isserver: bool = False, server_notakeover: bool = False +) -> str: + # client_notakeover=False not used for server + # compress wbit 8 does not support in zlib + if compress < 9 or compress > 15: + raise ValueError( + "Compress wbits must between 9 and 15, zlib does not support wbits=8" + ) + enabledext = ["permessage-deflate"] + if not isserver: + enabledext.append("client_max_window_bits") + + if compress < 15: + enabledext.append("server_max_window_bits=" + str(compress)) + if server_notakeover: + enabledext.append("server_no_context_takeover") + # if client_notakeover: + # enabledext.append('client_no_context_takeover') + return "; ".join(enabledext) diff --git a/venv/Lib/site-packages/aiohttp/_websocket/mask.cp311-win_amd64.pyd b/venv/Lib/site-packages/aiohttp/_websocket/mask.cp311-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..80ed45239e295c13755d8569042917cc8757f462 Binary files /dev/null and b/venv/Lib/site-packages/aiohttp/_websocket/mask.cp311-win_amd64.pyd differ diff --git a/venv/Lib/site-packages/aiohttp/_websocket/mask.pxd b/venv/Lib/site-packages/aiohttp/_websocket/mask.pxd new file mode 100644 index 0000000000000000000000000000000000000000..90ab78fd06aa6cd4639795f06f15eb07f5404834 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/mask.pxd @@ -0,0 +1,3 @@ +"""Cython declarations for websocket masking.""" + +cpdef void _websocket_mask_cython(bytes mask, bytearray data) diff --git a/venv/Lib/site-packages/aiohttp/_websocket/mask.pyx b/venv/Lib/site-packages/aiohttp/_websocket/mask.pyx new file mode 100644 index 0000000000000000000000000000000000000000..ef61b76217b5cf9071f716aeedee377a3bbb7517 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/mask.pyx @@ -0,0 +1,48 @@ +from cpython cimport PyBytes_AsString + + +#from cpython cimport PyByteArray_AsString # cython still not exports that +cdef extern from "Python.h": + char* PyByteArray_AsString(bytearray ba) except NULL + +from libc.stdint cimport uint32_t, uint64_t, uintmax_t + + +cpdef void _websocket_mask_cython(bytes mask, bytearray data): + """Note, this function mutates its `data` argument + """ + cdef: + Py_ssize_t data_len, i + # bit operations on signed integers are implementation-specific + unsigned char * in_buf + const unsigned char * mask_buf + uint32_t uint32_msk + uint64_t uint64_msk + + assert len(mask) == 4 + + data_len = len(data) + in_buf = PyByteArray_AsString(data) + mask_buf = PyBytes_AsString(mask) + uint32_msk = (mask_buf)[0] + + # TODO: align in_data ptr to achieve even faster speeds + # does it need in python ?! malloc() always aligns to sizeof(long) bytes + + if sizeof(size_t) >= 8: + uint64_msk = uint32_msk + uint64_msk = (uint64_msk << 32) | uint32_msk + + while data_len >= 8: + (in_buf)[0] ^= uint64_msk + in_buf += 8 + data_len -= 8 + + + while data_len >= 4: + (in_buf)[0] ^= uint32_msk + in_buf += 4 + data_len -= 4 + + for i in range(0, data_len): + in_buf[i] ^= mask_buf[i] diff --git a/venv/Lib/site-packages/aiohttp/_websocket/models.py b/venv/Lib/site-packages/aiohttp/_websocket/models.py new file mode 100644 index 0000000000000000000000000000000000000000..27373a1b122b0362261f67833a4b8ff9f7d1506e --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/models.py @@ -0,0 +1,107 @@ +"""Models for WebSocket protocol versions 13 and 8.""" + +import json +from collections.abc import Callable +from enum import IntEnum +from typing import Any, Final, NamedTuple, cast + +WS_DEFLATE_TRAILING: Final[bytes] = bytes([0x00, 0x00, 0xFF, 0xFF]) + + +class WSCloseCode(IntEnum): + OK = 1000 + GOING_AWAY = 1001 + PROTOCOL_ERROR = 1002 + UNSUPPORTED_DATA = 1003 + ABNORMAL_CLOSURE = 1006 + INVALID_TEXT = 1007 + POLICY_VIOLATION = 1008 + MESSAGE_TOO_BIG = 1009 + MANDATORY_EXTENSION = 1010 + INTERNAL_ERROR = 1011 + SERVICE_RESTART = 1012 + TRY_AGAIN_LATER = 1013 + BAD_GATEWAY = 1014 + + +class WSMsgType(IntEnum): + # websocket spec types + CONTINUATION = 0x0 + TEXT = 0x1 + BINARY = 0x2 + PING = 0x9 + PONG = 0xA + CLOSE = 0x8 + + # aiohttp specific types + CLOSING = 0x100 + CLOSED = 0x101 + ERROR = 0x102 + + text = TEXT + binary = BINARY + ping = PING + pong = PONG + close = CLOSE + closing = CLOSING + closed = CLOSED + error = ERROR + + +class WSMessage(NamedTuple): + type: WSMsgType + # To type correctly, this would need some kind of tagged union for each type. + data: Any + extra: str | None + + def json(self, *, loads: Callable[[Any], Any] = json.loads) -> Any: + """Return parsed JSON data. + + .. versionadded:: 0.22 + """ + return loads(self.data) + + +class WSMessageTextBytes(NamedTuple): + """WebSocket TEXT message with raw bytes (no UTF-8 decoding).""" + + type: WSMsgType + # To type correctly, this would need some kind of tagged union for each type. + # In 4.0, we use a union of message types to properly type data, but in 3.x + # we keep it as Any to avoid a breaking change. + data: Any + extra: str | None + + def json(self, *, loads: Callable[[Any], Any] = json.loads) -> Any: + """Return parsed JSON data.""" + return loads(self.data) + + +# Type aliases for message types based on decode_text setting +# When decode_text=True, TEXT messages have str data (WSMessage) +# When decode_text=False, TEXT messages have bytes data (WSMessageTextBytes) +WSMessageDecodeText = WSMessage +WSMessageNoDecodeText = WSMessage | WSMessageTextBytes + + +# Constructing the tuple directly to avoid the overhead of +# the lambda and arg processing since NamedTuples are constructed +# with a run time built lambda +# https://github.com/python/cpython/blob/d83fcf8371f2f33c7797bc8f5423a8bca8c46e5c/Lib/collections/__init__.py#L441 +WS_CLOSED_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSED, None, None)) +WS_CLOSING_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSING, None, None)) + + +class WebSocketError(Exception): + """WebSocket protocol parser error.""" + + def __init__(self, code: int, message: str) -> None: + self.code = code + super().__init__(code, message) + + def __str__(self) -> str: + return cast(str, self.args[1]) + + +class WSHandshakeError(Exception): + """WebSocket protocol handshake error.""" diff --git a/venv/Lib/site-packages/aiohttp/_websocket/reader.py b/venv/Lib/site-packages/aiohttp/_websocket/reader.py new file mode 100644 index 0000000000000000000000000000000000000000..e70858d3d5c49cb94b56d1d911aada5f150f63c7 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/reader.py @@ -0,0 +1,31 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +from typing import TYPE_CHECKING + +from ..helpers import NO_EXTENSIONS + +if TYPE_CHECKING or NO_EXTENSIONS: # pragma: no cover + from .reader_py import ( + WebSocketDataQueue as WebSocketDataQueuePython, + WebSocketReader as WebSocketReaderPython, + ) + + WebSocketReader = WebSocketReaderPython + WebSocketDataQueue = WebSocketDataQueuePython +else: + try: + from .reader_c import ( # type: ignore[import-not-found] + WebSocketDataQueue as WebSocketDataQueueCython, + WebSocketReader as WebSocketReaderCython, + ) + + WebSocketReader = WebSocketReaderCython + WebSocketDataQueue = WebSocketDataQueueCython + except ImportError: # pragma: no cover + from .reader_py import ( + WebSocketDataQueue as WebSocketDataQueuePython, + WebSocketReader as WebSocketReaderPython, + ) + + WebSocketReader = WebSocketReaderPython + WebSocketDataQueue = WebSocketDataQueuePython diff --git a/venv/Lib/site-packages/aiohttp/_websocket/reader_c.cp311-win_amd64.pyd b/venv/Lib/site-packages/aiohttp/_websocket/reader_c.cp311-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..a289f022e744f14343198031d049247c29fd044e --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/reader_c.cp311-win_amd64.pyd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3efc2e02a76c14b85eb5aa6beeebe4604cf30c1b957f972c877b10578a635ffb +size 141824 diff --git a/venv/Lib/site-packages/aiohttp/_websocket/reader_c.pxd b/venv/Lib/site-packages/aiohttp/_websocket/reader_c.pxd new file mode 100644 index 0000000000000000000000000000000000000000..2e298beb1c06a2ebdf149073b200dd6861e2f821 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/reader_c.pxd @@ -0,0 +1,112 @@ +import cython + +from .mask cimport _websocket_mask_cython as websocket_mask + + +cdef unsigned int READ_HEADER +cdef unsigned int READ_PAYLOAD_LENGTH +cdef unsigned int READ_PAYLOAD_MASK +cdef unsigned int READ_PAYLOAD + +cdef int OP_CODE_NOT_SET +cdef int OP_CODE_CONTINUATION +cdef int OP_CODE_TEXT +cdef int OP_CODE_BINARY +cdef int OP_CODE_CLOSE +cdef int OP_CODE_PING +cdef int OP_CODE_PONG + +cdef int COMPRESSED_NOT_SET +cdef int COMPRESSED_FALSE +cdef int COMPRESSED_TRUE + +cdef object UNPACK_LEN3 +cdef object UNPACK_CLOSE_CODE +cdef object TUPLE_NEW + +cdef object WSMsgType +cdef object WSMessage +cdef object WSMessageTextBytes + +cdef object WS_MSG_TYPE_TEXT +cdef object WS_MSG_TYPE_BINARY + +cdef set ALLOWED_CLOSE_CODES +cdef set MESSAGE_TYPES_WITH_CONTENT + +cdef tuple EMPTY_FRAME +cdef tuple EMPTY_FRAME_ERROR + +cdef class WebSocketDataQueue: + + cdef unsigned int _size + cdef public object _protocol + cdef unsigned int _limit + cdef object _loop + cdef bint _eof + cdef object _waiter + cdef object _exception + cdef public object _buffer + cdef object _get_buffer + cdef object _put_buffer + + cdef void _release_waiter(self) + + cpdef void feed_data(self, object data, unsigned int size) + + @cython.locals(size="unsigned int") + cdef _read_from_buffer(self) + +cdef class WebSocketReader: + + cdef WebSocketDataQueue queue + cdef unsigned int _max_msg_size + cdef bint _decode_text + + cdef Exception _exc + cdef bytearray _partial + cdef unsigned int _state + + cdef int _opcode + cdef bint _frame_fin + cdef int _frame_opcode + cdef list _payload_fragments + cdef Py_ssize_t _frame_payload_len + + cdef bytes _tail + cdef bint _has_mask + cdef bytes _frame_mask + cdef Py_ssize_t _payload_bytes_to_read + cdef unsigned int _payload_len_flag + cdef int _compressed + cdef object _decompressobj + cdef bint _compress + + cpdef tuple feed_data(self, object data) + + @cython.locals( + is_continuation=bint, + fin=bint, + has_partial=bint, + payload_merged=bytes, + ) + cpdef void _handle_frame(self, bint fin, int opcode, object payload, int compressed) except * + + @cython.locals( + start_pos=Py_ssize_t, + data_len=Py_ssize_t, + length=Py_ssize_t, + chunk_size=Py_ssize_t, + chunk_len=Py_ssize_t, + data_len=Py_ssize_t, + data_cstr="const unsigned char *", + first_byte="unsigned char", + second_byte="unsigned char", + f_start_pos=Py_ssize_t, + f_end_pos=Py_ssize_t, + has_mask=bint, + fin=bint, + had_fragments=Py_ssize_t, + payload_bytearray=bytearray, + ) + cpdef void _feed_data(self, bytes data) except * diff --git a/venv/Lib/site-packages/aiohttp/_websocket/reader_c.py b/venv/Lib/site-packages/aiohttp/_websocket/reader_c.py new file mode 100644 index 0000000000000000000000000000000000000000..40d495e59d7b28389869345eca6399566a75eb9e --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/reader_c.py @@ -0,0 +1,509 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +import asyncio +import builtins +from collections import deque +from typing import Final + +from ..base_protocol import BaseProtocol +from ..compression_utils import ZLibDecompressor +from ..helpers import _EXC_SENTINEL, set_exception +from ..streams import EofStream +from .helpers import UNPACK_CLOSE_CODE, UNPACK_LEN3, websocket_mask +from .models import ( + WS_DEFLATE_TRAILING, + WebSocketError, + WSCloseCode, + WSMessage, + WSMessageTextBytes, + WSMsgType, +) + +ALLOWED_CLOSE_CODES: Final[set[int]] = {int(i) for i in WSCloseCode} + +# States for the reader, used to parse the WebSocket frame +# integer values are used so they can be cythonized +READ_HEADER = 1 +READ_PAYLOAD_LENGTH = 2 +READ_PAYLOAD_MASK = 3 +READ_PAYLOAD = 4 + +WS_MSG_TYPE_BINARY = WSMsgType.BINARY +WS_MSG_TYPE_TEXT = WSMsgType.TEXT + +# WSMsgType values unpacked so they can by cythonized to ints +OP_CODE_NOT_SET = -1 +OP_CODE_CONTINUATION = WSMsgType.CONTINUATION.value +OP_CODE_TEXT = WSMsgType.TEXT.value +OP_CODE_BINARY = WSMsgType.BINARY.value +OP_CODE_CLOSE = WSMsgType.CLOSE.value +OP_CODE_PING = WSMsgType.PING.value +OP_CODE_PONG = WSMsgType.PONG.value + +EMPTY_FRAME_ERROR = (True, b"") +EMPTY_FRAME = (False, b"") + +COMPRESSED_NOT_SET = -1 +COMPRESSED_FALSE = 0 +COMPRESSED_TRUE = 1 + +TUPLE_NEW = tuple.__new__ + +cython_int = int # Typed to int in Python, but cython with use a signed int in the pxd + + +class WebSocketDataQueue: + """WebSocketDataQueue resumes and pauses an underlying stream. + + It is a destination for WebSocket data. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + self._loop = loop + self._eof = False + self._waiter: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._buffer: deque[tuple[WSMessage | WSMessageTextBytes, int]] = deque() + self._get_buffer = self._buffer.popleft + self._put_buffer = self._buffer.append + + def is_eof(self) -> bool: + return self._eof + + def exception(self) -> BaseException | None: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: builtins.BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def _release_waiter(self) -> None: + if (waiter := self._waiter) is None: + return + self._waiter = None + if not waiter.done(): + waiter.set_result(None) + + def feed_eof(self) -> None: + self._eof = True + self._release_waiter() + self._exception = None # Break cyclic references + + def feed_data( + self, data: "WSMessage | WSMessageTextBytes", size: "cython_int" + ) -> None: + self._size += size + self._put_buffer((data, size)) + self._release_waiter() + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> WSMessage | WSMessageTextBytes: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + return self._read_from_buffer() + + def _read_from_buffer(self) -> WSMessage | WSMessageTextBytes: + if self._buffer: + data, size = self._get_buffer() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + +class WebSocketReader: + def __init__( + self, + queue: WebSocketDataQueue, + max_msg_size: int, + compress: bool = True, + decode_text: bool = True, + ) -> None: + self.queue = queue + self._max_msg_size = max_msg_size + self._decode_text = decode_text + + self._exc: Exception | None = None + self._partial = bytearray() + self._state = READ_HEADER + + self._opcode: int = OP_CODE_NOT_SET + self._frame_fin = False + self._frame_opcode: int = OP_CODE_NOT_SET + self._payload_fragments: list[bytes] = [] + self._frame_payload_len = 0 + + self._tail: bytes = b"" + self._has_mask = False + self._frame_mask: bytes | None = None + self._payload_bytes_to_read = 0 + self._payload_len_flag = 0 + self._compressed: int = COMPRESSED_NOT_SET + self._decompressobj: ZLibDecompressor | None = None + self._compress = compress + + def feed_eof(self) -> None: + self.queue.feed_eof() + + # data can be bytearray on Windows because proactor event loop uses bytearray + # and asyncio types this to Union[bytes, bytearray, memoryview] so we need + # coerce data to bytes if it is not + def feed_data(self, data: bytes | bytearray | memoryview) -> tuple[bool, bytes]: + if type(data) is not bytes: + data = bytes(data) + + if self._exc is not None: + return True, data + + try: + self._feed_data(data) + except Exception as exc: + self._exc = exc + set_exception(self.queue, exc) + return EMPTY_FRAME_ERROR + + return EMPTY_FRAME + + def _handle_frame( + self, + fin: bool, + opcode: int | cython_int, # Union intended: Cython pxd uses C int + payload: bytes | bytearray, + compressed: int | cython_int, # Union intended: Cython pxd uses C int + ) -> None: + msg: WSMessage + if opcode in {OP_CODE_TEXT, OP_CODE_BINARY, OP_CODE_CONTINUATION}: + # Validate continuation frames before processing + if opcode == OP_CODE_CONTINUATION and self._opcode == OP_CODE_NOT_SET: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Continuation frame for non started message", + ) + + # load text/binary + if not fin: + # got partial frame payload + if opcode != OP_CODE_CONTINUATION: + self._opcode = opcode + self._partial += payload + return + + has_partial = bool(self._partial) + if opcode == OP_CODE_CONTINUATION: + opcode = self._opcode + self._opcode = OP_CODE_NOT_SET + # previous frame was non finished + # we should get continuation opcode + elif has_partial: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "The opcode in non-fin frame is expected " + f"to be zero, got {opcode!r}", + ) + + assembled_payload: bytes | bytearray + if has_partial: + assembled_payload = self._partial + payload + self._partial.clear() + else: + assembled_payload = payload + + # Decompress process must to be done after all packets + # received. + if compressed: + if not self._decompressobj: + self._decompressobj = ZLibDecompressor(suppress_deflate_header=True) + # XXX: It's possible that the zlib backend (isal is known to + # do this, maybe others too?) will return max_length bytes, + # but internally buffer more data such that the payload is + # >max_length, so we return one extra byte and if we're able + # to do that, then the message is too big. + payload_merged = self._decompressobj.decompress_sync( + assembled_payload + WS_DEFLATE_TRAILING, + ( + self._max_msg_size + 1 + if self._max_msg_size + else self._max_msg_size + ), + ) + if self._max_msg_size and len(payload_merged) > self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Decompressed message exceeds size limit {self._max_msg_size}", + ) + elif type(assembled_payload) is bytes: + payload_merged = assembled_payload + else: + payload_merged = bytes(assembled_payload) + + if opcode == OP_CODE_TEXT: + if self._decode_text: + try: + text = payload_merged.decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + + # XXX: The Text and Binary messages here can be a performance + # bottleneck, so we use tuple.__new__ to improve performance. + # This is not type safe, but many tests should fail in + # test_client_ws_functional.py if this is wrong. + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_TEXT, text, "")), + len(payload_merged), + ) + else: + # Return raw bytes for TEXT messages when decode_text=False + self.queue.feed_data( + TUPLE_NEW( + WSMessageTextBytes, (WS_MSG_TYPE_TEXT, payload_merged, "") + ), + len(payload_merged), + ) + else: + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_BINARY, payload_merged, "")), + len(payload_merged), + ) + elif opcode == OP_CODE_CLOSE: + if len(payload) >= 2: + close_code = UNPACK_CLOSE_CODE(payload[:2])[0] + if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close code: {close_code}", + ) + try: + close_message = payload[2:].decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, close_code, close_message)) + elif payload: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close frame: {fin} {opcode} {payload!r}", + ) + else: + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, 0, "")) + + self.queue.feed_data(msg, 0) + elif opcode == OP_CODE_PING: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PING, payload, "")) + self.queue.feed_data(msg, len(payload)) + elif opcode == OP_CODE_PONG: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PONG, payload, "")) + self.queue.feed_data(msg, len(payload)) + else: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}" + ) + + def _feed_data(self, data: bytes) -> None: + """Return the next frame from the socket.""" + if self._tail: + data, self._tail = self._tail + data, b"" + + start_pos: int = 0 + data_len = len(data) + data_cstr = data + + while True: + # read header + if self._state == READ_HEADER: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xF + + # frame-fin = %x0 ; more frames of this message follow + # / %x1 ; final frame of this message + # frame-rsv1 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv2 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv3 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # + # Remove rsv1 from this test for deflate development + if rsv2 or rsv3 or (rsv1 and not self._compress): + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + if opcode not in { + OP_CODE_CONTINUATION, + OP_CODE_TEXT, + OP_CODE_BINARY, + OP_CODE_CLOSE, + OP_CODE_PING, + OP_CODE_PONG, + }: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Unexpected opcode={opcode!r}", + ) + + if opcode > 0x7 and fin == 0: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received fragmented control frame", + ) + + has_mask = (second_byte >> 7) & 1 + length = second_byte & 0x7F + + # Control frames MUST have a payload + # length of 125 bytes or less + if opcode > 0x7 and length > 125: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Control frame payload cannot be larger than 125 bytes", + ) + + # Set compress status if last package is FIN + # OR set compress status if this is first fragment + # Raise error if not first fragment with rsv1 = 0x1 + if self._frame_fin or self._compressed == COMPRESSED_NOT_SET: + self._compressed = COMPRESSED_TRUE if rsv1 else COMPRESSED_FALSE + elif rsv1: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + self._frame_fin = bool(fin) + self._frame_opcode = opcode + self._has_mask = bool(has_mask) + self._payload_len_flag = length + self._state = READ_PAYLOAD_LENGTH + + # read payload length + if self._state == READ_PAYLOAD_LENGTH: + len_flag = self._payload_len_flag + if len_flag == 126: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + self._payload_bytes_to_read = first_byte << 8 | second_byte + elif len_flag > 126: + if data_len - start_pos < 8: + break + self._payload_bytes_to_read = UNPACK_LEN3(data, start_pos)[0] + start_pos += 8 + else: + self._payload_bytes_to_read = len_flag + + # Reject oversized data frames before buffering any payload + # bytes. Control frames are capped at 125 bytes (checked in + # READ_HEADER) so only text/binary/continuation need this. + if self._max_msg_size and self._frame_opcode in { + OP_CODE_TEXT, + OP_CODE_BINARY, + OP_CODE_CONTINUATION, + }: + projected_size = self._payload_bytes_to_read + len(self._partial) + if projected_size >= self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Message size {projected_size} " + f"exceeds limit {self._max_msg_size}", + ) + + self._state = READ_PAYLOAD_MASK if self._has_mask else READ_PAYLOAD + + # read payload mask + if self._state == READ_PAYLOAD_MASK: + if data_len - start_pos < 4: + break + self._frame_mask = data_cstr[start_pos : start_pos + 4] + start_pos += 4 + self._state = READ_PAYLOAD + + if self._state == READ_PAYLOAD: + chunk_len = data_len - start_pos + if self._payload_bytes_to_read >= chunk_len: + f_end_pos = data_len + self._payload_bytes_to_read -= chunk_len + else: + f_end_pos = start_pos + self._payload_bytes_to_read + self._payload_bytes_to_read = 0 + + had_fragments = self._frame_payload_len + self._frame_payload_len += f_end_pos - start_pos + f_start_pos = start_pos + start_pos = f_end_pos + + if self._payload_bytes_to_read != 0: + # If we don't have a complete frame, we need to save the + # data for the next call to feed_data. + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + break + + payload: bytes | bytearray + if had_fragments: + # We have to join the payload fragments get the payload + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + if self._has_mask: + assert self._frame_mask is not None + payload_bytearray = bytearray(b"".join(self._payload_fragments)) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = b"".join(self._payload_fragments) + self._payload_fragments.clear() + elif self._has_mask: + assert self._frame_mask is not None + payload_bytearray = data_cstr[f_start_pos:f_end_pos] # type: ignore[assignment] + if type(payload_bytearray) is not bytearray: # pragma: no branch + # Cython will do the conversion for us + # but we need to do it for Python and we + # will always get here in Python + payload_bytearray = bytearray(payload_bytearray) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = data_cstr[f_start_pos:f_end_pos] + + self._handle_frame( + self._frame_fin, self._frame_opcode, payload, self._compressed + ) + self._frame_payload_len = 0 + self._state = READ_HEADER + + # XXX: Cython needs slices to be bounded, so we can't omit the slice end here. + self._tail = data_cstr[start_pos:data_len] if start_pos < data_len else b"" diff --git a/venv/Lib/site-packages/aiohttp/_websocket/reader_py.py b/venv/Lib/site-packages/aiohttp/_websocket/reader_py.py new file mode 100644 index 0000000000000000000000000000000000000000..40d495e59d7b28389869345eca6399566a75eb9e --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/reader_py.py @@ -0,0 +1,509 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +import asyncio +import builtins +from collections import deque +from typing import Final + +from ..base_protocol import BaseProtocol +from ..compression_utils import ZLibDecompressor +from ..helpers import _EXC_SENTINEL, set_exception +from ..streams import EofStream +from .helpers import UNPACK_CLOSE_CODE, UNPACK_LEN3, websocket_mask +from .models import ( + WS_DEFLATE_TRAILING, + WebSocketError, + WSCloseCode, + WSMessage, + WSMessageTextBytes, + WSMsgType, +) + +ALLOWED_CLOSE_CODES: Final[set[int]] = {int(i) for i in WSCloseCode} + +# States for the reader, used to parse the WebSocket frame +# integer values are used so they can be cythonized +READ_HEADER = 1 +READ_PAYLOAD_LENGTH = 2 +READ_PAYLOAD_MASK = 3 +READ_PAYLOAD = 4 + +WS_MSG_TYPE_BINARY = WSMsgType.BINARY +WS_MSG_TYPE_TEXT = WSMsgType.TEXT + +# WSMsgType values unpacked so they can by cythonized to ints +OP_CODE_NOT_SET = -1 +OP_CODE_CONTINUATION = WSMsgType.CONTINUATION.value +OP_CODE_TEXT = WSMsgType.TEXT.value +OP_CODE_BINARY = WSMsgType.BINARY.value +OP_CODE_CLOSE = WSMsgType.CLOSE.value +OP_CODE_PING = WSMsgType.PING.value +OP_CODE_PONG = WSMsgType.PONG.value + +EMPTY_FRAME_ERROR = (True, b"") +EMPTY_FRAME = (False, b"") + +COMPRESSED_NOT_SET = -1 +COMPRESSED_FALSE = 0 +COMPRESSED_TRUE = 1 + +TUPLE_NEW = tuple.__new__ + +cython_int = int # Typed to int in Python, but cython with use a signed int in the pxd + + +class WebSocketDataQueue: + """WebSocketDataQueue resumes and pauses an underlying stream. + + It is a destination for WebSocket data. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + self._loop = loop + self._eof = False + self._waiter: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._buffer: deque[tuple[WSMessage | WSMessageTextBytes, int]] = deque() + self._get_buffer = self._buffer.popleft + self._put_buffer = self._buffer.append + + def is_eof(self) -> bool: + return self._eof + + def exception(self) -> BaseException | None: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: builtins.BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def _release_waiter(self) -> None: + if (waiter := self._waiter) is None: + return + self._waiter = None + if not waiter.done(): + waiter.set_result(None) + + def feed_eof(self) -> None: + self._eof = True + self._release_waiter() + self._exception = None # Break cyclic references + + def feed_data( + self, data: "WSMessage | WSMessageTextBytes", size: "cython_int" + ) -> None: + self._size += size + self._put_buffer((data, size)) + self._release_waiter() + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> WSMessage | WSMessageTextBytes: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + return self._read_from_buffer() + + def _read_from_buffer(self) -> WSMessage | WSMessageTextBytes: + if self._buffer: + data, size = self._get_buffer() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + +class WebSocketReader: + def __init__( + self, + queue: WebSocketDataQueue, + max_msg_size: int, + compress: bool = True, + decode_text: bool = True, + ) -> None: + self.queue = queue + self._max_msg_size = max_msg_size + self._decode_text = decode_text + + self._exc: Exception | None = None + self._partial = bytearray() + self._state = READ_HEADER + + self._opcode: int = OP_CODE_NOT_SET + self._frame_fin = False + self._frame_opcode: int = OP_CODE_NOT_SET + self._payload_fragments: list[bytes] = [] + self._frame_payload_len = 0 + + self._tail: bytes = b"" + self._has_mask = False + self._frame_mask: bytes | None = None + self._payload_bytes_to_read = 0 + self._payload_len_flag = 0 + self._compressed: int = COMPRESSED_NOT_SET + self._decompressobj: ZLibDecompressor | None = None + self._compress = compress + + def feed_eof(self) -> None: + self.queue.feed_eof() + + # data can be bytearray on Windows because proactor event loop uses bytearray + # and asyncio types this to Union[bytes, bytearray, memoryview] so we need + # coerce data to bytes if it is not + def feed_data(self, data: bytes | bytearray | memoryview) -> tuple[bool, bytes]: + if type(data) is not bytes: + data = bytes(data) + + if self._exc is not None: + return True, data + + try: + self._feed_data(data) + except Exception as exc: + self._exc = exc + set_exception(self.queue, exc) + return EMPTY_FRAME_ERROR + + return EMPTY_FRAME + + def _handle_frame( + self, + fin: bool, + opcode: int | cython_int, # Union intended: Cython pxd uses C int + payload: bytes | bytearray, + compressed: int | cython_int, # Union intended: Cython pxd uses C int + ) -> None: + msg: WSMessage + if opcode in {OP_CODE_TEXT, OP_CODE_BINARY, OP_CODE_CONTINUATION}: + # Validate continuation frames before processing + if opcode == OP_CODE_CONTINUATION and self._opcode == OP_CODE_NOT_SET: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Continuation frame for non started message", + ) + + # load text/binary + if not fin: + # got partial frame payload + if opcode != OP_CODE_CONTINUATION: + self._opcode = opcode + self._partial += payload + return + + has_partial = bool(self._partial) + if opcode == OP_CODE_CONTINUATION: + opcode = self._opcode + self._opcode = OP_CODE_NOT_SET + # previous frame was non finished + # we should get continuation opcode + elif has_partial: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "The opcode in non-fin frame is expected " + f"to be zero, got {opcode!r}", + ) + + assembled_payload: bytes | bytearray + if has_partial: + assembled_payload = self._partial + payload + self._partial.clear() + else: + assembled_payload = payload + + # Decompress process must to be done after all packets + # received. + if compressed: + if not self._decompressobj: + self._decompressobj = ZLibDecompressor(suppress_deflate_header=True) + # XXX: It's possible that the zlib backend (isal is known to + # do this, maybe others too?) will return max_length bytes, + # but internally buffer more data such that the payload is + # >max_length, so we return one extra byte and if we're able + # to do that, then the message is too big. + payload_merged = self._decompressobj.decompress_sync( + assembled_payload + WS_DEFLATE_TRAILING, + ( + self._max_msg_size + 1 + if self._max_msg_size + else self._max_msg_size + ), + ) + if self._max_msg_size and len(payload_merged) > self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Decompressed message exceeds size limit {self._max_msg_size}", + ) + elif type(assembled_payload) is bytes: + payload_merged = assembled_payload + else: + payload_merged = bytes(assembled_payload) + + if opcode == OP_CODE_TEXT: + if self._decode_text: + try: + text = payload_merged.decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + + # XXX: The Text and Binary messages here can be a performance + # bottleneck, so we use tuple.__new__ to improve performance. + # This is not type safe, but many tests should fail in + # test_client_ws_functional.py if this is wrong. + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_TEXT, text, "")), + len(payload_merged), + ) + else: + # Return raw bytes for TEXT messages when decode_text=False + self.queue.feed_data( + TUPLE_NEW( + WSMessageTextBytes, (WS_MSG_TYPE_TEXT, payload_merged, "") + ), + len(payload_merged), + ) + else: + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_BINARY, payload_merged, "")), + len(payload_merged), + ) + elif opcode == OP_CODE_CLOSE: + if len(payload) >= 2: + close_code = UNPACK_CLOSE_CODE(payload[:2])[0] + if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close code: {close_code}", + ) + try: + close_message = payload[2:].decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, close_code, close_message)) + elif payload: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close frame: {fin} {opcode} {payload!r}", + ) + else: + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, 0, "")) + + self.queue.feed_data(msg, 0) + elif opcode == OP_CODE_PING: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PING, payload, "")) + self.queue.feed_data(msg, len(payload)) + elif opcode == OP_CODE_PONG: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PONG, payload, "")) + self.queue.feed_data(msg, len(payload)) + else: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}" + ) + + def _feed_data(self, data: bytes) -> None: + """Return the next frame from the socket.""" + if self._tail: + data, self._tail = self._tail + data, b"" + + start_pos: int = 0 + data_len = len(data) + data_cstr = data + + while True: + # read header + if self._state == READ_HEADER: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xF + + # frame-fin = %x0 ; more frames of this message follow + # / %x1 ; final frame of this message + # frame-rsv1 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv2 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv3 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # + # Remove rsv1 from this test for deflate development + if rsv2 or rsv3 or (rsv1 and not self._compress): + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + if opcode not in { + OP_CODE_CONTINUATION, + OP_CODE_TEXT, + OP_CODE_BINARY, + OP_CODE_CLOSE, + OP_CODE_PING, + OP_CODE_PONG, + }: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Unexpected opcode={opcode!r}", + ) + + if opcode > 0x7 and fin == 0: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received fragmented control frame", + ) + + has_mask = (second_byte >> 7) & 1 + length = second_byte & 0x7F + + # Control frames MUST have a payload + # length of 125 bytes or less + if opcode > 0x7 and length > 125: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Control frame payload cannot be larger than 125 bytes", + ) + + # Set compress status if last package is FIN + # OR set compress status if this is first fragment + # Raise error if not first fragment with rsv1 = 0x1 + if self._frame_fin or self._compressed == COMPRESSED_NOT_SET: + self._compressed = COMPRESSED_TRUE if rsv1 else COMPRESSED_FALSE + elif rsv1: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + self._frame_fin = bool(fin) + self._frame_opcode = opcode + self._has_mask = bool(has_mask) + self._payload_len_flag = length + self._state = READ_PAYLOAD_LENGTH + + # read payload length + if self._state == READ_PAYLOAD_LENGTH: + len_flag = self._payload_len_flag + if len_flag == 126: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + self._payload_bytes_to_read = first_byte << 8 | second_byte + elif len_flag > 126: + if data_len - start_pos < 8: + break + self._payload_bytes_to_read = UNPACK_LEN3(data, start_pos)[0] + start_pos += 8 + else: + self._payload_bytes_to_read = len_flag + + # Reject oversized data frames before buffering any payload + # bytes. Control frames are capped at 125 bytes (checked in + # READ_HEADER) so only text/binary/continuation need this. + if self._max_msg_size and self._frame_opcode in { + OP_CODE_TEXT, + OP_CODE_BINARY, + OP_CODE_CONTINUATION, + }: + projected_size = self._payload_bytes_to_read + len(self._partial) + if projected_size >= self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Message size {projected_size} " + f"exceeds limit {self._max_msg_size}", + ) + + self._state = READ_PAYLOAD_MASK if self._has_mask else READ_PAYLOAD + + # read payload mask + if self._state == READ_PAYLOAD_MASK: + if data_len - start_pos < 4: + break + self._frame_mask = data_cstr[start_pos : start_pos + 4] + start_pos += 4 + self._state = READ_PAYLOAD + + if self._state == READ_PAYLOAD: + chunk_len = data_len - start_pos + if self._payload_bytes_to_read >= chunk_len: + f_end_pos = data_len + self._payload_bytes_to_read -= chunk_len + else: + f_end_pos = start_pos + self._payload_bytes_to_read + self._payload_bytes_to_read = 0 + + had_fragments = self._frame_payload_len + self._frame_payload_len += f_end_pos - start_pos + f_start_pos = start_pos + start_pos = f_end_pos + + if self._payload_bytes_to_read != 0: + # If we don't have a complete frame, we need to save the + # data for the next call to feed_data. + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + break + + payload: bytes | bytearray + if had_fragments: + # We have to join the payload fragments get the payload + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + if self._has_mask: + assert self._frame_mask is not None + payload_bytearray = bytearray(b"".join(self._payload_fragments)) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = b"".join(self._payload_fragments) + self._payload_fragments.clear() + elif self._has_mask: + assert self._frame_mask is not None + payload_bytearray = data_cstr[f_start_pos:f_end_pos] # type: ignore[assignment] + if type(payload_bytearray) is not bytearray: # pragma: no branch + # Cython will do the conversion for us + # but we need to do it for Python and we + # will always get here in Python + payload_bytearray = bytearray(payload_bytearray) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = data_cstr[f_start_pos:f_end_pos] + + self._handle_frame( + self._frame_fin, self._frame_opcode, payload, self._compressed + ) + self._frame_payload_len = 0 + self._state = READ_HEADER + + # XXX: Cython needs slices to be bounded, so we can't omit the slice end here. + self._tail = data_cstr[start_pos:data_len] if start_pos < data_len else b"" diff --git a/venv/Lib/site-packages/aiohttp/_websocket/writer.py b/venv/Lib/site-packages/aiohttp/_websocket/writer.py new file mode 100644 index 0000000000000000000000000000000000000000..f46ea69663d6b9ada3fd693e1615fe4c09675a3a --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/_websocket/writer.py @@ -0,0 +1,261 @@ +"""WebSocket protocol versions 13 and 8.""" + +import asyncio +import random +import sys +from functools import partial +from typing import Final, Optional, Set + +from ..base_protocol import BaseProtocol +from ..client_exceptions import ClientConnectionResetError +from ..compression_utils import ZLibBackend, ZLibCompressor +from ..helpers import DEFAULT_CHUNK_SIZE +from .helpers import ( + MASK_LEN, + MSG_SIZE, + PACK_CLOSE_CODE, + PACK_LEN1, + PACK_LEN2, + PACK_LEN3, + PACK_RANDBITS, + websocket_mask, +) +from .models import WS_DEFLATE_TRAILING, WSMsgType + +# WebSocket opcode boundary: opcodes 0-7 are data frames, 8-15 are control frames +# Control frames (ping, pong, close) are never compressed +WS_CONTROL_FRAME_OPCODE: Final[int] = 8 + +# For websockets, keeping latency low is extremely important as implementations +# generally expect to be able to send and receive messages quickly. We use a +# larger chunk size to reduce the number of executor calls and avoid task +# creation overhead, since both are significant sources of latency when chunks +# are small. A size of 16KiB was chosen as a balance between avoiding task +# overhead and not blocking the event loop too long with synchronous compression. + +WEBSOCKET_MAX_SYNC_CHUNK_SIZE = 16 * 1024 + + +class WebSocketWriter: + """WebSocket writer. + + The writer is responsible for sending messages to the client. It is + created by the protocol when a connection is established. The writer + should avoid implementing any application logic and should only be + concerned with the low-level details of the WebSocket protocol. + """ + + def __init__( + self, + protocol: BaseProtocol, + transport: asyncio.Transport, + *, + use_mask: bool = False, + limit: int = DEFAULT_CHUNK_SIZE, + random: random.Random = random.Random(), + compress: int = 0, + notakeover: bool = False, + ) -> None: + """Initialize a WebSocket writer.""" + self.protocol = protocol + self.transport = transport + self.use_mask = use_mask + self.get_random_bits = partial(random.getrandbits, 32) + self.compress = compress + self.notakeover = notakeover + self._closing = False + self._limit = limit + self._output_size = 0 + self._compressobj: Optional[ZLibCompressor] = None + self._send_lock = asyncio.Lock() + self._background_tasks: Set[asyncio.Task[None]] = set() + + async def send_frame( + self, message: bytes, opcode: int, compress: int | None = None + ) -> None: + """Send a frame over the websocket with message as its payload.""" + if self._closing and not (opcode & WSMsgType.CLOSE): + raise ClientConnectionResetError("Cannot write to closing transport") + + if not (compress or self.compress) or opcode >= WS_CONTROL_FRAME_OPCODE: + # Non-compressed frames don't need lock or shield + self._write_websocket_frame(message, opcode, 0) + elif len(message) <= WEBSOCKET_MAX_SYNC_CHUNK_SIZE: + # Small compressed payloads - compress synchronously in event loop + # We need the lock even though sync compression has no await points. + # This prevents small frames from interleaving with large frames that + # compress in the executor, avoiding compressor state corruption. + async with self._send_lock: + self._send_compressed_frame_sync(message, opcode, compress) + else: + # Large compressed frames need shield to prevent corruption + # For large compressed frames, the entire compress+send + # operation must be atomic. If cancelled after compression but + # before send, the compressor state would be advanced but data + # not sent, corrupting subsequent frames. + # Create a task to shield from cancellation + # The lock is acquired inside the shielded task so the entire + # operation (lock + compress + send) completes atomically. + # Use eager_start on Python 3.12+ to avoid scheduling overhead + loop = asyncio.get_running_loop() + coro = self._send_compressed_frame_async_locked(message, opcode, compress) + if sys.version_info >= (3, 12): + send_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + send_task = loop.create_task(coro) + # Keep a strong reference to prevent garbage collection + self._background_tasks.add(send_task) + send_task.add_done_callback(self._background_tasks.discard) + await asyncio.shield(send_task) + + # It is safe to return control to the event loop when using compression + # after this point as we have already sent or buffered all the data. + # Once we have written output_size up to the limit, we call the + # drain helper which waits for the transport to be ready to accept + # more data. This is a flow control mechanism to prevent the buffer + # from growing too large. The drain helper will return right away + # if the writer is not paused. + if self._output_size > self._limit: + self._output_size = 0 + if self.protocol._paused: + await self.protocol._drain_helper() + + def _write_websocket_frame(self, message: bytes, opcode: int, rsv: int) -> None: + """ + Write a websocket frame to the transport. + + This method handles frame header construction, masking, and writing to transport. + It does not handle compression or flow control - those are the responsibility + of the caller. + """ + msg_length = len(message) + + use_mask = self.use_mask + mask_bit = 0x80 if use_mask else 0 + + # Depending on the message length, the header is assembled differently. + # The first byte is reserved for the opcode and the RSV bits. + first_byte = 0x80 | rsv | opcode + if msg_length < 126: + header = PACK_LEN1(first_byte, msg_length | mask_bit) + header_len = 2 + elif msg_length < 65536: + header = PACK_LEN2(first_byte, 126 | mask_bit, msg_length) + header_len = 4 + else: + header = PACK_LEN3(first_byte, 127 | mask_bit, msg_length) + header_len = 10 + + if self.transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.3 + # If we are using a mask, we need to generate it randomly + # and apply it to the message before sending it. A mask is + # a 32-bit value that is applied to the message using a + # bitwise XOR operation. It is used to prevent certain types + # of attacks on the websocket protocol. The mask is only used + # when aiohttp is acting as a client. Servers do not use a mask. + if use_mask: + mask = PACK_RANDBITS(self.get_random_bits()) + message_arr = bytearray(message) + websocket_mask(mask, message_arr) + self.transport.write(header + mask + message_arr) + self._output_size += MASK_LEN + elif msg_length > MSG_SIZE: + self.transport.write(header) + self.transport.write(message) + else: + self.transport.write(header + message) + + self._output_size += header_len + msg_length + + def _get_compressor(self, compress: int | None) -> ZLibCompressor: + """Get or create a compressor object for the given compression level.""" + if compress: + # Do not set self._compress if compressing is for this frame + return ZLibCompressor( + level=ZLibBackend.Z_BEST_SPEED, + wbits=-compress, + max_sync_chunk_size=WEBSOCKET_MAX_SYNC_CHUNK_SIZE, + ) + if not self._compressobj: + self._compressobj = ZLibCompressor( + level=ZLibBackend.Z_BEST_SPEED, + wbits=-self.compress, + max_sync_chunk_size=WEBSOCKET_MAX_SYNC_CHUNK_SIZE, + ) + return self._compressobj + + def _send_compressed_frame_sync( + self, message: bytes, opcode: int, compress: int | None + ) -> None: + """ + Synchronous send for small compressed frames. + + This is used for small compressed payloads that compress synchronously in the event loop. + Since there are no await points, this is inherently cancellation-safe. + """ + # RSV are the reserved bits in the frame header. They are used to + # indicate that the frame is using an extension. + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.2 + compressobj = self._get_compressor(compress) + # (0x40) RSV1 is set for compressed frames + # https://datatracker.ietf.org/doc/html/rfc7692#section-7.2.3.1 + self._write_websocket_frame( + ( + compressobj.compress_sync(message) + + compressobj.flush( + ZLibBackend.Z_FULL_FLUSH + if self.notakeover + else ZLibBackend.Z_SYNC_FLUSH + ) + ).removesuffix(WS_DEFLATE_TRAILING), + opcode, + 0x40, + ) + + async def _send_compressed_frame_async_locked( + self, message: bytes, opcode: int, compress: int | None + ) -> None: + """ + Async send for large compressed frames with lock. + + Acquires the lock and compresses large payloads asynchronously in + the executor. The lock is held for the entire operation to ensure + the compressor state is not corrupted by concurrent sends. + + MUST be run shielded from cancellation. If cancelled after + compression but before sending, the compressor state would be + advanced but data not sent, corrupting subsequent frames. + """ + async with self._send_lock: + # RSV are the reserved bits in the frame header. They are used to + # indicate that the frame is using an extension. + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.2 + compressobj = self._get_compressor(compress) + # (0x40) RSV1 is set for compressed frames + # https://datatracker.ietf.org/doc/html/rfc7692#section-7.2.3.1 + self._write_websocket_frame( + ( + await compressobj.compress(message) + + compressobj.flush( + ZLibBackend.Z_FULL_FLUSH + if self.notakeover + else ZLibBackend.Z_SYNC_FLUSH + ) + ).removesuffix(WS_DEFLATE_TRAILING), + opcode, + 0x40, + ) + + async def close(self, code: int = 1000, message: bytes | str = b"") -> None: + """Close the websocket, sending the specified code and message.""" + if isinstance(message, str): + message = message.encode("utf-8") + try: + await self.send_frame( + PACK_CLOSE_CODE(code) + message, opcode=WSMsgType.CLOSE + ) + finally: + self._closing = True diff --git a/venv/Lib/site-packages/aiohttp/abc.py b/venv/Lib/site-packages/aiohttp/abc.py new file mode 100644 index 0000000000000000000000000000000000000000..ea4924d922615e58923155f5026b59eecdade947 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/abc.py @@ -0,0 +1,270 @@ +import asyncio +import logging +import socket +from abc import ABC, abstractmethod +from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence, Sized +from http.cookies import BaseCookie, Morsel, SimpleCookie +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, TypedDict + +from multidict import CIMultiDict +from yarl import URL + +from ._cookie_helpers import parse_set_cookie_headers +from .typedefs import LooseCookies + +if TYPE_CHECKING: + from .web_app import Application + from .web_exceptions import HTTPException + from .web_request import BaseRequest, Request + from .web_response import StreamResponse +else: + BaseRequest = Request = Application = StreamResponse = Any + HTTPException = Any + + +class AbstractRouter(ABC): + def __init__(self) -> None: + self._frozen = False + + def post_init(self, app: Application) -> None: + """Post init stage. + + Not an abstract method for sake of backward compatibility, + but if the router wants to be aware of the application + it can override this. + """ + + @property + def frozen(self) -> bool: + return self._frozen + + def freeze(self) -> None: + """Freeze router.""" + self._frozen = True + + @abstractmethod + async def resolve(self, request: Request) -> "AbstractMatchInfo": + """Return MATCH_INFO for given request""" + + +class AbstractMatchInfo(ABC): + + __slots__ = () + + @property # pragma: no branch + @abstractmethod + def handler(self) -> Callable[[Request], Awaitable[StreamResponse]]: + """Execute matched request handler""" + + @property + @abstractmethod + def expect_handler( + self, + ) -> Callable[[Request], Awaitable[StreamResponse | None]]: + """Expect handler for 100-continue processing""" + + @property # pragma: no branch + @abstractmethod + def http_exception(self) -> HTTPException | None: + """HTTPException instance raised on router's resolving, or None""" + + @abstractmethod # pragma: no branch + def get_info(self) -> dict[str, Any]: + """Return a dict with additional info useful for introspection""" + + @property # pragma: no branch + @abstractmethod + def apps(self) -> tuple[Application, ...]: + """Stack of nested applications. + + Top level application is left-most element. + + """ + + @abstractmethod + def add_app(self, app: Application) -> None: + """Add application to the nested apps stack.""" + + @abstractmethod + def freeze(self) -> None: + """Freeze the match info. + + The method is called after route resolution. + + After the call .add_app() is forbidden. + + """ + + +class AbstractView(ABC): + """Abstract class based view.""" + + def __init__(self, request: Request) -> None: + self._request = request + + @property + def request(self) -> Request: + """Request instance.""" + return self._request + + @abstractmethod + def __await__(self) -> Generator[None, None, StreamResponse]: + """Execute the view handler.""" + + +class ResolveResult(TypedDict): + """Resolve result. + + This is the result returned from an AbstractResolver's + resolve method. + + :param hostname: The hostname that was provided. + :param host: The IP address that was resolved. + :param port: The port that was resolved. + :param family: The address family that was resolved. + :param proto: The protocol that was resolved. + :param flags: The flags that were resolved. + """ + + hostname: str + host: str + port: int + family: int + proto: int + flags: int + + +class AbstractResolver(ABC): + """Abstract DNS resolver.""" + + @abstractmethod + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> list[ResolveResult]: + """Return IP address for given hostname""" + + @abstractmethod + async def close(self) -> None: + """Release resolver""" + + +if TYPE_CHECKING: + IterableBase = Iterable[Morsel[str]] +else: + IterableBase = Iterable + + +ClearCookiePredicate = Callable[["Morsel[str]"], bool] + + +class AbstractCookieJar(Sized, IterableBase): + """Abstract Cookie Jar.""" + + def __init__(self, *, loop: asyncio.AbstractEventLoop | None = None) -> None: + self._loop = loop or asyncio.get_running_loop() + + @property + @abstractmethod + def unsafe(self) -> bool: + """Return True if cookies can be used with IP addresses.""" + + @property + @abstractmethod + def quote_cookie(self) -> bool: + """Return True if cookies should be quoted.""" + + @property + @abstractmethod + def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]: + """Return the cookies stored in this jar.""" + + @property + @abstractmethod + def host_only_cookies(self) -> frozenset[tuple[str, str]]: + """Return the host-only cookies stored in this jar.""" + + @abstractmethod + def clear(self, predicate: ClearCookiePredicate | None = None) -> None: + """Clear all cookies if no predicate is passed.""" + + @abstractmethod + def clear_domain(self, domain: str) -> None: + """Clear all cookies for domain and all subdomains.""" + + @abstractmethod + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + """Update cookies.""" + + def update_cookies_from_headers( + self, headers: Sequence[str], response_url: URL + ) -> None: + """Update cookies from raw Set-Cookie headers.""" + if headers and (cookies_to_update := parse_set_cookie_headers(headers)): + self.update_cookies(cookies_to_update, response_url) + + @abstractmethod + def filter_cookies(self, request_url: URL) -> "BaseCookie[str]": + """Return the jar's cookies filtered by their attributes.""" + + +class AbstractStreamWriter(ABC): + """Abstract stream writer.""" + + buffer_size: int = 0 + output_size: int = 0 + length: int | None = 0 + + @abstractmethod + async def write(self, chunk: bytes | bytearray | memoryview) -> None: + """Write chunk into stream.""" + + @abstractmethod + async def write_eof(self, chunk: bytes = b"") -> None: + """Write last chunk.""" + + @abstractmethod + async def drain(self) -> None: + """Flush the write buffer.""" + + @abstractmethod + def enable_compression( + self, encoding: str = "deflate", strategy: int | None = None + ) -> None: + """Enable HTTP body compression""" + + @abstractmethod + def enable_chunking(self) -> None: + """Enable HTTP chunked mode""" + + @abstractmethod + async def write_headers( + self, status_line: str, headers: "CIMultiDict[str]" + ) -> None: + """Write HTTP headers""" + + def send_headers(self) -> None: + """Force sending buffered headers if not already sent. + + Required only if write_headers() buffers headers instead of sending immediately. + For backwards compatibility, this method does nothing by default. + """ + + +class AbstractAccessLogger(ABC): + """Abstract writer to access log.""" + + __slots__ = ("logger", "log_format") + + def __init__(self, logger: logging.Logger, log_format: str) -> None: + self.logger = logger + self.log_format = log_format + + @abstractmethod + def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None: + """Emit log to logger.""" + + @property + def enabled(self) -> bool: + """Check if logger is enabled.""" + return True diff --git a/venv/Lib/site-packages/aiohttp/base_protocol.py b/venv/Lib/site-packages/aiohttp/base_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..2be5728d6e8e667e326f2123e7cf72fdea715629 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/base_protocol.py @@ -0,0 +1,140 @@ +import asyncio +from typing import TYPE_CHECKING, Any, cast + +from .client_exceptions import ClientConnectionResetError +from .helpers import set_exception +from .tcp_helpers import tcp_nodelay + +if TYPE_CHECKING: + from .http_parser import HttpParser + +# Raised by transport.pause_reading()/resume_reading() when the transport +# does not support flow control; safe to ignore. +# NOTE: Catch these with a plain try/except/pass, never contextlib.suppress(): +# pause/resume run on the hot read path and suppress() is ~6x slower than +# try/except here (it builds a context manager and unpacks this tuple per call). +PAUSE_RESUME_READING_ERRORS = (AttributeError, NotImplementedError, RuntimeError) + + +class BaseProtocol(asyncio.Protocol): + __slots__ = ( + "_loop", + "_paused", + "_parser", + "_drain_waiter", + "_connection_lost", + "_reading_paused", + "_upgraded", + "transport", + ) + + def __init__( + self, loop: asyncio.AbstractEventLoop, parser: "HttpParser[Any] | None" = None + ) -> None: + self._loop: asyncio.AbstractEventLoop = loop + self._paused = False + self._drain_waiter: asyncio.Future[None] | None = None + self._reading_paused = False + self._parser = parser + self._upgraded = False + + self.transport: asyncio.Transport | None = None + + @property + def connected(self) -> bool: + """Return True if the connection is open.""" + return self.transport is not None + + @property + def writing_paused(self) -> bool: + return self._paused + + def pause_writing(self) -> None: + assert not self._paused + self._paused = True + + def resume_writing(self) -> None: + assert self._paused + self._paused = False + + waiter = self._drain_waiter + if waiter is not None: + self._drain_waiter = None + if not waiter.done(): + waiter.set_result(None) + + def pause_reading(self) -> None: + self._reading_paused = True + # Parser shouldn't be paused on websockets. + if not self._upgraded: + assert self._parser is not None + self._parser.pause_reading() + if self.transport is not None: + try: + self.transport.pause_reading() + except PAUSE_RESUME_READING_ERRORS: + # Transport lacks flow control; nothing to pause. Intentionally + # ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress). + pass + + def _reading_paused_for_msg_queue(self) -> bool: + """Keep the transport paused for protocol-specific reasons (overridden).""" + return False + + def resume_reading(self, resume_parser: bool = True) -> None: + self._reading_paused = False + + # This will resume parsing any unprocessed data from the last pause. + if not self._upgraded and resume_parser: + self.data_received(b"") + + # Reading may have been paused again in the above call if there was a lot of + # compressed data still pending. + if ( + not self._reading_paused + and not self._reading_paused_for_msg_queue() + and self.transport is not None + ): + try: + self.transport.resume_reading() + except PAUSE_RESUME_READING_ERRORS: + # Transport lacks flow control; nothing to resume. Intentionally + # ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress). + pass + self._reading_paused = False + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + tr = cast(asyncio.Transport, transport) + tcp_nodelay(tr, True) + self.transport = tr + + def connection_lost(self, exc: BaseException | None) -> None: + # Wake up the writer if currently paused. + self.transport = None + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + return + self._drain_waiter = None + if waiter.done(): + return + if exc is None: + waiter.set_result(None) + else: + set_exception( + waiter, + ConnectionError("Connection lost"), + exc, + ) + + async def _drain_helper(self) -> None: + if self.transport is None: + raise ClientConnectionResetError("Connection lost") + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + waiter = self._loop.create_future() + self._drain_waiter = waiter + await waiter diff --git a/venv/Lib/site-packages/aiohttp/client.py b/venv/Lib/site-packages/aiohttp/client.py new file mode 100644 index 0000000000000000000000000000000000000000..99eb72cf74900e5a6983ac04178ad2a593056a44 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/client.py @@ -0,0 +1,1808 @@ +"""HTTP Client for asyncio.""" + +import asyncio +import base64 +import hashlib +import json +import os +import sys +import traceback +import warnings +from collections.abc import ( + Awaitable, + Callable, + Coroutine, + Generator, + Iterable, + Sequence, +) +from contextlib import suppress +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Final, + Generic, + Literal, + TypedDict, + TypeVar, + overload, +) + +import attr +from multidict import CIMultiDict, MultiDict, MultiDictProxy, istr +from yarl import URL + +from . import hdrs, http, payload +from ._websocket.reader import WebSocketDataQueue +from .abc import AbstractCookieJar +from .client_exceptions import ( + ClientConnectionError, + ClientConnectionResetError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientError, + ClientHttpProxyError, + ClientOSError, + ClientPayloadError, + ClientProxyConnectionError, + ClientResponseError, + ClientSSLError, + ConnectionTimeoutError, + ContentTypeError, + InvalidURL, + InvalidUrlClientError, + InvalidUrlRedirectClientError, + NonHttpUrlClientError, + NonHttpUrlRedirectClientError, + RedirectClientError, + ServerConnectionError, + ServerDisconnectedError, + ServerFingerprintMismatch, + ServerTimeoutError, + SocketTimeoutError, + TooManyRedirects, + WSMessageTypeError, + WSServerHandshakeError, +) +from .client_middlewares import ClientMiddlewareType, build_client_middlewares +from .client_reqrep import ( + ClientRequest as ClientRequest, + ClientResponse as ClientResponse, + Fingerprint as Fingerprint, + RequestInfo as RequestInfo, + _merge_ssl_params, +) +from .client_ws import ( + DEFAULT_WS_CLIENT_TIMEOUT, + ClientWebSocketResponse as ClientWebSocketResponse, + ClientWSTimeout as ClientWSTimeout, +) +from .connector import ( + HTTP_AND_EMPTY_SCHEMA_SET, + BaseConnector as BaseConnector, + NamedPipeConnector as NamedPipeConnector, + TCPConnector as TCPConnector, + UnixConnector as UnixConnector, +) +from .cookiejar import CookieJar +from .helpers import ( + _SENTINEL, + DEBUG, + DEFAULT_CHUNK_SIZE, + EMPTY_BODY_METHODS, + BasicAuth, + TimeoutHandle, + basicauth_from_netrc, + get_env_proxy_for_url, + netrc_from_env, + sentinel, + strip_auth_from_url, +) +from .http import WS_KEY, HttpVersion, WebSocketReader, WebSocketWriter +from .http_websocket import WSHandshakeError, ws_ext_gen, ws_ext_parse +from .tracing import Trace, TraceConfig +from .typedefs import ( + JSONBytesEncoder, + JSONEncoder, + LooseCookies, + LooseHeaders, + Query, + StrOrURL, +) + +__all__ = ( + # client_exceptions + "ClientConnectionError", + "ClientConnectionResetError", + "ClientConnectorCertificateError", + "ClientConnectorDNSError", + "ClientConnectorError", + "ClientConnectorSSLError", + "ClientError", + "ClientHttpProxyError", + "ClientOSError", + "ClientPayloadError", + "ClientProxyConnectionError", + "ClientResponseError", + "ClientSSLError", + "ConnectionTimeoutError", + "ContentTypeError", + "InvalidURL", + "InvalidUrlClientError", + "RedirectClientError", + "NonHttpUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlRedirectClientError", + "ServerConnectionError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ServerTimeoutError", + "SocketTimeoutError", + "TooManyRedirects", + "WSServerHandshakeError", + # client_reqrep + "ClientRequest", + "ClientResponse", + "Fingerprint", + "RequestInfo", + # connector + "BaseConnector", + "TCPConnector", + "UnixConnector", + "NamedPipeConnector", + # client_ws + "ClientWebSocketResponse", + # client + "ClientSession", + "ClientTimeout", + "ClientWSTimeout", + "request", + "WSMessageTypeError", +) + + +if TYPE_CHECKING: + from ssl import SSLContext +else: + SSLContext = Any + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + from typing import Unpack + + +class _RequestOptions(TypedDict, total=False): + params: Query + data: Any + json: Any + cookies: LooseCookies | None + headers: LooseHeaders | None + skip_auto_headers: Iterable[str] | None + auth: BasicAuth | None + allow_redirects: bool + max_redirects: int + compress: str | bool | None + chunked: bool | None + expect100: bool + raise_for_status: None | bool | Callable[[ClientResponse], Awaitable[None]] + read_until_eof: bool + proxy: StrOrURL | None + proxy_auth: BasicAuth | None + timeout: "ClientTimeout | _SENTINEL | None" + ssl: SSLContext | bool | Fingerprint + server_hostname: str | None + proxy_headers: LooseHeaders | None + trace_request_ctx: object + read_bufsize: int | None + auto_decompress: bool | None + max_line_size: int | None + max_field_size: int | None + max_headers: int | None + middlewares: Sequence[ClientMiddlewareType] | None + + +class _WSConnectOptions(TypedDict, total=False): + method: str + protocols: Iterable[str] + timeout: "ClientWSTimeout | _SENTINEL" + receive_timeout: float | None + autoclose: bool + autoping: bool + heartbeat: float | None + auth: BasicAuth | None + origin: str | None + params: Query + headers: LooseHeaders | None + proxy: StrOrURL | None + proxy_auth: BasicAuth | None + ssl: SSLContext | bool | Fingerprint + verify_ssl: bool | None + fingerprint: bytes | None + ssl_context: SSLContext | None + server_hostname: str | None + proxy_headers: LooseHeaders | None + compress: int + max_msg_size: int + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ClientTimeout: + total: float | None = None + connect: float | None = None + sock_read: float | None = None + sock_connect: float | None = None + ceil_threshold: float = 5 + + # pool_queue_timeout: Optional[float] = None + # dns_resolution_timeout: Optional[float] = None + # socket_connect_timeout: Optional[float] = None + # connection_acquiring_timeout: Optional[float] = None + # new_connection_timeout: Optional[float] = None + # http_header_timeout: Optional[float] = None + # response_body_timeout: Optional[float] = None + + # to create a timeout specific for a single request, either + # - create a completely new one to overwrite the default + # - or use http://www.attrs.org/en/stable/api.html#attr.evolve + # to overwrite the defaults + + +# 5 Minute default read timeout +DEFAULT_TIMEOUT: Final[ClientTimeout] = ClientTimeout(total=5 * 60, sock_connect=30) + +# https://www.rfc-editor.org/rfc/rfc9110#section-9.2.2 +IDEMPOTENT_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "TRACE", "PUT", "DELETE"}) + +_RetType_co = TypeVar( + "_RetType_co", + bound="ClientResponse | ClientWebSocketResponse[bool]", + covariant=True, +) +_CharsetResolver = Callable[[ClientResponse, bytes], str] + + +class ClientSession: + """First-class interface for making HTTP requests.""" + + ATTRS = frozenset( + [ + "_base_url", + "_base_url_origin", + "_source_traceback", + "_connector", + "_loop", + "_cookie_jar", + "_connector_owner", + "_default_auth", + "_version", + "_json_serialize", + "_json_serialize_bytes", + "_requote_redirect_url", + "_timeout", + "_raise_for_status", + "_auto_decompress", + "_trust_env", + "_default_headers", + "_skip_auto_headers", + "_request_class", + "_response_class", + "_ws_response_class", + "_trace_configs", + "_read_bufsize", + "_max_line_size", + "_max_field_size", + "_max_headers", + "_resolve_charset", + "_default_proxy", + "_default_proxy_auth", + "_retry_connection", + "_middlewares", + "requote_redirect_url", + ] + ) + + _source_traceback: traceback.StackSummary | None = None + _connector: BaseConnector | None = None + + def __init__( + self, + base_url: StrOrURL | None = None, + *, + connector: BaseConnector | None = None, + loop: asyncio.AbstractEventLoop | None = None, + cookies: LooseCookies | None = None, + headers: LooseHeaders | None = None, + proxy: StrOrURL | None = None, + proxy_auth: BasicAuth | None = None, + skip_auto_headers: Iterable[str] | None = None, + auth: BasicAuth | None = None, + json_serialize: JSONEncoder = json.dumps, + json_serialize_bytes: JSONBytesEncoder | None = None, + request_class: type[ClientRequest] = ClientRequest, + response_class: type[ClientResponse] = ClientResponse, + ws_response_class: type[ClientWebSocketResponse] = ClientWebSocketResponse, + version: HttpVersion = http.HttpVersion11, + cookie_jar: AbstractCookieJar | None = None, + connector_owner: bool = True, + raise_for_status: bool | Callable[[ClientResponse], Awaitable[None]] = False, + read_timeout: float | _SENTINEL = sentinel, + conn_timeout: float | None = None, + timeout: object | ClientTimeout = sentinel, + auto_decompress: bool = True, + trust_env: bool = False, + requote_redirect_url: bool = True, + trace_configs: list[TraceConfig] | None = None, + read_bufsize: int = DEFAULT_CHUNK_SIZE, + max_line_size: int = 8190, + max_field_size: int = 8190, + max_headers: int = 128, + fallback_charset_resolver: _CharsetResolver = lambda r, b: "utf-8", + middlewares: Sequence[ClientMiddlewareType] = (), + ssl_shutdown_timeout: _SENTINEL | None | float = sentinel, + ) -> None: + # We initialise _connector to None immediately, as it's referenced in __del__() + # and could cause issues if an exception occurs during initialisation. + self._connector: BaseConnector | None = None + + if loop is None: + if connector is not None: + loop = connector._loop + + loop = loop or asyncio.get_running_loop() + + if base_url is None or isinstance(base_url, URL): + self._base_url: URL | None = base_url + self._base_url_origin = None if base_url is None else base_url.origin() + else: + self._base_url = URL(base_url) + self._base_url_origin = self._base_url.origin() + assert self._base_url.absolute, "Only absolute URLs are supported" + if self._base_url is not None and not self._base_url.path.endswith("/"): + raise ValueError("base_url must have a trailing '/'") + + if timeout is sentinel or timeout is None: + self._timeout = DEFAULT_TIMEOUT + if read_timeout is not sentinel: + warnings.warn( + "read_timeout is deprecated, use timeout argument instead", + DeprecationWarning, + stacklevel=2, + ) + self._timeout = attr.evolve(self._timeout, total=read_timeout) + if conn_timeout is not None: + self._timeout = attr.evolve(self._timeout, connect=conn_timeout) + warnings.warn( + "conn_timeout is deprecated, use timeout argument instead", + DeprecationWarning, + stacklevel=2, + ) + else: + if not isinstance(timeout, ClientTimeout): + raise ValueError( + f"timeout parameter cannot be of {type(timeout)} type, " + "please use 'timeout=ClientTimeout(...)'", + ) + self._timeout = timeout + if read_timeout is not sentinel: + raise ValueError( + "read_timeout and timeout parameters " + "conflict, please setup " + "timeout.read" + ) + if conn_timeout is not None: + raise ValueError( + "conn_timeout and timeout parameters " + "conflict, please setup " + "timeout.connect" + ) + + if ssl_shutdown_timeout is not sentinel: + warnings.warn( + "The ssl_shutdown_timeout parameter is deprecated and will be removed in aiohttp 4.0", + DeprecationWarning, + stacklevel=2, + ) + + if connector is None: + connector = TCPConnector( + loop=loop, ssl_shutdown_timeout=ssl_shutdown_timeout + ) + + if connector._loop is not loop: + raise RuntimeError("Session and connector has to use same event loop") + + self._loop = loop + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + if cookie_jar is None: + cookie_jar = CookieJar(loop=loop) + self._cookie_jar = cookie_jar + + if cookies: + self._cookie_jar.update_cookies(cookies) + + if auth is not None: + warnings.warn( + "The 'auth' parameter is deprecated and will be removed in v4;" + " pass headers={'Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=2, + ) + if proxy_auth is not None: + warnings.warn( + "The 'proxy_auth' parameter is deprecated and will be removed in v4;" + " pass proxy_headers={'Proxy-Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=2, + ) + self._connector = connector + self._connector_owner = connector_owner + self._default_auth = auth + self._version = version + self._json_serialize = json_serialize + self._json_serialize_bytes = json_serialize_bytes + self._raise_for_status = raise_for_status + self._auto_decompress = auto_decompress + self._trust_env = trust_env + self._requote_redirect_url = requote_redirect_url + self._read_bufsize = read_bufsize + self._max_line_size = max_line_size + self._max_field_size = max_field_size + self._max_headers = max_headers + + # Convert to list of tuples + if headers: + real_headers: CIMultiDict[str] = CIMultiDict(headers) + else: + real_headers = CIMultiDict() + self._default_headers: CIMultiDict[str] = real_headers + if skip_auto_headers is not None: + self._skip_auto_headers = frozenset(istr(i) for i in skip_auto_headers) + else: + self._skip_auto_headers = frozenset() + + self._request_class = request_class + self._response_class = response_class + self._ws_response_class = ws_response_class + + self._trace_configs = trace_configs or [] + for trace_config in self._trace_configs: + trace_config.freeze() + + self._resolve_charset = fallback_charset_resolver + + self._default_proxy = proxy + self._default_proxy_auth = proxy_auth + self._retry_connection: bool = True + self._middlewares = middlewares + + def __init_subclass__(cls: type["ClientSession"]) -> None: + warnings.warn( + f"Inheritance class {cls.__name__} from ClientSession is discouraged", + DeprecationWarning, + stacklevel=2, + ) + + if DEBUG: + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + f"Setting custom ClientSession.{name} attribute is discouraged", + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + def __del__(self, _warnings: Any = warnings) -> None: + if not self.closed: + kwargs = {"source": self} + _warnings.warn( + f"Unclosed client session {self!r}", ResourceWarning, **kwargs + ) + context = {"client_session": self, "message": "Unclosed client session"} + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + self, + method: str, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + else: + + def request( + self, method: str, url: StrOrURL, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP request.""" + return _RequestContextManager(self._request(method, url, **kwargs)) + + def _build_url(self, str_or_url: StrOrURL) -> URL: + url = URL(str_or_url) + if self._base_url and not url.absolute: + return self._base_url.join(url) + return url + + async def _request( + self, + method: str, + str_or_url: StrOrURL, + *, + params: Query = None, + data: Any = None, + json: Any = None, + cookies: LooseCookies | None = None, + headers: LooseHeaders | None = None, + skip_auto_headers: Iterable[str] | None = None, + auth: BasicAuth | None = None, + allow_redirects: bool = True, + max_redirects: int = 10, + compress: str | bool | None = None, + chunked: bool | None = None, + expect100: bool = False, + raise_for_status: ( + None | bool | Callable[[ClientResponse], Awaitable[None]] + ) = None, + read_until_eof: bool = True, + proxy: StrOrURL | None = None, + proxy_auth: BasicAuth | None = None, + timeout: ClientTimeout | _SENTINEL = sentinel, + verify_ssl: bool | None = None, + fingerprint: bytes | None = None, + ssl_context: SSLContext | None = None, + ssl: SSLContext | bool | Fingerprint = True, + server_hostname: str | None = None, + proxy_headers: LooseHeaders | None = None, + trace_request_ctx: object = None, + read_bufsize: int | None = None, + auto_decompress: bool | None = None, + max_line_size: int | None = None, + max_field_size: int | None = None, + max_headers: int | None = None, + middlewares: Sequence[ClientMiddlewareType] | None = None, + ) -> ClientResponse: + + # NOTE: timeout clamps existing connect and read timeouts. We cannot + # set the default to None because we need to detect if the user wants + # to use the existing timeouts by setting timeout to None. + + if self.closed: + raise RuntimeError("Session is closed") + + ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + if auth is not None: + warnings.warn( + "The 'auth' parameter is deprecated and will be removed in v4;" + " pass headers={'Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=3, + ) + if proxy_auth is not None: + warnings.warn( + "The 'proxy_auth' parameter is deprecated and will be removed in v4;" + " pass proxy_headers={'Proxy-Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=3, + ) + + if data is not None and json is not None: + raise ValueError( + "data and json parameters can not be used at the same time" + ) + elif json is not None: + if self._json_serialize_bytes is not None: + data = payload.JsonBytesPayload(json, dumps=self._json_serialize_bytes) + else: + data = payload.JsonPayload(json, dumps=self._json_serialize) + + if not isinstance(chunked, bool) and chunked is not None: + warnings.warn("Chunk size is deprecated #1615", DeprecationWarning) + + redirects = 0 + history: list[ClientResponse] = [] + version = self._version + params = params or {} + + # Merge with default headers and transform to CIMultiDict + headers = self._prepare_headers(headers) + + try: + url = self._build_url(str_or_url) + except ValueError as e: + raise InvalidUrlClientError(str_or_url) from e + + assert self._connector is not None + if url.scheme not in self._connector.allowed_protocol_schema_set: + raise NonHttpUrlClientError(url) + + skip_headers: Iterable[istr] | None + if skip_auto_headers is not None: + skip_headers = { + istr(i) for i in skip_auto_headers + } | self._skip_auto_headers + elif self._skip_auto_headers: + skip_headers = self._skip_auto_headers + else: + skip_headers = None + + if proxy is None: + proxy = self._default_proxy + if proxy_auth is None: + proxy_auth = self._default_proxy_auth + + if proxy is None: + proxy_headers = None + else: + proxy_headers = self._prepare_headers(proxy_headers) + try: + proxy = URL(proxy) + except ValueError as e: + raise InvalidURL(proxy) from e + + if timeout is sentinel: + real_timeout: ClientTimeout = self._timeout + else: + if not isinstance(timeout, ClientTimeout): + real_timeout = ClientTimeout(total=timeout) + else: + real_timeout = timeout + # timeout is cumulative for all request operations + # (request, redirects, responses, data consuming) + tm = TimeoutHandle( + self._loop, real_timeout.total, ceil_threshold=real_timeout.ceil_threshold + ) + handle = tm.start() + + if read_bufsize is None: + read_bufsize = self._read_bufsize + + if auto_decompress is None: + auto_decompress = self._auto_decompress + + if max_line_size is None: + max_line_size = self._max_line_size + + if max_field_size is None: + max_field_size = self._max_field_size + + if max_headers is None: + max_headers = self._max_headers + + traces = [ + Trace( + self, + trace_config, + trace_config.trace_config_ctx(trace_request_ctx=trace_request_ctx), + ) + for trace_config in self._trace_configs + ] + + for trace in traces: + await trace.send_request_start(method, url.update_query(params), headers) + + timer = tm.timer() + try: + with timer: + # https://www.rfc-editor.org/rfc/rfc9112.html#name-retrying-requests + retry_persistent_connection = ( + self._retry_connection and method in IDEMPOTENT_METHODS + ) + while True: + url, auth_from_url = strip_auth_from_url(url) + if not url.raw_host: + # NOTE: Bail early, otherwise, causes `InvalidURL` through + # NOTE: `self._request_class()` below. + err_exc_cls = ( + InvalidUrlRedirectClientError + if redirects + else InvalidUrlClientError + ) + raise err_exc_cls(url) + # If `auth` was passed for an already authenticated URL, + # disallow only if this is the initial URL; this is to avoid issues + # with sketchy redirects that are not the caller's responsibility + if not history and (auth and auth_from_url): + raise ValueError( + "Cannot combine AUTH argument with " + "credentials encoded in URL" + ) + + # Override the auth with the one from the URL only if we + # have no auth, or if we got an auth from a redirect URL + if auth is None or (history and auth_from_url is not None): + auth = auth_from_url + + if ( + auth is None + and self._default_auth + and ( + not self._base_url or self._base_url_origin == url.origin() + ) + ): + auth = self._default_auth + + # Try netrc if auth is still None and trust_env is enabled. + if auth is None and self._trust_env and url.host is not None: + auth = await self._loop.run_in_executor( + None, self._get_netrc_auth, url.host + ) + + # It would be confusing if we support explicit + # Authorization header with auth argument + if ( + headers is not None + and auth is not None + and hdrs.AUTHORIZATION in headers + ): + raise ValueError( + "Cannot combine AUTHORIZATION header " + "with AUTH argument or credentials " + "encoded in URL" + ) + + all_cookies = self._cookie_jar.filter_cookies(url) + + if cookies is not None: + tmp_cookie_jar = CookieJar( + unsafe=self._cookie_jar.unsafe, + quote_cookie=self._cookie_jar.quote_cookie, + ) + tmp_cookie_jar.update_cookies(cookies) + req_cookies = tmp_cookie_jar.filter_cookies(url) + if req_cookies: + all_cookies.load(req_cookies) + + proxy_: URL | None = None + if proxy is not None: + proxy_ = URL(proxy) + elif self._trust_env: + with suppress(LookupError): + proxy_, proxy_auth = await asyncio.to_thread( + get_env_proxy_for_url, url + ) + + req = self._request_class( + method, + url, + params=params, + headers=headers, + skip_auto_headers=skip_headers, + data=data, + cookies=all_cookies, + auth=auth, + version=version, + compress=compress, + chunked=chunked, + expect100=expect100, + loop=self._loop, + response_class=self._response_class, + proxy=proxy_, + proxy_auth=proxy_auth, + timer=timer, + session=self, + ssl=ssl if ssl is not None else True, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + traces=traces, + trust_env=self.trust_env, + ) + + async def _connect_and_send_request( + req: ClientRequest, + ) -> ClientResponse: + # connection timeout + assert self._connector is not None + try: + conn = await self._connector.connect( + req, traces=traces, timeout=real_timeout + ) + except asyncio.TimeoutError as exc: + raise ConnectionTimeoutError( + f"Connection timeout to host {req.url}" + ) from exc + + assert conn.protocol is not None + conn.protocol.set_response_params( + timer=timer, + skip_payload=req.method in EMPTY_BODY_METHODS, + read_until_eof=read_until_eof, + auto_decompress=auto_decompress, + read_timeout=real_timeout.sock_read, + read_bufsize=read_bufsize, + timeout_ceil_threshold=self._connector._timeout_ceil_threshold, + max_line_size=max_line_size, + max_field_size=max_field_size, + max_headers=max_headers, + ) + try: + resp = await req.send(conn) + try: + await resp.start(conn) + except BaseException: + resp.close() + raise + except BaseException: + conn.close() + raise + return resp + + # Apply middleware (if any) - per-request middleware overrides session middleware + effective_middlewares = ( + self._middlewares if middlewares is None else middlewares + ) + + if effective_middlewares: + handler = build_client_middlewares( + _connect_and_send_request, effective_middlewares + ) + else: + handler = _connect_and_send_request + + try: + resp = await handler(req) + # Client connector errors should not be retried + except ( + ConnectionTimeoutError, + ClientConnectorError, + ClientConnectorCertificateError, + ClientConnectorSSLError, + ): + raise + except (ClientOSError, ServerDisconnectedError): + if retry_persistent_connection: + retry_persistent_connection = False + continue + raise + except ClientError: + raise + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise ClientOSError(*exc.args) from exc + + # Update cookies from raw headers to preserve duplicates + if resp._raw_cookie_headers: + self._cookie_jar.update_cookies_from_headers( + resp._raw_cookie_headers, resp.url + ) + + # redirects + if resp.status in (301, 302, 303, 307, 308) and allow_redirects: + + for trace in traces: + await trace.send_request_redirect( + method, url.update_query(params), headers, resp + ) + + redirects += 1 + history.append(resp) + if max_redirects and redirects >= max_redirects: + if req._body is not None: + await req._body.close() + resp.close() + raise TooManyRedirects( + history[0].request_info, tuple(history) + ) + + # For 301 and 302, mimic IE, now changed in RFC + # https://github.com/kennethreitz/requests/pull/269 + if (resp.status == 303 and resp.method != hdrs.METH_HEAD) or ( + resp.status in (301, 302) and resp.method == hdrs.METH_POST + ): + method = hdrs.METH_GET + data = None + if headers.get(hdrs.CONTENT_LENGTH): + headers.pop(hdrs.CONTENT_LENGTH) + else: + # For 307/308, always preserve the request body + # For 301/302 with non-POST methods, preserve the request body + # https://www.rfc-editor.org/rfc/rfc9110#section-15.4.3-3.1 + # Use the existing payload to avoid recreating it from + # a potentially consumed file. + # + # If the payload is already consumed and cannot be replayed, + # fail fast instead of silently sending an empty body. + if req._body is not None and req._body.consumed: + resp.close() + raise ClientPayloadError( + "Cannot follow redirect with a consumed request " + "body. Use bytes, a seekable file-like object, " + "or set allow_redirects=False." + ) + data = req._body + + r_url = resp.headers.get(hdrs.LOCATION) or resp.headers.get( + hdrs.URI + ) + if r_url is None: + # see github.com/aio-libs/aiohttp/issues/2022 + break + else: + # reading from correct redirection + # response is forbidden + resp.release() + + try: + parsed_redirect_url = URL( + r_url, encoded=not self._requote_redirect_url + ) + except ValueError as e: + if req._body is not None: + await req._body.close() + resp.close() + raise InvalidUrlRedirectClientError( + r_url, + "Server attempted redirecting to a location that does not look like a URL", + ) from e + + scheme = parsed_redirect_url.scheme + if scheme not in HTTP_AND_EMPTY_SCHEMA_SET: + if req._body is not None: + await req._body.close() + resp.close() + raise NonHttpUrlRedirectClientError(r_url) + elif not scheme: + parsed_redirect_url = url.join(parsed_redirect_url) + + try: + redirect_origin = parsed_redirect_url.origin() + except ValueError as origin_val_err: + if req._body is not None: + await req._body.close() + resp.close() + raise InvalidUrlRedirectClientError( + parsed_redirect_url, + "Invalid redirect URL origin", + ) from origin_val_err + + if url.origin() != redirect_origin: + auth = None + cookies = None + headers.pop(hdrs.AUTHORIZATION, None) + headers.pop(hdrs.COOKIE, None) + headers.pop(hdrs.PROXY_AUTHORIZATION, None) + + url = parsed_redirect_url + params = {} + resp.release() + continue + + break + + if req._body is not None: + await req._body.close() + # check response status + if raise_for_status is None: + raise_for_status = self._raise_for_status + + if raise_for_status is None: + pass + elif callable(raise_for_status): + await raise_for_status(resp) + elif raise_for_status: + resp.raise_for_status() + + # register connection + if handle is not None: + if resp.connection is not None: + resp.connection.add_callback(handle.cancel) + else: + handle.cancel() + + resp._history = tuple(history) + + for trace in traces: + await trace.send_request_end( + method, url.update_query(params), headers, resp + ) + return resp + + except BaseException as e: + # cleanup timer + tm.close() + if handle: + handle.cancel() + handle = None + + for trace in traces: + await trace.send_request_exception( + method, url.update_query(params), headers, e + ) + raise + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + @overload + def ws_connect( + self, + url: StrOrURL, + *, + decode_text: Literal[True] = ..., + **kwargs: Unpack[_WSConnectOptions], + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[True]]]": ... + + @overload + def ws_connect( + self, + url: StrOrURL, + *, + decode_text: Literal[False], + **kwargs: Unpack[_WSConnectOptions], + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[False]]]": ... + + @overload + def ws_connect( + self, + url: StrOrURL, + *, + decode_text: bool = ..., + **kwargs: Unpack[_WSConnectOptions], + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": ... + + def ws_connect( + self, + url: StrOrURL, + *, + method: str = hdrs.METH_GET, + protocols: Iterable[str] = (), + timeout: ClientWSTimeout | _SENTINEL = sentinel, + receive_timeout: float | None = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: float | None = None, + auth: BasicAuth | None = None, + origin: str | None = None, + params: Query = None, + headers: LooseHeaders | None = None, + proxy: StrOrURL | None = None, + proxy_auth: BasicAuth | None = None, + ssl: SSLContext | bool | Fingerprint = True, + verify_ssl: bool | None = None, + fingerprint: bytes | None = None, + ssl_context: SSLContext | None = None, + server_hostname: str | None = None, + proxy_headers: LooseHeaders | None = None, + compress: int = 0, + max_msg_size: int = 4 * 1024 * 1024, + decode_text: bool = True, + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": + """Initiate websocket connection.""" + return _WSRequestContextManager( + self._ws_connect( + url, + method=method, + protocols=protocols, + timeout=timeout, + receive_timeout=receive_timeout, + autoclose=autoclose, + autoping=autoping, + heartbeat=heartbeat, + auth=auth, + origin=origin, + params=params, + headers=headers, + proxy=proxy, + proxy_auth=proxy_auth, + ssl=ssl, + verify_ssl=verify_ssl, + fingerprint=fingerprint, + ssl_context=ssl_context, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + compress=compress, + max_msg_size=max_msg_size, + decode_text=decode_text, + ) + ) + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + @overload + async def _ws_connect( + self, + url: StrOrURL, + *, + decode_text: Literal[True] = ..., + **kwargs: Unpack[_WSConnectOptions], + ) -> "ClientWebSocketResponse[Literal[True]]": ... + + @overload + async def _ws_connect( + self, + url: StrOrURL, + *, + decode_text: Literal[False], + **kwargs: Unpack[_WSConnectOptions], + ) -> "ClientWebSocketResponse[Literal[False]]": ... + + @overload + async def _ws_connect( + self, + url: StrOrURL, + *, + decode_text: bool = ..., + **kwargs: Unpack[_WSConnectOptions], + ) -> "ClientWebSocketResponse[bool]": ... + + async def _ws_connect( + self, + url: StrOrURL, + *, + method: str = hdrs.METH_GET, + protocols: Iterable[str] = (), + timeout: ClientWSTimeout | _SENTINEL = sentinel, + receive_timeout: float | None = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: float | None = None, + auth: BasicAuth | None = None, + origin: str | None = None, + params: Query = None, + headers: LooseHeaders | None = None, + proxy: StrOrURL | None = None, + proxy_auth: BasicAuth | None = None, + ssl: SSLContext | bool | Fingerprint = True, + verify_ssl: bool | None = None, + fingerprint: bytes | None = None, + ssl_context: SSLContext | None = None, + server_hostname: str | None = None, + proxy_headers: LooseHeaders | None = None, + compress: int = 0, + max_msg_size: int = 4 * 1024 * 1024, + decode_text: bool = True, + ) -> "ClientWebSocketResponse[bool]": + if auth is not None: + warnings.warn( + "The 'auth' parameter is deprecated and will be removed in v4;" + " pass headers={'Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=3, + ) + if proxy_auth is not None: + warnings.warn( + "The 'proxy_auth' parameter is deprecated and will be removed in v4;" + " pass proxy_headers={'Proxy-Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=3, + ) + if timeout is not sentinel: + if isinstance(timeout, ClientWSTimeout): + ws_timeout = timeout + else: + warnings.warn( + "parameter 'timeout' of type 'float' " + "is deprecated, please use " + "'timeout=ClientWSTimeout(ws_close=...)'", + DeprecationWarning, + stacklevel=2, + ) + ws_timeout = ClientWSTimeout(ws_close=timeout) + else: + ws_timeout = DEFAULT_WS_CLIENT_TIMEOUT + if receive_timeout is not None: + warnings.warn( + "float parameter 'receive_timeout' " + "is deprecated, please use parameter " + "'timeout=ClientWSTimeout(ws_receive=...)'", + DeprecationWarning, + stacklevel=2, + ) + ws_timeout = attr.evolve(ws_timeout, ws_receive=receive_timeout) + + if headers is None: + real_headers: CIMultiDict[str] = CIMultiDict() + else: + real_headers = CIMultiDict(headers) + + default_headers = { + hdrs.UPGRADE: "websocket", + hdrs.CONNECTION: "Upgrade", + hdrs.SEC_WEBSOCKET_VERSION: "13", + } + + for key, value in default_headers.items(): + real_headers.setdefault(key, value) + + sec_key = base64.b64encode(os.urandom(16)) + real_headers[hdrs.SEC_WEBSOCKET_KEY] = sec_key.decode() + + if protocols: + real_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = ",".join(protocols) + if origin is not None: + real_headers[hdrs.ORIGIN] = origin + if compress: + extstr = ws_ext_gen(compress=compress) + real_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = extstr + + # For the sake of backward compatibility, if user passes in None, convert it to True + if ssl is None: + warnings.warn( + "ssl=None is deprecated, please use ssl=True", + DeprecationWarning, + stacklevel=2, + ) + ssl = True + ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + # send request + resp = await self.request( + method, + url, + params=params, + headers=real_headers, + read_until_eof=False, + auth=auth, + proxy=proxy, + proxy_auth=proxy_auth, + ssl=ssl, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + ) + + try: + # check handshake + if resp.status != 101: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid response status", + status=resp.status, + headers=resp.headers, + ) + + if resp.headers.get(hdrs.UPGRADE, "").lower() != "websocket": + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid upgrade header", + status=resp.status, + headers=resp.headers, + ) + + if resp.headers.get(hdrs.CONNECTION, "").lower() != "upgrade": + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid connection header", + status=resp.status, + headers=resp.headers, + ) + + # key calculation + r_key = resp.headers.get(hdrs.SEC_WEBSOCKET_ACCEPT, "") + match = base64.b64encode(hashlib.sha1(sec_key + WS_KEY).digest()).decode() + if r_key != match: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid challenge response", + status=resp.status, + headers=resp.headers, + ) + + # websocket protocol + protocol = None + if protocols and hdrs.SEC_WEBSOCKET_PROTOCOL in resp.headers: + resp_protocols = [ + proto.strip() + for proto in resp.headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",") + ] + + for proto in resp_protocols: + if proto in protocols: + protocol = proto + break + + # websocket compress + notakeover = False + if compress: + compress_hdrs = resp.headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS) + if compress_hdrs: + try: + compress, notakeover = ws_ext_parse(compress_hdrs) + except WSHandshakeError as exc: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message=exc.args[0], + status=resp.status, + headers=resp.headers, + ) from exc + else: + compress = 0 + notakeover = False + + conn = resp.connection + assert conn is not None + conn_proto = conn.protocol + assert conn_proto is not None + + # For WS connection the read_timeout must be either receive_timeout or greater + # None == no timeout, i.e. infinite timeout, so None is the max timeout possible + if ws_timeout.ws_receive is None: + # Reset regardless + conn_proto.read_timeout = None + elif conn_proto.read_timeout is not None: + conn_proto.read_timeout = max( + ws_timeout.ws_receive, conn_proto.read_timeout + ) + + transport = conn.transport + assert transport is not None + reader = WebSocketDataQueue(conn_proto, DEFAULT_CHUNK_SIZE, loop=self._loop) + writer = WebSocketWriter( + conn_proto, + transport, + use_mask=True, + compress=compress, + notakeover=notakeover, + ) + except BaseException: + resp.close() + raise + else: + ws_resp = self._ws_response_class( + reader, + writer, + protocol, + resp, + ws_timeout, + autoclose, + autoping, + self._loop, + heartbeat=heartbeat, + compress=compress, + client_notakeover=notakeover, + ) + parser = WebSocketReader(reader, max_msg_size, decode_text=decode_text) + cb = None if heartbeat is None else ws_resp._on_data_received + conn_proto.set_parser(parser, reader, data_received_cb=cb) + return ws_resp + + def _prepare_headers(self, headers: LooseHeaders | None) -> "CIMultiDict[str]": + """Add default headers and transform it to CIMultiDict""" + # Convert headers to MultiDict + result = CIMultiDict(self._default_headers) + if headers: + if not isinstance(headers, (MultiDictProxy, MultiDict)): + headers = CIMultiDict(headers) + added_names: set[str] = set() + for key, value in headers.items(): + if key in added_names: + result.add(key, value) + else: + result[key] = value + added_names.add(key) + return result + + def _get_netrc_auth(self, host: str) -> BasicAuth | None: + """ + Get auth from netrc for the given host. + + This method is designed to be called in an executor to avoid + blocking I/O in the event loop. + """ + netrc_obj = netrc_from_env() + try: + return basicauth_from_netrc(netrc_obj, host) + except LookupError: + return None + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def get( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def options( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def head( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def post( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def put( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def patch( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def delete( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + else: + + def get( + self, url: StrOrURL, *, allow_redirects: bool = True, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP GET request.""" + return _RequestContextManager( + self._request( + hdrs.METH_GET, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def options( + self, url: StrOrURL, *, allow_redirects: bool = True, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP OPTIONS request.""" + return _RequestContextManager( + self._request( + hdrs.METH_OPTIONS, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def head( + self, url: StrOrURL, *, allow_redirects: bool = False, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP HEAD request.""" + return _RequestContextManager( + self._request( + hdrs.METH_HEAD, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def post( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP POST request.""" + return _RequestContextManager( + self._request(hdrs.METH_POST, url, data=data, **kwargs) + ) + + def put( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP PUT request.""" + return _RequestContextManager( + self._request(hdrs.METH_PUT, url, data=data, **kwargs) + ) + + def patch( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_PATCH, url, data=data, **kwargs) + ) + + def delete(self, url: StrOrURL, **kwargs: Any) -> "_RequestContextManager": + """Perform HTTP DELETE request.""" + return _RequestContextManager( + self._request(hdrs.METH_DELETE, url, **kwargs) + ) + + async def close(self) -> None: + """Close underlying connector. + + Release all acquired resources. + """ + if not self.closed: + if self._connector is not None and self._connector_owner: + await self._connector.close() + self._connector = None + + @property + def closed(self) -> bool: + """Is client session closed. + + A readonly property. + """ + return self._connector is None or self._connector.closed + + @property + def connector(self) -> BaseConnector | None: + """Connector instance used for the session.""" + return self._connector + + @property + def cookie_jar(self) -> AbstractCookieJar: + """The session cookies.""" + return self._cookie_jar + + @property + def version(self) -> tuple[int, int]: + """The session HTTP protocol version.""" + return self._version + + @property + def requote_redirect_url(self) -> bool: + """Do URL requoting on redirection handling.""" + return self._requote_redirect_url + + @requote_redirect_url.setter + def requote_redirect_url(self, val: bool) -> None: + """Do URL requoting on redirection handling.""" + warnings.warn( + "session.requote_redirect_url modification is deprecated #2778", + DeprecationWarning, + stacklevel=2, + ) + self._requote_redirect_url = val + + @property + def loop(self) -> asyncio.AbstractEventLoop: + """Session's loop.""" + warnings.warn( + "client.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + @property + def timeout(self) -> ClientTimeout: + """Timeout for the session.""" + return self._timeout + + @property + def headers(self) -> "CIMultiDict[str]": + """The default headers of the client session.""" + return self._default_headers + + @property + def skip_auto_headers(self) -> frozenset[istr]: + """Headers for which autogeneration should be skipped""" + return self._skip_auto_headers + + @property + def auth(self) -> BasicAuth | None: + """An object that represents HTTP Basic Authorization""" + return self._default_auth + + @property + def json_serialize(self) -> JSONEncoder: + """Json serializer callable""" + return self._json_serialize + + @property + def connector_owner(self) -> bool: + """Should connector be closed on session closing""" + return self._connector_owner + + @property + def raise_for_status( + self, + ) -> bool | Callable[[ClientResponse], Awaitable[None]]: + """Should `ClientResponse.raise_for_status()` be called for each response.""" + return self._raise_for_status + + @property + def auto_decompress(self) -> bool: + """Should the body response be automatically decompressed.""" + return self._auto_decompress + + @property + def trust_env(self) -> bool: + """ + Should proxies information from environment or netrc be trusted. + + Information is from HTTP_PROXY / HTTPS_PROXY environment variables + or ~/.netrc file if present. + """ + return self._trust_env + + @property + def trace_configs(self) -> list[TraceConfig]: + """A list of TraceConfig instances used for client tracing""" + return self._trace_configs + + def detach(self) -> None: + """Detach connector from session without closing the former. + + Session is switched to closed state anyway. + """ + self._connector = None + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> "ClientSession": + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + await self.close() + + +class _BaseRequestContextManager( + Coroutine[Any, Any, _RetType_co], Generic[_RetType_co] +): + + __slots__ = ("_coro", "_resp") + + def __init__(self, coro: Coroutine[asyncio.Future[Any], None, _RetType_co]) -> None: + self._coro: Coroutine[asyncio.Future[Any], None, _RetType_co] = coro + + def send(self, arg: None) -> asyncio.Future[Any]: + return self._coro.send(arg) + + def throw(self, *args: Any, **kwargs: Any) -> asyncio.Future[Any]: + return self._coro.throw(*args, **kwargs) + + def close(self) -> None: + return self._coro.close() + + def __await__(self) -> Generator[Any, None, _RetType_co]: + ret = self._coro.__await__() + return ret + + def __iter__(self) -> Generator[Any, None, _RetType_co]: + return self.__await__() + + async def __aenter__(self) -> _RetType_co: + self._resp: _RetType_co = await self._coro + return await self._resp.__aenter__() # type: ignore[return-value] + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + await self._resp.__aexit__(exc_type, exc, tb) + + +_RequestContextManager = _BaseRequestContextManager[ClientResponse] +_WSRequestContextManager = _BaseRequestContextManager[ClientWebSocketResponse[bool]] + + +class _SessionRequestContextManager: + + __slots__ = ("_coro", "_resp", "_session") + + def __init__( + self, + coro: Coroutine[asyncio.Future[Any], None, ClientResponse], + session: ClientSession, + ) -> None: + self._coro = coro + self._resp: ClientResponse | None = None + self._session = session + + async def __aenter__(self) -> ClientResponse: + try: + self._resp = await self._coro + except BaseException: + await self._session.close() + raise + else: + return self._resp + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + assert self._resp is not None + self._resp.close() + await self._session.close() + + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + method: str, + url: StrOrURL, + *, + version: HttpVersion = http.HttpVersion11, + connector: BaseConnector | None = None, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Unpack[_RequestOptions], + ) -> _SessionRequestContextManager: ... + +else: + + def request( + method: str, + url: StrOrURL, + *, + version: HttpVersion = http.HttpVersion11, + connector: BaseConnector | None = None, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Any, + ) -> _SessionRequestContextManager: + """Constructs and sends a request. + + Returns response object. + method - HTTP method + url - request url + params - (optional) Dictionary or bytes to be sent in the query + string of the new request + data - (optional) Dictionary, bytes, or file-like object to + send in the body of the request + json - (optional) Any json compatible python object + headers - (optional) Dictionary of HTTP Headers to send with + the request + cookies - (optional) Dict object to send with the request + auth - (optional) BasicAuth named tuple represent HTTP Basic Auth + auth - aiohttp.helpers.BasicAuth + allow_redirects - (optional) If set to False, do not follow + redirects + version - Request HTTP version. + compress - Set to True if request has to be compressed + with deflate encoding. + chunked - Set to chunk size for chunked transfer encoding. + expect100 - Expect 100-continue response from server. + connector - BaseConnector sub-class instance to support + connection pooling. + read_until_eof - Read response until eof if response + does not have Content-Length header. + loop - Optional event loop. + timeout - Optional ClientTimeout settings structure, 5min + total timeout by default. + Usage:: + >>> import aiohttp + >>> async with aiohttp.request('GET', 'http://python.org/') as resp: + ... print(resp) + ... data = await resp.read() + + """ + connector_owner = False + if connector is None: + connector_owner = True + connector = TCPConnector(loop=loop, force_close=True) + + session = ClientSession( + loop=loop, + cookies=kwargs.pop("cookies", None), + version=version, + timeout=kwargs.pop("timeout", sentinel), + connector=connector, + connector_owner=connector_owner, + ) + + return _SessionRequestContextManager( + session._request(method, url, **kwargs), + session, + ) diff --git a/venv/Lib/site-packages/aiohttp/client_exceptions.py b/venv/Lib/site-packages/aiohttp/client_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..260ea7b0efc4ccd10f210eb022f95d9595f11518 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/client_exceptions.py @@ -0,0 +1,426 @@ +"""HTTP related errors.""" + +import asyncio +import warnings +from typing import TYPE_CHECKING, Union + +from multidict import MultiMapping + +from .typedefs import StrOrURL + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = SSLContext = None # type: ignore[assignment] + +if TYPE_CHECKING: + from .client_reqrep import ClientResponse, ConnectionKey, Fingerprint, RequestInfo + from .http_parser import RawResponseMessage +else: + RequestInfo = ClientResponse = ConnectionKey = RawResponseMessage = None + +__all__ = ( + "ClientError", + "ClientConnectionError", + "ClientConnectionResetError", + "ClientOSError", + "ClientConnectorError", + "ClientProxyConnectionError", + "ClientSSLError", + "ClientConnectorDNSError", + "ClientConnectorSSLError", + "ClientConnectorCertificateError", + "ConnectionTimeoutError", + "SocketTimeoutError", + "ServerConnectionError", + "ServerTimeoutError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ClientResponseError", + "ClientHttpProxyError", + "WSServerHandshakeError", + "ContentTypeError", + "ClientPayloadError", + "InvalidURL", + "InvalidUrlClientError", + "RedirectClientError", + "NonHttpUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlRedirectClientError", + "WSMessageTypeError", +) + + +class ClientError(Exception): + """Base class for client connection errors.""" + + +class ClientResponseError(ClientError): + """Base class for exceptions that occur after getting a response. + + request_info: An instance of RequestInfo. + history: A sequence of responses, if redirects occurred. + status: HTTP status code. + message: Error message. + headers: Response headers. + """ + + def __init__( + self, + request_info: RequestInfo, + history: tuple[ClientResponse, ...], + *, + code: int | None = None, + status: int | None = None, + message: str = "", + headers: MultiMapping[str] | None = None, + ) -> None: + self.request_info = request_info + if code is not None: + if status is not None: + raise ValueError( + "Both code and status arguments are provided; " + "code is deprecated, use status instead" + ) + warnings.warn( + "code argument is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + if status is not None: + self.status = status + elif code is not None: + self.status = code + else: + self.status = 0 + self.message = message + self.headers = headers + self.history = history + self.args = (request_info, history) + + def __str__(self) -> str: + return f"{self.status}, message={self.message!r}, url={str(self.request_info.real_url)!r}" + + def __repr__(self) -> str: + args = f"{self.request_info!r}, {self.history!r}" + if self.status != 0: + args += f", status={self.status!r}" + if self.message != "": + args += f", message={self.message!r}" + if self.headers is not None: + args += f", headers={self.headers!r}" + return f"{type(self).__name__}({args})" + + @property + def code(self) -> int: + warnings.warn( + "code property is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + return self.status + + @code.setter + def code(self, value: int) -> None: + warnings.warn( + "code property is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + self.status = value + + +class ContentTypeError(ClientResponseError): + """ContentType found is not valid.""" + + +class WSServerHandshakeError(ClientResponseError): + """websocket server handshake error.""" + + +class ClientHttpProxyError(ClientResponseError): + """HTTP proxy error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + proxy responds with status other than ``200 OK`` + on ``CONNECT`` request. + """ + + +class TooManyRedirects(ClientResponseError): + """Client was redirected too many times.""" + + +class ClientConnectionError(ClientError): + """Base class for client socket errors.""" + + +class ClientConnectionResetError(ClientConnectionError, ConnectionResetError): + """ConnectionResetError""" + + +class ClientOSError(ClientConnectionError, OSError): + """OSError error.""" + + +class ClientConnectorError(ClientOSError): + """Client connector error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + a connection can not be established. + """ + + def __init__(self, connection_key: ConnectionKey, os_error: OSError) -> None: + self._conn_key = connection_key + self._os_error = os_error + super().__init__(os_error.errno, os_error.strerror) + self.args = (connection_key, os_error) + + @property + def os_error(self) -> OSError: + return self._os_error + + @property + def host(self) -> str: + return self._conn_key.host + + @property + def port(self) -> int | None: + return self._conn_key.port + + @property + def ssl(self) -> Union[SSLContext, bool, "Fingerprint"]: + return self._conn_key.ssl + + def __str__(self) -> str: + return "Cannot connect to host {0.host}:{0.port} ssl:{1} [{2}]".format( + self, "default" if self.ssl is True else self.ssl, self.strerror + ) + + # OSError.__reduce__ does too much black magick + __reduce__ = BaseException.__reduce__ + + +class ClientConnectorDNSError(ClientConnectorError): + """DNS resolution failed during client connection. + + Raised in :class:`aiohttp.connector.TCPConnector` if + DNS resolution fails. + """ + + +class ClientProxyConnectionError(ClientConnectorError): + """Proxy connection error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + connection to proxy can not be established. + """ + + +class UnixClientConnectorError(ClientConnectorError): + """Unix connector error. + + Raised in :py:class:`aiohttp.connector.UnixConnector` + if connection to unix socket can not be established. + """ + + def __init__( + self, path: str, connection_key: ConnectionKey, os_error: OSError + ) -> None: + self._path = path + super().__init__(connection_key, os_error) + + @property + def path(self) -> str: + return self._path + + def __str__(self) -> str: + return "Cannot connect to unix socket {0.path} ssl:{1} [{2}]".format( + self, "default" if self.ssl is True else self.ssl, self.strerror + ) + + +class ServerConnectionError(ClientConnectionError): + """Server connection errors.""" + + +class ServerDisconnectedError(ServerConnectionError): + """Server disconnected.""" + + def __init__(self, message: RawResponseMessage | str | None = None) -> None: + if message is None: + message = "Server disconnected" + + self.args = (message,) + self.message = message + + +class ServerTimeoutError(ServerConnectionError, asyncio.TimeoutError): + """Server timeout error.""" + + +class ConnectionTimeoutError(ServerTimeoutError): + """Connection timeout error.""" + + +class SocketTimeoutError(ServerTimeoutError): + """Socket timeout error.""" + + +class ServerFingerprintMismatch(ServerConnectionError): + """SSL certificate does not match expected fingerprint.""" + + def __init__(self, expected: bytes, got: bytes, host: str, port: int) -> None: + self.expected = expected + self.got = got + self.host = host + self.port = port + self.args = (expected, got, host, port) + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} expected={self.expected!r} got={self.got!r} host={self.host!r} port={self.port!r}>" + + +class ClientPayloadError(ClientError): + """Response payload error.""" + + +class InvalidURL(ClientError, ValueError): + """Invalid URL. + + URL used for fetching is malformed, e.g. it doesn't contains host + part. + """ + + # Derive from ValueError for backward compatibility + + def __init__(self, url: StrOrURL, description: str | None = None) -> None: + # The type of url is not yarl.URL because the exception can be raised + # on URL(url) call + self._url = url + self._description = description + + if description: + super().__init__(url, description) + else: + super().__init__(url) + + @property + def url(self) -> StrOrURL: + return self._url + + @property + def description(self) -> "str | None": + return self._description + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self}>" + + def __str__(self) -> str: + if self._description: + return f"{self._url} - {self._description}" + return str(self._url) + + +class InvalidUrlClientError(InvalidURL): + """Invalid URL client error.""" + + +class RedirectClientError(ClientError): + """Client redirect error.""" + + +class NonHttpUrlClientError(ClientError): + """Non http URL client error.""" + + +class InvalidUrlRedirectClientError(InvalidUrlClientError, RedirectClientError): + """Invalid URL redirect client error.""" + + +class NonHttpUrlRedirectClientError(NonHttpUrlClientError, RedirectClientError): + """Non http URL redirect client error.""" + + +class ClientSSLError(ClientConnectorError): + """Base error for ssl.*Errors.""" + + +if ssl is not None: + cert_errors = (ssl.CertificateError,) + cert_errors_bases = ( + ClientSSLError, + ssl.CertificateError, + ) + + ssl_errors = (ssl.SSLError,) + ssl_error_bases = (ClientSSLError, ssl.SSLError) +else: # pragma: no cover + cert_errors = tuple() + cert_errors_bases = ( + ClientSSLError, + ValueError, + ) + + ssl_errors = tuple() + ssl_error_bases = (ClientSSLError,) + + +class ClientConnectorSSLError(*ssl_error_bases): # type: ignore[misc] + """Response ssl error.""" + + +class ClientConnectorCertificateError(*cert_errors_bases): # type: ignore[misc] + """Response certificate error.""" + + _conn_key: ConnectionKey + + def __init__( + # TODO: If we require ssl in future, this can become ssl.CertificateError + self, + connection_key: ConnectionKey, + certificate_error: Exception, + ) -> None: + if isinstance(certificate_error, cert_errors + (OSError,)): + # ssl.CertificateError has errno and strerror, so we should be fine + os_error = certificate_error + else: + os_error = OSError() + + super().__init__(connection_key, os_error) + self._certificate_error = certificate_error + self.args = (connection_key, certificate_error) + + @property + def certificate_error(self) -> Exception: + return self._certificate_error + + @property + def host(self) -> str: + return self._conn_key.host + + @property + def port(self) -> int | None: + return self._conn_key.port + + @property + def ssl(self) -> bool: + return self._conn_key.is_ssl + + def __str__(self) -> str: + return ( + f"Cannot connect to host {self.host}:{self.port} ssl:{self.ssl} " + f"[{self.certificate_error.__class__.__name__}: " + f"{self.certificate_error.args}]" + ) + + +class WSMessageTypeError(TypeError): + """WebSocket message type is not valid.""" diff --git a/venv/Lib/site-packages/aiohttp/client_middleware_digest_auth.py b/venv/Lib/site-packages/aiohttp/client_middleware_digest_auth.py new file mode 100644 index 0000000000000000000000000000000000000000..4020b7abd1e1b19daa93ae068ad597f1533d2952 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/client_middleware_digest_auth.py @@ -0,0 +1,494 @@ +""" +Digest authentication middleware for aiohttp client. + +This middleware implements HTTP Digest Authentication according to RFC 7616, +providing a more secure alternative to Basic Authentication. It supports all +standard hash algorithms including MD5, SHA, SHA-256, SHA-512 and their session +variants, as well as both 'auth' and 'auth-int' quality of protection (qop) options. +""" + +import hashlib +import os +import re +import sys +import time +from collections.abc import Callable +from typing import Final, Literal, TypedDict + +from yarl import URL + +from . import hdrs +from .client_exceptions import ClientError +from .client_middlewares import ClientHandlerType +from .client_reqrep import ClientRequest, ClientResponse +from .payload import Payload + + +class DigestAuthChallenge(TypedDict, total=False): + realm: str + nonce: str + qop: str + algorithm: str + opaque: str + domain: str + stale: str + + +DigestFunctions: dict[str, Callable[[bytes], "hashlib._Hash"]] = { + "MD5": hashlib.md5, + "MD5-SESS": hashlib.md5, + "SHA": hashlib.sha1, + "SHA-SESS": hashlib.sha1, + "SHA256": hashlib.sha256, + "SHA256-SESS": hashlib.sha256, + "SHA-256": hashlib.sha256, + "SHA-256-SESS": hashlib.sha256, + "SHA512": hashlib.sha512, + "SHA512-SESS": hashlib.sha512, + "SHA-512": hashlib.sha512, + "SHA-512-SESS": hashlib.sha512, +} + + +# Compile the regex pattern once at module level for performance +_HEADER_PAIRS_PATTERN = re.compile( + r'(?:^|\s|,\s*)(\w+)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s,]+))' + if sys.version_info < (3, 11) + else r'(?:^|\s|,\s*)((?>\w+))\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s,]+))' + # +------------|--------|--|-|-|--|----|------|----|--||-----|-> Match valid start/sep + # +--------|--|-|-|--|----|------|----|--||-----|-> alphanumeric key (atomic + # | | | | | | | | || | group reduces backtracking) + # +--|-|-|--|----|------|----|--||-----|-> maybe whitespace + # | | | | | | | || | + # +-|-|--|----|------|----|--||-----|-> = (delimiter) + # +-|--|----|------|----|--||-----|-> maybe whitespace + # | | | | | || | + # +--|----|------|----|--||-----|-> group quoted or unquoted + # | | | | || | + # +----|------|----|--||-----|-> if quoted... + # +------|----|--||-----|-> anything but " or \ + # +----|--||-----|-> escaped characters allowed + # +--||-----|-> or can be empty string + # || | + # +|-----|-> if unquoted... + # +-----|-> anything but , or + # +-> at least one char req'd +) + + +# RFC 7616: Challenge parameters to extract +CHALLENGE_FIELDS: Final[ + tuple[ + Literal["realm", "nonce", "qop", "algorithm", "opaque", "domain", "stale"], ... + ] +] = ( + "realm", + "nonce", + "qop", + "algorithm", + "opaque", + "domain", + "stale", +) + +# Supported digest authentication algorithms +# Use a tuple of sorted keys for predictable documentation and error messages +SUPPORTED_ALGORITHMS: Final[tuple[str, ...]] = tuple(sorted(DigestFunctions.keys())) + +# RFC 7616: Fields that require quoting in the Digest auth header +# These fields must be enclosed in double quotes in the Authorization header. +# Algorithm, qop, and nc are never quoted per RFC specifications. +# This frozen set is used by the template-based header construction to +# automatically determine which fields need quotes. +QUOTED_AUTH_FIELDS: Final[frozenset[str]] = frozenset( + {"username", "realm", "nonce", "uri", "response", "opaque", "cnonce"} +) + + +def escape_quotes(value: str) -> str: + """Escape double quotes for HTTP header values.""" + return value.replace('"', '\\"') + + +def unescape_quotes(value: str) -> str: + """Unescape double quotes in HTTP header values.""" + return value.replace('\\"', '"') + + +def parse_header_pairs(header: str) -> dict[str, str]: + """ + Parse key-value pairs from WWW-Authenticate or similar HTTP headers. + + This function handles the complex format of WWW-Authenticate header values, + supporting both quoted and unquoted values, proper handling of commas in + quoted values, and whitespace variations per RFC 7616. + + Examples of supported formats: + - key1="value1", key2=value2 + - key1 = "value1" , key2="value, with, commas" + - key1=value1,key2="value2" + - realm="example.com", nonce="12345", qop="auth" + + Args: + header: The header value string to parse + + Returns: + Dictionary mapping parameter names to their values + """ + return { + stripped_key: unescape_quotes(quoted_val) if quoted_val else unquoted_val + for key, quoted_val, unquoted_val in _HEADER_PAIRS_PATTERN.findall(header) + if (stripped_key := key.strip()) + } + + +class DigestAuthMiddleware: + """ + HTTP digest authentication middleware for aiohttp client. + + This middleware intercepts 401 Unauthorized responses containing a Digest + authentication challenge, calculates the appropriate digest credentials, + and automatically retries the request with the proper Authorization header. + + Features: + - Handles all aspects of Digest authentication handshake automatically + - Supports all standard hash algorithms: + - MD5, MD5-SESS + - SHA, SHA-SESS + - SHA256, SHA256-SESS, SHA-256, SHA-256-SESS + - SHA512, SHA512-SESS, SHA-512, SHA-512-SESS + - Supports 'auth' and 'auth-int' quality of protection modes + - Properly handles quoted strings and parameter parsing + - Includes replay attack protection with client nonce count tracking + - Supports preemptive authentication per RFC 7616 Section 3.6 + + Origin scoping: + The credentials are scoped to the origin of the first request the + middleware handles. A request to a different origin is passed through + untouched, so it never receives a digest response computed from those + credentials, unless that origin falls within a protection space the + anchor origin advertised through the RFC 7616 ``domain`` directive. Make + the first request through the middleware against the intended origin, as + the anchor is pinned to it and not reset for the life of the instance. + + Standards compliance: + - RFC 7616: HTTP Digest Access Authentication (primary reference) + - RFC 2617: HTTP Authentication (deprecated by RFC 7616) + - RFC 1945: Section 11.1 (username restrictions) + + Implementation notes: + The core digest calculation is inspired by the implementation in + https://github.com/requests/requests/blob/v2.18.4/requests/auth.py + with added support for modern digest auth features and error handling. + """ + + def __init__( + self, + login: str, + password: str, + preemptive: bool = True, + ) -> None: + if login is None: + raise ValueError("None is not allowed as login value") + + if password is None: + raise ValueError("None is not allowed as password value") + + if ":" in login: + raise ValueError('A ":" is not allowed in username (RFC 1945#section-11.1)') + + self._login_str: Final[str] = login + self._login_bytes: Final[bytes] = login.encode("utf-8") + self._password_bytes: Final[bytes] = password.encode("utf-8") + + self._last_nonce_bytes = b"" + self._nonce_count = 0 + self._challenge: DigestAuthChallenge = {} + self._preemptive: bool = preemptive + # Set of URLs defining the protection space + self._protection_space: list[str] = [] + # Origin the credentials are scoped to; set on the first request. + self._origin: URL | None = None + + async def _encode(self, method: str, url: URL, body: Payload | Literal[b""]) -> str: + """ + Build digest authorization header for the current challenge. + + Args: + method: The HTTP method (GET, POST, etc.) + url: The request URL + body: The request body (used for qop=auth-int) + + Returns: + A fully formatted Digest authorization header string + + Raises: + ClientError: If the challenge is missing required parameters or + contains unsupported values + + """ + challenge = self._challenge + if "realm" not in challenge: + raise ClientError( + "Malformed Digest auth challenge: Missing 'realm' parameter" + ) + + if "nonce" not in challenge: + raise ClientError( + "Malformed Digest auth challenge: Missing 'nonce' parameter" + ) + + # Empty realm values are allowed per RFC 7616 (SHOULD, not MUST, contain host name) + realm = challenge["realm"] + nonce = challenge["nonce"] + + # Empty nonce values are not allowed as they are security-critical for replay protection + if not nonce: + raise ClientError( + "Security issue: Digest auth challenge contains empty 'nonce' value" + ) + + qop_raw = challenge.get("qop", "") + # Preserve original algorithm case for response while using uppercase for processing + algorithm_original = challenge.get("algorithm", "MD5") + algorithm = algorithm_original.upper() + opaque = challenge.get("opaque", "") + + # Convert string values to bytes once + nonce_bytes = nonce.encode("utf-8") + realm_bytes = realm.encode("utf-8") + # Use the encoded request-target (raw_path_qs) since that is what is + # transmitted on the wire and what the server signs against. Using the + # decoded form would cause digest verification to fail when the path + # or query string contains percent-encoded reserved characters. + path = URL(url).raw_path_qs + + # Process QoP + qop = "" + qop_bytes = b"" + if qop_raw: + valid_qops = {"auth", "auth-int"}.intersection( + {q.strip() for q in qop_raw.split(",") if q.strip()} + ) + if not valid_qops: + raise ClientError( + f"Digest auth error: Unsupported Quality of Protection (qop) value(s): {qop_raw}" + ) + + qop = "auth-int" if "auth-int" in valid_qops else "auth" + qop_bytes = qop.encode("utf-8") + + if algorithm not in DigestFunctions: + raise ClientError( + f"Digest auth error: Unsupported hash algorithm: {algorithm}. " + f"Supported algorithms: {', '.join(SUPPORTED_ALGORITHMS)}" + ) + hash_fn: Final = DigestFunctions[algorithm] + + def H(x: bytes) -> bytes: + """RFC 7616 Section 3: Hash function H(data) = hex(hash(data)).""" + return hash_fn(x).hexdigest().encode() + + def KD(s: bytes, d: bytes) -> bytes: + """RFC 7616 Section 3: KD(secret, data) = H(concat(secret, ":", data)).""" + return H(b":".join((s, d))) + + # Calculate A1 and A2 + A1 = b":".join((self._login_bytes, realm_bytes, self._password_bytes)) + A2 = f"{method.upper()}:{path}".encode() + if qop == "auth-int": + if isinstance(body, Payload): # will always be empty bytes unless Payload + entity_bytes = await body.as_bytes() # Get bytes from Payload + else: + entity_bytes = body + entity_hash = H(entity_bytes) + A2 = b":".join((A2, entity_hash)) + + HA1 = H(A1) + HA2 = H(A2) + + # Nonce count handling + if nonce_bytes == self._last_nonce_bytes: + self._nonce_count += 1 + else: + self._nonce_count = 1 + + self._last_nonce_bytes = nonce_bytes + ncvalue = f"{self._nonce_count:08x}" + ncvalue_bytes = ncvalue.encode("utf-8") + + # Generate client nonce + cnonce = hashlib.sha1( + b"".join( + [ + str(self._nonce_count).encode("utf-8"), + nonce_bytes, + time.ctime().encode("utf-8"), + os.urandom(8), + ] + ) + ).hexdigest()[:16] + cnonce_bytes = cnonce.encode("utf-8") + + # Special handling for session-based algorithms + if algorithm.upper().endswith("-SESS"): + HA1 = H(b":".join((HA1, nonce_bytes, cnonce_bytes))) + + # Calculate the response digest + if qop: + noncebit = b":".join( + (nonce_bytes, ncvalue_bytes, cnonce_bytes, qop_bytes, HA2) + ) + response_digest = KD(HA1, noncebit) + else: + response_digest = KD(HA1, b":".join((nonce_bytes, HA2))) + + # Define a dict mapping of header fields to their values + # Group fields into always-present, optional, and qop-dependent + header_fields = { + # Always present fields + "username": escape_quotes(self._login_str), + "realm": escape_quotes(realm), + "nonce": escape_quotes(nonce), + "uri": path, + "response": response_digest.decode(), + "algorithm": algorithm_original, + } + + # Optional fields + if opaque: + header_fields["opaque"] = escape_quotes(opaque) + + # QoP-dependent fields + if qop: + header_fields["qop"] = qop + header_fields["nc"] = ncvalue + header_fields["cnonce"] = cnonce + + # Build header using templates for each field type + pairs: list[str] = [] + for field, value in header_fields.items(): + if field in QUOTED_AUTH_FIELDS: + pairs.append(f'{field}="{value}"') + else: + pairs.append(f"{field}={value}") + + return f"Digest {', '.join(pairs)}" + + def _in_protection_space(self, url: URL) -> bool: + """ + Check if the given URL is within the current protection space. + + According to RFC 7616, a URI is in the protection space if any URI + in the protection space is a prefix of it (after both have been made absolute). + """ + request_str = str(url) + for space_str in self._protection_space: + # Check if request starts with space URL + if not request_str.startswith(space_str): + continue + # Exact match or space ends with / (proper directory prefix) + if len(request_str) == len(space_str) or space_str[-1] == "/": + return True + # Check next char is / to ensure proper path boundary + if request_str[len(space_str)] == "/": + return True + return False + + def _authenticate(self, response: ClientResponse) -> bool: + """ + Takes the given response and tries digest-auth, if needed. + + Returns true if the original request must be resent. + """ + if response.status != 401: + return False + + auth_header = response.headers.get("www-authenticate", "") + if not auth_header: + return False # No authentication header present + + method, sep, headers = auth_header.partition(" ") + if not sep: + # No space found in www-authenticate header + return False # Malformed auth header, missing scheme separator + + if method.lower() != "digest": + # Not a digest auth challenge (could be Basic, Bearer, etc.) + return False + + if not headers: + # We have a digest scheme but no parameters + return False # Malformed digest header, missing parameters + + # We have a digest auth header with content + if not (header_pairs := parse_header_pairs(headers)): + # Failed to parse any key-value pairs + return False # Malformed digest header, no valid parameters + + # Extract challenge parameters + self._challenge = {} + for field in CHALLENGE_FIELDS: + if (value := header_pairs.get(field)) is not None: + self._challenge[field] = value + + # Update protection space based on domain parameter or default to origin + origin = response.url.origin() + + if domain := self._challenge.get("domain"): + # Parse space-separated list of URIs + self._protection_space = [] + for uri in domain.split(): + # Remove quotes if present + uri = uri.strip('"') + if uri.startswith("/"): + # Path-absolute, relative to origin + self._protection_space.append(str(origin.join(URL(uri)))) + else: + # Absolute URI + self._protection_space.append(str(URL(uri))) + else: + # No domain specified, protection space is entire origin + self._protection_space = [str(origin)] + + # Return True only if we found at least one challenge parameter + return bool(self._challenge) + + async def __call__( + self, request: ClientRequest, handler: ClientHandlerType + ) -> ClientResponse: + """Run the digest auth middleware.""" + # Credentials are scoped to the first request's origin. Other origins + # pass through untouched unless a challenge from the anchor origin + # advertised them via RFC 7616 domain; mirrors aiohttp stripping + # Authorization on cross-origin redirects. + origin = request.url.origin() + if self._origin is None: + self._origin = origin + elif origin != self._origin and not self._in_protection_space(request.url): + return await handler(request) + + response = None + for retry_count in range(2): + # Apply authorization header if: + # 1. This is a retry after 401 (retry_count > 0), OR + # 2. Preemptive auth is enabled AND we have a challenge AND the URL is in protection space + if retry_count > 0 or ( + self._preemptive + and self._challenge + and self._in_protection_space(request.url) + ): + request.headers[hdrs.AUTHORIZATION] = await self._encode( + request.method, request.url, request.body + ) + + # Send the request + response = await handler(request) + + # Check if we need to authenticate + if not self._authenticate(response): + break + + # At this point, response is guaranteed to be defined + assert response is not None + return response diff --git a/venv/Lib/site-packages/aiohttp/client_middlewares.py b/venv/Lib/site-packages/aiohttp/client_middlewares.py new file mode 100644 index 0000000000000000000000000000000000000000..fd06568f62db8d36dda84f7a1c3271833e1a73bc --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/client_middlewares.py @@ -0,0 +1,55 @@ +"""Client middleware support.""" + +from collections.abc import Awaitable, Callable, Sequence + +from .client_reqrep import ClientRequest, ClientResponse + +__all__ = ("ClientMiddlewareType", "ClientHandlerType", "build_client_middlewares") + +# Type alias for client request handlers - functions that process requests and return responses +ClientHandlerType = Callable[[ClientRequest], Awaitable[ClientResponse]] + +# Type for client middleware - similar to server but uses ClientRequest/ClientResponse +ClientMiddlewareType = Callable[ + [ClientRequest, ClientHandlerType], Awaitable[ClientResponse] +] + + +def build_client_middlewares( + handler: ClientHandlerType, + middlewares: Sequence[ClientMiddlewareType], +) -> ClientHandlerType: + """ + Apply middlewares to request handler. + + The middlewares are applied in reverse order, so the first middleware + in the list wraps all subsequent middlewares and the handler. + + This implementation avoids using partial/update_wrapper to minimize overhead + and doesn't cache to avoid holding references to stateful middleware. + """ + # Optimize for single middleware case + if len(middlewares) == 1: + middleware = middlewares[0] + + async def single_middleware_handler(req: ClientRequest) -> ClientResponse: + return await middleware(req, handler) + + return single_middleware_handler + + # Build the chain for multiple middlewares + current_handler = handler + + for middleware in reversed(middlewares): + # Create a new closure that captures the current state + def make_wrapper( + mw: ClientMiddlewareType, next_h: ClientHandlerType + ) -> ClientHandlerType: + async def wrapped(req: ClientRequest) -> ClientResponse: + return await mw(req, next_h) + + return wrapped + + current_handler = make_wrapper(middleware, current_handler) + + return current_handler diff --git a/venv/Lib/site-packages/aiohttp/client_proto.py b/venv/Lib/site-packages/aiohttp/client_proto.py new file mode 100644 index 0000000000000000000000000000000000000000..836f0bf8d404bcb6452578395f5d0ca5bddf16c8 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/client_proto.py @@ -0,0 +1,370 @@ +import asyncio +from contextlib import suppress +from typing import Any, Callable + +from .base_protocol import BaseProtocol +from .client_exceptions import ( + ClientConnectionError, + ClientOSError, + ClientPayloadError, + ServerDisconnectedError, + SocketTimeoutError, +) +from .helpers import ( + _EXC_SENTINEL, + DEFAULT_CHUNK_SIZE, + EMPTY_BODY_STATUS_CODES, + BaseTimerContext, + set_exception, + set_result, +) +from .http import HttpResponseParser, RawResponseMessage +from .http_exceptions import HttpProcessingError +from .streams import EMPTY_PAYLOAD, DataQueue, StreamReader + + +class ResponseHandler(BaseProtocol, DataQueue[tuple[RawResponseMessage, StreamReader]]): + """Helper class to adapt between Protocol and StreamReader.""" + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + BaseProtocol.__init__(self, loop=loop, parser=None) + DataQueue.__init__(self, loop) + + self._should_close = False + + self._payload: StreamReader | None = None + self._skip_payload = False + self._payload_parser = None + self._data_received_cb: Callable[[], None] | None = None + + self._timer = None + self._tail = b"" + + self._read_timeout: float | None = None + self._read_timeout_handle: asyncio.TimerHandle | None = None + + self._timeout_ceil_threshold: float | None = 5 + + self._closed: None | asyncio.Future[None] = None + self._connection_lost_called = False + + @property + def closed(self) -> None | asyncio.Future[None]: + """Future that is set when the connection is closed. + + This property returns a Future that will be completed when the connection + is closed. The Future is created lazily on first access to avoid creating + futures that will never be awaited. + + Returns: + - A Future[None] if the connection is still open or was closed after + this property was accessed + - None if connection_lost() was already called before this property + was ever accessed (indicating no one is waiting for the closure) + """ + if self._closed is None and not self._connection_lost_called: + self._closed = self._loop.create_future() + return self._closed + + @property + def upgraded(self) -> bool: + return self._upgraded + + @property + def should_close(self) -> bool: + return bool( + self._should_close + or (self._payload is not None and not self._payload.is_eof()) + or self._upgraded + or self._exception is not None + or self._payload_parser is not None + or self._buffer + or self._tail + ) + + def force_close(self) -> None: + self._should_close = True + + def close(self) -> None: + self._exception = None # Break cyclic references + transport = self.transport + if transport is not None: + transport.close() + self.transport = None + self._payload = None + self._drop_timeout() + + def abort(self) -> None: + self._exception = None # Break cyclic references + transport = self.transport + if transport is not None: + transport.abort() + self.transport = None + self._payload = None + self._drop_timeout() + + def is_connected(self) -> bool: + return self.transport is not None and not self.transport.is_closing() + + def connection_lost(self, exc: BaseException | None) -> None: + self._connection_lost_called = True + self._drop_timeout() + + original_connection_error = exc + reraised_exc = original_connection_error + + connection_closed_cleanly = original_connection_error is None + + if self._closed is not None: + # If someone is waiting for the closed future, + # we should set it to None or an exception. If + # self._closed is None, it means that + # connection_lost() was called already + # or nobody is waiting for it. + if connection_closed_cleanly: + set_result(self._closed, None) + else: + assert original_connection_error is not None + set_exception( + self._closed, + ClientConnectionError( + f"Connection lost: {original_connection_error !s}", + ), + original_connection_error, + ) + + if self._payload_parser is not None: + with suppress(Exception): # FIXME: log this somehow? + self._payload_parser.feed_eof() + + uncompleted = None + if self._parser is not None: + try: + uncompleted = self._parser.feed_eof() + except Exception as underlying_exc: + if self._payload is not None: + client_payload_exc_msg = ( + f"Response payload is not completed: {underlying_exc !r}" + ) + if not connection_closed_cleanly: + client_payload_exc_msg = ( + f"{client_payload_exc_msg !s}. " + f"{original_connection_error !r}" + ) + set_exception( + self._payload, + ClientPayloadError(client_payload_exc_msg), + underlying_exc, + ) + + if not self.is_eof(): + if isinstance(original_connection_error, OSError): + reraised_exc = ClientOSError(*original_connection_error.args) + if connection_closed_cleanly: + reraised_exc = ServerDisconnectedError(uncompleted) + # assigns self._should_close to True as side effect, + # we do it anyway below + underlying_non_eof_exc = ( + _EXC_SENTINEL + if connection_closed_cleanly + else original_connection_error + ) + assert underlying_non_eof_exc is not None + assert reraised_exc is not None + self.set_exception(reraised_exc, underlying_non_eof_exc) + + self._should_close = True + self._parser = None + self._payload = None + self._payload_parser = None + self._reading_paused = False + + super().connection_lost(reraised_exc) + + def eof_received(self) -> None: + # should call parser.feed_eof() most likely + self._drop_timeout() + + def pause_reading(self) -> None: + super().pause_reading() + self._drop_timeout() + + def resume_reading(self, resume_parser: bool = True) -> None: + super().resume_reading(resume_parser) + self._reschedule_timeout() + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._should_close = True + self._drop_timeout() + super().set_exception(exc, exc_cause) + + def set_parser( + self, + parser: Any, + payload: Any, + data_received_cb: Callable[[], None] | None = None, + ) -> None: + # TODO: actual types are: + # parser: WebSocketReader + # payload: WebSocketDataQueue + # but they are not generi enough + # Need an ABC for both types + self._payload = payload + self._payload_parser = parser + self._data_received_cb = data_received_cb + + self._drop_timeout() + + if self._tail: + data, self._tail = self._tail, b"" + self.data_received(data) + + def set_response_params( + self, + *, + timer: BaseTimerContext | None = None, + skip_payload: bool = False, + read_until_eof: bool = False, + auto_decompress: bool = True, + read_timeout: float | None = None, + read_bufsize: int = DEFAULT_CHUNK_SIZE, + timeout_ceil_threshold: float = 5, + max_line_size: int = 8190, + max_field_size: int = 8190, + max_headers: int = 128, + ) -> None: + self._skip_payload = skip_payload + + self._read_timeout = read_timeout + + self._timeout_ceil_threshold = timeout_ceil_threshold + + self._parser = HttpResponseParser( + self, + self._loop, + read_bufsize, + timer=timer, + payload_exception=ClientPayloadError, + response_with_body=not skip_payload, + read_until_eof=read_until_eof, + auto_decompress=auto_decompress, + max_line_size=max_line_size, + max_field_size=max_field_size, + max_headers=max_headers, + ) + + if self._tail: + data, self._tail = self._tail, b"" + self.data_received(data) + + def _drop_timeout(self) -> None: + if self._read_timeout_handle is not None: + self._read_timeout_handle.cancel() + self._read_timeout_handle = None + + def _reschedule_timeout(self) -> None: + timeout = self._read_timeout + if self._read_timeout_handle is not None: + self._read_timeout_handle.cancel() + + if timeout: + self._read_timeout_handle = self._loop.call_later( + timeout, self._on_read_timeout + ) + else: + self._read_timeout_handle = None + + def start_timeout(self) -> None: + self._reschedule_timeout() + + @property + def read_timeout(self) -> float | None: + return self._read_timeout + + @read_timeout.setter + def read_timeout(self, read_timeout: float | None) -> None: + self._read_timeout = read_timeout + + def _on_read_timeout(self) -> None: + exc = SocketTimeoutError("Timeout on reading data from socket") + self.set_exception(exc) + if self._payload is not None: + set_exception(self._payload, exc) + + def data_received(self, data: bytes) -> None: + # If no data, then we are resuming decompression. We haven't received + # data from the socket, so we can avoid the reschedule overhead. + if data: + self._reschedule_timeout() + + # custom payload parser - currently always WebSocketReader + if self._payload_parser is not None: + if self._data_received_cb is not None: + self._data_received_cb() + eof, tail = self._payload_parser.feed_data(data) + if eof: + self._payload = None + self._payload_parser = None + + if tail: + self.data_received(tail) + return + + if self._upgraded or self._parser is None: + # i.e. websocket connection, websocket parser is not set yet + self._tail += data + return + + # parse http messages + try: + messages, upgraded, tail = self._parser.feed_data(data) + except BaseException as underlying_exc: + if self.transport is not None: + # connection.release() could be called BEFORE + # data_received(), the transport is already + # closed in this case + self.transport.close() + if not isinstance(underlying_exc, Exception): + raise + # should_close is True after the call + if isinstance(underlying_exc, HttpProcessingError): + exc = HttpProcessingError( + code=underlying_exc.code, + message=underlying_exc.message, + headers=underlying_exc.headers, + ) + else: + exc = HttpProcessingError() + self.set_exception(exc, underlying_exc) + return + + self._upgraded = upgraded + + payload: StreamReader | None = None + for message, payload in messages: + if message.should_close: + self._should_close = True + + self._payload = payload + + if self._skip_payload or message.code in EMPTY_BODY_STATUS_CODES: + self.feed_data((message, EMPTY_PAYLOAD), 0) + else: + self.feed_data((message, payload), 0) + + if payload is not None: + # new message(s) was processed + # register timeout handler unsubscribing + # either on end-of-stream or immediately for + # EMPTY_PAYLOAD + if payload is not EMPTY_PAYLOAD: + payload.on_eof(self._drop_timeout) + else: + self._drop_timeout() + + if upgraded and tail: + self.data_received(tail) diff --git a/venv/Lib/site-packages/aiohttp/client_reqrep.py b/venv/Lib/site-packages/aiohttp/client_reqrep.py new file mode 100644 index 0000000000000000000000000000000000000000..b0b741e749511ca59cb2234212d66acf723658d4 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/client_reqrep.py @@ -0,0 +1,1557 @@ +import asyncio +import codecs +import contextlib +import functools +import io +import re +import sys +import traceback +import warnings +from collections.abc import Callable, Iterable, Mapping +from hashlib import md5, sha1, sha256 +from http.cookies import Morsel, SimpleCookie +from types import MappingProxyType, TracebackType +from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Optional, Union + +import attr +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy +from yarl import URL + +from . import hdrs, helpers, http, multipart, payload +from ._cookie_helpers import ( + parse_cookie_header, + parse_set_cookie_headers, + preserve_morsel_with_coded_value, +) +from .abc import AbstractStreamWriter +from .client_exceptions import ( + ClientConnectionError, + ClientOSError, + ClientResponseError, + ContentTypeError, + InvalidURL, + ServerFingerprintMismatch, +) +from .compression_utils import HAS_BROTLI, HAS_ZSTD +from .formdata import FormData +from .helpers import ( + _SENTINEL, + BaseTimerContext, + BasicAuth, + HeadersMixin, + TimerNoop, + _basic_auth_no_warn, + noop, + reify, + sentinel, + set_exception, + set_result, +) +from .http import ( + SERVER_SOFTWARE, + HttpVersion, + HttpVersion10, + HttpVersion11, + StreamWriter, +) +from .streams import StreamReader +from .typedefs import ( + DEFAULT_JSON_DECODER, + JSONDecoder, + LooseCookies, + LooseHeaders, + Query, + RawHeaders, +) + +if TYPE_CHECKING: + import ssl + from ssl import SSLContext +else: + try: + import ssl + from ssl import SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + + +__all__ = ("ClientRequest", "ClientResponse", "RequestInfo", "Fingerprint") + + +if TYPE_CHECKING: + from .client import ClientSession + from .connector import Connection + from .tracing import Trace + + +_CONNECTION_CLOSED_EXCEPTION = ClientConnectionError("Connection closed") +_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") +json_re = re.compile(r"^application/(?:[\w.+-]+?\+)?json") +_DIGITS_RE = re.compile(r"\d+", re.ASCII) + + +def _gen_default_accept_encoding() -> str: + encodings = [ + "gzip", + "deflate", + ] + if HAS_BROTLI: + encodings.append("br") + if HAS_ZSTD: + encodings.append("zstd") + return ", ".join(encodings) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ContentDisposition: + type: str | None + parameters: "MappingProxyType[str, str]" + filename: str | None + + +class _RequestInfo(NamedTuple): + url: URL + method: str + headers: "CIMultiDictProxy[str]" + real_url: URL + + +class RequestInfo(_RequestInfo): + + def __new__( + cls, + url: URL, + method: str, + headers: "CIMultiDictProxy[str]", + real_url: URL | _SENTINEL = sentinel, + ) -> "RequestInfo": + """Create a new RequestInfo instance. + + For backwards compatibility, the real_url parameter is optional. + """ + return tuple.__new__( + cls, (url, method, headers, url if real_url is sentinel else real_url) + ) + + +class Fingerprint: + HASHFUNC_BY_DIGESTLEN = { + 16: md5, + 20: sha1, + 32: sha256, + } + + def __init__(self, fingerprint: bytes) -> None: + digestlen = len(fingerprint) + hashfunc = self.HASHFUNC_BY_DIGESTLEN.get(digestlen) + if not hashfunc: + raise ValueError("fingerprint has invalid length") + elif hashfunc is md5 or hashfunc is sha1: + raise ValueError("md5 and sha1 are insecure and not supported. Use sha256.") + self._hashfunc = hashfunc + self._fingerprint = fingerprint + + @property + def fingerprint(self) -> bytes: + return self._fingerprint + + def check(self, transport: asyncio.Transport) -> None: + if not transport.get_extra_info("sslcontext"): + return + sslobj = transport.get_extra_info("ssl_object") + cert = sslobj.getpeercert(binary_form=True) + got = self._hashfunc(cert).digest() + if got != self._fingerprint: + host, port, *_ = transport.get_extra_info("peername") + raise ServerFingerprintMismatch(self._fingerprint, got, host, port) + + +if ssl is not None: + SSL_ALLOWED_TYPES = (ssl.SSLContext, bool, Fingerprint, type(None)) +else: # pragma: no cover + SSL_ALLOWED_TYPES = (bool, type(None)) + + +def _merge_ssl_params( + ssl: Union["SSLContext", bool, Fingerprint], + verify_ssl: bool | None, + ssl_context: Optional["SSLContext"], + fingerprint: bytes | None, +) -> Union["SSLContext", bool, Fingerprint]: + if ssl is None: + ssl = True # Double check for backwards compatibility + if verify_ssl is not None and not verify_ssl: + warnings.warn( + "verify_ssl is deprecated, use ssl=False instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = False + if ssl_context is not None: + warnings.warn( + "ssl_context is deprecated, use ssl=context instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = ssl_context + if fingerprint is not None: + warnings.warn( + "fingerprint is deprecated, use ssl=Fingerprint(fingerprint) instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = Fingerprint(fingerprint) + if not isinstance(ssl, SSL_ALLOWED_TYPES): + raise TypeError( + "ssl should be SSLContext, bool, Fingerprint or None, " + f"got {ssl!r} instead." + ) + return ssl + + +_SSL_SCHEMES = frozenset(("https", "wss")) + + +# ConnectionKey is a NamedTuple because it is used as a key in a dict +# and a set in the connector. Since a NamedTuple is a tuple it uses +# the fast native tuple __hash__ and __eq__ implementation in CPython. +class ConnectionKey(NamedTuple): + # the key should contain an information about used proxy / TLS + # to prevent reusing wrong connections from a pool + host: str + port: int | None + is_ssl: bool + ssl: SSLContext | bool | Fingerprint + proxy: URL | None + proxy_auth: BasicAuth | None + proxy_headers_hash: int | None # hash(CIMultiDict) + server_hostname: str | None = None + + +def _is_expected_content_type( + response_content_type: str, expected_content_type: str +) -> bool: + if expected_content_type == "application/json": + return json_re.match(response_content_type) is not None + return expected_content_type in response_content_type + + +def _warn_if_unclosed_payload(payload: payload.Payload, stacklevel: int = 2) -> None: + """Warn if the payload is not closed. + + Callers must check that the body is a Payload before calling this method. + + Args: + payload: The payload to check + stacklevel: Stack level for the warning (default 2 for direct callers) + """ + if not payload.autoclose and not payload.consumed: + warnings.warn( + "The previous request body contains unclosed resources. " + "Use await request.update_body() instead of setting request.body " + "directly to properly close resources and avoid leaks.", + ResourceWarning, + stacklevel=stacklevel, + ) + + +class ClientResponse(HeadersMixin): + + # Some of these attributes are None when created, + # but will be set by the start() method. + # As the end user will likely never see the None values, we cheat the types below. + # from the Status-Line of the response + version: HttpVersion | None = None # HTTP-Version + status: int = None # type: ignore[assignment] # Status-Code + reason: str | None = None # Reason-Phrase + + content: StreamReader = None # type: ignore[assignment] # Payload stream + _body: bytes | None = None + _headers: CIMultiDictProxy[str] = None # type: ignore[assignment] + _history: tuple["ClientResponse", ...] = () + _raw_headers: RawHeaders = None # type: ignore[assignment] + + _connection: Optional["Connection"] = None # current connection + _cookies: SimpleCookie | None = None + _raw_cookie_headers: tuple[str, ...] | None = None + _continue: Optional["asyncio.Future[bool]"] = None + _source_traceback: traceback.StackSummary | None = None + _session: Optional["ClientSession"] = None + # set up by ClientRequest after ClientResponse object creation + # post-init stage allows to not change ctor signature + _closed = True # to allow __del__ for non-initialized properly response + _released = False + _in_context = False + + _resolve_charset: Callable[["ClientResponse", bytes], str] = lambda *_: "utf-8" + + __writer: Optional["asyncio.Task[None]"] = None + _stream_writer: Optional[AbstractStreamWriter] = None + _output_size: int = 0 + _upload_complete: Optional[asyncio.Future[None]] = None + + def __init__( + self, + method: str, + url: URL, + *, + writer: "asyncio.Task[None] | None", + continue100: Optional["asyncio.Future[bool]"], + timer: BaseTimerContext, + request_info: RequestInfo, + traces: list["Trace"], + loop: asyncio.AbstractEventLoop, + session: "ClientSession", + stream_writer: AbstractStreamWriter, + ) -> None: + # URL forbids subclasses, so a simple type check is enough. + assert type(url) is URL + + self.method = method + + self._real_url = url + self._url = url.with_fragment(None) if url.raw_fragment else url + if writer is None: # Request already sent + self._output_size = stream_writer.output_size + else: + self._stream_writer = stream_writer + self._writer = writer + if continue100 is not None: + self._continue = continue100 + self._request_info = request_info + self._timer = timer if timer is not None else TimerNoop() + self._cache: dict[str, Any] = {} + self._traces = traces + self._loop = loop + # Save reference to _resolve_charset, so that get_encoding() will still + # work after the response has finished reading the body. + # TODO: Fix session=None in tests (see ClientRequest.__init__). + if session is not None: + # store a reference to session #1985 + self._session = session + self._resolve_charset = session._resolve_charset + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + def __reset_writer(self, _: object = None) -> None: + self.__writer = None + if self._stream_writer is not None: + self._output_size = self._stream_writer.output_size + self._stream_writer = None + if self._upload_complete is not None and not self._upload_complete.done(): + self._upload_complete.set_result(None) + + @property + def _writer(self) -> Optional["asyncio.Task[None]"]: + """The writer task for streaming data. + + _writer is only provided for backwards compatibility + for subclasses that may need to access it. + """ + return self.__writer + + @_writer.setter + def _writer(self, writer: Optional["asyncio.Task[None]"]) -> None: + """Set the writer task for streaming data.""" + if self.__writer is not None: + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = writer + if writer is None: + return + if writer.done(): + # The writer is already done, so we can clear it immediately. + self.__reset_writer() + else: + writer.add_done_callback(self.__reset_writer) + + @property + def output_size(self) -> int: + """Number of bytes sent for this request.""" + if self._stream_writer is not None: + return self._stream_writer.output_size + return self._output_size + + @property + def upload_complete(self) -> "asyncio.Future[None]": + """Future set when the request body has been fully sent. + + Already done when the request had no body or was written eagerly. + """ + if self._upload_complete is None: + self._upload_complete = self._loop.create_future() + if self._stream_writer is None: # upload already finished + self._upload_complete.set_result(None) + return self._upload_complete + + @property + def cookies(self) -> SimpleCookie: + if self._cookies is None: + if self._raw_cookie_headers is not None: + # Parse cookies for response.cookies (SimpleCookie for backward compatibility) + cookies = SimpleCookie() + # Use parse_set_cookie_headers for more lenient parsing that handles + # malformed cookies better than SimpleCookie.load + cookies.update(parse_set_cookie_headers(self._raw_cookie_headers)) + self._cookies = cookies + else: + self._cookies = SimpleCookie() + return self._cookies + + @cookies.setter + def cookies(self, cookies: SimpleCookie) -> None: + self._cookies = cookies + # Generate raw cookie headers from the SimpleCookie + if cookies: + self._raw_cookie_headers = tuple( + morsel.OutputString() for morsel in cookies.values() + ) + else: + self._raw_cookie_headers = None + + @reify + def url(self) -> URL: + return self._url + + @reify + def url_obj(self) -> URL: + warnings.warn("Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) + return self._url + + @reify + def real_url(self) -> URL: + return self._real_url + + @reify + def host(self) -> str: + assert self._url.host is not None + return self._url.host + + @reify + def headers(self) -> "CIMultiDictProxy[str]": + return self._headers + + @reify + def raw_headers(self) -> RawHeaders: + return self._raw_headers + + @reify + def request_info(self) -> RequestInfo: + return self._request_info + + @reify + def content_disposition(self) -> ContentDisposition | None: + raw = self._headers.get(hdrs.CONTENT_DISPOSITION) + if raw is None: + return None + disposition_type, params_dct = multipart.parse_content_disposition(raw) + params = MappingProxyType(params_dct) + filename = multipart.content_disposition_filename(params) + return ContentDisposition(disposition_type, params, filename) + + def __del__(self, _warnings: Any = warnings) -> None: + if self._closed: + return + + if self._connection is not None: + self._connection.release() + self._cleanup_writer() + + if self._loop.get_debug(): + kwargs = {"source": self} + _warnings.warn(f"Unclosed response {self!r}", ResourceWarning, **kwargs) + context = {"client_response": self, "message": "Unclosed response"} + if self._source_traceback: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __repr__(self) -> str: + out = io.StringIO() + ascii_encodable_url = str(self.url) + if self.reason: + ascii_encodable_reason = self.reason.encode( + "ascii", "backslashreplace" + ).decode("ascii") + else: + ascii_encodable_reason = "None" + print( + f"", + file=out, + ) + print(self.headers, file=out) + return out.getvalue() + + @property + def connection(self) -> Optional["Connection"]: + return self._connection + + @reify + def history(self) -> tuple["ClientResponse", ...]: + """A sequence of of responses, if redirects occurred.""" + return self._history + + @reify + def links(self) -> "MultiDictProxy[MultiDictProxy[str | URL]]": + links_str = ", ".join(self.headers.getall("link", [])) + + if not links_str: + return MultiDictProxy(MultiDict()) + + links: MultiDict[MultiDictProxy[str | URL]] = MultiDict() + + for val in re.split(r",(?=\s*<)", links_str): + match = re.match(r"\s*<(.*)>(.*)", val) + if match is None: # pragma: no cover + # the check exists to suppress mypy error + continue + url, params_str = match.groups() + params = params_str.split(";")[1:] + + link: MultiDict[str | URL] = MultiDict() + + for param in params: + match = re.match(r"^\s*(\S*)\s*=\s*(['\"]?)(.*?)(\2)\s*$", param, re.M) + if match is None: # pragma: no cover + # the check exists to suppress mypy error + continue + key, _, value, _ = match.groups() + + link.add(key, value) + + key = link.get("rel", url) + + link.add("url", self.url.join(URL(url))) + + links.add(str(key), MultiDictProxy(link)) + + return MultiDictProxy(links) + + async def start(self, connection: "Connection") -> "ClientResponse": + """Start response processing.""" + self._closed = False + self._protocol = connection.protocol + self._connection = connection + + with self._timer: + while True: + # read response + try: + protocol = self._protocol + message, payload = await protocol.read() # type: ignore[union-attr] + except http.HttpProcessingError as exc: + raise ClientResponseError( + self.request_info, + self.history, + status=exc.code, + message=exc.message, + headers=exc.headers, + ) from exc + + if message.code < 100 or message.code > 199 or message.code == 101: + break + + if self._continue is not None: + set_result(self._continue, True) + self._continue = None + + # payload eof handler + payload.on_eof(self._response_eof) + + # response status + self.version = message.version + self.status = message.code + self.reason = message.reason + + # headers + self._headers = message.headers # type is CIMultiDictProxy + self._raw_headers = message.raw_headers # type is Tuple[bytes, bytes] + + # payload + self.content = payload + + # cookies + if cookie_hdrs := self.headers.getall(hdrs.SET_COOKIE, ()): + # Store raw cookie headers for CookieJar + self._raw_cookie_headers = tuple(cookie_hdrs) + return self + + def _response_eof(self) -> None: + if self._closed: + return + + # protocol could be None because connection could be detached + protocol = self._connection and self._connection.protocol + if protocol is not None and protocol.upgraded: + return + + self._closed = True + self._cleanup_writer() + self._release_connection() + + @property + def closed(self) -> bool: + return self._closed + + def close(self) -> None: + if not self._released: + self._notify_content() + + self._closed = True + if self._loop is None or self._loop.is_closed(): + return + + self._cleanup_writer() + if self._connection is not None: + self._connection.close() + self._connection = None + + def release(self) -> Any: + if not self._released: + self._notify_content() + + self._closed = True + + self._cleanup_writer() + self._release_connection() + return noop() + + @property + def ok(self) -> bool: + """Returns ``True`` if ``status`` is less than ``400``, ``False`` if not. + + This is **not** a check for ``200 OK`` but a check that the response + status is under 400. + """ + return 400 > self.status + + def raise_for_status(self) -> None: + if not self.ok: + # reason should always be not None for a started response + assert self.reason is not None + + # If we're in a context we can rely on __aexit__() to release as the + # exception propagates. + if not self._in_context: + self.release() + + raise ClientResponseError( + self.request_info, + self.history, + status=self.status, + message=self.reason, + headers=self.headers, + ) + + def _release_connection(self) -> None: + if self._connection is not None: + if self.__writer is None: + self._connection.release() + self._connection = None + else: + self.__writer.add_done_callback(lambda f: self._release_connection()) + + async def _wait_released(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + self._release_connection() + + def _cleanup_writer(self) -> None: + if self.__writer is not None: + self.__writer.cancel() + if self._stream_writer is not None: + self._output_size = self._stream_writer.output_size + self._stream_writer = None + self._session = None + + def _notify_content(self) -> None: + content = self.content + if content and content.exception() is None: + set_exception(content, _CONNECTION_CLOSED_EXCEPTION) + self._released = True + + async def wait_for_close(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + self.release() + + async def read(self) -> bytes: + """Read response payload.""" + if self._body is None: + try: + self._body = await self.content.read() + for trace in self._traces: + await trace.send_response_chunk_received( + self.method, self.url, self._body + ) + except BaseException: + self.close() + raise + elif self._released: # Response explicitly released + raise ClientConnectionError("Connection closed") + + protocol = self._connection and self._connection.protocol + if protocol is None or not protocol.upgraded: + await self._wait_released() # Underlying connection released + return self._body + + def get_encoding(self) -> str: + ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() + mimetype = helpers.parse_mimetype(ctype) + + encoding = mimetype.parameters.get("charset") + if encoding: + with contextlib.suppress(LookupError, ValueError): + return codecs.lookup(encoding).name + + if mimetype.type == "application" and ( + mimetype.subtype == "json" or mimetype.subtype == "rdap" + ): + # RFC 7159 states that the default encoding is UTF-8. + # RFC 7483 defines application/rdap+json + return "utf-8" + + if self._body is None: + raise RuntimeError( + "Cannot compute fallback encoding of a not yet read body" + ) + + return self._resolve_charset(self, self._body) + + async def text(self, encoding: str | None = None, errors: str = "strict") -> str: + """Read response payload and decode.""" + if self._body is None: + await self.read() + + if encoding is None: + encoding = self.get_encoding() + + return self._body.decode(encoding, errors=errors) # type: ignore[union-attr] + + async def json( + self, + *, + encoding: str | None = None, + loads: JSONDecoder = DEFAULT_JSON_DECODER, + content_type: str | None = "application/json", + ) -> Any: + """Read and decodes JSON response.""" + if self._body is None: + await self.read() + + if content_type: + ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() + if not _is_expected_content_type(ctype, content_type): + raise ContentTypeError( + self.request_info, + self.history, + status=self.status, + message=( + "Attempt to decode JSON with unexpected mimetype: %s" % ctype + ), + headers=self.headers, + ) + + stripped = self._body.strip() # type: ignore[union-attr] + if not stripped: + return None + + if encoding is None: + encoding = self.get_encoding() + + return loads(stripped.decode(encoding)) + + async def __aenter__(self) -> "ClientResponse": + self._in_context = True + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self._in_context = False + # similar to _RequestContextManager, we do not need to check + # for exceptions, response object can close connection + # if state is broken + self.release() + await self.wait_for_close() + + +class ClientRequest: + GET_METHODS = { + hdrs.METH_GET, + hdrs.METH_HEAD, + hdrs.METH_OPTIONS, + hdrs.METH_TRACE, + } + POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} + ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) + + DEFAULT_HEADERS = { + hdrs.ACCEPT: "*/*", + hdrs.ACCEPT_ENCODING: _gen_default_accept_encoding(), + } + + # Type of body depends on PAYLOAD_REGISTRY, which is dynamic. + _body: None | payload.Payload = None + auth = None + response = None + + __writer: Optional["asyncio.Task[None]"] = None # async task for streaming data + + # These class defaults help create_autospec() work correctly. + # If autospec is improved in future, maybe these can be removed. + url = URL() + method = "GET" + + _continue = None # waiter future for '100 Continue' response + + _skip_auto_headers: Optional["CIMultiDict[None]"] = None + + # N.B. + # Adding __del__ method with self._writer closing doesn't make sense + # because _writer is instance method, thus it keeps a reference to self. + # Until writer has finished finalizer will not be called. + + def __init__( + self, + method: str, + url: URL, + *, + params: Query = None, + headers: LooseHeaders | None = None, + skip_auto_headers: Iterable[str] | None = None, + data: Any = None, + cookies: LooseCookies | None = None, + auth: BasicAuth | None = None, + version: http.HttpVersion = http.HttpVersion11, + compress: str | bool | None = None, + chunked: bool | None = None, + expect100: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + response_class: type["ClientResponse"] | None = None, + proxy: URL | None = None, + proxy_auth: BasicAuth | None = None, + timer: BaseTimerContext | None = None, + session: Optional["ClientSession"] = None, + ssl: SSLContext | bool | Fingerprint = True, + proxy_headers: LooseHeaders | None = None, + traces: list["Trace"] | None = None, + trust_env: bool = False, + server_hostname: str | None = None, + ): + if loop is None: + loop = asyncio.get_event_loop() + if match := _CONTAINS_CONTROL_CHAR_RE.search(method): + raise ValueError( + f"Method cannot contain non-token characters {method!r} " + f"(found at least {match.group()!r})" + ) + # URL forbids subclasses, so a simple type check is enough. + assert type(url) is URL, url + if proxy is not None: + assert type(proxy) is URL, proxy + # FIXME: session is None in tests only, need to fix tests + # assert session is not None + if TYPE_CHECKING: + assert session is not None + self._session = session + if params: + url = url.extend_query(params) + self.original_url = url + self.url = url.with_fragment(None) if url.raw_fragment else url + self.method = method.upper() + self.chunked = chunked + self.compress = compress + self.loop = loop + self.length = None + if response_class is None: + real_response_class = ClientResponse + else: + real_response_class = response_class + self.response_class: type[ClientResponse] = real_response_class + self._timer = timer if timer is not None else TimerNoop() + self._ssl = ssl if ssl is not None else True + self.server_hostname = server_hostname + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + self.update_version(version) + self.update_host(url) + self.update_headers(headers) + self.update_auto_headers(skip_auto_headers) + self.update_cookies(cookies) + self.update_content_encoding(data) + self.update_auth(auth, trust_env) + self.update_proxy(proxy, proxy_auth, proxy_headers) + + self.update_body_from_data(data) + if data is not None or self.method not in self.GET_METHODS: + self.update_transfer_encoding() + self.update_expect_continue(expect100) + self._traces = [] if traces is None else traces + + def __reset_writer(self, _: object = None) -> None: + self.__writer = None + + def _get_content_length(self) -> int | None: + """Extract and validate Content-Length header value. + + Returns parsed Content-Length value or None if not set. + Raises ValueError if header exists but cannot be parsed as an integer. + """ + if hdrs.CONTENT_LENGTH not in self.headers: + return None + + content_length_hdr = self.headers[hdrs.CONTENT_LENGTH] + if not _DIGITS_RE.fullmatch(content_length_hdr): + raise ValueError(f"Invalid Content-Length header: {content_length_hdr!r}") + return int(content_length_hdr) + + @property + def skip_auto_headers(self) -> CIMultiDict[None]: + return self._skip_auto_headers or CIMultiDict() + + @property + def _writer(self) -> Optional["asyncio.Task[None]"]: + return self.__writer + + @_writer.setter + def _writer(self, writer: "asyncio.Task[None]") -> None: + if self.__writer is not None: + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = writer + writer.add_done_callback(self.__reset_writer) + + def is_ssl(self) -> bool: + return self.url.scheme in _SSL_SCHEMES + + @property + def ssl(self) -> Union["SSLContext", bool, Fingerprint]: + return self._ssl + + @property + def connection_key(self) -> ConnectionKey: + if proxy_headers := self.proxy_headers: + h: int | None = hash(tuple(proxy_headers.items())) + else: + h = None + url = self.url + return tuple.__new__( + ConnectionKey, + ( + url.raw_host or "", + url.port, + url.scheme in _SSL_SCHEMES, + self._ssl, + self.proxy, + self.proxy_auth, + h, + self.server_hostname, + ), + ) + + @property + def host(self) -> str: + ret = self.url.raw_host + assert ret is not None + return ret + + @property + def port(self) -> int | None: + return self.url.port + + @property + def body(self) -> payload.Payload | Literal[b""]: + """Request body.""" + # empty body is represented as bytes for backwards compatibility + return self._body or b"" + + @body.setter + def body(self, value: Any) -> None: + """Set request body with warning for non-autoclose payloads. + + WARNING: This setter must be called from within an event loop and is not + thread-safe. Setting body outside of an event loop may raise RuntimeError + when closing file-based payloads. + + DEPRECATED: Direct assignment to body is deprecated and will be removed + in a future version. Use await update_body() instead for proper resource + management. + """ + # Close existing payload if present + if self._body is not None: + # Warn if the payload needs manual closing + # stacklevel=3: user code -> body setter -> _warn_if_unclosed_payload + _warn_if_unclosed_payload(self._body, stacklevel=3) + # NOTE: In the future, when we remove sync close support, + # this setter will need to be removed and only the async + # update_body() method will be available. For now, we call + # _close() for backwards compatibility. + self._body._close() + self._update_body(value) + + @property + def request_info(self) -> RequestInfo: + headers: CIMultiDictProxy[str] = CIMultiDictProxy(self.headers) + # These are created on every request, so we use a NamedTuple + # for performance reasons. We don't use the RequestInfo.__new__ + # method because it has a different signature which is provided + # for backwards compatibility only. + return tuple.__new__( + RequestInfo, (self.url, self.method, headers, self.original_url) + ) + + @property + def session(self) -> "ClientSession": + """Return the ClientSession instance. + + This property provides access to the ClientSession that initiated + this request, allowing middleware to make additional requests + using the same session. + """ + return self._session + + def update_host(self, url: URL) -> None: + """Update destination host, port and connection type (ssl).""" + # get host/port + if not url.raw_host: + raise InvalidURL(url) + + # basic auth info + if url.raw_user or url.raw_password: + self.auth = _basic_auth_no_warn(url.user or "", url.password or "") + + def update_version(self, version: http.HttpVersion | str) -> None: + """Convert request version to two elements tuple. + + parser HTTP version '1.1' => (1, 1) + """ + if isinstance(version, str): + v = [part.strip() for part in version.split(".", 1)] + try: + version = http.HttpVersion(int(v[0]), int(v[1])) + except ValueError: + raise ValueError( + f"Can not parse http version number: {version}" + ) from None + self.version = version + + def update_headers(self, headers: LooseHeaders | None) -> None: + """Update request headers.""" + self.headers: CIMultiDict[str] = CIMultiDict() + + # Build the host header + host = self.url.host_port_subcomponent + + # host_port_subcomponent is None when the URL is a relative URL. + # but we know we do not have a relative URL here. + assert host is not None + self.headers[hdrs.HOST] = host + + if not headers: + return + + if isinstance(headers, (dict, MultiDictProxy, MultiDict)): + headers = headers.items() + + for key, value in headers: # type: ignore[str-unpack] + # A special case for Host header + if key in hdrs.HOST_ALL: + self.headers[key] = value + else: + self.headers.add(key, value) + + def update_auto_headers(self, skip_auto_headers: Iterable[str] | None) -> None: + if skip_auto_headers is not None: + self._skip_auto_headers = CIMultiDict( + (hdr, None) for hdr in sorted(skip_auto_headers) + ) + used_headers = self.headers.copy() + used_headers.extend(self._skip_auto_headers) # type: ignore[arg-type] + else: + # Fast path when there are no headers to skip + # which is the most common case. + used_headers = self.headers + + for hdr, val in self.DEFAULT_HEADERS.items(): + if hdr not in used_headers: + self.headers[hdr] = val + + if hdrs.USER_AGENT not in used_headers: + self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE + + def update_cookies(self, cookies: LooseCookies | None) -> None: + """Update request cookies header.""" + if not cookies: + return + + c = SimpleCookie() + if hdrs.COOKIE in self.headers: + # parse_cookie_header for RFC 6265 compliant Cookie header parsing + c.update(parse_cookie_header(self.headers.get(hdrs.COOKIE, ""))) + del self.headers[hdrs.COOKIE] + + if isinstance(cookies, Mapping): + iter_cookies = cookies.items() + else: + iter_cookies = cookies # type: ignore[assignment] + for name, value in iter_cookies: + if isinstance(value, Morsel): + # Use helper to preserve coded_value exactly as sent by server + c[name] = preserve_morsel_with_coded_value(value) + else: + c[name] = value # type: ignore[assignment] + + self.headers[hdrs.COOKIE] = c.output(header="", sep=";").strip() + + def update_content_encoding(self, data: Any) -> None: + """Set request content encoding.""" + if not data: + # Don't compress an empty body. + self.compress = None + return + + if self.headers.get(hdrs.CONTENT_ENCODING): + if self.compress: + raise ValueError( + "compress can not be set if Content-Encoding header is set" + ) + elif self.compress: + if not isinstance(self.compress, str): + self.compress = "deflate" + self.headers[hdrs.CONTENT_ENCODING] = self.compress + self.chunked = True # enable chunked, no need to deal with length + + def update_transfer_encoding(self) -> None: + """Analyze transfer-encoding header.""" + te = self.headers.get(hdrs.TRANSFER_ENCODING, "").lower() + + if "chunked" in te: + if self.chunked: + raise ValueError( + "chunked can not be set " + 'if "Transfer-Encoding: chunked" header is set' + ) + + elif self.chunked: + if hdrs.CONTENT_LENGTH in self.headers: + raise ValueError( + "chunked can not be set if Content-Length header is set" + ) + + self.headers[hdrs.TRANSFER_ENCODING] = "chunked" + + def update_auth(self, auth: BasicAuth | None, trust_env: bool = False) -> None: + """Set basic auth.""" + if auth is None: + auth = self.auth + if auth is None: + return + + if not isinstance(auth, helpers.BasicAuth): + raise TypeError("BasicAuth() tuple is required instead") + + self.headers[hdrs.AUTHORIZATION] = auth.encode() + + def update_body_from_data(self, body: Any, _stacklevel: int = 3) -> None: + """Update request body from data.""" + if self._body is not None: + _warn_if_unclosed_payload(self._body, stacklevel=_stacklevel) + + if body is None: + self._body = None + # Set Content-Length to 0 when body is None for methods that expect a body + if ( + self.method not in self.GET_METHODS + and not self.chunked + and hdrs.CONTENT_LENGTH not in self.headers + ): + self.headers[hdrs.CONTENT_LENGTH] = "0" + return + + # FormData + maybe_payload = body() if isinstance(body, FormData) else body + + try: + body_payload = payload.PAYLOAD_REGISTRY.get(maybe_payload, disposition=None) + except payload.LookupError: + body_payload = FormData(maybe_payload)() # type: ignore[arg-type] + + self._body = body_payload + # enable chunked encoding if needed + if not self.chunked and hdrs.CONTENT_LENGTH not in self.headers: + if (size := body_payload.size) is not None: + self.headers[hdrs.CONTENT_LENGTH] = str(size) + else: + self.chunked = True + + # copy payload headers + assert body_payload.headers + headers = self.headers + skip_headers = self._skip_auto_headers + for key, value in body_payload.headers.items(): + if key in headers or (skip_headers is not None and key in skip_headers): + continue + headers[key] = value + + def _update_body(self, body: Any) -> None: + """Update request body after its already been set.""" + # Remove existing Content-Length header since body is changing + if hdrs.CONTENT_LENGTH in self.headers: + del self.headers[hdrs.CONTENT_LENGTH] + + # Remove existing Transfer-Encoding header to avoid conflicts + if self.chunked and hdrs.TRANSFER_ENCODING in self.headers: + del self.headers[hdrs.TRANSFER_ENCODING] + + # Now update the body using the existing method + # Called from _update_body, add 1 to stacklevel from caller + self.update_body_from_data(body, _stacklevel=4) + + # Update transfer encoding headers if needed (same logic as __init__) + if body is not None or self.method not in self.GET_METHODS: + self.update_transfer_encoding() + + async def update_body(self, body: Any) -> None: + """ + Update request body and close previous payload if needed. + + This method safely updates the request body by first closing any existing + payload to prevent resource leaks, then setting the new body. + + IMPORTANT: Always use this method instead of setting request.body directly. + Direct assignment to request.body will leak resources if the previous body + contains file handles, streams, or other resources that need cleanup. + + Args: + body: The new body content. Can be: + - bytes/bytearray: Raw binary data + - str: Text data (will be encoded using charset from Content-Type) + - FormData: Form data that will be encoded as multipart/form-data + - Payload: A pre-configured payload object + - AsyncIterable: An async iterable of bytes chunks + - File-like object: Will be read and sent as binary data + - None: Clears the body + + Usage: + # CORRECT: Use update_body + await request.update_body(b"new request data") + + # WRONG: Don't set body directly + # request.body = b"new request data" # This will leak resources! + + # Update with form data + form_data = FormData() + form_data.add_field('field', 'value') + await request.update_body(form_data) + + # Clear body + await request.update_body(None) + + Note: + This method is async because it may need to close file handles or + other resources associated with the previous payload. Always await + this method to ensure proper cleanup. + + Warning: + Setting request.body directly is highly discouraged and can lead to: + - Resource leaks (unclosed file handles, streams) + - Memory leaks (unreleased buffers) + - Unexpected behavior with streaming payloads + + It is not recommended to change the payload type in middleware. If the + body was already set (e.g., as bytes), it's best to keep the same type + rather than converting it (e.g., to str) as this may result in unexpected + behavior. + + See Also: + - update_body_from_data: Synchronous body update without cleanup + - body property: Direct body access (STRONGLY DISCOURAGED) + + """ + # Close existing payload if it exists and needs closing + if self._body is not None: + await self._body.close() + self._update_body(body) + + def update_expect_continue(self, expect: bool = False) -> None: + if expect: + self.headers[hdrs.EXPECT] = "100-continue" + elif ( + hdrs.EXPECT in self.headers + and self.headers[hdrs.EXPECT].lower() == "100-continue" + ): + expect = True + + if expect: + self._continue = self.loop.create_future() + + def update_proxy( + self, + proxy: URL | None, + proxy_auth: BasicAuth | None, + proxy_headers: LooseHeaders | None, + ) -> None: + self.proxy = proxy + if proxy is None: + self.proxy_auth = None + self.proxy_headers = None + return + + if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): + raise ValueError("proxy_auth must be None or BasicAuth() tuple") + self.proxy_auth = proxy_auth + + if proxy_headers is not None and not isinstance( + proxy_headers, (MultiDict, MultiDictProxy) + ): + proxy_headers = CIMultiDict(proxy_headers) + self.proxy_headers = proxy_headers + + async def write_bytes( + self, + writer: AbstractStreamWriter, + conn: "Connection", + content_length: int | None = None, + ) -> None: + """ + Write the request body to the connection stream. + + This method handles writing different types of request bodies: + 1. Payload objects (using their specialized write_with_length method) + 2. Bytes/bytearray objects + 3. Iterable body content + + Args: + writer: The stream writer to write the body to + conn: The connection being used for this request + content_length: Optional maximum number of bytes to write from the body + (None means write the entire body) + + The method properly handles: + - Waiting for 100-Continue responses if required + - Content length constraints for chunked encoding + - Error handling for network issues, cancellation, and other exceptions + - Signaling EOF and timeout management + + Raises: + ClientOSError: When there's an OS-level error writing the body + ClientConnectionError: When there's a general connection error + asyncio.CancelledError: When the operation is cancelled + + """ + # 100 response + if self._continue is not None: + # Force headers to be sent before waiting for 100-continue + writer.send_headers() + await writer.drain() + await self._continue + + protocol = conn.protocol + assert protocol is not None + try: + # This should be a rare case but the + # self._body can be set to None while + # the task is being started or we wait above + # for the 100-continue response. + # The more likely case is we have an empty + # payload, but 100-continue is still expected. + if self._body is not None: + await self._body.write_with_length(writer, content_length) + except OSError as underlying_exc: + reraised_exc = underlying_exc + + # Distinguish between timeout and other OS errors for better error reporting + exc_is_not_timeout = underlying_exc.errno is not None or not isinstance( + underlying_exc, asyncio.TimeoutError + ) + if exc_is_not_timeout: + reraised_exc = ClientOSError( + underlying_exc.errno, + f"Can not write request body for {self.url !s}", + ) + + set_exception(protocol, reraised_exc, underlying_exc) + except asyncio.CancelledError: + # Body hasn't been fully sent, so connection can't be reused + conn.close() + raise + except Exception as underlying_exc: + set_exception( + protocol, + ClientConnectionError( + "Failed to send bytes into the underlying connection " + f"{conn !s}: {underlying_exc!r}", + ), + underlying_exc, + ) + else: + # Successfully wrote the body, signal EOF and start response timeout + await writer.write_eof() + protocol.start_timeout() + + async def send(self, conn: "Connection") -> "ClientResponse": + # Specify request target: + # - CONNECT request must send authority form URI + # - not CONNECT proxy must send absolute form URI + # - most common is origin form URI + if self.method == hdrs.METH_CONNECT: + connect_host = self.url.host_subcomponent + assert connect_host is not None + path = f"{connect_host}:{self.url.port}" + elif self.proxy and not self.is_ssl(): + path = str(self.url) + else: + path = self.url.raw_path_qs + + protocol = conn.protocol + assert protocol is not None + writer = StreamWriter( + protocol, + self.loop, + on_chunk_sent=( + functools.partial(self._on_chunk_request_sent, self.method, self.url) + if self._traces + else None + ), + on_headers_sent=( + functools.partial(self._on_headers_request_sent, self.method, self.url) + if self._traces + else None + ), + ) + + if self.compress: + writer.enable_compression(self.compress) # type: ignore[arg-type] + + if self.chunked is not None: + writer.enable_chunking() + + # set default content-type + if ( + self.method in self.POST_METHODS + and ( + self._skip_auto_headers is None + or hdrs.CONTENT_TYPE not in self._skip_auto_headers + ) + and hdrs.CONTENT_TYPE not in self.headers + ): + self.headers[hdrs.CONTENT_TYPE] = "application/octet-stream" + + v = self.version + if hdrs.CONNECTION not in self.headers: + if conn._connector.force_close: + if v == HttpVersion11: + self.headers[hdrs.CONNECTION] = "close" + elif v == HttpVersion10: + self.headers[hdrs.CONNECTION] = "keep-alive" + + # status + headers + status_line = f"{self.method} {path} HTTP/{v.major}.{v.minor}" + + # Buffer headers for potential coalescing with body + await writer.write_headers(status_line, self.headers) + + task: asyncio.Task[None] | None + if self._body or self._continue is not None or protocol.writing_paused: + coro = self.write_bytes(writer, conn, self._get_content_length()) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to write + # bytes immediately to avoid having to schedule + # the task on the event loop. + task = asyncio.Task(coro, loop=self.loop, eager_start=True) + else: + task = self.loop.create_task(coro) + if task.done(): + task = None + else: + self._writer = task + else: + # We have nothing to write because + # - there is no body + # - the protocol does not have writing paused + # - we are not waiting for a 100-continue response + protocol.start_timeout() + writer.set_eof() + task = None + response_class = self.response_class + assert response_class is not None + self.response = response_class( + self.method, + self.original_url, + writer=task, + continue100=self._continue, + timer=self._timer, + request_info=self.request_info, + traces=self._traces, + loop=self.loop, + session=self._session, + stream_writer=writer, + ) + return self.response + + async def close(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + + def terminate(self) -> None: + if self.__writer is not None: + if not self.loop.is_closed(): + self.__writer.cancel() + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = None + + async def _on_chunk_request_sent(self, method: str, url: URL, chunk: bytes) -> None: + for trace in self._traces: + await trace.send_request_chunk_sent(method, url, chunk) + + async def _on_headers_request_sent( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + for trace in self._traces: + await trace.send_request_headers(method, url, headers) diff --git a/venv/Lib/site-packages/aiohttp/client_ws.py b/venv/Lib/site-packages/aiohttp/client_ws.py new file mode 100644 index 0000000000000000000000000000000000000000..4314b3cd4293d088e872ac95574c19b636ef6beb --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/client_ws.py @@ -0,0 +1,560 @@ +"""WebSocket client for asyncio.""" + +import asyncio +import sys +from collections.abc import Callable +from types import TracebackType +from typing import Any, Generic, Literal, Optional, cast, overload + +import attr + +from ._websocket.reader import WebSocketDataQueue +from .client_exceptions import ClientError, ServerTimeoutError, WSMessageTypeError +from .client_reqrep import ClientResponse +from .helpers import calculate_timeout_when, set_result +from .http import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WebSocketError, + WSCloseCode, + WSMessage, + WSMessageDecodeText, + WSMessageNoDecodeText, + WSMsgType, +) +from .http_websocket import _INTERNAL_RECEIVE_TYPES, WebSocketWriter +from .streams import EofStream +from .typedefs import ( + DEFAULT_JSON_DECODER, + DEFAULT_JSON_ENCODER, + JSONBytesEncoder, + JSONDecoder, + JSONEncoder, +) + +if sys.version_info >= (3, 13): + from typing import TypeVar +else: + from typing_extensions import TypeVar + +if sys.version_info >= (3, 11): + import asyncio as async_timeout + from typing import Self +else: + import async_timeout + from typing_extensions import Self + +# TypeVar for whether text messages are decoded to str (True) or kept as bytes (False) +# Covariant because it only affects return types, not input types +_DecodeText = TypeVar("_DecodeText", bound=bool, covariant=True, default=Literal[True]) + + +@attr.s(frozen=True, slots=True) +class ClientWSTimeout: + ws_receive = attr.ib(type=Optional[float], default=None) + ws_close = attr.ib(type=Optional[float], default=None) + + +DEFAULT_WS_CLIENT_TIMEOUT = ClientWSTimeout(ws_receive=None, ws_close=10.0) + + +class ClientWebSocketResponse(Generic[_DecodeText]): + def __init__( + self, + reader: WebSocketDataQueue, + writer: WebSocketWriter, + protocol: str | None, + response: ClientResponse, + timeout: ClientWSTimeout, + autoclose: bool, + autoping: bool, + loop: asyncio.AbstractEventLoop, + *, + heartbeat: float | None = None, + compress: int = 0, + client_notakeover: bool = False, + ) -> None: + self._response = response + self._conn = response.connection + + self._writer = writer + self._reader = reader + self._protocol = protocol + self._closed = False + self._closing = False + self._close_code: int | None = None + self._timeout = timeout + self._autoclose = autoclose + self._autoping = autoping + self._heartbeat = heartbeat + self._heartbeat_cb: asyncio.TimerHandle | None = None + self._heartbeat_when: float = 0.0 + if heartbeat is not None: + self._pong_heartbeat = heartbeat / 2.0 + self._pong_response_cb: asyncio.TimerHandle | None = None + self._loop = loop + self._waiting: bool = False + self._close_wait: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._compress = compress + self._client_notakeover = client_notakeover + self._ping_task: asyncio.Task[None] | None = None + self._need_heartbeat_reset = False + self._heartbeat_reset_handle: asyncio.Handle | None = None + + self._reset_heartbeat() + + def _cancel_heartbeat(self) -> None: + self._cancel_pong_response_cb() + if self._heartbeat_reset_handle is not None: + self._heartbeat_reset_handle.cancel() + self._heartbeat_reset_handle = None + self._need_heartbeat_reset = False + if self._heartbeat_cb is not None: + self._heartbeat_cb.cancel() + self._heartbeat_cb = None + if self._ping_task is not None: + self._ping_task.cancel() + self._ping_task = None + + def _cancel_pong_response_cb(self) -> None: + if self._pong_response_cb is not None: + self._pong_response_cb.cancel() + self._pong_response_cb = None + + def _on_data_received(self) -> None: + if self._heartbeat is None or self._need_heartbeat_reset: + return + loop = self._loop + assert loop is not None + # Coalesce multiple chunks received in the same loop tick into a single + # heartbeat reset. Resetting immediately per chunk increases timer churn. + self._need_heartbeat_reset = True + self._heartbeat_reset_handle = loop.call_soon(self._flush_heartbeat_reset) + + def _flush_heartbeat_reset(self) -> None: + self._heartbeat_reset_handle = None + if not self._need_heartbeat_reset: + return + self._reset_heartbeat() + self._need_heartbeat_reset = False + + def _reset_heartbeat(self) -> None: + if self._heartbeat is None: + return + self._cancel_pong_response_cb() + loop = self._loop + assert loop is not None + conn = self._conn + timeout_ceil_threshold = ( + conn._connector._timeout_ceil_threshold if conn is not None else 5 + ) + now = loop.time() + when = calculate_timeout_when(now, self._heartbeat, timeout_ceil_threshold) + self._heartbeat_when = when + if self._heartbeat_cb is None: + # We do not cancel the previous heartbeat_cb here because + # it generates a significant amount of TimerHandle churn + # which causes asyncio to rebuild the heap frequently. + # Instead _send_heartbeat() will reschedule the next + # heartbeat if it fires too early. + self._heartbeat_cb = loop.call_at(when, self._send_heartbeat) + + def _send_heartbeat(self) -> None: + self._heartbeat_cb = None + + # If heartbeat reset is pending (data is being received), skip sending + # the ping and let the reset callback handle rescheduling the heartbeat. + if self._need_heartbeat_reset: + return + + loop = self._loop + now = loop.time() + if now < self._heartbeat_when: + # Heartbeat fired too early, reschedule + self._heartbeat_cb = loop.call_at( + self._heartbeat_when, self._send_heartbeat + ) + return + + conn = self._conn + timeout_ceil_threshold = ( + conn._connector._timeout_ceil_threshold if conn is not None else 5 + ) + when = calculate_timeout_when(now, self._pong_heartbeat, timeout_ceil_threshold) + self._cancel_pong_response_cb() + self._pong_response_cb = loop.call_at(when, self._pong_not_received) + + coro = self._writer.send_frame(b"", WSMsgType.PING) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send the ping + # immediately to avoid having to schedule + # the task on the event loop. + ping_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + ping_task = loop.create_task(coro) + + if not ping_task.done(): + self._ping_task = ping_task + ping_task.add_done_callback(self._ping_task_done) + else: + self._ping_task_done(ping_task) + + def _ping_task_done(self, task: "asyncio.Task[None]") -> None: + """Callback for when the ping task completes.""" + if not task.cancelled() and (exc := task.exception()): + self._handle_ping_pong_exception(exc) + self._ping_task = None + + def _pong_not_received(self) -> None: + self._handle_ping_pong_exception( + ServerTimeoutError(f"No PONG received after {self._pong_heartbeat} seconds") + ) + + def _handle_ping_pong_exception(self, exc: BaseException) -> None: + """Handle exceptions raised during ping/pong processing.""" + if self._closed: + return + self._set_closed() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + if self._waiting and not self._closing: + self._reader.feed_data(WSMessage(WSMsgType.ERROR, exc, None), 0) + + def _set_closed(self) -> None: + """Set the connection to closed. + + Cancel any heartbeat timers and set the closed flag. + """ + self._closed = True + self._cancel_heartbeat() + + def _set_closing(self) -> None: + """Set the connection to closing. + + Cancel any heartbeat timers and set the closing flag. + """ + self._closing = True + self._cancel_heartbeat() + + @property + def closed(self) -> bool: + return self._closed + + @property + def close_code(self) -> int | None: + return self._close_code + + @property + def protocol(self) -> str | None: + return self._protocol + + @property + def compress(self) -> int: + return self._compress + + @property + def client_notakeover(self) -> bool: + return self._client_notakeover + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """extra info from connection transport""" + conn = self._response.connection + if conn is None: + return default + transport = conn.transport + if transport is None: + return default + return transport.get_extra_info(name, default) + + def exception(self) -> BaseException | None: + return self._exception + + async def ping(self, message: bytes = b"") -> None: + await self._writer.send_frame(message, WSMsgType.PING) + + async def pong(self, message: bytes = b"") -> None: + await self._writer.send_frame(message, WSMsgType.PONG) + + async def send_frame( + self, message: bytes, opcode: WSMsgType, compress: int | None = None + ) -> None: + """Send a frame over the websocket.""" + await self._writer.send_frame(message, opcode, compress) + + async def send_str(self, data: str, compress: int | None = None) -> None: + if not isinstance(data, str): + raise TypeError("data argument must be str (%r)" % type(data)) + await self._writer.send_frame( + data.encode("utf-8"), WSMsgType.TEXT, compress=compress + ) + + async def send_bytes(self, data: bytes, compress: int | None = None) -> None: + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("data argument must be byte-ish (%r)" % type(data)) + await self._writer.send_frame(data, WSMsgType.BINARY, compress=compress) + + async def send_json( + self, + data: Any, + compress: int | None = None, + *, + dumps: JSONEncoder = DEFAULT_JSON_ENCODER, + ) -> None: + await self.send_str(dumps(data), compress=compress) + + async def send_json_bytes( + self, + data: Any, + compress: int | None = None, + *, + dumps: JSONBytesEncoder, + ) -> None: + """Send JSON data using a bytes-returning encoder as a binary frame. + + Use this when your JSON encoder (like orjson) returns bytes + instead of str, avoiding the encode/decode overhead. + """ + await self.send_bytes(dumps(data), compress=compress) + + async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool: + # we need to break `receive()` cycle first, + # `close()` may be called from different task + if self._waiting and not self._closing: + assert self._loop is not None + self._close_wait = self._loop.create_future() + self._set_closing() + self._reader.feed_data(WS_CLOSING_MESSAGE, 0) + await self._close_wait + + if self._closed: + return False + + self._set_closed() + try: + await self._writer.close(code, message) + except asyncio.CancelledError: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._response.close() + raise + except Exception as exc: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + return True + + if self._close_code: + self._response.close() + return True + + while True: + try: + async with async_timeout.timeout(self._timeout.ws_close): + msg = await self._reader.read() + except asyncio.CancelledError: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._response.close() + raise + except Exception as exc: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + return True + + if msg.type is WSMsgType.CLOSE: + self._close_code = msg.data + self._response.close() + return True + + @overload + async def receive( + self: "ClientWebSocketResponse[Literal[True]]", timeout: float | None = None + ) -> WSMessageDecodeText: ... + + @overload + async def receive( + self: "ClientWebSocketResponse[Literal[False]]", timeout: float | None = None + ) -> WSMessageNoDecodeText: ... + + @overload + async def receive( + self: "ClientWebSocketResponse[_DecodeText]", timeout: float | None = None + ) -> WSMessageDecodeText | WSMessageNoDecodeText: ... + + async def receive( + self, timeout: float | None = None + ) -> WSMessageDecodeText | WSMessageNoDecodeText: + receive_timeout = timeout or self._timeout.ws_receive + + while True: + if self._waiting: + raise RuntimeError("Concurrent call to receive() is not allowed") + + if self._closed: + return WS_CLOSED_MESSAGE + elif self._closing: + await self.close() + return WS_CLOSED_MESSAGE + + try: + self._waiting = True + try: + if receive_timeout: + # Entering the context manager and creating + # Timeout() object can take almost 50% of the + # run time in this loop so we avoid it if + # there is no read timeout. + async with async_timeout.timeout(receive_timeout): + msg = await self._reader.read() + else: + msg = await self._reader.read() + finally: + self._waiting = False + if self._close_wait: + set_result(self._close_wait, None) + except (asyncio.CancelledError, asyncio.TimeoutError): + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + raise + except EofStream: + self._close_code = WSCloseCode.OK + await self.close() + return WSMessage(WSMsgType.CLOSED, None, None) + except ClientError: + # Likely ServerDisconnectedError when connection is lost + self._set_closed() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + return WS_CLOSED_MESSAGE + except WebSocketError as exc: + self._close_code = exc.code + await self.close(code=exc.code) + return WSMessage(WSMsgType.ERROR, exc, None) + except Exception as exc: + self._exception = exc + self._set_closing() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + await self.close() + return WSMessage(WSMsgType.ERROR, exc, None) + + if msg.type not in _INTERNAL_RECEIVE_TYPES: + # If its not a close/closing/ping/pong message + # we can return it immediately + return msg + + if msg.type is WSMsgType.CLOSE: + self._set_closing() + self._close_code = msg.data + if not self._closed and self._autoclose: + await self.close() + elif msg.type is WSMsgType.CLOSING: + self._set_closing() + elif msg.type is WSMsgType.PING and self._autoping: + await self.pong(msg.data) + continue + elif msg.type is WSMsgType.PONG and self._autoping: + continue + + return msg + + @overload + async def receive_str( + self: "ClientWebSocketResponse[Literal[True]]", *, timeout: float | None = None + ) -> str: ... + + @overload + async def receive_str( + self: "ClientWebSocketResponse[Literal[False]]", *, timeout: float | None = None + ) -> bytes: ... + + @overload + async def receive_str( + self: "ClientWebSocketResponse[_DecodeText]", *, timeout: float | None = None + ) -> str | bytes: ... + + async def receive_str(self, *, timeout: float | None = None) -> str | bytes: + """Receive TEXT message. + + Returns str when decode_text=True (default), bytes when decode_text=False. + """ + msg = await self.receive(timeout) + if msg.type is not WSMsgType.TEXT: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.TEXT" + ) + return cast(str, msg.data) + + async def receive_bytes(self, *, timeout: float | None = None) -> bytes: + msg = await self.receive(timeout) + if msg.type is not WSMsgType.BINARY: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.BINARY" + ) + return cast(bytes, msg.data) + + @overload + async def receive_json( + self: "ClientWebSocketResponse[Literal[True]]", + *, + loads: JSONDecoder = ..., + timeout: float | None = None, + ) -> Any: ... + + @overload + async def receive_json( + self: "ClientWebSocketResponse[Literal[False]]", + *, + loads: Callable[[bytes], Any] = ..., + timeout: float | None = None, + ) -> Any: ... + + @overload + async def receive_json( + self: "ClientWebSocketResponse[_DecodeText]", + *, + loads: JSONDecoder | Callable[[bytes], Any] = ..., + timeout: float | None = None, + ) -> Any: ... + + async def receive_json( + self, + *, + loads: JSONDecoder | Callable[[bytes], Any] = DEFAULT_JSON_DECODER, + timeout: float | None = None, + ) -> Any: + data = await self.receive_str(timeout=timeout) + return loads(data) # type: ignore[arg-type] + + def __aiter__(self) -> Self: + return self + + @overload + async def __anext__( + self: "ClientWebSocketResponse[Literal[True]]", + ) -> WSMessageDecodeText: ... + + @overload + async def __anext__( + self: "ClientWebSocketResponse[Literal[False]]", + ) -> WSMessageNoDecodeText: ... + + @overload + async def __anext__( + self: "ClientWebSocketResponse[_DecodeText]", + ) -> WSMessageDecodeText | WSMessageNoDecodeText: ... + + async def __anext__(self) -> WSMessageDecodeText | WSMessageNoDecodeText: + msg = await self.receive() + if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED): + raise StopAsyncIteration + return msg + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + await self.close() diff --git a/venv/Lib/site-packages/aiohttp/compression_utils.py b/venv/Lib/site-packages/aiohttp/compression_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..712531e0fba126c94f3a46378143657ff5779e50 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/compression_utils.py @@ -0,0 +1,447 @@ +import asyncio +import sys +import zlib +from abc import ABC, abstractmethod +from concurrent.futures import Executor +from typing import Any, Final, Protocol, TypedDict, cast + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing import Union + + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +try: + try: + import brotlicffi as brotli + except ImportError: + import brotli + + HAS_BROTLI = True +except ImportError: # pragma: no cover + HAS_BROTLI = False + +try: + if sys.version_info >= (3, 14): + from compression.zstd import ZstdDecompressor # noqa: I900 + else: # TODO(PY314): Remove mentions of backports.zstd across codebase + from backports.zstd import ZstdDecompressor + + HAS_ZSTD = True +except ImportError: + HAS_ZSTD = False + + +MAX_SYNC_CHUNK_SIZE = 4096 + +# Unlimited decompression constants - different libraries use different conventions +ZLIB_MAX_LENGTH_UNLIMITED = 0 # zlib uses 0 to mean unlimited +ZSTD_MAX_LENGTH_UNLIMITED = -1 # zstd uses -1 to mean unlimited + + +class ZLibCompressObjProtocol(Protocol): + def compress(self, data: Buffer) -> bytes: ... + def flush(self, mode: int = ..., /) -> bytes: ... + + +class ZLibDecompressObjProtocol(Protocol): + def decompress(self, data: Buffer, max_length: int = ...) -> bytes: ... + def flush(self, length: int = ..., /) -> bytes: ... + + @property + def eof(self) -> bool: ... + + @property + def unconsumed_tail(self) -> bytes: ... + + @property + def unused_data(self) -> bytes: ... + + +class ZLibBackendProtocol(Protocol): + MAX_WBITS: int + Z_FULL_FLUSH: int + Z_SYNC_FLUSH: int + Z_BEST_SPEED: int + Z_FINISH: int + + def compressobj( + self, + level: int = ..., + method: int = ..., + wbits: int = ..., + memLevel: int = ..., + strategy: int = ..., + zdict: Buffer | None = ..., + ) -> ZLibCompressObjProtocol: ... + def decompressobj( + self, wbits: int = ..., zdict: Buffer = ... + ) -> ZLibDecompressObjProtocol: ... + + def compress( + self, data: Buffer, /, level: int = ..., wbits: int = ... + ) -> bytes: ... + def decompress( + self, data: Buffer, /, wbits: int = ..., bufsize: int = ... + ) -> bytes: ... + + +class CompressObjArgs(TypedDict, total=False): + wbits: int + strategy: int + level: int + + +class ZLibBackendWrapper: + def __init__(self, _zlib_backend: ZLibBackendProtocol): + self._zlib_backend: ZLibBackendProtocol = _zlib_backend + + @property + def name(self) -> str: + return getattr(self._zlib_backend, "__name__", "undefined") + + @property + def MAX_WBITS(self) -> int: + return self._zlib_backend.MAX_WBITS + + @property + def Z_FULL_FLUSH(self) -> int: + return self._zlib_backend.Z_FULL_FLUSH + + @property + def Z_SYNC_FLUSH(self) -> int: + return self._zlib_backend.Z_SYNC_FLUSH + + @property + def Z_BEST_SPEED(self) -> int: + return self._zlib_backend.Z_BEST_SPEED + + @property + def Z_FINISH(self) -> int: + return self._zlib_backend.Z_FINISH + + def compressobj(self, *args: Any, **kwargs: Any) -> ZLibCompressObjProtocol: + return self._zlib_backend.compressobj(*args, **kwargs) + + def decompressobj(self, *args: Any, **kwargs: Any) -> ZLibDecompressObjProtocol: + return self._zlib_backend.decompressobj(*args, **kwargs) + + def compress(self, data: Buffer, *args: Any, **kwargs: Any) -> bytes: + return self._zlib_backend.compress(data, *args, **kwargs) + + def decompress(self, data: Buffer, *args: Any, **kwargs: Any) -> bytes: + return self._zlib_backend.decompress(data, *args, **kwargs) + + # Everything not explicitly listed in the Protocol we just pass through + def __getattr__(self, attrname: str) -> Any: + return getattr(self._zlib_backend, attrname) + + +ZLibBackend: ZLibBackendWrapper = ZLibBackendWrapper(zlib) + + +def set_zlib_backend(new_zlib_backend: ZLibBackendProtocol) -> None: + ZLibBackend._zlib_backend = new_zlib_backend + + +def encoding_to_mode( + encoding: str | None = None, + suppress_deflate_header: bool = False, +) -> int: + if encoding == "gzip": + return 16 + ZLibBackend.MAX_WBITS + + return -ZLibBackend.MAX_WBITS if suppress_deflate_header else ZLibBackend.MAX_WBITS + + +class DecompressionBaseHandler(ABC): + def __init__( + self, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ): + """Base class for decompression handlers.""" + self._executor = executor + self._max_sync_chunk_size = max_sync_chunk_size + + @abstractmethod + def decompress_sync( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + """Decompress the given data.""" + + async def decompress( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + """Decompress the given data.""" + if ( + self._max_sync_chunk_size is not None + and len(data) > self._max_sync_chunk_size + ): + return await asyncio.get_event_loop().run_in_executor( + self._executor, self.decompress_sync, data, max_length + ) + return self.decompress_sync(data, max_length) + + @property + @abstractmethod + def data_available(self) -> bool: + """Return True if more output is available by passing b"".""" + + +class ZLibCompressor: + def __init__( + self, + encoding: str | None = None, + suppress_deflate_header: bool = False, + level: int | None = None, + wbits: int | None = None, + strategy: int | None = None, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ): + self._executor = executor + self._max_sync_chunk_size = max_sync_chunk_size + self._mode = ( + encoding_to_mode(encoding, suppress_deflate_header) + if wbits is None + else wbits + ) + self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend) + + kwargs: CompressObjArgs = {} + kwargs["wbits"] = self._mode + if strategy is not None: + kwargs["strategy"] = strategy + if level is not None: + kwargs["level"] = level + self._compressor = self._zlib_backend.compressobj(**kwargs) + + def compress_sync(self, data: Buffer) -> bytes: + return self._compressor.compress(data) + + async def compress(self, data: Buffer) -> bytes: + """Compress the data and returned the compressed bytes. + + Note that flush() must be called after the last call to compress() + + If the data size is large than the max_sync_chunk_size, the compression + will be done in the executor. Otherwise, the compression will be done + in the event loop. + + **WARNING: This method is NOT cancellation-safe when used with flush().** + If this operation is cancelled, the compressor state may be corrupted. + The connection MUST be closed after cancellation to avoid data corruption + in subsequent compress operations. + + For cancellation-safe compression (e.g., WebSocket), the caller MUST wrap + compress() + flush() + send operations in a shield and lock to ensure atomicity. + """ + # For large payloads, offload compression to executor to avoid blocking event loop + should_use_executor = ( + self._max_sync_chunk_size is not None + and len(data) > self._max_sync_chunk_size + ) + if should_use_executor: + return await asyncio.get_running_loop().run_in_executor( + self._executor, self._compressor.compress, data + ) + return self.compress_sync(data) + + def flush(self, mode: int | None = None) -> bytes: + """Flush the compressor synchronously. + + **WARNING: This method is NOT cancellation-safe when called after compress().** + The flush() operation accesses shared compressor state. If compress() was + cancelled, calling flush() may result in corrupted data. The connection MUST + be closed after compress() cancellation. + + For cancellation-safe compression (e.g., WebSocket), the caller MUST wrap + compress() + flush() + send operations in a shield and lock to ensure atomicity. + """ + return self._compressor.flush( + mode if mode is not None else self._zlib_backend.Z_FINISH + ) + + +class ZLibDecompressor(DecompressionBaseHandler): + def __init__( + self, + encoding: str | None = None, + suppress_deflate_header: bool = False, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ): + super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size) + self._mode = encoding_to_mode(encoding, suppress_deflate_header) + self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend) + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + self._last_empty = False + self._pending_unused_data: bytes | None = None + + def decompress_sync( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + if self._pending_unused_data is not None: + data = self._pending_unused_data + bytes(data) + self._pending_unused_data = None + result = self._decompressor.decompress( + self._decompressor.unconsumed_tail + data, max_length + ) + # Only way to know that isal has no further data is checking we get no output + self._last_empty = result == b"" + + # Handle concatenated gzip/deflate streams (multi-member). + # After a member ends, unused_data holds the start of the next member. + # Create a fresh decompressor for each subsequent member. + while self._decompressor.eof and self._decompressor.unused_data: + unused = self._decompressor.unused_data + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + if max_length != ZLIB_MAX_LENGTH_UNLIMITED: + max_length -= len(result) + if max_length <= 0: + self._pending_unused_data = unused + break + chunk = self._decompressor.decompress(unused, max_length) + self._last_empty = chunk == b"" + result += chunk + + # Member ended exactly at chunk boundary β€” no unused_data, but the + # next feed_data() call would fail on the spent decompressor. + # Only reset for gzip; deflate's feed_eof() relies on eof=True to + # confirm the stream is complete. + if self._decompressor.eof and self._mode > self._zlib_backend.MAX_WBITS: + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + + return result + + def flush(self, length: int = 0) -> bytes: + return ( + self._decompressor.flush(length) + if length > 0 + else self._decompressor.flush() + ) + + @property + def data_available(self) -> bool: + return ( + bool(self._decompressor.unconsumed_tail) + or not self._last_empty + or self._pending_unused_data is not None + ) + + @property + def eof(self) -> bool: + return self._decompressor.eof + + +class BrotliDecompressor(DecompressionBaseHandler): + # Supports both 'brotlipy' and 'Brotli' packages + # since they share an import name. The top branches + # are for 'brotlipy' and bottom branches for 'Brotli' + def __init__( + self, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ) -> None: + """Decompress data using the Brotli library.""" + if not HAS_BROTLI: + raise RuntimeError( + "The brotli decompression is not available. " + "Please install `Brotli` module" + ) + self._obj = brotli.Decompressor() + self._last_empty = False + super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size) + + def decompress_sync( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + """Decompress the given data.""" + if hasattr(self._obj, "decompress"): + if max_length == ZLIB_MAX_LENGTH_UNLIMITED: + result = cast(bytes, self._obj.decompress(data)) + else: + result = cast(bytes, self._obj.decompress(data, max_length)) + else: + if max_length == ZLIB_MAX_LENGTH_UNLIMITED: + result = cast(bytes, self._obj.process(data)) + else: + result = cast(bytes, self._obj.process(data, max_length)) + # Only way to know that brotli has no further data is checking we get no output + self._last_empty = result == b"" + return result + + def flush(self) -> bytes: + """Flush the decompressor.""" + if hasattr(self._obj, "flush"): + return cast(bytes, self._obj.flush()) + return b"" + + @property + def data_available(self) -> bool: + return not self._obj.is_finished() and not self._last_empty + + +class ZSTDDecompressor(DecompressionBaseHandler): + def __init__( + self, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ) -> None: + if not HAS_ZSTD: + raise RuntimeError( + "The zstd decompression is not available. " + "Please install `backports.zstd` module" + ) + self._obj = ZstdDecompressor() + self._pending_unused_data: bytes | None = None + super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size) + + def decompress_sync( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + # zstd uses -1 for unlimited, while zlib uses 0 for unlimited + # Convert the zlib convention (0=unlimited) to zstd convention (-1=unlimited) + zstd_max_length = ( + ZSTD_MAX_LENGTH_UNLIMITED + if max_length == ZLIB_MAX_LENGTH_UNLIMITED + else max_length + ) + if self._pending_unused_data is not None: + data = self._pending_unused_data + data + self._pending_unused_data = None + result = self._obj.decompress(data, zstd_max_length) + + # Handle multi-frame zstd streams. + # https://datatracker.ietf.org/doc/html/rfc8878#section-3.1.1 + # ZstdDecompressor handles one frame only. When a frame ends, + # eof becomes True and any trailing data goes to unused_data. + # We create a fresh decompressor to continue with the next frame. + while self._obj.eof and self._obj.unused_data: + unused_data = self._obj.unused_data + self._obj = ZstdDecompressor() + if zstd_max_length != ZSTD_MAX_LENGTH_UNLIMITED: + zstd_max_length -= len(result) + if zstd_max_length <= 0: + self._pending_unused_data = unused_data + break + result += self._obj.decompress(unused_data, zstd_max_length) + + # Frame ended exactly at chunk boundary β€” no unused_data, but the + # next feed_data() call would fail on the spent decompressor. + # Prepare a fresh one for the next chunk. + if self._obj.eof: + self._obj = ZstdDecompressor() + + return result + + def flush(self) -> bytes: + return b"" + + @property + def data_available(self) -> bool: + return ( + not self._obj.needs_input and not self._obj.eof + ) or self._pending_unused_data is not None diff --git a/venv/Lib/site-packages/aiohttp/connector.py b/venv/Lib/site-packages/aiohttp/connector.py new file mode 100644 index 0000000000000000000000000000000000000000..7cdcaad0b38cf6ba516919699043bafbe637c404 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/connector.py @@ -0,0 +1,1851 @@ +import asyncio +import functools +import random +import socket +import sys +import traceback +import warnings +from collections import OrderedDict, defaultdict, deque +from collections.abc import Awaitable, Callable, Iterator, Sequence +from contextlib import suppress +from http import HTTPStatus +from itertools import chain, cycle, islice +from time import monotonic +from types import TracebackType +from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast + +import aiohappyeyeballs +from aiohappyeyeballs import AddrInfoType, SocketFactoryType + +from . import hdrs, helpers +from .abc import AbstractResolver, ResolveResult +from .client_exceptions import ( + ClientConnectionError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientHttpProxyError, + ClientProxyConnectionError, + InvalidUrlClientError, + ServerFingerprintMismatch, + UnixClientConnectorError, + cert_errors, + ssl_errors, +) +from .client_proto import ResponseHandler +from .client_reqrep import ClientRequest, Fingerprint, _merge_ssl_params +from .helpers import ( + _SENTINEL, + ceil_timeout, + is_canonical_ipv4_address, + is_ip_address, + noop, + sentinel, + set_exception, + set_result, +) +from .log import client_logger +from .resolver import DefaultResolver + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + +EMPTY_SCHEMA_SET = frozenset({""}) +HTTP_SCHEMA_SET = frozenset({"http", "https"}) +WS_SCHEMA_SET = frozenset({"ws", "wss"}) + +HTTP_AND_EMPTY_SCHEMA_SET = HTTP_SCHEMA_SET | EMPTY_SCHEMA_SET +HIGH_LEVEL_SCHEMA_SET = HTTP_AND_EMPTY_SCHEMA_SET | WS_SCHEMA_SET + +NEEDS_CLEANUP_CLOSED = (3, 13, 0) <= sys.version_info < ( + 3, + 13, + 1, +) or sys.version_info < (3, 12, 8) +# Cleanup closed is no longer needed after https://github.com/python/cpython/pull/118960 +# which first appeared in Python 3.12.8 and 3.13.1 + + +__all__ = ( + "BaseConnector", + "TCPConnector", + "UnixConnector", + "NamedPipeConnector", + "AddrInfoType", + "SocketFactoryType", +) + + +if TYPE_CHECKING: + from .client import ClientTimeout + from .client_reqrep import ConnectionKey + from .tracing import Trace + + +class _DeprecationWaiter: + __slots__ = ("_awaitable", "_awaited") + + def __init__(self, awaitable: Awaitable[Any]) -> None: + self._awaitable = awaitable + self._awaited = False + + def __await__(self) -> Any: + self._awaited = True + return self._awaitable.__await__() + + def __del__(self) -> None: + if not self._awaited: + warnings.warn( + "Connector.close() is a coroutine, " + "please use await connector.close()", + DeprecationWarning, + ) + + +async def _wait_for_close(waiters: list[Awaitable[object]]) -> None: + """Wait for all waiters to finish closing.""" + results = await asyncio.gather(*waiters, return_exceptions=True) + for res in results: + if isinstance(res, Exception): + client_logger.debug("Error while closing connector: %r", res) + + +class Connection: + + _source_traceback = None + + def __init__( + self, + connector: "BaseConnector", + key: "ConnectionKey", + protocol: ResponseHandler, + loop: asyncio.AbstractEventLoop, + ) -> None: + self._key = key + self._connector = connector + self._loop = loop + self._protocol: ResponseHandler | None = protocol + self._callbacks: list[Callable[[], None]] = [] + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + def __repr__(self) -> str: + return f"Connection<{self._key}>" + + def __del__(self, _warnings: Any = warnings) -> None: + if self._protocol is not None: + kwargs = {"source": self} + _warnings.warn(f"Unclosed connection {self!r}", ResourceWarning, **kwargs) + if self._loop.is_closed(): + return + + self._connector._release(self._key, self._protocol, should_close=True) + + context = {"client_connection": self, "message": "Unclosed connection"} + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __bool__(self) -> Literal[True]: + """Force subclasses to not be falsy, to make checks simpler.""" + return True + + @property + def loop(self) -> asyncio.AbstractEventLoop: + warnings.warn( + "connector.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + @property + def transport(self) -> asyncio.Transport | None: + if self._protocol is None: + return None + return self._protocol.transport + + @property + def protocol(self) -> ResponseHandler | None: + return self._protocol + + def add_callback(self, callback: Callable[[], None]) -> None: + if callback is not None: + self._callbacks.append(callback) + + def _notify_release(self) -> None: + callbacks, self._callbacks = self._callbacks[:], [] + + for cb in callbacks: + with suppress(Exception): + cb() + + def close(self) -> None: + self._notify_release() + + if self._protocol is not None: + self._connector._release(self._key, self._protocol, should_close=True) + self._protocol = None + + def release(self) -> None: + self._notify_release() + + if self._protocol is not None: + self._connector._release(self._key, self._protocol) + self._protocol = None + + @property + def closed(self) -> bool: + return self._protocol is None or not self._protocol.is_connected() + + +class _ConnectTunnelConnection(Connection): + """Special connection wrapper for CONNECT tunnels that must never be pooled. + + This connection wraps the proxy connection that will be upgraded with TLS. + It must never be released to the pool because: + 1. Its 'closed' future will never complete, causing session.close() to hang + 2. It represents an intermediate state, not a reusable connection + 3. The real connection (with TLS) will be created separately + """ + + def release(self) -> None: + """Do nothing - don't pool or close the connection. + + These connections are an intermediate state during the CONNECT tunnel + setup and will be cleaned up naturally after the TLS upgrade. If they + were to be pooled, they would never be properly closed, causing + session.close() to wait forever for their 'closed' future. + """ + + +class _TransportPlaceholder: + """placeholder for BaseConnector.connect function""" + + __slots__ = ("closed", "transport") + + def __init__(self, closed_future: asyncio.Future[Exception | None]) -> None: + """Initialize a placeholder for a transport.""" + self.closed = closed_future + self.transport = None + + def close(self) -> None: + """Close the placeholder.""" + + def abort(self) -> None: + """Abort the placeholder (does nothing).""" + + +class BaseConnector: + """Base connector class. + + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + enable_cleanup_closed - Enables clean-up closed ssl transports. + Disabled by default. + timeout_ceil_threshold - Trigger ceiling of timeout values when + it's above timeout_ceil_threshold. + loop - Optional event loop. + """ + + _closed = True # prevent AttributeError in __del__ if ctor was failed + _source_traceback = None + + # abort transport after 2 seconds (cleanup broken connections) + _cleanup_closed_period = 2.0 + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET + + def __init__( + self, + *, + keepalive_timeout: object | None | float = sentinel, + force_close: bool = False, + limit: int = 100, + limit_per_host: int = 0, + enable_cleanup_closed: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + timeout_ceil_threshold: float = 5, + ) -> None: + + if force_close: + if keepalive_timeout is not None and keepalive_timeout is not sentinel: + raise ValueError( + "keepalive_timeout cannot be set if force_close is True" + ) + else: + if keepalive_timeout is sentinel: + keepalive_timeout = 15.0 + + loop = loop or asyncio.get_running_loop() + self._timeout_ceil_threshold = timeout_ceil_threshold + + self._closed = False + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + # Connection pool of reusable connections. + # We use a deque to store connections because it has O(1) popleft() + # and O(1) append() operations to implement a FIFO queue. + self._conns: defaultdict[ + ConnectionKey, deque[tuple[ResponseHandler, float]] + ] = defaultdict(deque) + self._limit = limit + self._limit_per_host = limit_per_host + self._acquired: set[ResponseHandler] = set() + self._acquired_per_host: defaultdict[ConnectionKey, set[ResponseHandler]] = ( + defaultdict(set) + ) + self._keepalive_timeout = cast(float, keepalive_timeout) + self._force_close = force_close + + # {host_key: FIFO list of waiters} + # The FIFO is implemented with an OrderedDict with None keys because + # python does not have an ordered set. + self._waiters: defaultdict[ + ConnectionKey, OrderedDict[asyncio.Future[None], None] + ] = defaultdict(OrderedDict) + + self._loop = loop + self._factory = functools.partial(ResponseHandler, loop=loop) + + # start keep-alive connection cleanup task + self._cleanup_handle: asyncio.TimerHandle | None = None + + # start cleanup closed transports task + self._cleanup_closed_handle: asyncio.TimerHandle | None = None + + if enable_cleanup_closed and not NEEDS_CLEANUP_CLOSED: + warnings.warn( + "enable_cleanup_closed ignored because " + "https://github.com/python/cpython/pull/118960 is fixed " + f"in Python version {sys.version_info}", + DeprecationWarning, + stacklevel=2, + ) + enable_cleanup_closed = False + + self._cleanup_closed_disabled = not enable_cleanup_closed + self._cleanup_closed_transports: list[asyncio.Transport | None] = [] + self._placeholder_future: asyncio.Future[Exception | None] = ( + loop.create_future() + ) + self._placeholder_future.set_result(None) + self._cleanup_closed() + + def __del__(self, _warnings: Any = warnings) -> None: + if self._closed: + return + if not self._conns: + return + + conns = [repr(c) for c in self._conns.values()] + + self._close() + + kwargs = {"source": self} + _warnings.warn(f"Unclosed connector {self!r}", ResourceWarning, **kwargs) + context = { + "connector": self, + "connections": conns, + "message": "Unclosed connector", + } + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __enter__(self) -> "BaseConnector": + warnings.warn( + '"with Connector():" is deprecated, ' + 'use "async with Connector():" instead', + DeprecationWarning, + ) + return self + + def __exit__(self, *exc: Any) -> None: + self._close() + + async def __aenter__(self) -> "BaseConnector": + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + exc_traceback: TracebackType | None = None, + ) -> None: + await self.close() + + @property + def force_close(self) -> bool: + """Ultimately close connection on releasing if True.""" + return self._force_close + + @property + def limit(self) -> int: + """The total number for simultaneous connections. + + If limit is 0 the connector has no limit. + The default limit size is 100. + """ + return self._limit + + @property + def limit_per_host(self) -> int: + """The limit for simultaneous connections to the same endpoint. + + Endpoints are the same if they are have equal + (host, port, is_ssl) triple. + """ + return self._limit_per_host + + def _cleanup(self) -> None: + """Cleanup unused transports.""" + if self._cleanup_handle: + self._cleanup_handle.cancel() + # _cleanup_handle should be unset, otherwise _release() will not + # recreate it ever! + self._cleanup_handle = None + + now = monotonic() + timeout = self._keepalive_timeout + + if self._conns: + connections = defaultdict(deque) + deadline = now - timeout + for key, conns in self._conns.items(): + alive: deque[tuple[ResponseHandler, float]] = deque() + for proto, use_time in conns: + if proto.is_connected() and use_time - deadline >= 0: + alive.append((proto, use_time)) + continue + transport = proto.transport + proto.close() + if not self._cleanup_closed_disabled and key.is_ssl: + self._cleanup_closed_transports.append(transport) + + if alive: + connections[key] = alive + + self._conns = connections + + if self._conns: + self._cleanup_handle = helpers.weakref_handle( + self, + "_cleanup", + timeout, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + def _cleanup_closed(self) -> None: + """Double confirmation for transport close. + + Some broken ssl servers may leave socket open without proper close. + """ + if self._cleanup_closed_handle: + self._cleanup_closed_handle.cancel() + + for transport in self._cleanup_closed_transports: + if transport is not None: + transport.abort() + + self._cleanup_closed_transports = [] + + if not self._cleanup_closed_disabled: + self._cleanup_closed_handle = helpers.weakref_handle( + self, + "_cleanup_closed", + self._cleanup_closed_period, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + def close(self, *, abort_ssl: bool = False) -> Awaitable[None]: + """Close all opened transports. + + :param abort_ssl: If True, SSL connections will be aborted immediately + without performing the shutdown handshake. This provides + faster cleanup at the cost of less graceful disconnection. + """ + if not (waiters := self._close(abort_ssl=abort_ssl)): + # If there are no connections to close, we can return a noop + # awaitable to avoid scheduling a task on the event loop. + return _DeprecationWaiter(noop()) + coro = _wait_for_close(waiters) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to close connections + # immediately to avoid having to schedule the task on the event loop. + task = asyncio.Task(coro, loop=self._loop, eager_start=True) + else: + task = self._loop.create_task(coro) + return _DeprecationWaiter(task) + + def _close(self, *, abort_ssl: bool = False) -> list[Awaitable[object]]: + waiters: list[Awaitable[object]] = [] + + if self._closed: + return waiters + + self._closed = True + + try: + if self._loop.is_closed(): + return waiters + + # cancel cleanup task + if self._cleanup_handle: + self._cleanup_handle.cancel() + + # cancel cleanup close task + if self._cleanup_closed_handle: + self._cleanup_closed_handle.cancel() + + for data in self._conns.values(): + for proto, _ in data: + if ( + abort_ssl + and proto.transport + and proto.transport.get_extra_info("sslcontext") is not None + ): + proto.abort() + else: + proto.close() + if closed := proto.closed: + waiters.append(closed) + + for proto in self._acquired: + if ( + abort_ssl + and proto.transport + and proto.transport.get_extra_info("sslcontext") is not None + ): + proto.abort() + else: + proto.close() + if closed := proto.closed: + waiters.append(closed) + + for transport in self._cleanup_closed_transports: + if transport is not None: + transport.abort() + + return waiters + + finally: + self._conns.clear() + self._acquired.clear() + for keyed_waiters in self._waiters.values(): + for keyed_waiter in keyed_waiters: + keyed_waiter.cancel() + self._waiters.clear() + self._cleanup_handle = None + self._cleanup_closed_transports.clear() + self._cleanup_closed_handle = None + + @property + def closed(self) -> bool: + """Is connector closed. + + A readonly property. + """ + return self._closed + + def _available_connections(self, key: "ConnectionKey") -> int: + """ + Return number of available connections. + + The limit, limit_per_host and the connection key are taken into account. + + If it returns less than 1 means that there are no connections + available. + """ + # check total available connections + # If there are no limits, this will always return 1 + total_remain = 1 + + if self._limit and (total_remain := self._limit - len(self._acquired)) <= 0: + return total_remain + + # check limit per host + if host_remain := self._limit_per_host: + if acquired := self._acquired_per_host.get(key): + host_remain -= len(acquired) + if total_remain > host_remain: + return host_remain + + return total_remain + + def _update_proxy_auth_header_and_build_proxy_req( + self, req: ClientRequest + ) -> ClientRequest: + """Set Proxy-Authorization header for non-SSL proxy requests and builds the proxy request for SSL proxy requests.""" + url = req.proxy + assert url is not None + headers: dict[str, str] = {} + if req.proxy_headers is not None: + headers = req.proxy_headers # type: ignore[assignment] + headers[hdrs.HOST] = req.headers[hdrs.HOST] + proxy_req = ClientRequest( + hdrs.METH_GET, + url, + headers=headers, + auth=req.proxy_auth, + loop=self._loop, + ssl=req.ssl, + ) + auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None) + if auth is not None: + if not req.is_ssl(): + req.headers[hdrs.PROXY_AUTHORIZATION] = auth + else: + proxy_req.headers[hdrs.PROXY_AUTHORIZATION] = auth + return proxy_req + + async def connect( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> Connection: + """Get from pool or create new connection.""" + key = req.connection_key + if (conn := await self._get(key, traces)) is not None: + # If we do not have to wait and we can get a connection from the pool + # we can avoid the timeout ceil logic and directly return the connection + if req.proxy: + self._update_proxy_auth_header_and_build_proxy_req(req) + return conn + + async with ceil_timeout(timeout.connect, timeout.ceil_threshold): + if self._available_connections(key) <= 0: + await self._wait_for_available_connection(key, traces) + if (conn := await self._get(key, traces)) is not None: + if req.proxy: + self._update_proxy_auth_header_and_build_proxy_req(req) + return conn + + placeholder = cast( + ResponseHandler, _TransportPlaceholder(self._placeholder_future) + ) + self._acquired.add(placeholder) + if self._limit_per_host: + self._acquired_per_host[key].add(placeholder) + + try: + # Traces are done inside the try block to ensure that the + # that the placeholder is still cleaned up if an exception + # is raised. + if traces: + for trace in traces: + await trace.send_connection_create_start() + proto = await self._create_connection(req, traces, timeout) + if traces: + for trace in traces: + await trace.send_connection_create_end() + except BaseException: + self._release_acquired(key, placeholder) + raise + else: + if self._closed: + proto.close() + raise ClientConnectionError("Connector is closed.") + + # The connection was successfully created, drop the placeholder + # and add the real connection to the acquired set. There should + # be no awaits after the proto is added to the acquired set + # to ensure that the connection is not left in the acquired set + # on cancellation. + self._acquired.remove(placeholder) + self._acquired.add(proto) + if self._limit_per_host: + acquired_per_host = self._acquired_per_host[key] + acquired_per_host.remove(placeholder) + acquired_per_host.add(proto) + return Connection(self, key, proto, self._loop) + + async def _wait_for_available_connection( + self, key: "ConnectionKey", traces: list["Trace"] + ) -> None: + """Wait for an available connection slot.""" + # We loop here because there is a race between + # the connection limit check and the connection + # being acquired. If the connection is acquired + # between the check and the await statement, we + # need to loop again to check if the connection + # slot is still available. + attempts = 0 + while True: + fut: asyncio.Future[None] = self._loop.create_future() + keyed_waiters = self._waiters[key] + keyed_waiters[fut] = None + if attempts: + # If we have waited before, we need to move the waiter + # to the front of the queue as otherwise we might get + # starved and hit the timeout. + keyed_waiters.move_to_end(fut, last=False) + + try: + # Traces happen in the try block to ensure that the + # the waiter is still cleaned up if an exception is raised. + if traces: + for trace in traces: + await trace.send_connection_queued_start() + await fut + if traces: + for trace in traces: + await trace.send_connection_queued_end() + finally: + # pop the waiter from the queue if its still + # there and not already removed by _release_waiter + keyed_waiters.pop(fut, None) + if not self._waiters.get(key, True): + del self._waiters[key] + + if self._available_connections(key) > 0: + break + attempts += 1 + + async def _get( + self, key: "ConnectionKey", traces: list["Trace"] + ) -> Connection | None: + """Get next reusable connection for the key or None. + + The connection will be marked as acquired. + """ + if (conns := self._conns.get(key)) is None: + return None + + t1 = monotonic() + while conns: + proto, t0 = conns.popleft() + # We will we reuse the connection if its connected and + # the keepalive timeout has not been exceeded + if proto.is_connected() and t1 - t0 <= self._keepalive_timeout: + if not conns: + # The very last connection was reclaimed: drop the key + del self._conns[key] + self._acquired.add(proto) + if self._limit_per_host: + self._acquired_per_host[key].add(proto) + if traces: + for trace in traces: + try: + await trace.send_connection_reuseconn() + except BaseException: + self._release_acquired(key, proto) + raise + return Connection(self, key, proto, self._loop) + + # Connection cannot be reused, close it + transport = proto.transport + proto.close() + # only for SSL transports + if not self._cleanup_closed_disabled and key.is_ssl: + self._cleanup_closed_transports.append(transport) + + # No more connections: drop the key + del self._conns[key] + return None + + def _release_waiter(self) -> None: + """ + Iterates over all waiters until one to be released is found. + + The one to be released is not finished and + belongs to a host that has available connections. + """ + if not self._waiters: + return + + # Having the dict keys ordered this avoids to iterate + # at the same order at each call. + queues = list(self._waiters) + random.shuffle(queues) + + for key in queues: + if self._available_connections(key) < 1: + continue + + waiters = self._waiters[key] + while waiters: + waiter, _ = waiters.popitem(last=False) + if not waiter.done(): + waiter.set_result(None) + return + + def _release_acquired(self, key: "ConnectionKey", proto: ResponseHandler) -> None: + """Release acquired connection.""" + if self._closed: + # acquired connection is already released on connector closing + return + + self._acquired.discard(proto) + if self._limit_per_host and (conns := self._acquired_per_host.get(key)): + conns.discard(proto) + if not conns: + del self._acquired_per_host[key] + self._release_waiter() + + def _release( + self, + key: "ConnectionKey", + protocol: ResponseHandler, + *, + should_close: bool = False, + ) -> None: + if self._closed: + # acquired connection is already released on connector closing + return + + self._release_acquired(key, protocol) + + if self._force_close or should_close or protocol.should_close: + transport = protocol.transport + protocol.close() + + if key.is_ssl and not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transport) + return + + self._conns[key].append((protocol, monotonic())) + + if self._cleanup_handle is None: + self._cleanup_handle = helpers.weakref_handle( + self, + "_cleanup", + self._keepalive_timeout, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + async def _create_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + raise NotImplementedError() + + +class _DNSCacheTable: + def __init__(self, ttl: float | None = None, max_size: int = 1000) -> None: + self._addrs_rr: OrderedDict[ + tuple[str, int], tuple[Iterator[ResolveResult], int] + ] = OrderedDict() + self._timestamps: dict[tuple[str, int], float] = {} + self._ttl = ttl + self._max_size = max_size + + def __contains__(self, host: object) -> bool: + return host in self._addrs_rr + + def add(self, key: tuple[str, int], addrs: list[ResolveResult]) -> None: + if key in self._addrs_rr: + self._addrs_rr.move_to_end(key) + + self._addrs_rr[key] = (cycle(addrs), len(addrs)) + + if self._ttl is not None: + self._timestamps[key] = monotonic() + + if len(self._addrs_rr) > self._max_size: + oldest_key, _ = self._addrs_rr.popitem(last=False) + self._timestamps.pop(oldest_key, None) + + def remove(self, key: tuple[str, int]) -> None: + self._addrs_rr.pop(key, None) + self._timestamps.pop(key, None) + + def clear(self) -> None: + self._addrs_rr.clear() + self._timestamps.clear() + + def next_addrs(self, key: tuple[str, int]) -> list[ResolveResult]: + loop, length = self._addrs_rr[key] + addrs = list(islice(loop, length)) + # Consume one more element to shift internal state of `cycle` + next(loop) + self._addrs_rr.move_to_end(key) + return addrs + + def expired(self, key: tuple[str, int]) -> bool: + if self._ttl is None: + return False + + return self._timestamps[key] + self._ttl < monotonic() + + +def _make_ssl_context(verified: bool) -> SSLContext: + """Create SSL context. + + This method is not async-friendly and should be called from a thread + because it will load certificates from disk and do other blocking I/O. + """ + if ssl is None: + # No ssl support + return None + if verified: + sslcontext = ssl.create_default_context() + else: + sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.options |= ssl.OP_NO_SSLv3 + sslcontext.check_hostname = False + sslcontext.verify_mode = ssl.CERT_NONE + sslcontext.options |= ssl.OP_NO_COMPRESSION + sslcontext.set_default_verify_paths() + sslcontext.set_alpn_protocols(("http/1.1",)) + return sslcontext + + +# The default SSLContext objects are created at import time +# since they do blocking I/O to load certificates from disk, +# and imports should always be done before the event loop starts +# or in a thread. +_SSL_CONTEXT_VERIFIED = _make_ssl_context(True) +_SSL_CONTEXT_UNVERIFIED = _make_ssl_context(False) + + +class TCPConnector(BaseConnector): + """TCP connector. + + verify_ssl - Set to True to check ssl certifications. + fingerprint - Pass the binary sha256 + digest of the expected certificate in DER format to verify + that the certificate the server presents matches. See also + https://en.wikipedia.org/wiki/HTTP_Public_Key_Pinning + resolver - Enable DNS lookups and use this + resolver + use_dns_cache - Use memory cache for DNS lookups. + ttl_dns_cache - Max seconds having cached a DNS entry, None forever. + family - socket address family + local_addr - local tuple of (host, port) to bind socket to + + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + enable_cleanup_closed - Enables clean-up closed ssl transports. + Disabled by default. + happy_eyeballs_delay - This is the β€œConnection Attempt Delay” + as defined in RFC 8305. To disable + the happy eyeballs algorithm, set to None. + interleave - β€œFirst Address Family Count” as defined in RFC 8305 + loop - Optional event loop. + socket_factory - A SocketFactoryType function that, if supplied, + will be used to create sockets given an + AddrInfoType. + ssl_shutdown_timeout - DEPRECATED. Will be removed in aiohttp 4.0. + Grace period for SSL shutdown handshake on TLS + connections. Default is 0 seconds (immediate abort). + This parameter allowed for a clean SSL shutdown by + notifying the remote peer of connection closure, + while avoiding excessive delays during connector cleanup. + Note: Only takes effect on Python 3.11+. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"tcp"}) + + def __init__( + self, + *, + verify_ssl: bool = True, + fingerprint: bytes | None = None, + use_dns_cache: bool = True, + ttl_dns_cache: int | None = 10, + dns_cache_max_size: int = 1000, + family: socket.AddressFamily = socket.AddressFamily.AF_UNSPEC, + ssl_context: SSLContext | None = None, + ssl: bool | Fingerprint | SSLContext = True, + local_addr: tuple[str, int] | None = None, + resolver: AbstractResolver | None = None, + keepalive_timeout: None | float | object = sentinel, + force_close: bool = False, + limit: int = 100, + limit_per_host: int = 0, + enable_cleanup_closed: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + timeout_ceil_threshold: float = 5, + happy_eyeballs_delay: float | None = 0.25, + interleave: int | None = None, + socket_factory: SocketFactoryType | None = None, + ssl_shutdown_timeout: _SENTINEL | None | float = sentinel, + ): + super().__init__( + keepalive_timeout=keepalive_timeout, + force_close=force_close, + limit=limit, + limit_per_host=limit_per_host, + enable_cleanup_closed=enable_cleanup_closed, + loop=loop, + timeout_ceil_threshold=timeout_ceil_threshold, + ) + + self._ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + self._resolver: AbstractResolver + if resolver is None: + self._resolver = DefaultResolver(loop=self._loop) + self._resolver_owner = True + else: + self._resolver = resolver + self._resolver_owner = False + + self._use_dns_cache = use_dns_cache + self._cached_hosts = _DNSCacheTable( + ttl=ttl_dns_cache, max_size=dns_cache_max_size + ) + self._throttle_dns_futures: dict[tuple[str, int], set[asyncio.Future[None]]] = ( + {} + ) + self._family = family + self._local_addr_infos = aiohappyeyeballs.addr_to_addr_infos(local_addr) + self._happy_eyeballs_delay = happy_eyeballs_delay + self._interleave = interleave + self._resolve_host_tasks: set[asyncio.Task[list[ResolveResult]]] = set() + self._socket_factory = socket_factory + self._ssl_shutdown_timeout: float | None + # Handle ssl_shutdown_timeout with warning for Python < 3.11 + if ssl_shutdown_timeout is sentinel: + self._ssl_shutdown_timeout = 0 + else: + # Deprecation warning for ssl_shutdown_timeout parameter + warnings.warn( + "The ssl_shutdown_timeout parameter is deprecated and will be removed in aiohttp 4.0", + DeprecationWarning, + stacklevel=2, + ) + if ( + sys.version_info < (3, 11) + and ssl_shutdown_timeout is not None + and ssl_shutdown_timeout != 0 + ): + warnings.warn( + f"ssl_shutdown_timeout={ssl_shutdown_timeout} is ignored on Python < 3.11; " + "only ssl_shutdown_timeout=0 is supported. The timeout will be ignored.", + RuntimeWarning, + stacklevel=2, + ) + self._ssl_shutdown_timeout = ssl_shutdown_timeout + + def _close(self, *, abort_ssl: bool = False) -> list[Awaitable[object]]: + """Close all ongoing DNS calls.""" + for fut in chain.from_iterable(self._throttle_dns_futures.values()): + fut.cancel() + + waiters = super()._close(abort_ssl=abort_ssl) + + for t in self._resolve_host_tasks: + t.cancel() + waiters.append(t) + + return waiters + + async def close(self, *, abort_ssl: bool = False) -> None: + """ + Close all opened transports. + + :param abort_ssl: If True, SSL connections will be aborted immediately + without performing the shutdown handshake. If False (default), + the behavior is determined by ssl_shutdown_timeout: + - If ssl_shutdown_timeout=0: connections are aborted + - If ssl_shutdown_timeout>0: graceful shutdown is performed + """ + # Use abort_ssl param if explicitly set, otherwise use ssl_shutdown_timeout default + await super().close(abort_ssl=abort_ssl or self._ssl_shutdown_timeout == 0) + if self._resolver_owner: + await self._resolver.close() + + @property + def family(self) -> int: + """Socket family like AF_INET.""" + return self._family + + @property + def use_dns_cache(self) -> bool: + """True if local DNS caching is enabled.""" + return self._use_dns_cache + + def clear_dns_cache(self, host: str | None = None, port: int | None = None) -> None: + """Remove specified host/port or clear all dns local cache.""" + if host is not None and port is not None: + self._cached_hosts.remove((host, port)) + elif host is not None or port is not None: + raise ValueError("either both host and port or none of them are allowed") + else: + self._cached_hosts.clear() + + async def _resolve_host( + self, host: str, port: int, traces: Sequence["Trace"] | None = None + ) -> list[ResolveResult]: + """Resolve host and return list of addresses.""" + if is_ip_address(host): + # Reject legacy numeric IPv4 forms (e.g. 2130706433, 127.1) that + # socket would map onto an address, slipping past a connector-level + # policy that only sees the raw host. + if ":" not in host and not is_canonical_ipv4_address(host): + raise InvalidUrlClientError(host, "is not a canonical IPv4 address") + return [ + { + "hostname": host, + "host": host, + "port": port, + "family": self._family, + "proto": 0, + "flags": 0, + } + ] + + if not self._use_dns_cache: + + if traces: + for trace in traces: + await trace.send_dns_resolvehost_start(host) + + if self._closed: + raise ClientConnectionError("Connector is closed") + + res = await self._resolver.resolve(host, port, family=self._family) + + if traces: + for trace in traces: + await trace.send_dns_resolvehost_end(host) + + return res + + key = (host, port) + if key in self._cached_hosts and not self._cached_hosts.expired(key): + # get result early, before any await (#4014) + result = self._cached_hosts.next_addrs(key) + + if traces: + for trace in traces: + await trace.send_dns_cache_hit(host) + return result + + futures: set[asyncio.Future[None]] + # + # If multiple connectors are resolving the same host, we wait + # for the first one to resolve and then use the result for all of them. + # We use a throttle to ensure that we only resolve the host once + # and then use the result for all the waiters. + # + if key in self._throttle_dns_futures: + # get futures early, before any await (#4014) + futures = self._throttle_dns_futures[key] + future: asyncio.Future[None] = self._loop.create_future() + futures.add(future) + if traces: + for trace in traces: + await trace.send_dns_cache_hit(host) + try: + await future + finally: + futures.discard(future) + return self._cached_hosts.next_addrs(key) + + # update dict early, before any await (#4014) + self._throttle_dns_futures[key] = futures = set() + # In this case we need to create a task to ensure that we can shield + # the task from cancellation as cancelling this lookup should not cancel + # the underlying lookup or else the cancel event will get broadcast to + # all the waiters across all connections. + # + coro = self._resolve_host_with_throttle(key, host, port, futures, traces) + loop = asyncio.get_running_loop() + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send immediately + resolved_host_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + resolved_host_task = loop.create_task(coro) + + if not resolved_host_task.done(): + self._resolve_host_tasks.add(resolved_host_task) + resolved_host_task.add_done_callback(self._resolve_host_tasks.discard) + + try: + return await asyncio.shield(resolved_host_task) + except asyncio.CancelledError: + + def drop_exception(fut: "asyncio.Future[list[ResolveResult]]") -> None: + with suppress(Exception, asyncio.CancelledError): + fut.result() + + resolved_host_task.add_done_callback(drop_exception) + raise + + async def _resolve_host_with_throttle( + self, + key: tuple[str, int], + host: str, + port: int, + futures: set["asyncio.Future[None]"], + traces: Sequence["Trace"] | None, + ) -> list[ResolveResult]: + """Resolve host and set result for all waiters. + + This method must be run in a task and shielded from cancellation + to avoid cancelling the underlying lookup. + """ + try: + if traces: + for trace in traces: + await trace.send_dns_cache_miss(host) + + for trace in traces: + await trace.send_dns_resolvehost_start(host) + + addrs = await self._resolver.resolve(host, port, family=self._family) + if traces: + for trace in traces: + await trace.send_dns_resolvehost_end(host) + + self._cached_hosts.add(key, addrs) + for fut in futures: + set_result(fut, None) + except BaseException as e: + # any DNS exception is set for the waiters to raise the same exception. + # This coro is always run in task that is shielded from cancellation so + # we should never be propagating cancellation here. + for fut in futures: + set_exception(fut, e) + raise + finally: + self._throttle_dns_futures.pop(key) + + return self._cached_hosts.next_addrs(key) + + async def _create_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + """Create connection. + + Has same keyword arguments as BaseEventLoop.create_connection. + """ + if req.proxy: + _, proto = await self._create_proxy_connection(req, traces, timeout) + else: + _, proto = await self._create_direct_connection(req, traces, timeout) + + return proto + + def _get_ssl_context(self, req: ClientRequest) -> SSLContext | None: + """Logic to get the correct SSL context + + 0. if req.ssl is false, return None + + 1. if ssl_context is specified in req, use it + 2. if _ssl_context is specified in self, use it + 3. otherwise: + 1. if verify_ssl is not specified in req, use self.ssl_context + (will generate a default context according to self.verify_ssl) + 2. if verify_ssl is True in req, generate a default SSL context + 3. if verify_ssl is False in req, generate a SSL context that + won't verify + """ + if not req.is_ssl(): + return None + + if ssl is None: # pragma: no cover + raise RuntimeError("SSL is not supported.") + sslcontext = req.ssl + if isinstance(sslcontext, ssl.SSLContext): + return sslcontext + if sslcontext is not True: + # not verified or fingerprinted + return _SSL_CONTEXT_UNVERIFIED + sslcontext = self._ssl + if isinstance(sslcontext, ssl.SSLContext): + return sslcontext + if sslcontext is not True: + # not verified or fingerprinted + return _SSL_CONTEXT_UNVERIFIED + return _SSL_CONTEXT_VERIFIED + + def _get_fingerprint(self, req: ClientRequest) -> Optional["Fingerprint"]: + ret = req.ssl + if isinstance(ret, Fingerprint): + return ret + ret = self._ssl + if isinstance(ret, Fingerprint): + return ret + return None + + async def _wrap_create_connection( + self, + *args: Any, + addr_infos: list[AddrInfoType], + req: ClientRequest, + timeout: "ClientTimeout", + client_error: type[Exception] = ClientConnectorError, + **kwargs: Any, + ) -> tuple[asyncio.Transport, ResponseHandler]: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + sock = await aiohappyeyeballs.start_connection( + addr_infos=addr_infos, + local_addr_infos=self._local_addr_infos, + happy_eyeballs_delay=self._happy_eyeballs_delay, + interleave=self._interleave, + loop=self._loop, + socket_factory=self._socket_factory, + ) + # Add ssl_shutdown_timeout for Python 3.11+ when SSL is used + if ( + kwargs.get("ssl") + and self._ssl_shutdown_timeout + and sys.version_info >= (3, 11) + ): + kwargs["ssl_shutdown_timeout"] = self._ssl_shutdown_timeout + return await self._loop.create_connection(*args, **kwargs, sock=sock) + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + + async def _wrap_existing_connection( + self, + *args: Any, + req: ClientRequest, + timeout: "ClientTimeout", + client_error: type[Exception] = ClientConnectorError, + **kwargs: Any, + ) -> tuple[asyncio.Transport, ResponseHandler]: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + return await self._loop.create_connection(*args, **kwargs) + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + + def _fail_on_no_start_tls(self, req: "ClientRequest") -> None: + """Raise a :py:exc:`RuntimeError` on missing ``start_tls()``. + + It is necessary for TLS-in-TLS so that it is possible to + send HTTPS queries through HTTPS proxies. + + This doesn't affect regular HTTP requests, though. + """ + if not req.is_ssl(): + return + + proxy_url = req.proxy + assert proxy_url is not None + if proxy_url.scheme != "https": + return + + self._check_loop_for_start_tls() + + def _check_loop_for_start_tls(self) -> None: + try: + self._loop.start_tls + except AttributeError as attr_exc: + raise RuntimeError( + "An HTTPS request is being sent through an HTTPS proxy. " + "This needs support for TLS in TLS but it is not implemented " + "in your runtime for the stdlib asyncio.\n\n" + "Please upgrade to Python 3.11 or higher. For more details, " + "please see:\n" + "* https://bugs.python.org/issue37179\n" + "* https://github.com/python/cpython/pull/28073\n" + "* https://docs.aiohttp.org/en/stable/" + "client_advanced.html#proxy-support\n" + "* https://github.com/aio-libs/aiohttp/discussions/6044\n", + ) from attr_exc + + def _loop_supports_start_tls(self) -> bool: + try: + self._check_loop_for_start_tls() + except RuntimeError: + return False + else: + return True + + def _warn_about_tls_in_tls( + self, + underlying_transport: asyncio.Transport, + req: ClientRequest, + ) -> None: + """Issue a warning if the requested URL has HTTPS scheme.""" + if req.request_info.url.scheme != "https": + return + + # TLS-in-TLS only applies when the proxy itself is HTTPS. + # When the proxy is HTTP, start_tls upgrades a plain TCP connection, + # which is standard TLS and works on all event loops and Python versions. + if req.proxy is None or req.proxy.scheme != "https": + return + + # Check if uvloop is being used, which supports TLS in TLS, + # otherwise assume that asyncio's native transport is being used. + if type(underlying_transport).__module__.startswith("uvloop"): + return + + # Support in asyncio was added in Python 3.11 (bpo-44011) + asyncio_supports_tls_in_tls = sys.version_info >= (3, 11) or getattr( + underlying_transport, + "_start_tls_compatible", + False, + ) + + if asyncio_supports_tls_in_tls: + return + + warnings.warn( + "An HTTPS request is being sent through an HTTPS proxy. " + "This support for TLS in TLS is known to be disabled " + "in the stdlib asyncio (Python <3.11). This is why you'll probably see " + "an error in the log below.\n\n" + "It is possible to enable it via monkeypatching. " + "For more details, see:\n" + "* https://bugs.python.org/issue37179\n" + "* https://github.com/python/cpython/pull/28073\n\n" + "You can temporarily patch this as follows:\n" + "* https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support\n" + "* https://github.com/aio-libs/aiohttp/discussions/6044\n", + RuntimeWarning, + source=self, + # Why `4`? At least 3 of the calls in the stack originate + # from the methods in this class. + stacklevel=3, + ) + + async def _start_tls_connection( + self, + underlying_transport: asyncio.Transport, + req: ClientRequest, + timeout: "ClientTimeout", + client_error: type[Exception] = ClientConnectorError, + ) -> tuple[asyncio.BaseTransport, ResponseHandler]: + """Wrap the raw TCP transport with TLS.""" + tls_proto = self._factory() # Create a brand new proto for TLS + sslcontext = self._get_ssl_context(req) + if TYPE_CHECKING: + # _start_tls_connection is unreachable in the current code path + # if sslcontext is None. + assert sslcontext is not None + + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + try: + # ssl_shutdown_timeout is only available in Python 3.11+ + if sys.version_info >= (3, 11) and self._ssl_shutdown_timeout: + tls_transport = await self._loop.start_tls( + underlying_transport, + tls_proto, + sslcontext, + server_hostname=req.server_hostname or req.host, + ssl_handshake_timeout=timeout.total or None, + ssl_shutdown_timeout=self._ssl_shutdown_timeout, + ) + else: + tls_transport = await self._loop.start_tls( + underlying_transport, + tls_proto, + sslcontext, + server_hostname=req.server_hostname or req.host, + ssl_handshake_timeout=timeout.total or None, + ) + except BaseException: + # We need to close the underlying transport since + # `start_tls()` probably failed before it had a + # chance to do this: + if self._ssl_shutdown_timeout == 0: + underlying_transport.abort() + else: + underlying_transport.close() + raise + if isinstance(tls_transport, asyncio.Transport): + fingerprint = self._get_fingerprint(req) + if fingerprint: + try: + fingerprint.check(tls_transport) + except ServerFingerprintMismatch: + tls_transport.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(tls_transport) + raise + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + except TypeError as type_err: + # Example cause looks like this: + # TypeError: transport is not supported by start_tls() + + raise ClientConnectionError( + "Cannot initialize a TLS-in-TLS connection to host " + f"{req.host!s}:{req.port:d} through an underlying connection " + f"to an HTTPS proxy {req.proxy!s} ssl:{req.ssl or 'default'} " + f"[{type_err!s}]" + ) from type_err + else: + if tls_transport is None: + msg = "Failed to start TLS (possibly caused by closing transport)" + raise client_error(req.connection_key, OSError(msg)) + tls_proto.connection_made( + tls_transport + ) # Kick the state machine of the new TLS protocol + + return tls_transport, tls_proto + + def _convert_hosts_to_addr_infos( + self, hosts: list[ResolveResult] + ) -> list[AddrInfoType]: + """Converts the list of hosts to a list of addr_infos. + + The list of hosts is the result of a DNS lookup. The list of + addr_infos is the result of a call to `socket.getaddrinfo()`. + """ + addr_infos: list[AddrInfoType] = [] + for hinfo in hosts: + host = hinfo["host"] + is_ipv6 = ":" in host + family = socket.AF_INET6 if is_ipv6 else socket.AF_INET + if self._family and self._family != family: + continue + addr = (host, hinfo["port"], 0, 0) if is_ipv6 else (host, hinfo["port"]) + addr_infos.append( + (family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", addr) + ) + return addr_infos + + async def _create_direct_connection( + self, + req: ClientRequest, + traces: list["Trace"], + timeout: "ClientTimeout", + *, + client_error: type[Exception] = ClientConnectorError, + ) -> tuple[asyncio.Transport, ResponseHandler]: + sslcontext = self._get_ssl_context(req) + fingerprint = self._get_fingerprint(req) + + host = req.url.raw_host + assert host is not None + # Replace multiple trailing dots with a single one. + # A trailing dot is only present for fully-qualified domain names. + # See https://github.com/aio-libs/aiohttp/pull/7364. + if host.endswith(".."): + host = host.rstrip(".") + "." + port = req.port + assert port is not None + try: + # Cancelling this lookup should not cancel the underlying lookup + # or else the cancel event will get broadcast to all the waiters + # across all connections. + hosts = await self._resolve_host(host, port, traces=traces) + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + # in case of proxy it is not ClientProxyConnectionError + # it is problem of resolving proxy ip itself + raise ClientConnectorDNSError(req.connection_key, exc) from exc + + last_exc: Exception | None = None + addr_infos = self._convert_hosts_to_addr_infos(hosts) + while addr_infos: + # Strip trailing dots, certificates contain FQDN without dots. + # See https://github.com/aio-libs/aiohttp/issues/3636 + server_hostname = ( + (req.server_hostname or host).rstrip(".") if sslcontext else None + ) + + try: + transp, proto = await self._wrap_create_connection( + self._factory, + timeout=timeout, + ssl=sslcontext, + addr_infos=addr_infos, + server_hostname=server_hostname, + req=req, + client_error=client_error, + ) + except (ClientConnectorError, asyncio.TimeoutError) as exc: + last_exc = exc + aiohappyeyeballs.pop_addr_infos_interleave(addr_infos, self._interleave) + continue + + if req.is_ssl() and fingerprint: + try: + fingerprint.check(transp) + except ServerFingerprintMismatch as exc: + transp.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transp) + last_exc = exc + # Remove the bad peer from the list of addr_infos + sock: socket.socket = transp.get_extra_info("socket") + bad_peer = sock.getpeername() + aiohappyeyeballs.remove_addr_infos(addr_infos, bad_peer) + continue + + return transp, proto + else: + assert last_exc is not None + raise last_exc + + async def _create_proxy_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> tuple[asyncio.BaseTransport, ResponseHandler]: + self._fail_on_no_start_tls(req) + runtime_has_start_tls = self._loop_supports_start_tls() + proxy_req = self._update_proxy_auth_header_and_build_proxy_req(req) + + # create connection to proxy server + transport, proto = await self._create_direct_connection( + proxy_req, [], timeout, client_error=ClientProxyConnectionError + ) + + if req.is_ssl(): + if runtime_has_start_tls: + self._warn_about_tls_in_tls(transport, req) + + # For HTTPS requests over HTTP proxy + # we must notify proxy to tunnel connection + # so we send CONNECT command: + # CONNECT www.python.org:443 HTTP/1.1 + # Host: www.python.org + # + # next we must do TLS handshake and so on + # to do this we must wrap raw socket into secure one + # asyncio handles this perfectly + proxy_req.method = hdrs.METH_CONNECT + proxy_req.url = req.url + key = req.connection_key._replace( + proxy=None, proxy_auth=None, proxy_headers_hash=None + ) + conn = _ConnectTunnelConnection(self, key, proto, self._loop) + proxy_resp = await proxy_req.send(conn) + try: + protocol = conn._protocol + assert protocol is not None + + # read_until_eof=True will ensure the connection isn't closed + # once the response is received and processed allowing + # START_TLS to work on the connection below. + protocol.set_response_params( + read_until_eof=runtime_has_start_tls, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + resp = await proxy_resp.start(conn) + except BaseException: + proxy_resp.close() + conn.close() + raise + else: + conn._protocol = None + try: + if resp.status != 200: + message = resp.reason + if message is None: + message = HTTPStatus(resp.status).phrase + raise ClientHttpProxyError( + proxy_resp.request_info, + resp.history, + status=resp.status, + message=message, + headers=resp.headers, + ) + if not runtime_has_start_tls: + rawsock = transport.get_extra_info("socket", default=None) + if rawsock is None: + raise RuntimeError( + "Transport does not expose socket instance" + ) + # Duplicate the socket, so now we can close proxy transport + rawsock = rawsock.dup() + except BaseException: + # It shouldn't be closed in `finally` because it's fed to + # `loop.start_tls()` and the docs say not to touch it after + # passing there. + transport.close() + raise + finally: + if not runtime_has_start_tls: + transport.close() + + if not runtime_has_start_tls: + # HTTP proxy with support for upgrade to HTTPS + sslcontext = self._get_ssl_context(req) + return await self._wrap_existing_connection( + self._factory, + timeout=timeout, + ssl=sslcontext, + sock=rawsock, + server_hostname=req.host, + req=req, + ) + + return await self._start_tls_connection( + # Access the old transport for the last time before it's + # closed and forgotten forever: + transport, + req=req, + timeout=timeout, + ) + finally: + proxy_resp.close() + + return transport, proto + + +class UnixConnector(BaseConnector): + """Unix socket connector. + + path - Unix socket path. + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + loop - Optional event loop. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"unix"}) + + def __init__( + self, + path: str, + force_close: bool = False, + keepalive_timeout: object | float | None = sentinel, + limit: int = 100, + limit_per_host: int = 0, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + super().__init__( + force_close=force_close, + keepalive_timeout=keepalive_timeout, + limit=limit, + limit_per_host=limit_per_host, + loop=loop, + ) + self._path = path + + @property + def path(self) -> str: + """Path to unix socket.""" + return self._path + + async def _create_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + _, proto = await self._loop.create_unix_connection( + self._factory, self._path + ) + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise UnixClientConnectorError(self.path, req.connection_key, exc) from exc + + return proto + + +class NamedPipeConnector(BaseConnector): + """Named pipe connector. + + Only supported by the proactor event loop. + See also: https://docs.python.org/3/library/asyncio-eventloop.html + + path - Windows named pipe path. + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + loop - Optional event loop. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"npipe"}) + + def __init__( + self, + path: str, + force_close: bool = False, + keepalive_timeout: object | float | None = sentinel, + limit: int = 100, + limit_per_host: int = 0, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + super().__init__( + force_close=force_close, + keepalive_timeout=keepalive_timeout, + limit=limit, + limit_per_host=limit_per_host, + loop=loop, + ) + if not isinstance( + self._loop, + asyncio.ProactorEventLoop, # type: ignore[attr-defined] + ): + raise RuntimeError( + "Named Pipes only available in proactor loop under windows" + ) + self._path = path + + @property + def path(self) -> str: + """Path to the named pipe.""" + return self._path + + async def _create_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + _, proto = await self._loop.create_pipe_connection( # type: ignore[attr-defined] + self._factory, self._path + ) + # the drain is required so that the connection_made is called + # and transport is set otherwise it is not set before the + # `assert conn.transport is not None` + # in client.py's _request method + await asyncio.sleep(0) + # other option is to manually set transport like + # `proto.transport = trans` + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise ClientConnectorError(req.connection_key, exc) from exc + + return cast(ResponseHandler, proto) diff --git a/venv/Lib/site-packages/aiohttp/cookiejar.py b/venv/Lib/site-packages/aiohttp/cookiejar.py new file mode 100644 index 0000000000000000000000000000000000000000..d7b91887a5d6e90de48aeeaad34b55cb693b1089 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/cookiejar.py @@ -0,0 +1,678 @@ +import asyncio +import calendar +import contextlib +import datetime +import heapq +import itertools +import json +import os +import pathlib +import pickle +import re +import time +import warnings +from collections import defaultdict +from collections.abc import Iterable, Iterator, Mapping +from http.cookies import BaseCookie, Morsel, SimpleCookie +from types import MappingProxyType +from typing import Union + +from yarl import URL + +from ._cookie_helpers import preserve_morsel_with_coded_value +from .abc import AbstractCookieJar, ClearCookiePredicate +from .helpers import is_ip_address +from .typedefs import LooseCookies, PathLike, StrOrURL + +__all__ = ("CookieJar", "DummyCookieJar") + + +CookieItem = Union[str, "Morsel[str]"] + +# We cache these string methods here as their use is in performance critical code. +_FORMAT_PATH = "{}/{}".format +_FORMAT_DOMAIN_REVERSED = "{1}.{0}".format + +# The minimum number of scheduled cookie expirations before we start cleaning up +# the expiration heap. This is a performance optimization to avoid cleaning up the +# heap too often when there are only a few scheduled expirations. +_MIN_SCHEDULED_COOKIE_EXPIRATION = 100 +_SIMPLE_COOKIE = SimpleCookie() + +# Not persisted; the absolute deadline is saved instead. +_RELATIVE_EXPIRY_ATTRS = frozenset(("max-age", "expires")) + + +class _RestrictedCookieUnpickler(pickle._Unpickler): + """A restricted unpickler that only allows cookie-related types. + + This prevents arbitrary code execution when loading pickled cookie data + from untrusted sources. Only types that are expected in a serialized + CookieJar are permitted. + + Subclasses :class:`pickle._Unpickler` (the pure-Python implementation) + rather than :class:`pickle.Unpickler` because the accelerated unpickler + on some implementations (notably PyPy) does not dispatch through + :meth:`find_class` overrides. + + See: https://docs.python.org/3/library/pickle.html#restricting-globals + """ + + _ALLOWED_CLASSES: frozenset[tuple[str, str]] = frozenset( + { + # Core cookie types + ("http.cookies", "SimpleCookie"), + ("http.cookies", "Morsel"), + # Container types used by CookieJar._cookies + ("collections", "defaultdict"), + # builtins that pickle uses for reconstruction + ("builtins", "tuple"), + ("builtins", "set"), + ("builtins", "frozenset"), + ("builtins", "dict"), + } + ) + + def find_class(self, module: str, name: str) -> type: + if (module, name) not in self._ALLOWED_CLASSES: + raise pickle.UnpicklingError( + f"Forbidden class: {module}.{name}. " + "CookieJar.load() only allows cookie-related types for security. " + "See https://docs.python.org/3/library/pickle.html#restricting-globals" + ) + return super().find_class(module, name) # type: ignore[no-any-return] + + +class CookieJar(AbstractCookieJar): + """Implements cookie storage adhering to RFC 6265.""" + + DATE_TOKENS_RE = re.compile( + r"[\x09\x20-\x2F\x3B-\x40\x5B-\x60\x7B-\x7E]*" + r"(?P[\x00-\x08\x0A-\x1F\d:a-zA-Z\x7F-\xFF]+)" + ) + + DATE_HMS_TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})") + + DATE_DAY_OF_MONTH_RE = re.compile(r"(\d{1,2})") + + DATE_MONTH_RE = re.compile( + "(jan)|(feb)|(mar)|(apr)|(may)|(jun)|(jul)|(aug)|(sep)|(oct)|(nov)|(dec)", + re.I, + ) + + DATE_YEAR_RE = re.compile(r"(\d{2,4})") + + # calendar.timegm() fails for timestamps after datetime.datetime.max + # Minus one as a loss of precision occurs when timestamp() is called. + MAX_TIME = ( + int(datetime.datetime.max.replace(tzinfo=datetime.timezone.utc).timestamp()) - 1 + ) + try: + calendar.timegm(time.gmtime(MAX_TIME)) + except OSError: + # Hit the maximum representable time on Windows + # https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/localtime-localtime32-localtime64 + MAX_TIME = calendar.timegm((3000, 12, 31, 23, 59, 59, -1, -1, -1)) + except OverflowError: + # #4515: datetime.max may not be representable on 32-bit platforms + MAX_TIME = 2**31 - 1 + # Avoid minuses in the future, 3x faster + SUB_MAX_TIME = MAX_TIME - 1 + + def __init__( + self, + *, + unsafe: bool = False, + quote_cookie: bool = True, + treat_as_secure_origin: StrOrURL | list[StrOrURL] | None = None, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + super().__init__(loop=loop) + self._cookies: defaultdict[tuple[str, str], SimpleCookie] = defaultdict( + SimpleCookie + ) + self._morsel_cache: defaultdict[tuple[str, str], dict[str, Morsel[str]]] = ( + defaultdict(dict) + ) + self._host_only_cookies: set[tuple[str, str]] = set() + self._unsafe = unsafe + self._quote_cookie = quote_cookie + if treat_as_secure_origin is None: + treat_as_secure_origin = [] + elif isinstance(treat_as_secure_origin, URL): + treat_as_secure_origin = [treat_as_secure_origin.origin()] + elif isinstance(treat_as_secure_origin, str): + treat_as_secure_origin = [URL(treat_as_secure_origin).origin()] + else: + treat_as_secure_origin = [ + URL(url).origin() if isinstance(url, str) else url.origin() + for url in treat_as_secure_origin + ] + self._treat_as_secure_origin = treat_as_secure_origin + self._expire_heap: list[tuple[float, tuple[str, str, str]]] = [] + self._expirations: dict[tuple[str, str, str], float] = {} + + @property + def unsafe(self) -> bool: + return self._unsafe + + @property + def quote_cookie(self) -> bool: + return self._quote_cookie + + @property + def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]: + """Return the cookies stored in this jar.""" + return MappingProxyType(self._cookies) + + @property + def host_only_cookies(self) -> frozenset[tuple[str, str]]: + """Return the host-only cookies stored in this jar.""" + return frozenset(self._host_only_cookies) + + def save(self, file_path: PathLike) -> None: + """Save cookies to a file using JSON format. + + :param file_path: Path to file where cookies will be serialized, + :class:`str` or :class:`pathlib.Path` instance. + """ + file_path = pathlib.Path(file_path) + data: dict[str, dict[str, dict[str, str | bool | float]]] = {} + for (domain, path), cookie in self._cookies.items(): + key = f"{domain}|{path}" + data[key] = {} + for name, morsel in cookie.items(): + morsel_data: dict[str, str | bool | float] = { + "key": morsel.key, + "value": morsel.value, + "coded_value": morsel.coded_value, + } + # Skip relative expiry; the absolute deadline is saved below. + for attr in morsel._reserved: # type: ignore[attr-defined] + if attr in _RELATIVE_EXPIRY_ATTRS: + continue + attr_val = morsel[attr] + if attr_val: + morsel_data[attr] = attr_val + # Persist or it reloads as a domain cookie and leaks to subdomains. + if (domain, name) in self._host_only_cookies: + morsel_data["host_only"] = True + if (exp := self._expirations.get((domain, path, name))) is not None: + morsel_data["expires_timestamp"] = exp + data[key][name] = morsel_data + + # Cookie persistence may include authentication/session tokens. + # Use 0o600 at creation time to avoid umask-dependent overexposure + # and enforce least-privilege access to sensitive credential data. + with open( + file_path, + mode="w", + encoding="utf-8", + opener=lambda path, flags: os.open(path, flags, 0o600), + ) as f: + json.dump(data, f, indent=2) + + def load(self, file_path: PathLike) -> None: + """Load cookies from a file. + + Tries to load JSON format first. Falls back to loading legacy + pickle format (using a restricted unpickler) for backward + compatibility with existing cookie files. + + Replaces the current jar contents; loaded cookies pass through the + same acceptance rules as :meth:`update_cookies`. + + :param file_path: Path to file from where cookies will be + imported, :class:`str` or :class:`pathlib.Path` instance. + """ + file_path = pathlib.Path(file_path) + # Try JSON format first + try: + with file_path.open(mode="r", encoding="utf-8") as f: + data = json.load(f) + self._load_json_data(data) + except (json.JSONDecodeError, UnicodeDecodeError, ValueError): + # Fall back to legacy pickle format with restricted unpickler + with file_path.open(mode="rb") as f: + self._cookies = _RestrictedCookieUnpickler(f).load() + + def _load_json_data( + self, data: dict[str, dict[str, dict[str, str | bool | float]]] + ) -> None: + """Replace contents, routing cookies through update_cookies().""" + self.clear() + for compound_key, cookie_data in data.items(): + domain, path = compound_key.split("|", 1) + for name, morsel_data in cookie_data.items(): + morsel: Morsel[str] = Morsel() + # Use __setstate__ to bypass validation, same pattern + # used in _build_morsel and _cookie_helpers. + morsel.__setstate__( # type: ignore[attr-defined] + { + "key": morsel_data["key"], + "value": morsel_data["value"], + "coded_value": morsel_data["coded_value"], + } + ) + # Restore morsel attributes + for attr in morsel._reserved: # type: ignore[attr-defined] + if attr in morsel_data and attr not in ( + "key", + "value", + "coded_value", + ): + morsel[attr] = morsel_data[attr] + # Drop the domain so update_cookies() re-marks it host-only. + if morsel_data.get("host_only"): + morsel["domain"] = "" + response_url = ( + URL.build(scheme="https", host=domain) if domain else URL() + ) + self.update_cookies({name: morsel}, response_url) + # Restore the absolute deadline; update_cookies() schedules none. + if (exp := morsel_data.get("expires_timestamp")) is not None: + self._expire_cookie(float(exp), domain, path, name) + self._do_expiration() + + def clear(self, predicate: ClearCookiePredicate | None = None) -> None: + if predicate is None: + self._expire_heap.clear() + self._cookies.clear() + self._morsel_cache.clear() + self._host_only_cookies.clear() + self._expirations.clear() + return + + now = time.time() + to_del = [ + key + for (domain, path), cookie in self._cookies.items() + for name, morsel in cookie.items() + if ( + (key := (domain, path, name)) in self._expirations + and self._expirations[key] <= now + ) + or predicate(morsel) + ] + if to_del: + self._delete_cookies(to_del) + + def clear_domain(self, domain: str) -> None: + self.clear(lambda x: self._is_domain_match(domain, x["domain"])) + + def __iter__(self) -> "Iterator[Morsel[str]]": + self._do_expiration() + for val in self._cookies.values(): + yield from val.values() + + def __len__(self) -> int: + """Return number of cookies. + + This function does not iterate self to avoid unnecessary expiration + checks. + """ + return sum(len(cookie.values()) for cookie in self._cookies.values()) + + def _do_expiration(self) -> None: + """Remove expired cookies.""" + if not (expire_heap_len := len(self._expire_heap)): + return + + # If the expiration heap grows larger than the number expirations + # times two, we clean it up to avoid keeping expired entries in + # the heap and consuming memory. We guard this with a minimum + # threshold to avoid cleaning up the heap too often when there are + # only a few scheduled expirations. + if ( + expire_heap_len > _MIN_SCHEDULED_COOKIE_EXPIRATION + and expire_heap_len > len(self._expirations) * 2 + ): + # Remove any expired entries from the expiration heap + # that do not match the expiration time in the expirations + # as it means the cookie has been re-added to the heap + # with a different expiration time. + self._expire_heap = [ + entry + for entry in self._expire_heap + if self._expirations.get(entry[1]) == entry[0] + ] + heapq.heapify(self._expire_heap) + + now = time.time() + to_del: list[tuple[str, str, str]] = [] + # Find any expired cookies and add them to the to-delete list + while self._expire_heap: + when, cookie_key = self._expire_heap[0] + if when > now: + break + heapq.heappop(self._expire_heap) + # Check if the cookie hasn't been re-added to the heap + # with a different expiration time as it will be removed + # later when it reaches the top of the heap and its + # expiration time is met. + if self._expirations.get(cookie_key) == when: + to_del.append(cookie_key) + + if to_del: + self._delete_cookies(to_del) + + def _delete_cookies(self, to_del: list[tuple[str, str, str]]) -> None: + for domain, path, name in to_del: + self._host_only_cookies.discard((domain, name)) + self._cookies[(domain, path)].pop(name, None) + self._morsel_cache[(domain, path)].pop(name, None) + self._expirations.pop((domain, path, name), None) + + def _expire_cookie(self, when: float, domain: str, path: str, name: str) -> None: + cookie_key = (domain, path, name) + if self._expirations.get(cookie_key) == when: + # Avoid adding duplicates to the heap + return + heapq.heappush(self._expire_heap, (when, cookie_key)) + self._expirations[cookie_key] = when + + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + """Update cookies.""" + hostname = response_url.raw_host + + if not self._unsafe and is_ip_address(hostname): + # Don't accept cookies from IPs + return + + if isinstance(cookies, Mapping): + cookies = cookies.items() + + for name, cookie in cookies: + if not isinstance(cookie, Morsel): + tmp = SimpleCookie() + tmp[name] = cookie # type: ignore[assignment] + cookie = tmp[name] + + domain = cookie["domain"] + + # ignore domains with trailing dots + if domain and domain[-1] == ".": + domain = "" + del cookie["domain"] + + if not domain and hostname is not None: + # Set the cookie's domain to the response hostname + # and set its host-only-flag + self._host_only_cookies.add((hostname, name)) + domain = cookie["domain"] = hostname + + if domain and domain[0] == ".": + # Remove leading dot + domain = domain[1:] + cookie["domain"] = domain + + if hostname and not self._is_domain_match(domain, hostname): + # Setting cookies for different domains is not allowed + continue + + path = cookie["path"] + if not path or path[0] != "/": + # Set the cookie's path to the response path + path = response_url.path + if not path.startswith("/"): + path = "/" + else: + # Cut everything from the last slash to the end + path = "/" + path[1 : path.rfind("/")] + cookie["path"] = path + path = path.rstrip("/") + + if max_age := cookie["max-age"]: + try: + delta_seconds = int(max_age) + max_age_expiration = min(time.time() + delta_seconds, self.MAX_TIME) + self._expire_cookie(max_age_expiration, domain, path, name) + except ValueError: + cookie["max-age"] = "" + + elif expires := cookie["expires"]: + if expire_time := self._parse_date(expires): + self._expire_cookie(expire_time, domain, path, name) + else: + cookie["expires"] = "" + + key = (domain, path) + if self._cookies[key].get(name) != cookie: + # Don't blow away the cache if the same + # cookie gets set again + self._cookies[key][name] = cookie + self._morsel_cache[key].pop(name, None) + + self._do_expiration() + + def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": + """Returns this jar's cookies filtered by their attributes.""" + # We always use BaseCookie now since all + # cookies set on on filtered are fully constructed + # Morsels, not just names and values. + filtered: BaseCookie[str] = BaseCookie() + if not self._cookies: + # Skip do_expiration() if there are no cookies. + return filtered + self._do_expiration() + if not self._cookies: + # Skip rest of function if no non-expired cookies. + return filtered + if type(request_url) is not URL: + warnings.warn( + "filter_cookies expects yarl.URL instances only," + f"and will stop working in 4.x, got {type(request_url)}", + DeprecationWarning, + stacklevel=2, + ) + request_url = URL(request_url) + hostname = request_url.raw_host or "" + + is_not_secure = request_url.scheme not in ("https", "wss") + if is_not_secure and self._treat_as_secure_origin: + request_origin = URL() + with contextlib.suppress(ValueError): + request_origin = request_url.origin() + is_not_secure = request_origin not in self._treat_as_secure_origin + + # Send shared cookie + key = ("", "") + for c in self._cookies[key].values(): + # Check cache first + if c.key in self._morsel_cache[key]: + filtered[c.key] = self._morsel_cache[key][c.key] + continue + + # Build and cache the morsel + mrsl_val = self._build_morsel(c) + self._morsel_cache[key][c.key] = mrsl_val + filtered[c.key] = mrsl_val + + if is_ip_address(hostname): + if not self._unsafe: + return filtered + domains: Iterable[str] = (hostname,) + else: + # Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com") + domains = itertools.accumulate( + reversed(hostname.split(".")), _FORMAT_DOMAIN_REVERSED + ) + + # Get all the path prefixes that might match a cookie (e.g. "", "/foo", "/foo/bar") + paths = itertools.accumulate(request_url.path.split("/"), _FORMAT_PATH) + # Create every combination of (domain, path) pairs. + pairs = itertools.product(domains, paths) + + path_len = len(request_url.path) + # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4 + for p in pairs: + if p not in self._cookies: + continue + for name, cookie in self._cookies[p].items(): + domain = cookie["domain"] + + if (domain, name) in self._host_only_cookies and domain != hostname: + continue + + # Skip edge case when the cookie has a trailing slash but request doesn't. + if len(cookie["path"]) > path_len: + continue + + if is_not_secure and cookie["secure"]: + continue + + # We already built the Morsel so reuse it here + if name in self._morsel_cache[p]: + filtered[name] = self._morsel_cache[p][name] + continue + + # Build and cache the morsel + mrsl_val = self._build_morsel(cookie) + self._morsel_cache[p][name] = mrsl_val + filtered[name] = mrsl_val + + return filtered + + def _build_morsel(self, cookie: Morsel[str]) -> Morsel[str]: + """Build a morsel for sending, respecting quote_cookie setting.""" + if self._quote_cookie and cookie.coded_value and cookie.coded_value[0] == '"': + return preserve_morsel_with_coded_value(cookie) + morsel: Morsel[str] = Morsel() + if self._quote_cookie: + value, coded_value = _SIMPLE_COOKIE.value_encode(cookie.value) + else: + coded_value = value = cookie.value + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + morsel.__setstate__({"key": cookie.key, "value": value, "coded_value": coded_value}) # type: ignore[attr-defined] + return morsel + + @staticmethod + def _is_domain_match(domain: str, hostname: str) -> bool: + """Implements domain matching adhering to RFC 6265.""" + if hostname == domain: + return True + + if not hostname.endswith(domain): + return False + + non_matching = hostname[: -len(domain)] + + if not non_matching.endswith("."): + return False + + return not is_ip_address(hostname) + + @classmethod + def _parse_date(cls, date_str: str) -> int | None: + """Implements date string parsing adhering to RFC 6265.""" + if not date_str: + return None + + found_time = False + found_day = False + found_month = False + found_year = False + + hour = minute = second = 0 + day = 0 + month = 0 + year = 0 + + for token_match in cls.DATE_TOKENS_RE.finditer(date_str): + + token = token_match.group("token") + + if not found_time: + time_match = cls.DATE_HMS_TIME_RE.match(token) + if time_match: + found_time = True + hour, minute, second = (int(s) for s in time_match.groups()) + continue + + if not found_day: + day_match = cls.DATE_DAY_OF_MONTH_RE.match(token) + if day_match: + found_day = True + day = int(day_match.group()) + continue + + if not found_month: + month_match = cls.DATE_MONTH_RE.match(token) + if month_match: + found_month = True + assert month_match.lastindex is not None + month = month_match.lastindex + continue + + if not found_year: + year_match = cls.DATE_YEAR_RE.match(token) + if year_match: + found_year = True + year = int(year_match.group()) + + if 70 <= year <= 99: + year += 1900 + elif 0 <= year <= 69: + year += 2000 + + if False in (found_day, found_month, found_year, found_time): + return None + + if not 1 <= day <= 31: + return None + + if year < 1601 or hour > 23 or minute > 59 or second > 59: + return None + + return calendar.timegm((year, month, day, hour, minute, second, -1, -1, -1)) + + +class DummyCookieJar(AbstractCookieJar): + """Implements a dummy cookie storage. + + It can be used with the ClientSession when no cookie processing is needed. + + """ + + def __init__(self, *, loop: asyncio.AbstractEventLoop | None = None) -> None: + super().__init__(loop=loop) + + def __iter__(self) -> "Iterator[Morsel[str]]": + while False: + yield None + + def __len__(self) -> int: + return 0 + + @property + def unsafe(self) -> bool: + return False + + @property + def quote_cookie(self) -> bool: + return True + + @property + def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]: + """Return an empty mapping.""" + return MappingProxyType({}) + + @property + def host_only_cookies(self) -> frozenset[tuple[str, str]]: + """Return an empty frozenset.""" + return frozenset() + + def clear(self, predicate: ClearCookiePredicate | None = None) -> None: + pass + + def clear_domain(self, domain: str) -> None: + pass + + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + pass + + def filter_cookies(self, request_url: URL) -> "BaseCookie[str]": + return SimpleCookie() diff --git a/venv/Lib/site-packages/aiohttp/formdata.py b/venv/Lib/site-packages/aiohttp/formdata.py new file mode 100644 index 0000000000000000000000000000000000000000..efa7f39312c9557bc7ea8bbba4c8d2c56c56cd01 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/formdata.py @@ -0,0 +1,184 @@ +import io +import warnings +from collections.abc import Iterable +from typing import Any +from urllib.parse import urlencode + +from multidict import MultiDict, MultiDictProxy + +from . import hdrs, multipart, payload +from .helpers import guess_filename +from .http_writer import _safe_header +from .payload import Payload + +__all__ = ("FormData",) + + +class FormData: + """Helper class for form body generation. + + Supports multipart/form-data and application/x-www-form-urlencoded. + """ + + def __init__( + self, + fields: Iterable[Any] = (), + quote_fields: bool = True, + charset: str | None = None, + *, + default_to_multipart: bool = False, + ) -> None: + self._writer = multipart.MultipartWriter("form-data") + self._fields: list[Any] = [] + self._is_multipart = default_to_multipart + self._quote_fields = quote_fields + self._charset = charset + + if isinstance(fields, dict): + fields = list(fields.items()) + elif not isinstance(fields, (list, tuple)): + fields = (fields,) + self.add_fields(*fields) + + @property + def is_multipart(self) -> bool: + return self._is_multipart + + def add_field( + self, + name: str, + value: Any, + *, + content_type: str | None = None, + filename: str | None = None, + content_transfer_encoding: str | None = None, + ) -> None: + + if isinstance(value, io.IOBase): + self._is_multipart = True + elif isinstance(value, (bytes, bytearray, memoryview)): + msg = ( + "In v4, passing bytes will no longer create a file field. " + "Please explicitly use the filename parameter or pass a BytesIO object." + ) + if filename is None and content_transfer_encoding is None: + warnings.warn(msg, DeprecationWarning) + filename = name + + _safe_header(name) + type_options: MultiDict[str] = MultiDict({"name": name}) + if filename is not None and not isinstance(filename, str): + raise TypeError("filename must be an instance of str. Got: %s" % filename) + if filename is None and isinstance(value, io.IOBase): + filename = guess_filename(value, name) + if filename is not None: + _safe_header(filename) + type_options["filename"] = filename + self._is_multipart = True + + headers = {} + if content_type is not None: + if not isinstance(content_type, str): + raise TypeError( + "content_type must be an instance of str. Got: %s" % content_type + ) + _safe_header(content_type) + headers[hdrs.CONTENT_TYPE] = content_type + self._is_multipart = True + if content_transfer_encoding is not None: + if not isinstance(content_transfer_encoding, str): + raise TypeError( + "content_transfer_encoding must be an instance" + " of str. Got: %s" % content_transfer_encoding + ) + msg = ( + "content_transfer_encoding is deprecated. " + "To maintain compatibility with v4 please pass a BytesPayload." + ) + warnings.warn(msg, DeprecationWarning) + self._is_multipart = True + + self._fields.append((type_options, headers, value)) + + def add_fields(self, *fields: Any) -> None: + to_add = list(fields) + + while to_add: + rec = to_add.pop(0) + + if isinstance(rec, io.IOBase): + k = guess_filename(rec, "unknown") + self.add_field(k, rec) # type: ignore[arg-type] + + elif isinstance(rec, (MultiDictProxy, MultiDict)): + to_add.extend(rec.items()) + + elif isinstance(rec, (list, tuple)) and len(rec) == 2: + k, fp = rec + self.add_field(k, fp) + + else: + raise TypeError( + "Only io.IOBase, multidict and (name, file) " + "pairs allowed, use .add_field() for passing " + f"more complex parameters, got {rec!r}" + ) + + def _gen_form_urlencoded(self) -> payload.BytesPayload: + # form data (x-www-form-urlencoded) + data = [] + for type_options, _, value in self._fields: + data.append((type_options["name"], value)) + + charset = self._charset if self._charset is not None else "utf-8" + + if charset == "utf-8": + content_type = "application/x-www-form-urlencoded" + else: + content_type = "application/x-www-form-urlencoded; charset=%s" % charset + + return payload.BytesPayload( + urlencode(data, doseq=True, encoding=charset).encode(), + content_type=content_type, + ) + + def _gen_form_data(self) -> multipart.MultipartWriter: + """Encode a list of fields using the multipart/form-data MIME format""" + for dispparams, headers, value in self._fields: + try: + if hdrs.CONTENT_TYPE in headers: + part = payload.get_payload( + value, + content_type=headers[hdrs.CONTENT_TYPE], + headers=headers, + encoding=self._charset, + ) + else: + part = payload.get_payload( + value, headers=headers, encoding=self._charset + ) + except Exception as exc: + raise TypeError( + "Can not serialize value type: %r\n " + "headers: %r\n value: %r" % (type(value), headers, value) + ) from exc + + if dispparams: + part.set_content_disposition( + "form-data", quote_fields=self._quote_fields, **dispparams + ) + # FIXME cgi.FieldStorage doesn't likes body parts with + # Content-Length which were sent via chunked transfer encoding + assert part.headers is not None + part.headers.popall(hdrs.CONTENT_LENGTH, None) + + self._writer.append_payload(part) + + self._fields.clear() + return self._writer + + def __call__(self) -> Payload: + if self._is_multipart: + return self._gen_form_data() + else: + return self._gen_form_urlencoded() diff --git a/venv/Lib/site-packages/aiohttp/hdrs.py b/venv/Lib/site-packages/aiohttp/hdrs.py new file mode 100644 index 0000000000000000000000000000000000000000..356b0d98dd349168f42e2cdf1545ab9f70b99d62 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/hdrs.py @@ -0,0 +1,121 @@ +"""HTTP Headers constants.""" + +# After changing the file content call ./tools/gen.py +# to regenerate the headers parser +import itertools +from typing import Final + +from multidict import istr + +METH_ANY: Final[str] = "*" +METH_CONNECT: Final[str] = "CONNECT" +METH_HEAD: Final[str] = "HEAD" +METH_GET: Final[str] = "GET" +METH_DELETE: Final[str] = "DELETE" +METH_OPTIONS: Final[str] = "OPTIONS" +METH_PATCH: Final[str] = "PATCH" +METH_POST: Final[str] = "POST" +METH_PUT: Final[str] = "PUT" +METH_TRACE: Final[str] = "TRACE" + +METH_ALL: Final[set[str]] = { + METH_CONNECT, + METH_HEAD, + METH_GET, + METH_DELETE, + METH_OPTIONS, + METH_PATCH, + METH_POST, + METH_PUT, + METH_TRACE, +} + +ACCEPT: Final[istr] = istr("Accept") +ACCEPT_CHARSET: Final[istr] = istr("Accept-Charset") +ACCEPT_ENCODING: Final[istr] = istr("Accept-Encoding") +ACCEPT_LANGUAGE: Final[istr] = istr("Accept-Language") +ACCEPT_RANGES: Final[istr] = istr("Accept-Ranges") +ACCESS_CONTROL_MAX_AGE: Final[istr] = istr("Access-Control-Max-Age") +ACCESS_CONTROL_ALLOW_CREDENTIALS: Final[istr] = istr("Access-Control-Allow-Credentials") +ACCESS_CONTROL_ALLOW_HEADERS: Final[istr] = istr("Access-Control-Allow-Headers") +ACCESS_CONTROL_ALLOW_METHODS: Final[istr] = istr("Access-Control-Allow-Methods") +ACCESS_CONTROL_ALLOW_ORIGIN: Final[istr] = istr("Access-Control-Allow-Origin") +ACCESS_CONTROL_EXPOSE_HEADERS: Final[istr] = istr("Access-Control-Expose-Headers") +ACCESS_CONTROL_REQUEST_HEADERS: Final[istr] = istr("Access-Control-Request-Headers") +ACCESS_CONTROL_REQUEST_METHOD: Final[istr] = istr("Access-Control-Request-Method") +AGE: Final[istr] = istr("Age") +ALLOW: Final[istr] = istr("Allow") +AUTHORIZATION: Final[istr] = istr("Authorization") +CACHE_CONTROL: Final[istr] = istr("Cache-Control") +CONNECTION: Final[istr] = istr("Connection") +CONTENT_DISPOSITION: Final[istr] = istr("Content-Disposition") +CONTENT_ENCODING: Final[istr] = istr("Content-Encoding") +CONTENT_LANGUAGE: Final[istr] = istr("Content-Language") +CONTENT_LENGTH: Final[istr] = istr("Content-Length") +CONTENT_LOCATION: Final[istr] = istr("Content-Location") +CONTENT_MD5: Final[istr] = istr("Content-MD5") +CONTENT_RANGE: Final[istr] = istr("Content-Range") +CONTENT_TRANSFER_ENCODING: Final[istr] = istr("Content-Transfer-Encoding") +CONTENT_TYPE: Final[istr] = istr("Content-Type") +COOKIE: Final[istr] = istr("Cookie") +DATE: Final[istr] = istr("Date") +DESTINATION: Final[istr] = istr("Destination") +DIGEST: Final[istr] = istr("Digest") +ETAG: Final[istr] = istr("Etag") +EXPECT: Final[istr] = istr("Expect") +EXPIRES: Final[istr] = istr("Expires") +FORWARDED: Final[istr] = istr("Forwarded") +FROM: Final[istr] = istr("From") +HOST: Final[istr] = istr("Host") +IF_MATCH: Final[istr] = istr("If-Match") +IF_MODIFIED_SINCE: Final[istr] = istr("If-Modified-Since") +IF_NONE_MATCH: Final[istr] = istr("If-None-Match") +IF_RANGE: Final[istr] = istr("If-Range") +IF_UNMODIFIED_SINCE: Final[istr] = istr("If-Unmodified-Since") +KEEP_ALIVE: Final[istr] = istr("Keep-Alive") +LAST_EVENT_ID: Final[istr] = istr("Last-Event-ID") +LAST_MODIFIED: Final[istr] = istr("Last-Modified") +LINK: Final[istr] = istr("Link") +LOCATION: Final[istr] = istr("Location") +MAX_FORWARDS: Final[istr] = istr("Max-Forwards") +ORIGIN: Final[istr] = istr("Origin") +PRAGMA: Final[istr] = istr("Pragma") +PROXY_AUTHENTICATE: Final[istr] = istr("Proxy-Authenticate") +PROXY_AUTHORIZATION: Final[istr] = istr("Proxy-Authorization") +RANGE: Final[istr] = istr("Range") +REFERER: Final[istr] = istr("Referer") +RETRY_AFTER: Final[istr] = istr("Retry-After") +SEC_WEBSOCKET_ACCEPT: Final[istr] = istr("Sec-WebSocket-Accept") +SEC_WEBSOCKET_VERSION: Final[istr] = istr("Sec-WebSocket-Version") +SEC_WEBSOCKET_PROTOCOL: Final[istr] = istr("Sec-WebSocket-Protocol") +SEC_WEBSOCKET_EXTENSIONS: Final[istr] = istr("Sec-WebSocket-Extensions") +SEC_WEBSOCKET_KEY: Final[istr] = istr("Sec-WebSocket-Key") +SEC_WEBSOCKET_KEY1: Final[istr] = istr("Sec-WebSocket-Key1") +SERVER: Final[istr] = istr("Server") +SET_COOKIE: Final[istr] = istr("Set-Cookie") +TE: Final[istr] = istr("TE") +TRAILER: Final[istr] = istr("Trailer") +TRANSFER_ENCODING: Final[istr] = istr("Transfer-Encoding") +UPGRADE: Final[istr] = istr("Upgrade") +URI: Final[istr] = istr("URI") +USER_AGENT: Final[istr] = istr("User-Agent") +VARY: Final[istr] = istr("Vary") +VIA: Final[istr] = istr("Via") +WANT_DIGEST: Final[istr] = istr("Want-Digest") +WARNING: Final[istr] = istr("Warning") +WWW_AUTHENTICATE: Final[istr] = istr("WWW-Authenticate") +X_FORWARDED_FOR: Final[istr] = istr("X-Forwarded-For") +X_FORWARDED_HOST: Final[istr] = istr("X-Forwarded-Host") +X_FORWARDED_PROTO: Final[istr] = istr("X-Forwarded-Proto") + +# These are the upper/lower case variants of the headers/methods +# Example: {'hOst', 'host', 'HoST', 'HOSt', 'hOsT', 'HosT', 'hoSt', ...} +METH_HEAD_ALL: Final = frozenset( + map("".join, itertools.product(*zip(METH_HEAD.upper(), METH_HEAD.lower()))) +) +METH_CONNECT_ALL: Final = frozenset( + map("".join, itertools.product(*zip(METH_CONNECT.upper(), METH_CONNECT.lower()))) +) +HOST_ALL: Final = frozenset( + map("".join, itertools.product(*zip(HOST.upper(), HOST.lower()))) +) diff --git a/venv/Lib/site-packages/aiohttp/helpers.py b/venv/Lib/site-packages/aiohttp/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..f1203eba613035862936586d3272688c5518a90b --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/helpers.py @@ -0,0 +1,1046 @@ +"""Various helper functions""" + +import asyncio +import base64 +import binascii +import contextlib +import datetime +import enum +import functools +import inspect +import netrc +import os +import platform +import re +import sys +import time +import warnings +import weakref +from collections import namedtuple +from collections.abc import Callable, Generator, Iterable, Iterator, Mapping +from contextlib import suppress +from email.message import EmailMessage +from email.parser import HeaderParser +from email.policy import HTTP +from email.utils import parsedate +from math import ceil +from pathlib import Path +from types import MappingProxyType, TracebackType +from typing import ( + Any, + ContextManager, + Generic, + Optional, + Protocol, + TypeVar, + get_args, + overload, +) +from urllib.parse import quote +from urllib.request import getproxies, proxy_bypass + +import attr +from multidict import MultiDict, MultiDictProxy, MultiMapping +from propcache.api import under_cached_property as reify +from yarl import URL + +from . import hdrs +from .log import client_logger + +if sys.version_info >= (3, 11): + import asyncio as async_timeout +else: + import async_timeout + +__all__ = ("BasicAuth", "ChainMapProxy", "ETag", "reify") + +IS_MACOS = platform.system() == "Darwin" +IS_WINDOWS = platform.system() == "Windows" + +PY_311 = sys.version_info >= (3, 11) + +# This is the default size/limit for several operations. +# Matches the max size we receive from sockets: +# https://github.com/python/cpython/blob/1857a40807daeae3a1bf5efb682de9c9ae6df845/Lib/asyncio/selector_events.py#L766 +DEFAULT_CHUNK_SIZE = 2**18 # 256 KiB + +_T = TypeVar("_T") +_S = TypeVar("_S") + +_SENTINEL = enum.Enum("_SENTINEL", "sentinel") +sentinel = _SENTINEL.sentinel + +NO_EXTENSIONS = bool(os.environ.get("AIOHTTP_NO_EXTENSIONS")) + +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1 +EMPTY_BODY_STATUS_CODES = frozenset((204, 304, *range(100, 200))) +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1 +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.2 +EMPTY_BODY_METHODS = hdrs.METH_HEAD_ALL + +DEBUG = sys.flags.dev_mode or ( + not sys.flags.ignore_environment and bool(os.environ.get("PYTHONASYNCIODEBUG")) +) + + +CHAR = {chr(i) for i in range(0, 128)} +CTL = {chr(i) for i in range(0, 32)} | { + chr(127), +} +SEPARATORS = { + "(", + ")", + "<", + ">", + "@", + ",", + ";", + ":", + "\\", + '"', + "/", + "[", + "]", + "?", + "=", + "{", + "}", + " ", + chr(9), +} +TOKEN = CHAR ^ CTL ^ SEPARATORS + + +class noop: + def __await__(self) -> Generator[None, None, None]: + yield + + +def encode_basic_auth(login: str, password: str = "", encoding: str = "utf-8") -> str: + """Encode HTTP Basic Authentication credentials as an Authorization header value. + + Returns a string of the form ``"Basic "`` suitable for use as the + value of the ``Authorization`` (or ``Proxy-Authorization``) header. + """ + if ":" in login: + raise ValueError('A ":" is not allowed in login (RFC 7617#section-2)') + creds = f"{login}:{password}".encode(encoding) + return "Basic " + base64.b64encode(creds).decode(encoding) + + +class BasicAuth(namedtuple("BasicAuth", ["login", "password", "encoding"])): + """Http basic authentication helper.""" + + def __new__( + cls, login: str, password: str = "", encoding: str = "latin1" + ) -> "BasicAuth": + if login is None: + raise ValueError("None is not allowed as login value") + + if password is None: + raise ValueError("None is not allowed as password value") + + if ":" in login: + raise ValueError('A ":" is not allowed in login (RFC 1945#section-11.1)') + + warnings.warn( + "BasicAuth is deprecated and will be removed in aiohttp 4.0; " + "use aiohttp.encode_basic_auth() with " + "headers={'Authorization': ...} instead", + DeprecationWarning, + stacklevel=2, + ) + return super().__new__(cls, login, password, encoding) + + @classmethod + def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth": + """Create a BasicAuth object from an Authorization HTTP header.""" + try: + auth_type, encoded_credentials = auth_header.split(" ", 1) + except ValueError: + raise ValueError("Could not parse authorization header.") + + if auth_type.lower() != "basic": + raise ValueError("Unknown authorization method %s" % auth_type) + + try: + decoded = base64.b64decode( + encoded_credentials.encode("ascii"), validate=True + ).decode(encoding) + except binascii.Error: + raise ValueError("Invalid base64 encoding.") + + try: + # RFC 2617 HTTP Authentication + # https://www.ietf.org/rfc/rfc2617.txt + # the colon must be present, but the username and password may be + # otherwise blank. + username, password = decoded.split(":", 1) + except ValueError: + raise ValueError("Invalid credentials.") + + return _basic_auth_no_warn(username, password, encoding) + + @classmethod + def from_url(cls, url: URL, *, encoding: str = "latin1") -> Optional["BasicAuth"]: + """Create BasicAuth from url.""" + if not isinstance(url, URL): + raise TypeError("url should be yarl.URL instance") + # Check raw_user and raw_password first as yarl is likely + # to already have these values parsed from the netloc in the cache. + if url.raw_user is None and url.raw_password is None: + return None + return _basic_auth_no_warn(url.user or "", url.password or "", encoding) + + def encode(self) -> str: + """Encode credentials.""" + return encode_basic_auth(self.login, self.password, self.encoding) + + +def _basic_auth_no_warn( + login: str, password: str = "", encoding: str = "latin1" +) -> BasicAuth: + """Construct a BasicAuth without emitting the deprecation warning. + + For internal use only. Bypasses BasicAuth.__new__ so that aiohttp's own + machinery doesn't trigger deprecation warnings in user code. + """ + return tuple.__new__(BasicAuth, (login, password, encoding)) + + +def strip_auth_from_url(url: URL) -> tuple[URL, BasicAuth | None]: + """Remove user and password from URL if present and return BasicAuth object.""" + # Check raw_user and raw_password first as yarl is likely + # to already have these values parsed from the netloc in the cache. + if url.raw_user is None and url.raw_password is None: + return url, None + return url.with_user(None), _basic_auth_no_warn(url.user or "", url.password or "") + + +def netrc_from_env() -> netrc.netrc | None: + """Load netrc from file. + + Attempt to load it from the path specified by the env-var + NETRC or in the default location in the user's home directory. + + Returns None if it couldn't be found or fails to parse. + """ + netrc_env = os.environ.get("NETRC") + + if netrc_env is not None: + netrc_path = Path(netrc_env) + else: + try: + home_dir = Path.home() + except RuntimeError as e: # pragma: no cover + # if pathlib can't resolve home, it may raise a RuntimeError + client_logger.debug( + "Could not resolve home directory when " + "trying to look for .netrc file: %s", + e, + ) + return None + + netrc_path = home_dir / ("_netrc" if IS_WINDOWS else ".netrc") + + try: + return netrc.netrc(str(netrc_path)) + except netrc.NetrcParseError as e: + client_logger.warning("Could not parse .netrc file: %s", e) + except OSError as e: + netrc_exists = False + with contextlib.suppress(OSError): + netrc_exists = netrc_path.is_file() + # we couldn't read the file (doesn't exist, permissions, etc.) + if netrc_env or netrc_exists: + # only warn if the environment wanted us to load it, + # or it appears like the default file does actually exist + client_logger.warning("Could not read .netrc file: %s", e) + + return None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ProxyInfo: + proxy: URL + proxy_auth: BasicAuth | None + + +def basicauth_from_netrc(netrc_obj: netrc.netrc | None, host: str) -> BasicAuth: + """ + Return :py:class:`~aiohttp.BasicAuth` credentials for ``host`` from ``netrc_obj``. + + :raises LookupError: if ``netrc_obj`` is :py:data:`None` or if no + entry is found for the ``host``. + """ + if netrc_obj is None: + raise LookupError("No .netrc file found") + auth_from_netrc = netrc_obj.authenticators(host) + + if auth_from_netrc is None: + raise LookupError(f"No entry for {host!s} found in the `.netrc` file.") + login, account, password = auth_from_netrc + + # TODO(PY311): username = login or account + # Up to python 3.10, account could be None if not specified, + # and login will be empty string if not specified. From 3.11, + # login and account will be empty string if not specified. + username = login if (login or account is None) else account + + # TODO(PY311): Remove this, as password will be empty string + # if not specified + if password is None: + password = "" + + return _basic_auth_no_warn(username, password) + + +def proxies_from_env() -> dict[str, ProxyInfo]: + proxy_urls = { + k: URL(v) + for k, v in getproxies().items() + if k in ("http", "https", "ws", "wss") + } + netrc_obj = netrc_from_env() + stripped = {k: strip_auth_from_url(v) for k, v in proxy_urls.items()} + ret = {} + for proto, val in stripped.items(): + proxy, auth = val + if proxy.scheme in ("https", "wss"): + client_logger.warning( + "%s proxies %s are not supported, ignoring", proxy.scheme.upper(), proxy + ) + continue + if netrc_obj and auth is None: + if proxy.host is not None: + try: + auth = basicauth_from_netrc(netrc_obj, proxy.host) + except LookupError: + auth = None + ret[proto] = ProxyInfo(proxy, auth) + return ret + + +def get_env_proxy_for_url(url: URL) -> tuple[URL, BasicAuth | None]: + """Get a permitted proxy for the given URL from the env.""" + if url.host is not None and proxy_bypass(url.host): + raise LookupError(f"Proxying is disallowed for `{url.host!r}`") + + proxies_in_env = proxies_from_env() + try: + proxy_info = proxies_in_env[url.scheme] + except KeyError: + raise LookupError(f"No proxies found for `{url!s}` in the env") + else: + return proxy_info.proxy, proxy_info.proxy_auth + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class MimeType: + type: str + subtype: str + suffix: str + parameters: "MultiDictProxy[str]" + + +@functools.lru_cache(maxsize=56) +def parse_mimetype(mimetype: str) -> MimeType: + """Parses a MIME type into its components. + + mimetype is a MIME type string. + + Returns a MimeType object. + + Example: + + >>> parse_mimetype('text/html; charset=utf-8') + MimeType(type='text', subtype='html', suffix='', + parameters={'charset': 'utf-8'}) + + """ + if not mimetype: + return MimeType( + type="", subtype="", suffix="", parameters=MultiDictProxy(MultiDict()) + ) + + parts = mimetype.split(";") + params: MultiDict[str] = MultiDict() + for item in parts[1:]: + if not item: + continue + key, _, value = item.partition("=") + params.add(key.lower().strip(), value.strip(' "')) + + fulltype = parts[0].strip().lower() + if fulltype == "*": + fulltype = "*/*" + + mtype, _, stype = fulltype.partition("/") + stype, _, suffix = stype.partition("+") + + return MimeType( + type=mtype, subtype=stype, suffix=suffix, parameters=MultiDictProxy(params) + ) + + +class EnsureOctetStream(EmailMessage): + def __init__(self) -> None: + super().__init__() + # https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5 + self.set_default_type("application/octet-stream") + + def get_content_type(self) -> str: + """Re-implementation from Message + + Returns application/octet-stream in place of plain/text when + value is wrong. + + The way this class is used guarantees that content-type will + be present so simplify the checks wrt to the base implementation. + """ + value = self.get("content-type", "").lower() + + # Based on the implementation of _splitparam in the standard library + ctype, _, _ = value.partition(";") + ctype = ctype.strip() + if ctype.count("/") != 1: + return self.get_default_type() + return ctype + + +@functools.lru_cache(maxsize=56) +def parse_content_type(raw: str) -> tuple[str, MappingProxyType[str, str]]: + """Parse Content-Type header. + + Returns a tuple of the parsed content type and a + MappingProxyType of parameters. The default returned value + is `application/octet-stream` + """ + msg = HeaderParser(EnsureOctetStream, policy=HTTP).parsestr(f"Content-Type: {raw}") + content_type = msg.get_content_type() + params = msg.get_params(()) + content_dict = dict(params[1:]) # First element is content type again + return content_type, MappingProxyType(content_dict) + + +def guess_filename(obj: Any, default: str | None = None) -> str | None: + name = getattr(obj, "name", None) + if name and isinstance(name, str) and name[0] != "<" and name[-1] != ">": + return Path(name).name + return default + + +not_qtext_re = re.compile(r"[^\041\043-\133\135-\176]") +QCONTENT = {chr(i) for i in range(0x20, 0x7F)} | {"\t"} + + +def quoted_string(content: str) -> str: + """Return 7-bit content as quoted-string. + + Format content into a quoted-string as defined in RFC5322 for + Internet Message Format. Notice that this is not the 8-bit HTTP + format, but the 7-bit email format. Content must be in usascii or + a ValueError is raised. + """ + if not (QCONTENT > set(content)): + raise ValueError(f"bad content for quoted-string {content!r}") + return not_qtext_re.sub(lambda x: "\\" + x.group(0), content) + + +def content_disposition_header( + disptype: str, quote_fields: bool = True, _charset: str = "utf-8", **params: str +) -> str: + """Sets ``Content-Disposition`` header for MIME. + + This is the MIME payload Content-Disposition header from RFC 2183 + and RFC 7579 section 4.2, not the HTTP Content-Disposition from + RFC 6266. + + disptype is a disposition type: inline, attachment, form-data. + Should be valid extension token (see RFC 2183) + + quote_fields performs value quoting to 7-bit MIME headers + according to RFC 7578. Set to quote_fields to False if recipient + can take 8-bit file names and field values. + + _charset specifies the charset to use when quote_fields is True. + + params is a dict with disposition params. + """ + if not disptype or not (TOKEN > set(disptype)): + raise ValueError(f"bad content disposition type {disptype!r}") + + value = disptype + if params: + lparams = [] + for key, val in params.items(): + if not key or not (TOKEN > set(key)): + raise ValueError(f"bad content disposition parameter {key!r}={val!r}") + if quote_fields: + if key.lower() == "filename": + qval = quote(val, "", encoding=_charset) + lparams.append((key, '"%s"' % qval)) + else: + try: + qval = quoted_string(val) + except ValueError: + qval = "".join( + (_charset, "''", quote(val, "", encoding=_charset)) + ) + lparams.append((key + "*", qval)) + else: + lparams.append((key, '"%s"' % qval)) + else: + qval = val.replace("\\", "\\\\").replace('"', '\\"') + lparams.append((key, '"%s"' % qval)) + sparams = "; ".join("=".join(pair) for pair in lparams) + value = "; ".join((value, sparams)) + return value + + +def is_ip_address(host: str | None) -> bool: + """Check if host looks like an IP Address. + + This check is only meant as a heuristic to ensure that + a host is not a domain name. + """ + if not host: + return False + # For a host to be an ipv4 address, it must be all numeric. + # The host must contain a colon to be an IPv6 address. + return ":" in host or host.replace(".", "").isdigit() + + +def is_canonical_ipv4_address(host: str) -> bool: + """Check if host is a canonical dotted-quad IPv4 address. + + Rejects the legacy numeric forms that ``socket`` still accepts and + maps onto an address, e.g. ``2130706433``, ``017700000001``, ``127.1``. + """ + parts = host.split(".") + if len(parts) != 4: + return False + for part in parts: + # Each octet must be 1-3 ASCII digits; reject unicode digits + # (which ``str.isdigit`` accepts but ``int`` may not), octal + # leading zeros, and values above 255. + if not (1 <= len(part) <= 3) or not part.isascii() or not part.isdigit(): + return False + if part[0] == "0" and len(part) != 1: + return False + if int(part) > 255: + return False + return True + + +_cached_current_datetime: int | None = None +_cached_formatted_datetime = "" + + +def rfc822_formatted_time() -> str: + global _cached_current_datetime + global _cached_formatted_datetime + + now = int(time.time()) + if now != _cached_current_datetime: + # Weekday and month names for HTTP date/time formatting; + # always English! + # Tuples are constants stored in codeobject! + _weekdayname = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") + _monthname = ( + "", # Dummy so we can use 1-based month numbers + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ) + + year, month, day, hh, mm, ss, wd, *tail = time.gmtime(now) + _cached_formatted_datetime = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + _weekdayname[wd], + day, + _monthname[month], + year, + hh, + mm, + ss, + ) + _cached_current_datetime = now + return _cached_formatted_datetime + + +def _weakref_handle(info: "tuple[weakref.ref[object], str]") -> None: + ref, name = info + ob = ref() + if ob is not None: + with suppress(Exception): + getattr(ob, name)() + + +def weakref_handle( + ob: object, + name: str, + timeout: float, + loop: asyncio.AbstractEventLoop, + timeout_ceil_threshold: float = 5, +) -> asyncio.TimerHandle | None: + if timeout is not None and timeout > 0: + when = loop.time() + timeout + if timeout >= timeout_ceil_threshold: + when = ceil(when) + + return loop.call_at(when, _weakref_handle, (weakref.ref(ob), name)) + return None + + +def call_later( + cb: Callable[[], Any], + timeout: float, + loop: asyncio.AbstractEventLoop, + timeout_ceil_threshold: float = 5, +) -> asyncio.TimerHandle | None: + if timeout is None or timeout <= 0: + return None + now = loop.time() + when = calculate_timeout_when(now, timeout, timeout_ceil_threshold) + return loop.call_at(when, cb) + + +def calculate_timeout_when( + loop_time: float, + timeout: float, + timeout_ceiling_threshold: float, +) -> float: + """Calculate when to execute a timeout.""" + when = loop_time + timeout + if timeout > timeout_ceiling_threshold: + return ceil(when) + return when + + +class TimeoutHandle: + """Timeout handle""" + + __slots__ = ("_timeout", "_loop", "_ceil_threshold", "_callbacks") + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + timeout: float | None, + ceil_threshold: float = 5, + ) -> None: + self._timeout = timeout + self._loop = loop + self._ceil_threshold = ceil_threshold + self._callbacks: list[ + tuple[Callable[..., None], tuple[Any, ...], dict[str, Any]] + ] = [] + + def register( + self, callback: Callable[..., None], *args: Any, **kwargs: Any + ) -> None: + self._callbacks.append((callback, args, kwargs)) + + def close(self) -> None: + self._callbacks.clear() + + def start(self) -> asyncio.TimerHandle | None: + timeout = self._timeout + if timeout is not None and timeout > 0: + when = self._loop.time() + timeout + if timeout >= self._ceil_threshold: + when = ceil(when) + return self._loop.call_at(when, self.__call__) + else: + return None + + def timer(self) -> "BaseTimerContext": + if self._timeout is not None and self._timeout > 0: + timer = TimerContext(self._loop) + self.register(timer.timeout) + return timer + else: + return TimerNoop() + + def __call__(self) -> None: + for cb, args, kwargs in self._callbacks: + with suppress(Exception): + cb(*args, **kwargs) + + self._callbacks.clear() + + +class BaseTimerContext(ContextManager["BaseTimerContext"]): + + __slots__ = () + + def assert_timeout(self) -> None: + """Raise TimeoutError if timeout has been exceeded.""" + + +class TimerNoop(BaseTimerContext): + + __slots__ = () + + def __enter__(self) -> BaseTimerContext: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + return + + +class TimerContext(BaseTimerContext): + """Low resolution timeout context manager""" + + __slots__ = ("_loop", "_tasks", "_cancelled", "_cancelling") + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._tasks: list[asyncio.Task[Any]] = [] + self._cancelled = False + self._cancelling = 0 + + def assert_timeout(self) -> None: + """Raise TimeoutError if timer has already been cancelled.""" + if self._cancelled: + raise asyncio.TimeoutError from None + + def __enter__(self) -> BaseTimerContext: + task = asyncio.current_task(loop=self._loop) + if task is None: + raise RuntimeError("Timeout context manager should be used inside a task") + + if sys.version_info >= (3, 11): + # Remember if the task was already cancelling + # so when we __exit__ we can decide if we should + # raise asyncio.TimeoutError or let the cancellation propagate + self._cancelling = task.cancelling() + + if self._cancelled: + raise asyncio.TimeoutError from None + + self._tasks.append(task) + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: + enter_task: asyncio.Task[Any] | None = None + if self._tasks: + enter_task = self._tasks.pop() + + if exc_type is asyncio.CancelledError and self._cancelled: + assert enter_task is not None + # The timeout was hit, and the task was cancelled + # so we need to uncancel the last task that entered the context manager + # since the cancellation should not leak out of the context manager + if sys.version_info >= (3, 11): + # If the task was already cancelling don't raise + # asyncio.TimeoutError and instead return None + # to allow the cancellation to propagate + if enter_task.uncancel() > self._cancelling: + return None + raise asyncio.TimeoutError from exc_val + return None + + def timeout(self) -> None: + if not self._cancelled: + for task in set(self._tasks): + task.cancel() + + self._cancelled = True + + +def ceil_timeout( + delay: float | None, ceil_threshold: float = 5 +) -> async_timeout.Timeout: + if delay is None or delay <= 0: + return async_timeout.timeout(None) + + loop = asyncio.get_running_loop() + now = loop.time() + when = now + delay + if delay > ceil_threshold: + when = ceil(when) + return async_timeout.timeout_at(when) + + +class HeadersMixin: + """Mixin for handling headers.""" + + ATTRS = frozenset(["_content_type", "_content_dict", "_stored_content_type"]) + + _headers: MultiMapping[str] + _content_type: str | None = None + _content_dict: dict[str, str] | None = None + _stored_content_type: str | None | _SENTINEL = sentinel + + def _parse_content_type(self, raw: str | None) -> None: + self._stored_content_type = raw + if raw is None: + # default value according to RFC 2616 + self._content_type = "application/octet-stream" + self._content_dict = {} + else: + content_type, content_mapping_proxy = parse_content_type(raw) + self._content_type = content_type + # _content_dict needs to be mutable so we can update it + self._content_dict = content_mapping_proxy.copy() + + @property + def content_type(self) -> str: + """The value of content part for Content-Type HTTP header.""" + raw = self._headers.get(hdrs.CONTENT_TYPE) + if self._stored_content_type != raw: + self._parse_content_type(raw) + assert self._content_type is not None + return self._content_type + + @property + def charset(self) -> str | None: + """The value of charset part for Content-Type HTTP header.""" + raw = self._headers.get(hdrs.CONTENT_TYPE) + if self._stored_content_type != raw: + self._parse_content_type(raw) + assert self._content_dict is not None + return self._content_dict.get("charset") + + @property + def content_length(self) -> int | None: + """The value of Content-Length HTTP header.""" + content_length = self._headers.get(hdrs.CONTENT_LENGTH) + return None if content_length is None else int(content_length) + + +def set_result(fut: "asyncio.Future[_T]", result: _T) -> None: + if not fut.done(): + fut.set_result(result) + + +_EXC_SENTINEL = BaseException() + + +class ErrorableProtocol(Protocol): + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = ..., + ) -> None: ... # pragma: no cover + + +def set_exception( + fut: "asyncio.Future[_T] | ErrorableProtocol", + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, +) -> None: + """Set future exception. + + If the future is marked as complete, this function is a no-op. + + :param exc_cause: An exception that is a direct cause of ``exc``. + Only set if provided. + """ + if asyncio.isfuture(fut) and fut.done(): + return + + exc_is_sentinel = exc_cause is _EXC_SENTINEL + exc_causes_itself = exc is exc_cause + if not exc_is_sentinel and not exc_causes_itself: + exc.__cause__ = exc_cause + + fut.set_exception(exc) + + +@functools.total_ordering +class BaseKey(Generic[_T]): + """Base for concrete context storage key classes. + + Each storage is provided with its own sub-class for the sake of some additional type safety. + """ + + __slots__ = ("_name", "_t", "__orig_class__") + + # This may be set by Python when instantiating with a generic type. We need to + # support this, in order to support types that are not concrete classes, + # like Iterable, which can't be passed as the second parameter to __init__. + __orig_class__: type[object] + + def __init__(self, name: str, t: type[_T] | None = None): + # Prefix with module name to help deduplicate key names. + frame = inspect.currentframe() + while frame: + if frame.f_code.co_name == "": + module: str = frame.f_globals["__name__"] + break + frame = frame.f_back + + self._name = module + "." + name + self._t = t + + def __lt__(self, other: object) -> bool: + if isinstance(other, BaseKey): + return self._name < other._name + return True # Order BaseKey above other types. + + def __repr__(self) -> str: + t = self._t + if t is None: + with suppress(AttributeError): + # Set to type arg. + t = get_args(self.__orig_class__)[0] + + if t is None: + t_repr = "<>" + elif isinstance(t, type): + if t.__module__ == "builtins": + t_repr = t.__qualname__ + else: + t_repr = f"{t.__module__}.{t.__qualname__}" + else: + t_repr = repr(t) + return f"<{self.__class__.__name__}({self._name}, type={t_repr})>" + + +class AppKey(BaseKey[_T]): + """Keys for static typing support in Application.""" + + +class RequestKey(BaseKey[_T]): + """Keys for static typing support in Request.""" + + +class ResponseKey(BaseKey[_T]): + """Keys for static typing support in Response.""" + + +class ChainMapProxy(Mapping[str | AppKey[Any], Any]): + __slots__ = ("_maps",) + + def __init__(self, maps: Iterable[Mapping[str | AppKey[Any], Any]]) -> None: + self._maps = tuple(maps) + + def __init_subclass__(cls) -> None: + raise TypeError( + f"Inheritance class {cls.__name__} from ChainMapProxy is forbidden" + ) + + @overload # type: ignore[override] + def __getitem__(self, key: AppKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: str | AppKey[_T]) -> Any: + for mapping in self._maps: + try: + return mapping[key] + except KeyError: + pass + raise KeyError(key) + + @overload # type: ignore[override] + def get(self, key: AppKey[_T], default: _S) -> _T | _S: ... + + @overload + def get(self, key: AppKey[_T], default: None = ...) -> _T | None: ... + + @overload + def get(self, key: str, default: Any = ...) -> Any: ... + + def get(self, key: str | AppKey[_T], default: Any = None) -> Any: + try: + return self[key] + except KeyError: + return default + + def __len__(self) -> int: + # reuses stored hash values if possible + return len(set().union(*self._maps)) + + def __iter__(self) -> Iterator[str | AppKey[Any]]: + d: dict[str | AppKey[Any], Any] = {} + for mapping in reversed(self._maps): + # reuses stored hash values if possible + d.update(mapping) + return iter(d) + + def __contains__(self, key: object) -> bool: + return any(key in m for m in self._maps) + + def __bool__(self) -> bool: + return any(self._maps) + + def __repr__(self) -> str: + content = ", ".join(map(repr, self._maps)) + return f"ChainMapProxy({content})" + + +# https://tools.ietf.org/html/rfc7232#section-2.3 +_ETAGC = r"[!\x23-\x7E\x80-\xff]+" +_ETAGC_RE = re.compile(_ETAGC) +_QUOTED_ETAG = rf'(W/)?"({_ETAGC})"' +QUOTED_ETAG_RE = re.compile(_QUOTED_ETAG) +LIST_QUOTED_ETAG_RE = re.compile(rf"({_QUOTED_ETAG})(?:\s*,\s*|$)|(.)") + +ETAG_ANY = "*" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ETag: + value: str + is_weak: bool = False + + +def validate_etag_value(value: str) -> None: + if value != ETAG_ANY and not _ETAGC_RE.fullmatch(value): + raise ValueError( + f"Value {value!r} is not a valid etag. Maybe it contains '\"'?" + ) + + +def parse_http_date(date_str: str | None) -> datetime.datetime | None: + """Process a date string, return a datetime object""" + if date_str is not None: + timetuple = parsedate(date_str) + if timetuple is not None: + with suppress(ValueError): + return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) + return None + + +@functools.lru_cache +def must_be_empty_body(method: str, code: int) -> bool: + """Check if a request must return an empty body.""" + return ( + code in EMPTY_BODY_STATUS_CODES + or method in EMPTY_BODY_METHODS + or (200 <= code < 300 and method in hdrs.METH_CONNECT_ALL) + ) + + +def should_remove_content_length(method: str, code: int) -> bool: + """Check if a Content-Length header should be removed. + + This should always be a subset of must_be_empty_body + """ + # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-8 + # https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4.5-4 + return code in EMPTY_BODY_STATUS_CODES or ( + 200 <= code < 300 and method in hdrs.METH_CONNECT_ALL + ) diff --git a/venv/Lib/site-packages/aiohttp/http.py b/venv/Lib/site-packages/aiohttp/http.py new file mode 100644 index 0000000000000000000000000000000000000000..67d5de119fe38dd8e1b0d4e3a80ff5ba52542cac --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/http.py @@ -0,0 +1,78 @@ +import sys +from collections.abc import Mapping +from http import HTTPStatus + +from . import __version__ +from .http_exceptions import HttpProcessingError as HttpProcessingError +from .http_parser import ( + HeadersParser as HeadersParser, + HttpParser as HttpParser, + HttpRequestParser as HttpRequestParser, + HttpResponseParser as HttpResponseParser, + RawRequestMessage as RawRequestMessage, + RawResponseMessage as RawResponseMessage, +) +from .http_websocket import ( + WS_CLOSED_MESSAGE as WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE as WS_CLOSING_MESSAGE, + WS_KEY as WS_KEY, + WebSocketError as WebSocketError, + WebSocketReader as WebSocketReader, + WebSocketWriter as WebSocketWriter, + WSCloseCode as WSCloseCode, + WSMessage as WSMessage, + WSMessageDecodeText as WSMessageDecodeText, + WSMessageNoDecodeText as WSMessageNoDecodeText, + WSMessageTextBytes as WSMessageTextBytes, + WSMsgType as WSMsgType, + ws_ext_gen as ws_ext_gen, + ws_ext_parse as ws_ext_parse, +) +from .http_writer import ( + HttpVersion as HttpVersion, + HttpVersion10 as HttpVersion10, + HttpVersion11 as HttpVersion11, + StreamWriter as StreamWriter, +) + +__all__ = ( + "HttpProcessingError", + "RESPONSES", + "SERVER_SOFTWARE", + # .http_writer + "StreamWriter", + "HttpVersion", + "HttpVersion10", + "HttpVersion11", + # .http_parser + "HeadersParser", + "HttpParser", + "HttpRequestParser", + "HttpResponseParser", + "RawRequestMessage", + "RawResponseMessage", + # .http_websocket + "WS_CLOSED_MESSAGE", + "WS_CLOSING_MESSAGE", + "WS_KEY", + "WebSocketReader", + "WebSocketWriter", + "ws_ext_gen", + "ws_ext_parse", + "WSMessage", + "WSMessageDecodeText", + "WSMessageNoDecodeText", + "WSMessageTextBytes", + "WebSocketError", + "WSMsgType", + "WSCloseCode", +) + + +SERVER_SOFTWARE: str = ( + f"Python/{sys.version_info[0]}.{sys.version_info[1]} aiohttp/{__version__}" +) + +RESPONSES: Mapping[int, tuple[str, str]] = { + v: (v.phrase, v.description) for v in HTTPStatus.__members__.values() +} diff --git a/venv/Lib/site-packages/aiohttp/http_exceptions.py b/venv/Lib/site-packages/aiohttp/http_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..c2bb87be5ceb0b209f3b99ad2eeb2ae6a8b2f25f --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/http_exceptions.py @@ -0,0 +1,118 @@ +"""Low-level http related exceptions.""" + +from textwrap import indent + +from .typedefs import _CIMultiDict + +__all__ = ("HttpProcessingError",) + + +class HttpProcessingError(Exception): + """HTTP error. + + Shortcut for raising HTTP errors with custom code, message and headers. + + code: HTTP Error code. + message: (optional) Error message. + headers: (optional) Headers to be sent in response, a list of pairs + """ + + code = 0 + message = "" + headers = None + + def __init__( + self, + *, + code: int | None = None, + message: str = "", + headers: _CIMultiDict | None = None, + ) -> None: + if code is not None: + self.code = code + self.headers = headers + self.message = message + + def __str__(self) -> str: + msg = indent(self.message, " ") + return f"{self.code}, message:\n{msg}" + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {self.code}, message={self.message!r}>" + + +class BadHttpMessage(HttpProcessingError): + + code = 400 + message = "Bad Request" + + def __init__(self, message: str, *, headers: _CIMultiDict | None = None) -> None: + super().__init__(message=message, headers=headers) + self.args = (message,) + + +class HttpBadRequest(BadHttpMessage): + + code = 400 + message = "Bad Request" + + +class PayloadEncodingError(BadHttpMessage): + """Base class for payload errors""" + + +class ContentEncodingError(PayloadEncodingError): + """Content encoding error.""" + + +class TransferEncodingError(PayloadEncodingError): + """transfer encoding error.""" + + +class ContentLengthError(PayloadEncodingError): + """Not enough data to satisfy content length header.""" + + +class DecompressSizeError(PayloadEncodingError): + """Deprecated. Removed in v4.""" + + +class LineTooLong(BadHttpMessage): + def __init__( + self, + line: str | bytes, + limit: str | int = "Unknown", + actual_size: str = "Unknown", + ) -> None: + super().__init__(f"Got more than {limit} bytes when reading: {line!r}.") + self.args = (line, limit, actual_size) + + +class InvalidHeader(BadHttpMessage): + def __init__(self, hdr: bytes | str) -> None: + hdr_s = hdr.decode(errors="backslashreplace") if isinstance(hdr, bytes) else hdr + super().__init__(f"Invalid HTTP header: {hdr!r}") + self.hdr = hdr_s + self.args = (hdr,) + + +class BadStatusLine(BadHttpMessage): + def __init__(self, line: str = "", error: str | None = None) -> None: + if not isinstance(line, str): + line = repr(line) + super().__init__(error or f"Bad status line {line!r}") + self.args = (line,) + self.line = line + + +class BadHttpMethod(BadStatusLine): + """Invalid HTTP method in status line.""" + + def __init__(self, line: str = "", error: str | None = None) -> None: + if error is None and line.startswith("\x16\x03"): + error = "Received HTTPS traffic on an HTTP port" + super().__init__(line, error or f"Bad HTTP method in status line {line!r}") + + +class InvalidURLError(BadHttpMessage): + pass diff --git a/venv/Lib/site-packages/aiohttp/http_parser.py b/venv/Lib/site-packages/aiohttp/http_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..ed928e1858e7768a5f96b05ccd0bb5bb1c4a92ec --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/http_parser.py @@ -0,0 +1,1217 @@ +import abc +import asyncio +import re +import string +import sys +from contextlib import suppress +from enum import IntEnum +from re import Pattern +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Final, + Generic, + Literal, + NamedTuple, + TypeVar, +) + +from multidict import CIMultiDict, CIMultiDictProxy, istr +from yarl import URL + +from . import hdrs +from .base_protocol import BaseProtocol +from .compression_utils import ( + HAS_BROTLI, + HAS_ZSTD, + BrotliDecompressor, + ZLibDecompressor, + ZSTDDecompressor, +) +from .helpers import ( + _EXC_SENTINEL, + DEBUG, + DEFAULT_CHUNK_SIZE, + EMPTY_BODY_METHODS, + EMPTY_BODY_STATUS_CODES, + NO_EXTENSIONS, + BaseTimerContext, + set_exception, +) +from .http_exceptions import ( + BadHttpMessage, + BadHttpMethod, + BadStatusLine, + ContentEncodingError, + ContentLengthError, + InvalidHeader, + InvalidURLError, + LineTooLong, + TransferEncodingError, +) +from .http_writer import HttpVersion, HttpVersion10, HttpVersion11 +from .streams import EMPTY_PAYLOAD, StreamReader +from .typedefs import RawHeaders + +if TYPE_CHECKING: + from .client_proto import ResponseHandler + +__all__ = ( + "HeadersParser", + "HttpParser", + "HttpRequestParser", + "HttpResponseParser", + "RawRequestMessage", + "RawResponseMessage", +) + +_SEP = Literal[b"\r\n", b"\n"] + +ASCIISET: Final[set[str]] = set(string.printable) + +# See https://www.rfc-editor.org/rfc/rfc9110.html#name-overview +# and https://www.rfc-editor.org/rfc/rfc9110.html#name-tokens +# +# method = token +# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / +# "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA +# token = 1*tchar +_TCHAR_SPECIALS: Final[str] = re.escape("!#$%&'*+-.^_`|~") +TOKENRE: Final[Pattern[str]] = re.compile(f"[0-9A-Za-z{_TCHAR_SPECIALS}]+") +VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d)\.(\d)", re.ASCII) +DIGITS: Final[Pattern[str]] = re.compile(r"\d+", re.ASCII) +HEXDIGITS: Final[Pattern[bytes]] = re.compile(rb"[0-9a-fA-F]+") +# https://www.rfc-editor.org/rfc/rfc9110#section-5.5-5 +_FIELD_VALUE_FORBIDDEN_CTL_RE: Final[Pattern[str]] = re.compile( + r"[\x00-\x08\x0a-\x1f\x7f]" +) + +# RFC 9110 singleton headers β€” duplicates are rejected in strict mode. +# In lax mode (response parser default), the check is skipped entirely +# since real-world servers (e.g. Google APIs, Werkzeug) commonly send +# duplicate headers like Content-Type or Server. +# Lowercased for case-insensitive matching against wire names. +SINGLETON_HEADERS: Final[frozenset[str]] = frozenset( + { + "content-length", + "content-location", + "content-range", + "content-type", + "etag", + "host", + "max-forwards", + "server", + "transfer-encoding", + "user-agent", + } +) + + +class RawRequestMessage(NamedTuple): + method: str + path: str + version: HttpVersion + headers: "CIMultiDictProxy[str]" + raw_headers: RawHeaders + should_close: bool + compression: str | None + upgrade: bool + chunked: bool + url: URL + + +class RawResponseMessage(NamedTuple): + version: HttpVersion + code: int + reason: str + headers: CIMultiDictProxy[str] + raw_headers: RawHeaders + should_close: bool + compression: str | None + upgrade: bool + chunked: bool + + +_MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage) + + +class PayloadState(IntEnum): + PAYLOAD_COMPLETE = 0 + PAYLOAD_NEEDS_INPUT = 1 + PAYLOAD_HAS_PENDING_INPUT = 2 + + +class ParseState(IntEnum): + + PARSE_NONE = 0 + PARSE_LENGTH = 1 + PARSE_CHUNKED = 2 + PARSE_UNTIL_EOF = 3 + + +class ChunkState(IntEnum): + PARSE_CHUNKED_SIZE = 0 + PARSE_CHUNKED_CHUNK = 1 + PARSE_CHUNKED_CHUNK_EOF = 2 + PARSE_MAYBE_TRAILERS = 3 + PARSE_TRAILERS = 4 + + +class HeadersParser: + def __init__( + self, + max_line_size: int = 8190, + max_headers: int = 32768, + max_field_size: int = 8190, + lax: bool = False, + ) -> None: + self.max_line_size = max_line_size + self.max_headers = max_headers + self.max_field_size = max_field_size + self._lax = lax + + def parse_headers( + self, lines: list[bytes] + ) -> tuple["CIMultiDictProxy[str]", RawHeaders]: + headers: CIMultiDict[str] = CIMultiDict() + # note: "raw" does not mean inclusion of OWS before/after the field value + raw_headers = [] + + lines_idx = 0 + line = lines[lines_idx] + line_count = len(lines) + + while line: + # Parse initial header name : value pair. + try: + bname, bvalue = line.split(b":", 1) + except ValueError: + raise InvalidHeader(line) from None + + if len(bname) == 0: + raise InvalidHeader(bname) + + # https://www.rfc-editor.org/rfc/rfc9112.html#section-5.1-2 + if {bname[0], bname[-1]} & {32, 9}: # {" ", "\t"} + raise InvalidHeader(line) + + bvalue = bvalue.lstrip(b" \t") + name = bname.decode("utf-8", "surrogateescape") + if not TOKENRE.fullmatch(name): + raise InvalidHeader(bname) + + # next line + lines_idx += 1 + line = lines[lines_idx] + + # consume continuation lines + continuation = self._lax and line and line[0] in (32, 9) # (' ', '\t') + + # Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding + if continuation: + header_length = len(bvalue) + bvalue_lst = [bvalue] + while continuation: + header_length += len(line) + if header_length > self.max_field_size: + header_line = bname + b": " + b"".join(bvalue_lst) + raise LineTooLong( + header_line[:100] + b"...", self.max_field_size + ) + bvalue_lst.append(line) + + # next line + lines_idx += 1 + if lines_idx < line_count: + line = lines[lines_idx] + if line: + continuation = line[0] in (32, 9) # (' ', '\t') + else: + line = b"" + break + bvalue = b"".join(bvalue_lst) + + bvalue = bvalue.strip(b" \t") + value = bvalue.decode("utf-8", "surrogateescape") + + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-5 + if self._lax: + if "\n" in value or "\r" in value or "\x00" in value: + raise InvalidHeader(bvalue) + elif _FIELD_VALUE_FORBIDDEN_CTL_RE.search(value): + raise InvalidHeader(bvalue) + + if not self._lax and name in headers and name.lower() in SINGLETON_HEADERS: + raise BadHttpMessage(f"Duplicate '{name}' header found.") + headers.add(name, value) + raw_headers.append((bname, bvalue)) + + return (CIMultiDictProxy(headers), tuple(raw_headers)) + + +def _is_supported_upgrade(headers: CIMultiDictProxy[str]) -> bool: + """Check if the upgrade header is supported.""" + u = headers.get(hdrs.UPGRADE, "") + # .lower() can transform non-ascii characters. + return u.isascii() and u.lower() in {"tcp", "websocket"} + + +class HttpParser(abc.ABC, Generic[_MsgT]): + lax: ClassVar[bool] = False + + def __init__( + self, + protocol: BaseProtocol | None = None, + loop: asyncio.AbstractEventLoop | None = None, + limit: int = 2**16, + max_line_size: int = 8190, + max_headers: int = 128, + max_field_size: int = 8190, + timer: BaseTimerContext | None = None, + code: int | None = None, + method: str | None = None, + payload_exception: type[BaseException] | None = None, + response_with_body: bool = True, + read_until_eof: bool = False, + auto_decompress: bool = True, + max_msg_queue_size: int = 0, + ) -> None: + self.protocol = protocol + self.loop = loop + self.max_line_size = max_line_size + self.max_headers = max_headers + self.max_field_size = max_field_size + self.max_headers = max_headers + self.timer = timer + self.code = code + self.method = method + self.payload_exception = payload_exception + self.response_with_body = response_with_body + self.read_until_eof = read_until_eof + + self._lines: list[bytes] = [] + self._tail = b"" + self._upgraded = False + self._payload = None + self._payload_parser: HttpPayloadParser | None = None + self._payload_has_more_data = False + self._auto_decompress = auto_decompress + self._limit = limit + self._headers_parser = HeadersParser( + max_line_size, max_headers, max_field_size, self.lax + ) + # Stop emitting messages once this many are queued unconsumed (0 = off). + self._max_msg_queue_size = max_msg_queue_size + self._msg_in_flight = 0 + + @abc.abstractmethod + def parse_message(self, lines: list[bytes]) -> _MsgT: ... + + @abc.abstractmethod + def _is_chunked_te(self, te: str) -> bool: ... + + def pause_reading(self) -> None: + assert self._payload_parser is not None + self._payload_parser.pause_reading() + + def message_consumed(self) -> None: + """Protocol drained a queued message; free a slot for parsing.""" + if self._msg_in_flight > 0: + self._msg_in_flight -= 1 + + def feed_eof(self) -> _MsgT | None: + if self._payload_parser is not None: + self._payload_parser.feed_eof() + if self._payload_parser.done: + self._payload_parser = None + else: + # try to extract partial message + if self._tail: + self._lines.append(self._tail) + + if self._lines: + if self._lines[-1] != "\r\n": + self._lines.append(b"") + with suppress(Exception): + return self.parse_message(self._lines) + return None + + def feed_data( + self, + data: bytes, + SEP: _SEP = b"\r\n", + EMPTY: bytes = b"", + CONTENT_LENGTH: istr = hdrs.CONTENT_LENGTH, + METH_CONNECT: str = hdrs.METH_CONNECT, + SEC_WEBSOCKET_KEY1: istr = hdrs.SEC_WEBSOCKET_KEY1, + ) -> tuple[list[tuple[_MsgT, StreamReader]], bool, bytes]: + + messages = [] + + if self._tail: + data, self._tail = self._tail + data, b"" + + data_len = len(data) + start_pos = 0 + loop = self.loop + max_line_length = self.max_line_size + + should_close = False + while start_pos < data_len or self._payload_has_more_data: + # read HTTP message (request/response line + headers), \r\n\r\n + # and split by lines + if self._payload_parser is None and not self._upgraded: + if ( + self._max_msg_queue_size + and self._msg_in_flight >= self._max_msg_queue_size + ): + # Queue full: buffer the rest and stop. Safe pause point; + # any preceding body is consumed before the next request + # line. Resumes via feed_data(b"") when the queue drains. + self._tail = data[start_pos:] + break + pos = data.find(SEP, start_pos) + # consume \r\n + if pos == start_pos and not self._lines: + start_pos = pos + len(SEP) + continue + + if pos >= start_pos: + if should_close: + raise BadHttpMessage("Data after `Connection: close`") + + # line found + line = data[start_pos:pos] + if SEP == b"\n": # For lax response parsing + line = line.rstrip(b"\r") + if len(line) > max_line_length: + raise LineTooLong(line[:100] + b"...", max_line_length) + + self._lines.append(line) + # After processing the status/request line, everything is a header. + max_line_length = self.max_field_size + + if len(self._lines) > self.max_headers: + raise BadHttpMessage("Too many headers received") + + start_pos = pos + len(SEP) + + # \r\n\r\n found + if self._lines[-1] == EMPTY: + max_trailers = self.max_headers - len(self._lines) + try: + msg: _MsgT = self.parse_message(self._lines) + finally: + self._lines.clear() + + def get_content_length() -> int | None: + # payload length + length_hdr = msg.headers.get(CONTENT_LENGTH) + if length_hdr is None: + return None + + # Shouldn't allow +/- or other number formats. + # https://www.rfc-editor.org/rfc/rfc9110#section-8.6-2 + # msg.headers is already stripped of leading/trailing wsp + if not DIGITS.fullmatch(length_hdr): + raise InvalidHeader(CONTENT_LENGTH) + + return int(length_hdr) + + length = get_content_length() + # do not support old websocket spec + if SEC_WEBSOCKET_KEY1 in msg.headers: + raise InvalidHeader(SEC_WEBSOCKET_KEY1) + + self._upgraded = msg.upgrade and _is_supported_upgrade( + msg.headers + ) + + method = getattr(msg, "method", self.method) + # code is only present on responses + code = getattr(msg, "code", 0) + + assert self.protocol is not None + # calculate payload + empty_body = code in EMPTY_BODY_STATUS_CODES or bool( + method and method in EMPTY_BODY_METHODS + ) + if not empty_body and ( + ((length is not None and length > 0) or msg.chunked) + and not self._upgraded + ): + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + payload_parser = HttpPayloadParser( + payload, + length=length, + chunked=msg.chunked, + method=method, + compression=msg.compression, + code=self.code, + response_with_body=self.response_with_body, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, + limit=self._limit, + ) + if not payload_parser.done: + self._payload_parser = payload_parser + elif method == METH_CONNECT: + assert isinstance(msg, RawRequestMessage) + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + self._upgraded = True + self._payload_parser = HttpPayloadParser( + payload, + method=msg.method, + compression=msg.compression, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, + limit=self._limit, + ) + elif not empty_body and length is None and self.read_until_eof: + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + payload_parser = HttpPayloadParser( + payload, + length=length, + chunked=msg.chunked, + method=method, + compression=msg.compression, + code=self.code, + response_with_body=self.response_with_body, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, + limit=self._limit, + ) + if not payload_parser.done: + self._payload_parser = payload_parser + else: + payload = EMPTY_PAYLOAD + + messages.append((msg, payload)) + if self._max_msg_queue_size: + self._msg_in_flight += 1 + should_close = msg.should_close + else: + self._tail = data[start_pos:] + if len(self._tail) > self.max_line_size: + raise LineTooLong(self._tail[:100] + b"...", self.max_line_size) + data = EMPTY + break + + # no parser, just store + elif self._payload_parser is None and self._upgraded: + assert not self._lines + break + + # feed payload + else: + assert not self._lines + assert self._payload_parser is not None + try: + payload_state, data = self._payload_parser.feed_data( + data[start_pos:], SEP + ) + except Exception as underlying_exc: + reraised_exc: BaseException = underlying_exc + if self.payload_exception is not None: + reraised_exc = self.payload_exception(str(underlying_exc)) + + set_exception( + self._payload_parser.payload, + reraised_exc, + underlying_exc, + ) + + payload_state = PayloadState.PAYLOAD_COMPLETE + data = b"" + if isinstance( + underlying_exc, (InvalidHeader, TransferEncodingError) + ): + raise + + self._payload_has_more_data = ( + payload_state == PayloadState.PAYLOAD_HAS_PENDING_INPUT + ) + + if payload_state is not PayloadState.PAYLOAD_COMPLETE: + # We've either consumed all available data, or we're pausing + # until the reader buffer is freed up. + break + + start_pos = 0 + data_len = len(data) + self._payload_parser = None + + if data and start_pos < data_len: + data = data[start_pos:] + else: + data = EMPTY + + return messages, self._upgraded, data + + def parse_headers( + self, lines: list[bytes] + ) -> tuple[ + "CIMultiDictProxy[str]", RawHeaders, bool | None, str | None, bool, bool + ]: + """Parses RFC 5322 headers from a stream. + + Line continuations are supported. Returns list of header name + and value pairs. Header name is in upper case. + """ + headers, raw_headers = self._headers_parser.parse_headers(lines) + close_conn = None + encoding = None + upgrade = False + chunked = False + + # keep-alive and protocol switching + # RFC 9110 section 7.6.1 defines Connection as a comma-separated list. + conn_values = headers.getall(hdrs.CONNECTION, ()) + if conn_values: + conn_tokens = { + token.lower() + for conn_value in conn_values + for token in (part.strip(" \t") for part in conn_value.split(",")) + if token and token.isascii() + } + + if "close" in conn_tokens: + close_conn = True + elif "keep-alive" in conn_tokens: + close_conn = False + + # https://www.rfc-editor.org/rfc/rfc9110.html#name-101-switching-protocols + if "upgrade" in conn_tokens and headers.get(hdrs.UPGRADE): + upgrade = True + + # encoding + enc = headers.get(hdrs.CONTENT_ENCODING, "") + if enc.isascii() and enc.lower() in {"gzip", "deflate", "br", "zstd"}: + encoding = enc + + # chunking + te = headers.get(hdrs.TRANSFER_ENCODING) + if te is not None: + if self._is_chunked_te(te): + chunked = True + + if hdrs.CONTENT_LENGTH in headers: + raise BadHttpMessage( + "Transfer-Encoding can't be present with Content-Length", + ) + + return (headers, raw_headers, close_conn, encoding, upgrade, chunked) + + def set_upgraded(self, val: bool) -> None: + """Set connection upgraded (to websocket) mode. + + :param bool val: new state. + """ + self._upgraded = val + + +class HttpRequestParser(HttpParser[RawRequestMessage]): + """Read request status line. + + Exception .http_exceptions.BadStatusLine + could be raised in case of any errors in status line. + Returns RawRequestMessage. + """ + + def parse_message(self, lines: list[bytes]) -> RawRequestMessage: + # request line + line = lines[0].decode("utf-8", "surrogateescape") + try: + method, path, version = line.split(" ", maxsplit=2) + except ValueError: + raise BadHttpMethod(line) from None + + # method + if not TOKENRE.fullmatch(method): + raise BadHttpMethod(method) + + # version + match = VERSRE.fullmatch(version) + if match is None: + raise BadStatusLine(line) + version_o = HttpVersion(int(match.group(1)), int(match.group(2))) + + if method == "CONNECT": + # authority-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3 + url = URL.build(authority=path, encoded=True) + elif path.startswith("/"): + # origin-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1 + path_part, _hash_separator, url_fragment = path.partition("#") + path_part, _question_mark_separator, qs_part = path_part.partition("?") + + # NOTE: `yarl.URL.build()` is used to mimic what the Cython-based + # NOTE: parser does, otherwise it results into the same + # NOTE: HTTP Request-Line input producing different + # NOTE: `yarl.URL()` objects + url = URL.build( + path=path_part, + query_string=qs_part, + fragment=url_fragment, + encoded=True, + ) + elif path == "*" and method == "OPTIONS": + # asterisk-form, + url = URL(path, encoded=True) + else: + # absolute-form for proxy maybe, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2 + url = URL(path, encoded=True) + if url.scheme == "": + # not absolute-form + raise InvalidURLError( + path.encode(errors="surrogateescape").decode("latin1") + ) + + # read headers + ( + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) = self.parse_headers(lines[1:]) + + if version_o == HttpVersion11 and hdrs.HOST not in headers: + raise BadHttpMessage("Missing 'Host' header in request.") + + if close is None: # then the headers weren't set in the request + if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close + close = True + else: # HTTP 1.1 must ask to close. + close = False + + return RawRequestMessage( + method, + path, + version_o, + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + url, + ) + + def _is_chunked_te(self, te: str) -> bool: + te = te.rsplit(",", maxsplit=1)[-1].strip(" \t") + # .lower() transforms some non-ascii chars, so must check first. + if te.isascii() and te.lower() == "chunked": + return True + # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.3 + raise BadHttpMessage("Request has invalid `Transfer-Encoding`") + + +class HttpResponseParser(HttpParser[RawResponseMessage]): + """Read response status line and headers. + + BadStatusLine could be raised in case of any errors in status line. + Returns RawResponseMessage. + """ + + protocol: "ResponseHandler" + + # Lax mode should only be enabled on response parser. + lax = not DEBUG + + def feed_data( + self, + data: bytes, + SEP: _SEP | None = None, + *args: Any, + **kwargs: Any, + ) -> tuple[list[tuple[RawResponseMessage, StreamReader]], bool, bytes]: + if SEP is None: + SEP = b"\r\n" if DEBUG else b"\n" + return super().feed_data(data, SEP, *args, **kwargs) + + def parse_message(self, lines: list[bytes]) -> RawResponseMessage: + line = lines[0].decode("utf-8", "surrogateescape") + try: + version, status = line.split(maxsplit=1) + except ValueError: + raise BadStatusLine(line) from None + + try: + status, reason = status.split(maxsplit=1) + except ValueError: + status = status.strip() + reason = "" + + # version + match = VERSRE.fullmatch(version) + if match is None: + raise BadStatusLine(line) + version_o = HttpVersion(int(match.group(1)), int(match.group(2))) + + # The status code is a three-digit ASCII number, no padding + if len(status) != 3 or not DIGITS.fullmatch(status): + raise BadStatusLine(line) + status_i = int(status) + + # read headers + ( + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) = self.parse_headers(lines[1:]) + + if close is None: + if version_o <= HttpVersion10: + close = True + # https://www.rfc-editor.org/rfc/rfc9112.html#name-message-body-length + elif 100 <= status_i < 200 or status_i in {204, 304}: + close = False + elif hdrs.CONTENT_LENGTH in headers or hdrs.TRANSFER_ENCODING in headers: + close = False + else: + # https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3-2.8 + close = True + + return RawResponseMessage( + version_o, + status_i, + reason.strip(), + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) + + def _is_chunked_te(self, te: str) -> bool: + # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.2 + return te.rsplit(",", maxsplit=1)[-1].strip(" \t").lower() == "chunked" + + +class HttpPayloadParser: + def __init__( + self, + payload: StreamReader, + length: int | None = None, + chunked: bool = False, + compression: str | None = None, + code: int | None = None, + method: str | None = None, + response_with_body: bool = True, + auto_decompress: bool = True, + lax: bool = False, + *, + headers_parser: HeadersParser, + max_line_size: int = 8190, + max_field_size: int = 8190, + max_trailers: int = 128, + limit: int = DEFAULT_CHUNK_SIZE, + ) -> None: + self._length = 0 + self._paused = False + self._type = ParseState.PARSE_UNTIL_EOF + self._chunk = ChunkState.PARSE_CHUNKED_SIZE + self._chunk_size = 0 + self._chunk_tail = b"" + self._auto_decompress = auto_decompress + self._lax = lax + self._headers_parser = headers_parser + self._max_line_size = max_line_size + self._max_field_size = max_field_size + self._max_trailers = max_trailers + self._more_data_available = False + self._trailer_lines: list[bytes] = [] + self.done = False + self._eof_pending = False + + # payload decompression wrapper + if response_with_body and compression and self._auto_decompress: + real_payload: StreamReader | DeflateBuffer = DeflateBuffer( + payload, compression, max_decompress_size=limit + ) + else: + real_payload = payload + + # payload parser + if not response_with_body: + # don't parse payload if it's not expected to be received + self._type = ParseState.PARSE_NONE + real_payload.feed_eof() + self.done = True + elif chunked: + self._type = ParseState.PARSE_CHUNKED + elif length is not None: + self._type = ParseState.PARSE_LENGTH + self._length = length + self._length_expected = length + if self._length == 0: + real_payload.feed_eof() + self.done = True + + self.payload = real_payload + + def pause_reading(self) -> None: + self._paused = True + + def feed_eof(self) -> None: + if self._type == ParseState.PARSE_UNTIL_EOF: + self._eof_pending = True + while self._more_data_available: + if self._paused: + self._paused = False + return # Will resume via feed_data(b"") later + self._more_data_available = self.payload.feed_data(b"", 0) + self.payload.feed_eof() + self.done = True + self._eof_pending = False + elif self._type == ParseState.PARSE_LENGTH: + received = self._length_expected - self._length + raise ContentLengthError( + f"Not enough data to satisfy content length header " + f"(received {received} of {self._length_expected} bytes)." + ) + elif self._type == ParseState.PARSE_CHUNKED: + raise TransferEncodingError( + "Not enough data to satisfy transfer length header." + ) + + def feed_data( + self, chunk: bytes, SEP: _SEP = b"\r\n", CHUNK_EXT: bytes = b";" + ) -> tuple[PayloadState, bytes]: + """Receive a chunk of data to process. + + Return: + PayloadState - The current state of payload processing. + This function may be called with empty bytes after returning + PAYLOAD_HAS_PENDING_INPUT to continue processing after a pause. + bytes - If payload is complete, this is the unconsumed bytes intended for the + next message/payload, b"" otherwise. + """ + # Read specified amount of bytes + if self._type == ParseState.PARSE_LENGTH: + if self._chunk_tail: + chunk = self._chunk_tail + chunk + self._chunk_tail = b"" + + required = self._length + self._length = max(required - len(chunk), 0) + self._more_data_available = self.payload.feed_data( + chunk[:required], required + ) + while self._more_data_available: + if self._paused: + self._paused = False + self._chunk_tail = chunk[required:] + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + self._more_data_available = self.payload.feed_data(b"", 0) + + if self._length == 0: + self.payload.feed_eof() + return PayloadState.PAYLOAD_COMPLETE, chunk[required:] + # Chunked transfer encoding parser + elif self._type == ParseState.PARSE_CHUNKED: + if self._chunk_tail: + # We should check the length is sane when not processing payload body. + if self._chunk != ChunkState.PARSE_CHUNKED_CHUNK: + max_line_length = self._max_line_size + if self._chunk == ChunkState.PARSE_TRAILERS: + max_line_length = self._max_field_size + if len(self._chunk_tail) > max_line_length: + raise LineTooLong( + self._chunk_tail[:100] + b"...", max_line_length + ) + + chunk = self._chunk_tail + chunk + self._chunk_tail = b"" + + while chunk or self._more_data_available: + # read next chunk size + if self._chunk == ChunkState.PARSE_CHUNKED_SIZE: + pos = chunk.find(SEP) + if pos >= 0: + # Only chunk-size lines reach here; trailers enforce + # _max_field_size separately in PARSE_TRAILERS below. + if pos > self._max_line_size: + raise LineTooLong(chunk[:100] + b"...", self._max_line_size) + i = chunk.find(CHUNK_EXT, 0, pos) + if i >= 0: + size_b = chunk[:i] # strip chunk-extensions + # Verify no LF in the chunk-extension + if b"\n" in (ext := chunk[i:pos]): + exc = TransferEncodingError( + f"Unexpected LF in chunk-extension: {ext!r}" + ) + set_exception(self.payload, exc) + raise exc + else: + size_b = chunk[:pos] + + if self._lax: # Allow whitespace in lax mode. + size_b = size_b.strip() + + if not re.fullmatch(HEXDIGITS, size_b): + exc = TransferEncodingError( + chunk[:pos].decode("ascii", "surrogateescape") + ) + set_exception(self.payload, exc) + raise exc + size = int(bytes(size_b), 16) + + chunk = chunk[pos + len(SEP) :] + if size == 0: # eof marker + self._chunk = ChunkState.PARSE_TRAILERS + if self._lax and chunk.startswith(b"\r"): + chunk = chunk[1:] + else: + self._chunk = ChunkState.PARSE_CHUNKED_CHUNK + self._chunk_size = size + self.payload.begin_http_chunk_receiving() + else: + self._chunk_tail = chunk + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + + # read chunk and feed buffer + if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK: + if self._paused: + self._paused = False + self._chunk_tail = chunk + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + + required = self._chunk_size + self._chunk_size = max(required - len(chunk), 0) + self._more_data_available = self.payload.feed_data( + chunk[:required], required + ) + chunk = chunk[required:] + + if self._more_data_available: + continue + + if self._chunk_size: + self._paused = False + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF + self.payload.end_http_chunk_receiving() + + # toss the CRLF at the end of the chunk + if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK_EOF: + if self._lax and chunk.startswith(b"\r"): + chunk = chunk[1:] + if chunk[: len(SEP)] == SEP: + chunk = chunk[len(SEP) :] + self._chunk = ChunkState.PARSE_CHUNKED_SIZE + elif len(chunk) >= len(SEP) or chunk != SEP[: len(chunk)]: + exc = TransferEncodingError( + "Chunk size mismatch: expected CRLF after chunk data" + ) + set_exception(self.payload, exc) + raise exc + else: + self._chunk_tail = chunk + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + + if self._chunk == ChunkState.PARSE_TRAILERS: + pos = chunk.find(SEP) + if pos < 0: # No line found + self._chunk_tail = chunk + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + + line = chunk[:pos] + chunk = chunk[pos + len(SEP) :] + if SEP == b"\n": # For lax response parsing + line = line.rstrip(b"\r") + + if len(line) > self._max_field_size: + raise LineTooLong(line[:100] + b"...", self._max_field_size) + + self._trailer_lines.append(line) + + if len(self._trailer_lines) > self._max_trailers: + raise BadHttpMessage("Too many trailers received") + + # \r\n\r\n found, end of stream + if self._trailer_lines[-1] == b"": + # Headers and trailers are defined the same way, + # so we reuse the HeadersParser here. + try: + trailers, raw_trailers = self._headers_parser.parse_headers( + self._trailer_lines + ) + finally: + self._trailer_lines.clear() + self.payload.feed_eof() + return PayloadState.PAYLOAD_COMPLETE, chunk + + # Read all bytes until eof + elif self._type == ParseState.PARSE_UNTIL_EOF: + self._more_data_available = self.payload.feed_data(chunk, len(chunk)) + while self._more_data_available: + if self._paused: + self._paused = False + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + self._more_data_available = self.payload.feed_data(b"", 0) + + if self._eof_pending: + self.payload.feed_eof() + self.done = True + self._eof_pending = False + return PayloadState.PAYLOAD_COMPLETE, b"" + + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + + +class DeflateBuffer: + """DeflateStream decompress stream and feed data into specified stream.""" + + decompressor: Any + + def __init__( + self, + out: StreamReader, + encoding: str | None, + max_decompress_size: int = DEFAULT_CHUNK_SIZE, + ) -> None: + self.out = out + self.size = 0 + out.total_compressed_bytes = self.size + self.encoding = encoding + self._started_decoding = False + + self.decompressor: BrotliDecompressor | ZLibDecompressor | ZSTDDecompressor + if encoding == "br": + if not HAS_BROTLI: # pragma: no cover + raise ContentEncodingError( + "Can not decode content-encoding: brotli (br). " + "Please install `Brotli`" + ) + self.decompressor = BrotliDecompressor() + elif encoding == "zstd": + if not HAS_ZSTD: + raise ContentEncodingError( + "Can not decode content-encoding: zstandard (zstd). " + "Please install `backports.zstd`" + ) + self.decompressor = ZSTDDecompressor() + else: + self.decompressor = ZLibDecompressor(encoding=encoding) + + self._max_decompress_size = max_decompress_size + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + set_exception(self.out, exc, exc_cause) + + def feed_data(self, chunk: bytes, size: int) -> bool: + self.size += size + self.out.total_compressed_bytes = self.size + + # RFC1950 + # bits 0..3 = CM = 0b1000 = 8 = "deflate" + # bits 4..7 = CINFO = 1..7 = windows size. + if ( + not self._started_decoding + and self.encoding == "deflate" + and chunk[0] & 0xF != 8 + ): + # Change the decoder to decompress incorrectly compressed data + # Actually we should issue a warning about non-RFC-compliant data. + self.decompressor = ZLibDecompressor( + encoding=self.encoding, suppress_deflate_header=True + ) + + low_water = self.out._low_water + max_length = ( + 0 if low_water >= sys.maxsize else max(self._max_decompress_size, low_water) + ) + try: + chunk = self.decompressor.decompress_sync(chunk, max_length=max_length) + except Exception: + raise ContentEncodingError( + "Can not decode content-encoding: %s" % self.encoding + ) + + self._started_decoding = True + + if chunk: + self.out.feed_data(chunk, len(chunk)) + return self.decompressor.data_available # type: ignore[no-any-return] + + def feed_eof(self) -> None: + chunk = self.decompressor.flush() + # This should never contain data as we defer the call until exhausting + # the decompression. If .flush() is returning data, this may indicate a + # zip bomb vulnerability as it will decompress all remaining data at once. + assert not chunk + + if self.size > 0: + if self.encoding == "deflate" and not self.decompressor.eof: + raise ContentEncodingError("deflate") + + self.out.feed_eof() + + def begin_http_chunk_receiving(self) -> None: + self.out.begin_http_chunk_receiving() + + def end_http_chunk_receiving(self) -> None: + self.out.end_http_chunk_receiving() + + +HttpRequestParserPy = HttpRequestParser +HttpResponseParserPy = HttpResponseParser +RawRequestMessagePy = RawRequestMessage +RawResponseMessagePy = RawResponseMessage + +try: + if not NO_EXTENSIONS: + from ._http_parser import ( # type: ignore[import-not-found,no-redef] + HttpRequestParser, + HttpResponseParser, + RawRequestMessage, + RawResponseMessage, + ) + + HttpRequestParserC = HttpRequestParser + HttpResponseParserC = HttpResponseParser + RawRequestMessageC = RawRequestMessage + RawResponseMessageC = RawResponseMessage +except ImportError: # pragma: no cover + pass diff --git a/venv/Lib/site-packages/aiohttp/http_websocket.py b/venv/Lib/site-packages/aiohttp/http_websocket.py new file mode 100644 index 0000000000000000000000000000000000000000..db81d5221a608da1929624cf3b83e42d14c1be37 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/http_websocket.py @@ -0,0 +1,41 @@ +"""WebSocket protocol versions 13 and 8.""" + +from ._websocket.helpers import WS_KEY, ws_ext_gen, ws_ext_parse +from ._websocket.models import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WebSocketError, + WSCloseCode, + WSHandshakeError, + WSMessage, + WSMessageDecodeText, + WSMessageNoDecodeText, + WSMessageTextBytes, + WSMsgType, +) +from ._websocket.reader import WebSocketReader +from ._websocket.writer import WebSocketWriter + +# Messages that the WebSocketResponse.receive needs to handle internally +_INTERNAL_RECEIVE_TYPES = frozenset( + (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.PING, WSMsgType.PONG) +) + + +__all__ = ( + "WS_CLOSED_MESSAGE", + "WS_CLOSING_MESSAGE", + "WS_KEY", + "WebSocketReader", + "WebSocketWriter", + "WSMessage", + "WSMessageDecodeText", + "WSMessageNoDecodeText", + "WSMessageTextBytes", + "WebSocketError", + "WSMsgType", + "WSCloseCode", + "ws_ext_gen", + "ws_ext_parse", + "WSHandshakeError", +) diff --git a/venv/Lib/site-packages/aiohttp/http_writer.py b/venv/Lib/site-packages/aiohttp/http_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..89f9d71e870e3a390c4f84e0bc393e95e504fcd7 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/http_writer.py @@ -0,0 +1,381 @@ +"""Http related parsers and protocol.""" + +import asyncio +import re +import sys +from typing import ( # noqa + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Iterable, + List, + NamedTuple, + Optional, +) + +from multidict import CIMultiDict + +from .abc import AbstractStreamWriter +from .base_protocol import BaseProtocol +from .client_exceptions import ClientConnectionResetError +from .compression_utils import ZLibCompressor +from .helpers import NO_EXTENSIONS + +__all__ = ("StreamWriter", "HttpVersion", "HttpVersion10", "HttpVersion11") + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing import Union + + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + + +MIN_PAYLOAD_FOR_WRITELINES = 2048 +IS_PY313_BEFORE_313_2 = (3, 13, 0) <= sys.version_info < (3, 13, 2) +IS_PY_BEFORE_312_9 = sys.version_info < (3, 12, 9) +SKIP_WRITELINES = IS_PY313_BEFORE_313_2 or IS_PY_BEFORE_312_9 +# writelines is not safe for use +# on Python 3.12+ until 3.12.9 +# on Python 3.13+ until 3.13.2 +# and on older versions it not any faster than write +# CVE-2024-12254: https://github.com/python/cpython/pull/127656 + + +class HttpVersion(NamedTuple): + major: int + minor: int + + +HttpVersion10 = HttpVersion(1, 0) +HttpVersion11 = HttpVersion(1, 1) + + +_T_OnChunkSent = Optional[Callable[[Buffer], Awaitable[None]]] +_T_OnHeadersSent = Optional[Callable[["CIMultiDict[str]"], Awaitable[None]]] + + +class StreamWriter(AbstractStreamWriter): + + length: int | None = None + chunked: bool = False + _eof: bool = False + _compress: ZLibCompressor | None = None + + def __init__( + self, + protocol: BaseProtocol, + loop: asyncio.AbstractEventLoop, + on_chunk_sent: _T_OnChunkSent = None, + on_headers_sent: _T_OnHeadersSent = None, + ) -> None: + self._protocol = protocol + self.loop = loop + self._on_chunk_sent: _T_OnChunkSent = on_chunk_sent + self._on_headers_sent: _T_OnHeadersSent = on_headers_sent + self._headers_buf: bytes | None = None + self._headers_written: bool = False + + @property + def transport(self) -> asyncio.Transport | None: + return self._protocol.transport + + @property + def protocol(self) -> BaseProtocol: + return self._protocol + + def enable_chunking(self) -> None: + self.chunked = True + + def enable_compression( + self, encoding: str = "deflate", strategy: int | None = None + ) -> None: + self._compress = ZLibCompressor(encoding=encoding, strategy=strategy) + + def _write(self, chunk: Buffer) -> None: + size = len(chunk) + self.buffer_size += size + self.output_size += size + transport = self._protocol.transport + if transport is None or transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + transport.write(chunk) + + def _writelines(self, chunks: Iterable[Buffer]) -> None: + size = 0 + for chunk in chunks: + size += len(chunk) + self.buffer_size += size + self.output_size += size + transport = self._protocol.transport + if transport is None or transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + if SKIP_WRITELINES or size < MIN_PAYLOAD_FOR_WRITELINES: + transport.write(b"".join(chunks)) + else: + transport.writelines(chunks) + + def _write_chunked_payload(self, chunk: Buffer) -> None: + """Write a chunk with proper chunked encoding.""" + chunk_len_pre = f"{len(chunk):x}\r\n".encode("ascii") + self._writelines((chunk_len_pre, chunk, b"\r\n")) + + def _send_headers_with_payload(self, chunk: Buffer, is_eof: bool) -> None: + """Send buffered headers with payload, coalescing into single write.""" + # Mark headers as written + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because callers (write() and write_eof()) only invoke this method + # after checking that self._headers_buf is truthy + assert headers_buf is not None + + if not self.chunked: + # Non-chunked: coalesce headers with body + if chunk: + self._writelines((headers_buf, chunk)) + else: + self._write(headers_buf) + return + + # Coalesce headers with chunked data + if chunk: + chunk_len_pre = f"{len(chunk):x}\r\n".encode("ascii") + if is_eof: + self._writelines((headers_buf, chunk_len_pre, chunk, b"\r\n0\r\n\r\n")) + else: + self._writelines((headers_buf, chunk_len_pre, chunk, b"\r\n")) + elif is_eof: + self._writelines((headers_buf, b"0\r\n\r\n")) + else: + self._write(headers_buf) + + async def write( + self, chunk: Buffer, *, drain: bool = True, LIMIT: int = 0x10000 + ) -> None: + """ + Writes chunk of data to a stream. + + write_eof() indicates end of stream. + writer can't be used after write_eof() method being called. + write() return drain future. + """ + if self._on_chunk_sent is not None: + await self._on_chunk_sent(chunk) + + if isinstance(chunk, memoryview): + if chunk.nbytes != len(chunk): + # just reshape it + chunk = chunk.cast("c") + + if self._compress is not None: + chunk = await self._compress.compress(chunk) + if not chunk: + return + + if self.length is not None: + chunk_len = len(chunk) + if self.length >= chunk_len: + self.length = self.length - chunk_len + else: + chunk = chunk[: self.length] + self.length = 0 + if not chunk: + return + + # Handle buffered headers for small payload optimization + if self._headers_buf and not self._headers_written: + self._send_headers_with_payload(chunk, False) + if drain and self.buffer_size > LIMIT: + self.buffer_size = 0 + await self.drain() + return + + if chunk: + if self.chunked: + self._write_chunked_payload(chunk) + else: + self._write(chunk) + + if drain and self.buffer_size > LIMIT: + self.buffer_size = 0 + await self.drain() + + async def write_headers( + self, status_line: str, headers: "CIMultiDict[str]" + ) -> None: + """Write headers to the stream.""" + if self._on_headers_sent is not None: + await self._on_headers_sent(headers) + # status + headers + buf = _serialize_headers(status_line, headers) + self._headers_written = False + self._headers_buf = buf + + def send_headers(self) -> None: + """Force sending buffered headers if not already sent.""" + if not self._headers_buf or self._headers_written: + return + + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because we only enter this block when self._headers_buf is truthy + assert headers_buf is not None + + self._write(headers_buf) + + def set_eof(self) -> None: + """Indicate that the message is complete.""" + if self._eof: + return + + # If headers haven't been sent yet, send them now + # This handles the case where there's no body at all + if self._headers_buf and not self._headers_written: + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because we only enter this block when self._headers_buf is truthy + assert headers_buf is not None + + # Combine headers and chunked EOF marker in a single write + if self.chunked: + self._writelines((headers_buf, b"0\r\n\r\n")) + else: + self._write(headers_buf) + elif self.chunked and self._headers_written: + # Headers already sent, just send the final chunk marker + self._write(b"0\r\n\r\n") + + self._eof = True + + async def write_eof(self, chunk: bytes = b"") -> None: + if self._eof: + return + + if chunk and self._on_chunk_sent is not None: + await self._on_chunk_sent(chunk) + + # Handle body/compression + if self._compress: + chunks: list[bytes] = [] + chunks_len = 0 + if chunk and (compressed_chunk := await self._compress.compress(chunk)): + chunks_len = len(compressed_chunk) + chunks.append(compressed_chunk) + + flush_chunk = self._compress.flush() + chunks_len += len(flush_chunk) + chunks.append(flush_chunk) + assert chunks_len + + # Send buffered headers with compressed data if not yet sent + if self._headers_buf and not self._headers_written: + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if self.chunked: + # Coalesce headers with compressed chunked data + chunk_len_pre = f"{chunks_len:x}\r\n".encode("ascii") + self._writelines( + (headers_buf, chunk_len_pre, *chunks, b"\r\n0\r\n\r\n") + ) + else: + # Coalesce headers with compressed data + self._writelines((headers_buf, *chunks)) + await self.drain() + self._eof = True + return + + # Headers already sent, just write compressed data + if self.chunked: + chunk_len_pre = f"{chunks_len:x}\r\n".encode("ascii") + self._writelines((chunk_len_pre, *chunks, b"\r\n0\r\n\r\n")) + elif len(chunks) > 1: + self._writelines(chunks) + else: + self._write(chunks[0]) + await self.drain() + self._eof = True + return + + # No compression - send buffered headers if not yet sent + if self._headers_buf and not self._headers_written: + # Use helper to send headers with payload + self._send_headers_with_payload(chunk, True) + await self.drain() + self._eof = True + return + + # Handle remaining body + if self.chunked: + if chunk: + # Write final chunk with EOF marker + self._writelines( + (f"{len(chunk):x}\r\n".encode("ascii"), chunk, b"\r\n0\r\n\r\n") + ) + else: + self._write(b"0\r\n\r\n") + await self.drain() + self._eof = True + return + + if chunk: + self._write(chunk) + await self.drain() + + self._eof = True + + async def drain(self) -> None: + """Flush the write buffer. + + The intended use is to write + + await w.write(data) + await w.drain() + """ + protocol = self._protocol + if protocol.transport is not None and protocol._paused: + await protocol._drain_helper() + + +# https://www.rfc-editor.org/info/rfc9110/#section-5.5-5 +# https://www.rfc-editor.org/info/rfc9112/#section-4-3 +_FORBIDDEN_HEADER_CHARS_RE = re.compile(r"[\x00-\x08\x0a-\x1f\x7f]") + + +def _safe_header(string: str) -> str: + if _FORBIDDEN_HEADER_CHARS_RE.search(string) is not None: + raise ValueError( + "Forbidden control character detected in headers. " + "Potential header injection attack." + ) + return string + + +def _py_serialize_headers(status_line: str, headers: "CIMultiDict[str]") -> bytes: + _safe_header(status_line) + headers_gen = (_safe_header(k) + ": " + _safe_header(v) for k, v in headers.items()) + line = status_line + "\r\n" + "\r\n".join(headers_gen) + "\r\n\r\n" + return line.encode("utf-8") + + +_serialize_headers = _py_serialize_headers + +try: + import aiohttp._http_writer as _http_writer # type: ignore[import-not-found] + + _c_serialize_headers = _http_writer._serialize_headers + if not NO_EXTENSIONS: + _serialize_headers = _c_serialize_headers +except ImportError: + pass diff --git a/venv/Lib/site-packages/aiohttp/log.py b/venv/Lib/site-packages/aiohttp/log.py new file mode 100644 index 0000000000000000000000000000000000000000..d1314145b262658992db308631a13bac6948f7f8 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/log.py @@ -0,0 +1,8 @@ +import logging + +access_logger = logging.getLogger("aiohttp.access") +client_logger = logging.getLogger("aiohttp.client") +internal_logger = logging.getLogger("aiohttp.internal") +server_logger = logging.getLogger("aiohttp.server") +web_logger = logging.getLogger("aiohttp.web") +ws_logger = logging.getLogger("aiohttp.websocket") diff --git a/venv/Lib/site-packages/aiohttp/multipart.py b/venv/Lib/site-packages/aiohttp/multipart.py new file mode 100644 index 0000000000000000000000000000000000000000..a970f2c810773f8a7ecda91608ed40efd29daa9f --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/multipart.py @@ -0,0 +1,1228 @@ +import base64 +import binascii +import json +import re +import sys +import uuid +import warnings +from collections import deque +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from types import TracebackType +from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from urllib.parse import parse_qsl, unquote, urlencode + +from multidict import CIMultiDict, CIMultiDictProxy + +from .abc import AbstractStreamWriter +from .compression_utils import ZLibCompressor, ZLibDecompressor +from .hdrs import ( + CONTENT_DISPOSITION, + CONTENT_ENCODING, + CONTENT_LENGTH, + CONTENT_TRANSFER_ENCODING, + CONTENT_TYPE, +) +from .helpers import CHAR, DEFAULT_CHUNK_SIZE, TOKEN, parse_mimetype, reify +from .http import HeadersParser +from .http_exceptions import BadHttpMessage +from .log import internal_logger +from .payload import ( + JsonPayload, + LookupError, + Order, + Payload, + StringPayload, + get_payload, + payload_type, +) +from .streams import StreamReader + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = TypeVar("Self", bound="BodyPartReader") + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +_Buffer = TypeVar("_Buffer", bound=Buffer) + +__all__ = ( + "MultipartReader", + "MultipartWriter", + "BodyPartReader", + "BadContentDispositionHeader", + "BadContentDispositionParam", + "parse_content_disposition", + "content_disposition_filename", +) + + +if TYPE_CHECKING: + from .client_reqrep import ClientResponse + + +class BadContentDispositionHeader(RuntimeWarning): + pass + + +class BadContentDispositionParam(RuntimeWarning): + pass + + +def parse_content_disposition( + header: str | None, +) -> tuple[str | None, dict[str, str]]: + def is_token(string: str) -> bool: + return bool(string) and TOKEN >= set(string) + + def is_quoted(string: str) -> bool: + return string[0] == string[-1] == '"' + + def is_rfc5987(string: str) -> bool: + return is_token(string) and string.count("'") == 2 + + def is_extended_param(string: str) -> bool: + return string.endswith("*") + + def is_continuous_param(string: str) -> bool: + pos = string.find("*") + 1 + if not pos: + return False + substring = string[pos:-1] if string.endswith("*") else string[pos:] + return substring.isdigit() + + def unescape(text: str, *, chars: str = "".join(map(re.escape, CHAR))) -> str: + return re.sub(f"\\\\([{chars}])", "\\1", text) + + if not header: + return None, {} + + disptype, *parts = header.split(";") + if not is_token(disptype): + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + params: dict[str, str] = {} + while parts: + item = parts.pop(0) + + if not item: # To handle trailing semicolons + warnings.warn(BadContentDispositionHeader(header)) + continue + + if "=" not in item: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + key, value = item.split("=", 1) + key = key.lower().strip() + value = value.lstrip() + + if key in params: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + if not is_token(key): + warnings.warn(BadContentDispositionParam(item)) + continue + + elif is_continuous_param(key): + if is_quoted(value): + value = unescape(value[1:-1]) + elif not is_token(value): + warnings.warn(BadContentDispositionParam(item)) + continue + + elif is_extended_param(key): + if is_rfc5987(value): + encoding, _, value = value.split("'", 2) + encoding = encoding or "utf-8" + else: + warnings.warn(BadContentDispositionParam(item)) + continue + + try: + value = unquote(value, encoding, "strict") + except UnicodeDecodeError: # pragma: nocover + warnings.warn(BadContentDispositionParam(item)) + continue + + else: + failed = True + if is_quoted(value): + failed = False + value = unescape(value[1:-1].lstrip("\\/")) + elif is_token(value): + failed = False + elif parts: + # maybe just ; in filename, in any case this is just + # one case fix, for proper fix we need to redesign parser + _value = f"{value};{parts[0]}" + if is_quoted(_value): + parts.pop(0) + value = unescape(_value[1:-1].lstrip("\\/")) + failed = False + + if failed: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + params[key] = value + + return disptype.lower(), params + + +def content_disposition_filename( + params: Mapping[str, str], name: str = "filename" +) -> str | None: + name_suf = "%s*" % name + if not params: + return None + elif name_suf in params: + return params[name_suf] + elif name in params: + return params[name] + else: + parts = [] + fnparams = sorted( + (key, value) for key, value in params.items() if key.startswith(name_suf) + ) + for num, (key, value) in enumerate(fnparams): + _, tail = key.split("*", 1) + if tail.endswith("*"): + tail = tail[:-1] + if tail == str(num): + parts.append(value) + else: + break + if not parts: + return None + value = "".join(parts) + if "'" in value: + encoding, _, value = value.split("'", 2) + encoding = encoding or "utf-8" + return unquote(value, encoding, "strict") + return value + + +class MultipartResponseWrapper: + """Wrapper around the MultipartReader. + + It takes care about + underlying connection and close it when it needs in. + """ + + def __init__( + self, + resp: "ClientResponse", + stream: "MultipartReader", + ) -> None: + self.resp = resp + self.stream = stream + + def __aiter__(self) -> "MultipartResponseWrapper": + return self + + async def __anext__( + self, + ) -> Union["MultipartReader", "BodyPartReader"]: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + def at_eof(self) -> bool: + """Returns True when all response data had been read.""" + return self.resp.content.at_eof() + + async def next( + self, + ) -> Union["MultipartReader", "BodyPartReader"] | None: + """Emits next multipart reader object.""" + item = await self.stream.next() + if self.stream.at_eof(): + await self.release() + return item + + async def release(self) -> None: + """Release the connection gracefully. + + All remaining content is read to the void. + """ + await self.resp.release() + + +class BodyPartReader: + """Multipart reader for single body part.""" + + chunk_size = 8192 + + def __init__( + self, + boundary: bytes, + headers: "CIMultiDictProxy[str]", + content: StreamReader, + *, + subtype: str = "mixed", + default_charset: str | None = None, + max_decompress_size: int = DEFAULT_CHUNK_SIZE, + client_max_size: int = sys.maxsize, + max_size_error_cls: type[Exception] = ValueError, + ) -> None: + self.headers = headers + self._boundary = boundary + self._boundary_len = len(boundary) + 2 # Boundary + \r\n + self._content = content + self._default_charset = default_charset + self._at_eof = False + self._is_form_data = subtype == "form-data" + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + length = None if self._is_form_data else self.headers.get(CONTENT_LENGTH, None) + self._length = int(length) if length is not None else None + self._read_bytes = 0 + self._unread: deque[bytes] = deque() + self._prev_chunk: bytes | None = None + self._content_eof = 0 + self._cache: dict[str, Any] = {} + self._max_decompress_size = max_decompress_size + self._client_max_size = client_max_size + self._max_size_error_cls = max_size_error_cls + + def __aiter__(self: Self) -> Self: + return self + + async def __anext__(self) -> bytes: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + async def next(self) -> bytes | None: + item = await self.read() + if not item: + return None + return item + + async def read(self, *, decode: bool = False) -> bytes: + """Reads body part data. + + decode: Decodes data following by encoding + method from Content-Encoding header. If it missed + data remains untouched + """ + if self._at_eof: + return b"" + data = bytearray() + while not self._at_eof: + data.extend(await self.read_chunk(self.chunk_size)) + if len(data) > self._client_max_size: + raise self._max_size_error_cls(self._client_max_size) + if decode: + decoded_data = bytearray() + async for d in self.decode_iter(data): + decoded_data.extend(d) + if len(decoded_data) > self._client_max_size: + raise self._max_size_error_cls(self._client_max_size) + return decoded_data + return data + + async def read_chunk(self, size: int = chunk_size) -> bytes: + """Reads body part content chunk of the specified size. + + size: chunk size + """ + if self._at_eof: + return b"" + if self._length: + chunk = await self._read_chunk_from_length(size) + else: + chunk = await self._read_chunk_from_stream(size) + + # For the case of base64 data, we must read a fragment of size with a + # remainder of 0 by dividing by 4 for string without symbols \n or \r + encoding = self.headers.get(CONTENT_TRANSFER_ENCODING) + if encoding and encoding.lower() == "base64": + stripped_chunk = b"".join(chunk.split()) + remainder = len(stripped_chunk) % 4 + + while remainder != 0 and not self.at_eof(): + over_chunk_size = 4 - remainder + over_chunk = b"" + + if self._prev_chunk: + over_chunk = self._prev_chunk[:over_chunk_size] + self._prev_chunk = self._prev_chunk[len(over_chunk) :] + + if len(over_chunk) != over_chunk_size: + over_chunk += await self._content.read(4 - len(over_chunk)) + + if not over_chunk: + self._at_eof = True + + stripped_chunk += b"".join(over_chunk.split()) + chunk += over_chunk + remainder = len(stripped_chunk) % 4 + + self._read_bytes += len(chunk) + if self._read_bytes == self._length: + self._at_eof = True + if self._at_eof and await self._content.readline() != b"\r\n": + raise ValueError("Reader did not read all the data or it is malformed") + return chunk + + async def _read_chunk_from_length(self, size: int) -> bytes: + # Reads body part content chunk of the specified size. + # The body part must has Content-Length header with proper value. + assert self._length is not None, "Content-Length required for chunked read" + chunk_size = min(size, self._length - self._read_bytes) + chunk = await self._content.read(chunk_size) + if self._content.at_eof(): + self._at_eof = True + return chunk + + async def _read_chunk_from_stream(self, size: int) -> bytes: + # Reads content chunk of body part with unknown length. + # The Content-Length header for body part is not necessary. + assert ( + size >= self._boundary_len + ), "Chunk size must be greater or equal than boundary length + 2" + first_chunk = self._prev_chunk is None + if first_chunk: + # We need to re-add the CRLF that got removed from headers parsing. + self._prev_chunk = b"\r\n" + await self._content.read(size) + + chunk = b"" + # content.read() may return less than size, so we need to loop to ensure + # we have enough data to detect the boundary. + while len(chunk) < self._boundary_len: + chunk += await self._content.read(size) + self._content_eof += int(self._content.at_eof()) + if self._content_eof > 2: + raise ValueError("Reading after EOF") + if self._content_eof: + break + if len(chunk) > size: + self._content.unread_data(chunk[size:]) + chunk = chunk[:size] + + assert self._prev_chunk is not None + window = self._prev_chunk + chunk + sub = b"\r\n" + self._boundary + if first_chunk: + idx = window.find(sub) + else: + idx = window.find(sub, max(0, len(self._prev_chunk) - len(sub))) + if idx >= 0: + # pushing boundary back to content + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + self._content.unread_data(window[idx:]) + self._prev_chunk = self._prev_chunk[:idx] + chunk = window[len(self._prev_chunk) : idx] + if not chunk: + self._at_eof = True + result = self._prev_chunk[2 if first_chunk else 0 :] # Strip initial CRLF + self._prev_chunk = chunk + return result + + async def readline(self) -> bytes: + """Reads body part by line by line.""" + if self._at_eof: + return b"" + + if self._unread: + line = self._unread.popleft() + else: + line = await self._content.readline() + + if line.startswith(self._boundary): + # the very last boundary may not come with \r\n, + # so set single rules for everyone + sline = line.rstrip(b"\r\n") + boundary = self._boundary + last_boundary = self._boundary + b"--" + # ensure that we read exactly the boundary, not something alike + if sline == boundary or sline == last_boundary: + self._at_eof = True + self._unread.append(line) + return b"" + else: + next_line = await self._content.readline() + if next_line.startswith(self._boundary): + line = line[:-2] # strip CRLF but only once + self._unread.append(next_line) + + return line + + async def release(self) -> None: + """Like read(), but reads all the data to the void.""" + if self._at_eof: + return + while not self._at_eof: + await self.read_chunk(self.chunk_size) + + async def text(self, *, encoding: str | None = None) -> str: + """Like read(), but assumes that body part contains text data.""" + data = await self.read(decode=True) + # see https://www.w3.org/TR/html5/forms.html#multipart/form-data-encoding-algorithm + # and https://dvcs.w3.org/hg/xhr/raw-file/tip/Overview.html#dom-xmlhttprequest-send + encoding = encoding or self.get_charset(default="utf-8") + return data.decode(encoding) + + async def json(self, *, encoding: str | None = None) -> dict[str, Any] | None: + """Like read(), but assumes that body parts contains JSON data.""" + data = await self.read(decode=True) + if not data: + return None + encoding = encoding or self.get_charset(default="utf-8") + return cast(dict[str, Any], json.loads(data.decode(encoding))) + + async def form(self, *, encoding: str | None = None) -> list[tuple[str, str]]: + """Like read(), but assumes that body parts contain form urlencoded data.""" + data = await self.read(decode=True) + if not data: + return [] + if encoding is not None: + real_encoding = encoding + else: + real_encoding = self.get_charset(default="utf-8") + try: + decoded_data = data.rstrip().decode(real_encoding) + except UnicodeDecodeError: + raise ValueError("data cannot be decoded with %s encoding" % real_encoding) + + return parse_qsl( + decoded_data, + keep_blank_values=True, + encoding=real_encoding, + ) + + def at_eof(self) -> bool: + """Returns True if the boundary was reached or False otherwise.""" + return self._at_eof + + def _apply_content_transfer_decoding(self, data: _Buffer) -> _Buffer | bytes: + """Apply Content-Transfer-Encoding decoding if header is present.""" + if CONTENT_TRANSFER_ENCODING in self.headers: + return self._decode_content_transfer(data) + return data + + def _needs_content_decoding(self) -> bool: + """Check if Content-Encoding decoding should be applied.""" + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + return not self._is_form_data and CONTENT_ENCODING in self.headers + + def decode(self, data: _Buffer) -> _Buffer | bytes: + """Decodes data synchronously. + + Decodes data according the specified Content-Encoding + or Content-Transfer-Encoding headers value. + + Note: For large payloads, consider using decode_iter() instead + to avoid blocking the event loop during decompression. + """ + decoded = self._apply_content_transfer_decoding(data) + if self._needs_content_decoding(): + return self._decode_content(decoded) + return decoded + + async def decode_iter(self, data: _Buffer) -> AsyncIterator[_Buffer | bytes]: + """Async generator that yields decoded data chunks. + + Decodes data according the specified Content-Encoding + or Content-Transfer-Encoding headers value. + + This method offloads decompression to an executor for large payloads + to avoid blocking the event loop. + """ + decoded = self._apply_content_transfer_decoding(data) + if self._needs_content_decoding(): + async for d in self._decode_content_async(decoded): + yield d + else: + yield decoded + + def _decode_content(self, data: _Buffer) -> _Buffer | bytes: + encoding = self.headers.get(CONTENT_ENCODING, "").lower() + if encoding == "identity": + return data + if encoding in {"deflate", "gzip"}: + return ZLibDecompressor( + encoding=encoding, + suppress_deflate_header=True, + ).decompress_sync(data, max_length=self._max_decompress_size) + + raise RuntimeError(f"unknown content encoding: {encoding}") + + async def _decode_content_async( + self, data: _Buffer + ) -> AsyncIterator[_Buffer | bytes]: + encoding = self.headers.get(CONTENT_ENCODING, "").lower() + if encoding == "identity": + yield data + elif encoding in {"deflate", "gzip"}: + d = ZLibDecompressor( + encoding=encoding, + suppress_deflate_header=True, + ) + yield await d.decompress(data, max_length=self._max_decompress_size) + while d.data_available: + yield await d.decompress(b"", max_length=self._max_decompress_size) + else: + raise RuntimeError(f"unknown content encoding: {encoding}") + + def _decode_content_transfer(self, data: _Buffer) -> _Buffer | bytes: + encoding = self.headers.get(CONTENT_TRANSFER_ENCODING, "").lower() + + if encoding == "base64": + return base64.b64decode(data) + elif encoding == "quoted-printable": + return binascii.a2b_qp(data) + elif encoding in ("binary", "8bit", "7bit"): + return data + else: + raise RuntimeError(f"unknown content transfer encoding: {encoding}") + + def get_charset(self, default: str) -> str: + """Returns charset parameter from Content-Type header or default.""" + ctype = self.headers.get(CONTENT_TYPE, "") + mimetype = parse_mimetype(ctype) + return mimetype.parameters.get("charset", self._default_charset or default) + + @reify + def name(self) -> str | None: + """Returns name specified in Content-Disposition header. + + If the header is missing or malformed, returns None. + """ + _, params = parse_content_disposition(self.headers.get(CONTENT_DISPOSITION)) + return content_disposition_filename(params, "name") + + @reify + def filename(self) -> str | None: + """Returns filename specified in Content-Disposition header. + + Returns None if the header is missing or malformed. + """ + _, params = parse_content_disposition(self.headers.get(CONTENT_DISPOSITION)) + return content_disposition_filename(params, "filename") + + +@payload_type(BodyPartReader, order=Order.try_first) +class BodyPartReaderPayload(Payload): + _value: BodyPartReader + # _autoclose = False (inherited) - Streaming reader that may have resources + + def __init__(self, value: BodyPartReader, *args: Any, **kwargs: Any) -> None: + super().__init__(value, *args, **kwargs) + + params: dict[str, str] = {} + if value.name is not None: + params["name"] = value.name + if value.filename is not None: + params["filename"] = value.filename + + if params: + self.set_content_disposition("attachment", True, **params) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + raise TypeError("Unable to decode.") + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """Raises TypeError as body parts should be consumed via write(). + + This is intentional: BodyPartReader payloads are designed for streaming + large data (potentially gigabytes) and must be consumed only once via + the write() method to avoid memory exhaustion. They cannot be buffered + in memory for reuse. + """ + raise TypeError("Unable to read body part as bytes. Use write() to consume.") + + async def write(self, writer: AbstractStreamWriter) -> None: + field = self._value + while chunk := await field.read_chunk(size=DEFAULT_CHUNK_SIZE): + async for d in field.decode_iter(chunk): + await writer.write(d) + + +class MultipartReader: + """Multipart body reader.""" + + #: Response wrapper, used when multipart readers constructs from response. + response_wrapper_cls = MultipartResponseWrapper + #: Multipart reader class, used to handle multipart/* body parts. + #: None points to type(self) + multipart_reader_cls: type["MultipartReader"] | None = None + #: Body part reader class for non multipart/* content types. + part_reader_cls = BodyPartReader + + def __init__( + self, + headers: Mapping[str, str], + content: StreamReader, + *, + client_max_size: int = sys.maxsize, + max_field_size: int = 8190, + max_headers: int = 128, + max_size_error_cls: type[Exception] = ValueError, + ) -> None: + self._mimetype = parse_mimetype(headers[CONTENT_TYPE]) + assert self._mimetype.type == "multipart", "multipart/* content type expected" + if "boundary" not in self._mimetype.parameters: + raise ValueError( + "boundary missed for Content-Type: %s" % headers[CONTENT_TYPE] + ) + + self.headers = headers + self._boundary = ("--" + self._get_boundary()).encode() + self._client_max_size = client_max_size + self._content = content + self._default_charset: str | None = None + self._last_part: MultipartReader | BodyPartReader | None = None + self._max_field_size = max_field_size + self._max_headers = max_headers + self._max_size_error_cls = max_size_error_cls + self._at_eof = False + self._at_bof = True + self._unread: list[bytes] = [] + + def __aiter__(self: Self) -> Self: + return self + + async def __anext__( + self, + ) -> Union["MultipartReader", BodyPartReader] | None: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + @classmethod + def from_response( + cls, + response: "ClientResponse", + ) -> MultipartResponseWrapper: + """Constructs reader instance from HTTP response. + + :param response: :class:`~aiohttp.client.ClientResponse` instance + """ + obj = cls.response_wrapper_cls( + response, cls(response.headers, response.content) + ) + return obj + + def at_eof(self) -> bool: + """Returns True if the final boundary was reached, false otherwise.""" + return self._at_eof + + async def next( + self, + ) -> Union["MultipartReader", BodyPartReader] | None: + """Emits the next multipart body part.""" + # So, if we're at BOF, we need to skip till the boundary. + if self._at_eof: + return None + await self._maybe_release_last_part() + if self._at_bof: + await self._read_until_first_boundary() + self._at_bof = False + else: + await self._read_boundary() + if self._at_eof: # we just read the last boundary, nothing to do there + return None + + part = await self.fetch_next_part() + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.6 + if ( + self._last_part is None + and self._mimetype.subtype == "form-data" + and isinstance(part, BodyPartReader) + ): + _, params = parse_content_disposition(part.headers.get(CONTENT_DISPOSITION)) + if params.get("name") == "_charset_": + # Longest encoding in https://encoding.spec.whatwg.org/encodings.json + # is 19 characters, so 32 should be more than enough for any valid encoding. + charset = await part.read_chunk(32) + if len(charset) > 31: + raise RuntimeError("Invalid default charset") + self._default_charset = charset.strip().decode() + part = await self.fetch_next_part() + self._last_part = part + return self._last_part + + async def release(self) -> None: + """Reads all the body parts to the void till the final boundary.""" + while not self._at_eof: + item = await self.next() + if item is None: + break + await item.release() + + async def fetch_next_part( + self, + ) -> Union["MultipartReader", BodyPartReader]: + """Returns the next body part reader.""" + headers = await self._read_headers() + return self._get_part_reader(headers) + + def _get_part_reader( + self, + headers: "CIMultiDictProxy[str]", + ) -> Union["MultipartReader", BodyPartReader]: + """Dispatches the response by the `Content-Type` header. + + Returns a suitable reader instance. + + :param dict headers: Response headers + """ + ctype = headers.get(CONTENT_TYPE, "") + mimetype = parse_mimetype(ctype) + + if mimetype.type == "multipart": + if self.multipart_reader_cls is None: + return type(self)( + headers, + self._content, + client_max_size=self._client_max_size, + max_field_size=self._max_field_size, + max_headers=self._max_headers, + max_size_error_cls=self._max_size_error_cls, + ) + return self.multipart_reader_cls( + headers, + self._content, + client_max_size=self._client_max_size, + max_field_size=self._max_field_size, + max_headers=self._max_headers, + max_size_error_cls=self._max_size_error_cls, + ) + else: + return self.part_reader_cls( + self._boundary, + headers, + self._content, + subtype=self._mimetype.subtype, + default_charset=self._default_charset, + client_max_size=self._client_max_size, + max_size_error_cls=self._max_size_error_cls, + ) + + def _get_boundary(self) -> str: + boundary = self._mimetype.parameters["boundary"] + if len(boundary) > 70: + raise ValueError("boundary %r is too long (70 chars max)" % boundary) + + return boundary + + async def _readline(self) -> bytes: + if self._unread: + return self._unread.pop() + return await self._content.readline() + + async def _read_until_first_boundary(self) -> None: + while True: + chunk = await self._readline() + if chunk == b"": + raise ValueError( + "Could not find starting boundary %r" % (self._boundary) + ) + chunk = chunk.rstrip() + if chunk == self._boundary: + return + elif chunk == self._boundary + b"--": + self._at_eof = True + return + + async def _read_boundary(self) -> None: + chunk = (await self._readline()).rstrip() + if chunk == self._boundary: + pass + elif chunk == self._boundary + b"--": + self._at_eof = True + epilogue = await self._readline() + next_line = await self._readline() + + # the epilogue is expected and then either the end of input or the + # parent multipart boundary, if the parent boundary is found then + # it should be marked as unread and handed to the parent for + # processing + if next_line[:2] == b"--": + self._unread.append(next_line) + # otherwise the request is likely missing an epilogue and both + # lines should be passed to the parent for processing + # (this handles the old behavior gracefully) + else: + self._unread.extend([next_line, epilogue]) + else: + raise ValueError(f"Invalid boundary {chunk!r}, expected {self._boundary!r}") + + async def _read_headers(self) -> "CIMultiDictProxy[str]": + lines = [] + while True: + chunk = await self._content.readline(max_line_length=self._max_field_size) + chunk = chunk.rstrip(b"\r\n") + lines.append(chunk) + if not chunk: + break + if len(lines) > self._max_headers: + raise BadHttpMessage("Too many headers received") + parser = HeadersParser(max_field_size=self._max_field_size) + headers, raw_headers = parser.parse_headers(lines) + return headers + + async def _maybe_release_last_part(self) -> None: + """Ensures that the last read body part is read completely.""" + if self._last_part is not None: + if not self._last_part.at_eof(): + await self._last_part.release() + self._unread.extend(self._last_part._unread) + self._last_part = None + + +_Part = tuple[Payload, str, str] + + +class MultipartWriter(Payload): + """Multipart body writer.""" + + _value: None + # _consumed = False (inherited) - Can be encoded multiple times + _autoclose = True # No file handles, just collects parts in memory + + def __init__(self, subtype: str = "mixed", boundary: str | None = None) -> None: + boundary = boundary if boundary is not None else uuid.uuid4().hex + # The underlying Payload API demands a str (utf-8), not bytes, + # so we need to ensure we don't lose anything during conversion. + # As a result, require the boundary to be ASCII only. + # In both situations. + + try: + self._boundary = boundary.encode("ascii") + except UnicodeEncodeError: + raise ValueError("boundary should contain ASCII only chars") from None + ctype = f"multipart/{subtype}; boundary={self._boundary_value}" + + super().__init__(None, content_type=ctype) + + self._parts: list[_Part] = [] + self._is_form_data = subtype == "form-data" + + def __enter__(self) -> "MultipartWriter": + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + pass + + def __iter__(self) -> Iterator[_Part]: + return iter(self._parts) + + def __len__(self) -> int: + return len(self._parts) + + def __bool__(self) -> bool: + return True + + _valid_tchar_regex = re.compile(rb"\A[!#$%&'*+\-.^_`|~\w]+\Z") + _invalid_qdtext_char_regex = re.compile(rb"[\x00-\x08\x0A-\x1F\x7F]") + + @property + def _boundary_value(self) -> str: + """Wrap boundary parameter value in quotes, if necessary. + + Reads self.boundary and returns a unicode string. + """ + # Refer to RFCs 7231, 7230, 5234. + # + # parameter = token "=" ( token / quoted-string ) + # token = 1*tchar + # quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE + # qdtext = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text + # obs-text = %x80-FF + # quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + # tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + # / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + # / DIGIT / ALPHA + # ; any VCHAR, except delimiters + # VCHAR = %x21-7E + value = self._boundary + if re.match(self._valid_tchar_regex, value): + return value.decode("ascii") # cannot fail + + if re.search(self._invalid_qdtext_char_regex, value): + raise ValueError("boundary value contains invalid characters") + + # escape %x5C and %x22 + quoted_value_content = value.replace(b"\\", b"\\\\") + quoted_value_content = quoted_value_content.replace(b'"', b'\\"') + + return '"' + quoted_value_content.decode("ascii") + '"' + + @property + def boundary(self) -> str: + return self._boundary.decode("ascii") + + def append(self, obj: Any, headers: Mapping[str, str] | None = None) -> Payload: + if headers is None: + headers = CIMultiDict() + + if isinstance(obj, Payload): + obj.headers.update(headers) + return self.append_payload(obj) + else: + try: + payload = get_payload(obj, headers=headers) + except LookupError: + raise TypeError("Cannot create payload from %r" % obj) + else: + return self.append_payload(payload) + + def append_payload(self, payload: Payload) -> Payload: + """Adds a new body part to multipart writer.""" + encoding: str | None = None + te_encoding: str | None = None + if self._is_form_data: + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.7 + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + assert ( + not {CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TRANSFER_ENCODING} + & payload.headers.keys() + ) + # Set default Content-Disposition in case user doesn't create one + if CONTENT_DISPOSITION not in payload.headers: + name = f"section-{len(self._parts)}" + payload.set_content_disposition("form-data", name=name) + else: + # compression + encoding = payload.headers.get(CONTENT_ENCODING, "").lower() + if encoding and encoding not in ("deflate", "gzip", "identity"): + raise RuntimeError(f"unknown content encoding: {encoding}") + if encoding == "identity": + encoding = None + + # te encoding + te_encoding = payload.headers.get(CONTENT_TRANSFER_ENCODING, "").lower() + if te_encoding not in ("", "base64", "quoted-printable", "binary"): + raise RuntimeError(f"unknown content transfer encoding: {te_encoding}") + if te_encoding == "binary": + te_encoding = None + + # size + size = payload.size + if size is not None and not (encoding or te_encoding): + payload.headers[CONTENT_LENGTH] = str(size) + + self._parts.append((payload, encoding, te_encoding)) # type: ignore[arg-type] + return payload + + def append_json( + self, obj: Any, headers: Mapping[str, str] | None = None + ) -> Payload: + """Helper to append JSON part.""" + if headers is None: + headers = CIMultiDict() + + return self.append_payload(JsonPayload(obj, headers=headers)) + + def append_form( + self, + obj: Sequence[tuple[str, str]] | Mapping[str, str], + headers: Mapping[str, str] | None = None, + ) -> Payload: + """Helper to append form urlencoded part.""" + assert isinstance(obj, (Sequence, Mapping)) + + if headers is None: + headers = CIMultiDict() + + if isinstance(obj, Mapping): + obj = list(obj.items()) + data = urlencode(obj, doseq=True) + + return self.append_payload( + StringPayload( + data, headers=headers, content_type="application/x-www-form-urlencoded" + ) + ) + + @property + def size(self) -> int | None: + """Size of the payload.""" + total = 0 + for part, encoding, te_encoding in self._parts: + part_size = part.size + if encoding or te_encoding or part_size is None: + return None + + total += int( + 2 + + len(self._boundary) + + 2 + + part_size # b'--'+self._boundary+b'\r\n' + + len(part._binary_headers) + + 2 # b'\r\n' + ) + + total += 2 + len(self._boundary) + 4 # b'--'+self._boundary+b'--\r\n' + return total + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """Return string representation of the multipart data. + + WARNING: This method may do blocking I/O if parts contain file payloads. + It should not be called in the event loop. Use as_bytes().decode() instead. + """ + return "".join( + "--" + + self.boundary + + "\r\n" + + part._binary_headers.decode(encoding, errors) + + part.decode() + for part, _e, _te in self._parts + ) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """Return bytes representation of the multipart data. + + This method is async-safe and calls as_bytes on underlying payloads. + """ + parts: list[bytes] = [] + + # Process each part + for part, _e, _te in self._parts: + # Add boundary + parts.append(b"--" + self._boundary + b"\r\n") + + # Add headers + parts.append(part._binary_headers) + + # Add payload content using as_bytes for async safety + part_bytes = await part.as_bytes(encoding, errors) + parts.append(part_bytes) + + # Add trailing CRLF + parts.append(b"\r\n") + + # Add closing boundary + parts.append(b"--" + self._boundary + b"--\r\n") + + return b"".join(parts) + + async def write( + self, writer: AbstractStreamWriter, close_boundary: bool = True + ) -> None: + """Write body.""" + for part, encoding, te_encoding in self._parts: + if self._is_form_data: + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.2 + assert CONTENT_DISPOSITION in part.headers + assert "name=" in part.headers[CONTENT_DISPOSITION] + + await writer.write(b"--" + self._boundary + b"\r\n") + await writer.write(part._binary_headers) + + if encoding or te_encoding: + w = MultipartPayloadWriter(writer) + if encoding: + w.enable_compression(encoding) + if te_encoding: + w.enable_encoding(te_encoding) + await part.write(w) # type: ignore[arg-type] + await w.write_eof() + else: + await part.write(writer) + + await writer.write(b"\r\n") + + if close_boundary: + await writer.write(b"--" + self._boundary + b"--\r\n") + + async def close(self) -> None: + """ + Close all part payloads that need explicit closing. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + """ + if self._consumed: + return + self._consumed = True + + # Close all parts that need explicit closing + # We catch and log exceptions to ensure all parts get a chance to close + # we do not use asyncio.gather() here because we are not allowed + # to suspend given we may be called during cleanup + for idx, (part, _, _) in enumerate(self._parts): + if not part.autoclose and not part.consumed: + try: + await part.close() + except Exception as exc: + internal_logger.error( + "Failed to close multipart part %d: %s", idx, exc, exc_info=True + ) + + +class MultipartPayloadWriter: + def __init__(self, writer: AbstractStreamWriter) -> None: + self._writer = writer + self._encoding: str | None = None + self._compress: ZLibCompressor | None = None + self._encoding_buffer: bytearray | None = None + + def enable_encoding(self, encoding: str) -> None: + if encoding == "base64": + self._encoding = encoding + self._encoding_buffer = bytearray() + elif encoding == "quoted-printable": + self._encoding = "quoted-printable" + + def enable_compression( + self, encoding: str = "deflate", strategy: int | None = None + ) -> None: + self._compress = ZLibCompressor( + encoding=encoding, + suppress_deflate_header=True, + strategy=strategy, + ) + + async def write_eof(self) -> None: + if self._compress is not None: + chunk = self._compress.flush() + if chunk: + self._compress = None + await self.write(chunk) + + if self._encoding == "base64": + if self._encoding_buffer: + await self._writer.write(base64.b64encode(self._encoding_buffer)) + + async def write(self, chunk: bytes) -> None: + if self._compress is not None: + if chunk: + chunk = await self._compress.compress(chunk) + if not chunk: + return + + if self._encoding == "base64": + buf = self._encoding_buffer + assert buf is not None + buf.extend(chunk) + + if buf: + div, mod = divmod(len(buf), 3) + enc_chunk, self._encoding_buffer = (buf[: div * 3], buf[div * 3 :]) + if enc_chunk: + b64chunk = base64.b64encode(enc_chunk) + await self._writer.write(b64chunk) + elif self._encoding == "quoted-printable": + await self._writer.write(binascii.b2a_qp(chunk)) + else: + await self._writer.write(chunk) diff --git a/venv/Lib/site-packages/aiohttp/payload.py b/venv/Lib/site-packages/aiohttp/payload.py new file mode 100644 index 0000000000000000000000000000000000000000..5f2177ace32732a5a06793f45c535933e922b6ee --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/payload.py @@ -0,0 +1,1138 @@ +import asyncio +import enum +import io +import json +import mimetypes +import os +import sys +import warnings +from abc import ABC, abstractmethod +from collections.abc import Iterable +from itertools import chain +from typing import IO, TYPE_CHECKING, Any, Final, TextIO + +from multidict import CIMultiDict + +from . import hdrs +from .abc import AbstractStreamWriter +from .helpers import ( + _SENTINEL, + DEFAULT_CHUNK_SIZE, + content_disposition_header, + guess_filename, + parse_mimetype, + sentinel, +) +from .http_writer import _safe_header +from .streams import StreamReader +from .typedefs import JSONBytesEncoder, JSONEncoder, _CIMultiDict + +__all__ = ( + "PAYLOAD_REGISTRY", + "get_payload", + "payload_type", + "Payload", + "BytesPayload", + "StringPayload", + "IOBasePayload", + "BytesIOPayload", + "BufferedReaderPayload", + "TextIOPayload", + "StringIOPayload", + "JsonPayload", + "JsonBytesPayload", + "AsyncIterablePayload", +) + +TOO_LARGE_BYTES_BODY: Final[int] = 2**20 # 1 MB +_CLOSE_FUTURES: set[asyncio.Future[None]] = set() + + +class LookupError(Exception): + """Raised when no payload factory is found for the given data type.""" + + +class Order(str, enum.Enum): + normal = "normal" + try_first = "try_first" + try_last = "try_last" + + +def get_payload(data: Any, *args: Any, **kwargs: Any) -> "Payload": + return PAYLOAD_REGISTRY.get(data, *args, **kwargs) + + +def register_payload( + factory: type["Payload"], type: Any, *, order: Order = Order.normal +) -> None: + PAYLOAD_REGISTRY.register(factory, type, order=order) + + +class payload_type: + def __init__(self, type: Any, *, order: Order = Order.normal) -> None: + self.type = type + self.order = order + + def __call__(self, factory: type["Payload"]) -> type["Payload"]: + register_payload(factory, self.type, order=self.order) + return factory + + +PayloadType = type["Payload"] +_PayloadRegistryItem = tuple[PayloadType, Any] + + +class PayloadRegistry: + """Payload registry. + + note: we need zope.interface for more efficient adapter search + """ + + __slots__ = ("_first", "_normal", "_last", "_normal_lookup") + + def __init__(self) -> None: + self._first: list[_PayloadRegistryItem] = [] + self._normal: list[_PayloadRegistryItem] = [] + self._last: list[_PayloadRegistryItem] = [] + self._normal_lookup: dict[Any, PayloadType] = {} + + def get( + self, + data: Any, + *args: Any, + _CHAIN: "type[chain[_PayloadRegistryItem]]" = chain, + **kwargs: Any, + ) -> "Payload": + if self._first: + for factory, type_ in self._first: + if isinstance(data, type_): + return factory(data, *args, **kwargs) + # Try the fast lookup first + if lookup_factory := self._normal_lookup.get(type(data)): + return lookup_factory(data, *args, **kwargs) + # Bail early if its already a Payload + if isinstance(data, Payload): + return data + # Fallback to the slower linear search + for factory, type_ in _CHAIN(self._normal, self._last): + if isinstance(data, type_): + return factory(data, *args, **kwargs) + raise LookupError() + + def register( + self, factory: PayloadType, type: Any, *, order: Order = Order.normal + ) -> None: + if order is Order.try_first: + self._first.append((factory, type)) + elif order is Order.normal: + self._normal.append((factory, type)) + if isinstance(type, Iterable): + for t in type: + self._normal_lookup[t] = factory + else: + self._normal_lookup[type] = factory + elif order is Order.try_last: + self._last.append((factory, type)) + else: + raise ValueError(f"Unsupported order {order!r}") + + +class Payload(ABC): + + _default_content_type: str = "application/octet-stream" + _size: int | None = None + _consumed: bool = False # Default: payload has not been consumed yet + _autoclose: bool = False # Default: assume resource needs explicit closing + + def __init__( + self, + value: Any, + headers: ( + _CIMultiDict | dict[str, str] | Iterable[tuple[str, str]] | None + ) = None, + content_type: str | None | _SENTINEL = sentinel, + filename: str | None = None, + encoding: str | None = None, + **kwargs: Any, + ) -> None: + self._encoding = encoding + self._filename = filename + self._headers: _CIMultiDict = CIMultiDict() + self._value = value + if content_type is not sentinel and content_type is not None: + self._headers[hdrs.CONTENT_TYPE] = content_type + elif self._filename is not None: + if sys.version_info >= (3, 13): + guesser = mimetypes.guess_file_type + else: + guesser = mimetypes.guess_type + content_type = guesser(self._filename)[0] + if content_type is None: + content_type = self._default_content_type + self._headers[hdrs.CONTENT_TYPE] = content_type + else: + self._headers[hdrs.CONTENT_TYPE] = self._default_content_type + if headers: + self._headers.update(headers) + + @property + def size(self) -> int | None: + """Size of the payload in bytes. + + Returns the number of bytes that will be transmitted when the payload + is written. For string payloads, this is the size after encoding to bytes, + not the length of the string. + """ + return self._size + + @property + def filename(self) -> str | None: + """Filename of the payload.""" + return self._filename + + @property + def headers(self) -> _CIMultiDict: + """Custom item headers""" + return self._headers + + @property + def _binary_headers(self) -> bytes: + return ( + "".join( + _safe_header(k) + ": " + _safe_header(v) + "\r\n" + for k, v in self.headers.items() + ).encode("utf-8") + + b"\r\n" + ) + + @property + def encoding(self) -> str | None: + """Payload encoding""" + return self._encoding + + @property + def content_type(self) -> str: + """Content type""" + return self._headers[hdrs.CONTENT_TYPE] + + @property + def consumed(self) -> bool: + """Whether the payload has been consumed and cannot be reused.""" + return self._consumed + + @property + def autoclose(self) -> bool: + """ + Whether the payload can close itself automatically. + + Returns True if the payload has no file handles or resources that need + explicit closing. If False, callers must await close() to release resources. + """ + return self._autoclose + + def set_content_disposition( + self, + disptype: str, + quote_fields: bool = True, + _charset: str = "utf-8", + **params: Any, + ) -> None: + """Sets ``Content-Disposition`` header.""" + self._headers[hdrs.CONTENT_DISPOSITION] = content_disposition_header( + disptype, quote_fields=quote_fields, _charset=_charset, **params + ) + + @abstractmethod + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + This is named decode() to allow compatibility with bytes objects. + """ + + @abstractmethod + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This is a legacy method that writes the entire payload without length constraints. + + Important: + For new implementations, use write_with_length() instead of this method. + This method is maintained for backwards compatibility and will eventually + delegate to write_with_length(writer, None) in all implementations. + + All payload subclasses must override this method for backwards compatibility, + but new code should use write_with_length for more flexibility and control. + + """ + + # write_with_length is new in aiohttp 3.12 + # it should be overridden by subclasses + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method allows writing payload content with a specific length constraint, + which is particularly useful for HTTP responses with Content-Length header. + + Note: + This is the base implementation that provides backwards compatibility + for subclasses that don't override this method. Specific payload types + should override this method to implement proper length-constrained writing. + + """ + # Backwards compatibility for subclasses that don't override this method + # and for the default implementation + await self.write(writer) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This is a convenience method that calls decode() and encodes the result + to bytes using the specified encoding. + """ + # Use instance encoding if available, otherwise use parameter + actual_encoding = self._encoding or encoding + return self.decode(actual_encoding, errors).encode(actual_encoding) + + def _close(self) -> None: + """ + Async safe synchronous close operations for backwards compatibility. + + This method exists only for backwards compatibility with code that + needs to clean up payloads synchronously. In the future, we will + drop this method and only support the async close() method. + + WARNING: This method must be safe to call from within the event loop + without blocking. Subclasses should not perform any blocking I/O here. + + WARNING: This method must be called from within an event loop for + certain payload types (e.g., IOBasePayload). Calling it outside an + event loop may raise RuntimeError. + """ + # This is a no-op by default, but subclasses can override it + # for non-blocking cleanup operations. + + async def close(self) -> None: + """ + Close the payload if it holds any resources. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + + In the future, this will be the only close method supported. + """ + self._close() + + +class BytesPayload(Payload): + _value: bytes + # _consumed = False (inherited) - Bytes are immutable and can be reused + _autoclose = True # No file handle, just bytes in memory + + def __init__( + self, value: bytes | bytearray | memoryview, *args: Any, **kwargs: Any + ) -> None: + if "content_type" not in kwargs: + kwargs["content_type"] = "application/octet-stream" + + super().__init__(value, *args, **kwargs) + + if isinstance(value, memoryview): + self._size = value.nbytes + elif isinstance(value, (bytes, bytearray)): + self._size = len(value) + else: + raise TypeError(f"value argument must be byte-ish, not {type(value)!r}") + + if self._size > TOO_LARGE_BYTES_BODY: + kwargs = {"source": self} + warnings.warn( + "Sending a large body directly with raw bytes might" + " lock the event loop. You should probably pass an " + "io.BytesIO object instead", + ResourceWarning, + **kwargs, + ) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + return self._value.decode(encoding, errors) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method returns the raw bytes content of the payload. + It is equivalent to accessing the _value attribute directly. + """ + return self._value + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire bytes payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method writes the entire bytes content without any length constraint. + + Note: + For new implementations that need length control, use write_with_length(). + This method is maintained for backwards compatibility and is equivalent + to write_with_length(writer, None). + + """ + await writer.write(self._value) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write bytes payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method writes either the entire byte sequence or a slice of it + up to the specified content_length. For BytesPayload, this operation + is performed efficiently using array slicing. + + """ + if content_length is not None: + await writer.write(self._value[:content_length]) + else: + await writer.write(self._value) + + +class StringPayload(BytesPayload): + def __init__( + self, + value: str, + *args: Any, + encoding: str | None = None, + content_type: str | None = None, + **kwargs: Any, + ) -> None: + + if encoding is None: + if content_type is None: + real_encoding = "utf-8" + content_type = "text/plain; charset=utf-8" + else: + mimetype = parse_mimetype(content_type) + real_encoding = mimetype.parameters.get("charset", "utf-8") + else: + if content_type is None: + content_type = "text/plain; charset=%s" % encoding + real_encoding = encoding + + super().__init__( + value.encode(real_encoding), + encoding=real_encoding, + content_type=content_type, + *args, + **kwargs, + ) + + +class StringIOPayload(StringPayload): + def __init__(self, value: IO[str], *args: Any, **kwargs: Any) -> None: + super().__init__(value.read(), *args, **kwargs) + + +class IOBasePayload(Payload): + _value: io.IOBase + # _consumed = False (inherited) - File can be re-read from the same position + _start_position: int | None = None + # _autoclose = False (inherited) - Has file handle that needs explicit closing + + def __init__( + self, value: IO[Any], disposition: str = "attachment", *args: Any, **kwargs: Any + ) -> None: + if "filename" not in kwargs: + kwargs["filename"] = guess_filename(value) + + super().__init__(value, *args, **kwargs) + + if self._filename is not None and disposition is not None: + if hdrs.CONTENT_DISPOSITION not in self.headers: + self.set_content_disposition(disposition, filename=self._filename) + + def _set_or_restore_start_position(self) -> None: + """Set or restore the start position of the file-like object.""" + if self._start_position is None: + try: + self._start_position = self._value.tell() + except (OSError, AttributeError): + self._consumed = True # Cannot seek, mark as consumed + return + try: + self._value.seek(self._start_position) + except (OSError, AttributeError): + # Failed to seek back - mark as consumed since we've already read + self._consumed = True + + def _read_and_available_len( + self, remaining_content_len: int | None + ) -> tuple[int | None, bytes]: + """ + Read the file-like object and return both its total size and the first chunk. + + Args: + remaining_content_len: Optional limit on how many bytes to read in this operation. + If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size. + + Returns: + A tuple containing: + - The total size of the remaining unread content (None if size cannot be determined) + - The first chunk of bytes read from the file object + + This method is optimized to perform both size calculation and initial read + in a single operation, which is executed in a single executor job to minimize + context switches and file operations when streaming content. + + """ + self._set_or_restore_start_position() + size = self.size # Call size only once since it does I/O + return size, self._value.read( + min( + DEFAULT_CHUNK_SIZE, + size or DEFAULT_CHUNK_SIZE, + remaining_content_len or DEFAULT_CHUNK_SIZE, + ) + ) + + def _read(self, remaining_content_len: int | None) -> bytes: + """ + Read a chunk of data from the file-like object. + + Args: + remaining_content_len: Optional maximum number of bytes to read. + If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size. + + Returns: + A chunk of bytes read from the file object, respecting the + remaining_content_len limit if specified. + + This method is used for subsequent reads during streaming after + the initial _read_and_available_len call has been made. + + """ + return self._value.read(remaining_content_len or DEFAULT_CHUNK_SIZE) # type: ignore[no-any-return] + + @property + def size(self) -> int | None: + """ + Size of the payload in bytes. + + Returns the total size of the payload content from the initial position. + This ensures consistent Content-Length for requests, including 307/308 redirects + where the same payload instance is reused. + + Returns None if the size cannot be determined (e.g., for unseekable streams). + """ + try: + # Store the start position on first access. + # This is critical when the same payload instance is reused (e.g., 307/308 + # redirects). Without storing the initial position, after the payload is + # read once, the file position would be at EOF, which would cause the + # size calculation to return 0 (file_size - EOF position). + # By storing the start position, we ensure the size calculation always + # returns the correct total size for any subsequent use. + if self._start_position is None: + self._start_position = self._value.tell() + + # Return the total size from the start position + # This ensures Content-Length is correct even after reading + return os.fstat(self._value.fileno()).st_size - self._start_position + except (AttributeError, OSError): + return None + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire file-like payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method writes the entire file content without any length constraint. + It delegates to write_with_length() with no length limit for implementation + consistency. + + Note: + For new implementations that need length control, use write_with_length() directly. + This method is maintained for backwards compatibility with existing code. + + """ + await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write file-like payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method implements optimized streaming of file content with length constraints: + + 1. File reading is performed in a thread pool to avoid blocking the event loop + 2. Content is read and written in chunks to maintain memory efficiency + 3. Writing stops when either: + - All available file content has been written (when size is known) + - The specified content_length has been reached + 4. File resources are properly closed even if the operation is cancelled + + The implementation carefully handles both known-size and unknown-size payloads, + as well as constrained and unconstrained content lengths. + + """ + loop = asyncio.get_running_loop() + total_written_len = 0 + remaining_content_len = content_length + + # Get initial data and available length + available_len, chunk = await loop.run_in_executor( + None, self._read_and_available_len, remaining_content_len + ) + # Process data chunks until done + while chunk: + chunk_len = len(chunk) + + # Write data with or without length constraint + if remaining_content_len is None: + await writer.write(chunk) + else: + await writer.write(chunk[:remaining_content_len]) + remaining_content_len -= chunk_len + + total_written_len += chunk_len + + # Check if we're done writing + if self._should_stop_writing( + available_len, total_written_len, remaining_content_len + ): + return + + # Read next chunk + chunk = await loop.run_in_executor( + None, + self._read, + ( + min(DEFAULT_CHUNK_SIZE, remaining_content_len) + if remaining_content_len is not None + else DEFAULT_CHUNK_SIZE + ), + ) + + def _should_stop_writing( + self, + available_len: int | None, + total_written_len: int, + remaining_content_len: int | None, + ) -> bool: + """ + Determine if we should stop writing data. + + Args: + available_len: Known size of the payload if available (None if unknown) + total_written_len: Number of bytes already written + remaining_content_len: Remaining bytes to be written for content-length limited responses + + Returns: + True if we should stop writing data, based on either: + - Having written all available data (when size is known) + - Having written all requested content (when content-length is specified) + + """ + return (available_len is not None and total_written_len >= available_len) or ( + remaining_content_len is not None and remaining_content_len <= 0 + ) + + def _close(self) -> None: + """ + Async safe synchronous close operations for backwards compatibility. + + This method exists only for backwards + compatibility. Use the async close() method instead. + + WARNING: This method MUST be called from within an event loop. + Calling it outside an event loop will raise RuntimeError. + """ + # Skip if already consumed + if self._consumed: + return + self._consumed = True # Mark as consumed to prevent further writes + # Schedule file closing without awaiting to prevent cancellation issues + loop = asyncio.get_running_loop() + close_future = loop.run_in_executor(None, self._value.close) + # Hold a strong reference to the future to prevent it from being + # garbage collected before it completes. + _CLOSE_FUTURES.add(close_future) + close_future.add_done_callback(_CLOSE_FUTURES.remove) + + async def close(self) -> None: + """ + Close the payload if it holds any resources. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + """ + self._close() + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + WARNING: This method does blocking I/O and should not be called in the event loop. + """ + return self._read_all().decode(encoding, errors) + + def _read_all(self) -> bytes: + """Read the entire file-like object and return its content as bytes.""" + self._set_or_restore_start_position() + # Use readlines() to ensure we get all content + return b"".join(self._value.readlines()) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire file content and returns it as bytes. + It is equivalent to reading the file-like object directly. + The file reading is performed in an executor to avoid blocking the event loop. + """ + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, self._read_all) + + +class TextIOPayload(IOBasePayload): + _value: io.TextIOBase + # _autoclose = False (inherited) - Has text file handle that needs explicit closing + + def __init__( + self, + value: TextIO, + *args: Any, + encoding: str | None = None, + content_type: str | None = None, + **kwargs: Any, + ) -> None: + + if encoding is None: + if content_type is None: + encoding = "utf-8" + content_type = "text/plain; charset=utf-8" + else: + mimetype = parse_mimetype(content_type) + encoding = mimetype.parameters.get("charset", "utf-8") + else: + if content_type is None: + content_type = "text/plain; charset=%s" % encoding + + super().__init__( + value, + content_type=content_type, + encoding=encoding, + *args, + **kwargs, + ) + + def _read_and_available_len( + self, remaining_content_len: int | None + ) -> tuple[int | None, bytes]: + """ + Read the text file-like object and return both its total size and the first chunk. + + Args: + remaining_content_len: Optional limit on how many bytes to read in this operation. + If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size. + + Returns: + A tuple containing: + - The total size of the remaining unread content (None if size cannot be determined) + - The first chunk of bytes read from the file object, encoded using the payload's encoding + + This method is optimized to perform both size calculation and initial read + in a single operation, which is executed in a single executor job to minimize + context switches and file operations when streaming content. + + Note: + TextIOPayload handles encoding of the text content before writing it + to the stream. If no encoding is specified, UTF-8 is used as the default. + + """ + self._set_or_restore_start_position() + size = self.size + chunk = self._value.read( + min( + DEFAULT_CHUNK_SIZE, + size or DEFAULT_CHUNK_SIZE, + remaining_content_len or DEFAULT_CHUNK_SIZE, + ) + ) + return size, chunk.encode(self._encoding) if self._encoding else chunk.encode() + + def _read(self, remaining_content_len: int | None) -> bytes: + """ + Read a chunk of data from the text file-like object. + + Args: + remaining_content_len: Optional maximum number of bytes to read. + If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size. + + Returns: + A chunk of bytes read from the file object and encoded using the payload's + encoding. The data is automatically converted from text to bytes. + + This method is used for subsequent reads during streaming after + the initial _read_and_available_len call has been made. It properly + handles text encoding, converting the text content to bytes using + the specified encoding (or UTF-8 if none was provided). + + """ + chunk = self._value.read(remaining_content_len or DEFAULT_CHUNK_SIZE) + return chunk.encode(self._encoding) if self._encoding else chunk.encode() + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + WARNING: This method does blocking I/O and should not be called in the event loop. + """ + self._set_or_restore_start_position() + return self._value.read() + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire text file content and returns it as bytes. + It encodes the text content using the specified encoding. + The file reading is performed in an executor to avoid blocking the event loop. + """ + loop = asyncio.get_running_loop() + + # Use instance encoding if available, otherwise use parameter + actual_encoding = self._encoding or encoding + + def _read_and_encode() -> bytes: + self._set_or_restore_start_position() + # TextIO read() always returns the full content + return self._value.read().encode(actual_encoding, errors) + + return await loop.run_in_executor(None, _read_and_encode) + + +class BytesIOPayload(IOBasePayload): + _value: io.BytesIO + _size: int # Always initialized in __init__ + _autoclose = True # BytesIO is in-memory, safe to auto-close + + def __init__(self, value: io.BytesIO, *args: Any, **kwargs: Any) -> None: + super().__init__(value, *args, **kwargs) + # Calculate size once during initialization + self._size = len(self._value.getbuffer()) - self._value.tell() + + @property + def size(self) -> int: + """Size of the payload in bytes. + + Returns the number of bytes in the BytesIO buffer that will be transmitted. + This is calculated once during initialization for efficiency. + """ + return self._size + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + self._set_or_restore_start_position() + return self._value.read().decode(encoding, errors) + + async def write(self, writer: AbstractStreamWriter) -> None: + return await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write BytesIO payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This implementation is specifically optimized for BytesIO objects: + + 1. Reads content in chunks to maintain memory efficiency + 2. Yields control back to the event loop periodically to prevent blocking + when dealing with large BytesIO objects + 3. Respects content_length constraints when specified + 4. Properly cleans up by closing the BytesIO object when done or on error + + The periodic yielding to the event loop is important for maintaining + responsiveness when processing large in-memory buffers. + + """ + self._set_or_restore_start_position() + loop_count = 0 + remaining_bytes = content_length + while chunk := self._value.read(DEFAULT_CHUNK_SIZE): + if loop_count > 0: + # Avoid blocking the event loop + # if they pass a large BytesIO object + # and we are not in the first iteration + # of the loop + await asyncio.sleep(0) + if remaining_bytes is None: + await writer.write(chunk) + else: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + if remaining_bytes <= 0: + return + loop_count += 1 + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire BytesIO content and returns it as bytes. + It is equivalent to accessing the _value attribute directly. + """ + self._set_or_restore_start_position() + return self._value.read() + + async def close(self) -> None: + """ + Close the BytesIO payload. + + This does nothing since BytesIO is in-memory and does not require explicit closing. + """ + + +class BufferedReaderPayload(IOBasePayload): + _value: io.BufferedIOBase + # _autoclose = False (inherited) - Has buffered file handle that needs explicit closing + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + self._set_or_restore_start_position() + return self._value.read().decode(encoding, errors) + + +class JsonPayload(BytesPayload): + def __init__( + self, + value: Any, + encoding: str = "utf-8", + content_type: str = "application/json", + dumps: JSONEncoder = json.dumps, + *args: Any, + **kwargs: Any, + ) -> None: + + super().__init__( + dumps(value).encode(encoding), + content_type=content_type, + encoding=encoding, + *args, + **kwargs, + ) + + +class JsonBytesPayload(BytesPayload): + """JSON payload for encoders that return bytes directly. + + Use this when your JSON encoder (like orjson) returns bytes + instead of str, avoiding the encode/decode overhead. + """ + + def __init__( + self, + value: Any, + dumps: JSONBytesEncoder, + content_type: str = "application/json", + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__( + dumps(value), + content_type=content_type, + *args, + **kwargs, + ) + + +if TYPE_CHECKING: + from collections.abc import AsyncIterable, AsyncIterator + + _AsyncIterator = AsyncIterator[bytes] + _AsyncIterable = AsyncIterable[bytes] +else: + from collections.abc import AsyncIterable, AsyncIterator + + _AsyncIterator = AsyncIterator + _AsyncIterable = AsyncIterable + + +class AsyncIterablePayload(Payload): + + _iter: _AsyncIterator | None = None + _value: _AsyncIterable + _cached_chunks: list[bytes] | None = None + # _consumed stays False to allow reuse with cached content + _autoclose = True # Iterator doesn't need explicit closing + + def __init__(self, value: _AsyncIterable, *args: Any, **kwargs: Any) -> None: + if not isinstance(value, AsyncIterable): + raise TypeError( + "value argument must support " + "collections.abc.AsyncIterable interface, " + f"got {type(value)!r}" + ) + + if "content_type" not in kwargs: + kwargs["content_type"] = "application/octet-stream" + + super().__init__(value, *args, **kwargs) + + self._iter = value.__aiter__() + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire async iterable payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method iterates through the async iterable and writes each chunk + to the writer without any length constraint. + + Note: + For new implementations that need length control, use write_with_length() directly. + This method is maintained for backwards compatibility with existing code. + + """ + await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write async iterable payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This implementation handles streaming of async iterable content with length constraints: + + 1. If cached chunks are available, writes from them + 2. Otherwise iterates through the async iterable one chunk at a time + 3. Respects content_length constraints when specified + 4. Does NOT generate cache - that's done by as_bytes() + + """ + # If we have cached chunks, use them + if self._cached_chunks is not None: + remaining_bytes = content_length + for chunk in self._cached_chunks: + if remaining_bytes is None: + await writer.write(chunk) + elif remaining_bytes > 0: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + else: + break + return + + # If iterator is exhausted and we don't have cached chunks, nothing to write + if self._iter is None: + return + + # Stream from the iterator + remaining_bytes = content_length + + try: + while True: + chunk = await anext(self._iter) + if remaining_bytes is None: + await writer.write(chunk) + # If we have a content length limit + elif remaining_bytes > 0: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + # We still want to exhaust the iterator even + # if we have reached the content length limit + # since the file handle may not get closed by + # the iterator if we don't do this + except StopAsyncIteration: + # Iterator is exhausted + self._iter = None + self._consumed = True # Mark as consumed when streamed without caching + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """Decode the payload content as a string if cached chunks are available.""" + if self._cached_chunks is not None: + return b"".join(self._cached_chunks).decode(encoding, errors) + raise TypeError("Unable to decode - content not cached. Call as_bytes() first.") + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire async iterable content and returns it as bytes. + It generates and caches the chunks for future reuse. + """ + # If we have cached chunks, return them joined + if self._cached_chunks is not None: + return b"".join(self._cached_chunks) + + # If iterator is exhausted and no cache, return empty + if self._iter is None: + return b"" + + # Read all chunks and cache them + chunks: list[bytes] = [] + async for chunk in self._iter: + chunks.append(chunk) + + # Iterator is exhausted, cache the chunks + self._iter = None + self._cached_chunks = chunks + # Keep _consumed as False to allow reuse with cached chunks + + return b"".join(chunks) + + +class StreamReaderPayload(AsyncIterablePayload): + def __init__(self, value: StreamReader, *args: Any, **kwargs: Any) -> None: + super().__init__(value.iter_any(), *args, **kwargs) + + +PAYLOAD_REGISTRY = PayloadRegistry() +PAYLOAD_REGISTRY.register(BytesPayload, (bytes, bytearray, memoryview)) +PAYLOAD_REGISTRY.register(StringPayload, str) +PAYLOAD_REGISTRY.register(StringIOPayload, io.StringIO) +PAYLOAD_REGISTRY.register(TextIOPayload, io.TextIOBase) +PAYLOAD_REGISTRY.register(BytesIOPayload, io.BytesIO) +PAYLOAD_REGISTRY.register(BufferedReaderPayload, (io.BufferedReader, io.BufferedRandom)) +PAYLOAD_REGISTRY.register(IOBasePayload, io.IOBase) +PAYLOAD_REGISTRY.register(StreamReaderPayload, StreamReader) +# try_last for giving a chance to more specialized async interables like +# multipart.BodyPartReaderPayload override the default +PAYLOAD_REGISTRY.register(AsyncIterablePayload, AsyncIterable, order=Order.try_last) diff --git a/venv/Lib/site-packages/aiohttp/payload_streamer.py b/venv/Lib/site-packages/aiohttp/payload_streamer.py new file mode 100644 index 0000000000000000000000000000000000000000..095a88a2e35244e9f0e5c6f45893ff7bc0288ec5 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/payload_streamer.py @@ -0,0 +1,79 @@ +""" +Payload implementation for coroutines as data provider. + +As a simple case, you can upload data from file:: + + @aiohttp.streamer + async def file_sender(writer, file_name=None): + with open(file_name, 'rb') as f: + chunk = f.read(2**16) + while chunk: + await writer.write(chunk) + + chunk = f.read(2**16) + +Then you can use `file_sender` like this: + + async with session.post('http://httpbin.org/post', + data=file_sender(file_name='huge_file')) as resp: + print(await resp.text()) + +..note:: Coroutine must accept `writer` as first argument + +""" + +import types +import warnings +from collections.abc import Awaitable, Callable +from typing import Any + +from .abc import AbstractStreamWriter +from .payload import Payload, payload_type + +__all__ = ("streamer",) + + +class _stream_wrapper: + def __init__( + self, + coro: Callable[..., Awaitable[None]], + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> None: + self.coro = types.coroutine(coro) + self.args = args + self.kwargs = kwargs + + async def __call__(self, writer: AbstractStreamWriter) -> None: + await self.coro(writer, *self.args, **self.kwargs) + + +class streamer: + def __init__(self, coro: Callable[..., Awaitable[None]]) -> None: + warnings.warn( + "@streamer is deprecated, use async generators instead", + DeprecationWarning, + stacklevel=2, + ) + self.coro = coro + + def __call__(self, *args: Any, **kwargs: Any) -> _stream_wrapper: + return _stream_wrapper(self.coro, args, kwargs) + + +@payload_type(_stream_wrapper) +class StreamWrapperPayload(Payload): + async def write(self, writer: AbstractStreamWriter) -> None: + await self._value(writer) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + raise TypeError("Unable to decode.") + + +@payload_type(streamer) +class StreamPayload(StreamWrapperPayload): + def __init__(self, value: Any, *args: Any, **kwargs: Any) -> None: + super().__init__(value(), *args, **kwargs) + + async def write(self, writer: AbstractStreamWriter) -> None: + await self._value(writer) diff --git a/venv/Lib/site-packages/aiohttp/py.typed b/venv/Lib/site-packages/aiohttp/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..1d9f9a52a2f18944337009f77ba0d3cee2752144 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/py.typed @@ -0,0 +1 @@ +Marker diff --git a/venv/Lib/site-packages/aiohttp/pytest_plugin.py b/venv/Lib/site-packages/aiohttp/pytest_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..e8efc41606833cb836ce82af9a207af7da12f624 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/pytest_plugin.py @@ -0,0 +1,439 @@ +import asyncio +import contextlib +import inspect +import warnings +from collections.abc import Awaitable, Callable, Iterator +from typing import Any, Protocol, overload + +import pytest + +from .test_utils import ( + BaseTestServer, + RawTestServer, + TestClient, + TestServer, + loop_context, + setup_test_loop, + teardown_test_loop, + unused_port as _unused_port, +) +from .web import Application, BaseRequest, Request +from .web_protocol import _RequestHandler + +try: + import uvloop +except ImportError: # pragma: no cover + uvloop = None # type: ignore[assignment] + + +class AiohttpClient(Protocol): + @overload + async def __call__( + self, + __param: Application, + *, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[Request, Application]: ... + @overload + async def __call__( + self, + __param: BaseTestServer, + *, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[BaseRequest, None]: ... + + +class AiohttpServer(Protocol): + def __call__( + self, app: Application, *, port: int | None = None, **kwargs: Any + ) -> Awaitable[TestServer]: ... + + +class AiohttpRawServer(Protocol): + def __call__( + self, handler: _RequestHandler, *, port: int | None = None, **kwargs: Any + ) -> Awaitable[RawTestServer]: ... + + +def pytest_addoption(parser): # type: ignore[no-untyped-def] + parser.addoption( + "--aiohttp-fast", + action="store_true", + default=False, + help="run tests faster by disabling extra checks", + ) + parser.addoption( + "--aiohttp-loop", + action="store", + default="pyloop", + help="run tests with specific loop: pyloop, uvloop or all", + ) + parser.addoption( + "--aiohttp-enable-loop-debug", + action="store_true", + default=False, + help="enable event loop debug mode", + ) + + +def pytest_fixture_setup(fixturedef): # type: ignore[no-untyped-def] + """Set up pytest fixture. + + Allow fixtures to be coroutines. Run coroutine fixtures in an event loop. + """ + func = fixturedef.func + + if inspect.isasyncgenfunction(func): + # async generator fixture + is_async_gen = True + elif inspect.iscoroutinefunction(func): + # regular async fixture + is_async_gen = False + else: + # not an async fixture, nothing to do + return + + strip_request = False + if "request" not in fixturedef.argnames: + fixturedef.argnames += ("request",) + strip_request = True + + def wrapper(*args, **kwargs): # type: ignore[no-untyped-def] + request = kwargs["request"] + if strip_request: + del kwargs["request"] + + # if neither the fixture nor the test use the 'loop' fixture, + # 'getfixturevalue' will fail because the test is not parameterized + # (this can be removed someday if 'loop' is no longer parameterized) + if "loop" not in request.fixturenames: + raise Exception( + "Asynchronous fixtures must depend on the 'loop' fixture or " + "be used in tests depending from it." + ) + + _loop = request.getfixturevalue("loop") + + if is_async_gen: + # for async generators, we need to advance the generator once, + # then advance it again in a finalizer + gen = func(*args, **kwargs) + + def finalizer(): # type: ignore[no-untyped-def] + try: + return _loop.run_until_complete(gen.__anext__()) + except StopAsyncIteration: + pass + + request.addfinalizer(finalizer) + return _loop.run_until_complete(gen.__anext__()) + else: + return _loop.run_until_complete(func(*args, **kwargs)) + + fixturedef.func = wrapper + + +@pytest.fixture +def fast(request): # type: ignore[no-untyped-def] + """--fast config option""" + return request.config.getoption("--aiohttp-fast") + + +@pytest.fixture +def loop_debug(request): # type: ignore[no-untyped-def] + """--enable-loop-debug config option""" + return request.config.getoption("--aiohttp-enable-loop-debug") + + +@contextlib.contextmanager +def _runtime_warning_context(): # type: ignore[no-untyped-def] + """Context manager which checks for RuntimeWarnings. + + This exists specifically to + avoid "coroutine 'X' was never awaited" warnings being missed. + + If RuntimeWarnings occur in the context a RuntimeError is raised. + """ + with warnings.catch_warnings(record=True) as _warnings: + yield + rw = [ + f"{w.filename}:{w.lineno}:{w.message}" + for w in _warnings + if w.category == RuntimeWarning + ] + if rw: + raise RuntimeError( + "{} Runtime Warning{},\n{}".format( + len(rw), "" if len(rw) == 1 else "s", "\n".join(rw) + ) + ) + + +@contextlib.contextmanager +def _passthrough_loop_context(loop, fast=False): # type: ignore[no-untyped-def] + """Passthrough loop context. + + Sets up and tears down a loop unless one is passed in via the loop + argument when it's passed straight through. + """ + if loop: + # loop already exists, pass it straight through + yield loop + else: + # this shadows loop_context's standard behavior + loop = setup_test_loop() + yield loop + teardown_test_loop(loop, fast=fast) + + +def pytest_pycollect_makeitem(collector, name, obj): # type: ignore[no-untyped-def] + """Fix pytest collecting for coroutines.""" + if collector.funcnamefilter(name) and inspect.iscoroutinefunction(obj): + return list(collector._genfunctions(name, obj)) + + +def pytest_pyfunc_call(pyfuncitem): # type: ignore[no-untyped-def] + """Run coroutines in an event loop instead of a normal function call.""" + fast = pyfuncitem.config.getoption("--aiohttp-fast") + if inspect.iscoroutinefunction(pyfuncitem.function): + warnings.warn( + "aiohttp.pytest_plugin will be removed in v4. Please install pytest-aiohttp.", + DeprecationWarning, + ) + existing_loop = ( + pyfuncitem.funcargs.get("proactor_loop") + or pyfuncitem.funcargs.get("selector_loop") + or pyfuncitem.funcargs.get("uvloop_loop") + or pyfuncitem.funcargs.get("loop", None) + ) + + with _runtime_warning_context(): + with _passthrough_loop_context(existing_loop, fast=fast) as _loop: + testargs = { + arg: pyfuncitem.funcargs[arg] + for arg in pyfuncitem._fixtureinfo.argnames + } + _loop.run_until_complete(pyfuncitem.obj(**testargs)) + + return True + + +def pytest_generate_tests(metafunc): # type: ignore[no-untyped-def] + if "loop_factory" not in metafunc.fixturenames: + return + + loops = metafunc.config.option.aiohttp_loop + avail_factories: dict[str, Callable[[], asyncio.AbstractEventLoop]] + avail_factories = {"pyloop": asyncio.new_event_loop} + + if uvloop is not None: # pragma: no cover + avail_factories["uvloop"] = uvloop.new_event_loop + + if loops == "all": + loops = "pyloop,uvloop?" + + factories = {} # type: ignore[var-annotated] + for name in loops.split(","): + required = not name.endswith("?") + name = name.strip(" ?") + if name not in avail_factories: # pragma: no cover + if required: + raise ValueError( + "Unknown loop '%s', available loops: %s" + % (name, list(factories.keys())) + ) + else: + continue + factories[name] = avail_factories[name] + metafunc.parametrize( + "loop_factory", list(factories.values()), ids=list(factories.keys()) + ) + + +@pytest.fixture +def loop( + loop_factory: Callable[[], asyncio.AbstractEventLoop], + fast: bool, + loop_debug: bool, +) -> Iterator[asyncio.AbstractEventLoop]: + """Return an instance of the event loop.""" + with loop_context(loop_factory, fast=fast) as _loop: + if loop_debug: + _loop.set_debug(True) # pragma: no cover + asyncio.set_event_loop(_loop) + yield _loop + + +@pytest.fixture +def proactor_loop() -> Iterator[asyncio.AbstractEventLoop]: + factory = asyncio.ProactorEventLoop # type: ignore[attr-defined] + + with loop_context(factory) as _loop: + asyncio.set_event_loop(_loop) + yield _loop + + +@pytest.fixture +def unused_port(aiohttp_unused_port: Callable[[], int]) -> Callable[[], int]: + warnings.warn( + "Deprecated, use aiohttp_unused_port fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_unused_port + + +@pytest.fixture +def aiohttp_unused_port() -> Callable[[], int]: + """Return a port that is unused on the current host.""" + return _unused_port + + +@pytest.fixture +def aiohttp_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpServer]: + """Factory to create a TestServer instance, given an app. + + aiohttp_server(app, **kwargs) + """ + servers = [] + + async def go( + app: Application, + *, + host: str = "127.0.0.1", + port: int | None = None, + **kwargs: Any, + ) -> TestServer: + server = TestServer(app, host=host, port=port) + await server.start_server(loop=loop, **kwargs) + servers.append(server) + return server + + yield go + + async def finalize() -> None: + while servers: + await servers.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def test_server(aiohttp_server): # type: ignore[no-untyped-def] # pragma: no cover + warnings.warn( + "Deprecated, use aiohttp_server fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_server + + +@pytest.fixture +def aiohttp_raw_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpRawServer]: + """Factory to create a RawTestServer instance, given a web handler. + + aiohttp_raw_server(handler, **kwargs) + """ + servers = [] + + async def go( + handler: _RequestHandler, *, port: int | None = None, **kwargs: Any + ) -> RawTestServer: + server = RawTestServer(handler, port=port) + await server.start_server(loop=loop, **kwargs) + servers.append(server) + return server + + yield go + + async def finalize() -> None: + while servers: + await servers.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def raw_test_server( # type: ignore[no-untyped-def] # pragma: no cover + aiohttp_raw_server, +): + warnings.warn( + "Deprecated, use aiohttp_raw_server fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_raw_server + + +@pytest.fixture +def aiohttp_client(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpClient]: + """Factory to create a TestClient instance. + + aiohttp_client(app, **kwargs) + aiohttp_client(server, **kwargs) + aiohttp_client(raw_server, **kwargs) + """ + clients = [] + + @overload + async def go( + __param: Application, + *, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[Request, Application]: ... + + @overload + async def go( + __param: BaseTestServer, + *, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[BaseRequest, None]: ... + + async def go( + __param: Application | BaseTestServer, + *args: Any, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[Any, Any]: + if isinstance(__param, Callable) and not isinstance( # type: ignore[arg-type] + __param, (Application, BaseTestServer) + ): + __param = __param(loop, *args, **kwargs) + kwargs = {} + else: + assert not args, "args should be empty" + + if isinstance(__param, Application): + server_kwargs = server_kwargs or {} + server = TestServer(__param, loop=loop, **server_kwargs) + client = TestClient(server, loop=loop, **kwargs) + elif isinstance(__param, BaseTestServer): + client = TestClient(__param, loop=loop, **kwargs) + else: + raise ValueError("Unknown argument type: %r" % type(__param)) + + await client.start_server() + clients.append(client) + return client + + yield go + + async def finalize() -> None: + while clients: + await clients.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def test_client(aiohttp_client): # type: ignore[no-untyped-def] # pragma: no cover + warnings.warn( + "Deprecated, use aiohttp_client fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_client diff --git a/venv/Lib/site-packages/aiohttp/resolver.py b/venv/Lib/site-packages/aiohttp/resolver.py new file mode 100644 index 0000000000000000000000000000000000000000..46f7897da585cabb55543b5e4315eafb04381757 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/resolver.py @@ -0,0 +1,273 @@ +import asyncio +import socket +import weakref +from typing import Any, Final, Optional + +from .abc import AbstractResolver, ResolveResult + +__all__ = ("ThreadedResolver", "AsyncResolver", "DefaultResolver") + + +try: + import aiodns + + aiodns_default = hasattr(aiodns.DNSResolver, "getaddrinfo") +except ImportError: # pragma: no cover + aiodns = None # type: ignore[assignment] + aiodns_default = False + + +_NUMERIC_SOCKET_FLAGS = socket.AI_NUMERICHOST | socket.AI_NUMERICSERV +_NAME_SOCKET_FLAGS = socket.NI_NUMERICHOST | socket.NI_NUMERICSERV +_AI_ADDRCONFIG = socket.AI_ADDRCONFIG +if hasattr(socket, "AI_MASK"): + _AI_ADDRCONFIG &= socket.AI_MASK + + +class ThreadedResolver(AbstractResolver): + """Threaded resolver. + + Uses an Executor for synchronous getaddrinfo() calls. + concurrent.futures.ThreadPoolExecutor is used by default. + """ + + def __init__(self, loop: asyncio.AbstractEventLoop | None = None) -> None: + self._loop = loop or asyncio.get_running_loop() + + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> list[ResolveResult]: + infos = await self._loop.getaddrinfo( + host, + port, + type=socket.SOCK_STREAM, + family=family, + flags=_AI_ADDRCONFIG, + ) + + hosts: list[ResolveResult] = [] + for family, _, proto, _, address in infos: + if family == socket.AF_INET6: + if len(address) < 3: + # IPv6 is not supported by Python build, + # or IPv6 is not enabled in the host + continue + if address[3]: + # This is essential for link-local IPv6 addresses. + # LL IPv6 is a VERY rare case. Strictly speaking, we should use + # getnameinfo() unconditionally, but performance makes sense. + resolved_host, _port = await self._loop.getnameinfo( + address, _NAME_SOCKET_FLAGS + ) + port = int(_port) + else: + resolved_host, port = address[:2] + else: # IPv4 + assert family == socket.AF_INET + resolved_host, port = address # type: ignore[misc] + hosts.append( + ResolveResult( + hostname=host, + host=resolved_host, + port=port, + family=family, + proto=proto, + flags=_NUMERIC_SOCKET_FLAGS, + ) + ) + + return hosts + + async def close(self) -> None: + pass + + +class AsyncResolver(AbstractResolver): + """Use the `aiodns` package to make asynchronous DNS lookups""" + + def __init__( + self, + loop: asyncio.AbstractEventLoop | None = None, + *args: Any, + **kwargs: Any, + ) -> None: + if aiodns is None: + raise RuntimeError("Resolver requires aiodns library") + + self._loop = loop or asyncio.get_running_loop() + self._manager: _DNSResolverManager | None = None + # If custom args are provided, create a dedicated resolver instance + # This means each AsyncResolver with custom args gets its own + # aiodns.DNSResolver instance + if args or kwargs: + self._resolver = aiodns.DNSResolver(*args, **kwargs) + return + # Use the shared resolver from the manager for default arguments + self._manager = _DNSResolverManager() + self._resolver = self._manager.get_resolver(self, self._loop) + + if not hasattr(self._resolver, "gethostbyname"): + # aiodns 1.1 is not available, fallback to DNSResolver.query + self.resolve = self._resolve_with_query # type: ignore + + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> list[ResolveResult]: + try: + resp = await self._resolver.getaddrinfo( + host, + port=port, + type=socket.SOCK_STREAM, + family=family, + flags=_AI_ADDRCONFIG, + ) + except aiodns.error.DNSError as exc: + msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" + raise OSError(None, msg) from exc + hosts: list[ResolveResult] = [] + for node in resp.nodes: + address: tuple[bytes, int] | tuple[bytes, int, int, int] = node.addr + if node.family == socket.AF_INET6: + if len(address) > 3 and address[3]: + # This is essential for link-local IPv6 addresses. + # LL IPv6 is a VERY rare case. Strictly speaking, we should use + # getnameinfo() unconditionally, but performance makes sense. + result = await self._resolver.getnameinfo( + (address[0].decode("ascii"), *address[1:]), + _NAME_SOCKET_FLAGS, + ) + resolved_host = result.node + else: + resolved_host = address[0].decode("ascii") + port = address[1] + else: # IPv4 + assert node.family == socket.AF_INET + resolved_host = address[0].decode("ascii") + port = address[1] + hosts.append( + ResolveResult( + hostname=host, + host=resolved_host, + port=port, + family=node.family, + proto=0, + flags=_NUMERIC_SOCKET_FLAGS, + ) + ) + + if not hosts: + raise OSError(None, "DNS lookup failed") + + return hosts + + async def _resolve_with_query( + self, host: str, port: int = 0, family: int = socket.AF_INET + ) -> list[dict[str, Any]]: + qtype: Final = "AAAA" if family == socket.AF_INET6 else "A" + + try: + resp = await self._resolver.query(host, qtype) + except aiodns.error.DNSError as exc: + msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" + raise OSError(None, msg) from exc + + hosts = [] + for rr in resp: + hosts.append( + { + "hostname": host, + "host": rr.host, + "port": port, + "family": family, + "proto": 0, + "flags": socket.AI_NUMERICHOST, + } + ) + + if not hosts: + raise OSError(None, "DNS lookup failed") + + return hosts + + async def close(self) -> None: + if self._manager: + # Release the resolver from the manager if using the shared resolver + self._manager.release_resolver(self, self._loop) + self._manager = None # Clear reference to manager + self._resolver = None # type: ignore[assignment] # Clear reference to resolver + return + # Otherwise cancel our dedicated resolver + if self._resolver is not None: + self._resolver.cancel() + self._resolver = None # type: ignore[assignment] # Clear reference + + +class _DNSResolverManager: + """Manager for aiodns.DNSResolver objects. + + This class manages shared aiodns.DNSResolver instances + with no custom arguments across different event loops. + """ + + _instance: Optional["_DNSResolverManager"] = None + + def __new__(cls) -> "_DNSResolverManager": + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._init() + return cls._instance + + def _init(self) -> None: + # Use WeakKeyDictionary to allow event loops to be garbage collected + self._loop_data: weakref.WeakKeyDictionary[ + asyncio.AbstractEventLoop, + tuple[aiodns.DNSResolver, weakref.WeakSet[AsyncResolver]], + ] = weakref.WeakKeyDictionary() + + def get_resolver( + self, client: "AsyncResolver", loop: asyncio.AbstractEventLoop + ) -> "aiodns.DNSResolver": + """Get or create the shared aiodns.DNSResolver instance for a specific event loop. + + Args: + client: The AsyncResolver instance requesting the resolver. + This is required to track resolver usage. + loop: The event loop to use for the resolver. + """ + # Create a new resolver and client set for this loop if it doesn't exist + if loop not in self._loop_data: + resolver = aiodns.DNSResolver(loop=loop) + client_set: weakref.WeakSet[AsyncResolver] = weakref.WeakSet() + self._loop_data[loop] = (resolver, client_set) + else: + # Get the existing resolver and client set + resolver, client_set = self._loop_data[loop] + + # Register this client with the loop + client_set.add(client) + return resolver + + def release_resolver( + self, client: "AsyncResolver", loop: asyncio.AbstractEventLoop + ) -> None: + """Release the resolver for an AsyncResolver client when it's closed. + + Args: + client: The AsyncResolver instance to release. + loop: The event loop the resolver was using. + """ + # Remove client from its loop's tracking + current_loop_data = self._loop_data.get(loop) + if current_loop_data is None: + return + resolver, client_set = current_loop_data + client_set.discard(client) + # If no more clients for this loop, cancel and remove its resolver + if not client_set: + if resolver is not None: + resolver.cancel() + del self._loop_data[loop] + + +_DefaultType = type[AsyncResolver | ThreadedResolver] +DefaultResolver: _DefaultType = AsyncResolver if aiodns_default else ThreadedResolver diff --git a/venv/Lib/site-packages/aiohttp/streams.py b/venv/Lib/site-packages/aiohttp/streams.py new file mode 100644 index 0000000000000000000000000000000000000000..6708ed41088bafd889bde3db80f850190257cfd4 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/streams.py @@ -0,0 +1,760 @@ +import asyncio +import collections +import sys +import warnings +from collections.abc import Awaitable, Callable +from typing import Final, Generic, TypeVar + +from .base_protocol import BaseProtocol +from .helpers import ( + _EXC_SENTINEL, + DEFAULT_CHUNK_SIZE, + BaseTimerContext, + TimerNoop, + set_exception, + set_result, +) +from .http_exceptions import LineTooLong +from .log import internal_logger + +__all__ = ( + "EMPTY_PAYLOAD", + "EofStream", + "StreamReader", + "DataQueue", +) + +_T = TypeVar("_T") + + +class EofStream(Exception): + """eof stream indication.""" + + +class AsyncStreamIterator(Generic[_T]): + + __slots__ = ("read_func",) + + def __init__(self, read_func: Callable[[], Awaitable[_T]]) -> None: + self.read_func = read_func + + def __aiter__(self) -> "AsyncStreamIterator[_T]": + return self + + async def __anext__(self) -> _T: + try: + rv = await self.read_func() + except EofStream: + raise StopAsyncIteration + if rv == b"": + raise StopAsyncIteration + return rv + + +class ChunkTupleAsyncStreamIterator: + + __slots__ = ("_stream",) + + def __init__(self, stream: "StreamReader") -> None: + self._stream = stream + + def __aiter__(self) -> "ChunkTupleAsyncStreamIterator": + return self + + async def __anext__(self) -> tuple[bytes, bool]: + rv = await self._stream.readchunk() + if rv == (b"", False): + raise StopAsyncIteration + return rv + + +class StreamReader: + """An enhancement of asyncio.StreamReader. + + Supports asynchronous iteration by line, chunk or as available:: + + async for line in reader: + ... + async for chunk in reader.iter_chunked(1024): + ... + async for slice in reader.iter_any(): + ... + + """ + + __slots__ = ( + "_protocol", + "_low_water", + "_high_water", + "_low_water_chunks", + "_high_water_chunks", + "_loop", + "_size", + "_cursor", + "_http_chunk_splits", + "_buffer", + "_buffer_offset", + "_eof", + "_waiter", + "_eof_waiter", + "_exception", + "_timer", + "_eof_callbacks", + "_eof_counter", + "total_bytes", + "total_compressed_bytes", + ) + + def __init__( + self, + protocol: BaseProtocol, + limit: int, + *, + timer: BaseTimerContext | None = None, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + self._protocol = protocol + self._low_water = limit + self._high_water = limit * 2 + if loop is None: + loop = asyncio.get_event_loop() + # Use max(4, ...) because there's always at least 1 chunk split remaining + # (the current position), so we need low_water >= 2 to allow resume. + # limit // 16 gets us a reasonable value of 16k with default 256KiB limit. + self._high_water_chunks = max(4, limit // 16) + self._low_water_chunks = self._high_water_chunks // 2 + self._loop = loop + self._size = 0 + self._cursor = 0 + self._http_chunk_splits: collections.deque[int] | None = None + self._buffer: collections.deque[bytes] = collections.deque() + self._buffer_offset = 0 + self._eof = False + self._waiter: asyncio.Future[None] | None = None + self._eof_waiter: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._timer = TimerNoop() if timer is None else timer + self._eof_callbacks: list[Callable[[], None]] = [] + self._eof_counter = 0 + self.total_bytes = 0 + self.total_compressed_bytes: int | None = None + + def __repr__(self) -> str: + info = [self.__class__.__name__] + if self._size: + info.append("%d bytes" % self._size) + if self._eof: + info.append("eof") + if self._low_water != DEFAULT_CHUNK_SIZE: + info.append("low=%d high=%d" % (self._low_water, self._high_water)) + if self._waiter: + info.append("w=%r" % self._waiter) + if self._exception: + info.append("e=%r" % self._exception) + return "<%s>" % " ".join(info) + + def __aiter__(self) -> AsyncStreamIterator[bytes]: + return AsyncStreamIterator(self.readline) + + def iter_chunked(self, n: int) -> AsyncStreamIterator[bytes]: + """Returns an asynchronous iterator that yields chunks of size n.""" + self.set_read_chunk_size(n) + return AsyncStreamIterator(lambda: self.read(n)) + + def iter_any(self) -> AsyncStreamIterator[bytes]: + """Yield all available data as soon as it is received.""" + return AsyncStreamIterator(self.readany) + + def iter_chunks(self) -> ChunkTupleAsyncStreamIterator: + """Yield chunks of data as they are received by the server. + + The yielded objects are tuples + of (bytes, bool) as returned by the StreamReader.readchunk method. + """ + return ChunkTupleAsyncStreamIterator(self) + + def get_read_buffer_limits(self) -> tuple[int, int]: + return (self._low_water, self._high_water) + + def set_read_chunk_size(self, n: int) -> None: + """Raise buffer limits to match the consumer's chunk size.""" + if n > self._low_water: + self._low_water = n + self._high_water = n * 2 + + def exception(self) -> BaseException | None: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._exception = exc + self._eof_callbacks.clear() + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + waiter = self._eof_waiter + if waiter is not None: + self._eof_waiter = None + set_exception(waiter, exc, exc_cause) + + def on_eof(self, callback: Callable[[], None]) -> None: + if self._eof: + try: + callback() + except Exception: + internal_logger.exception("Exception in eof callback") + else: + self._eof_callbacks.append(callback) + + def feed_eof(self) -> None: + self._eof = True + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + waiter = self._eof_waiter + if waiter is not None: + self._eof_waiter = None + set_result(waiter, None) + + # At EOF the parser is done, there won't be unprocessed data. + self._protocol.resume_reading(resume_parser=False) + + for cb in self._eof_callbacks: + try: + cb() + except Exception: + internal_logger.exception("Exception in eof callback") + + self._eof_callbacks.clear() + + def is_eof(self) -> bool: + """Return True if 'feed_eof' was called.""" + return self._eof + + def at_eof(self) -> bool: + """Return True if the buffer is empty and 'feed_eof' was called.""" + return self._eof and not self._buffer + + async def wait_eof(self) -> None: + if self._eof: + return + + assert self._eof_waiter is None + self._eof_waiter = self._loop.create_future() + try: + await self._eof_waiter + finally: + self._eof_waiter = None + + @property + def total_raw_bytes(self) -> int: + if self.total_compressed_bytes is None: + return self.total_bytes + return self.total_compressed_bytes + + def unread_data(self, data: bytes) -> None: + """rollback reading some data from stream, inserting it to buffer head.""" + warnings.warn( + "unread_data() is deprecated " + "and will be removed in future releases (#3260)", + DeprecationWarning, + stacklevel=2, + ) + if not data: + return + + if self._buffer_offset: + self._buffer[0] = self._buffer[0][self._buffer_offset :] + self._buffer_offset = 0 + self._size += len(data) + self._cursor -= len(data) + self._buffer.appendleft(data) + self._eof_counter = 0 + + # TODO: size is ignored, remove the param later + def feed_data(self, data: bytes, size: int = 0) -> bool: + assert not self._eof, "feed_data after feed_eof" + + if not data: + return False + + data_len = len(data) + self._size += data_len + self._buffer.append(data) + self.total_bytes += data_len + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + if self._size > self._high_water: + self._protocol.pause_reading() + return False + + def begin_http_chunk_receiving(self) -> None: + if self._http_chunk_splits is None: + if self.total_bytes: + raise RuntimeError( + "Called begin_http_chunk_receiving when some data was already fed" + ) + self._http_chunk_splits = collections.deque() + + def end_http_chunk_receiving(self) -> None: + if self._http_chunk_splits is None: + raise RuntimeError( + "Called end_chunk_receiving without calling " + "begin_chunk_receiving first" + ) + + # self._http_chunk_splits contains logical byte offsets from start of + # the body transfer. Each offset is the offset of the end of a chunk. + # "Logical" means bytes, accessible for a user. + # If no chunks containing logical data were received, current position + # is difinitely zero. + pos = self._http_chunk_splits[-1] if self._http_chunk_splits else 0 + + if self.total_bytes == pos: + # We should not add empty chunks here. So we check for that. + # Note, when chunked + gzip is used, we can receive a chunk + # of compressed data, but that data may not be enough for gzip FSM + # to yield any uncompressed data. That's why current position may + # not change after receiving a chunk. + return + + self._http_chunk_splits.append(self.total_bytes) + + # If we get too many small chunks before self._high_water is reached, then any + # .read() call becomes computationally expensive, and could block the event loop + # for too long, hence an additional self._high_water_chunks here. + if len(self._http_chunk_splits) > self._high_water_chunks: + self._protocol.pause_reading() + + # wake up readchunk when end of http chunk received + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + async def _wait(self, func_name: str) -> None: + if not self._protocol.connected: + raise RuntimeError("Connection closed.") + + # StreamReader uses a future to link the protocol feed_data() method + # to a read coroutine. Running two read coroutines at the same time + # would have an unexpected behaviour. It would not possible to know + # which coroutine would get the next data. + if self._waiter is not None: + raise RuntimeError( + "%s() called while another coroutine is " + "already waiting for incoming data" % func_name + ) + + waiter = self._waiter = self._loop.create_future() + try: + with self._timer: + await waiter + finally: + self._waiter = None + + async def readline(self, *, max_line_length: int | None = None) -> bytes: + return await self.readuntil(max_size=max_line_length) + + async def readuntil( + self, separator: bytes = b"\n", *, max_size: int | None = None + ) -> bytes: + seplen = len(separator) + if seplen == 0: + raise ValueError("Separator should be at least one-byte string") + + if self._exception is not None: + raise self._exception + + chunk = b"" + chunk_size = 0 + not_enough = True + max_size = max_size or self._high_water + + while not_enough: + while self._buffer and not_enough: + offset = self._buffer_offset + ichar = self._buffer[0].find(separator, offset) + 1 + # Read from current offset to found separator or to the end. + data = self._read_nowait_chunk( + ichar - offset + seplen - 1 if ichar else -1 + ) + chunk += data + chunk_size += len(data) + if ichar: + not_enough = False + + if chunk_size > max_size: + raise LineTooLong(chunk[:100] + b"...", max_size) + + if self._eof: + break + + if not_enough: + await self._wait("readuntil") + + return chunk + + async def read(self, n: int = -1) -> bytes: + if self._exception is not None: + raise self._exception + + # migration problem; with DataQueue you have to catch + # EofStream exception, so common way is to run payload.read() inside + # infinite loop. what can cause real infinite loop with StreamReader + # lets keep this code one major release. + if __debug__: + if self._eof and not self._buffer: + self._eof_counter = getattr(self, "_eof_counter", 0) + 1 + if self._eof_counter > 5: + internal_logger.warning( + "Multiple access to StreamReader in eof state, " + "might be infinite loop.", + stack_info=True, + ) + + if not n: + return b"" + + if n < 0: + # Reading everything β€” remove decompression chunk limit. + self.set_read_chunk_size(sys.maxsize) + blocks = [] + while True: + block = await self.readany() + if not block: + break + blocks.append(block) + return b"".join(blocks) + + self.set_read_chunk_size(n) + # TODO: should be `if` instead of `while` + # because waiter maybe triggered on chunk end, + # without feeding any data + while not self._buffer and not self._eof: + await self._wait("read") + + return self._read_nowait(n) + + async def readany(self) -> bytes: + if self._exception is not None: + raise self._exception + + # TODO: should be `if` instead of `while` + # because waiter maybe triggered on chunk end, + # without feeding any data + while not self._buffer and not self._eof: + await self._wait("readany") + + return self._read_nowait(-1) + + async def readchunk(self) -> tuple[bytes, bool]: + """Returns a tuple of (data, end_of_http_chunk). + + When chunked transfer + encoding is used, end_of_http_chunk is a boolean indicating if the end + of the data corresponds to the end of a HTTP chunk , otherwise it is + always False. + """ + while True: + if self._exception is not None: + raise self._exception + + while self._http_chunk_splits: + pos = self._http_chunk_splits.popleft() + if pos == self._cursor: + return (b"", True) + if pos > self._cursor: + return (self._read_nowait(pos - self._cursor), True) + internal_logger.warning( + "Skipping HTTP chunk end due to data " + "consumption beyond chunk boundary" + ) + + if self._buffer: + return (self._read_nowait_chunk(-1), False) + # return (self._read_nowait(-1), False) + + if self._eof: + # Special case for signifying EOF. + # (b'', True) is not a final return value actually. + return (b"", False) + + await self._wait("readchunk") + + async def readexactly(self, n: int) -> bytes: + if self._exception is not None: + raise self._exception + + blocks: list[bytes] = [] + while n > 0: + block = await self.read(n) + if not block: + partial = b"".join(blocks) + raise asyncio.IncompleteReadError(partial, len(partial) + n) + blocks.append(block) + n -= len(block) + + return b"".join(blocks) + + def read_nowait(self, n: int = -1) -> bytes: + # default was changed to be consistent with .read(-1) + # + # I believe the most users don't know about the method and + # they are not affected. + if self._exception is not None: + raise self._exception + + if self._waiter and not self._waiter.done(): + raise RuntimeError( + "Called while some coroutine is waiting for incoming data." + ) + + return self._read_nowait(n) + + def _read_nowait_chunk(self, n: int) -> bytes: + first_buffer = self._buffer[0] + offset = self._buffer_offset + if n != -1 and len(first_buffer) - offset > n: + data = first_buffer[offset : offset + n] + self._buffer_offset += n + + elif offset: + self._buffer.popleft() + data = first_buffer[offset:] + self._buffer_offset = 0 + + else: + data = self._buffer.popleft() + + data_len = len(data) + self._size -= data_len + self._cursor += data_len + + chunk_splits = self._http_chunk_splits + # Prevent memory leak: drop useless chunk splits + while chunk_splits and chunk_splits[0] < self._cursor: + chunk_splits.popleft() + + if self._size < self._low_water and ( + self._http_chunk_splits is None + or len(self._http_chunk_splits) < self._low_water_chunks + ): + self._protocol.resume_reading() + return data + + def _read_nowait(self, n: int) -> bytes: + """Read not more than n bytes, or whole buffer if n == -1""" + self._timer.assert_timeout() + + if n == -1: + # Drain only chunks present now; _read_nowait_chunk() can + # re-entrantly resume_reading() and refill the buffer. + count = len(self._buffer) + if count == 1: + return self._read_nowait_chunk(-1) + return b"".join([self._read_nowait_chunk(-1) for _ in range(count)]) + + chunks: list[bytes] = [] + while self._buffer: + chunk = self._read_nowait_chunk(n) + chunks.append(chunk) + n -= len(chunk) + if n == 0: + break + + return b"".join(chunks) if chunks else b"" + + +class EmptyStreamReader(StreamReader): # lgtm [py/missing-call-to-init] + + __slots__ = ("_read_eof_chunk",) + + def __init__(self) -> None: + self._read_eof_chunk = False + self.total_bytes = 0 + + def __repr__(self) -> str: + return "<%s>" % self.__class__.__name__ + + def exception(self) -> BaseException | None: + return None + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + pass + + def on_eof(self, callback: Callable[[], None]) -> None: + try: + callback() + except Exception: + internal_logger.exception("Exception in eof callback") + + def feed_eof(self) -> None: + pass + + def is_eof(self) -> bool: + return True + + def at_eof(self) -> bool: + return True + + async def wait_eof(self) -> None: + return + + def feed_data(self, data: bytes, n: int = 0) -> bool: + return False + + def set_read_chunk_size(self, n: int) -> None: + return + + async def readline(self, *, max_line_length: int | None = None) -> bytes: + return b"" + + async def read(self, n: int = -1) -> bytes: + return b"" + + # TODO add async def readuntil + + async def readany(self) -> bytes: + return b"" + + async def readchunk(self) -> tuple[bytes, bool]: + if not self._read_eof_chunk: + self._read_eof_chunk = True + return (b"", False) + + return (b"", True) + + async def readexactly(self, n: int) -> bytes: + raise asyncio.IncompleteReadError(b"", n) + + def read_nowait(self, n: int = -1) -> bytes: + return b"" + + +EMPTY_PAYLOAD: Final[StreamReader] = EmptyStreamReader() + + +class DataQueue(Generic[_T]): + """DataQueue is a general-purpose blocking queue with one reader.""" + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._eof = False + self._waiter: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._buffer: collections.deque[tuple[_T, int]] = collections.deque() + + def __len__(self) -> int: + return len(self._buffer) + + def is_eof(self) -> bool: + return self._eof + + def at_eof(self) -> bool: + return self._eof and not self._buffer + + def exception(self) -> BaseException | None: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def feed_data(self, data: _T, size: int = 0) -> None: + self._buffer.append((data, size)) + if (waiter := self._waiter) is not None: + self._waiter = None + set_result(waiter, None) + + def feed_eof(self) -> None: + self._eof = True + if (waiter := self._waiter) is not None: + self._waiter = None + set_result(waiter, None) + + async def read(self) -> _T: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + if self._buffer: + data, _ = self._buffer.popleft() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + def __aiter__(self) -> AsyncStreamIterator[_T]: + return AsyncStreamIterator(self.read) + + +class FlowControlDataQueue(DataQueue[_T]): + """FlowControlDataQueue resumes and pauses an underlying stream. + + It is a destination for parsed data. + + This class is deprecated and will be removed in version 4.0. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + super().__init__(loop=loop) + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + + def feed_data(self, data: _T, size: int = 0) -> None: + super().feed_data(data, size) + self._size += size + + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> _T: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + if self._buffer: + data, size = self._buffer.popleft() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream diff --git a/venv/Lib/site-packages/aiohttp/tcp_helpers.py b/venv/Lib/site-packages/aiohttp/tcp_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..8d39ba9e3c818281d45f8525d0cdf112871e2017 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/tcp_helpers.py @@ -0,0 +1,37 @@ +"""Helper methods to tune a TCP connection""" + +import asyncio +import socket +from contextlib import suppress +from typing import Optional # noqa + +__all__ = ("tcp_keepalive", "tcp_nodelay") + + +if hasattr(socket, "SO_KEEPALIVE"): + + def tcp_keepalive(transport: asyncio.Transport) -> None: + sock = transport.get_extra_info("socket") + if sock is not None: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + +else: + + def tcp_keepalive(transport: asyncio.Transport) -> None: # pragma: no cover + pass + + +def tcp_nodelay(transport: asyncio.Transport, value: bool) -> None: + sock = transport.get_extra_info("socket") + + if sock is None: + return + + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return + + value = bool(value) + + # socket may be closed already, on windows OSError get raised + with suppress(OSError): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, value) diff --git a/venv/Lib/site-packages/aiohttp/test_utils.py b/venv/Lib/site-packages/aiohttp/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..10e38a156c4533c5ef813d4a03eea04f1c6e3e72 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/test_utils.py @@ -0,0 +1,808 @@ +"""Utilities shared by tests.""" + +import asyncio +import contextlib +import gc +import inspect +import ipaddress +import os +import socket +import sys +import warnings +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterator +from types import TracebackType +from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar, cast, overload +from unittest import IsolatedAsyncioTestCase, mock + +from aiosignal import Signal +from multidict import CIMultiDict, CIMultiDictProxy +from yarl import URL + +import aiohttp +from aiohttp.client import ( + _BaseRequestContextManager, + _RequestContextManager, + _RequestOptions, + _WSRequestContextManager, +) + +from . import ClientSession, hdrs +from .abc import AbstractCookieJar +from .client_reqrep import ClientResponse +from .client_ws import ClientWebSocketResponse +from .helpers import sentinel +from .http import HttpVersion, RawRequestMessage +from .streams import EMPTY_PAYLOAD, StreamReader +from .typedefs import StrOrURL +from .web import ( + Application, + AppRunner, + BaseRequest, + BaseRunner, + Request, + Server, + ServerRunner, + SockSite, + UrlMappingMatchInfo, +) +from .web_protocol import _RequestHandler + +if TYPE_CHECKING: + from ssl import SSLContext +else: + SSLContext = Any + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + from typing import Unpack + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = Any + +_ApplicationNone = TypeVar("_ApplicationNone", Application, None) +_Request = TypeVar("_Request", bound=BaseRequest) + +REUSE_ADDRESS = os.name == "posix" and sys.platform != "cygwin" + + +def get_unused_port_socket( + host: str, family: socket.AddressFamily = socket.AF_INET +) -> socket.socket: + return get_port_socket(host, 0, family) + + +def get_port_socket( + host: str, port: int, family: socket.AddressFamily +) -> socket.socket: + s = socket.socket(family, socket.SOCK_STREAM) + if REUSE_ADDRESS: + # Windows has different semantics for SO_REUSEADDR, + # so don't set it. Ref: + # https://docs.microsoft.com/en-us/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + return s + + +def unused_port() -> int: + """Return a port that is unused on the current host.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return cast(int, s.getsockname()[1]) + + +class BaseTestServer(ABC): + __test__ = False + + def __init__( + self, + *, + scheme: str = "", + loop: asyncio.AbstractEventLoop | None = None, + host: str = "127.0.0.1", + port: int | None = None, + skip_url_asserts: bool = False, + socket_factory: Callable[ + [str, int, socket.AddressFamily], socket.socket + ] = get_port_socket, + **kwargs: Any, + ) -> None: + self._loop = loop + self.runner: BaseRunner | None = None + self._root: URL | None = None + self.host = host + self.port = port + self._closed = False + self.scheme = scheme + self.skip_url_asserts = skip_url_asserts + self.socket_factory = socket_factory + + async def start_server( + self, loop: asyncio.AbstractEventLoop | None = None, **kwargs: Any + ) -> None: + if self.runner: + return + self._loop = loop + self._ssl = kwargs.pop("ssl", None) + self.runner = await self._make_runner(handler_cancellation=True, **kwargs) + await self.runner.setup() + if not self.port: + self.port = 0 + absolute_host = self.host + try: + version = ipaddress.ip_address(self.host).version + except ValueError: + version = 4 + if version == 6: + absolute_host = f"[{self.host}]" + family = socket.AF_INET6 if version == 6 else socket.AF_INET + _sock = self.socket_factory(self.host, self.port, family) + self.host, self.port = _sock.getsockname()[:2] + site = SockSite(self.runner, sock=_sock, ssl_context=self._ssl) + await site.start() + server = site._server + assert server is not None + sockets = server.sockets # type: ignore[attr-defined] + assert sockets is not None + self.port = sockets[0].getsockname()[1] + if not self.scheme: + self.scheme = "https" if self._ssl else "http" + self._root = URL(f"{self.scheme}://{absolute_host}:{self.port}") + + @abstractmethod # pragma: no cover + async def _make_runner(self, **kwargs: Any) -> BaseRunner: + pass + + def make_url(self, path: StrOrURL) -> URL: + assert self._root is not None + url = URL(path) + if not self.skip_url_asserts: + assert not url.absolute + return self._root.join(url) + else: + return URL(str(self._root) + str(path)) + + @property + def started(self) -> bool: + return self.runner is not None + + @property + def closed(self) -> bool: + return self._closed + + @property + def handler(self) -> Server: + # for backward compatibility + # web.Server instance + runner = self.runner + assert runner is not None + assert runner.server is not None + return runner.server + + async def close(self) -> None: + """Close all fixtures created by the test client. + + After that point, the TestClient is no longer usable. + + This is an idempotent function: running close multiple times + will not have any additional effects. + + close is also run when the object is garbage collected, and on + exit when used as a context manager. + + """ + if self.started and not self.closed: + assert self.runner is not None + await self.runner.cleanup() + self._root = None + self.port = None + self._closed = True + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> "BaseTestServer": + await self.start_server(loop=self._loop) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self.close() + + +class TestServer(BaseTestServer): + def __init__( + self, + app: Application, + *, + scheme: str = "", + host: str = "127.0.0.1", + port: int | None = None, + **kwargs: Any, + ): + self.app = app + super().__init__(scheme=scheme, host=host, port=port, **kwargs) + + async def _make_runner(self, **kwargs: Any) -> BaseRunner: + return AppRunner(self.app, **kwargs) + + +class RawTestServer(BaseTestServer): + def __init__( + self, + handler: _RequestHandler, + *, + scheme: str = "", + host: str = "127.0.0.1", + port: int | None = None, + **kwargs: Any, + ) -> None: + self._handler = handler + super().__init__(scheme=scheme, host=host, port=port, **kwargs) + + async def _make_runner(self, debug: bool = True, **kwargs: Any) -> ServerRunner: + srv = Server(self._handler, loop=self._loop, debug=debug, **kwargs) + return ServerRunner(srv, debug=debug, **kwargs) + + +class TestClient(Generic[_Request, _ApplicationNone]): + """ + A test client implementation. + + To write functional tests for aiohttp based servers. + + """ + + __test__ = False + + @overload + def __init__( + self: "TestClient[Request, Application]", + server: TestServer, + *, + cookie_jar: AbstractCookieJar | None = None, + **kwargs: Any, + ) -> None: ... + @overload + def __init__( + self: "TestClient[_Request, None]", + server: BaseTestServer, + *, + cookie_jar: AbstractCookieJar | None = None, + **kwargs: Any, + ) -> None: ... + def __init__( + self, + server: BaseTestServer, + *, + cookie_jar: AbstractCookieJar | None = None, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Any, + ) -> None: + if not isinstance(server, BaseTestServer): + raise TypeError( + "server must be TestServer instance, found type: %r" % type(server) + ) + self._server = server + self._loop = loop + if cookie_jar is None: + cookie_jar = aiohttp.CookieJar(unsafe=True, loop=loop) + self._session = ClientSession(loop=loop, cookie_jar=cookie_jar, **kwargs) + self._session._retry_connection = False + self._closed = False + self._responses: list[ClientResponse] = [] + self._websockets: list[ClientWebSocketResponse[bool]] = [] + + async def start_server(self) -> None: + await self._server.start_server(loop=self._loop) + + @property + def host(self) -> str: + return self._server.host + + @property + def port(self) -> int | None: + return self._server.port + + @property + def server(self) -> BaseTestServer: + return self._server + + @property + def app(self) -> _ApplicationNone: + return getattr(self._server, "app", None) # type: ignore[return-value] + + @property + def session(self) -> ClientSession: + """An internal aiohttp.ClientSession. + + Unlike the methods on the TestClient, client session requests + do not automatically include the host in the url queried, and + will require an absolute path to the resource. + + """ + return self._session + + def make_url(self, path: StrOrURL) -> URL: + return self._server.make_url(path) + + async def _request( + self, method: str, path: StrOrURL, **kwargs: Any + ) -> ClientResponse: + resp = await self._session.request(method, self.make_url(path), **kwargs) + # save it to close later + self._responses.append(resp) + return resp + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + self, method: str, path: StrOrURL, **kwargs: Unpack[_RequestOptions] + ) -> _RequestContextManager: ... + + def get( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def options( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def head( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def post( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def put( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def patch( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def delete( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + else: + + def request( + self, method: str, path: StrOrURL, **kwargs: Any + ) -> _RequestContextManager: + """Routes a request to tested http server. + + The interface is identical to aiohttp.ClientSession.request, + except the loop kwarg is overridden by the instance used by the + test server. + + """ + return _RequestContextManager(self._request(method, path, **kwargs)) + + def get(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP GET request.""" + return _RequestContextManager(self._request(hdrs.METH_GET, path, **kwargs)) + + def post(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP POST request.""" + return _RequestContextManager(self._request(hdrs.METH_POST, path, **kwargs)) + + def options(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP OPTIONS request.""" + return _RequestContextManager( + self._request(hdrs.METH_OPTIONS, path, **kwargs) + ) + + def head(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP HEAD request.""" + return _RequestContextManager(self._request(hdrs.METH_HEAD, path, **kwargs)) + + def put(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PUT request.""" + return _RequestContextManager(self._request(hdrs.METH_PUT, path, **kwargs)) + + def patch(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_PATCH, path, **kwargs) + ) + + def delete(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_DELETE, path, **kwargs) + ) + + @overload + def ws_connect( + self, path: StrOrURL, *, decode_text: Literal[True] = ..., **kwargs: Any + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[True]]]": ... + + @overload + def ws_connect( + self, path: StrOrURL, *, decode_text: Literal[False], **kwargs: Any + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[False]]]": ... + + @overload + def ws_connect( + self, path: StrOrURL, *, decode_text: bool = ..., **kwargs: Any + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": ... + + def ws_connect( + self, path: StrOrURL, *, decode_text: bool = True, **kwargs: Any + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": + """Initiate websocket connection. + + The api corresponds to aiohttp.ClientSession.ws_connect. + + """ + return _WSRequestContextManager( + self._ws_connect(path, decode_text=decode_text, **kwargs) + ) + + @overload + async def _ws_connect( + self, path: StrOrURL, *, decode_text: Literal[True] = ..., **kwargs: Any + ) -> "ClientWebSocketResponse[Literal[True]]": ... + + @overload + async def _ws_connect( + self, path: StrOrURL, *, decode_text: Literal[False], **kwargs: Any + ) -> "ClientWebSocketResponse[Literal[False]]": ... + + @overload + async def _ws_connect( + self, path: StrOrURL, *, decode_text: bool = ..., **kwargs: Any + ) -> "ClientWebSocketResponse[bool]": ... + + async def _ws_connect( + self, path: StrOrURL, *, decode_text: bool = True, **kwargs: Any + ) -> "ClientWebSocketResponse[bool]": + ws = await self._session.ws_connect( + self.make_url(path), decode_text=decode_text, **kwargs + ) + self._websockets.append(ws) + return ws + + async def close(self) -> None: + """Close all fixtures created by the test client. + + After that point, the TestClient is no longer usable. + + This is an idempotent function: running close multiple times + will not have any additional effects. + + close is also run on exit when used as a(n) (asynchronous) + context manager. + + """ + if not self._closed: + for resp in self._responses: + resp.close() + for ws in self._websockets: + await ws.close() + await self._session.close() + await self._server.close() + self._closed = True + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> Self: + await self.start_server() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + await self.close() + + +class AioHTTPTestCase(IsolatedAsyncioTestCase): + """A base class to allow for unittest web applications using aiohttp. + + Provides the following: + + * self.client (aiohttp.test_utils.TestClient): an aiohttp test client. + * self.loop (asyncio.BaseEventLoop): the event loop in which the + application and server are running. + * self.app (aiohttp.web.Application): the application returned by + self.get_application() + + Note that the TestClient's methods are asynchronous: you have to + execute function on the test client using asynchronous methods. + """ + + async def get_application(self) -> Application: + """Get application. + + This method should be overridden + to return the aiohttp.web.Application + object to test. + """ + return self.get_app() + + def get_app(self) -> Application: + """Obsolete method used to constructing web application. + + Use .get_application() coroutine instead. + """ + raise RuntimeError("Did you forget to define get_application()?") + + async def asyncSetUp(self) -> None: + self.loop = asyncio.get_running_loop() + return await self.setUpAsync() + + async def setUpAsync(self) -> None: + self.app = await self.get_application() + self.server = await self.get_server(self.app) + self.client = await self.get_client(self.server) + + await self.client.start_server() + + async def asyncTearDown(self) -> None: + return await self.tearDownAsync() + + async def tearDownAsync(self) -> None: + await self.client.close() + + async def get_server(self, app: Application) -> TestServer: + """Return a TestServer instance.""" + return TestServer(app, loop=self.loop) + + async def get_client(self, server: TestServer) -> TestClient[Request, Application]: + """Return a TestClient instance.""" + return TestClient(server, loop=self.loop) + + +def unittest_run_loop(func: Any, *args: Any, **kwargs: Any) -> Any: + """ + A decorator dedicated to use with asynchronous AioHTTPTestCase test methods. + + In 3.8+, this does nothing. + """ + warnings.warn( + "Decorator `@unittest_run_loop` is no longer needed in aiohttp 3.8+", + DeprecationWarning, + stacklevel=2, + ) + return func + + +_LOOP_FACTORY = Callable[[], asyncio.AbstractEventLoop] + + +@contextlib.contextmanager +def loop_context( + loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, fast: bool = False +) -> Iterator[asyncio.AbstractEventLoop]: + """A contextmanager that creates an event_loop, for test purposes. + + Handles the creation and cleanup of a test loop. + """ + loop = setup_test_loop(loop_factory) + yield loop + teardown_test_loop(loop, fast=fast) + + +def setup_test_loop( + loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, +) -> asyncio.AbstractEventLoop: + """Create and return an asyncio.BaseEventLoop instance. + + The caller should also call teardown_test_loop, + once they are done with the loop. + """ + loop = loop_factory() + asyncio.set_event_loop(loop) + return loop + + +def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None: + """Teardown and cleanup an event_loop created by setup_test_loop.""" + closed = loop.is_closed() + if not closed: + loop.call_soon(loop.stop) + loop.run_forever() + loop.close() + + if not fast: + gc.collect() + + asyncio.set_event_loop(None) + + +def _create_app_mock() -> mock.MagicMock: + def get_dict(app: Any, key: str) -> Any: + return app.__app_dict[key] + + def set_dict(app: Any, key: str, value: Any) -> None: + app.__app_dict[key] = value + + app = mock.MagicMock(spec=Application) + app.__app_dict = {} + app.__getitem__ = get_dict + app.__setitem__ = set_dict + + app._debug = False + app.on_response_prepare = Signal(app) + app.on_response_prepare.freeze() + return app + + +def _create_transport(sslcontext: SSLContext | None = None) -> mock.Mock: + transport = mock.Mock() + + def get_extra_info(key: str) -> SSLContext | tuple[str, int] | None: + if key == "sslcontext": + return sslcontext + return ("127.0.0.1", 80) if key == "sockname" else None + + transport.get_extra_info.side_effect = get_extra_info + return transport + + +def make_mocked_request( + method: str, + path: str, + headers: Any = None, + *, + match_info: Any = sentinel, + version: HttpVersion = HttpVersion(1, 1), + closing: bool = False, + app: Any = None, + writer: Any = sentinel, + protocol: Any = sentinel, + transport: Any = sentinel, + payload: StreamReader = EMPTY_PAYLOAD, + sslcontext: SSLContext | None = None, + client_max_size: int = 1024**2, + loop: Any = ..., +) -> Request: + """Creates mocked web.Request testing purposes. + + Useful in unit tests, when spinning full web server is overkill or + specific conditions and errors are hard to trigger. + """ + task = mock.Mock() + if loop is ...: + # no loop passed, try to get the current one if + # its is running as we need a real loop to create + # executor jobs to be able to do testing + # with a real executor + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = mock.Mock() + loop.create_future.return_value = () + + if version < HttpVersion(1, 1): + closing = True + + if headers: + headers = CIMultiDictProxy(CIMultiDict(headers)) + raw_hdrs = tuple( + (k.encode("utf-8"), v.encode("utf-8")) for k, v in headers.items() + ) + else: + headers = CIMultiDictProxy(CIMultiDict()) + raw_hdrs = () + + chunked = "chunked" in headers.get(hdrs.TRANSFER_ENCODING, "").lower() + upgrade = headers.get(hdrs.CONNECTION, "").lower() == "upgrade" and bool( + headers.get(hdrs.UPGRADE) + ) + + message = RawRequestMessage( + method, + path, + version, + headers, + raw_hdrs, + closing, + None, + upgrade, + chunked, + URL(path), + ) + if app is None: + app = _create_app_mock() + + if transport is sentinel: + transport = _create_transport(sslcontext) + + if protocol is sentinel: + protocol = mock.Mock() + protocol.max_field_size = 8190 + protocol.max_line_length = 8190 + protocol.max_headers = 128 + protocol.transport = transport + type(protocol).peername = mock.PropertyMock( + return_value=transport.get_extra_info("peername") + ) + type(protocol).sockname = mock.PropertyMock( + return_value=transport.get_extra_info("sockname") + ) + type(protocol).ssl_context = mock.PropertyMock(return_value=sslcontext) + + if writer is sentinel: + writer = mock.Mock() + writer.write_headers = make_mocked_coro(None) + writer.write = make_mocked_coro(None) + writer.write_eof = make_mocked_coro(None) + writer.drain = make_mocked_coro(None) + writer.transport = transport + + protocol.transport = transport + protocol.writer = writer + + req = Request( + message, payload, protocol, writer, task, loop, client_max_size=client_max_size + ) + + match_info = UrlMappingMatchInfo( + {} if match_info is sentinel else match_info, mock.Mock() + ) + match_info.add_app(app) + req._match_info = match_info + + return req + + +def make_mocked_coro( + return_value: Any = sentinel, raise_exception: Any = sentinel +) -> Any: + """Creates a coroutine mock.""" + + async def mock_coro(*args: Any, **kwargs: Any) -> Any: + if raise_exception is not sentinel: + raise raise_exception + if not inspect.isawaitable(return_value): + return return_value + await return_value + + return mock.Mock(wraps=mock_coro) diff --git a/venv/Lib/site-packages/aiohttp/tracing.py b/venv/Lib/site-packages/aiohttp/tracing.py new file mode 100644 index 0000000000000000000000000000000000000000..16cf55710f1763f14b7c862804e0de56d402bef2 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/tracing.py @@ -0,0 +1,453 @@ +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any, TypeVar + +import attr +from aiosignal import Signal +from multidict import CIMultiDict +from yarl import URL + +from .client_reqrep import ClientResponse + +if TYPE_CHECKING: + from .client import ClientSession + + _ParamT_contra = TypeVar("_ParamT_contra", contravariant=True) + _TracingSignal = Signal[ClientSession, SimpleNamespace, _ParamT_contra] + + +__all__ = ( + "TraceConfig", + "TraceRequestStartParams", + "TraceRequestEndParams", + "TraceRequestExceptionParams", + "TraceConnectionQueuedStartParams", + "TraceConnectionQueuedEndParams", + "TraceConnectionCreateStartParams", + "TraceConnectionCreateEndParams", + "TraceConnectionReuseconnParams", + "TraceDnsResolveHostStartParams", + "TraceDnsResolveHostEndParams", + "TraceDnsCacheHitParams", + "TraceDnsCacheMissParams", + "TraceRequestRedirectParams", + "TraceRequestChunkSentParams", + "TraceResponseChunkReceivedParams", + "TraceRequestHeadersSentParams", +) + + +class TraceConfig: + """First-class used to trace requests launched via ClientSession objects.""" + + def __init__( + self, trace_config_ctx_factory: type[SimpleNamespace] = SimpleNamespace + ) -> None: + self._on_request_start: _TracingSignal[TraceRequestStartParams] = Signal(self) + self._on_request_chunk_sent: _TracingSignal[TraceRequestChunkSentParams] = ( + Signal(self) + ) + self._on_response_chunk_received: _TracingSignal[ + TraceResponseChunkReceivedParams + ] = Signal(self) + self._on_request_end: _TracingSignal[TraceRequestEndParams] = Signal(self) + self._on_request_exception: _TracingSignal[TraceRequestExceptionParams] = ( + Signal(self) + ) + self._on_request_redirect: _TracingSignal[TraceRequestRedirectParams] = Signal( + self + ) + self._on_connection_queued_start: _TracingSignal[ + TraceConnectionQueuedStartParams + ] = Signal(self) + self._on_connection_queued_end: _TracingSignal[ + TraceConnectionQueuedEndParams + ] = Signal(self) + self._on_connection_create_start: _TracingSignal[ + TraceConnectionCreateStartParams + ] = Signal(self) + self._on_connection_create_end: _TracingSignal[ + TraceConnectionCreateEndParams + ] = Signal(self) + self._on_connection_reuseconn: _TracingSignal[ + TraceConnectionReuseconnParams + ] = Signal(self) + self._on_dns_resolvehost_start: _TracingSignal[ + TraceDnsResolveHostStartParams + ] = Signal(self) + self._on_dns_resolvehost_end: _TracingSignal[TraceDnsResolveHostEndParams] = ( + Signal(self) + ) + self._on_dns_cache_hit: _TracingSignal[TraceDnsCacheHitParams] = Signal(self) + self._on_dns_cache_miss: _TracingSignal[TraceDnsCacheMissParams] = Signal(self) + self._on_request_headers_sent: _TracingSignal[TraceRequestHeadersSentParams] = ( + Signal(self) + ) + + self._trace_config_ctx_factory = trace_config_ctx_factory + + def trace_config_ctx(self, trace_request_ctx: Any = None) -> SimpleNamespace: + """Return a new trace_config_ctx instance""" + return self._trace_config_ctx_factory(trace_request_ctx=trace_request_ctx) + + def freeze(self) -> None: + self._on_request_start.freeze() + self._on_request_chunk_sent.freeze() + self._on_response_chunk_received.freeze() + self._on_request_end.freeze() + self._on_request_exception.freeze() + self._on_request_redirect.freeze() + self._on_connection_queued_start.freeze() + self._on_connection_queued_end.freeze() + self._on_connection_create_start.freeze() + self._on_connection_create_end.freeze() + self._on_connection_reuseconn.freeze() + self._on_dns_resolvehost_start.freeze() + self._on_dns_resolvehost_end.freeze() + self._on_dns_cache_hit.freeze() + self._on_dns_cache_miss.freeze() + self._on_request_headers_sent.freeze() + + @property + def on_request_start(self) -> "_TracingSignal[TraceRequestStartParams]": + return self._on_request_start + + @property + def on_request_chunk_sent( + self, + ) -> "_TracingSignal[TraceRequestChunkSentParams]": + return self._on_request_chunk_sent + + @property + def on_response_chunk_received( + self, + ) -> "_TracingSignal[TraceResponseChunkReceivedParams]": + return self._on_response_chunk_received + + @property + def on_request_end(self) -> "_TracingSignal[TraceRequestEndParams]": + return self._on_request_end + + @property + def on_request_exception( + self, + ) -> "_TracingSignal[TraceRequestExceptionParams]": + return self._on_request_exception + + @property + def on_request_redirect( + self, + ) -> "_TracingSignal[TraceRequestRedirectParams]": + return self._on_request_redirect + + @property + def on_connection_queued_start( + self, + ) -> "_TracingSignal[TraceConnectionQueuedStartParams]": + return self._on_connection_queued_start + + @property + def on_connection_queued_end( + self, + ) -> "_TracingSignal[TraceConnectionQueuedEndParams]": + return self._on_connection_queued_end + + @property + def on_connection_create_start( + self, + ) -> "_TracingSignal[TraceConnectionCreateStartParams]": + return self._on_connection_create_start + + @property + def on_connection_create_end( + self, + ) -> "_TracingSignal[TraceConnectionCreateEndParams]": + return self._on_connection_create_end + + @property + def on_connection_reuseconn( + self, + ) -> "_TracingSignal[TraceConnectionReuseconnParams]": + return self._on_connection_reuseconn + + @property + def on_dns_resolvehost_start( + self, + ) -> "_TracingSignal[TraceDnsResolveHostStartParams]": + return self._on_dns_resolvehost_start + + @property + def on_dns_resolvehost_end( + self, + ) -> "_TracingSignal[TraceDnsResolveHostEndParams]": + return self._on_dns_resolvehost_end + + @property + def on_dns_cache_hit(self) -> "_TracingSignal[TraceDnsCacheHitParams]": + return self._on_dns_cache_hit + + @property + def on_dns_cache_miss(self) -> "_TracingSignal[TraceDnsCacheMissParams]": + return self._on_dns_cache_miss + + @property + def on_request_headers_sent( + self, + ) -> "_TracingSignal[TraceRequestHeadersSentParams]": + return self._on_request_headers_sent + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestStartParams: + """Parameters sent by the `on_request_start` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestChunkSentParams: + """Parameters sent by the `on_request_chunk_sent` signal""" + + method: str + url: URL + chunk: bytes + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceResponseChunkReceivedParams: + """Parameters sent by the `on_response_chunk_received` signal""" + + method: str + url: URL + chunk: bytes + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestEndParams: + """Parameters sent by the `on_request_end` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + response: ClientResponse + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestExceptionParams: + """Parameters sent by the `on_request_exception` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + exception: BaseException + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestRedirectParams: + """Parameters sent by the `on_request_redirect` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + response: ClientResponse + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionQueuedStartParams: + """Parameters sent by the `on_connection_queued_start` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionQueuedEndParams: + """Parameters sent by the `on_connection_queued_end` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionCreateStartParams: + """Parameters sent by the `on_connection_create_start` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionCreateEndParams: + """Parameters sent by the `on_connection_create_end` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionReuseconnParams: + """Parameters sent by the `on_connection_reuseconn` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsResolveHostStartParams: + """Parameters sent by the `on_dns_resolvehost_start` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsResolveHostEndParams: + """Parameters sent by the `on_dns_resolvehost_end` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsCacheHitParams: + """Parameters sent by the `on_dns_cache_hit` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsCacheMissParams: + """Parameters sent by the `on_dns_cache_miss` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestHeadersSentParams: + """Parameters sent by the `on_request_headers_sent` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + + +class Trace: + """Internal dependency holder class. + + Used to keep together the main dependencies used + at the moment of send a signal. + """ + + def __init__( + self, + session: "ClientSession", + trace_config: TraceConfig, + trace_config_ctx: SimpleNamespace, + ) -> None: + self._trace_config = trace_config + self._trace_config_ctx = trace_config_ctx + self._session = session + + async def send_request_start( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + return await self._trace_config.on_request_start.send( + self._session, + self._trace_config_ctx, + TraceRequestStartParams(method, url, headers), + ) + + async def send_request_chunk_sent( + self, method: str, url: URL, chunk: bytes + ) -> None: + return await self._trace_config.on_request_chunk_sent.send( + self._session, + self._trace_config_ctx, + TraceRequestChunkSentParams(method, url, chunk), + ) + + async def send_response_chunk_received( + self, method: str, url: URL, chunk: bytes + ) -> None: + return await self._trace_config.on_response_chunk_received.send( + self._session, + self._trace_config_ctx, + TraceResponseChunkReceivedParams(method, url, chunk), + ) + + async def send_request_end( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + response: ClientResponse, + ) -> None: + return await self._trace_config.on_request_end.send( + self._session, + self._trace_config_ctx, + TraceRequestEndParams(method, url, headers, response), + ) + + async def send_request_exception( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + exception: BaseException, + ) -> None: + return await self._trace_config.on_request_exception.send( + self._session, + self._trace_config_ctx, + TraceRequestExceptionParams(method, url, headers, exception), + ) + + async def send_request_redirect( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + response: ClientResponse, + ) -> None: + return await self._trace_config._on_request_redirect.send( + self._session, + self._trace_config_ctx, + TraceRequestRedirectParams(method, url, headers, response), + ) + + async def send_connection_queued_start(self) -> None: + return await self._trace_config.on_connection_queued_start.send( + self._session, self._trace_config_ctx, TraceConnectionQueuedStartParams() + ) + + async def send_connection_queued_end(self) -> None: + return await self._trace_config.on_connection_queued_end.send( + self._session, self._trace_config_ctx, TraceConnectionQueuedEndParams() + ) + + async def send_connection_create_start(self) -> None: + return await self._trace_config.on_connection_create_start.send( + self._session, self._trace_config_ctx, TraceConnectionCreateStartParams() + ) + + async def send_connection_create_end(self) -> None: + return await self._trace_config.on_connection_create_end.send( + self._session, self._trace_config_ctx, TraceConnectionCreateEndParams() + ) + + async def send_connection_reuseconn(self) -> None: + return await self._trace_config.on_connection_reuseconn.send( + self._session, self._trace_config_ctx, TraceConnectionReuseconnParams() + ) + + async def send_dns_resolvehost_start(self, host: str) -> None: + return await self._trace_config.on_dns_resolvehost_start.send( + self._session, self._trace_config_ctx, TraceDnsResolveHostStartParams(host) + ) + + async def send_dns_resolvehost_end(self, host: str) -> None: + return await self._trace_config.on_dns_resolvehost_end.send( + self._session, self._trace_config_ctx, TraceDnsResolveHostEndParams(host) + ) + + async def send_dns_cache_hit(self, host: str) -> None: + return await self._trace_config.on_dns_cache_hit.send( + self._session, self._trace_config_ctx, TraceDnsCacheHitParams(host) + ) + + async def send_dns_cache_miss(self, host: str) -> None: + return await self._trace_config.on_dns_cache_miss.send( + self._session, self._trace_config_ctx, TraceDnsCacheMissParams(host) + ) + + async def send_request_headers( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + return await self._trace_config._on_request_headers_sent.send( + self._session, + self._trace_config_ctx, + TraceRequestHeadersSentParams(method, url, headers), + ) diff --git a/venv/Lib/site-packages/aiohttp/typedefs.py b/venv/Lib/site-packages/aiohttp/typedefs.py new file mode 100644 index 0000000000000000000000000000000000000000..b08b0c824c51f23ef9954861a0c1a98e2e672c47 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/typedefs.py @@ -0,0 +1,61 @@ +import json +import os +from collections.abc import Awaitable, Callable, Iterable, Mapping +from typing import TYPE_CHECKING, Any, Protocol, Union + +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy, istr +from yarl import URL, Query as _Query + +Query = _Query + +DEFAULT_JSON_ENCODER = json.dumps +DEFAULT_JSON_DECODER = json.loads + +if TYPE_CHECKING: + _CIMultiDict = CIMultiDict[str] + _CIMultiDictProxy = CIMultiDictProxy[str] + _MultiDict = MultiDict[str] + _MultiDictProxy = MultiDictProxy[str] + from http.cookies import BaseCookie, Morsel + + from .web import Request, StreamResponse +else: + _CIMultiDict = CIMultiDict + _CIMultiDictProxy = CIMultiDictProxy + _MultiDict = MultiDict + _MultiDictProxy = MultiDictProxy + +Byteish = Union[bytes, bytearray, memoryview] +JSONEncoder = Callable[[Any], str] +JSONBytesEncoder = Callable[[Any], bytes] +JSONDecoder = Callable[[str], Any] +LooseHeaders = Union[ + Mapping[str, str], + Mapping[istr, str], + _CIMultiDict, + _CIMultiDictProxy, + Iterable[tuple[str | istr, str]], +] +RawHeaders = tuple[tuple[bytes, bytes], ...] +StrOrURL = Union[str, URL] + +LooseCookiesMappings = Mapping[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]] +LooseCookiesIterables = Iterable[ + tuple[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]] +] +LooseCookies = Union[ + LooseCookiesMappings, + LooseCookiesIterables, + "BaseCookie[str]", +] + +Handler = Callable[["Request"], Awaitable["StreamResponse"]] + + +class Middleware(Protocol): + def __call__( + self, request: "Request", handler: Handler + ) -> Awaitable["StreamResponse"]: ... + + +PathLike = Union[str, "os.PathLike[str]"] diff --git a/venv/Lib/site-packages/aiohttp/web.py b/venv/Lib/site-packages/aiohttp/web.py new file mode 100644 index 0000000000000000000000000000000000000000..67614043d54010f18fec2935517ed553ee8402de --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web.py @@ -0,0 +1,593 @@ +import asyncio +import logging +import os +import socket +import sys +import warnings +from argparse import ArgumentParser +from collections.abc import Awaitable, Callable, Iterable, Iterable as TypingIterable +from contextlib import suppress +from importlib import import_module +from typing import TYPE_CHECKING, Any, cast + +from .abc import AbstractAccessLogger +from .helpers import AppKey, RequestKey, ResponseKey +from .log import access_logger +from .typedefs import PathLike +from .web_app import Application as Application, CleanupError as CleanupError +from .web_exceptions import ( + HTTPAccepted as HTTPAccepted, + HTTPBadGateway as HTTPBadGateway, + HTTPBadRequest as HTTPBadRequest, + HTTPClientError as HTTPClientError, + HTTPConflict as HTTPConflict, + HTTPCreated as HTTPCreated, + HTTPError as HTTPError, + HTTPException as HTTPException, + HTTPExpectationFailed as HTTPExpectationFailed, + HTTPFailedDependency as HTTPFailedDependency, + HTTPForbidden as HTTPForbidden, + HTTPFound as HTTPFound, + HTTPGatewayTimeout as HTTPGatewayTimeout, + HTTPGone as HTTPGone, + HTTPInsufficientStorage as HTTPInsufficientStorage, + HTTPInternalServerError as HTTPInternalServerError, + HTTPLengthRequired as HTTPLengthRequired, + HTTPMethodNotAllowed as HTTPMethodNotAllowed, + HTTPMisdirectedRequest as HTTPMisdirectedRequest, + HTTPMove as HTTPMove, + HTTPMovedPermanently as HTTPMovedPermanently, + HTTPMultipleChoices as HTTPMultipleChoices, + HTTPNetworkAuthenticationRequired as HTTPNetworkAuthenticationRequired, + HTTPNoContent as HTTPNoContent, + HTTPNonAuthoritativeInformation as HTTPNonAuthoritativeInformation, + HTTPNotAcceptable as HTTPNotAcceptable, + HTTPNotExtended as HTTPNotExtended, + HTTPNotFound as HTTPNotFound, + HTTPNotImplemented as HTTPNotImplemented, + HTTPNotModified as HTTPNotModified, + HTTPOk as HTTPOk, + HTTPPartialContent as HTTPPartialContent, + HTTPPaymentRequired as HTTPPaymentRequired, + HTTPPermanentRedirect as HTTPPermanentRedirect, + HTTPPreconditionFailed as HTTPPreconditionFailed, + HTTPPreconditionRequired as HTTPPreconditionRequired, + HTTPProxyAuthenticationRequired as HTTPProxyAuthenticationRequired, + HTTPRedirection as HTTPRedirection, + HTTPRequestEntityTooLarge as HTTPRequestEntityTooLarge, + HTTPRequestHeaderFieldsTooLarge as HTTPRequestHeaderFieldsTooLarge, + HTTPRequestRangeNotSatisfiable as HTTPRequestRangeNotSatisfiable, + HTTPRequestTimeout as HTTPRequestTimeout, + HTTPRequestURITooLong as HTTPRequestURITooLong, + HTTPResetContent as HTTPResetContent, + HTTPSeeOther as HTTPSeeOther, + HTTPServerError as HTTPServerError, + HTTPServiceUnavailable as HTTPServiceUnavailable, + HTTPSuccessful as HTTPSuccessful, + HTTPTemporaryRedirect as HTTPTemporaryRedirect, + HTTPTooManyRequests as HTTPTooManyRequests, + HTTPUnauthorized as HTTPUnauthorized, + HTTPUnavailableForLegalReasons as HTTPUnavailableForLegalReasons, + HTTPUnprocessableEntity as HTTPUnprocessableEntity, + HTTPUnsupportedMediaType as HTTPUnsupportedMediaType, + HTTPUpgradeRequired as HTTPUpgradeRequired, + HTTPUseProxy as HTTPUseProxy, + HTTPVariantAlsoNegotiates as HTTPVariantAlsoNegotiates, + HTTPVersionNotSupported as HTTPVersionNotSupported, + NotAppKeyWarning as NotAppKeyWarning, +) +from .web_fileresponse import FileResponse as FileResponse +from .web_log import AccessLogger +from .web_middlewares import ( + middleware as middleware, + normalize_path_middleware as normalize_path_middleware, +) +from .web_protocol import ( + PayloadAccessError as PayloadAccessError, + RequestHandler as RequestHandler, + RequestPayloadError as RequestPayloadError, +) +from .web_request import ( + BaseRequest as BaseRequest, + FileField as FileField, + Request as Request, +) +from .web_response import ( + ContentCoding as ContentCoding, + Response as Response, + StreamResponse as StreamResponse, + json_bytes_response as json_bytes_response, + json_response as json_response, +) +from .web_routedef import ( + AbstractRouteDef as AbstractRouteDef, + RouteDef as RouteDef, + RouteTableDef as RouteTableDef, + StaticDef as StaticDef, + delete as delete, + get as get, + head as head, + options as options, + patch as patch, + post as post, + put as put, + route as route, + static as static, + view as view, +) +from .web_runner import ( + AppRunner as AppRunner, + BaseRunner as BaseRunner, + BaseSite as BaseSite, + GracefulExit as GracefulExit, + NamedPipeSite as NamedPipeSite, + ServerRunner as ServerRunner, + SockSite as SockSite, + TCPSite as TCPSite, + UnixSite as UnixSite, +) +from .web_server import Server as Server +from .web_urldispatcher import ( + AbstractResource as AbstractResource, + AbstractRoute as AbstractRoute, + DynamicResource as DynamicResource, + PlainResource as PlainResource, + PrefixedSubAppResource as PrefixedSubAppResource, + Resource as Resource, + ResourceRoute as ResourceRoute, + StaticResource as StaticResource, + UrlDispatcher as UrlDispatcher, + UrlMappingMatchInfo as UrlMappingMatchInfo, + View as View, +) +from .web_ws import ( + WebSocketReady as WebSocketReady, + WebSocketResponse as WebSocketResponse, + WSMsgType as WSMsgType, +) + +__all__ = ( + # web_app + "AppKey", + "Application", + "CleanupError", + # web_exceptions + "NotAppKeyWarning", + "HTTPAccepted", + "HTTPBadGateway", + "HTTPBadRequest", + "HTTPClientError", + "HTTPConflict", + "HTTPCreated", + "HTTPError", + "HTTPException", + "HTTPExpectationFailed", + "HTTPFailedDependency", + "HTTPForbidden", + "HTTPFound", + "HTTPGatewayTimeout", + "HTTPGone", + "HTTPInsufficientStorage", + "HTTPInternalServerError", + "HTTPLengthRequired", + "HTTPMethodNotAllowed", + "HTTPMisdirectedRequest", + "HTTPMove", + "HTTPMovedPermanently", + "HTTPMultipleChoices", + "HTTPNetworkAuthenticationRequired", + "HTTPNoContent", + "HTTPNonAuthoritativeInformation", + "HTTPNotAcceptable", + "HTTPNotExtended", + "HTTPNotFound", + "HTTPNotImplemented", + "HTTPNotModified", + "HTTPOk", + "HTTPPartialContent", + "HTTPPaymentRequired", + "HTTPPermanentRedirect", + "HTTPPreconditionFailed", + "HTTPPreconditionRequired", + "HTTPProxyAuthenticationRequired", + "HTTPRedirection", + "HTTPRequestEntityTooLarge", + "HTTPRequestHeaderFieldsTooLarge", + "HTTPRequestRangeNotSatisfiable", + "HTTPRequestTimeout", + "HTTPRequestURITooLong", + "HTTPResetContent", + "HTTPSeeOther", + "HTTPServerError", + "HTTPServiceUnavailable", + "HTTPSuccessful", + "HTTPTemporaryRedirect", + "HTTPTooManyRequests", + "HTTPUnauthorized", + "HTTPUnavailableForLegalReasons", + "HTTPUnprocessableEntity", + "HTTPUnsupportedMediaType", + "HTTPUpgradeRequired", + "HTTPUseProxy", + "HTTPVariantAlsoNegotiates", + "HTTPVersionNotSupported", + # web_fileresponse + "FileResponse", + # web_middlewares + "middleware", + "normalize_path_middleware", + # web_protocol + "PayloadAccessError", + "RequestHandler", + "RequestPayloadError", + # web_request + "BaseRequest", + "FileField", + "Request", + "RequestKey", + # web_response + "ContentCoding", + "Response", + "StreamResponse", + "json_bytes_response", + "json_response", + "ResponseKey", + # web_routedef + "AbstractRouteDef", + "RouteDef", + "RouteTableDef", + "StaticDef", + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "route", + "static", + "view", + # web_runner + "AppRunner", + "BaseRunner", + "BaseSite", + "GracefulExit", + "ServerRunner", + "SockSite", + "TCPSite", + "UnixSite", + "NamedPipeSite", + # web_server + "Server", + # web_urldispatcher + "AbstractResource", + "AbstractRoute", + "DynamicResource", + "PlainResource", + "PrefixedSubAppResource", + "Resource", + "ResourceRoute", + "StaticResource", + "UrlDispatcher", + "UrlMappingMatchInfo", + "View", + # web_ws + "WebSocketReady", + "WebSocketResponse", + "WSMsgType", + # web + "run_app", +) + + +if TYPE_CHECKING: + from ssl import SSLContext +else: + try: + from ssl import SSLContext + except ImportError: # pragma: no cover + SSLContext = object # type: ignore[misc,assignment] + +# Only display warning when using -Wdefault, -We, -X dev or similar. +warnings.filterwarnings("ignore", category=NotAppKeyWarning, append=True) + +HostSequence = TypingIterable[str] + + +async def _run_app( + app: Application | Awaitable[Application], + *, + host: str | HostSequence | None = None, + port: int | None = None, + path: PathLike | TypingIterable[PathLike] | None = None, + sock: socket.socket | TypingIterable[socket.socket] | None = None, + ssl_context: SSLContext | None = None, + print: Callable[..., None] | None = print, + backlog: int = 128, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + **kwargs: Any, # TODO(PY311): Use Unpack +) -> None: + # An internal function to actually do all dirty job for application running + if asyncio.iscoroutine(app): + app = await app + + app = cast(Application, app) + + runner = AppRunner(app, **kwargs) + + await runner.setup() + + sites: list[BaseSite] = [] + + try: + if host is not None: + if isinstance(host, str): + sites.append( + TCPSite( + runner, + host, + port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + else: + for h in host: + sites.append( + TCPSite( + runner, + h, + port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + elif path is None and sock is None or port is not None: + sites.append( + TCPSite( + runner, + port=port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + + if path is not None: + if isinstance(path, (str, os.PathLike)): + sites.append( + UnixSite( + runner, + path, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + else: + for p in path: + sites.append( + UnixSite( + runner, + p, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + + if sock is not None: + if not isinstance(sock, Iterable): + sites.append( + SockSite( + runner, + sock, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + else: + for s in sock: + sites.append( + SockSite( + runner, + s, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + for site in sites: + await site.start() + + if print: # pragma: no branch + names = sorted(str(s.name) for s in runner.sites) + print( + "======== Running on {} ========\n" + "(Press CTRL+C to quit)".format(", ".join(names)) + ) + + # sleep forever by 1 hour intervals, + while True: + await asyncio.sleep(3600) + finally: + await runner.cleanup() + + +def _cancel_tasks( + to_cancel: set["asyncio.Task[Any]"], loop: asyncio.AbstractEventLoop +) -> None: + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + + loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) + + for task in to_cancel: + if task.cancelled(): + continue + if task.exception() is not None: + loop.call_exception_handler( + { + "message": "unhandled exception during asyncio.run() shutdown", + "exception": task.exception(), + "task": task, + } + ) + + +def run_app( + app: Application | Awaitable[Application], + *, + host: str | HostSequence | None = None, + port: int | None = None, + path: PathLike | TypingIterable[PathLike] | None = None, + sock: socket.socket | TypingIterable[socket.socket] | None = None, + shutdown_timeout: float = 60.0, + keepalive_timeout: float = 75.0, + ssl_context: SSLContext | None = None, + print: Callable[..., None] | None = print, + backlog: int = 128, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + access_log_format: str = AccessLogger.LOG_FORMAT, + access_log: logging.Logger | None = access_logger, + handle_signals: bool = True, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + handler_cancellation: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Any, +) -> None: + """Run an app locally""" + if loop is None: + loop = asyncio.new_event_loop() + + # Configure if and only if in debugging mode and using the default logger + if loop.get_debug() and access_log and access_log.name == "aiohttp.access": + if access_log.level == logging.NOTSET: + access_log.setLevel(logging.DEBUG) + if not access_log.hasHandlers(): + access_log.addHandler(logging.StreamHandler()) + + main_task = loop.create_task( + _run_app( + app, + host=host, + port=port, + path=path, + sock=sock, + shutdown_timeout=shutdown_timeout, + keepalive_timeout=keepalive_timeout, + ssl_context=ssl_context, + print=print, + backlog=backlog, + access_log_class=access_log_class, + access_log_format=access_log_format, + access_log=access_log, + handle_signals=handle_signals, + reuse_address=reuse_address, + reuse_port=reuse_port, + handler_cancellation=handler_cancellation, + **kwargs, + ) + ) + + try: + asyncio.set_event_loop(loop) + loop.run_until_complete(main_task) + except (GracefulExit, KeyboardInterrupt): # pragma: no cover + pass + finally: + try: + # Skip when ``main_task`` is already done (e.g. raised during startup). + # Re-running ``loop.run_until_complete`` on a finished task calls + # ``Future.result`` again, which does + # ``raise self._exception.with_traceback(self._exception_tb)`` and + # resets ``exc.__traceback__`` to the originally saved tb β€” by then + # shallow β€” clobbering the deep traceback the caller would otherwise + # see (frames from ``cleanup_ctx`` / ``on_startup`` and the user code + # that actually raised). + if not main_task.done(): + main_task.cancel() + with suppress(asyncio.CancelledError): + loop.run_until_complete(main_task) + finally: + _cancel_tasks(asyncio.all_tasks(loop), loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + +def main(argv: list[str]) -> None: + arg_parser = ArgumentParser( + description="aiohttp.web Application server", prog="aiohttp.web" + ) + arg_parser.add_argument( + "entry_func", + help=( + "Callable returning the `aiohttp.web.Application` instance to " + "run. Should be specified in the 'module:function' syntax." + ), + metavar="entry-func", + ) + arg_parser.add_argument( + "-H", + "--hostname", + help="TCP/IP hostname to serve on (default: localhost)", + default=None, + ) + arg_parser.add_argument( + "-P", + "--port", + help="TCP/IP port to serve on (default: %(default)r)", + type=int, + default=8080, + ) + arg_parser.add_argument( + "-U", + "--path", + help="Unix file system path to serve on. Can be combined with hostname " + "to serve on both Unix and TCP.", + ) + args, extra_argv = arg_parser.parse_known_args(argv) + + # Import logic + mod_str, _, func_str = args.entry_func.partition(":") + if not func_str or not mod_str: + arg_parser.error("'entry-func' not in 'module:function' syntax") + if mod_str.startswith("."): + arg_parser.error("relative module names not supported") + try: + module = import_module(mod_str) + except ImportError as ex: + arg_parser.error(f"unable to import {mod_str}: {ex}") + try: + func = getattr(module, func_str) + except AttributeError: + arg_parser.error(f"module {mod_str!r} has no attribute {func_str!r}") + + # Compatibility logic + if args.path is not None and not hasattr(socket, "AF_UNIX"): + arg_parser.error( + "file system paths not supported by your operating environment" + ) + + logging.basicConfig(level=logging.DEBUG) + + if args.path and args.hostname is None: + host = port = None + else: + host = args.hostname or "localhost" + port = args.port + + app = func(extra_argv) + run_app(app, host=host, port=port, path=args.path) + arg_parser.exit(message="Stopped\n") + + +if __name__ == "__main__": # pragma: no branch + main(sys.argv[1:]) # pragma: no cover diff --git a/venv/Lib/site-packages/aiohttp/web_app.py b/venv/Lib/site-packages/aiohttp/web_app.py new file mode 100644 index 0000000000000000000000000000000000000000..e859b865805f21e98d72e8ac0ea4af72f74cdfd5 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_app.py @@ -0,0 +1,610 @@ +import asyncio +import logging +import warnings +from collections.abc import ( + AsyncIterator, + Awaitable, + Callable, + Iterable, + Iterator, + Mapping, + MutableMapping, + Sequence, +) +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from functools import lru_cache, partial, update_wrapper +from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast, overload + +from aiosignal import Signal +from frozenlist import FrozenList + +from . import hdrs +from .abc import ( + AbstractAccessLogger, + AbstractMatchInfo, + AbstractRouter, + AbstractStreamWriter, +) +from .helpers import DEBUG, AppKey +from .http_parser import RawRequestMessage +from .log import web_logger +from .streams import StreamReader +from .typedefs import Handler, Middleware +from .web_exceptions import NotAppKeyWarning +from .web_log import AccessLogger +from .web_middlewares import _fix_request_current_app +from .web_protocol import RequestHandler +from .web_request import Request +from .web_response import StreamResponse +from .web_routedef import AbstractRouteDef +from .web_server import Server +from .web_urldispatcher import ( + AbstractResource, + AbstractRoute, + Domain, + MaskDomain, + MatchedSubAppResource, + PrefixedSubAppResource, + SystemRoute, + UrlDispatcher, +) + +__all__ = ("Application", "CleanupError") + + +if TYPE_CHECKING: + _AppSignal = Signal["Application"] + _RespPrepareSignal = Signal[Request, StreamResponse] + _Middlewares = FrozenList[Middleware] + _MiddlewaresHandlers = Optional[Sequence[tuple[Middleware, bool]]] + _Subapps = list["Application"] +else: + # No type checker mode, skip types + _AppSignal = Signal + _RespPrepareSignal = Signal + _Middlewares = FrozenList + _MiddlewaresHandlers = Optional[Sequence] + _Subapps = list + +_T = TypeVar("_T") +_U = TypeVar("_U") +_Resource = TypeVar("_Resource", bound=AbstractResource) + + +def _build_middlewares( + handler: Handler, apps: tuple["Application", ...] +) -> Callable[[Request], Awaitable[StreamResponse]]: + """Apply middlewares to handler.""" + for app in apps[::-1]: + for m, _ in app._middlewares_handlers: # type: ignore[union-attr] + handler = update_wrapper(partial(m, handler=handler), handler) + return handler + + +_cached_build_middleware = lru_cache(maxsize=1024)(_build_middlewares) + + +class Application(MutableMapping[str | AppKey[Any], Any]): + ATTRS = frozenset( + [ + "logger", + "_debug", + "_router", + "_loop", + "_handler_args", + "_middlewares", + "_middlewares_handlers", + "_has_legacy_middlewares", + "_run_middlewares", + "_state", + "_frozen", + "_pre_frozen", + "_subapps", + "_on_response_prepare", + "_on_startup", + "_on_shutdown", + "_on_cleanup", + "_client_max_size", + "_cleanup_ctx", + ] + ) + + def __init__( + self, + *, + logger: logging.Logger = web_logger, + router: UrlDispatcher | None = None, + middlewares: Iterable[Middleware] = (), + handler_args: Mapping[str, Any] | None = None, + client_max_size: int = 1024**2, + loop: asyncio.AbstractEventLoop | None = None, + debug: Any = ..., # mypy doesn't support ellipsis + ) -> None: + if router is None: + router = UrlDispatcher() + else: + warnings.warn( + "router argument is deprecated", DeprecationWarning, stacklevel=2 + ) + assert isinstance(router, AbstractRouter), router + + if loop is not None: + warnings.warn( + "loop argument is deprecated", DeprecationWarning, stacklevel=2 + ) + + if debug is not ...: + warnings.warn( + "debug argument is deprecated", DeprecationWarning, stacklevel=2 + ) + self._debug = debug + self._router: UrlDispatcher = router + self._loop = loop + self._handler_args = handler_args + self.logger = logger + + self._middlewares: _Middlewares = FrozenList(middlewares) + + # initialized on freezing + self._middlewares_handlers: _MiddlewaresHandlers = None + # initialized on freezing + self._run_middlewares: bool | None = None + self._has_legacy_middlewares: bool = True + + self._state: dict[AppKey[Any] | str, object] = {} + self._frozen = False + self._pre_frozen = False + self._subapps: _Subapps = [] + + self._on_response_prepare: _RespPrepareSignal = Signal(self) + self._on_startup: _AppSignal = Signal(self) + self._on_shutdown: _AppSignal = Signal(self) + self._on_cleanup: _AppSignal = Signal(self) + self._cleanup_ctx = CleanupContext() + self._on_startup.append(self._cleanup_ctx._on_startup) + self._on_cleanup.append(self._cleanup_ctx._on_cleanup) + self._client_max_size = client_max_size + + def __init_subclass__(cls: type["Application"]) -> None: + warnings.warn( + f"Inheritance class {cls.__name__} from web.Application is discouraged", + DeprecationWarning, + stacklevel=3, + ) + + if DEBUG: # pragma: no cover + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + f"Setting custom web.Application.{name} attribute " + "is discouraged", + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + # MutableMapping API + + def __eq__(self, other: object) -> bool: + return self is other + + @overload # type: ignore[override] + def __getitem__(self, key: AppKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: str | AppKey[_T]) -> Any: + return self._state[key] + + def _check_frozen(self) -> None: + if self._frozen: + warnings.warn( + "Changing state of started or joined application is deprecated", + DeprecationWarning, + stacklevel=3, + ) + + @overload # type: ignore[override] + def __setitem__(self, key: AppKey[_T], value: _T) -> None: ... + + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + + def __setitem__(self, key: str | AppKey[_T], value: Any) -> None: + self._check_frozen() + if not isinstance(key, AppKey): + warnings.warn( + "It is recommended to use web.AppKey instances for keys.\n" + + "https://docs.aiohttp.org/en/stable/web_advanced.html" + + "#application-s-config", + category=NotAppKeyWarning, + stacklevel=2, + ) + self._state[key] = value + + def __delitem__(self, key: str | AppKey[_T]) -> None: + self._check_frozen() + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[str | AppKey[Any]]: + return iter(self._state) + + def __hash__(self) -> int: + return id(self) + + @overload # type: ignore[override] + def get(self, key: AppKey[_T], default: None = ...) -> _T | None: ... + + @overload + def get(self, key: AppKey[_T], default: _U) -> _T | _U: ... + + @overload + def get(self, key: str, default: Any = ...) -> Any: ... + + def get(self, key: str | AppKey[_T], default: Any = None) -> Any: + return self._state.get(key, default) + + ######## + @property + def loop(self) -> asyncio.AbstractEventLoop: + # Technically the loop can be None + # but we mask it by explicit type cast + # to provide more convenient type annotation + warnings.warn("loop property is deprecated", DeprecationWarning, stacklevel=2) + return cast(asyncio.AbstractEventLoop, self._loop) + + def _set_loop(self, loop: asyncio.AbstractEventLoop | None) -> None: + if loop is None: + loop = asyncio.get_event_loop() + if self._loop is not None and self._loop is not loop: + raise RuntimeError( + "web.Application instance initialized with different loop" + ) + + self._loop = loop + + # set loop debug + if self._debug is ...: + self._debug = loop.get_debug() + + # set loop to sub applications + for subapp in self._subapps: + subapp._set_loop(loop) + + @property + def pre_frozen(self) -> bool: + return self._pre_frozen + + def pre_freeze(self) -> None: + if self._pre_frozen: + return + + self._pre_frozen = True + self._middlewares.freeze() + self._router.freeze() + self._on_response_prepare.freeze() + self._cleanup_ctx.freeze() + self._on_startup.freeze() + self._on_shutdown.freeze() + self._on_cleanup.freeze() + self._middlewares_handlers = tuple(self._prepare_middleware()) + self._has_legacy_middlewares = any( + not new_style for _, new_style in self._middlewares_handlers + ) + + # If current app and any subapp do not have middlewares avoid run all + # of the code footprint that it implies, which have a middleware + # hardcoded per app that sets up the current_app attribute. If no + # middlewares are configured the handler will receive the proper + # current_app without needing all of this code. + self._run_middlewares = True if self.middlewares else False + + for subapp in self._subapps: + subapp.pre_freeze() + self._run_middlewares = self._run_middlewares or subapp._run_middlewares + + @property + def frozen(self) -> bool: + return self._frozen + + def freeze(self) -> None: + if self._frozen: + return + + self.pre_freeze() + self._frozen = True + for subapp in self._subapps: + subapp.freeze() + + @property + def debug(self) -> bool: + warnings.warn("debug property is deprecated", DeprecationWarning, stacklevel=2) + return self._debug # type: ignore[no-any-return] + + def _reg_subapp_signals(self, subapp: "Application") -> None: + def reg_handler(signame: str) -> None: + subsig = getattr(subapp, signame) + + async def handler(app: "Application") -> None: + await subsig.send(subapp) + + appsig = getattr(self, signame) + appsig.append(handler) + + reg_handler("on_startup") + reg_handler("on_shutdown") + reg_handler("on_cleanup") + + def add_subapp(self, prefix: str, subapp: "Application") -> PrefixedSubAppResource: + if not isinstance(prefix, str): + raise TypeError("Prefix must be str") + prefix = prefix.rstrip("/") + if not prefix: + raise ValueError("Prefix cannot be empty") + factory = partial(PrefixedSubAppResource, prefix, subapp) + return self._add_subapp(factory, subapp) + + def _add_subapp( + self, resource_factory: Callable[[], _Resource], subapp: "Application" + ) -> _Resource: + if self.frozen: + raise RuntimeError("Cannot add sub application to frozen application") + if subapp.frozen: + raise RuntimeError("Cannot add frozen application") + resource = resource_factory() + self.router.register_resource(resource) + self._reg_subapp_signals(subapp) + self._subapps.append(subapp) + subapp.pre_freeze() + if self._loop is not None: + subapp._set_loop(self._loop) + return resource + + def add_domain(self, domain: str, subapp: "Application") -> MatchedSubAppResource: + if not isinstance(domain, str): + raise TypeError("Domain must be str") + elif "*" in domain: + rule: Domain = MaskDomain(domain) + else: + rule = Domain(domain) + factory = partial(MatchedSubAppResource, rule, subapp) + return self._add_subapp(factory, subapp) + + def add_routes(self, routes: Iterable[AbstractRouteDef]) -> list[AbstractRoute]: + return self.router.add_routes(routes) + + @property + def on_response_prepare(self) -> _RespPrepareSignal: + return self._on_response_prepare + + @property + def on_startup(self) -> _AppSignal: + return self._on_startup + + @property + def on_shutdown(self) -> _AppSignal: + return self._on_shutdown + + @property + def on_cleanup(self) -> _AppSignal: + return self._on_cleanup + + @property + def cleanup_ctx(self) -> "CleanupContext": + return self._cleanup_ctx + + @property + def router(self) -> UrlDispatcher: + return self._router + + @property + def middlewares(self) -> _Middlewares: + return self._middlewares + + def _make_handler( + self, + *, + loop: asyncio.AbstractEventLoop | None = None, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> Server: + + if not issubclass(access_log_class, AbstractAccessLogger): + raise TypeError( + "access_log_class must be subclass of " + f"aiohttp.abc.AbstractAccessLogger, got {access_log_class}" + ) + + self._set_loop(loop) + self.freeze() + + kwargs["debug"] = self._debug + kwargs["access_log_class"] = access_log_class + if self._handler_args: + for k, v in self._handler_args.items(): + kwargs[k] = v + + return Server( + self._handle, # type: ignore[arg-type] + request_factory=self._make_request, + loop=self._loop, + **kwargs, + ) + + def make_handler( + self, + *, + loop: asyncio.AbstractEventLoop | None = None, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> Server: + + warnings.warn( + "Application.make_handler(...) is deprecated, use AppRunner API instead", + DeprecationWarning, + stacklevel=2, + ) + + return self._make_handler( + loop=loop, access_log_class=access_log_class, **kwargs + ) + + async def startup(self) -> None: + """Causes on_startup signal + + Should be called in the event loop along with the request handler. + """ + await self.on_startup.send(self) + + async def shutdown(self) -> None: + """Causes on_shutdown signal + + Should be called before cleanup() + """ + await self.on_shutdown.send(self) + + async def cleanup(self) -> None: + """Causes on_cleanup signal + + Should be called after shutdown() + """ + if self.on_cleanup.frozen: + await self.on_cleanup.send(self) + else: + # If an exception occurs in startup, ensure cleanup contexts are completed. + await self._cleanup_ctx._on_cleanup(self) + + def _make_request( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: RequestHandler, + writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + _cls: type[Request] = Request, + ) -> Request: + if TYPE_CHECKING: + assert self._loop is not None + return _cls( + message, + payload, + protocol, + writer, + task, + self._loop, + client_max_size=self._client_max_size, + ) + + def _prepare_middleware(self) -> Iterator[tuple[Middleware, bool]]: + for m in reversed(self._middlewares): + if getattr(m, "__middleware_version__", None) == 1: + yield m, True + else: + warnings.warn( + f'old-style middleware "{m!r}" deprecated, see #2252', + DeprecationWarning, + stacklevel=2, + ) + yield m, False + + yield _fix_request_current_app(self), True + + async def _handle(self, request: Request) -> StreamResponse: + loop = asyncio.get_event_loop() + debug = loop.get_debug() + match_info = await self._router.resolve(request) + if debug: # pragma: no cover + if not isinstance(match_info, AbstractMatchInfo): + raise TypeError( + "match_info should be AbstractMatchInfo " + f"instance, not {match_info!r}" + ) + match_info.add_app(self) + + match_info.freeze() + + request._match_info = match_info + + if request.headers.get(hdrs.EXPECT): + resp = await match_info.expect_handler(request) + await request.writer.drain() + if resp is not None: + return resp + + handler = match_info.handler + + if self._run_middlewares: + # If its a SystemRoute, don't cache building the middlewares since + # they are constructed for every MatchInfoError as a new handler + # is made each time. + if not self._has_legacy_middlewares and not isinstance( + match_info.route, SystemRoute + ): + handler = _cached_build_middleware(handler, match_info.apps) + else: + for app in match_info.apps[::-1]: + for m, new_style in app._middlewares_handlers: # type: ignore[union-attr] + if new_style: + handler = update_wrapper( + partial(m, handler=handler), handler + ) + else: + handler = await m(app, handler) # type: ignore[arg-type,assignment] + + return await handler(request) + + def __call__(self) -> "Application": + """gunicorn compatibility""" + return self + + def __repr__(self) -> str: + return f"" + + def __bool__(self) -> bool: + return True + + +class CleanupError(RuntimeError): + @property + def exceptions(self) -> list[BaseException]: + return cast(list[BaseException], self.args[1]) + + +_CleanupContextCallable = ( + Callable[[Application], AbstractAsyncContextManager[None]] + | Callable[[Application], AsyncIterator[None]] +) + + +class CleanupContext(FrozenList[_CleanupContextCallable]): + def __init__(self) -> None: + super().__init__() + self._exits: list[AbstractAsyncContextManager[None]] = [] + + async def _on_startup(self, app: Application) -> None: + for cb in self: + ctx = cb(app) + + if not isinstance(ctx, AbstractAsyncContextManager): + ctx = asynccontextmanager(cb)(app) # type: ignore[arg-type] + + await ctx.__aenter__() + self._exits.append(ctx) + + async def _on_cleanup(self, app: Application) -> None: + errors = [] + for it in reversed(self._exits): + try: + await it.__aexit__(None, None, None) + except (Exception, asyncio.CancelledError) as exc: + errors.append(exc) + if errors: + if len(errors) == 1: + raise errors[0] + else: + raise CleanupError("Multiple errors on cleanup stage", errors) diff --git a/venv/Lib/site-packages/aiohttp/web_exceptions.py b/venv/Lib/site-packages/aiohttp/web_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..b763d658b924a97df5b37aacbe95113d096f31b7 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_exceptions.py @@ -0,0 +1,450 @@ +import warnings +from typing import Any, Dict, Iterable, List, Optional, Set # noqa + +from yarl import URL + +from .typedefs import LooseHeaders, StrOrURL +from .web_response import Response + +__all__ = ( + "HTTPException", + "HTTPError", + "HTTPRedirection", + "HTTPSuccessful", + "HTTPOk", + "HTTPCreated", + "HTTPAccepted", + "HTTPNonAuthoritativeInformation", + "HTTPNoContent", + "HTTPResetContent", + "HTTPPartialContent", + "HTTPMove", + "HTTPMultipleChoices", + "HTTPMovedPermanently", + "HTTPFound", + "HTTPSeeOther", + "HTTPNotModified", + "HTTPUseProxy", + "HTTPTemporaryRedirect", + "HTTPPermanentRedirect", + "HTTPClientError", + "HTTPBadRequest", + "HTTPUnauthorized", + "HTTPPaymentRequired", + "HTTPForbidden", + "HTTPNotFound", + "HTTPMethodNotAllowed", + "HTTPNotAcceptable", + "HTTPProxyAuthenticationRequired", + "HTTPRequestTimeout", + "HTTPConflict", + "HTTPGone", + "HTTPLengthRequired", + "HTTPPreconditionFailed", + "HTTPRequestEntityTooLarge", + "HTTPRequestURITooLong", + "HTTPUnsupportedMediaType", + "HTTPRequestRangeNotSatisfiable", + "HTTPExpectationFailed", + "HTTPMisdirectedRequest", + "HTTPUnprocessableEntity", + "HTTPFailedDependency", + "HTTPUpgradeRequired", + "HTTPPreconditionRequired", + "HTTPTooManyRequests", + "HTTPRequestHeaderFieldsTooLarge", + "HTTPUnavailableForLegalReasons", + "HTTPServerError", + "HTTPInternalServerError", + "HTTPNotImplemented", + "HTTPBadGateway", + "HTTPServiceUnavailable", + "HTTPGatewayTimeout", + "HTTPVersionNotSupported", + "HTTPVariantAlsoNegotiates", + "HTTPInsufficientStorage", + "HTTPNotExtended", + "HTTPNetworkAuthenticationRequired", +) + + +class NotAppKeyWarning(UserWarning): + """Warning when not using AppKey in Application.""" + + +############################################################ +# HTTP Exceptions +############################################################ + + +class HTTPException(Response, Exception): + + # You should set in subclasses: + # status = 200 + + status_code = -1 + empty_body = False + + __http_exception__ = True + + def __init__( + self, + *, + headers: LooseHeaders | None = None, + reason: str | None = None, + body: Any = None, + text: str | None = None, + content_type: str | None = None, + ) -> None: + if body is not None: + warnings.warn( + "body argument is deprecated for http web exceptions", + DeprecationWarning, + ) + if reason is not None and ("\r" in reason or "\n" in reason): + raise ValueError("Reason cannot contain \\r or \\n") + Response.__init__( + self, + status=self.status_code, + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + Exception.__init__(self, self.reason) + if self.body is None and not self.empty_body: + self.text = f"{self.status}: {self.reason}" + + def __bool__(self) -> bool: + return True + + +class HTTPError(HTTPException): + """Base class for exceptions with status codes in the 400s and 500s.""" + + +class HTTPRedirection(HTTPException): + """Base class for exceptions with status codes in the 300s.""" + + +class HTTPSuccessful(HTTPException): + """Base class for exceptions with status codes in the 200s.""" + + +class HTTPOk(HTTPSuccessful): + status_code = 200 + + +class HTTPCreated(HTTPSuccessful): + status_code = 201 + + +class HTTPAccepted(HTTPSuccessful): + status_code = 202 + + +class HTTPNonAuthoritativeInformation(HTTPSuccessful): + status_code = 203 + + +class HTTPNoContent(HTTPSuccessful): + status_code = 204 + empty_body = True + + +class HTTPResetContent(HTTPSuccessful): + status_code = 205 + empty_body = True + + +class HTTPPartialContent(HTTPSuccessful): + status_code = 206 + + +############################################################ +# 3xx redirection +############################################################ + + +class HTTPMove(HTTPRedirection): + def __init__( + self, + location: StrOrURL, + *, + headers: LooseHeaders | None = None, + reason: str | None = None, + body: Any = None, + text: str | None = None, + content_type: str | None = None, + ) -> None: + if not location: + raise ValueError("HTTP redirects need a location to redirect to.") + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self.headers["Location"] = str(URL(location)) + self.location = location + + +class HTTPMultipleChoices(HTTPMove): + status_code = 300 + + +class HTTPMovedPermanently(HTTPMove): + status_code = 301 + + +class HTTPFound(HTTPMove): + status_code = 302 + + +# This one is safe after a POST (the redirected location will be +# retrieved with GET): +class HTTPSeeOther(HTTPMove): + status_code = 303 + + +class HTTPNotModified(HTTPRedirection): + # FIXME: this should include a date or etag header + status_code = 304 + empty_body = True + + +class HTTPUseProxy(HTTPMove): + # Not a move, but looks a little like one + status_code = 305 + + +class HTTPTemporaryRedirect(HTTPMove): + status_code = 307 + + +class HTTPPermanentRedirect(HTTPMove): + status_code = 308 + + +############################################################ +# 4xx client error +############################################################ + + +class HTTPClientError(HTTPError): + pass + + +class HTTPBadRequest(HTTPClientError): + status_code = 400 + + +class HTTPUnauthorized(HTTPClientError): + status_code = 401 + + +class HTTPPaymentRequired(HTTPClientError): + status_code = 402 + + +class HTTPForbidden(HTTPClientError): + status_code = 403 + + +class HTTPNotFound(HTTPClientError): + status_code = 404 + + +class HTTPMethodNotAllowed(HTTPClientError): + status_code = 405 + + def __init__( + self, + method: str, + allowed_methods: Iterable[str], + *, + headers: LooseHeaders | None = None, + reason: str | None = None, + body: Any = None, + text: str | None = None, + content_type: str | None = None, + ) -> None: + allow = ",".join(sorted(allowed_methods)) + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self.headers["Allow"] = allow + self.allowed_methods: set[str] = set(allowed_methods) + self.method = method.upper() + + +class HTTPNotAcceptable(HTTPClientError): + status_code = 406 + + +class HTTPProxyAuthenticationRequired(HTTPClientError): + status_code = 407 + + +class HTTPRequestTimeout(HTTPClientError): + status_code = 408 + + +class HTTPConflict(HTTPClientError): + status_code = 409 + + +class HTTPGone(HTTPClientError): + status_code = 410 + + +class HTTPLengthRequired(HTTPClientError): + status_code = 411 + + +class HTTPPreconditionFailed(HTTPClientError): + status_code = 412 + + +class HTTPRequestEntityTooLarge(HTTPClientError): + status_code = 413 + + def __init__(self, max_size: float, actual_size: float = 0, **kwargs: Any) -> None: + kwargs.setdefault("text", f"Maximum request body size {max_size} exceeded.") + super().__init__(**kwargs) + + +class HTTPRequestURITooLong(HTTPClientError): + status_code = 414 + + +class HTTPUnsupportedMediaType(HTTPClientError): + status_code = 415 + + +class HTTPRequestRangeNotSatisfiable(HTTPClientError): + status_code = 416 + + +class HTTPExpectationFailed(HTTPClientError): + status_code = 417 + + +class HTTPMisdirectedRequest(HTTPClientError): + status_code = 421 + + +class HTTPUnprocessableEntity(HTTPClientError): + status_code = 422 + + +class HTTPFailedDependency(HTTPClientError): + status_code = 424 + + +class HTTPUpgradeRequired(HTTPClientError): + status_code = 426 + + +class HTTPPreconditionRequired(HTTPClientError): + status_code = 428 + + +class HTTPTooManyRequests(HTTPClientError): + status_code = 429 + + +class HTTPRequestHeaderFieldsTooLarge(HTTPClientError): + status_code = 431 + + +class HTTPUnavailableForLegalReasons(HTTPClientError): + status_code = 451 + + def __init__( + self, + link: StrOrURL | None, + *, + headers: LooseHeaders | None = None, + reason: str | None = None, + body: Any = None, + text: str | None = None, + content_type: str | None = None, + ) -> None: + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self._link = None + if link: + self._link = URL(link) + self.headers["Link"] = f'<{str(self._link)}>; rel="blocked-by"' + + @property + def link(self) -> URL | None: + return self._link + + +############################################################ +# 5xx Server Error +############################################################ +# Response status codes beginning with the digit "5" indicate cases in +# which the server is aware that it has erred or is incapable of +# performing the request. Except when responding to a HEAD request, the +# server SHOULD include an entity containing an explanation of the error +# situation, and whether it is a temporary or permanent condition. User +# agents SHOULD display any included entity to the user. These response +# codes are applicable to any request method. + + +class HTTPServerError(HTTPError): + pass + + +class HTTPInternalServerError(HTTPServerError): + status_code = 500 + + +class HTTPNotImplemented(HTTPServerError): + status_code = 501 + + +class HTTPBadGateway(HTTPServerError): + status_code = 502 + + +class HTTPServiceUnavailable(HTTPServerError): + status_code = 503 + + +class HTTPGatewayTimeout(HTTPServerError): + status_code = 504 + + +class HTTPVersionNotSupported(HTTPServerError): + status_code = 505 + + +class HTTPVariantAlsoNegotiates(HTTPServerError): + status_code = 506 + + +class HTTPInsufficientStorage(HTTPServerError): + status_code = 507 + + +class HTTPNotExtended(HTTPServerError): + status_code = 510 + + +class HTTPNetworkAuthenticationRequired(HTTPServerError): + status_code = 511 diff --git a/venv/Lib/site-packages/aiohttp/web_fileresponse.py b/venv/Lib/site-packages/aiohttp/web_fileresponse.py new file mode 100644 index 0000000000000000000000000000000000000000..aad5aaf397e5fb076d251bf3eab2cd00b2d4cc41 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_fileresponse.py @@ -0,0 +1,419 @@ +import asyncio +import io +import os +import pathlib +import sys +from contextlib import suppress +from enum import Enum, auto +from mimetypes import MimeTypes +from stat import S_ISREG +from types import MappingProxyType +from typing import ( # noqa + IO, + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Final, + Iterator, + List, + Optional, + Set, + Tuple, + Union, + cast, +) + +from . import hdrs +from .abc import AbstractStreamWriter +from .helpers import DEFAULT_CHUNK_SIZE, ETAG_ANY, ETag, must_be_empty_body +from .typedefs import LooseHeaders, PathLike +from .web_exceptions import ( + HTTPForbidden, + HTTPNotFound, + HTTPNotModified, + HTTPPartialContent, + HTTPPreconditionFailed, + HTTPRequestRangeNotSatisfiable, +) +from .web_response import StreamResponse + +__all__ = ("FileResponse",) + +if TYPE_CHECKING: + from .web_request import BaseRequest + + +_T_OnChunkSent = Optional[Callable[[bytes], Awaitable[None]]] + + +NOSENDFILE: Final[bool] = bool(os.environ.get("AIOHTTP_NOSENDFILE")) + +CONTENT_TYPES: Final[MimeTypes] = MimeTypes() + +# File extension to IANA encodings map that will be checked in the order defined. +ENCODING_EXTENSIONS = MappingProxyType( + {ext: CONTENT_TYPES.encodings_map[ext] for ext in (".br", ".gz")} +) + +FALLBACK_CONTENT_TYPE = "application/octet-stream" + +# Provide additional MIME type/extension pairs to be recognized. +# https://en.wikipedia.org/wiki/List_of_archive_formats#Compression_only +ADDITIONAL_CONTENT_TYPES = MappingProxyType( + { + "application/gzip": ".gz", + "application/x-brotli": ".br", + "application/x-bzip2": ".bz2", + "application/x-compress": ".Z", + "application/x-xz": ".xz", + } +) + + +class _FileResponseResult(Enum): + """The result of the file response.""" + + SEND_FILE = auto() # Ie a regular file to send + NOT_ACCEPTABLE = auto() # Ie a socket, or non-regular file + PRE_CONDITION_FAILED = auto() # Ie If-Match or If-None-Match failed + NOT_MODIFIED = auto() # 304 Not Modified + + +# Add custom pairs and clear the encodings map so guess_type ignores them. +CONTENT_TYPES.encodings_map.clear() +for content_type, extension in ADDITIONAL_CONTENT_TYPES.items(): + CONTENT_TYPES.add_type(content_type, extension) + + +_CLOSE_FUTURES: set[asyncio.Future[None]] = set() + + +class FileResponse(StreamResponse): + """A response object can be used to send files.""" + + def __init__( + self, + path: PathLike, + chunk_size: int = DEFAULT_CHUNK_SIZE, + status: int = 200, + reason: str | None = None, + headers: LooseHeaders | None = None, + ) -> None: + super().__init__(status=status, reason=reason, headers=headers) + + self._path = pathlib.Path(path) + self._chunk_size = chunk_size + + def _seek_and_read(self, fobj: IO[Any], offset: int, chunk_size: int) -> bytes: + fobj.seek(offset) + return fobj.read(chunk_size) # type: ignore[no-any-return] + + async def _sendfile_fallback( + self, writer: AbstractStreamWriter, fobj: IO[Any], offset: int, count: int + ) -> AbstractStreamWriter: + # To keep memory usage low,fobj is transferred in chunks + # controlled by the constructor's chunk_size argument. + + chunk_size = self._chunk_size + loop = asyncio.get_event_loop() + chunk = await loop.run_in_executor( + None, self._seek_and_read, fobj, offset, min(chunk_size, count) + ) + while chunk: + await writer.write(chunk) + count = count - len(chunk) + if count <= 0: + break + chunk = await loop.run_in_executor(None, fobj.read, min(chunk_size, count)) + + await writer.drain() + return writer + + async def _sendfile( + self, request: "BaseRequest", fobj: IO[Any], offset: int, count: int + ) -> AbstractStreamWriter: + writer = await super().prepare(request) + assert writer is not None + + if NOSENDFILE or self.compression: + return await self._sendfile_fallback(writer, fobj, offset, count) + + loop = request._loop + transport = request.transport + if transport is None: + raise ConnectionResetError("Connection lost") + + try: + await loop.sendfile(transport, fobj, offset, count) + except NotImplementedError: + return await self._sendfile_fallback(writer, fobj, offset, count) + + await super().write_eof() + return writer + + @staticmethod + def _etag_match(etag_value: str, etags: tuple[ETag, ...], *, weak: bool) -> bool: + if len(etags) == 1 and etags[0].value == ETAG_ANY: + return True + return any( + etag.value == etag_value for etag in etags if weak or not etag.is_weak + ) + + async def _not_modified( + self, request: "BaseRequest", etag_value: str, last_modified: float + ) -> AbstractStreamWriter | None: + self.set_status(HTTPNotModified.status_code) + self._length_check = False + self.etag = etag_value + self.last_modified = last_modified + # Delete any Content-Length headers provided by user. HTTP 304 + # should always have empty response body + return await super().prepare(request) + + async def _precondition_failed( + self, request: "BaseRequest" + ) -> AbstractStreamWriter | None: + self.set_status(HTTPPreconditionFailed.status_code) + self.content_length = 0 + return await super().prepare(request) + + def _make_response( + self, request: "BaseRequest", accept_encoding: str + ) -> tuple[ + _FileResponseResult, io.BufferedReader | None, os.stat_result, str | None + ]: + """Return the response result, io object, stat result, and encoding. + + If an uncompressed file is returned, the encoding is set to + :py:data:`None`. + + This method should be called from a thread executor + since it calls os.stat which may block. + """ + file_path, st, file_encoding = self._get_file_path_stat_encoding( + accept_encoding + ) + if not file_path: + return _FileResponseResult.NOT_ACCEPTABLE, None, st, None + + etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}" + + # https://www.rfc-editor.org/rfc/rfc9110#section-13.1.1-2 + if (ifmatch := request.if_match) is not None and not self._etag_match( + etag_value, ifmatch, weak=False + ): + return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding + + if ( + (unmodsince := request.if_unmodified_since) is not None + and ifmatch is None + and st.st_mtime > unmodsince.timestamp() + ): + return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding + + # https://www.rfc-editor.org/rfc/rfc9110#section-13.1.2-2 + if (ifnonematch := request.if_none_match) is not None and self._etag_match( + etag_value, ifnonematch, weak=True + ): + return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding + + if ( + (modsince := request.if_modified_since) is not None + and ifnonematch is None + and st.st_mtime <= modsince.timestamp() + ): + return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding + + fobj = file_path.open("rb") + with suppress(OSError): + # fstat() may not be available on all platforms + # Once we open the file, we want the fstat() to ensure + # the file has not changed between the first stat() + # and the open(). + st = os.stat(fobj.fileno()) + return _FileResponseResult.SEND_FILE, fobj, st, file_encoding + + def _get_file_path_stat_encoding( + self, accept_encoding: str + ) -> tuple[pathlib.Path | None, os.stat_result, str | None]: + file_path = self._path + for file_extension, file_encoding in ENCODING_EXTENSIONS.items(): + if file_encoding not in accept_encoding: + continue + + compressed_path = file_path.with_suffix(file_path.suffix + file_extension) + with suppress(OSError): + # Do not follow symlinks and ignore any non-regular files. + st = compressed_path.lstat() + if S_ISREG(st.st_mode): + return compressed_path, st, file_encoding + + # Fallback to the uncompressed file + st = file_path.stat() + return file_path if S_ISREG(st.st_mode) else None, st, None + + async def prepare(self, request: "BaseRequest") -> AbstractStreamWriter | None: + loop = asyncio.get_running_loop() + # Encoding comparisons should be case-insensitive + # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1 + accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() + try: + response_result, fobj, st, file_encoding = await loop.run_in_executor( + None, self._make_response, request, accept_encoding + ) + except PermissionError: + self.set_status(HTTPForbidden.status_code) + return await super().prepare(request) + except OSError: + # Most likely to be FileNotFoundError or OSError for circular + # symlinks in python >= 3.13, so respond with 404. + self.set_status(HTTPNotFound.status_code) + return await super().prepare(request) + + # Forbid special files like sockets, pipes, devices, etc. + if response_result is _FileResponseResult.NOT_ACCEPTABLE: + self.set_status(HTTPForbidden.status_code) + return await super().prepare(request) + + if response_result is _FileResponseResult.PRE_CONDITION_FAILED: + return await self._precondition_failed(request) + + if response_result is _FileResponseResult.NOT_MODIFIED: + etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}" + last_modified = st.st_mtime + return await self._not_modified(request, etag_value, last_modified) + + assert fobj is not None + try: + return await self._prepare_open_file(request, fobj, st, file_encoding) + finally: + # We do not await here because we do not want to wait + # for the executor to finish before returning the response + # so the connection can begin servicing another request + # as soon as possible. + close_future = loop.run_in_executor(None, fobj.close) + # Hold a strong reference to the future to prevent it from being + # garbage collected before it completes. + _CLOSE_FUTURES.add(close_future) + close_future.add_done_callback(_CLOSE_FUTURES.remove) + + async def _prepare_open_file( + self, + request: "BaseRequest", + fobj: io.BufferedReader, + st: os.stat_result, + file_encoding: str | None, + ) -> AbstractStreamWriter | None: + status = self._status + file_size: int = st.st_size + file_mtime: float = st.st_mtime + count: int = file_size + start: int | None = None + + if (ifrange := request.if_range) is None or file_mtime <= ifrange.timestamp(): + # If-Range header check: + # condition = cached date >= last modification date + # return 206 if True else 200. + # if False: + # Range header would not be processed, return 200 + # if True but Range header missing + # return 200 + try: + rng = request.http_range + start = rng.start + end: int | None = rng.stop + except ValueError: + # https://tools.ietf.org/html/rfc7233: + # A server generating a 416 (Range Not Satisfiable) response to + # a byte-range request SHOULD send a Content-Range header field + # with an unsatisfied-range value. + # The complete-length in a 416 response indicates the current + # length of the selected representation. + # + # Will do the same below. Many servers ignore this and do not + # send a Content-Range header with HTTP 416 + self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}" + self.set_status(HTTPRequestRangeNotSatisfiable.status_code) + return await super().prepare(request) + + # If a range request has been made, convert start, end slice + # notation into file pointer offset and count + if start is not None: + if start < 0 and end is None: # return tail of file + start += file_size + if start < 0: + # if Range:bytes=-1000 in request header but file size + # is only 200, there would be trouble without this + start = 0 + count = file_size - start + else: + # rfc7233:If the last-byte-pos value is + # absent, or if the value is greater than or equal to + # the current length of the representation data, + # the byte range is interpreted as the remainder + # of the representation (i.e., the server replaces the + # value of last-byte-pos with a value that is one less than + # the current length of the selected representation). + count = ( + min(end if end is not None else file_size, file_size) - start + ) + + if start >= file_size: + # HTTP 416 should be returned in this case. + # + # According to https://tools.ietf.org/html/rfc7233: + # If a valid byte-range-set includes at least one + # byte-range-spec with a first-byte-pos that is less than + # the current length of the representation, or at least one + # suffix-byte-range-spec with a non-zero suffix-length, + # then the byte-range-set is satisfiable. Otherwise, the + # byte-range-set is unsatisfiable. + self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}" + self.set_status(HTTPRequestRangeNotSatisfiable.status_code) + return await super().prepare(request) + + status = HTTPPartialContent.status_code + # Even though you are sending the whole file, you should still + # return a HTTP 206 for a Range request. + self.set_status(status) + + # If the Content-Type header is not already set, guess it based on the + # extension of the request path. The encoding returned by guess_type + # can be ignored since the map was cleared above. + if hdrs.CONTENT_TYPE not in self._headers: + if sys.version_info >= (3, 13): + guesser = CONTENT_TYPES.guess_file_type + else: + guesser = CONTENT_TYPES.guess_type + self.content_type = guesser(self._path)[0] or FALLBACK_CONTENT_TYPE + + if file_encoding: + self._headers[hdrs.CONTENT_ENCODING] = file_encoding + self._headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING + # Disable compression if we are already sending + # a compressed file since we don't want to double + # compress. + self._compression = False + + self.etag = f"{st.st_mtime_ns:x}-{st.st_size:x}" + self.last_modified = file_mtime + self.content_length = count + + self._headers[hdrs.ACCEPT_RANGES] = "bytes" + + if status == HTTPPartialContent.status_code: + real_start = start + assert real_start is not None + self._headers[hdrs.CONTENT_RANGE] = ( + f"bytes {real_start}-{real_start + count - 1}/{file_size}" + ) + + # If we are sending 0 bytes calling sendfile() will throw a ValueError + if count == 0 or must_be_empty_body(request.method, status): + return await super().prepare(request) + + # be aware that start could be None or int=0 here. + offset = start or 0 + + return await self._sendfile(request, fobj, offset, count) diff --git a/venv/Lib/site-packages/aiohttp/web_log.py b/venv/Lib/site-packages/aiohttp/web_log.py new file mode 100644 index 0000000000000000000000000000000000000000..e6158635091e5f5cc952463a3aca308682f0748c --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_log.py @@ -0,0 +1,231 @@ +import datetime +import functools +import logging +import os +import re +import time as time_mod +from collections.abc import Iterable +from typing import Callable, ClassVar, NamedTuple + +from .abc import AbstractAccessLogger +from .web_request import BaseRequest +from .web_response import StreamResponse + + +class KeyMethod(NamedTuple): + key: str | tuple[str, str] + method: Callable[[BaseRequest, StreamResponse, float], str] + + +class AccessLogger(AbstractAccessLogger): + """Helper object to log access. + + Usage: + log = logging.getLogger("spam") + log_format = "%a %{User-Agent}i" + access_logger = AccessLogger(log, log_format) + access_logger.log(request, response, time) + + Format: + %% The percent sign + %a Remote IP-address (IP-address of proxy if using reverse proxy) + %t Time when the request was started to process + %P The process ID of the child that serviced the request + %r First line of request + %s Response status code + %b Size of response in bytes, including HTTP headers + %T Time taken to serve the request, in seconds + %Tf Time taken to serve the request, in seconds with floating fraction + in .06f format + %D Time taken to serve the request, in microseconds + %{FOO}i request.headers['FOO'] + %{FOO}o response.headers['FOO'] + %{FOO}e os.environ['FOO'] + + """ + + LOG_FORMAT_MAP = { + "a": "remote_address", + "t": "request_start_time", + "P": "process_id", + "r": "first_request_line", + "s": "response_status", + "b": "response_size", + "T": "request_time", + "Tf": "request_time_frac", + "D": "request_time_micro", + "i": "request_header", + "o": "response_header", + } + + LOG_FORMAT = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"' + FORMAT_RE = re.compile(r"%(\{([A-Za-z0-9\-_]+)\}([ioe])|[atPrsbOD]|Tf?)") + CLEANUP_RE = re.compile(r"(%[^s])") + _FORMAT_CACHE: dict[str, tuple[str, list[KeyMethod]]] = {} + + _cached_tz: ClassVar[datetime.timezone | None] = None + _cached_tz_expires: ClassVar[float] = 0.0 + + def __init__(self, logger: logging.Logger, log_format: str = LOG_FORMAT) -> None: + """Initialise the logger. + + logger is a logger object to be used for logging. + log_format is a string with apache compatible log format description. + + """ + super().__init__(logger, log_format=log_format) + + _compiled_format = AccessLogger._FORMAT_CACHE.get(log_format) + if not _compiled_format: + _compiled_format = self.compile_format(log_format) + AccessLogger._FORMAT_CACHE[log_format] = _compiled_format + + self._log_format, self._methods = _compiled_format + + def compile_format(self, log_format: str) -> tuple[str, list[KeyMethod]]: + """Translate log_format into form usable by modulo formatting + + All known atoms will be replaced with %s + Also methods for formatting of those atoms will be added to + _methods in appropriate order + + For example we have log_format = "%a %t" + This format will be translated to "%s %s" + Also contents of _methods will be + [self._format_a, self._format_t] + These method will be called and results will be passed + to translated string format. + + Each _format_* method receive 'args' which is list of arguments + given to self.log + + Exceptions are _format_e, _format_i and _format_o methods which + also receive key name (by functools.partial) + + """ + # list of (key, method) tuples, we don't use an OrderedDict as users + # can repeat the same key more than once + methods = list() + + for atom in self.FORMAT_RE.findall(log_format): + if atom[1] == "": + format_key1 = self.LOG_FORMAT_MAP[atom[0]] + m = getattr(AccessLogger, "_format_%s" % atom[0]) + key_method = KeyMethod(format_key1, m) + else: + format_key2 = (self.LOG_FORMAT_MAP[atom[2]], atom[1]) + m = getattr(AccessLogger, "_format_%s" % atom[2]) + key_method = KeyMethod(format_key2, functools.partial(m, atom[1])) + + methods.append(key_method) + + log_format = self.FORMAT_RE.sub(r"%s", log_format) + log_format = self.CLEANUP_RE.sub(r"%\1", log_format) + return log_format, methods + + @staticmethod + def _format_i( + key: str, request: BaseRequest, response: StreamResponse, time: float + ) -> str: + if request is None: + return "(no headers)" + + # suboptimal, make istr(key) once + return request.headers.get(key, "-") + + @staticmethod + def _format_o( + key: str, request: BaseRequest, response: StreamResponse, time: float + ) -> str: + # suboptimal, make istr(key) once + return response.headers.get(key, "-") + + @staticmethod + def _format_a(request: BaseRequest, response: StreamResponse, time: float) -> str: + if request is None: + return "-" + ip = request.remote + return ip if ip is not None else "-" + + @classmethod + def _get_local_time(cls) -> datetime.datetime: + if cls._cached_tz is None or time_mod.time() >= cls._cached_tz_expires: + gmtoff = time_mod.localtime().tm_gmtoff + cls._cached_tz = tz = datetime.timezone(datetime.timedelta(seconds=gmtoff)) + + now = datetime.datetime.now(tz) + # Expire at every 30 mins, as any DST change should occur at 0/30 mins past. + d = now + datetime.timedelta(minutes=30) + d = d.replace(minute=30 if d.minute >= 30 else 0, second=0, microsecond=0) + cls._cached_tz_expires = d.timestamp() + return now + + return datetime.datetime.now(cls._cached_tz) + + @staticmethod + def _format_t(request: BaseRequest, response: StreamResponse, time: float) -> str: + now = AccessLogger._get_local_time() + start_time = now - datetime.timedelta(seconds=time) + return start_time.strftime("[%d/%b/%Y:%H:%M:%S %z]") + + @staticmethod + def _format_P(request: BaseRequest, response: StreamResponse, time: float) -> str: + return "<%s>" % os.getpid() + + @staticmethod + def _format_r(request: BaseRequest, response: StreamResponse, time: float) -> str: + if request is None: + return "-" + return f"{request.method} {request.path_qs} HTTP/{request.version.major}.{request.version.minor}" + + @staticmethod + def _format_s(request: BaseRequest, response: StreamResponse, time: float) -> int: + return response.status + + @staticmethod + def _format_b(request: BaseRequest, response: StreamResponse, time: float) -> int: + return response.body_length + + @staticmethod + def _format_T(request: BaseRequest, response: StreamResponse, time: float) -> str: + return str(round(time)) + + @staticmethod + def _format_Tf(request: BaseRequest, response: StreamResponse, time: float) -> str: + return "%06f" % time + + @staticmethod + def _format_D(request: BaseRequest, response: StreamResponse, time: float) -> str: + return str(round(time * 1000000)) + + def _format_line( + self, request: BaseRequest, response: StreamResponse, time: float + ) -> Iterable[tuple[str | tuple[str, str], str]]: + return [(key, method(request, response, time)) for key, method in self._methods] + + @property + def enabled(self) -> bool: + """Check if logger is enabled.""" + # Avoid formatting the log line if it will not be emitted. + return self.logger.isEnabledFor(logging.INFO) + + def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None: + try: + fmt_info = self._format_line(request, response, time) + + values = list() + extra: dict[str, str | dict[str, str]] = dict() + for key, value in fmt_info: + values.append(value) + + if isinstance(key, str): + extra[key] = value + else: + k1, k2 = key + dct: dict[str, str] = extra.get(k1, {}) # type: ignore[assignment] + dct[k2] = value + extra[k1] = dct + + self.logger.info(self._log_format % tuple(values), extra=extra) + except Exception: + self.logger.exception("Error in logging") diff --git a/venv/Lib/site-packages/aiohttp/web_middlewares.py b/venv/Lib/site-packages/aiohttp/web_middlewares.py new file mode 100644 index 0000000000000000000000000000000000000000..472c15345253a748c6cb06035441a5811736569b --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_middlewares.py @@ -0,0 +1,121 @@ +import re +from typing import TYPE_CHECKING, TypeVar + +from .typedefs import Handler, Middleware +from .web_exceptions import HTTPMove, HTTPPermanentRedirect +from .web_request import Request +from .web_response import StreamResponse +from .web_urldispatcher import SystemRoute + +__all__ = ( + "middleware", + "normalize_path_middleware", +) + +if TYPE_CHECKING: + from .web_app import Application + +_Func = TypeVar("_Func") + + +async def _check_request_resolves(request: Request, path: str) -> tuple[bool, Request]: + alt_request = request.clone(rel_url=path) + + match_info = await request.app.router.resolve(alt_request) + alt_request._match_info = match_info + + if match_info.http_exception is None: + return True, alt_request + + return False, request + + +def middleware(f: _Func) -> _Func: + f.__middleware_version__ = 1 # type: ignore[attr-defined] + return f + + +def normalize_path_middleware( + *, + append_slash: bool = True, + remove_slash: bool = False, + merge_slashes: bool = True, + redirect_class: type[HTTPMove] = HTTPPermanentRedirect, +) -> Middleware: + """Factory for producing a middleware that normalizes the path of a request. + + Normalizing means: + - Add or remove a trailing slash to the path. + - Double slashes are replaced by one. + + The middleware returns as soon as it finds a path that resolves + correctly. The order if both merge and append/remove are enabled is + 1) merge slashes + 2) append/remove slash + 3) both merge slashes and append/remove slash. + If the path resolves with at least one of those conditions, it will + redirect to the new path. + + Only one of `append_slash` and `remove_slash` can be enabled. If both + are `True` the factory will raise an assertion error + + If `append_slash` is `True` the middleware will append a slash when + needed. If a resource is defined with trailing slash and the request + comes without it, it will append it automatically. + + If `remove_slash` is `True`, `append_slash` must be `False`. When enabled + the middleware will remove trailing slashes and redirect if the resource + is defined + + If merge_slashes is True, merge multiple consecutive slashes in the + path into one. + """ + correct_configuration = not (append_slash and remove_slash) + assert correct_configuration, "Cannot both remove and append slash" + + @middleware + async def impl(request: Request, handler: Handler) -> StreamResponse: + if isinstance(request.match_info.route, SystemRoute): + paths_to_check = [] + if "?" in request.raw_path: + path, query = request.raw_path.split("?", 1) + query = "?" + query + else: + query = "" + path = request.raw_path + + if merge_slashes: + paths_to_check.append(re.sub("//+", "/", path)) + if append_slash and not request.path.endswith("/"): + paths_to_check.append(path + "/") + if remove_slash and request.path.endswith("/"): + paths_to_check.append(path[:-1]) + if merge_slashes and append_slash: + paths_to_check.append(re.sub("//+", "/", path + "/")) + if merge_slashes and remove_slash: + merged_slashes = re.sub("//+", "/", path) + paths_to_check.append(merged_slashes[:-1]) + + for path in paths_to_check: + path = re.sub("^//+", "/", path) # SECURITY: GHSA-v6wp-4m6f-gcjg + resolves, request = await _check_request_resolves(request, path) + if resolves: + raise redirect_class(request.raw_path + query) + + return await handler(request) + + return impl + + +def _fix_request_current_app(app: "Application") -> Middleware: + @middleware + async def impl(request: Request, handler: Handler) -> StreamResponse: + match_info = request.match_info + prev = match_info.current_app + match_info.current_app = app + try: + return await handler(request) + finally: + match_info.current_app = prev + + return impl diff --git a/venv/Lib/site-packages/aiohttp/web_protocol.py b/venv/Lib/site-packages/aiohttp/web_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..e2225cade10aedfc676617f30f0377efcf9a2081 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_protocol.py @@ -0,0 +1,874 @@ +import asyncio +import asyncio.streams +import sys +import traceback +import warnings +from collections import deque +from collections.abc import Awaitable, Callable, Sequence +from contextlib import suppress +from html import escape as html_escape +from http import HTTPStatus +from logging import Logger +from typing import TYPE_CHECKING, Any, Optional, cast + +import attr +import yarl +from propcache import under_cached_property + +from .abc import AbstractAccessLogger, AbstractStreamWriter +from .base_protocol import PAUSE_RESUME_READING_ERRORS, BaseProtocol +from .helpers import DEFAULT_CHUNK_SIZE, ceil_timeout +from .http import ( + HttpProcessingError, + HttpRequestParser, + HttpVersion10, + RawRequestMessage, + StreamWriter, + WebSocketReader, +) +from .http_exceptions import BadHttpMethod +from .log import access_logger, server_logger +from .streams import EMPTY_PAYLOAD, StreamReader +from .tcp_helpers import tcp_keepalive +from .web_exceptions import HTTPException, HTTPInternalServerError +from .web_log import AccessLogger +from .web_request import BaseRequest +from .web_response import Response, StreamResponse + +__all__ = ("RequestHandler", "RequestPayloadError", "PayloadAccessError") + +# Max parsed-but-unhandled pipelined requests buffered per connection before +# reading is paused. Bounds memory a client can pin by keeping one handler busy +# and pipelining behind it; reading resumes as the queue drains. +MAX_MSG_QUEUE_SIZE = 32 + +if TYPE_CHECKING: + import ssl + + from .web_server import Server + + +_RequestFactory = Callable[ + [ + RawRequestMessage, + StreamReader, + "RequestHandler", + AbstractStreamWriter, + "asyncio.Task[None]", + ], + BaseRequest, +] + +_RequestHandler = Callable[[BaseRequest], Awaitable[StreamResponse]] + +ERROR = RawRequestMessage( + "UNKNOWN", + "/", + HttpVersion10, + {}, # type: ignore[arg-type] + {}, # type: ignore[arg-type] + True, + None, + False, + False, + yarl.URL("/"), +) + + +class RequestPayloadError(Exception): + """Payload parsing error.""" + + +class PayloadAccessError(Exception): + """Payload was accessed after response was sent.""" + + +_PAYLOAD_ACCESS_ERROR = PayloadAccessError() + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _ErrInfo: + status: int + exc: BaseException + message: str + + +_MsgType = tuple[RawRequestMessage | _ErrInfo, StreamReader] + + +class RequestHandler(BaseProtocol): + """HTTP protocol implementation. + + RequestHandler handles incoming HTTP request. It reads request line, + request headers and request payload and calls handle_request() method. + By default it always returns with 404 response. + + RequestHandler handles errors in incoming request, like bad + status line, bad headers or incomplete payload. If any error occurs, + connection gets closed. + + keepalive_timeout -- number of seconds before closing + keep-alive connection + + tcp_keepalive -- TCP keep-alive is on, default is on + + debug -- enable debug mode + + logger -- custom logger object + + access_log_class -- custom class for access_logger + + access_log -- custom logging object + + access_log_format -- access log format string + + loop -- Optional event loop + + max_line_size -- Optional maximum header line size + + max_field_size -- Optional maximum header field size + + max_headers -- Optional maximum header size + + timeout_ceil_threshold -- Optional value to specify + threshold to ceil() timeout + values + + """ + + __slots__ = ( + "max_field_size", + "max_headers", + "max_line_size", + "_request_count", + "_keepalive", + "_manager", + "_request_handler", + "_request_factory", + "_tcp_keepalive", + "_next_keepalive_close_time", + "_keepalive_handle", + "_keepalive_timeout", + "_lingering_time", + "_messages", + "_max_msg_queue_size", + "_msg_queue_resume_size", + "_msg_queue_paused", + "_message_tail", + "_handler_waiter", + "_waiter", + "_task_handler", + "_payload_parser", + "_data_received_cb", + "logger", + "debug", + "access_log", + "access_logger", + "_close", + "_force_close", + "_current_request", + "_timeout_ceil_threshold", + "_request_in_progress", + "_logging_enabled", + "_cache", + ) + + def __init__( + self, + manager: "Server", + *, + loop: asyncio.AbstractEventLoop, + # Default should be high enough that it's likely longer than a reverse proxy. + keepalive_timeout: float = 3630, + tcp_keepalive: bool = True, + logger: Logger = server_logger, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + access_log: Logger = access_logger, + access_log_format: str = AccessLogger.LOG_FORMAT, + debug: bool = False, + max_line_size: int = 8190, + max_headers: int = 128, + max_field_size: int = 8190, + lingering_time: float = 10.0, + read_bufsize: int = DEFAULT_CHUNK_SIZE, + auto_decompress: bool = True, + timeout_ceil_threshold: float = 5, + ): + self._max_msg_queue_size = MAX_MSG_QUEUE_SIZE + # Low-water mark: resume reading once the queue drains to half the limit + # so we refill in batches instead of churning pause/resume per request. + self._msg_queue_resume_size = MAX_MSG_QUEUE_SIZE // 2 + # Set before super().__init__ so _reading_paused_for_msg_queue() is safe + # if BaseProtocol ever triggers a resume during init. + self._msg_queue_paused = False + parser = HttpRequestParser( + self, + loop, + read_bufsize, + max_line_size=max_line_size, + max_field_size=max_field_size, + max_headers=max_headers, + payload_exception=RequestPayloadError, + auto_decompress=auto_decompress, + max_msg_queue_size=MAX_MSG_QUEUE_SIZE, + ) + super().__init__(loop, parser) + + # _request_count is the number of requests processed with the same connection. + self._request_count = 0 + self._keepalive = False + self._current_request: BaseRequest | None = None + self._manager: Server | None = manager + self._request_handler: _RequestHandler | None = manager.request_handler + self._request_factory: _RequestFactory | None = manager.request_factory + + self.max_line_size = max_line_size + self.max_headers = max_headers + self.max_field_size = max_field_size + + self._tcp_keepalive = tcp_keepalive + # placeholder to be replaced on keepalive timeout setup + self._next_keepalive_close_time = 0.0 + self._keepalive_handle: asyncio.Handle | None = None + self._keepalive_timeout = keepalive_timeout + self._lingering_time = float(lingering_time) + + self._messages: deque[_MsgType] = deque() + self._message_tail = b"" + self._data_received_cb: Callable[[], None] | None = None + + self._waiter: asyncio.Future[None] | None = None + self._handler_waiter: asyncio.Future[None] | None = None + self._task_handler: asyncio.Task[None] | None = None + self._payload_parser: Any = None + + self._timeout_ceil_threshold: float = 5 + try: + self._timeout_ceil_threshold = float(timeout_ceil_threshold) + except (TypeError, ValueError): + pass + + self.logger = logger + self.debug = debug + self.access_log = access_log + if access_log: + self.access_logger: AbstractAccessLogger | None = access_log_class( + access_log, access_log_format + ) + self._logging_enabled = self.access_logger.enabled + else: + self.access_logger = None + self._logging_enabled = False + + self._close = False + self._force_close = False + self._request_in_progress = False + self._cache: dict[str, Any] = {} + + def __repr__(self) -> str: + return "<{} {}>".format( + self.__class__.__name__, + "connected" if self.transport is not None else "disconnected", + ) + + @under_cached_property + def ssl_context(self) -> Optional["ssl.SSLContext"]: + """Return SSLContext if available.""" + return ( + None + if self.transport is None + else self.transport.get_extra_info("sslcontext") + ) + + @under_cached_property + def peername( + self, + ) -> str | tuple[str, int, int, int] | tuple[str, int] | None: + """Return peername if available.""" + return ( + None + if self.transport is None + else self.transport.get_extra_info("peername") + ) + + @under_cached_property + def sockname( + self, + ) -> str | tuple[str, int, int, int] | tuple[str, int] | None: + """Return sockname if available.""" + return ( + None + if self.transport is None + else self.transport.get_extra_info("sockname") + ) + + @property + def keepalive_timeout(self) -> float: + return self._keepalive_timeout + + async def shutdown(self, timeout: float | None = 15.0) -> None: + """Do worker process exit preparations. + + We need to clean up everything and stop accepting requests. + It is especially important for keep-alive connections. + """ + self._force_close = True + + if self._keepalive_handle is not None: + self._keepalive_handle.cancel() + + # Wait for graceful handler completion + if self._request_in_progress: + # The future is only created when we are shutting + # down while the handler is still processing a request + # to avoid creating a future for every request. + self._handler_waiter = self._loop.create_future() + try: + async with ceil_timeout(timeout): + await self._handler_waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._handler_waiter = None + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + # Then cancel handler and wait + try: + async with ceil_timeout(timeout): + if self._current_request is not None: + self._current_request._cancel(asyncio.CancelledError()) + + if self._task_handler is not None and not self._task_handler.done(): + await asyncio.shield(self._task_handler) + except (asyncio.CancelledError, asyncio.TimeoutError): + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + + # force-close non-idle handler + if self._task_handler is not None: + self._task_handler.cancel() + + self.force_close() + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + super().connection_made(transport) + + real_transport = cast(asyncio.Transport, transport) + if self._tcp_keepalive: + tcp_keepalive(real_transport) + + assert self._manager is not None + self._manager.connection_made(self, real_transport) + + loop = self._loop + if sys.version_info >= (3, 12): + task = asyncio.Task(self.start(), loop=loop, eager_start=True) + else: + task = loop.create_task(self.start()) + self._task_handler = task + + def connection_lost(self, exc: BaseException | None) -> None: + if self._manager is None: + return + self._manager.connection_lost(self, exc) + + # Grab value before setting _manager to None. + handler_cancellation = self._manager.handler_cancellation + + self.force_close() + super().connection_lost(exc) + self._manager = None + self._request_factory = None + self._request_handler = None + self._parser = None + + if self._keepalive_handle is not None: + self._keepalive_handle.cancel() + + if self._current_request is not None: + if exc is None: + exc = ConnectionResetError("Connection lost") + self._current_request._cancel(exc) + + if handler_cancellation and self._task_handler is not None: + self._task_handler.cancel() + + self._task_handler = None + + if self._payload_parser is not None: + self._payload_parser.feed_eof() + self._payload_parser = None + + def set_parser( + self, + parser: WebSocketReader, + data_received_cb: Callable[[], None] | None = None, + ) -> None: + assert self._payload_parser is None + + self._payload_parser = parser + self._data_received_cb = data_received_cb + + if self._message_tail: + self._payload_parser.feed_data(self._message_tail) + self._message_tail = b"" + + def eof_received(self) -> None: + pass + + def data_received(self, data: bytes) -> None: + if self._force_close or self._close: + return + # parse http messages + messages: Sequence[_MsgType] + if self._payload_parser is None and not self._upgraded: + assert self._parser is not None + try: + messages, upgraded, tail = self._parser.feed_data(data) + except HttpProcessingError as exc: + messages = [ + (_ErrInfo(status=400, exc=exc, message=exc.message), EMPTY_PAYLOAD) + ] + upgraded = False + tail = b"" + + for msg, payload in messages: + self._request_count += 1 + self._messages.append((msg, payload)) + + waiter = self._waiter + if messages and waiter is not None and not waiter.done(): + # don't set result twice + waiter.set_result(None) + + # Queue full: pause the transport (the parser already stopped + # emitting). start() resumes as it drains the queue. + if ( + not self._msg_queue_paused + and len(self._messages) >= self._max_msg_queue_size + ): + self._pause_msg_queue_reading() + + self._upgraded = upgraded + if upgraded and tail: + self._message_tail = tail + + # no parser, just store + elif self._payload_parser is None and self._upgraded and data: + self._message_tail += data + + # feed payload + elif data: + if self._data_received_cb is not None: + self._data_received_cb() + eof, tail = self._payload_parser.feed_data(data) + if eof: + self.close() + + def _reading_paused_for_msg_queue(self) -> bool: + return self._msg_queue_paused + + def _pause_msg_queue_reading(self) -> None: + self._msg_queue_paused = True + if self.transport is not None: + try: + self.transport.pause_reading() + except PAUSE_RESUME_READING_ERRORS: + # Transport lacks flow control; nothing to pause. Intentionally + # ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress). + pass + + def _resume_msg_queue_reading(self) -> None: + if not self._upgraded: + # Reparse buffered pipelined requests while still marked paused so + # a refill past the limit does not re-pause an already-paused + # transport; only resume below once it stayed under the limit. + self.data_received(b"") + if len(self._messages) >= self._max_msg_queue_size: + return + self._msg_queue_paused = False + if not self._reading_paused and self.transport is not None: + try: + self.transport.resume_reading() + except PAUSE_RESUME_READING_ERRORS: + # Transport lacks flow control; nothing to resume. Intentionally + # ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress). + pass + + def keep_alive(self, val: bool) -> None: + """Set keep-alive connection mode. + + :param bool val: new state. + """ + self._keepalive = val + if self._keepalive_handle: + self._keepalive_handle.cancel() + self._keepalive_handle = None + + def close(self) -> None: + """Close connection. + + Stop accepting new pipelining messages and close + connection when handlers done processing messages. + """ + self._close = True + if self._waiter: + self._waiter.cancel() + + def force_close(self) -> None: + """Forcefully close connection.""" + self._force_close = True + if self._waiter: + self._waiter.cancel() + if self.transport is not None: + self.transport.close() + self.transport = None + + def log_access( + self, request: BaseRequest, response: StreamResponse, time: float | None + ) -> None: + if self._logging_enabled and self.access_logger is not None: + if TYPE_CHECKING: + assert time is not None + self.access_logger.log(request, response, self._loop.time() - time) + + def log_debug(self, *args: Any, **kw: Any) -> None: + if self.debug: + self.logger.debug(*args, **kw) + + def log_exception(self, *args: Any, **kw: Any) -> None: + self.logger.exception(*args, **kw) + + def _process_keepalive(self) -> None: + self._keepalive_handle = None + if self._force_close or not self._keepalive: + return + + loop = self._loop + now = loop.time() + close_time = self._next_keepalive_close_time + if now < close_time: + # Keep alive close check fired too early, reschedule + self._keepalive_handle = loop.call_at(close_time, self._process_keepalive) + return + + # handler in idle state + if self._waiter and not self._waiter.done(): + self.force_close() + + async def _handle_request( + self, + request: BaseRequest, + start_time: float | None, + request_handler: Callable[[BaseRequest], Awaitable[StreamResponse]], + ) -> tuple[StreamResponse, bool]: + self._request_in_progress = True + try: + try: + self._current_request = request + resp = await request_handler(request) + finally: + self._current_request = None + except HTTPException as exc: + resp = exc + resp, reset = await self.finish_response(request, resp, start_time) + except asyncio.CancelledError: + raise + except asyncio.TimeoutError as exc: + self.log_debug("Request handler timed out.", exc_info=exc) + resp = self.handle_error(request, 504) + resp, reset = await self.finish_response(request, resp, start_time) + except Exception as exc: + resp = self.handle_error(request, 500, exc) + resp, reset = await self.finish_response(request, resp, start_time) + else: + # Deprecation warning (See #2415) + if getattr(resp, "__http_exception__", False): + warnings.warn( + "returning HTTPException object is deprecated " + "(#2415) and will be removed, " + "please raise the exception instead", + DeprecationWarning, + ) + + resp, reset = await self.finish_response(request, resp, start_time) + finally: + self._request_in_progress = False + if self._handler_waiter is not None: + self._handler_waiter.set_result(None) + + return resp, reset + + async def start(self) -> None: + """Process incoming request. + + It reads request line, request headers and request payload, then + calls handle_request() method. Subclass has to override + handle_request(). start() handles various exceptions in request + or response handling. Connection is being closed always unless + keep_alive(True) specified. + """ + loop = self._loop + manager = self._manager + assert manager is not None + keepalive_timeout = self._keepalive_timeout + resp = None + assert self._request_factory is not None + assert self._request_handler is not None + + while not self._force_close: + if not self._messages: + try: + # wait for next request + self._waiter = loop.create_future() + await self._waiter + finally: + self._waiter = None + + message, payload = self._messages.popleft() + + # Free a parser slot; resume reading once drained to low water so + # pipelining keeps flowing while this request is handled. + # no branch: _parser is only None after connection_lost, whose path + # exits this loop, so the None case is not reachably exercisable. + if self._parser is not None: # pragma: no branch + self._parser.message_consumed() + if ( + self._msg_queue_paused + and len(self._messages) <= self._msg_queue_resume_size + ): + self._resume_msg_queue_reading() + + # time is only fetched if logging is enabled as otherwise + # its thrown away and never used. + start = loop.time() if self._logging_enabled else None + + manager.requests_count += 1 + writer = StreamWriter(self, loop) + if isinstance(message, _ErrInfo): + # make request_factory work + request_handler = self._make_error_handler(message) + message = ERROR + else: + request_handler = self._request_handler + + # Important don't hold a reference to the current task + # as on traceback it will prevent the task from being + # collected and will cause a memory leak. + request = self._request_factory( + message, + payload, + self, + writer, + self._task_handler or asyncio.current_task(loop), # type: ignore[arg-type] + ) + try: + # a new task is used for copy context vars (#3406) + coro = self._handle_request(request, start, request_handler) + if sys.version_info >= (3, 12): + task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + task = loop.create_task(coro) + try: + resp, reset = await task + except ConnectionError: + self.log_debug("Ignored premature client disconnection") + break + + # Drop the processed task from asyncio.Task.all_tasks() early + del task + if reset: + self.log_debug("Ignored premature client disconnection 2") + break + + # notify server about keep-alive + self._keepalive = bool(resp.keep_alive) + + # check payload + if not payload.is_eof(): + lingering_time = self._lingering_time + if not self._force_close and lingering_time: + self.log_debug( + "Start lingering close timer for %s sec.", lingering_time + ) + + now = loop.time() + end_t = now + lingering_time + + try: + while not payload.is_eof() and now < end_t: + async with ceil_timeout(end_t - now): + # read and ignore + await payload.readany() + now = loop.time() + except (asyncio.CancelledError, asyncio.TimeoutError): + if ( + sys.version_info >= (3, 11) + and (t := asyncio.current_task()) + and t.cancelling() + ): + raise + + # if payload still uncompleted + if not payload.is_eof() and not self._force_close: + self.log_debug("Uncompleted request.") + self.close() + + payload.set_exception(_PAYLOAD_ACCESS_ERROR) + + except asyncio.CancelledError: + self.log_debug("Ignored premature client disconnection") + self.force_close() + raise + except Exception as exc: + self.log_exception("Unhandled exception", exc_info=exc) + self.force_close() + except BaseException: + self.force_close() + raise + finally: + request._task = None # type: ignore[assignment] # Break reference cycle in case of exception + if self.transport is None and resp is not None: + self.log_debug("Ignored premature client disconnection.") + + if self._keepalive and not self._close and not self._force_close: + # start keep-alive timer + close_time = loop.time() + keepalive_timeout + self._next_keepalive_close_time = close_time + if self._keepalive_handle is None: + self._keepalive_handle = loop.call_at( + close_time, self._process_keepalive + ) + else: + break + + # remove handler, close transport if no handlers left + if not self._force_close: + self._task_handler = None + if self.transport is not None: + self.transport.close() + + async def finish_response( + self, request: BaseRequest, resp: StreamResponse, start_time: float | None + ) -> tuple[StreamResponse, bool]: + """Prepare the response and write_eof, then log access. + + This has to + be called within the context of any exception so the access logger + can get exception information. Returns True if the client disconnects + prematurely. + """ + request._finish() + if self._parser is not None: + self._parser.set_upgraded(False) + self._upgraded = False + if self._message_tail: + messages, _upgraded, tail = self._parser.feed_data(self._message_tail) + self._message_tail = tail + for msg, payload in messages: + self._request_count += 1 + self._messages.append((msg, payload)) + # This shouldn't be possible. If a future refactor results in this + # failing, then the code may need to be updated to set the waiter. + assert self._waiter is None + try: + prepare_meth = resp.prepare + except AttributeError: + if resp is None: + self.log_exception("Missing return statement on request handler") + else: + self.log_exception( + f"Web-handler should return a response instance, got {resp!r}" + ) + exc = HTTPInternalServerError() + resp = Response( + status=exc.status, reason=exc.reason, text=exc.text, headers=exc.headers + ) + prepare_meth = resp.prepare + try: + await prepare_meth(request) + await resp.write_eof() + except ConnectionError: + self.log_access(request, resp, start_time) + return resp, True + + self.log_access(request, resp, start_time) + return resp, False + + def handle_error( + self, + request: BaseRequest, + status: int = 500, + exc: BaseException | None = None, + message: str | None = None, + ) -> StreamResponse: + """Handle errors. + + Returns HTTP response with specific status code. Logs additional + information. It always closes current connection. + """ + if self._request_count == 1 and isinstance(exc, BadHttpMethod): + # BadHttpMethod is common when a client sends non-HTTP + # or encrypted traffic to an HTTP port. This is expected + # to happen when connected to the public internet so we log + # it at the debug level as to not fill logs with noise. + self.logger.debug( + "Error handling request from %s", request.remote, exc_info=exc + ) + else: + self.log_exception( + "Error handling request from %s", request.remote, exc_info=exc + ) + + # some data already got sent, connection is broken + if request.writer.output_size > 0: + raise ConnectionError( + "Response is sent already, cannot send another response " + "with the error message" + ) + + ct = "text/plain" + if status == HTTPStatus.INTERNAL_SERVER_ERROR: + title = f"{HTTPStatus.INTERNAL_SERVER_ERROR.value} {HTTPStatus.INTERNAL_SERVER_ERROR.phrase}" + msg = HTTPStatus.INTERNAL_SERVER_ERROR.description + tb = None + if self.debug: + with suppress(Exception): + tb = traceback.format_exc() + + if "text/html" in request.headers.get("Accept", ""): + if tb: + tb = html_escape(tb) + msg = f"

Traceback:

\n
{tb}
" + message = ( + "" + f"{title}" + f"\n

{title}

" + f"\n{msg}\n\n" + ) + ct = "text/html" + else: + if tb: + msg = tb + message = title + "\n\n" + msg + + resp = Response(status=status, text=message, content_type=ct) + resp.force_close() + + return resp + + def _make_error_handler( + self, err_info: _ErrInfo + ) -> Callable[[BaseRequest], Awaitable[StreamResponse]]: + async def handler(request: BaseRequest) -> StreamResponse: + return self.handle_error( + request, err_info.status, err_info.exc, err_info.message + ) + + return handler diff --git a/venv/Lib/site-packages/aiohttp/web_request.py b/venv/Lib/site-packages/aiohttp/web_request.py new file mode 100644 index 0000000000000000000000000000000000000000..74dbd4846cc7354ca638444965e95e276cb4caa0 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_request.py @@ -0,0 +1,948 @@ +import asyncio +import datetime +import io +import re +import string +import tempfile +import types +import warnings +from collections.abc import Iterator, Mapping, MutableMapping +from re import Pattern +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar, cast, overload +from urllib.parse import parse_qsl + +import attr +from multidict import ( + CIMultiDict, + CIMultiDictProxy, + MultiDict, + MultiDictProxy, + MultiMapping, +) +from yarl import URL + +from . import hdrs +from ._cookie_helpers import parse_cookie_header +from .abc import AbstractStreamWriter +from .helpers import ( + _SENTINEL, + DEBUG, + DEFAULT_CHUNK_SIZE, + ETAG_ANY, + LIST_QUOTED_ETAG_RE, + ChainMapProxy, + ETag, + HeadersMixin, + RequestKey, + parse_http_date, + reify, + sentinel, + set_exception, +) +from .http_parser import RawRequestMessage +from .http_writer import HttpVersion +from .multipart import BodyPartReader, MultipartReader +from .streams import EmptyStreamReader, StreamReader +from .typedefs import ( + DEFAULT_JSON_DECODER, + JSONDecoder, + LooseHeaders, + RawHeaders, + StrOrURL, +) +from .web_exceptions import HTTPRequestEntityTooLarge, NotAppKeyWarning +from .web_response import StreamResponse + +__all__ = ("BaseRequest", "FileField", "Request") + + +if TYPE_CHECKING: + from .web_app import Application + from .web_protocol import RequestHandler + from .web_urldispatcher import UrlMappingMatchInfo + + +_T = TypeVar("_T") + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class FileField: + name: str + filename: str + file: io.BufferedReader + content_type: str + headers: CIMultiDictProxy[str] + + +_Post = str | bytes | bytearray | FileField +_TCHAR: Final[str] = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-" +# '-' at the end to prevent interpretation as range in a char class + +_TOKEN: Final[str] = rf"[{_TCHAR}]+" + +_QDTEXT: Final[str] = r"[{}]".format( + r"".join(chr(c) for c in (0x09, 0x20, 0x21) + tuple(range(0x23, 0x7F))) +) +# qdtext includes 0x5C to escape 0x5D ('\]') +# qdtext excludes obs-text (because obsoleted, and encoding not specified) + +_QUOTED_PAIR: Final[str] = r"\\[\t !-~]" + +_QUOTED_STRING: Final[str] = rf'"(?:{_QUOTED_PAIR}|{_QDTEXT})*"' + +# This does not have a ReDOS/performance concern as long as it used with re.match(). +_FORWARDED_PAIR: Final[str] = rf"({_TOKEN})=({_TOKEN}|{_QUOTED_STRING})(:\d{{1,4}})?" + +_QUOTED_PAIR_REPLACE_RE: Final[Pattern[str]] = re.compile(r"\\([\t !-~])") +# same pattern as _QUOTED_PAIR but contains a capture group + +_FORWARDED_PAIR_RE: Final[Pattern[str]] = re.compile(_FORWARDED_PAIR) + +############################################################ +# HTTP Request +############################################################ + + +class BaseRequest(MutableMapping[str | RequestKey[Any], Any], HeadersMixin): + POST_METHODS = { + hdrs.METH_PATCH, + hdrs.METH_POST, + hdrs.METH_PUT, + hdrs.METH_TRACE, + hdrs.METH_DELETE, + } + + ATTRS = HeadersMixin.ATTRS | frozenset( + [ + "_message", + "_protocol", + "_payload_writer", + "_payload", + "_headers", + "_method", + "_version", + "_rel_url", + "_post", + "_read_bytes", + "_state", + "_cache", + "_task", + "_client_max_size", + "_loop", + "_transport_sslcontext", + "_transport_peername", + ] + ) + _post: MultiDictProxy[_Post] | None = None + _read_bytes: bytes | None = None + _seen_str_keys: set[str] = set() + + def __init__( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: "RequestHandler", + payload_writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + loop: asyncio.AbstractEventLoop, + *, + client_max_size: int = 1024**2, + state: dict[RequestKey[Any] | str, Any] | None = None, + scheme: str | None = None, + host: str | None = None, + remote: str | None = None, + ) -> None: + self._message = message + self._protocol = protocol + self._payload_writer = payload_writer + + self._payload = payload + self._headers: CIMultiDictProxy[str] = message.headers + self._method = message.method + self._version = message.version + self._cache: dict[str, Any] = {} + url = message.url + if url.absolute: + if scheme is not None: + url = url.with_scheme(scheme) + if host is not None: + url = url.with_host(host) + # absolute URL is given, + # override auto-calculating url, host, and scheme + # all other properties should be good + self._cache["url"] = url + self._cache["host"] = url.host + self._cache["scheme"] = url.scheme + self._rel_url = url.relative() + else: + self._rel_url = url + if scheme is not None: + self._cache["scheme"] = scheme + if host is not None: + self._cache["host"] = host + + self._state = {} if state is None else state + self._task = task + self._client_max_size = client_max_size + self._loop = loop + + self._transport_sslcontext = protocol.ssl_context + self._transport_peername = protocol.peername + self._transport_sockname = protocol.sockname + + if remote is not None: + self._cache["remote"] = remote + + def clone( + self, + *, + method: str | _SENTINEL = sentinel, + rel_url: StrOrURL | _SENTINEL = sentinel, + headers: LooseHeaders | _SENTINEL = sentinel, + scheme: str | _SENTINEL = sentinel, + host: str | _SENTINEL = sentinel, + remote: str | _SENTINEL = sentinel, + client_max_size: int | _SENTINEL = sentinel, + ) -> "BaseRequest": + """Clone itself with replacement some attributes. + + Creates and returns a new instance of Request object. If no parameters + are given, an exact copy is returned. If a parameter is not passed, it + will reuse the one from the current request object. + """ + if self._read_bytes: + raise RuntimeError("Cannot clone request after reading its content") + + dct: dict[str, Any] = {} + if method is not sentinel: + dct["method"] = method + if rel_url is not sentinel: + new_url: URL = URL(rel_url) + dct["url"] = new_url + dct["path"] = str(new_url) + if headers is not sentinel: + # a copy semantic + dct["headers"] = CIMultiDictProxy(CIMultiDict(headers)) + dct["raw_headers"] = tuple( + (k.encode("utf-8"), v.encode("utf-8")) + for k, v in dct["headers"].items() + ) + + message = self._message._replace(**dct) + + kwargs = {} + if scheme is not sentinel: + kwargs["scheme"] = scheme + if host is not sentinel: + kwargs["host"] = host + if remote is not sentinel: + kwargs["remote"] = remote + if client_max_size is sentinel: + client_max_size = self._client_max_size + + return self.__class__( + message, + self._payload, + self._protocol, + self._payload_writer, + self._task, + self._loop, + client_max_size=client_max_size, + state=self._state.copy(), + **kwargs, + ) + + @property + def task(self) -> "asyncio.Task[None]": + return self._task + + @property + def protocol(self) -> "RequestHandler": + return self._protocol + + @property + def transport(self) -> asyncio.Transport | None: + if self._protocol is None: + return None + return self._protocol.transport + + @property + def writer(self) -> AbstractStreamWriter: + return self._payload_writer + + @property + def client_max_size(self) -> int: + return self._client_max_size + + @reify + def message(self) -> RawRequestMessage: + warnings.warn("Request.message is deprecated", DeprecationWarning, stacklevel=3) + return self._message + + @reify + def rel_url(self) -> URL: + return self._rel_url + + @reify + def loop(self) -> asyncio.AbstractEventLoop: + warnings.warn( + "request.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + # MutableMapping API + + @overload # type: ignore[override] + def __getitem__(self, key: RequestKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: str | RequestKey[_T]) -> Any: + return self._state[key] + + @overload # type: ignore[override] + def __setitem__(self, key: RequestKey[_T], value: _T) -> None: ... + + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + + def __setitem__(self, key: str | RequestKey[_T], value: Any) -> None: + if not isinstance(key, RequestKey) and key not in BaseRequest._seen_str_keys: + BaseRequest._seen_str_keys.add(key) + warnings.warn( + "It is recommended to use web.RequestKey instances for keys.\n" + + "https://docs.aiohttp.org/en/stable/web_advanced.html" + + "#request-s-storage", + category=NotAppKeyWarning, + stacklevel=2, + ) + self._state[key] = value + + def __delitem__(self, key: str | RequestKey[_T]) -> None: + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[str | RequestKey[Any]]: + return iter(self._state) + + ######## + + @reify + def secure(self) -> bool: + """A bool indicating if the request is handled with SSL.""" + return self.scheme == "https" + + @reify + def forwarded(self) -> tuple[Mapping[str, str], ...]: + """A tuple containing all parsed Forwarded header(s). + + Makes an effort to parse Forwarded headers as specified by RFC 7239: + + - It adds one (immutable) dictionary per Forwarded 'field-value', ie + per proxy. The element corresponds to the data in the Forwarded + field-value added by the first proxy encountered by the client. Each + subsequent item corresponds to those added by later proxies. + - It checks that every value has valid syntax in general as specified + in section 4: either a 'token' or a 'quoted-string'. + - It un-escapes found escape sequences. + - It does NOT validate 'by' and 'for' contents as specified in section + 6. + - It does NOT validate 'host' contents (Host ABNF). + - It does NOT validate 'proto' contents for valid URI scheme names. + + Returns a tuple containing one or more immutable dicts + """ + elems = [] + for field_value in self._message.headers.getall(hdrs.FORWARDED, ()): + length = len(field_value) + pos = 0 + need_separator = False + elem: dict[str, str] = {} + elems.append(types.MappingProxyType(elem)) + while 0 <= pos < length: + match = _FORWARDED_PAIR_RE.match(field_value, pos) + if match is not None: # got a valid forwarded-pair + if need_separator: + # bad syntax here, skip to next comma + pos = field_value.find(",", pos) + else: + name, value, port = match.groups() + if value[0] == '"': + # quoted string: remove quotes and unescape + value = _QUOTED_PAIR_REPLACE_RE.sub(r"\1", value[1:-1]) + if port: + value += port + elem[name.lower()] = value + pos += len(match.group(0)) + need_separator = True + elif field_value[pos] == ",": # next forwarded-element + need_separator = False + elem = {} + elems.append(types.MappingProxyType(elem)) + pos += 1 + elif field_value[pos] == ";": # next forwarded-pair + need_separator = False + pos += 1 + elif field_value[pos] in " \t": + # Allow whitespace even between forwarded-pairs, though + # RFC 7239 doesn't. This simplifies code and is in line + # with Postel's law. + pos += 1 + else: + # bad syntax here, skip to next comma + pos = field_value.find(",", pos) + return tuple(elems) + + @reify + def scheme(self) -> str: + """A string representing the scheme of the request. + + Hostname is resolved in this order: + + - overridden value by .clone(scheme=new_scheme) call. + - type of connection to peer: HTTPS if socket is SSL, HTTP otherwise. + + 'http' or 'https'. + """ + if self._transport_sslcontext: + return "https" + else: + return "http" + + @reify + def method(self) -> str: + """Read only property for getting HTTP method. + + The value is upper-cased str like 'GET', 'POST', 'PUT' etc. + """ + return self._method + + @reify + def version(self) -> HttpVersion: + """Read only property for getting HTTP version of request. + + Returns aiohttp.protocol.HttpVersion instance. + """ + return self._version + + @reify + def host(self) -> str: + """Hostname of the request. + + Hostname is resolved in this order: + + - overridden value by .clone(host=new_host) call. + - HOST HTTP header + - local socket address the request arrived on + (transport ``sockname``) + - empty string if no transport information is available + + For example, 'example.com' or 'localhost:8080'. + + For historical reasons, the port number may be included. + """ + host = self._message.headers.get(hdrs.HOST) + if host is not None: + return host + sockname = self._transport_sockname + if sockname is None: + return "" + if isinstance(sockname, tuple): + # AF_INET6 returns a 4-tuple (host, port, flowinfo, scopeid); + # bracket the bare address so it matches the Host-header shape + # and is a valid URL authority component. + if len(sockname) == 4: + return f"[{sockname[0]}]" + return str(sockname[0]) + return str(sockname) + + @reify + def remote(self) -> str | None: + """Remote IP of client initiated HTTP request. + + The IP is resolved in this order: + + - overridden value by .clone(remote=new_remote) call. + - peername of opened socket + """ + if self._transport_peername is None: + return None + if isinstance(self._transport_peername, (list, tuple)): + return str(self._transport_peername[0]) + return str(self._transport_peername) + + @reify + def url(self) -> URL: + """The full URL of the request.""" + # authority is used here because it may include the port number + # and we want yarl to parse it correctly + return URL.build(scheme=self.scheme, authority=self.host).join(self._rel_url) + + @reify + def path(self) -> str: + """The URL including *PATH INFO* without the host or scheme. + + E.g., ``/app/blog`` + """ + return self._rel_url.path + + @reify + def path_qs(self) -> str: + """The URL including PATH_INFO and the query string. + + E.g, /app/blog?id=10 + """ + return str(self._rel_url) + + @reify + def raw_path(self) -> str: + """The URL including raw *PATH INFO* without the host or scheme. + + Warning, the path is unquoted and may contains non valid URL characters + + E.g., ``/my%2Fpath%7Cwith%21some%25strange%24characters`` + """ + return self._message.path + + @reify + def query(self) -> "MultiMapping[str]": + """A multidict with all the variables in the query string.""" + return self._rel_url.query + + @reify + def query_string(self) -> str: + """The query string in the URL. + + E.g., id=10 + """ + return self._rel_url.query_string + + @reify + def headers(self) -> CIMultiDictProxy[str]: + """A case-insensitive multidict proxy with all headers.""" + return self._headers + + @reify + def raw_headers(self) -> RawHeaders: + """A sequence of pairs for all headers.""" + return self._message.raw_headers + + @reify + def if_modified_since(self) -> datetime.datetime | None: + """The value of If-Modified-Since HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_MODIFIED_SINCE)) + + @reify + def if_unmodified_since(self) -> datetime.datetime | None: + """The value of If-Unmodified-Since HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_UNMODIFIED_SINCE)) + + @staticmethod + def _etag_values(etag_header: str) -> Iterator[ETag]: + """Extract `ETag` objects from raw header.""" + if etag_header == ETAG_ANY: + yield ETag( + is_weak=False, + value=ETAG_ANY, + ) + else: + for match in LIST_QUOTED_ETAG_RE.finditer(etag_header): + is_weak, value, garbage = match.group(2, 3, 4) + # Any symbol captured by 4th group means + # that the following sequence is invalid. + if garbage: + break + + yield ETag( + is_weak=bool(is_weak), + value=value, + ) + + @classmethod + def _if_match_or_none_impl( + cls, header_value: str | None + ) -> tuple[ETag, ...] | None: + if not header_value: + return None + + return tuple(cls._etag_values(header_value)) + + @reify + def if_match(self) -> tuple[ETag, ...] | None: + """The value of If-Match HTTP header, or None. + + This header is represented as a `tuple` of `ETag` objects. + """ + return self._if_match_or_none_impl(self.headers.get(hdrs.IF_MATCH)) + + @reify + def if_none_match(self) -> tuple[ETag, ...] | None: + """The value of If-None-Match HTTP header, or None. + + This header is represented as a `tuple` of `ETag` objects. + """ + return self._if_match_or_none_impl(self.headers.get(hdrs.IF_NONE_MATCH)) + + @reify + def if_range(self) -> datetime.datetime | None: + """The value of If-Range HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_RANGE)) + + @reify + def keep_alive(self) -> bool: + """Is keepalive enabled by client?""" + return not self._message.should_close + + @reify + def cookies(self) -> Mapping[str, str]: + """Return request cookies. + + A read-only dictionary-like object. + """ + # Use parse_cookie_header for RFC 6265 compliant Cookie header parsing + # that accepts special characters in cookie names (fixes #2683) + parsed = parse_cookie_header(self.headers.get(hdrs.COOKIE, "")) + # Extract values from Morsel objects + return MappingProxyType({name: morsel.value for name, morsel in parsed}) + + @reify + def http_range(self) -> slice: + """The content of Range HTTP header. + + Return a slice instance. + + """ + rng = self._headers.get(hdrs.RANGE) + start, end = None, None + if rng is not None: + try: + pattern = r"^bytes=(\d*)-(\d*)$" + start, end = re.findall(pattern, rng, re.ASCII)[0] + except IndexError: # pattern was not found in header + raise ValueError("range not in acceptable format") + + end = int(end) if end else None + start = int(start) if start else None + + if start is None and end is not None: + # end with no start is to return tail of content + start = -end + end = None + + if start is not None and end is not None: + # end is inclusive in range header, exclusive for slice + end += 1 + + if start >= end: + raise ValueError("start cannot be after end") + + if start is end is None: # No valid range supplied + raise ValueError("No start or end of range specified") + + return slice(start, end, 1) + + @reify + def content(self) -> StreamReader: + """Return raw payload stream.""" + return self._payload + + @property + def has_body(self) -> bool: + """Return True if request's HTTP BODY can be read, False otherwise.""" + warnings.warn( + "Deprecated, use .can_read_body #2005", DeprecationWarning, stacklevel=2 + ) + return not self._payload.at_eof() + + @property + def can_read_body(self) -> bool: + """Return True if request's HTTP BODY can be read, False otherwise.""" + return not self._payload.at_eof() + + @reify + def body_exists(self) -> bool: + """Return True if request has HTTP BODY, False otherwise.""" + return type(self._payload) is not EmptyStreamReader + + async def release(self) -> None: + """Release request. + + Eat unread part of HTTP BODY if present. + """ + while not self._payload.at_eof(): + await self._payload.readany() + + async def read(self) -> bytes: + """Read request body if present. + + Returns bytes object with full request content. + """ + if self._read_bytes is None: + # Raise the buffer limits so compressed payloads decompress in + # larger chunks instead of many small pause/resume cycles. + if self._client_max_size: + self._payload.set_read_chunk_size(self._client_max_size) + body = bytearray() + while True: + chunk = await self._payload.readany() + body.extend(chunk) + if self._client_max_size: + body_size = len(body) + if body_size > self._client_max_size: + raise HTTPRequestEntityTooLarge(self._client_max_size) + if not chunk: + break + self._read_bytes = bytes(body) + return self._read_bytes + + async def text(self) -> str: + """Return BODY as text using encoding from .charset.""" + bytes_body = await self.read() + encoding = self.charset or "utf-8" + return bytes_body.decode(encoding) + + async def json(self, *, loads: JSONDecoder = DEFAULT_JSON_DECODER) -> Any: + """Return BODY as JSON.""" + body = await self.text() + return loads(body) + + async def multipart(self) -> MultipartReader: + """Return async iterator to process BODY as multipart.""" + return MultipartReader( + self._headers, + self._payload, + client_max_size=self._client_max_size, + max_field_size=self._protocol.max_field_size, + max_headers=self._protocol.max_headers, + max_size_error_cls=HTTPRequestEntityTooLarge, + ) + + async def post(self) -> "MultiDictProxy[_Post]": + """Return POST parameters.""" + if self._post is not None: + return self._post + if self._method not in self.POST_METHODS: + self._post = MultiDictProxy(MultiDict()) + return self._post + + content_type = self.content_type + if content_type not in ( + "", + "application/x-www-form-urlencoded", + "multipart/form-data", + ): + self._post = MultiDictProxy(MultiDict()) + return self._post + + out: MultiDict[_Post] = MultiDict() + + if content_type == "multipart/form-data": + multipart = await self.multipart() + max_size = self._client_max_size + + size = 0 + while (field := await multipart.next()) is not None: + field_ct = field.headers.get(hdrs.CONTENT_TYPE) + + if isinstance(field, BodyPartReader): + if field.name is None: + raise ValueError("Multipart field missing name.") + + # Note that according to RFC 7578, the Content-Type header + # is optional, even for files, so we can't assume it's + # present. + # https://tools.ietf.org/html/rfc7578#section-4.4 + if field.filename: + # store file in temp file + tmp = await self._loop.run_in_executor( + None, tempfile.TemporaryFile + ) + while chunk := await field.read_chunk(size=DEFAULT_CHUNK_SIZE): + async for decoded_chunk in field.decode_iter(chunk): + await self._loop.run_in_executor( + None, tmp.write, decoded_chunk + ) + size += len(decoded_chunk) + if 0 < max_size < size: + await self._loop.run_in_executor(None, tmp.close) + raise HTTPRequestEntityTooLarge(max_size) + await self._loop.run_in_executor(None, tmp.seek, 0) + + if field_ct is None: + field_ct = "application/octet-stream" + + ff = FileField( + field.name, + field.filename, + cast(io.BufferedReader, tmp), + field_ct, + field.headers, + ) + out.add(field.name, ff) + else: + # deal with ordinary data + raw_data = bytearray() + while chunk := await field.read_chunk(): + size += len(chunk) + if 0 < max_size < size: + raise HTTPRequestEntityTooLarge(max_size) + raw_data.extend(chunk) + + value = bytearray() + # form-data doesn't support compression, so don't need to check size again. + async for d in field.decode_iter(raw_data): + value.extend(d) + + if field_ct is None or field_ct.startswith("text/"): + charset = field.get_charset(default="utf-8") + out.add(field.name, value.decode(charset)) + else: + out.add(field.name, value) + else: + raise ValueError( + "To decode nested multipart you need to use custom reader", + ) + else: + data = await self.read() + if data: + charset = self.charset or "utf-8" + out.extend( + parse_qsl( + data.rstrip().decode(charset), + keep_blank_values=True, + encoding=charset, + ) + ) + + self._post = MultiDictProxy(out) + return self._post + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """Extra info from protocol transport""" + protocol = self._protocol + if protocol is None: + return default + + transport = protocol.transport + if transport is None: + return default + + return transport.get_extra_info(name, default) + + def __repr__(self) -> str: + ascii_encodable_path = self.path.encode("ascii", "backslashreplace").decode( + "ascii" + ) + return f"<{self.__class__.__name__} {self._method} {ascii_encodable_path} >" + + def __eq__(self, other: object) -> bool: + return id(self) == id(other) + + def __bool__(self) -> bool: + return True + + async def _prepare_hook(self, response: StreamResponse) -> None: + return + + def _cancel(self, exc: BaseException) -> None: + set_exception(self._payload, exc) + + def _finish(self) -> None: + if self._post is None or self.content_type != "multipart/form-data": + return + + # NOTE: Release file descriptors for the + # NOTE: `tempfile.Temporaryfile`-created `_io.BufferedRandom` + # NOTE: instances of files sent within multipart request body + # NOTE: via HTTP POST request. + for file_name, file_field_object in self._post.items(): + if isinstance(file_field_object, FileField): + file_field_object.file.close() + + +class Request(BaseRequest): + + ATTRS = BaseRequest.ATTRS | frozenset(["_match_info"]) + + _match_info: Optional["UrlMappingMatchInfo"] = None + + if DEBUG: + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + f"Setting custom {self.__class__.__name__}.{name} attribute " + "is discouraged", + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + def clone( + self, + *, + method: str | _SENTINEL = sentinel, + rel_url: StrOrURL | _SENTINEL = sentinel, + headers: LooseHeaders | _SENTINEL = sentinel, + scheme: str | _SENTINEL = sentinel, + host: str | _SENTINEL = sentinel, + remote: str | _SENTINEL = sentinel, + client_max_size: int | _SENTINEL = sentinel, + ) -> "Request": + ret = super().clone( + method=method, + rel_url=rel_url, + headers=headers, + scheme=scheme, + host=host, + remote=remote, + client_max_size=client_max_size, + ) + new_ret = cast(Request, ret) + new_ret._match_info = self._match_info + return new_ret + + @reify + def match_info(self) -> "UrlMappingMatchInfo": + """Result of route resolving.""" + match_info = self._match_info + assert match_info is not None + return match_info + + @property + def app(self) -> "Application": + """Application instance.""" + match_info = self._match_info + assert match_info is not None + return match_info.current_app + + @property + def config_dict(self) -> ChainMapProxy: + match_info = self._match_info + assert match_info is not None + lst = match_info.apps + app = self.app + idx = lst.index(app) + sublist = list(reversed(lst[: idx + 1])) + return ChainMapProxy(sublist) + + async def _prepare_hook(self, response: StreamResponse) -> None: + match_info = self._match_info + if match_info is None: + return + for app in match_info._apps: + if on_response_prepare := app.on_response_prepare: + await on_response_prepare.send(self, response) diff --git a/venv/Lib/site-packages/aiohttp/web_response.py b/venv/Lib/site-packages/aiohttp/web_response.py new file mode 100644 index 0000000000000000000000000000000000000000..23f6bb016ed3a2e4a28dd3fa541324e118cb75eb --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_response.py @@ -0,0 +1,909 @@ +import asyncio +import collections.abc +import datetime +import enum +import json +import math +import time +import warnings +from collections.abc import Iterator, MutableMapping +from concurrent.futures import Executor +from http import HTTPStatus +from http.cookies import SimpleCookie +from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast, overload + +from multidict import CIMultiDict, istr + +from . import hdrs, payload +from .abc import AbstractStreamWriter +from .compression_utils import MAX_SYNC_CHUNK_SIZE, ZLibCompressor +from .helpers import ( + ETAG_ANY, + QUOTED_ETAG_RE, + ETag, + HeadersMixin, + ResponseKey, + must_be_empty_body, + parse_http_date, + rfc822_formatted_time, + sentinel, + should_remove_content_length, + validate_etag_value, +) +from .http import SERVER_SOFTWARE, HttpVersion10, HttpVersion11 +from .payload import Payload +from .typedefs import JSONBytesEncoder, JSONEncoder, LooseHeaders + +REASON_PHRASES = {http_status.value: http_status.phrase for http_status in HTTPStatus} + +__all__ = ( + "ContentCoding", + "StreamResponse", + "Response", + "json_response", + "json_bytes_response", +) + + +if TYPE_CHECKING: + from .web_request import BaseRequest + + BaseClass = MutableMapping[str, Any] +else: + BaseClass = collections.abc.MutableMapping + + +_T = TypeVar("_T") + + +# TODO(py311): Convert to StrEnum for wider use +class ContentCoding(enum.Enum): + # The content codings that we have support for. + # + # Additional registered codings are listed at: + # https://www.iana.org/assignments/http-parameters/http-parameters.xhtml#content-coding + deflate = "deflate" + gzip = "gzip" + identity = "identity" + + +CONTENT_CODINGS = {coding.value: coding for coding in ContentCoding} + +############################################################ +# HTTP Response classes +############################################################ + + +class StreamResponse(MutableMapping[str | ResponseKey[Any], Any], HeadersMixin): + + _body: None | bytes | bytearray | Payload + _length_check = True + _body = None + _keep_alive: bool | None = None + _chunked: bool = False + _compression: bool = False + _compression_strategy: int | None = None + _compression_force: ContentCoding | None = None + _req: Optional["BaseRequest"] = None + _payload_writer: AbstractStreamWriter | None = None + _eof_sent: bool = False + _must_be_empty_body: bool | None = None + _body_length = 0 + _cookies: SimpleCookie | None = None + _send_headers_immediately = True + _seen_str_keys: set[str] = set() + + def __init__( + self, + *, + status: int = 200, + reason: str | None = None, + headers: LooseHeaders | None = None, + _real_headers: CIMultiDict[str] | None = None, + ) -> None: + """Initialize a new stream response object. + + _real_headers is an internal parameter used to pass a pre-populated + headers object. It is used by the `Response` class to avoid copying + the headers when creating a new response object. It is not intended + to be used by external code. + """ + self._state: dict[str | ResponseKey[Any], Any] = {} + + if _real_headers is not None: + self._headers = _real_headers + elif headers is not None: + self._headers: CIMultiDict[str] = CIMultiDict(headers) + else: + self._headers = CIMultiDict() + + self._set_status(status, reason) + + @property + def prepared(self) -> bool: + return self._eof_sent or self._payload_writer is not None + + @property + def task(self) -> "asyncio.Task[None] | None": + if self._req: + return self._req.task + else: + return None + + @property + def status(self) -> int: + return self._status + + @property + def chunked(self) -> bool: + return self._chunked + + @property + def compression(self) -> bool: + return self._compression + + @property + def reason(self) -> str: + return self._reason + + def set_status( + self, + status: int, + reason: str | None = None, + ) -> None: + assert ( + not self.prepared + ), "Cannot change the response status code after the headers have been sent" + self._set_status(status, reason) + + def _set_status(self, status: int, reason: str | None) -> None: + self._status = int(status) + if reason is None: + reason = REASON_PHRASES.get(self._status, "") + elif "\r" in reason or "\n" in reason: + raise ValueError("Reason cannot contain \\r or \\n") + self._reason = reason + + @property + def keep_alive(self) -> bool | None: + return self._keep_alive + + def force_close(self) -> None: + self._keep_alive = False + + @property + def body_length(self) -> int: + return self._body_length + + @property + def output_length(self) -> int: + warnings.warn("output_length is deprecated", DeprecationWarning) + assert self._payload_writer + return self._payload_writer.buffer_size + + def enable_chunked_encoding(self, chunk_size: int | None = None) -> None: + """Enables automatic chunked transfer encoding.""" + if hdrs.CONTENT_LENGTH in self._headers: + raise RuntimeError( + "You can't enable chunked encoding when a content length is set" + ) + if chunk_size is not None: + warnings.warn("Chunk size is deprecated #1615", DeprecationWarning) + self._chunked = True + + def enable_compression( + self, + force: bool | ContentCoding | None = None, + strategy: int | None = None, + ) -> None: + """Enables response compression encoding.""" + # Backwards compatibility for when force was a bool <0.17. + if isinstance(force, bool): + force = ContentCoding.deflate if force else ContentCoding.identity + warnings.warn( + "Using boolean for force is deprecated #3318", DeprecationWarning + ) + elif force is not None: + assert isinstance( + force, ContentCoding + ), "force should one of None, bool or ContentEncoding" + + self._compression = True + self._compression_force = force + self._compression_strategy = strategy + + @property + def headers(self) -> "CIMultiDict[str]": + return self._headers + + @property + def cookies(self) -> SimpleCookie: + if self._cookies is None: + self._cookies = SimpleCookie() + return self._cookies + + def set_cookie( + self, + name: str, + value: str, + *, + expires: str | None = None, + domain: str | None = None, + max_age: int | str | None = None, + path: str = "/", + secure: bool | None = None, + httponly: bool | None = None, + version: str | None = None, + samesite: str | None = None, + partitioned: bool | None = None, + ) -> None: + """Set or update response cookie. + + Sets new cookie or updates existent with new value. + Also updates only those params which are not None. + """ + if self._cookies is None: + self._cookies = SimpleCookie() + + self._cookies[name] = value + c = self._cookies[name] + + if expires is not None: + c["expires"] = expires + elif c.get("expires") == "Thu, 01 Jan 1970 00:00:00 GMT": + del c["expires"] + + if domain is not None: + c["domain"] = domain + + if max_age is not None: + c["max-age"] = str(max_age) + elif "max-age" in c: + del c["max-age"] + + c["path"] = path + + if secure is not None: + c["secure"] = secure + if httponly is not None: + c["httponly"] = httponly + if version is not None: + c["version"] = version + if samesite is not None: + c["samesite"] = samesite + + if partitioned is not None: + c["partitioned"] = partitioned + + def del_cookie( + self, + name: str, + *, + domain: str | None = None, + path: str = "/", + secure: bool | None = None, + httponly: bool | None = None, + samesite: str | None = None, + ) -> None: + """Delete cookie. + + Creates new empty expired cookie. + """ + # TODO: do we need domain/path here? + if self._cookies is not None: + self._cookies.pop(name, None) + self.set_cookie( + name, + "", + max_age=0, + expires="Thu, 01 Jan 1970 00:00:00 GMT", + domain=domain, + path=path, + secure=secure, + httponly=httponly, + samesite=samesite, + ) + + @property + def content_length(self) -> int | None: + # Just a placeholder for adding setter + return super().content_length + + @content_length.setter + def content_length(self, value: int | None) -> None: + if value is not None: + value = int(value) + if self._chunked: + raise RuntimeError( + "You can't set content length when chunked encoding is enable" + ) + self._headers[hdrs.CONTENT_LENGTH] = str(value) + else: + self._headers.pop(hdrs.CONTENT_LENGTH, None) + + @property + def content_type(self) -> str: + # Just a placeholder for adding setter + return super().content_type + + @content_type.setter + def content_type(self, value: str) -> None: + self.content_type # read header values if needed + self._content_type = str(value) + self._generate_content_type_header() + + @property + def charset(self) -> str | None: + # Just a placeholder for adding setter + return super().charset + + @charset.setter + def charset(self, value: str | None) -> None: + ctype = self.content_type # read header values if needed + if ctype == "application/octet-stream": + raise RuntimeError( + "Setting charset for application/octet-stream " + "doesn't make sense, setup content_type first" + ) + assert self._content_dict is not None + if value is None: + self._content_dict.pop("charset", None) + else: + self._content_dict["charset"] = str(value).lower() + self._generate_content_type_header() + + @property + def last_modified(self) -> datetime.datetime | None: + """The value of Last-Modified HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self._headers.get(hdrs.LAST_MODIFIED)) + + @last_modified.setter + def last_modified( + self, value: int | float | datetime.datetime | str | None + ) -> None: + if value is None: + self._headers.pop(hdrs.LAST_MODIFIED, None) + elif isinstance(value, (int, float)): + self._headers[hdrs.LAST_MODIFIED] = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value)) + ) + elif isinstance(value, datetime.datetime): + self._headers[hdrs.LAST_MODIFIED] = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple() + ) + elif isinstance(value, str): + self._headers[hdrs.LAST_MODIFIED] = value + else: + msg = f"Unsupported type for last_modified: {type(value).__name__}" + raise TypeError(msg) + + @property + def etag(self) -> ETag | None: + quoted_value = self._headers.get(hdrs.ETAG) + if not quoted_value: + return None + elif quoted_value == ETAG_ANY: + return ETag(value=ETAG_ANY) + match = QUOTED_ETAG_RE.fullmatch(quoted_value) + if not match: + return None + is_weak, value = match.group(1, 2) + return ETag( + is_weak=bool(is_weak), + value=value, + ) + + @etag.setter + def etag(self, value: ETag | str | None) -> None: + if value is None: + self._headers.pop(hdrs.ETAG, None) + elif (isinstance(value, str) and value == ETAG_ANY) or ( + isinstance(value, ETag) and value.value == ETAG_ANY + ): + self._headers[hdrs.ETAG] = ETAG_ANY + elif isinstance(value, str): + validate_etag_value(value) + self._headers[hdrs.ETAG] = f'"{value}"' + elif isinstance(value, ETag) and isinstance(value.value, str): + validate_etag_value(value.value) + hdr_value = f'W/"{value.value}"' if value.is_weak else f'"{value.value}"' + self._headers[hdrs.ETAG] = hdr_value + else: + raise ValueError( + f"Unsupported etag type: {type(value)}. " + f"etag must be str, ETag or None" + ) + + def _generate_content_type_header( + self, CONTENT_TYPE: istr = hdrs.CONTENT_TYPE + ) -> None: + assert self._content_dict is not None + assert self._content_type is not None + params = "; ".join(f"{k}={v}" for k, v in self._content_dict.items()) + if params: + ctype = self._content_type + "; " + params + else: + ctype = self._content_type + self._headers[CONTENT_TYPE] = ctype + + async def _do_start_compression(self, coding: ContentCoding) -> None: + if coding is ContentCoding.identity: + return + assert self._payload_writer is not None + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._payload_writer.enable_compression( + coding.value, self._compression_strategy + ) + # Compressed payload may have different content length, + # remove the header + self._headers.popall(hdrs.CONTENT_LENGTH, None) + + async def _start_compression(self, request: "BaseRequest") -> None: + if self._compression_force: + await self._do_start_compression(self._compression_force) + return + # Encoding comparisons should be case-insensitive + # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1 + accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() + for value, coding in CONTENT_CODINGS.items(): + if value in accept_encoding: + await self._do_start_compression(coding) + return + + async def prepare(self, request: "BaseRequest") -> AbstractStreamWriter | None: + if self._eof_sent: + return None + if self._payload_writer is not None: + return self._payload_writer + self._must_be_empty_body = must_be_empty_body(request.method, self.status) + return await self._start(request) + + async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: + self._req = request + writer = self._payload_writer = request._payload_writer + + await self._prepare_headers() + await request._prepare_hook(self) + await self._write_headers() + + return writer + + async def _prepare_headers(self) -> None: + request = self._req + assert request is not None + writer = self._payload_writer + assert writer is not None + keep_alive = self._keep_alive + if keep_alive is None: + keep_alive = request.keep_alive + self._keep_alive = keep_alive + + version = request.version + + headers = self._headers + if self._cookies: + for cookie in self._cookies.values(): + value = cookie.output(header="")[1:] + headers.add(hdrs.SET_COOKIE, value) + + if self._compression: + await self._start_compression(request) + + if self._chunked: + if version != HttpVersion11: + raise RuntimeError( + "Using chunked encoding is forbidden " + f"for HTTP/{request.version.major}.{request.version.minor}" + ) + if not self._must_be_empty_body: + writer.enable_chunking() + headers[hdrs.TRANSFER_ENCODING] = "chunked" + elif self._length_check: # Disabled for WebSockets + writer.length = self.content_length + if writer.length is None: + if version >= HttpVersion11: + if not self._must_be_empty_body: + writer.enable_chunking() + headers[hdrs.TRANSFER_ENCODING] = "chunked" + elif not self._must_be_empty_body: + keep_alive = False + + # HTTP 1.1: https://tools.ietf.org/html/rfc7230#section-3.3.2 + # HTTP 1.0: https://tools.ietf.org/html/rfc1945#section-10.4 + if self._must_be_empty_body: + if hdrs.CONTENT_LENGTH in headers and should_remove_content_length( + request.method, self.status + ): + del headers[hdrs.CONTENT_LENGTH] + # https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-10 + # https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-13 + if hdrs.TRANSFER_ENCODING in headers: + del headers[hdrs.TRANSFER_ENCODING] + elif (writer.length if self._length_check else self.content_length) != 0: + # https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5 + headers.setdefault(hdrs.CONTENT_TYPE, "application/octet-stream") + headers.setdefault(hdrs.DATE, rfc822_formatted_time()) + headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE) + + # connection header + if hdrs.CONNECTION not in headers: + if keep_alive: + if version == HttpVersion10: + headers[hdrs.CONNECTION] = "keep-alive" + elif version == HttpVersion11: + headers[hdrs.CONNECTION] = "close" + + async def _write_headers(self) -> None: + request = self._req + assert request is not None + writer = self._payload_writer + assert writer is not None + # status line + version = request.version + status_line = f"HTTP/{version[0]}.{version[1]} {self._status} {self._reason}" + await writer.write_headers(status_line, self._headers) + # Send headers immediately if not opted into buffering + if self._send_headers_immediately: + writer.send_headers() + + async def write(self, data: bytes | bytearray | memoryview) -> None: + assert isinstance( + data, (bytes, bytearray, memoryview) + ), "data argument must be byte-ish (%r)" % type(data) + + if self._eof_sent: + raise RuntimeError("Cannot call write() after write_eof()") + if self._payload_writer is None: + raise RuntimeError("Cannot call write() before prepare()") + + await self._payload_writer.write(data) + + async def drain(self) -> None: + assert not self._eof_sent, "EOF has already been sent" + assert self._payload_writer is not None, "Response has not been started" + warnings.warn( + "drain method is deprecated, use await resp.write()", + DeprecationWarning, + stacklevel=2, + ) + await self._payload_writer.drain() + + async def write_eof(self, data: bytes = b"") -> None: + assert isinstance( + data, (bytes, bytearray, memoryview) + ), "data argument must be byte-ish (%r)" % type(data) + + if self._eof_sent: + return + + assert self._payload_writer is not None, "Response has not been started" + + await self._payload_writer.write_eof(data) + self._eof_sent = True + self._req = None + self._body_length = self._payload_writer.output_size + self._payload_writer = None + + def __repr__(self) -> str: + if self._eof_sent: + info = "eof" + elif self.prepared: + assert self._req is not None + info = f"{self._req.method} {self._req.path} " + else: + info = "not prepared" + return f"<{self.__class__.__name__} {self.reason} {info}>" + + @overload # type: ignore[override] + def __getitem__(self, key: ResponseKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: str | ResponseKey[_T]) -> Any: + return self._state[key] + + @overload # type: ignore[override] + def __setitem__(self, key: ResponseKey[_T], value: _T) -> None: ... + + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + + def __setitem__(self, key: str | ResponseKey[_T], value: Any) -> None: + if ( + not isinstance(key, ResponseKey) + and key not in StreamResponse._seen_str_keys + ): + # Import here to break circular dependency + from .web_exceptions import NotAppKeyWarning + + StreamResponse._seen_str_keys.add(key) + warnings.warn( + "It is recommended to use web.ResponseKey instances for keys.\n" + + "https://docs.aiohttp.org/en/stable/web_advanced.html" + + "#response-s-storage", + category=NotAppKeyWarning, + stacklevel=2, + ) + self._state[key] = value + + def __delitem__(self, key: str | ResponseKey[_T]) -> None: + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[str | ResponseKey[Any]]: + return iter(self._state) + + def __hash__(self) -> int: + return hash(id(self)) + + def __eq__(self, other: object) -> bool: + return self is other + + def __bool__(self) -> bool: + return True + + +class Response(StreamResponse): + + _compressed_body: bytes | None = None + _send_headers_immediately = False + + def __init__( + self, + *, + body: Any = None, + status: int = 200, + reason: str | None = None, + text: str | None = None, + headers: LooseHeaders | None = None, + content_type: str | None = None, + charset: str | None = None, + zlib_executor_size: int = MAX_SYNC_CHUNK_SIZE, + zlib_executor: Executor | None = None, + ) -> None: + if body is not None and text is not None: + raise ValueError("body and text are not allowed together") + + if headers is None: + real_headers: CIMultiDict[str] = CIMultiDict() + else: + real_headers = CIMultiDict(headers) + + if content_type is not None and "charset" in content_type: + raise ValueError("charset must not be in content_type argument") + + if text is not None: + if hdrs.CONTENT_TYPE in real_headers: + if content_type or charset: + raise ValueError( + "passing both Content-Type header and " + "content_type or charset params " + "is forbidden" + ) + else: + # fast path for filling headers + if not isinstance(text, str): + raise TypeError("text argument must be str (%r)" % type(text)) + if content_type is None: + content_type = "text/plain" + if charset is None: + charset = "utf-8" + real_headers[hdrs.CONTENT_TYPE] = content_type + "; charset=" + charset + body = text.encode(charset) + text = None + elif hdrs.CONTENT_TYPE in real_headers: + if content_type is not None or charset is not None: + raise ValueError( + "passing both Content-Type header and " + "content_type or charset params " + "is forbidden" + ) + elif content_type is not None: + if charset is not None: + content_type += "; charset=" + charset + real_headers[hdrs.CONTENT_TYPE] = content_type + + super().__init__(status=status, reason=reason, _real_headers=real_headers) + + if text is not None: + self.text = text + else: + self.body = body + + self._zlib_executor_size = zlib_executor_size + self._zlib_executor = zlib_executor + + @property + def body(self) -> bytes | bytearray | Payload | None: + return self._body + + @body.setter + def body(self, body: Any) -> None: + if body is None: + self._body = None + elif isinstance(body, (bytes, bytearray)): + self._body = body + else: + try: + self._body = body = payload.PAYLOAD_REGISTRY.get(body) + except payload.LookupError: + raise ValueError("Unsupported body type %r" % type(body)) + + headers = self._headers + + # set content-type + if hdrs.CONTENT_TYPE not in headers: + headers[hdrs.CONTENT_TYPE] = body.content_type + + # copy payload headers + if body.headers: + for key, value in body.headers.items(): + if key not in headers: + headers[key] = value + + self._compressed_body = None + + @property + def text(self) -> str | None: + if self._body is None: + return None + # Note: When _body is a Payload (e.g. FilePayload), this may do blocking I/O + # This is generally safe as most common payloads (BytesPayload, StringPayload) + # don't do blocking I/O, but be careful with file-based payloads + return self._body.decode(self.charset or "utf-8") + + @text.setter + def text(self, text: str) -> None: + assert text is None or isinstance( + text, str + ), "text argument must be str (%r)" % type(text) + + if self.content_type == "application/octet-stream": + self.content_type = "text/plain" + if self.charset is None: + self.charset = "utf-8" + + self._body = text.encode(self.charset) + self._compressed_body = None + + @property + def content_length(self) -> int | None: + if self._chunked: + return None + + if hdrs.CONTENT_LENGTH in self._headers: + return int(self._headers[hdrs.CONTENT_LENGTH]) + + if self._compressed_body is not None: + # Return length of the compressed body + return len(self._compressed_body) + elif isinstance(self._body, Payload): + # A payload without content length, or a compressed payload + return None + elif self._body is not None: + return len(self._body) + else: + return 0 + + @content_length.setter + def content_length(self, value: int | None) -> None: + raise RuntimeError("Content length is set automatically") + + async def write_eof(self, data: bytes = b"") -> None: + if self._eof_sent: + return + if self._compressed_body is None: + body = self._body + else: + body = self._compressed_body + assert not data, f"data arg is not supported, got {data!r}" + assert self._req is not None + assert self._payload_writer is not None + if body is None or self._must_be_empty_body: + await super().write_eof() + elif isinstance(self._body, Payload): + try: + await self._body.write(self._payload_writer) + finally: + await self._body.close() + await super().write_eof() + else: + await super().write_eof(cast(bytes, body)) + + async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: + if hdrs.CONTENT_LENGTH in self._headers: + if should_remove_content_length(request.method, self.status): + del self._headers[hdrs.CONTENT_LENGTH] + elif not self._chunked: + if isinstance(self._body, Payload): + if (size := self._body.size) is not None: + self._headers[hdrs.CONTENT_LENGTH] = str(size) + else: + body_len = len(self._body) if self._body else "0" + # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-7 + if body_len != "0" or ( + self.status != 304 and request.method not in hdrs.METH_HEAD_ALL + ): + self._headers[hdrs.CONTENT_LENGTH] = str(body_len) + + return await super()._start(request) + + async def _do_start_compression(self, coding: ContentCoding) -> None: + if self._chunked or isinstance(self._body, Payload): + return await super()._do_start_compression(coding) + if coding is ContentCoding.identity: + return + # Instead of using _payload_writer.enable_compression, + # compress the whole body + compressor = ZLibCompressor( + encoding=coding.value, + max_sync_chunk_size=self._zlib_executor_size, + executor=self._zlib_executor, + ) + assert self._body is not None + self._compressed_body = ( + await compressor.compress(self._body) + compressor.flush() + ) + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._headers[hdrs.CONTENT_LENGTH] = str(len(self._compressed_body)) + + +def json_response( + data: Any = sentinel, + *, + text: str | None = None, + body: bytes | None = None, + status: int = 200, + reason: str | None = None, + headers: LooseHeaders | None = None, + content_type: str = "application/json", + dumps: JSONEncoder = json.dumps, +) -> Response: + if data is not sentinel: + if text or body: + raise ValueError("only one of data, text, or body should be specified") + else: + text = dumps(data) + return Response( + text=text, + body=body, + status=status, + reason=reason, + headers=headers, + content_type=content_type, + ) + + +def json_bytes_response( + data: Any = sentinel, + *, + dumps: JSONBytesEncoder, + body: bytes | None = None, + status: int = 200, + reason: str | None = None, + headers: LooseHeaders | None = None, + content_type: str = "application/json", +) -> Response: + """Create a JSON response using a bytes-returning encoder. + + Use this when your JSON encoder (like orjson) returns bytes + instead of str, avoiding the encode/decode overhead. + """ + if data is not sentinel: + if body is not None: + raise ValueError("only one of data or body should be specified") + else: + body = dumps(data) + return Response( + body=body, + status=status, + reason=reason, + headers=headers, + content_type=content_type, + ) diff --git a/venv/Lib/site-packages/aiohttp/web_routedef.py b/venv/Lib/site-packages/aiohttp/web_routedef.py new file mode 100644 index 0000000000000000000000000000000000000000..66082772af1b55da744b4b110082d9c172f3a9e3 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_routedef.py @@ -0,0 +1,203 @@ +import abc +import os # noqa +from collections.abc import Callable, Iterator, Sequence +from typing import TYPE_CHECKING, Any, Union, overload + +import attr + +from . import hdrs +from .abc import AbstractView +from .typedefs import Handler, PathLike + +if TYPE_CHECKING: + from .web_request import Request + from .web_response import StreamResponse + from .web_urldispatcher import AbstractRoute, UrlDispatcher +else: + Request = StreamResponse = UrlDispatcher = AbstractRoute = None + + +__all__ = ( + "AbstractRouteDef", + "RouteDef", + "StaticDef", + "RouteTableDef", + "head", + "options", + "get", + "post", + "patch", + "put", + "delete", + "route", + "view", + "static", +) + + +class AbstractRouteDef(abc.ABC): + @abc.abstractmethod + def register(self, router: UrlDispatcher) -> list[AbstractRoute]: + pass # pragma: no cover + + +_HandlerType = Union[type[AbstractView], Handler] + + +@attr.s(auto_attribs=True, frozen=True, repr=False, slots=True) +class RouteDef(AbstractRouteDef): + method: str + path: str + handler: _HandlerType + kwargs: dict[str, Any] + + def __repr__(self) -> str: + info = [] + for name, value in sorted(self.kwargs.items()): + info.append(f", {name}={value!r}") + return " {handler.__name__!r}{info}>".format( + method=self.method, path=self.path, handler=self.handler, info="".join(info) + ) + + def register(self, router: UrlDispatcher) -> list[AbstractRoute]: + if self.method in hdrs.METH_ALL: + reg = getattr(router, "add_" + self.method.lower()) + return [reg(self.path, self.handler, **self.kwargs)] + else: + return [ + router.add_route(self.method, self.path, self.handler, **self.kwargs) + ] + + +@attr.s(auto_attribs=True, frozen=True, repr=False, slots=True) +class StaticDef(AbstractRouteDef): + prefix: str + path: PathLike + kwargs: dict[str, Any] + + def __repr__(self) -> str: + info = [] + for name, value in sorted(self.kwargs.items()): + info.append(f", {name}={value!r}") + return " {path}{info}>".format( + prefix=self.prefix, path=self.path, info="".join(info) + ) + + def register(self, router: UrlDispatcher) -> list[AbstractRoute]: + resource = router.add_static(self.prefix, self.path, **self.kwargs) + routes = resource.get_info().get("routes", {}) + return list(routes.values()) + + +def route(method: str, path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return RouteDef(method, path, handler, kwargs) + + +def head(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_HEAD, path, handler, **kwargs) + + +def options(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_OPTIONS, path, handler, **kwargs) + + +def get( + path: str, + handler: _HandlerType, + *, + name: str | None = None, + allow_head: bool = True, + **kwargs: Any, +) -> RouteDef: + return route( + hdrs.METH_GET, path, handler, name=name, allow_head=allow_head, **kwargs + ) + + +def post(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_POST, path, handler, **kwargs) + + +def put(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_PUT, path, handler, **kwargs) + + +def patch(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_PATCH, path, handler, **kwargs) + + +def delete(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_DELETE, path, handler, **kwargs) + + +def view(path: str, handler: type[AbstractView], **kwargs: Any) -> RouteDef: + return route(hdrs.METH_ANY, path, handler, **kwargs) + + +def static(prefix: str, path: PathLike, **kwargs: Any) -> StaticDef: + return StaticDef(prefix, path, kwargs) + + +_Deco = Callable[[_HandlerType], _HandlerType] + + +class RouteTableDef(Sequence[AbstractRouteDef]): + """Route definition table""" + + def __init__(self) -> None: + self._items: list[AbstractRouteDef] = [] + + def __repr__(self) -> str: + return f"" + + @overload + def __getitem__(self, index: int) -> AbstractRouteDef: ... + + @overload + def __getitem__(self, index: slice) -> list[AbstractRouteDef]: ... + + def __getitem__(self, index): # type: ignore[no-untyped-def] + return self._items[index] + + def __iter__(self) -> Iterator[AbstractRouteDef]: + return iter(self._items) + + def __len__(self) -> int: + return len(self._items) + + def __contains__(self, item: object) -> bool: + return item in self._items + + def route(self, method: str, path: str, **kwargs: Any) -> _Deco: + def inner(handler: _HandlerType) -> _HandlerType: + self._items.append(RouteDef(method, path, handler, kwargs)) + return handler + + return inner + + def head(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_HEAD, path, **kwargs) + + def get(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_GET, path, **kwargs) + + def post(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_POST, path, **kwargs) + + def put(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_PUT, path, **kwargs) + + def patch(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_PATCH, path, **kwargs) + + def delete(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_DELETE, path, **kwargs) + + def options(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_OPTIONS, path, **kwargs) + + def view(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_ANY, path, **kwargs) + + def static(self, prefix: str, path: PathLike, **kwargs: Any) -> None: + self._items.append(StaticDef(prefix, path, kwargs)) diff --git a/venv/Lib/site-packages/aiohttp/web_runner.py b/venv/Lib/site-packages/aiohttp/web_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..387e692dbe69d54f70da0b26227ae13f5154509a --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_runner.py @@ -0,0 +1,425 @@ +import asyncio +import signal +import socket +import warnings +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +from yarl import URL + +from .abc import AbstractAccessLogger +from .typedefs import PathLike +from .web_app import Application +from .web_log import AccessLogger +from .web_server import Server + +if TYPE_CHECKING: + from ssl import SSLContext +else: + try: + from ssl import SSLContext + except ImportError: # pragma: no cover + SSLContext = object # type: ignore[misc,assignment] + +__all__ = ( + "BaseSite", + "TCPSite", + "UnixSite", + "NamedPipeSite", + "SockSite", + "BaseRunner", + "AppRunner", + "ServerRunner", + "GracefulExit", +) + + +class GracefulExit(SystemExit): + code = 1 + + +def _raise_graceful_exit() -> None: + raise GracefulExit() + + +class BaseSite(ABC): + __slots__ = ("_runner", "_ssl_context", "_backlog", "_server") + + def __init__( + self, + runner: "BaseRunner", + *, + shutdown_timeout: float = 60.0, + ssl_context: SSLContext | None = None, + backlog: int = 128, + ) -> None: + if runner.server is None: + raise RuntimeError("Call runner.setup() before making a site") + if shutdown_timeout != 60.0: + msg = "shutdown_timeout should be set on BaseRunner" + warnings.warn(msg, DeprecationWarning, stacklevel=2) + runner._shutdown_timeout = shutdown_timeout + self._runner = runner + self._ssl_context = ssl_context + self._backlog = backlog + self._server: asyncio.AbstractServer | None = None + + @property + @abstractmethod + def name(self) -> str: + pass # pragma: no cover + + @abstractmethod + async def start(self) -> None: + self._runner._reg_site(self) + + async def stop(self) -> None: + self._runner._check_site(self) + if self._server is not None: # Maybe not started yet + self._server.close() + + self._runner._unreg_site(self) + + +class TCPSite(BaseSite): + __slots__ = ("_host", "_port", "_bound_port", "_reuse_address", "_reuse_port") + + def __init__( + self, + runner: "BaseRunner", + host: str | None = None, + port: int | None = None, + *, + shutdown_timeout: float = 60.0, + ssl_context: SSLContext | None = None, + backlog: int = 128, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._host = host + if port is None: + port = 8443 if self._ssl_context else 8080 + self._port = port + self._bound_port: int | None = None + self._reuse_address = reuse_address + self._reuse_port = reuse_port + + @property + def port(self) -> int: + """The port the server is listening on. + + If the server hasn't been started yet, this returns the requested port + (which might be 0 for a dynamic port). + After the server starts, it returns the actual bound port. This is + especially useful when port=0 was requested, as it allows retrieving the + dynamically assigned port after the site has started. + """ + if self._bound_port is not None: + return self._bound_port + return self._port + + @property + def name(self) -> str: + scheme = "https" if self._ssl_context else "http" + host = "0.0.0.0" if not self._host else self._host + return str(URL.build(scheme=scheme, host=host, port=self.port)) + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_server( + server, + self._host, + self._port, + ssl=self._ssl_context, + backlog=self._backlog, + reuse_address=self._reuse_address, + reuse_port=self._reuse_port, + ) + if self._server.sockets: + self._bound_port = self._server.sockets[0].getsockname()[1] + else: + self._bound_port = self._port + + +class UnixSite(BaseSite): + __slots__ = ("_path",) + + def __init__( + self, + runner: "BaseRunner", + path: PathLike, + *, + shutdown_timeout: float = 60.0, + ssl_context: SSLContext | None = None, + backlog: int = 128, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._path = path + + @property + def name(self) -> str: + scheme = "https" if self._ssl_context else "http" + return f"{scheme}://unix:{self._path}:" + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_unix_server( + server, + self._path, + ssl=self._ssl_context, + backlog=self._backlog, + ) + + +class NamedPipeSite(BaseSite): + __slots__ = ("_path",) + + def __init__( + self, runner: "BaseRunner", path: str, *, shutdown_timeout: float = 60.0 + ) -> None: + loop = asyncio.get_event_loop() + if not isinstance( + loop, asyncio.ProactorEventLoop # type: ignore[attr-defined] + ): + raise RuntimeError( + "Named Pipes only available in proactor loop under windows" + ) + super().__init__(runner, shutdown_timeout=shutdown_timeout) + self._path = path + + @property + def name(self) -> str: + return self._path + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + _server = await loop.start_serving_pipe( # type: ignore[attr-defined] + server, self._path + ) + self._server = _server[0] + + +class SockSite(BaseSite): + __slots__ = ("_sock", "_name") + + def __init__( + self, + runner: "BaseRunner", + sock: socket.socket, + *, + shutdown_timeout: float = 60.0, + ssl_context: SSLContext | None = None, + backlog: int = 128, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._sock = sock + scheme = "https" if self._ssl_context else "http" + if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: + name = f"{scheme}://unix:{sock.getsockname()}:" + else: + host, port = sock.getsockname()[:2] + name = str(URL.build(scheme=scheme, host=host, port=port)) + self._name = name + + @property + def name(self) -> str: + return self._name + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_server( + server, sock=self._sock, ssl=self._ssl_context, backlog=self._backlog + ) + + +class BaseRunner(ABC): + __slots__ = ("_handle_signals", "_kwargs", "_server", "_sites", "_shutdown_timeout") + + def __init__( + self, + *, + handle_signals: bool = False, + shutdown_timeout: float = 60.0, + **kwargs: Any, + ) -> None: + self._handle_signals = handle_signals + self._kwargs = kwargs + self._server: Server | None = None + self._sites: list[BaseSite] = [] + self._shutdown_timeout = shutdown_timeout + + @property + def server(self) -> Server | None: + return self._server + + @property + def addresses(self) -> list[Any]: + ret: list[Any] = [] + for site in self._sites: + server = site._server + if server is not None: + sockets = server.sockets # type: ignore[attr-defined] + if sockets is not None: + for sock in sockets: + ret.append(sock.getsockname()) + return ret + + @property + def sites(self) -> set[BaseSite]: + return set(self._sites) + + async def setup(self) -> None: + loop = asyncio.get_event_loop() + + if self._handle_signals: + try: + loop.add_signal_handler(signal.SIGINT, _raise_graceful_exit) + loop.add_signal_handler(signal.SIGTERM, _raise_graceful_exit) + except NotImplementedError: # pragma: no cover + # add_signal_handler is not implemented on Windows + pass + + self._server = await self._make_server() + + @abstractmethod + async def shutdown(self) -> None: + """Call any shutdown hooks to help server close gracefully.""" + + async def cleanup(self) -> None: + # The loop over sites is intentional, an exception on gather() + # leaves self._sites in unpredictable state. + # The loop guaranties that a site is either deleted on success or + # still present on failure + for site in list(self._sites): + await site.stop() + + if self._server: # If setup succeeded + # Yield to event loop to ensure incoming requests prior to stopping the sites + # have all started to be handled before we proceed to close idle connections. + await asyncio.sleep(0) + self._server.pre_shutdown() + await self.shutdown() + await self._server.shutdown(self._shutdown_timeout) + await self._cleanup_server() + + self._server = None + if self._handle_signals: + loop = asyncio.get_running_loop() + try: + loop.remove_signal_handler(signal.SIGINT) + loop.remove_signal_handler(signal.SIGTERM) + except NotImplementedError: # pragma: no cover + # remove_signal_handler is not implemented on Windows + pass + + @abstractmethod + async def _make_server(self) -> Server: + pass # pragma: no cover + + @abstractmethod + async def _cleanup_server(self) -> None: + pass # pragma: no cover + + def _reg_site(self, site: BaseSite) -> None: + if site in self._sites: + raise RuntimeError(f"Site {site} is already registered in runner {self}") + self._sites.append(site) + + def _check_site(self, site: BaseSite) -> None: + if site not in self._sites: + raise RuntimeError(f"Site {site} is not registered in runner {self}") + + def _unreg_site(self, site: BaseSite) -> None: + if site not in self._sites: + raise RuntimeError(f"Site {site} is not registered in runner {self}") + self._sites.remove(site) + + +class ServerRunner(BaseRunner): + """Low-level web server runner""" + + __slots__ = ("_web_server",) + + def __init__( + self, web_server: Server, *, handle_signals: bool = False, **kwargs: Any + ) -> None: + super().__init__(handle_signals=handle_signals, **kwargs) + self._web_server = web_server + + async def shutdown(self) -> None: + pass + + async def _make_server(self) -> Server: + return self._web_server + + async def _cleanup_server(self) -> None: + pass + + +class AppRunner(BaseRunner): + """Web Application runner""" + + __slots__ = ("_app",) + + def __init__( + self, + app: Application, + *, + handle_signals: bool = False, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> None: + super().__init__(handle_signals=handle_signals, **kwargs) + if not isinstance(app, Application): + raise TypeError( + f"The first argument should be web.Application instance, got {app!r}" + ) + self._kwargs["access_log_class"] = access_log_class + self._app = app + + @property + def app(self) -> Application: + return self._app + + async def shutdown(self) -> None: + await self._app.shutdown() + + async def _make_server(self) -> Server: + loop = asyncio.get_event_loop() + self._app._set_loop(loop) + self._app.on_startup.freeze() + await self._app.startup() + self._app.freeze() + + return self._app._make_handler(loop=loop, **self._kwargs) + + async def _cleanup_server(self) -> None: + await self._app.cleanup() diff --git a/venv/Lib/site-packages/aiohttp/web_server.py b/venv/Lib/site-packages/aiohttp/web_server.py new file mode 100644 index 0000000000000000000000000000000000000000..3272c1d78925ff0cf26a232b1fad536e04f7f31d --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_server.py @@ -0,0 +1,91 @@ +"""Low level HTTP server.""" + +import asyncio +from typing import Any, Awaitable, Callable, Dict, List, Optional # noqa + +from .abc import AbstractStreamWriter +from .http_parser import RawRequestMessage +from .streams import StreamReader +from .web_protocol import RequestHandler, _RequestFactory, _RequestHandler +from .web_request import BaseRequest + +__all__ = ("Server",) + + +class Server: + def __init__( + self, + handler: _RequestHandler, + *, + request_factory: _RequestFactory | None = None, + handler_cancellation: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Any, + ) -> None: + self._loop = loop or asyncio.get_running_loop() + self._connections: dict[RequestHandler, asyncio.Transport] = {} + self._kwargs = kwargs + # requests_count is the number of requests being processed by the server + # for the lifetime of the server. + self.requests_count = 0 + self.request_handler = handler + self.request_factory = request_factory or self._make_request + self.handler_cancellation = handler_cancellation + + @property + def connections(self) -> list[RequestHandler]: + return list(self._connections.keys()) + + def connection_made( + self, handler: RequestHandler, transport: asyncio.Transport + ) -> None: + self._connections[handler] = transport + + def connection_lost( + self, handler: RequestHandler, exc: BaseException | None = None + ) -> None: + if handler in self._connections: + if handler._task_handler: + handler._task_handler.add_done_callback( + lambda f: self._connections.pop(handler, None) + ) + else: + del self._connections[handler] + + def _make_request( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: RequestHandler, + writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + ) -> BaseRequest: + return BaseRequest(message, payload, protocol, writer, task, self._loop) + + def pre_shutdown(self) -> None: + for conn in self._connections: + conn.close() + + async def shutdown(self, timeout: float | None = None) -> None: + coros = (conn.shutdown(timeout) for conn in self._connections) + await asyncio.gather(*coros) + self._connections.clear() + + def __call__(self) -> RequestHandler: + try: + return RequestHandler(self, loop=self._loop, **self._kwargs) + except TypeError: + # Failsafe creation: remove all custom handler_args + kwargs = { + k: v + for k, v in self._kwargs.items() + if k in ["debug", "access_log_class"] + } + handler = RequestHandler(self, loop=self._loop, **kwargs) + handler.logger.warning( + "Failed to create request handler with custom kwargs %r, " + "falling back to filtered kwargs. This may indicate a " + "misconfiguration.", + self._kwargs, + ) + return handler diff --git a/venv/Lib/site-packages/aiohttp/web_urldispatcher.py b/venv/Lib/site-packages/aiohttp/web_urldispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..305dbc3c01400f8e1db51d6a473860ec7a2ab787 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_urldispatcher.py @@ -0,0 +1,1281 @@ +import abc +import asyncio +import base64 +import functools +import hashlib +import html +import inspect +import keyword +import os +import platform +import re +import sys +import warnings +from collections.abc import ( + Awaitable, + Callable, + Container, + Generator, + Iterable, + Iterator, + Mapping, + Sized, +) +from functools import wraps +from pathlib import Path +from re import Pattern +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, NoReturn, Optional, TypedDict, cast + +from yarl import URL, __version__ as yarl_version + +from . import hdrs +from .abc import AbstractMatchInfo, AbstractRouter, AbstractView +from .helpers import DEBUG, DEFAULT_CHUNK_SIZE +from .http import HttpVersion11 +from .typedefs import Handler, PathLike +from .web_exceptions import ( + HTTPException, + HTTPExpectationFailed, + HTTPForbidden, + HTTPMethodNotAllowed, + HTTPNotFound, +) +from .web_fileresponse import FileResponse +from .web_request import Request +from .web_response import Response, StreamResponse +from .web_routedef import AbstractRouteDef + +__all__ = ( + "UrlDispatcher", + "UrlMappingMatchInfo", + "AbstractResource", + "Resource", + "PlainResource", + "DynamicResource", + "AbstractRoute", + "ResourceRoute", + "StaticResource", + "View", +) + + +if TYPE_CHECKING: + from .web_app import Application + + BaseDict = dict[str, str] +else: + BaseDict = dict + +CIRCULAR_SYMLINK_ERROR = (RuntimeError,) if sys.version_info < (3, 13) else () + +YARL_VERSION: Final[tuple[int, ...]] = tuple(map(int, yarl_version.split(".")[:2])) + +HTTP_METHOD_RE: Final[Pattern[str]] = re.compile( + r"^[0-9A-Za-z!#\$%&'\*\+\-\.\^_`\|~]+$" +) +ROUTE_RE: Final[Pattern[str]] = re.compile( + r"(\{[_a-zA-Z][^{}]*(?:\{[^{}]*\}[^{}]*)*\})" +) +PATH_SEP: Final[str] = re.escape("/") + +IS_WINDOWS: Final[bool] = platform.system() == "Windows" + +_ExpectHandler = Callable[[Request], Awaitable[StreamResponse | None]] +_Resolve = tuple[Optional["UrlMappingMatchInfo"], set[str]] + +html_escape = functools.partial(html.escape, quote=True) + + +class _InfoDict(TypedDict, total=False): + path: str + + formatter: str + pattern: Pattern[str] + + directory: Path + prefix: str + routes: Mapping[str, "AbstractRoute"] + + app: "Application" + + domain: str + + rule: "AbstractRuleMatching" + + http_exception: HTTPException + + +class AbstractResource(Sized, Iterable["AbstractRoute"]): + def __init__(self, *, name: str | None = None) -> None: + self._name = name + + @property + def name(self) -> str | None: + return self._name + + @property + @abc.abstractmethod + def canonical(self) -> str: + """Exposes the resource's canonical path. + + For example '/foo/bar/{name}' + + """ + + @abc.abstractmethod # pragma: no branch + def url_for(self, **kwargs: str) -> URL: + """Construct url for resource with additional params.""" + + @abc.abstractmethod # pragma: no branch + async def resolve(self, request: Request) -> _Resolve: + """Resolve resource. + + Return (UrlMappingMatchInfo, allowed_methods) pair. + """ + + @abc.abstractmethod + def add_prefix(self, prefix: str) -> None: + """Add a prefix to processed URLs. + + Required for subapplications support. + """ + + @abc.abstractmethod + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + def freeze(self) -> None: + pass + + @abc.abstractmethod + def raw_match(self, path: str) -> bool: + """Perform a raw match against path""" + + +class AbstractRoute(abc.ABC): + def __init__( + self, + method: str, + handler: Handler | type[AbstractView], + *, + expect_handler: _ExpectHandler | None = None, + resource: AbstractResource | None = None, + ) -> None: + + if expect_handler is None: + expect_handler = _default_expect_handler + + assert inspect.iscoroutinefunction(expect_handler) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(expect_handler) + ), f"Coroutine is expected, got {expect_handler!r}" + + method = method.upper() + if not HTTP_METHOD_RE.match(method): + raise ValueError(f"{method} is not allowed HTTP method") + + assert callable(handler), handler + if inspect.iscoroutinefunction(handler) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(handler) + ): + pass + elif inspect.isgeneratorfunction(handler): + if TYPE_CHECKING: + assert False + warnings.warn( + "Bare generators are deprecated, use @coroutine wrapper", + DeprecationWarning, + ) + elif isinstance(handler, type) and issubclass(handler, AbstractView): + pass + else: + warnings.warn( + "Bare functions are deprecated, use async ones", DeprecationWarning + ) + + @wraps(handler) + async def handler_wrapper(request: Request) -> StreamResponse: + result = old_handler(request) # type: ignore[call-arg] + if asyncio.iscoroutine(result): + result = await result + assert isinstance(result, StreamResponse) + return result + + old_handler = handler + handler = handler_wrapper + + self._method = method + self._handler = handler + self._expect_handler = expect_handler + self._resource = resource + + @property + def method(self) -> str: + return self._method + + @property + def handler(self) -> Handler: + return self._handler + + @property + @abc.abstractmethod + def name(self) -> str | None: + """Optional route's name, always equals to resource's name.""" + + @property + def resource(self) -> AbstractResource | None: + return self._resource + + @abc.abstractmethod + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + @abc.abstractmethod # pragma: no branch + def url_for(self, *args: str, **kwargs: str) -> URL: + """Construct url for route with additional params.""" + + async def handle_expect_header(self, request: Request) -> StreamResponse | None: + return await self._expect_handler(request) + + +class UrlMappingMatchInfo(BaseDict, AbstractMatchInfo): + + __slots__ = ("_route", "_apps", "_current_app", "_frozen") + + def __init__(self, match_dict: dict[str, str], route: AbstractRoute) -> None: + super().__init__(match_dict) + self._route = route + self._apps: list[Application] = [] + self._current_app: Application | None = None + self._frozen = False + + @property + def handler(self) -> Handler: + return self._route.handler + + @property + def route(self) -> AbstractRoute: + return self._route + + @property + def expect_handler(self) -> _ExpectHandler: + return self._route.handle_expect_header + + @property + def http_exception(self) -> HTTPException | None: + return None + + def get_info(self) -> _InfoDict: # type: ignore[override] + return self._route.get_info() + + @property + def apps(self) -> tuple["Application", ...]: + return tuple(self._apps) + + def add_app(self, app: "Application") -> None: + if self._frozen: + raise RuntimeError("Cannot change apps stack after .freeze() call") + if self._current_app is None: + self._current_app = app + self._apps.insert(0, app) + + @property + def current_app(self) -> "Application": + app = self._current_app + assert app is not None + return app + + @current_app.setter + def current_app(self, app: "Application") -> None: + if DEBUG: # pragma: no cover + if app not in self._apps: + raise RuntimeError( + f"Expected one of the following apps {self._apps!r}, got {app!r}" + ) + self._current_app = app + + def freeze(self) -> None: + self._frozen = True + + def __repr__(self) -> str: + return f"" + + +class MatchInfoError(UrlMappingMatchInfo): + + __slots__ = ("_exception",) + + def __init__(self, http_exception: HTTPException) -> None: + self._exception = http_exception + super().__init__({}, SystemRoute(self._exception)) + + @property + def http_exception(self) -> HTTPException: + return self._exception + + def __repr__(self) -> str: + return f"" + + +async def _default_expect_handler(request: Request) -> None: + """Default handler for Expect header. + + Just send "100 Continue" to client. + raise HTTPExpectationFailed if value of header is not "100-continue" + """ + expect = request.headers.get(hdrs.EXPECT, "") + if request.version == HttpVersion11: + if expect.lower() == "100-continue": + await request.writer.write(b"HTTP/1.1 100 Continue\r\n\r\n") + # Reset output_size as we haven't started the main body yet. + request.writer.output_size = 0 + else: + raise HTTPExpectationFailed(text="Unknown Expect: %s" % expect) + + +class Resource(AbstractResource): + def __init__(self, *, name: str | None = None) -> None: + super().__init__(name=name) + self._routes: dict[str, ResourceRoute] = {} + self._any_route: ResourceRoute | None = None + self._allowed_methods: set[str] = set() + + def add_route( + self, + method: str, + handler: type[AbstractView] | Handler, + *, + expect_handler: _ExpectHandler | None = None, + ) -> "ResourceRoute": + if route := self._routes.get(method, self._any_route): + raise RuntimeError( + "Added route will never be executed, " + f"method {route.method} is already " + "registered" + ) + + route_obj = ResourceRoute(method, handler, self, expect_handler=expect_handler) + self.register_route(route_obj) + return route_obj + + def register_route(self, route: "ResourceRoute") -> None: + assert isinstance( + route, ResourceRoute + ), f"Instance of Route class is required, got {route!r}" + if route.method == hdrs.METH_ANY: + self._any_route = route + self._allowed_methods.add(route.method) + self._routes[route.method] = route + + async def resolve(self, request: Request) -> _Resolve: + if (match_dict := self._match(request.rel_url.path_safe)) is None: + return None, set() + if route := self._routes.get(request.method, self._any_route): + return UrlMappingMatchInfo(match_dict, route), self._allowed_methods + return None, self._allowed_methods + + @abc.abstractmethod + def _match(self, path: str) -> dict[str, str] | None: + pass # pragma: no cover + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator["ResourceRoute"]: + return iter(self._routes.values()) + + # TODO: implement all abstract methods + + +class PlainResource(Resource): + def __init__(self, path: str, *, name: str | None = None) -> None: + super().__init__(name=name) + assert not path or path.startswith("/") + self._path = path + + @property + def canonical(self) -> str: + return self._path + + def freeze(self) -> None: + if not self._path: + self._path = "/" + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._path = prefix + self._path + + def _match(self, path: str) -> dict[str, str] | None: + # string comparison is about 10 times faster than regexp matching + if self._path == path: + return {} + return None + + def raw_match(self, path: str) -> bool: + return self._path == path + + def get_info(self) -> _InfoDict: + return {"path": self._path} + + def url_for(self) -> URL: # type: ignore[override] + return URL.build(path=self._path, encoded=True) + + def __repr__(self) -> str: + name = "'" + self.name + "' " if self.name is not None else "" + return f"" + + +class DynamicResource(Resource): + + DYN = re.compile(r"\{(?P[_a-zA-Z][_a-zA-Z0-9]*)\}") + DYN_WITH_RE = re.compile(r"\{(?P[_a-zA-Z][_a-zA-Z0-9]*):(?P.+)\}") + GOOD = r"[^{}/]+" + + def __init__(self, path: str, *, name: str | None = None) -> None: + super().__init__(name=name) + self._orig_path = path + pattern = "" + formatter = "" + for part in ROUTE_RE.split(path): + match = self.DYN.fullmatch(part) + if match: + pattern += "(?P<{}>{})".format(match.group("var"), self.GOOD) + formatter += "{" + match.group("var") + "}" + continue + + match = self.DYN_WITH_RE.fullmatch(part) + if match: + pattern += "(?P<{var}>{re})".format(**match.groupdict()) + formatter += "{" + match.group("var") + "}" + continue + + if "{" in part or "}" in part: + raise ValueError(f"Invalid path '{path}'['{part}']") + + part = _requote_path(part) + formatter += part + pattern += re.escape(part) + + try: + compiled = re.compile(pattern) + except re.error as exc: + raise ValueError(f"Bad pattern '{pattern}': {exc}") from None + assert compiled.pattern.startswith(PATH_SEP) + assert formatter.startswith("/") + self._pattern = compiled + self._formatter = formatter + + @property + def canonical(self) -> str: + return self._formatter + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._pattern = re.compile(re.escape(prefix) + self._pattern.pattern) + self._formatter = prefix + self._formatter + + def _match(self, path: str) -> dict[str, str] | None: + match = self._pattern.fullmatch(path) + if match is None: + return None + return { + key: _unquote_path_safe(value) for key, value in match.groupdict().items() + } + + def raw_match(self, path: str) -> bool: + return self._orig_path == path + + def get_info(self) -> _InfoDict: + return {"formatter": self._formatter, "pattern": self._pattern} + + def url_for(self, **parts: str) -> URL: + url = self._formatter.format_map({k: _quote_path(v) for k, v in parts.items()}) + return URL.build(path=url, encoded=True) + + def __repr__(self) -> str: + name = "'" + self.name + "' " if self.name is not None else "" + return f"" + + +class PrefixResource(AbstractResource): + def __init__(self, prefix: str, *, name: str | None = None) -> None: + assert not prefix or prefix.startswith("/"), prefix + assert prefix in ("", "/") or not prefix.endswith("/"), prefix + super().__init__(name=name) + self._prefix = _requote_path(prefix) + self._prefix2 = self._prefix + "/" + + @property + def canonical(self) -> str: + return self._prefix + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._prefix = prefix + self._prefix + self._prefix2 = self._prefix + "/" + + def raw_match(self, prefix: str) -> bool: + return False + + # TODO: impl missing abstract methods + + +class StaticResource(PrefixResource): + VERSION_KEY = "v" + + def __init__( + self, + prefix: str, + directory: PathLike, + *, + name: str | None = None, + expect_handler: _ExpectHandler | None = None, + chunk_size: int = DEFAULT_CHUNK_SIZE, + show_index: bool = False, + follow_symlinks: bool = False, + append_version: bool = False, + ) -> None: + super().__init__(prefix, name=name) + try: + directory = Path(directory).expanduser().resolve(strict=True) + except FileNotFoundError as error: + raise ValueError(f"'{directory}' does not exist") from error + if not directory.is_dir(): + raise ValueError(f"'{directory}' is not a directory") + self._directory = directory + self._show_index = show_index + self._chunk_size = chunk_size + self._follow_symlinks = follow_symlinks + self._expect_handler = expect_handler + self._append_version = append_version + + self._routes = { + "GET": ResourceRoute( + "GET", self._handle, self, expect_handler=expect_handler + ), + "HEAD": ResourceRoute( + "HEAD", self._handle, self, expect_handler=expect_handler + ), + } + self._allowed_methods = set(self._routes) + + def url_for( # type: ignore[override] + self, + *, + filename: PathLike, + append_version: bool | None = None, + ) -> URL: + if append_version is None: + append_version = self._append_version + filename = str(filename).lstrip("/") + + url = URL.build(path=self._prefix, encoded=True) + # filename is not encoded + if YARL_VERSION < (1, 6): + url = url / filename.replace("%", "%25") + else: + url = url / filename + + if append_version: + unresolved_path = self._directory.joinpath(filename) + try: + if self._follow_symlinks: + normalized_path = Path(os.path.normpath(unresolved_path)) + normalized_path.relative_to(self._directory) + filepath = normalized_path.resolve() + else: + filepath = unresolved_path.resolve() + filepath.relative_to(self._directory) + except (ValueError, FileNotFoundError): + # ValueError for case when path point to symlink + # with follow_symlinks is False + return url # relatively safe + if filepath.is_file(): + # TODO cache file content + # with file watcher for cache invalidation + with filepath.open("rb") as f: + file_bytes = f.read() + h = self._get_file_hash(file_bytes) + url = url.with_query({self.VERSION_KEY: h}) + return url + return url + + @staticmethod + def _get_file_hash(byte_array: bytes) -> str: + m = hashlib.sha256() # todo sha256 can be configurable param + m.update(byte_array) + b64 = base64.urlsafe_b64encode(m.digest()) + return b64.decode("ascii") + + def get_info(self) -> _InfoDict: + return { + "directory": self._directory, + "prefix": self._prefix, + "routes": self._routes, + } + + def set_options_route(self, handler: Handler) -> None: + if "OPTIONS" in self._routes: + raise RuntimeError("OPTIONS route was set already") + self._routes["OPTIONS"] = ResourceRoute( + "OPTIONS", handler, self, expect_handler=self._expect_handler + ) + self._allowed_methods.add("OPTIONS") + + async def resolve(self, request: Request) -> _Resolve: + path = request.rel_url.path_safe + method = request.method + # We normalise here to avoid matches that traverse below the static root. + # e.g. /static/../../../../home/user/webapp/static/ + norm_path = os.path.normpath(path) + if IS_WINDOWS: + norm_path = norm_path.replace("\\", "/") + if not norm_path.startswith(self._prefix2) and norm_path != self._prefix: + return None, set() + + allowed_methods = self._allowed_methods + if method not in allowed_methods: + return None, allowed_methods + + match_dict = {"filename": _unquote_path_safe(path[len(self._prefix) + 1 :])} + return (UrlMappingMatchInfo(match_dict, self._routes[method]), allowed_methods) + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator[AbstractRoute]: + return iter(self._routes.values()) + + async def _handle(self, request: Request) -> StreamResponse: + filename = request.match_info["filename"] + if Path(filename).is_absolute(): + # filename is an absolute path e.g. //network/share or D:\path + # which could be a UNC path leading to NTLM credential theft + raise HTTPNotFound() + unresolved_path = self._directory.joinpath(filename) + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, self._resolve_path_to_response, unresolved_path + ) + + def _resolve_path_to_response(self, unresolved_path: Path) -> StreamResponse: + """Take the unresolved path and query the file system to form a response.""" + # Check for access outside the root directory. For follow symlinks, URI + # cannot traverse out, but symlinks can. Otherwise, no access outside + # root is permitted. + try: + if self._follow_symlinks: + normalized_path = Path(os.path.normpath(unresolved_path)) + normalized_path.relative_to(self._directory) + file_path = normalized_path.resolve() + else: + file_path = unresolved_path.resolve() + file_path.relative_to(self._directory) + except (ValueError, *CIRCULAR_SYMLINK_ERROR) as error: + # ValueError is raised for the relative check. Circular symlinks + # raise here on resolving for python < 3.13. + raise HTTPNotFound() from error + + # if path is a directory, return the contents if permitted. Note the + # directory check will raise if a segment is not readable. + try: + if file_path.is_dir(): + if self._show_index: + return Response( + text=self._directory_as_html(file_path), + content_type="text/html", + ) + else: + raise HTTPForbidden() + except PermissionError as error: + raise HTTPForbidden() from error + + # Return the file response, which handles all other checks. + return FileResponse(file_path, chunk_size=self._chunk_size) + + def _directory_as_html(self, dir_path: Path) -> str: + """returns directory's index as html.""" + assert dir_path.is_dir() + + relative_path_to_dir = dir_path.relative_to(self._directory).as_posix() + index_of = f"Index of /{html_escape(relative_path_to_dir)}" + h1 = f"

{index_of}

" + + index_list = [] + dir_index = dir_path.iterdir() + for _file in sorted(dir_index): + # show file url as relative to static path + rel_path = _file.relative_to(self._directory).as_posix() + quoted_file_url = _quote_path(f"{self._prefix}/{rel_path}") + + # if file is a directory, add '/' to the end of the name + if _file.is_dir(): + file_name = f"{_file.name}/" + else: + file_name = _file.name + + index_list.append( + f'
  • {html_escape(file_name)}
  • ' + ) + ul = "
      \n{}\n
    ".format("\n".join(index_list)) + body = f"\n{h1}\n{ul}\n" + + head_str = f"\n{index_of}\n" + html = f"\n{head_str}\n{body}\n" + + return html + + def __repr__(self) -> str: + name = "'" + self.name + "'" if self.name is not None else "" + return f" {self._directory!r}>" + + +class PrefixedSubAppResource(PrefixResource): + def __init__(self, prefix: str, app: "Application") -> None: + super().__init__(prefix) + self._app = app + self._add_prefix_to_resources(prefix) + + def add_prefix(self, prefix: str) -> None: + super().add_prefix(prefix) + self._add_prefix_to_resources(prefix) + + def _add_prefix_to_resources(self, prefix: str) -> None: + router = self._app.router + for resource in router.resources(): + # Since the canonical path of a resource is about + # to change, we need to unindex it and then reindex + router.unindex_resource(resource) + resource.add_prefix(prefix) + router.index_resource(resource) + + def url_for(self, *args: str, **kwargs: str) -> URL: + raise RuntimeError(".url_for() is not supported by sub-application root") + + def get_info(self) -> _InfoDict: + return {"app": self._app, "prefix": self._prefix} + + async def resolve(self, request: Request) -> _Resolve: + match_info = await self._app.router.resolve(request) + match_info.add_app(self._app) + if isinstance(match_info.http_exception, HTTPMethodNotAllowed): + methods = match_info.http_exception.allowed_methods + else: + methods = set() + return match_info, methods + + def __len__(self) -> int: + return len(self._app.router.routes()) + + def __iter__(self) -> Iterator[AbstractRoute]: + return iter(self._app.router.routes()) + + def __repr__(self) -> str: + return f" {self._app!r}>" + + +class AbstractRuleMatching(abc.ABC): + @abc.abstractmethod # pragma: no branch + async def match(self, request: Request) -> bool: + """Return bool if the request satisfies the criteria""" + + @abc.abstractmethod # pragma: no branch + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + @property + @abc.abstractmethod # pragma: no branch + def canonical(self) -> str: + """Return a str""" + + +class Domain(AbstractRuleMatching): + re_part = re.compile(r"(?!-)[a-z\d-]{1,63}(? None: + super().__init__() + self._domain = self.validation(domain) + + @property + def canonical(self) -> str: + return self._domain + + def validation(self, domain: str) -> str: + if not isinstance(domain, str): + raise TypeError("Domain must be str") + domain = domain.rstrip(".").lower() + if not domain: + raise ValueError("Domain cannot be empty") + elif "://" in domain: + raise ValueError("Scheme not supported") + url = URL("http://" + domain) + assert url.raw_host is not None + if not all(self.re_part.fullmatch(x) for x in url.raw_host.split(".")): + raise ValueError("Domain not valid") + if url.port == 80: + return url.raw_host + return f"{url.raw_host}:{url.port}" + + async def match(self, request: Request) -> bool: + host = request.headers.get(hdrs.HOST) + if not host: + return False + return self.match_domain(host) + + def match_domain(self, host: str) -> bool: + return host.lower() == self._domain + + def get_info(self) -> _InfoDict: + return {"domain": self._domain} + + +class MaskDomain(Domain): + re_part = re.compile(r"(?!-)[a-z\d\*-]{1,63}(? None: + super().__init__(domain) + mask = self._domain.replace(".", r"\.").replace("*", ".*") + self._mask = re.compile(mask) + + @property + def canonical(self) -> str: + return self._mask.pattern + + def match_domain(self, host: str) -> bool: + return self._mask.fullmatch(host) is not None + + +class MatchedSubAppResource(PrefixedSubAppResource): + def __init__(self, rule: AbstractRuleMatching, app: "Application") -> None: + AbstractResource.__init__(self) + self._prefix = "" + self._app = app + self._rule = rule + + @property + def canonical(self) -> str: + return self._rule.canonical + + def get_info(self) -> _InfoDict: + return {"app": self._app, "rule": self._rule} + + async def resolve(self, request: Request) -> _Resolve: + if not await self._rule.match(request): + return None, set() + match_info = await self._app.router.resolve(request) + match_info.add_app(self._app) + if isinstance(match_info.http_exception, HTTPMethodNotAllowed): + methods = match_info.http_exception.allowed_methods + else: + methods = set() + return match_info, methods + + def __repr__(self) -> str: + return f" {self._app!r}>" + + +class ResourceRoute(AbstractRoute): + """A route with resource""" + + def __init__( + self, + method: str, + handler: Handler | type[AbstractView], + resource: AbstractResource, + *, + expect_handler: _ExpectHandler | None = None, + ) -> None: + super().__init__( + method, handler, expect_handler=expect_handler, resource=resource + ) + + def __repr__(self) -> str: + return f" {self.handler!r}" + + @property + def name(self) -> str | None: + if self._resource is None: + return None + return self._resource.name + + def url_for(self, *args: str, **kwargs: str) -> URL: + """Construct url for route with additional params.""" + assert self._resource is not None + return self._resource.url_for(*args, **kwargs) + + def get_info(self) -> _InfoDict: + assert self._resource is not None + return self._resource.get_info() + + +class SystemRoute(AbstractRoute): + def __init__(self, http_exception: HTTPException) -> None: + super().__init__(hdrs.METH_ANY, self._handle) + self._http_exception = http_exception + + def url_for(self, *args: str, **kwargs: str) -> URL: + raise RuntimeError(".url_for() is not allowed for SystemRoute") + + @property + def name(self) -> str | None: + return None + + def get_info(self) -> _InfoDict: + return {"http_exception": self._http_exception} + + async def _handle(self, request: Request) -> StreamResponse: + raise self._http_exception + + @property + def status(self) -> int: + return self._http_exception.status + + @property + def reason(self) -> str: + return self._http_exception.reason + + def __repr__(self) -> str: + return f"" + + +class View(AbstractView): + async def _iter(self) -> StreamResponse: + if self.request.method not in hdrs.METH_ALL: + self._raise_allowed_methods() + method: Callable[[], Awaitable[StreamResponse]] | None + method = getattr(self, self.request.method.lower(), None) + if method is None: + self._raise_allowed_methods() + ret = await method() + assert isinstance(ret, StreamResponse) + return ret + + def __await__(self) -> Generator[None, None, StreamResponse]: + return self._iter().__await__() + + def _raise_allowed_methods(self) -> NoReturn: + allowed_methods = {m for m in hdrs.METH_ALL if hasattr(self, m.lower())} + raise HTTPMethodNotAllowed(self.request.method, allowed_methods) + + +class ResourcesView(Sized, Iterable[AbstractResource], Container[AbstractResource]): + def __init__(self, resources: list[AbstractResource]) -> None: + self._resources = resources + + def __len__(self) -> int: + return len(self._resources) + + def __iter__(self) -> Iterator[AbstractResource]: + yield from self._resources + + def __contains__(self, resource: object) -> bool: + return resource in self._resources + + +class RoutesView(Sized, Iterable[AbstractRoute], Container[AbstractRoute]): + def __init__(self, resources: list[AbstractResource]): + self._routes: list[AbstractRoute] = [] + for resource in resources: + for route in resource: + self._routes.append(route) + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator[AbstractRoute]: + yield from self._routes + + def __contains__(self, route: object) -> bool: + return route in self._routes + + +class UrlDispatcher(AbstractRouter, Mapping[str, AbstractResource]): + + NAME_SPLIT_RE = re.compile(r"[.:-]") + + def __init__(self) -> None: + super().__init__() + self._resources: list[AbstractResource] = [] + self._named_resources: dict[str, AbstractResource] = {} + self._resource_index: dict[str, list[AbstractResource]] = {} + self._matched_sub_app_resources: list[MatchedSubAppResource] = [] + + async def resolve(self, request: Request) -> UrlMappingMatchInfo: + resource_index = self._resource_index + allowed_methods: set[str] = set() + + # MatchedSubAppResource is primarily used to match on domain names + # (though custom rules could match on other things). This means that + # the traversal algorithm below can't be applied, and that we likely + # need to check these first so a sub app that defines the same path + # as a parent app will get priority if there's a domain match. + # + # For most cases we do not expect there to be many of these since + # currently they are only added by `.add_domain()`. + for resource in self._matched_sub_app_resources: + match_dict, allowed = await resource.resolve(request) + if match_dict is not None: + return match_dict + else: + allowed_methods |= allowed + + # Walk the url parts looking for candidates. We walk the url backwards + # to ensure the most explicit match is found first. If there are multiple + # candidates for a given url part because there are multiple resources + # registered for the same canonical path, we resolve them in a linear + # fashion to ensure registration order is respected. + url_part = request.rel_url.path_safe + while url_part: + for candidate in resource_index.get(url_part, ()): + match_dict, allowed = await candidate.resolve(request) + if match_dict is not None: + return match_dict + else: + allowed_methods |= allowed + if url_part == "/": + break + url_part = url_part.rpartition("/")[0] or "/" + + if allowed_methods: + return MatchInfoError(HTTPMethodNotAllowed(request.method, allowed_methods)) + + return MatchInfoError(HTTPNotFound()) + + def __iter__(self) -> Iterator[str]: + return iter(self._named_resources) + + def __len__(self) -> int: + return len(self._named_resources) + + def __contains__(self, resource: object) -> bool: + return resource in self._named_resources + + def __getitem__(self, name: str) -> AbstractResource: + return self._named_resources[name] + + def resources(self) -> ResourcesView: + return ResourcesView(self._resources) + + def routes(self) -> RoutesView: + return RoutesView(self._resources) + + def named_resources(self) -> Mapping[str, AbstractResource]: + return MappingProxyType(self._named_resources) + + def register_resource(self, resource: AbstractResource) -> None: + assert isinstance( + resource, AbstractResource + ), f"Instance of AbstractResource class is required, got {resource!r}" + if self.frozen: + raise RuntimeError("Cannot register a resource into frozen router.") + + name = resource.name + + if name is not None: + parts = self.NAME_SPLIT_RE.split(name) + for part in parts: + if keyword.iskeyword(part): + raise ValueError( + f"Incorrect route name {name!r}, " + "python keywords cannot be used " + "for route name" + ) + if not part.isidentifier(): + raise ValueError( + f"Incorrect route name {name!r}, " + "the name should be a sequence of " + "python identifiers separated " + "by dash, dot or column" + ) + if name in self._named_resources: + raise ValueError( + f"Duplicate {name!r}, " + f"already handled by {self._named_resources[name]!r}" + ) + self._named_resources[name] = resource + self._resources.append(resource) + + if isinstance(resource, MatchedSubAppResource): + # We cannot index match sub-app resources because they have match rules + self._matched_sub_app_resources.append(resource) + else: + self.index_resource(resource) + + def _get_resource_index_key(self, resource: AbstractResource) -> str: + """Return a key to index the resource in the resource index.""" + if "{" in (index_key := resource.canonical): + # strip at the first { to allow for variables, and than + # rpartition at / to allow for variable parts in the path + # For example if the canonical path is `/core/locations{tail:.*}` + # the index key will be `/core` since index is based on the + # url parts split by `/` + index_key = index_key.partition("{")[0].rpartition("/")[0] + return index_key.rstrip("/") or "/" + + def index_resource(self, resource: AbstractResource) -> None: + """Add a resource to the resource index.""" + resource_key = self._get_resource_index_key(resource) + # There may be multiple resources for a canonical path + # so we keep them in a list to ensure that registration + # order is respected. + self._resource_index.setdefault(resource_key, []).append(resource) + + def unindex_resource(self, resource: AbstractResource) -> None: + """Remove a resource from the resource index.""" + resource_key = self._get_resource_index_key(resource) + self._resource_index[resource_key].remove(resource) + + def add_resource(self, path: str, *, name: str | None = None) -> Resource: + if path and not path.startswith("/"): + raise ValueError("path should be started with / or be empty") + # Reuse last added resource if path and name are the same + if self._resources: + resource = self._resources[-1] + if resource.name == name and resource.raw_match(path): + return cast(Resource, resource) + if not ("{" in path or "}" in path or ROUTE_RE.search(path)): + resource = PlainResource(path, name=name) + self.register_resource(resource) + return resource + resource = DynamicResource(path, name=name) + self.register_resource(resource) + return resource + + def add_route( + self, + method: str, + path: str, + handler: Handler | type[AbstractView], + *, + name: str | None = None, + expect_handler: _ExpectHandler | None = None, + ) -> AbstractRoute: + resource = self.add_resource(path, name=name) + return resource.add_route(method, handler, expect_handler=expect_handler) + + def add_static( + self, + prefix: str, + path: PathLike, + *, + name: str | None = None, + expect_handler: _ExpectHandler | None = None, + chunk_size: int = DEFAULT_CHUNK_SIZE, + show_index: bool = False, + follow_symlinks: bool = False, + append_version: bool = False, + ) -> AbstractResource: + """Add static files view. + + prefix - url prefix + path - folder with files + + """ + assert prefix.startswith("/") + if prefix.endswith("/"): + prefix = prefix[:-1] + resource = StaticResource( + prefix, + path, + name=name, + expect_handler=expect_handler, + chunk_size=chunk_size, + show_index=show_index, + follow_symlinks=follow_symlinks, + append_version=append_version, + ) + self.register_resource(resource) + return resource + + def add_head(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method HEAD.""" + return self.add_route(hdrs.METH_HEAD, path, handler, **kwargs) + + def add_options(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method OPTIONS.""" + return self.add_route(hdrs.METH_OPTIONS, path, handler, **kwargs) + + def add_get( + self, + path: str, + handler: Handler, + *, + name: str | None = None, + allow_head: bool = True, + **kwargs: Any, + ) -> AbstractRoute: + """Shortcut for add_route with method GET. + + If allow_head is true, another + route is added allowing head requests to the same endpoint. + """ + resource = self.add_resource(path, name=name) + if allow_head: + resource.add_route(hdrs.METH_HEAD, handler, **kwargs) + return resource.add_route(hdrs.METH_GET, handler, **kwargs) + + def add_post(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method POST.""" + return self.add_route(hdrs.METH_POST, path, handler, **kwargs) + + def add_put(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method PUT.""" + return self.add_route(hdrs.METH_PUT, path, handler, **kwargs) + + def add_patch(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method PATCH.""" + return self.add_route(hdrs.METH_PATCH, path, handler, **kwargs) + + def add_delete(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method DELETE.""" + return self.add_route(hdrs.METH_DELETE, path, handler, **kwargs) + + def add_view( + self, path: str, handler: type[AbstractView], **kwargs: Any + ) -> AbstractRoute: + """Shortcut for add_route with ANY methods for a class-based view.""" + return self.add_route(hdrs.METH_ANY, path, handler, **kwargs) + + def freeze(self) -> None: + super().freeze() + for resource in self._resources: + resource.freeze() + + def add_routes(self, routes: Iterable[AbstractRouteDef]) -> list[AbstractRoute]: + """Append routes to route table. + + Parameter should be a sequence of RouteDef objects. + + Returns a list of registered AbstractRoute instances. + """ + registered_routes = [] + for route_def in routes: + registered_routes.extend(route_def.register(self)) + return registered_routes + + +def _quote_path(value: str) -> str: + if YARL_VERSION < (1, 6): + value = value.replace("%", "%25") + return URL.build(path=value, encoded=False).raw_path + + +def _unquote_path_safe(value: str) -> str: + if "%" not in value: + return value + return value.replace("%2F", "/").replace("%25", "%") + + +def _requote_path(value: str) -> str: + # Quote non-ascii characters and other characters which must be quoted, + # but preserve existing %-sequences. + result = _quote_path(value) + if "%" in value: + result = result.replace("%25", "%") + return result diff --git a/venv/Lib/site-packages/aiohttp/web_ws.py b/venv/Lib/site-packages/aiohttp/web_ws.py new file mode 100644 index 0000000000000000000000000000000000000000..1893c959b5d9a7488cd10b710f7e2d0fc22a3d78 --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/web_ws.py @@ -0,0 +1,783 @@ +import asyncio +import base64 +import binascii +import hashlib +import json +import sys +from collections.abc import Callable, Iterable +from typing import Any, Final, Generic, Literal, cast, overload + +import attr +from multidict import CIMultiDict + +from . import hdrs +from ._websocket.reader import WebSocketDataQueue +from .abc import AbstractStreamWriter +from .client_exceptions import WSMessageTypeError +from .helpers import ( + DEFAULT_CHUNK_SIZE, + calculate_timeout_when, + set_exception, + set_result, +) +from .http import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WS_KEY, + WebSocketError, + WebSocketReader, + WebSocketWriter, + WSCloseCode, + WSMessage, + WSMessageDecodeText, + WSMessageNoDecodeText, + WSMsgType as WSMsgType, + ws_ext_gen, + ws_ext_parse, +) +from .http_websocket import _INTERNAL_RECEIVE_TYPES +from .log import ws_logger +from .streams import EofStream +from .typedefs import JSONBytesEncoder, JSONDecoder, JSONEncoder +from .web_exceptions import HTTPBadRequest, HTTPException +from .web_request import BaseRequest +from .web_response import StreamResponse + +if sys.version_info >= (3, 13): + from typing import TypeVar +else: + from typing_extensions import TypeVar + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing import Union + + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +if sys.version_info >= (3, 11): + import asyncio as async_timeout + from typing import Self +else: + import async_timeout + from typing_extensions import Self + +__all__ = ( + "WebSocketResponse", + "WebSocketReady", + "WSMsgType", +) + +THRESHOLD_CONNLOST_ACCESS: Final[int] = 5 + +# TypeVar for whether text messages are decoded to str (True) or kept as bytes (False) +_DecodeText = TypeVar("_DecodeText", bound=bool, covariant=True, default=Literal[True]) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class WebSocketReady: + ok: bool + protocol: str | None + + def __bool__(self) -> bool: + return self.ok + + +class WebSocketResponse(StreamResponse, Generic[_DecodeText]): + + _length_check: bool = False + _ws_protocol: str | None = None + _writer: WebSocketWriter | None = None + _reader: WebSocketDataQueue | None = None + _closed: bool = False + _closing: bool = False + _conn_lost: int = 0 + _close_code: int | None = None + _loop: asyncio.AbstractEventLoop | None = None + _waiting: bool = False + _close_wait: asyncio.Future[None] | None = None + _exception: BaseException | None = None + _heartbeat_when: float = 0.0 + _heartbeat_cb: asyncio.TimerHandle | None = None + _pong_response_cb: asyncio.TimerHandle | None = None + _ping_task: asyncio.Task[None] | None = None + _need_heartbeat_reset: bool = False + _heartbeat_reset_handle: asyncio.Handle | None = None + + def __init__( + self, + *, + timeout: float = 10.0, + receive_timeout: float | None = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: float | None = None, + protocols: Iterable[str] = (), + compress: bool = True, + max_msg_size: int = 4 * 1024 * 1024, + writer_limit: int = DEFAULT_CHUNK_SIZE, + decode_text: bool = True, + ) -> None: + super().__init__(status=101) + self._protocols = protocols + self._timeout = timeout + self._receive_timeout = receive_timeout + self._autoclose = autoclose + self._autoping = autoping + self._heartbeat = heartbeat + if heartbeat is not None: + self._pong_heartbeat = heartbeat / 2.0 + self._compress: bool | int = compress + self._max_msg_size = max_msg_size + self._writer_limit = writer_limit + self._decode_text = decode_text + self._need_heartbeat_reset = False + self._heartbeat_reset_handle = None + + def _cancel_heartbeat(self) -> None: + self._cancel_pong_response_cb() + if self._heartbeat_reset_handle is not None: + self._heartbeat_reset_handle.cancel() + self._heartbeat_reset_handle = None + self._need_heartbeat_reset = False + if self._heartbeat_cb is not None: + self._heartbeat_cb.cancel() + self._heartbeat_cb = None + if self._ping_task is not None: + self._ping_task.cancel() + self._ping_task = None + + def _cancel_pong_response_cb(self) -> None: + if self._pong_response_cb is not None: + self._pong_response_cb.cancel() + self._pong_response_cb = None + + def _on_data_received(self) -> None: + if self._heartbeat is None or self._need_heartbeat_reset: + return + loop = self._loop + assert loop is not None + # Coalesce multiple chunks received in the same loop tick into a single + # heartbeat reset. Resetting immediately per chunk increases timer churn. + self._need_heartbeat_reset = True + self._heartbeat_reset_handle = loop.call_soon(self._flush_heartbeat_reset) + + def _flush_heartbeat_reset(self) -> None: + self._heartbeat_reset_handle = None + if not self._need_heartbeat_reset: + return + self._reset_heartbeat() + self._need_heartbeat_reset = False + + def _reset_heartbeat(self) -> None: + if self._heartbeat is None: + return + self._cancel_pong_response_cb() + req = self._req + timeout_ceil_threshold = ( + req._protocol._timeout_ceil_threshold if req is not None else 5 + ) + loop = self._loop + assert loop is not None + now = loop.time() + when = calculate_timeout_when(now, self._heartbeat, timeout_ceil_threshold) + self._heartbeat_when = when + if self._heartbeat_cb is None: + # We do not cancel the previous heartbeat_cb here because + # it generates a significant amount of TimerHandle churn + # which causes asyncio to rebuild the heap frequently. + # Instead _send_heartbeat() will reschedule the next + # heartbeat if it fires too early. + self._heartbeat_cb = loop.call_at(when, self._send_heartbeat) + + def _send_heartbeat(self) -> None: + self._heartbeat_cb = None + + # If heartbeat reset is pending (data is being received), skip sending + # the ping and let the reset callback handle rescheduling the heartbeat. + if self._need_heartbeat_reset: + return + + loop = self._loop + assert loop is not None and self._writer is not None + now = loop.time() + if now < self._heartbeat_when: + # Heartbeat fired too early, reschedule + self._heartbeat_cb = loop.call_at( + self._heartbeat_when, self._send_heartbeat + ) + return + + req = self._req + timeout_ceil_threshold = ( + req._protocol._timeout_ceil_threshold if req is not None else 5 + ) + when = calculate_timeout_when(now, self._pong_heartbeat, timeout_ceil_threshold) + self._cancel_pong_response_cb() + self._pong_response_cb = loop.call_at(when, self._pong_not_received) + + coro = self._writer.send_frame(b"", WSMsgType.PING) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send the ping + # immediately to avoid having to schedule + # the task on the event loop. + ping_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + ping_task = loop.create_task(coro) + + if not ping_task.done(): + self._ping_task = ping_task + ping_task.add_done_callback(self._ping_task_done) + else: + self._ping_task_done(ping_task) + + def _ping_task_done(self, task: "asyncio.Task[None]") -> None: + """Callback for when the ping task completes.""" + if not task.cancelled() and (exc := task.exception()): + self._handle_ping_pong_exception(exc) + self._ping_task = None + + def _pong_not_received(self) -> None: + if self._req is not None and self._req.transport is not None: + self._handle_ping_pong_exception( + asyncio.TimeoutError( + f"No PONG received after {self._pong_heartbeat} seconds" + ) + ) + + def _handle_ping_pong_exception(self, exc: BaseException) -> None: + """Handle exceptions raised during ping/pong processing.""" + if self._closed: + return + self._set_closed() + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + self._exception = exc + if self._waiting and not self._closing and self._reader is not None: + self._reader.feed_data(WSMessage(WSMsgType.ERROR, exc, None), 0) + + def _set_closed(self) -> None: + """Set the connection to closed. + + Cancel any heartbeat timers and set the closed flag. + """ + self._closed = True + self._cancel_heartbeat() + + async def prepare(self, request: BaseRequest) -> AbstractStreamWriter: + # make pre-check to don't hide it by do_handshake() exceptions + if self._payload_writer is not None: + return self._payload_writer + + protocol, writer = self._pre_start(request) + payload_writer = await super().prepare(request) + assert payload_writer is not None + self._post_start(request, protocol, writer) + await payload_writer.drain() + return payload_writer + + def _handshake( + self, request: BaseRequest + ) -> tuple["CIMultiDict[str]", str | None, int, bool]: + headers = request.headers + if "websocket" != headers.get(hdrs.UPGRADE, "").lower().strip(): + raise HTTPBadRequest( + text=( + f"No WebSocket UPGRADE hdr: {headers.get(hdrs.UPGRADE)}\n Can " + '"Upgrade" only to "WebSocket".' + ) + ) + + if not request._message.upgrade: + raise HTTPBadRequest( + text=f"No CONNECTION upgrade hdr: {headers.get(hdrs.CONNECTION)}" + ) + + # find common sub-protocol between client and server + protocol: str | None = None + if hdrs.SEC_WEBSOCKET_PROTOCOL in headers: + req_protocols = [ + str(proto.strip()) + for proto in headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",") + ] + + for proto in req_protocols: + if proto in self._protocols: + protocol = proto + break + else: + # No overlap found: Return no protocol as per spec + ws_logger.warning( + "%s: Client protocols %r don’t overlap server-known ones %r", + request.remote, + req_protocols, + self._protocols, + ) + + # check supported version + version = headers.get(hdrs.SEC_WEBSOCKET_VERSION, "") + if version not in ("13", "8", "7"): + raise HTTPBadRequest(text=f"Unsupported version: {version}") + + # check client handshake for validity + key = headers.get(hdrs.SEC_WEBSOCKET_KEY) + try: + if not key or len(base64.b64decode(key)) != 16: + raise HTTPBadRequest(text=f"Handshake error: {key!r}") + except binascii.Error: + raise HTTPBadRequest(text=f"Handshake error: {key!r}") from None + + accept_val = base64.b64encode( + hashlib.sha1(key.encode() + WS_KEY).digest() + ).decode() + response_headers = CIMultiDict( + { + hdrs.UPGRADE: "websocket", + hdrs.CONNECTION: "upgrade", + hdrs.SEC_WEBSOCKET_ACCEPT: accept_val, + } + ) + + notakeover = False + compress = 0 + if self._compress: + extensions = headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS) + # Server side always get return with no exception. + # If something happened, just drop compress extension + compress, notakeover = ws_ext_parse(extensions, isserver=True) + if compress: + enabledext = ws_ext_gen( + compress=compress, isserver=True, server_notakeover=notakeover + ) + response_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = enabledext + + if protocol: + response_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = protocol + return ( + response_headers, + protocol, + compress, + notakeover, + ) + + def _pre_start(self, request: BaseRequest) -> tuple[str | None, WebSocketWriter]: + self._loop = request._loop + + headers, protocol, compress, notakeover = self._handshake(request) + + self.set_status(101) + self.headers.update(headers) + self.force_close() + self._compress = compress + transport = request._protocol.transport + if transport is None: + raise ConnectionResetError("Connection lost") + writer = WebSocketWriter( + request._protocol, + transport, + compress=compress, + notakeover=notakeover, + limit=self._writer_limit, + ) + + return protocol, writer + + def _post_start( + self, request: BaseRequest, protocol: str | None, writer: WebSocketWriter + ) -> None: + self._ws_protocol = protocol + self._writer = writer + + self._reset_heartbeat() + + loop = self._loop + assert loop is not None + self._reader = WebSocketDataQueue( + request._protocol, DEFAULT_CHUNK_SIZE, loop=loop + ) + parser = WebSocketReader( + self._reader, + self._max_msg_size, + compress=bool(self._compress), + decode_text=self._decode_text, + ) + cb = None if self._heartbeat is None else self._on_data_received + request.protocol.set_parser(parser, data_received_cb=cb) + # disable HTTP keepalive for WebSocket + request.protocol.keep_alive(False) + + def can_prepare(self, request: BaseRequest) -> WebSocketReady: + if self._writer is not None: + raise RuntimeError("Already started") + try: + _, protocol, _, _ = self._handshake(request) + except HTTPException: + return WebSocketReady(False, None) + else: + return WebSocketReady(True, protocol) + + @property + def prepared(self) -> bool: + return self._writer is not None + + @property + def closed(self) -> bool: + return self._closed + + @property + def close_code(self) -> int | None: + return self._close_code + + @property + def ws_protocol(self) -> str | None: + return self._ws_protocol + + @property + def compress(self) -> int | bool: + return self._compress + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """Get optional transport information. + + If no value associated with ``name`` is found, ``default`` is returned. + """ + writer = self._writer + if writer is None: + return default + transport = writer.transport + if transport is None: + return default + return transport.get_extra_info(name, default) + + def exception(self) -> BaseException | None: + return self._exception + + async def ping(self, message: bytes = b"") -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, WSMsgType.PING) + + async def pong(self, message: bytes = b"") -> None: + # unsolicited pong + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, WSMsgType.PONG) + + async def send_frame( + self, message: bytes, opcode: WSMsgType, compress: int | None = None + ) -> None: + """Send a frame over the websocket.""" + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, opcode, compress) + + async def send_str(self, data: str, compress: int | None = None) -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + if not isinstance(data, str): + raise TypeError("data argument must be str (%r)" % type(data)) + await self._writer.send_frame( + data.encode("utf-8"), WSMsgType.TEXT, compress=compress + ) + + async def send_bytes(self, data: bytes, compress: int | None = None) -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("data argument must be byte-ish (%r)" % type(data)) + await self._writer.send_frame(data, WSMsgType.BINARY, compress=compress) + + async def send_json( + self, + data: Any, + compress: int | None = None, + *, + dumps: JSONEncoder = json.dumps, + ) -> None: + await self.send_str(dumps(data), compress=compress) + + async def send_json_bytes( + self, + data: Any, + compress: int | None = None, + *, + dumps: JSONBytesEncoder, + ) -> None: + """Send JSON data using a bytes-returning encoder as a binary frame. + + Use this when your JSON encoder (like orjson) returns bytes + instead of str, avoiding the encode/decode overhead. + """ + await self.send_bytes(dumps(data), compress=compress) + + async def write_eof(self) -> None: # type: ignore[override] + if self._eof_sent: + return + if self._payload_writer is None: + raise RuntimeError("Response has not been started") + + await self.close() + self._eof_sent = True + + async def close( + self, *, code: int = WSCloseCode.OK, message: bytes = b"", drain: bool = True + ) -> bool: + """Close websocket connection.""" + if self._writer is None: + raise RuntimeError("Call .prepare() first") + + if self._closed: + return False + self._set_closed() + + try: + await self._writer.close(code, message) + writer = self._payload_writer + assert writer is not None + if drain: + await writer.drain() + except (asyncio.CancelledError, asyncio.TimeoutError): + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + raise + except Exception as exc: + self._exception = exc + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + return True + + reader = self._reader + assert reader is not None + # we need to break `receive()` cycle before we can call + # `reader.read()` as `close()` may be called from different task + if self._waiting: + assert self._loop is not None + assert self._close_wait is None + self._close_wait = self._loop.create_future() + reader.feed_data(WS_CLOSING_MESSAGE, 0) + await self._close_wait + + if self._closing: + self._close_transport() + return True + + try: + async with async_timeout.timeout(self._timeout): + while True: + msg = await reader.read() + if msg.type is WSMsgType.CLOSE: + self._set_code_close_transport(msg.data) + return True + except asyncio.CancelledError: + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + raise + except Exception as exc: + self._exception = exc + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + return True + + def _set_closing(self, code: WSCloseCode) -> None: + """Set the close code and mark the connection as closing.""" + self._closing = True + self._close_code = code + self._cancel_heartbeat() + + def _set_code_close_transport(self, code: WSCloseCode) -> None: + """Set the close code and close the transport.""" + self._close_code = code + self._close_transport() + + def _close_transport(self) -> None: + """Close the transport.""" + if self._req is not None and self._req.transport is not None: + self._req.transport.close() + + @overload + async def receive( + self: "WebSocketResponse[Literal[True]]", timeout: float | None = None + ) -> WSMessageDecodeText: ... + + @overload + async def receive( + self: "WebSocketResponse[Literal[False]]", timeout: float | None = None + ) -> WSMessageNoDecodeText: ... + + @overload + async def receive( + self: "WebSocketResponse[_DecodeText]", timeout: float | None = None + ) -> WSMessageDecodeText | WSMessageNoDecodeText: ... + + async def receive( + self, timeout: float | None = None + ) -> WSMessageDecodeText | WSMessageNoDecodeText: + if self._reader is None: + raise RuntimeError("Call .prepare() first") + + receive_timeout = timeout or self._receive_timeout + while True: + if self._waiting: + raise RuntimeError("Concurrent call to receive() is not allowed") + + if self._closed: + self._conn_lost += 1 + if self._conn_lost >= THRESHOLD_CONNLOST_ACCESS: + raise RuntimeError("WebSocket connection is closed.") + return WS_CLOSED_MESSAGE + elif self._closing: + return WS_CLOSING_MESSAGE + + try: + self._waiting = True + try: + if receive_timeout: + # Entering the context manager and creating + # Timeout() object can take almost 50% of the + # run time in this loop so we avoid it if + # there is no read timeout. + async with async_timeout.timeout(receive_timeout): + msg = await self._reader.read() + else: + msg = await self._reader.read() + finally: + self._waiting = False + if self._close_wait: + set_result(self._close_wait, None) + except asyncio.TimeoutError: + raise + except EofStream: + self._close_code = WSCloseCode.OK + await self.close() + return WSMessage(WSMsgType.CLOSED, None, None) + except WebSocketError as exc: + self._close_code = exc.code + await self.close(code=exc.code) + return WSMessage(WSMsgType.ERROR, exc, None) + except Exception as exc: + self._exception = exc + self._set_closing(WSCloseCode.ABNORMAL_CLOSURE) + await self.close() + return WSMessage(WSMsgType.ERROR, exc, None) + + if msg.type not in _INTERNAL_RECEIVE_TYPES: + # If its not a close/closing/ping/pong message + # we can return it immediately + return msg + + if msg.type is WSMsgType.CLOSE: + self._set_closing(msg.data) + # Could be closed while awaiting reader. + if not self._closed and self._autoclose: + # The client is likely going to close the + # connection out from under us so we do not + # want to drain any pending writes as it will + # likely result writing to a broken pipe. + await self.close(drain=False) + elif msg.type is WSMsgType.CLOSING: + self._set_closing(WSCloseCode.OK) + elif msg.type is WSMsgType.PING and self._autoping: + await self.pong(msg.data) + continue + elif msg.type is WSMsgType.PONG and self._autoping: + continue + + return msg + + @overload + async def receive_str( + self: "WebSocketResponse[Literal[True]]", *, timeout: float | None = None + ) -> str: ... + + @overload + async def receive_str( + self: "WebSocketResponse[Literal[False]]", *, timeout: float | None = None + ) -> bytes: ... + + @overload + async def receive_str( + self: "WebSocketResponse[_DecodeText]", *, timeout: float | None = None + ) -> str | bytes: ... + + async def receive_str(self, *, timeout: float | None = None) -> str | bytes: + """Receive TEXT message. + + Returns str when decode_text=True (default), bytes when decode_text=False. + """ + msg = await self.receive(timeout) + if msg.type is not WSMsgType.TEXT: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.TEXT" + ) + return cast(str, msg.data) + + async def receive_bytes(self, *, timeout: float | None = None) -> bytes: + msg = await self.receive(timeout) + if msg.type is not WSMsgType.BINARY: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.BINARY" + ) + return cast(bytes, msg.data) + + @overload + async def receive_json( + self: "WebSocketResponse[Literal[True]]", + *, + loads: JSONDecoder = ..., + timeout: float | None = None, + ) -> Any: ... + + @overload + async def receive_json( + self: "WebSocketResponse[Literal[False]]", + *, + loads: Callable[[bytes], Any] = ..., + timeout: float | None = None, + ) -> Any: ... + + @overload + async def receive_json( + self: "WebSocketResponse[_DecodeText]", + *, + loads: JSONDecoder | Callable[[bytes], Any] = ..., + timeout: float | None = None, + ) -> Any: ... + + async def receive_json( + self, + *, + loads: JSONDecoder | Callable[[bytes], Any] = json.loads, + timeout: float | None = None, + ) -> Any: + data = await self.receive_str(timeout=timeout) + return loads(data) # type: ignore[arg-type] + + async def write(self, data: Buffer) -> None: + raise RuntimeError("Cannot call .write() for websocket") + + def __aiter__(self) -> Self: + return self + + @overload + async def __anext__( + self: "WebSocketResponse[Literal[True]]", + ) -> WSMessageDecodeText: ... + + @overload + async def __anext__( + self: "WebSocketResponse[Literal[False]]", + ) -> WSMessageNoDecodeText: ... + + @overload + async def __anext__( + self: "WebSocketResponse[_DecodeText]", + ) -> WSMessageDecodeText | WSMessageNoDecodeText: ... + + async def __anext__(self) -> WSMessageDecodeText | WSMessageNoDecodeText: + msg = await self.receive() + if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED): + raise StopAsyncIteration + return msg + + def _cancel(self, exc: BaseException) -> None: + # web_protocol calls this from connection_lost + # or when the server is shutting down. + self._closing = True + self._cancel_heartbeat() + if self._reader is not None: + set_exception(self._reader, exc) diff --git a/venv/Lib/site-packages/aiohttp/worker.py b/venv/Lib/site-packages/aiohttp/worker.py new file mode 100644 index 0000000000000000000000000000000000000000..22b41e4172ae47386503933c74103e4a7a69b85b --- /dev/null +++ b/venv/Lib/site-packages/aiohttp/worker.py @@ -0,0 +1,266 @@ +"""Async gunicorn worker for aiohttp.web""" + +import asyncio +import inspect +import os +import re +import signal +import sys +from types import FrameType +from typing import TYPE_CHECKING, Any, Optional + +from gunicorn.config import AccessLogFormat as GunicornAccessLogFormat +from gunicorn.workers import base + +from aiohttp import web + +from .helpers import set_result +from .web_app import Application +from .web_log import AccessLogger + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + + +__all__ = ("GunicornWebWorker", "GunicornUVLoopWebWorker") + + +class GunicornWebWorker(base.Worker): # type: ignore[misc,no-any-unimported] + DEFAULT_AIOHTTP_LOG_FORMAT = AccessLogger.LOG_FORMAT + DEFAULT_GUNICORN_LOG_FORMAT = GunicornAccessLogFormat.default + + def __init__(self, *args: Any, **kw: Any) -> None: # pragma: no cover + super().__init__(*args, **kw) + + self._task: asyncio.Task[None] | None = None + self.exit_code = 0 + self._notify_waiter: asyncio.Future[bool] | None = None + + def init_process(self) -> None: + # create new event_loop after fork + try: + asyncio.get_event_loop().close() + except RuntimeError: + # No loop was running + pass + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + super().init_process() + + def run(self) -> None: + self._task = self.loop.create_task(self._run()) + + try: # ignore all finalization problems + self.loop.run_until_complete(self._task) + except Exception: + self.log.exception("Exception in gunicorn worker") + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.loop.close() + + sys.exit(self.exit_code) + + async def _run(self) -> None: + runner = None + if isinstance(self.wsgi, Application): + app = self.wsgi + elif inspect.iscoroutinefunction(self.wsgi) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(self.wsgi) + ): + wsgi = await self.wsgi() + if isinstance(wsgi, web.AppRunner): + runner = wsgi + app = runner.app + else: + app = wsgi + else: + raise RuntimeError( + "wsgi app should be either Application or " + f"async function returning Application, got {self.wsgi}" + ) + + if runner is None: + access_log = self.log.access_log if self.cfg.accesslog else None + runner = web.AppRunner( + app, + logger=self.log, + keepalive_timeout=self.cfg.keepalive, + access_log=access_log, + access_log_format=self._get_valid_log_format( + self.cfg.access_log_format + ), + shutdown_timeout=self.cfg.graceful_timeout / 100 * 95, + ) + await runner.setup() + + ctx = self._create_ssl_context(self.cfg) if self.cfg.is_ssl else None + + runner = runner + assert runner is not None + server = runner.server + assert server is not None + for sock in self.sockets: + site = web.SockSite( + runner, + sock, + ssl_context=ctx, + ) + await site.start() + + # If our parent changed then we shut down. + pid = os.getpid() + try: + while self.alive: # type: ignore[has-type] + self.notify() + + cnt = server.requests_count + if self.max_requests and cnt > self.max_requests: + self.alive = False + self.log.info("Max requests, shutting down: %s", self) + + elif pid == os.getpid() and self.ppid != os.getppid(): + self.alive = False + self.log.info("Parent changed, shutting down: %s", self) + else: + await self._wait_next_notify() + except Exception: + pass + + await runner.cleanup() + + def _wait_next_notify(self) -> "asyncio.Future[bool]": + self._notify_waiter_done() + + loop = self.loop + assert loop is not None + self._notify_waiter = waiter = loop.create_future() + self.loop.call_later(1.0, self._notify_waiter_done, waiter) + + return waiter + + def _notify_waiter_done( + self, waiter: Optional["asyncio.Future[bool]"] = None + ) -> None: + if waiter is None: + waiter = self._notify_waiter + if waiter is not None: + set_result(waiter, True) + + if waiter is self._notify_waiter: + self._notify_waiter = None + + def init_signals(self) -> None: + # Set up signals through the event loop API. + + self.loop.add_signal_handler( + signal.SIGQUIT, self.handle_quit, signal.SIGQUIT, None + ) + + self.loop.add_signal_handler( + signal.SIGTERM, self.handle_exit, signal.SIGTERM, None + ) + + self.loop.add_signal_handler( + signal.SIGINT, self.handle_quit, signal.SIGINT, None + ) + + self.loop.add_signal_handler( + signal.SIGWINCH, self.handle_winch, signal.SIGWINCH, None + ) + + self.loop.add_signal_handler( + signal.SIGUSR1, self.handle_usr1, signal.SIGUSR1, None + ) + + self.loop.add_signal_handler( + signal.SIGABRT, self.handle_abort, signal.SIGABRT, None + ) + + # Don't let SIGTERM and SIGUSR1 disturb active requests + # by interrupting system calls + signal.siginterrupt(signal.SIGTERM, False) + signal.siginterrupt(signal.SIGUSR1, False) + + # Reset SIGCHLD to default so Gunicorn doesn't swallow subprocess + # return codes. Without this, workers inherit the master arbiter's + # SIGCHLD handler, causing spurious "Worker exited" errors when + # application code spawns subprocesses. + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + def handle_quit(self, sig: int, frame: FrameType | None) -> None: + self.alive = False + + # worker_int callback + self.cfg.worker_int(self) + + # wakeup closing process + self._notify_waiter_done() + + def handle_abort(self, sig: int, frame: FrameType | None) -> None: + self.alive = False + self.exit_code = 1 + self.cfg.worker_abort(self) + sys.exit(1) + + @staticmethod + def _create_ssl_context(cfg: Any) -> "SSLContext": + """Creates SSLContext instance for usage in asyncio.create_server. + + See ssl.SSLSocket.__init__ for more details. + """ + if ssl is None: # pragma: no cover + raise RuntimeError("SSL is not supported.") + + ctx = ssl.SSLContext(cfg.ssl_version) + ctx.load_cert_chain(cfg.certfile, cfg.keyfile) + ctx.verify_mode = cfg.cert_reqs + if cfg.ca_certs: + ctx.load_verify_locations(cfg.ca_certs) + if cfg.ciphers: + ctx.set_ciphers(cfg.ciphers) + return ctx + + def _get_valid_log_format(self, source_format: str) -> str: + if source_format == self.DEFAULT_GUNICORN_LOG_FORMAT: + return self.DEFAULT_AIOHTTP_LOG_FORMAT + elif re.search(r"%\([^\)]+\)", source_format): + raise ValueError( + "Gunicorn's style options in form of `%(name)s` are not " + "supported for the log formatting. Please use aiohttp's " + "format specification to configure access log formatting: " + "http://docs.aiohttp.org/en/stable/logging.html" + "#format-specification" + ) + else: + return source_format + + +class GunicornUVLoopWebWorker(GunicornWebWorker): + def init_process(self) -> None: + import uvloop + + # Close any existing event loop before setting a + # new policy. + try: + asyncio.get_event_loop().close() + except RuntimeError: + # No loop was running + pass + + # Setup uvloop policy, so that every + # asyncio.get_event_loop() will create an instance + # of uvloop event loop. + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + super().init_process() diff --git a/venv/images/imagehash.png b/venv/images/imagehash.png new file mode 100644 index 0000000000000000000000000000000000000000..48db6a3c2c48160f2942f3f413f0fefa054663f9 --- /dev/null +++ b/venv/images/imagehash.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35134f60585946fba3e705db56d4941241bb174207a92e2cd46498efa055a936 +size 292247